├── .gitignore ├── data_prep.py ├── data_prepared └── .placeholder ├── data_raw └── .placeholder ├── elect_preprocessing.py ├── elect_test.py ├── evaluate.py ├── models ├── elect │ ├── ConvTransformer_elect2020-05-10 11:13:36.815714.pth │ ├── ConvTransformer_elect2020-05-10 11:52:33.147730.pth │ ├── model_config.json │ └── train_config.json └── electricity_dt_1 │ ├── ConvTransformer_electricity_dt_1_kernel6_start_3heads.pth │ ├── model_config.json │ └── train_config.json ├── net.py ├── prediction.py ├── predictions ├── .placeholder └── elect_01 │ ├── 0.png │ ├── 1.png │ ├── 10.png │ ├── 11.png │ ├── 12.png │ ├── 13.png │ ├── 14.png │ ├── 15.png │ ├── 16.png │ ├── 17.png │ ├── 18.png │ ├── 19.png │ ├── 2.png │ ├── 20.png │ ├── 21.png │ ├── 22.png │ ├── 23.png │ ├── 24.png │ ├── 25.png │ ├── 26.png │ ├── 27.png │ ├── 28.png │ ├── 29.png │ ├── 3.png │ ├── 30.png │ ├── 31.png │ ├── 32.png │ ├── 33.png │ ├── 34.png │ ├── 35.png │ ├── 36.png │ ├── 37.png │ ├── 38.png │ ├── 39.png │ ├── 4.png │ ├── 40.png │ ├── 41.png │ ├── 42.png │ ├── 43.png │ ├── 44.png │ ├── 45.png │ ├── 46.png │ ├── 47.png │ ├── 48.png │ ├── 49.png │ ├── 5.png │ ├── 50.png │ ├── 51.png │ ├── 52.png │ ├── 53.png │ ├── 54.png │ ├── 55.png │ ├── 56.png │ ├── 57.png │ ├── 58.png │ ├── 59.png │ ├── 6.png │ ├── 60.png │ ├── 61.png │ ├── 62.png │ ├── 63.png │ ├── 64.png │ ├── 65.png │ ├── 66.png │ ├── 67.png │ ├── 68.png │ ├── 69.png │ ├── 7.png │ ├── 70.png │ ├── 71.png │ ├── 72.png │ ├── 73.png │ ├── 74.png │ ├── 75.png │ ├── 76.png │ ├── 77.png │ ├── 78.png │ ├── 79.png │ ├── 8.png │ ├── 80.png │ ├── 81.png │ ├── 82.png │ ├── 83.png │ ├── 84.png │ ├── 85.png │ ├── 86.png │ ├── 87.png │ ├── 88.png │ ├── 89.png │ ├── 9.png │ ├── 90.png │ ├── 91.png │ ├── 92.png │ ├── 93.png │ ├── 94.png │ ├── 95.png │ ├── 96.png │ ├── 97.png │ ├── 98.png │ ├── 99.png │ ├── estimation.npy │ └── sigma.npy ├── readme.md └── train.py /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | data_prepared/* 3 | 4 | not_include/ 5 | 6 | data_prep/* 7 | 8 | data_raw/* 9 | 10 | *.pyc 11 | -------------------------------------------------------------------------------- /data_prep.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import numpy as np 3 | import matplotlib.pyplot as plt 4 | import os 5 | from scipy import stats 6 | import argparse 7 | import json 8 | 9 | from sklearn.preprocessing import StandardScaler 10 | from datetime import datetime, timedelta 11 | from pandas.tseries.frequencies import to_offset 12 | 13 | from evaluate import * 14 | import pickle 15 | 16 | def gen_covariates(times): 17 | ''' 18 | Function to create independent covariate variables and normalize them: weekday, hour, mouth 19 | 20 | # times : date times 21 | 22 | returns : Normalized covariates variables 23 | ''' 24 | num_covariates = 3 25 | covariates = np.zeros((times.shape[0], num_covariates)) 26 | for i, input_time in enumerate(times): 27 | covariates[i, 0] = input_time.weekday() 28 | covariates[i, 1] = input_time.hour 29 | covariates[i, 2] = input_time.month 30 | return covariates[:, :num_covariates] 31 | 32 | def prep_data(data, window_size, stride_size, data_start, timeseries_list, covariates=None, covariates_var=None, transformation_covariates_used = None, transformation_covariates_var_used = None): 33 | ''' 34 | Function to prepare data for the model 35 | 36 | # data : 2D array, contains the times series : e.g : (274,5) : 274 timestamps, 5 timeseries 37 | # window_size : int, the sliding windows size : e.g : 24 38 | # stride_size : int, the stride during the window_size sliding 39 | # covariates : 2D array, covariate variables, independents of the series : e.g (274, 3) : 274 timestamps, 3 covariate variables 40 | # covariates_var : 3D array, covariate variables that changes in time : e.g : (274,5,2) : 274 timestamps, 5 timeseries, 2 covariate variables 41 | # data_start : 1D array, the index in the timestamps where each timeseries start : e.g (5,) 42 | # timeseries_list : 1D array, the list of the names of the time series : e.G (5,) 43 | 44 | returns : x_input, label, v_input, dict_timeseries 45 | 46 | # x_input : 3D array, contains the input which will be feeded to the model, combaining data and covariates : e.g (63,24,7) : 63 number of windows created from the sliding window on all the series, 24 the windows size, 7 : the variables : 1 (time serie value) + 3 (covariate value) + 2 (covariate_var value normalized)+ 1 (time serie id) 47 | # label : 2D array, contains the values of the time serie on the window shifted by one : e.g : (63, 24) : 63 : 63 number of windows created from the sliding window on all the series, 24 the window size 48 | # v_input : 2D array, the normalization factor by window : e.g : (63,2) : 63 , 2 : 1 (the normalizaiton factor) +1 (zeros) 49 | # dict_timeseries : dict, contains the series number which figure in x_input[:,:,-1] corresponding serie name 50 | ''' 51 | # Initial global informations 52 | 53 | time_len = data.shape[0] # time series global lenght 54 | num_series = data.shape[1] # number of time series 55 | input_size = window_size-stride_size # Input size 56 | 57 | # Verification : 58 | 59 | if not covariates_var is None : 60 | # Verification of index matching 61 | assert covariates_var.shape[0] == data.shape[0] 62 | # Verification of time series matching 63 | assert covariates_var.shape[1] == data.shape[1] 64 | 65 | if not covariates is None : 66 | # Verification of index matching 67 | assert covariates.shape[0] == data.shape[0] 68 | 69 | # keep only time series with at least one window size 70 | 71 | kept_timeseries = time_len - data_start > window_size 72 | data = data[:,kept_timeseries] 73 | data_start = data_start[kept_timeseries] 74 | covariates_var = None if covariates_var is None else covariates_var[:,kept_timeseries,:] 75 | timeseries_list = timeseries_list[kept_timeseries] 76 | 77 | 78 | # Update the parametres 79 | 80 | time_len = data.shape[0]-1 # minus the last one to use it as the la predsict 81 | num_series = data.shape[1] 82 | 83 | num_cov_simple = 0 if covariates is None else covariates.shape[1] 84 | num_cov_var = 0 if covariates_var is None else covariates_var.shape[2] 85 | 86 | num_covariates = num_cov_simple + num_cov_var 87 | 88 | dict_timeseries = dict(zip(timeseries_list,range(num_series))) 89 | 90 | # Number of Windows 91 | 92 | windows_per_series = np.full((num_series), (time_len-input_size) // stride_size) # number of windows in the interval 93 | windows_per_series -= (data_start+stride_size-1) // stride_size 94 | 95 | # Data structre definition 96 | 97 | total_windows = np.sum(windows_per_series) 98 | 99 | x_input = np.zeros((total_windows, window_size, 1+num_covariates + 1), dtype='float32') 100 | label = np.zeros((total_windows, window_size), dtype='float32') 101 | v_input = np.zeros((total_windows, 2), dtype='float32') 102 | 103 | # transformation 104 | 105 | if not covariates_var is None: 106 | transformation_covariates_var = dict()# np.zeros((num_series,num_cov_var,2)) StandardScaler 107 | covariates_var = covariates_var.astype(float) 108 | else : 109 | transformation_covariates_var = None 110 | 111 | if not covariates is None: 112 | transformation_covariates = dict() #np.zeros((num_series,num_cov_simple,2)) 113 | covariates = covariates.astype(float) 114 | else : 115 | transformation_covariates = None 116 | 117 | ## Scaling the covariates 118 | 119 | for i in range(num_cov_simple): 120 | if transformation_covariates_used is None and not covariates is None: 121 | # creating a scaler 122 | transfomer_cov_simple = StandardScaler() 123 | covariates[:,i] = transfomer_cov_simple.fit_transform(covariates[:,i].reshape(-1,1))[:,0] 124 | transformation_covariates[i] = transfomer_cov_simple 125 | elif not covariates is None : 126 | # loeading one 127 | transfomer_cov_simple = transformation_covariates_used[i] 128 | covariates[:,i] = transfomer_cov_simple.transform(covariates[:,i].reshape(-1,1))[:,0] 129 | transformation_covariates[i] = transfomer_cov_simple 130 | 131 | # Values assignation 132 | 133 | count = 0 134 | for series in range(num_series): 135 | ## Scaling the covariate variables 136 | if transformation_covariates_var_used is None and not covariates_var is None: 137 | # creating a scaler 138 | transfomer_cov_var = StandardScaler() 139 | covariates_var[data_start[series]:, series,:] = transfomer_cov_var.fit_transform(covariates_var[data_start[series]:, series,:]) 140 | elif not covariates_var is None: 141 | # loading the used one for example in the training 142 | transfomer_cov_var = transformation_covariates_var_used[series] 143 | covariates_var[data_start[series]:, series,:] = transfomer_cov_var.transform(covariates_var[data_start[series]:, series,:]) 144 | 145 | if not covariates_var is None: 146 | transformation_covariates_var[series] = transfomer_cov_var 147 | 148 | # Windows creation 149 | 150 | for i in range(windows_per_series[series]): 151 | # In training windowing begun at the start of the time serie 152 | 153 | window_start = stride_size*i+data_start[series] 154 | window_end = window_start+window_size 155 | 156 | # x_input shape is : timeserie_id, steps, timeserie_value+covariate_variables 157 | 158 | x_input[count, :, 0] = data[window_start:window_end, series] # timeserie_value, we start always by 0 159 | x_input[count, :, 1:1+num_cov_simple] = 0 if num_cov_simple==0 else covariates[window_start+1:window_end+1, :] # covariates_values 160 | x_input[count, :, 1+num_cov_simple:-1] = 0 if num_cov_var == 0 else covariates_var[window_start+1:window_end+1, series,:] # covariates than depend on the time serie 161 | x_input[count, :, -1] = series # timeserie id / label / token which the name is in "timeseries_list" 162 | 163 | # target affectation / label 164 | 165 | label[count, :] = data[window_start+1:window_end+1, series] 166 | 167 | # Time serie scale normalization using the window sum. 168 | 169 | nonzero_sum = (x_input[count, :, 0]!=0).sum() 170 | 171 | if nonzero_sum == 0: 172 | v_input[count, 0] = 1 173 | else: 174 | v_input[count, 0] = np.true_divide(x_input[count, :, 0].sum(),nonzero_sum)+1 175 | x_input[count, :, 0] = x_input[count, :, 0]/v_input[count, 0] 176 | label[count, :] = label[count, :]/v_input[count, 0] 177 | count += 1 178 | return x_input, label, v_input, dict_timeseries, transformation_covariates_var, transformation_covariates 179 | 180 | def prep_data_for_prediction(model, data, window_size, steps, data_start, timeseries_list,device = 'cpu' , covariates=None, covariates_var=None, transformation_covariates_used = None, transformation_covariates_var_used = None, sampling = False): 181 | ''' 182 | Function to prepare data for the model 183 | # model : the pytorch Transformer model to use in prediction 184 | # data : 2D array, contains the times series : e.g : (274,5) : 274 timestamps, 5 timeseries 185 | # window_size : int, the sliding windows size : e.g : 24 186 | # stride_size : int, the stride during the window_size sliding 187 | # steps : int , steps to predict which needs to be < window_size 188 | # covariates : 2D array, covariate variables, independents of the series : e.g (274, 3) : 274 timestamps, 3 covariate variables 189 | # covariates_var : 3D array, covariate variables that changes in time : e.g : (274,5,2) : 274 timestamps, 5 timeseries, 2 covariate variables 190 | # data_start : 1D array, the index in the timestamps where each timeseries start : e.g (5,) 191 | # timeseries_list : 1D array, the list of the names of the time series : e.G (5,) 192 | # transformation_covariates_used : used Transformation during train 193 | # transformation_covariates_var_used : used Transformation during train 194 | # sampling : Boolean, for using the mean of the distribution or doing a sampling 195 | 196 | returns : estimation_modeles 197 | 198 | # estimation_model : 2D array, containing the predictions for the series 199 | ''' 200 | 201 | # Stride is 1 and not variable in the prediction mode 202 | 203 | stride_size = 1 204 | 205 | # torch device 206 | if device == None : 207 | if torch.cuda.is_available(): 208 | torch.set_default_tensor_type(torch.cuda.FloatTensor) 209 | device = 'cuda' 210 | # Initial global informations 211 | 212 | time_len = data.shape[0] # time series global lenght 213 | num_series = data.shape[1] # number of time series 214 | input_size = window_size-stride_size # Input size 215 | 216 | # Verification for prediction: 217 | 218 | assert steps < window_size, 'The steps='+str(steps)+' is > window_size='+str(window_size) 219 | assert steps > 0, 'The steps='+str(steps)+' needs to be > 0' 220 | 221 | if not covariates_var is None : 222 | assert not transformation_covariates_var_used is None, 'YOU NEED TRANSFMATION FOR THE COVARIATES VAR' 223 | # Verification of index matching 224 | assert covariates_var.shape[0] == data.shape[0] + steps, 'covariates_var.shape[0] '+str(covariates_var.shape[0])+' and data.shape[0] + steps '+str(data.shape[0] + steps)+' didn t match' 225 | # Verification of time series matching 226 | assert covariates_var.shape[1] == data.shape[1] 227 | 228 | if not covariates is None : 229 | assert not transformation_covariates_used is None, 'YOU NEED TRANSFMATION FOR THE COVARIATES' 230 | # Verification of index matching 231 | assert covariates.shape[0] == data.shape[0] + steps, 'covariates.shape[0] '+str(covariates.shape[0])+' and data.shape[0] + steps '+str(data.shape[0] + steps)+' didn t match' 232 | 233 | # Append -1 to the new steps to predict to the predict 234 | # keep only time series with at least one window size 235 | 236 | kept_timeseries = time_len - data_start > window_size 237 | data = data[:,kept_timeseries] 238 | data_start = data_start[kept_timeseries] 239 | covariates_var = None if covariates_var is None else covariates_var[:,kept_timeseries,:] 240 | timeseries_list = timeseries_list[kept_timeseries] 241 | 242 | # Number of series to predict : 243 | q = 100 244 | data = data[:,:q] 245 | data_start = data_start[:q] 246 | covariates_var = None if covariates_var is None else covariates_var[:,:q,:] 247 | 248 | # Update the parametres 249 | 250 | time_len = data.shape[0] 251 | num_series = data.shape[1] 252 | 253 | num_cov_simple = 0 if covariates is None else covariates.shape[1] 254 | num_cov_var = 0 if covariates_var is None else covariates_var.shape[2] 255 | 256 | num_covariates = num_cov_simple + num_cov_var 257 | 258 | dict_timeseries = dict(zip(timeseries_list,range(num_series))) 259 | 260 | # Number of Windows 261 | 262 | windows_per_series = np.full((num_series), (time_len-input_size) // stride_size) # number of windows in the interval 263 | windows_per_series -= (data_start+stride_size-1) // stride_size 264 | 265 | # Data structre definition 266 | 267 | total_windows = np.sum(windows_per_series) 268 | 269 | 270 | # transformation 271 | if not covariates_var is None : 272 | transformation_covariates_var = dict()# np.zeros((num_series,num_cov_var,2)) StandardScaler 273 | else : 274 | transformation_covariates_var = None 275 | 276 | if not covariates is None : 277 | transformation_covariates = dict() #np.zeros((num_series,num_cov_simple,2)) 278 | else : 279 | transformation_covariates = None 280 | 281 | ## Scaling the covariates 282 | 283 | for i in range(num_cov_simple): 284 | if transformation_covariates_used is None: 285 | # creating a scaler 286 | transfomer_cov_simple = StandardScaler() 287 | covariates[:,i] = transfomer_cov_simple.fit_transform(covariates[:,i].reshape((-1,1)))[:,0] 288 | transformation_covariates[i] = transfomer_cov_simple 289 | else : 290 | # creating a scaler 291 | transfomer_cov_simple = transformation_covariates_used[i] 292 | covariates[:,i] = transfomer_cov_simple.fit_transform(covariates[:,i].reshape((-1,1)))[:,0] 293 | transformation_covariates[i] = transfomer_cov_simple 294 | 295 | # Values assignation 296 | data = np.append(data, -1*np.ones((steps,num_series)), axis=0) # add the steps to predict 297 | estimation_model = np.empty((data.shape[0]-1,data.shape[1])) 298 | sigma_model = np.empty((data.shape[0]-1,data.shape[1])) 299 | 300 | count = 0 301 | print('# Device :', device) 302 | for series in range(num_series): 303 | ## Scaling the covariate variables 304 | if transformation_covariates_var_used is None and not covariates_var is None: 305 | # creating a scaler 306 | transfomer_cov_var = StandardScaler() 307 | covariates_var[data_start[series]:, series,:] = transfomer_cov_var.fit_transform(covariates_var[data_start[series]:, series,:]) 308 | transformation_covariates_var[series] = transfomer_cov_var 309 | elif not covariates_var is None: 310 | # creating a scaler 311 | transfomer_cov_var = transformation_covariates_var_used[series] 312 | covariates_var[data_start[series]:, series,:] = transfomer_cov_var.transform(covariates_var[data_start[series]:, series,:]) 313 | transformation_covariates_var[series] = transfomer_cov_var 314 | 315 | # Windows creation 316 | window_start = data_start[series] 317 | i = 0 318 | 319 | while stride_size*i + data_start[series] + window_size < time_len + steps - 1 - data_start[series] : # time_len + steps - 1 to match the new data size - 1 320 | # In training windowing begun at the start of the time serie 321 | 322 | window_start = stride_size*i + data_start[series] 323 | window_end = window_start + window_size 324 | 325 | x_input = np.zeros((1, window_size, 1 + num_covariates + 1), dtype='float32') 326 | 327 | # x_input shape is : timeserie_id, steps, timeserie_value+covariate_variables 328 | 329 | x_input[0, :, 0] = data[window_start:window_end, series] # timeserie_value, we start always by 0 330 | x_input[0, :, 1:1+num_cov_simple] = 0 if num_cov_simple==0 else covariates[window_start + 1:window_end + 1, :] # covariates_values 331 | x_input[0, :, 1+num_cov_simple:-1] = 0 if num_cov_var == 0 else covariates_var[window_start + 1:window_end + 1, series,:] # covariates than depend on the time serie 332 | x_input[0, :, -1] = series # timeserie id / label / token which the name is in "timeseries_list" 333 | 334 | # Time serie scale normalization using the window sum 335 | 336 | v_input = np.zeros((1, 2), dtype='float32') 337 | 338 | nonzero_sum = (x_input[0, :, 0]!=0).sum() 339 | 340 | if nonzero_sum == 0: 341 | v_input[0, 0] = 1 342 | else: 343 | v_input[0, 0] = np.true_divide(x_input[0, :, 0].sum(),nonzero_sum)+1 344 | x_input[0, :, 0] = x_input[0, :, 0]/v_input[0, 0] 345 | 346 | # To tensor 347 | v_input = torch.tensor(v_input) 348 | x_input = torch.tensor(x_input) 349 | 350 | # Expanding v_input 351 | v_input_expanded = v_input.expand(window_size,1,2).transpose(1,0) 352 | 353 | # tocken 354 | token = np.array([series]) 355 | token = torch.tensor(token) 356 | 357 | # model application 358 | x_input = x_input[:,:,:-1] # since the token info is given separately 359 | 360 | x_input = x_input.to(device) 361 | v_input_expanded = v_input_expanded.to(device) 362 | token = token.to(device) 363 | 364 | if sampling : 365 | samples, estimation, mu, sigma = evaluate(model, x_input, token, v_input_expanded, window_size, sampling, number_of_samples = 100, scaled_param = True) 366 | else : 367 | estimation, mu, sigma = evaluate(model, x_input, token, v_input_expanded, window_size, sampling, number_of_samples = 100, scaled_param = True) 368 | 369 | estimation_model[window_start + 1:window_end + 1, series] = estimation[0,:,0].detach().cpu().numpy() 370 | sigma_model[window_start + 1:window_end + 1, series] = sigma[0,:,0].detach().cpu().numpy() 371 | 372 | if not window_end < time_len : 373 | data[window_end,series] = estimation[0,-1,0].detach().cpu().numpy() 374 | i+=1 375 | count += 1 376 | 377 | return estimation_model, sigma_model 378 | 379 | def save_dataset(save_path,save_name,xtrain_input,v_input_train,label_train,timeseries_dict_train,transformation_covariates_var, transformation_covariates,xtest_input,v_input_test,label_test,timeseries_dict_test, split_dates): 380 | ''' 381 | Function to save the prepared data 382 | # save_path 383 | # save_name 384 | #.... 385 | 386 | ''' 387 | # saving train 388 | prefix = os.path.join(save_path, 'train_') 389 | 390 | np.save(prefix+'data_'+save_name, xtrain_input) 391 | np.save(prefix+'v_'+save_name, v_input_train) 392 | np.save(prefix+'label_'+save_name, label_train) 393 | np.save(prefix+'timeseries_dict_'+save_name, timeseries_dict_train) 394 | 395 | # saving test 396 | prefix = os.path.join(save_path, 'test_') 397 | np.save(prefix+'data_'+save_name, xtest_input) 398 | np.save(prefix+'v_'+save_name, v_input_test) 399 | np.save(prefix+'label_'+save_name, label_test) 400 | np.save(prefix+'timeseries_dict_'+save_name, timeseries_dict_test) 401 | 402 | # savinf transformation 403 | with open(save_path+'/transformation_covariates_var_'+save_name+'.pkl', 'wb') as file: 404 | pickle.dump(transformation_covariates_var, file, pickle.HIGHEST_PROTOCOL) 405 | 406 | with open(save_path+'/transformation_covariates_'+save_name+'.pkl', 'wb') as file: 407 | pickle.dump(transformation_covariates, file, pickle.HIGHEST_PROTOCOL) 408 | 409 | # saving meta informations 410 | num_timeseries_kinds = len(timeseries_dict_train) 411 | 412 | window_size = xtrain_input.shape[1] 413 | number_of_vars = xtrain_input.shape[2]-1 # -1 cz the last variable is the kind of the time serie 414 | 415 | meta_information = dict({ 416 | 'num_timeseries_kinds' : num_timeseries_kinds, 417 | 'window_size' : window_size, 418 | 'number_of_vars' : number_of_vars, 419 | }) 420 | train_start,train_end,test_start,test_end = split_dates 421 | 422 | meta_information['transformation_covariates_var_path'] = save_path+'/transformation_covariates_var_'+save_name+'.pkl' 423 | meta_information['transformation_covariates'] = save_path+'/transformation_covariates_'+save_name+'.pkl' 424 | 425 | meta_information['train_start'] = str(train_start) 426 | meta_information['train_end'] = str(train_end) 427 | meta_information['test_start'] = str(test_start) 428 | meta_information['test_end'] = str(test_end) 429 | 430 | with open(save_path+'/meta_information.json', 'w') as file: 431 | json.dump(meta_information, file) 432 | 433 | # Creation of the default model configuration && default train config 434 | model_config = dict() 435 | model_config['headers'] = 3 436 | model_config['depth'] = 3 437 | model_config['kernel_size'] = 6 438 | model_config['default'] = True 439 | 440 | config_folder_path = os.path.join('models',save_name) 441 | os.makedirs(config_folder_path, exist_ok=True) 442 | 443 | config_path = os.path.join(config_folder_path,'model_config.json') 444 | 445 | with open(config_path, 'w') as file: 446 | json.dump(model_config, file) 447 | 448 | # Creation of the default model configuration 449 | train_config = dict() 450 | train_config['num_epochs'] = 5 451 | train_config['lr_warmup'] = 1000 452 | train_config['learning_rate'] = 0.001 453 | train_config['default'] = True 454 | train_config['batch_size'] = 32 455 | train_config['predict_batch_size'] = 32 456 | 457 | train_path = os.path.join(config_folder_path,'train_config.json') 458 | with open(train_path, 'w') as file: 459 | json.dump(train_config, file) 460 | 461 | def generate_fake_data(interval= ['2012-12-31','2015-07-29']): 462 | ''' 463 | Function to generate a fake test data 464 | ''' 465 | # Main time serie to predict 466 | Table = pd.DataFrame(pd.date_range(interval[0],interval[1]), columns=['date']) 467 | 468 | num_tot = len(Table) 469 | serie_normal = [] 470 | serie_sin_normal = [] 471 | for i in range(num_tot): 472 | serie_normal.append(np.random.normal(2*i,scale=50)) #np.sin(200*np.pi*i/num_tot)*10+ 473 | serie_sin_normal.append(np.random.normal(2*i,scale=50)+np.sin(200*np.pi*i/num_tot)*500) 474 | 475 | plt.plot(serie_normal) 476 | plt.plot(serie_sin_normal) 477 | plt.show() 478 | 479 | Table['serie_normal'] = serie_normal 480 | Table['serie_sin_normal'] = serie_sin_normal 481 | Table.set_index('date',inplace=True) 482 | 483 | return Table 484 | 485 | def generate_fake_data_covariates(interval= ['2012-12-31','2015-07-29']): 486 | ''' 487 | Function to generate a fake test data 488 | ''' 489 | covariates_var = [] 490 | # var 1 491 | # Main time serie to predict 492 | Table = pd.DataFrame(pd.date_range(interval[0],interval[1]), columns=['date']) 493 | 494 | num_tot = len(Table) 495 | serie_cos_normal = [] 496 | for i in range(num_tot): 497 | serie_cos_normal.append(np.random.normal(2*i,scale=50)+np.cos(200*np.pi*i/num_tot)*500) #np.sin(200*np.pi*i/num_tot)*10+ 498 | 499 | plt.plot(serie_cos_normal) 500 | plt.show() 501 | 502 | Table['serie_cos_normal'] = serie_cos_normal 503 | Table.set_index('date',inplace=True) 504 | 505 | # Adding to the covariates_var 506 | covariates_var.append(Table) 507 | 508 | # var 2 509 | Table = pd.DataFrame(pd.date_range(interval[0],interval[1]), columns=['date']) 510 | 511 | num_tot = len(Table) 512 | serie_tan_normal = [] 513 | for i in range(num_tot): 514 | serie_tan_normal.append(np.random.normal(2*i,scale=50)+np.tan(200*np.pi*i/num_tot)*500) #np.sin(200*np.pi*i/num_tot)*10+ 515 | 516 | plt.plot(serie_tan_normal) 517 | plt.show() 518 | 519 | Table['serie_tan_normal'] = serie_tan_normal 520 | Table.set_index('date',inplace=True) 521 | 522 | # Adding to the covariates_var 523 | covariates_var.append(Table) 524 | 525 | return covariates_var 526 | 527 | def main(data_path,data_covariates_path,date_column,cast_float16,window_size,stride_size,test_ratio,save_name): 528 | 529 | # Data loading 530 | if data_path : 531 | print('## Data reading :') 532 | if date_column : 533 | Table = pd.read_csv(data_path) 534 | Table[date_column] = pd.to_datetime(Table[date_column]) 535 | Table.set_index(date_column,inplace=True) 536 | else : 537 | Table = pd.read_csv(data_path, index_col=0, parse_dates=True) 538 | 539 | if cast_float16 : 540 | Table = Table.astype('float16') 541 | inf_index = Table.index[np.isinf(Table).any(1)] 542 | inf_col = Table.columns.to_series()[np.isinf(Table).any()] 543 | else : 544 | print('## Data generation :') 545 | Table = generate_fake_data() 546 | 547 | if data_covariates_path: 548 | print('## Data covariates reading :') 549 | covariates_var = [] 550 | for file in os.listdir(data_covariates_path): 551 | if file.endswith(".csv"): 552 | if date_column : 553 | var = pd.read_csv(data_covariates_path+'/'+file) 554 | var[date_column] = pd.to_datetime(var[date_column]) 555 | var.set_index(date_column, inplace=True) 556 | else : 557 | var = pd.read_csv(data_covariates_path+'/'+file, index_col = 0, parse_date=True) 558 | covariates_var.append(var) 559 | else : 560 | covariates_var = None 561 | if not data_path : 562 | covariates_var = generate_fake_data_covariates() 563 | 564 | print('# Preparing the data :') 565 | 566 | # Global param 567 | 568 | print("\t # Window_size :",window_size) 569 | print("\t # Stride_size :",stride_size) 570 | print("\t # test_ratio :",test_ratio) 571 | 572 | l_total = len(Table) 573 | 574 | # Train and Test split 575 | train_start = Table.index[0] 576 | train_end = Table.index[int(l_total*(1-test_ratio))] 577 | 578 | freq = pd.infer_freq(Table.index) 579 | 580 | freq = to_offset(freq) 581 | 582 | test_start = train_end - freq*8 #Table.index[int(l_total*(1-test_ratio))] 583 | test_end = Table.index[-1] 584 | 585 | # List of time series 586 | timeseries_list = np.array(list(Table.columns)) 587 | 588 | train_data = Table[train_start:train_end].values 589 | test_data = Table[test_start:test_end].values 590 | 591 | print('\t #train_data index range:',train_start,train_end) 592 | print('\t #test_data index range:',test_start,test_end) 593 | 594 | # Covariate variables preparation 595 | 596 | print('# Covariate variables preparation :') 597 | if not covariates_var is None : 598 | train_covariates_var = [] 599 | for var in covariates_var : 600 | train_covariates_var.append(var[train_start:train_end].values[:,:,None]) 601 | train_covariates_var = np.concatenate(train_covariates_var,axis=2) 602 | 603 | test_covariates_var = [] 604 | for var in covariates_var : 605 | test_covariates_var.append(var[test_start:test_end].values[:,:,None]) 606 | test_covariates_var = np.concatenate(test_covariates_var,axis=2) 607 | else : 608 | train_covariates_var = None 609 | test_covariates_var = None 610 | 611 | ## Train covariates 612 | covariates_train = gen_covariates(Table[train_start:train_end].index) 613 | ## Test covariates 614 | covariates_test = gen_covariates(Table[test_start:test_end].index) 615 | 616 | # data_start 617 | data_start_train = (train_data!=0).argmax(axis=0) #find first nonzero value in each time series 618 | data_start_test = (test_data!=0).argmax(axis=0) #find first nonzero value in each time series 619 | 620 | # Apply the transformation 621 | print('# Transforming the data :') 622 | 623 | window_size_train = window_size 624 | window_size_test = window_size 625 | 626 | xtrain_input, label_train, v_input_train, timeseries_dict_train, transformation_covariates_var, transformation_covariates = prep_data(train_data, window_size=window_size_train, stride_size=stride_size, data_start=data_start_train, timeseries_list=timeseries_list, covariates=covariates_train, covariates_var=train_covariates_var) 627 | xtest_input, label_test, v_input_test, timeseries_dict_test, _, _ = prep_data(test_data, window_size=window_size_test, stride_size=stride_size, data_start=data_start_test, timeseries_list=timeseries_list, covariates=covariates_test, covariates_var=test_covariates_var,transformation_covariates_used = transformation_covariates, transformation_covariates_var_used = transformation_covariates_var) 628 | 629 | # Saving 630 | save_path = 'data_prepared/' 631 | os.makedirs(save_path+save_name, exist_ok = True) 632 | save_path = save_path+save_name 633 | 634 | print('# Saving th data .') 635 | print('\t # Used save_name:',save_name) 636 | save_dataset(save_path,save_name,xtrain_input,v_input_train,label_train,timeseries_dict_train,transformation_covariates_var, transformation_covariates,xtest_input,v_input_test,label_test,timeseries_dict_test, [train_start,train_end,test_start,test_end]) 637 | 638 | if __name__ == "__main__": 639 | print('# Loading the data :') 640 | 641 | # Parsing the arguments 642 | parser = argparse.ArgumentParser() 643 | parser.add_argument('--data-path', type=str, dest='data_path', help='data csv path', default=None) 644 | parser.add_argument('--data-covariates-path', type=str, dest='data_covariates_path', help='data csv path', default=None) 645 | parser.add_argument('--time-column-name', type=str, dest='date_column', help='timme column name', default=None) 646 | parser.add_argument('--save-name', type=str, dest='save_name', help='Save name', default='data_prepared') 647 | 648 | parser.add_argument('--window-size', type=int, dest='window_size', help='window_size', default=100) 649 | parser.add_argument('--stride-size', type=int, dest='stride_size', help='stride_size', default=4) 650 | parser.add_argument('--test-ratio', type=float, dest='test_ratio', help='test_ratio', default=0.30) 651 | 652 | parser.add_argument('--cast-float16', type=bool, dest='cast_float16', help='cast_float16', default=False) 653 | 654 | args = parser.parse_args() 655 | 656 | # Extracting arguments : 657 | data_path = args.data_path 658 | data_covariates_path = args.data_covariates_path 659 | date_column = args.date_column 660 | cast_float16 = args.cast_float16 661 | 662 | window_size = args.window_size 663 | stride_size = args.stride_size 664 | test_ratio = args.test_ratio 665 | save_name = args.save_name 666 | 667 | # Main 668 | main(data_path,data_covariates_path,date_column,cast_float16,window_size,stride_size,test_ratio,save_name) 669 | 670 | -------------------------------------------------------------------------------- /data_prepared/.placeholder: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/data_prepared/.placeholder -------------------------------------------------------------------------------- /data_raw/.placeholder: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/data_raw/.placeholder -------------------------------------------------------------------------------- /elect_preprocessing.py: -------------------------------------------------------------------------------- 1 | from io import BytesIO 2 | from urllib.request import urlopen 3 | from zipfile import ZipFile 4 | import os 5 | import pandas as pd 6 | 7 | 8 | if __name__ == '__main__': 9 | save_name = 'elect' 10 | name = 'LD2011_2014.txt' 11 | save_path = os.path.join('data_raw', save_name) 12 | if not os.path.exists(save_path): 13 | os.makedirs(save_path) 14 | 15 | csv_path = os.path.join(save_path, name) 16 | if not os.path.exists(csv_path): 17 | zipurl = 'https://archive.ics.uci.edu/ml/machine-learning-databases/00321/LD2011_2014.txt.zip' 18 | print('Start download') 19 | with urlopen(zipurl) as zipresp: 20 | print('Start unzip') 21 | with ZipFile(BytesIO(zipresp.read())) as zfile: 22 | zfile.extractall(save_path) 23 | print('Treating and saving') 24 | data_frame = pd.read_csv(csv_path, sep=";", index_col=0, parse_dates=True, decimal=',') 25 | #data_frame = data_frame.resample('1H',label = 'left',closed = 'right').sum() 26 | data_frame.fillna(0, inplace=True) 27 | data_frame.to_csv(csv_path) -------------------------------------------------------------------------------- /elect_test.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import os 3 | import json 4 | 5 | data_path = 'data_raw/elect/LD2011_2014.txt' 6 | data = pd.read_csv(data_path, index_col=0, parse_dates=True) 7 | 8 | 9 | meta_information_path = 'data_prepared/elect/meta_information.json' 10 | with open(meta_information_path, 'r') as file: 11 | meta_information = json.load(file) 12 | 13 | test_start = meta_information['test_start'] 14 | 15 | loc_end = data.index.get_loc(test_start) 16 | data_test = data.iloc[loc_end-100:] 17 | 18 | os.makedirs('data_raw/elect_prediction/',exist_ok=True) 19 | data_test.iloc[:400].to_csv('data_raw/elect_prediction/500_LD2011_2014.txt') 20 | -------------------------------------------------------------------------------- /evaluate.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | import torch.nn.functional as F 4 | 5 | import numpy as np 6 | from torch.autograd import Variable 7 | 8 | import math 9 | import random 10 | from torchtext import data, datasets, vocab 11 | 12 | import os 13 | 14 | class QuantileLoss(nn.Module): 15 | def __init__(self, quantiles): 16 | super().__init__() 17 | self.quantiles = quantiles 18 | 19 | def forward(self, target, preds_down,preds_middle,preds_up): 20 | preds = torch.cat([preds_down,preds_middle,preds_up], axis=2) 21 | assert not target.requires_grad 22 | assert preds.size(0) == target.size(0) 23 | losses = [] 24 | for i, q in enumerate(self.quantiles): 25 | errors = target[:,:,0] - preds[:,:,i] 26 | losses.append( 27 | torch.max( 28 | (q-1) * errors, 29 | q * errors 30 | ).unsqueeze(1)) 31 | loss = torch.mean( 32 | torch.sum(torch.cat(losses, dim=1), dim=1)) 33 | return loss 34 | 35 | def loss_fct(mu, sigma, labels, loss_kind): 36 | if loss_kind=='nll_dist_normal': 37 | indices= (labels != 0) 38 | log_likelihood = torch.distributions.Normal(mu[indices], sigma[indices]).log_prob(labels[indices]) 39 | loss = -log_likelihood.mean() 40 | elif loss_kind == 'nll_dist_lognormal': 41 | indices= (labels != 0) 42 | log_likelihood = torch.distributions.log_normal.LogNormal(mu[indices], sigma[indices]).log_prob(labels[indices]) 43 | loss = -log_likelihood.mean() 44 | elif loss_kind == 'quantileloss_dist_normal': 45 | lossQ = QuantileLoss([0.1587,0.5,0.8413]) 46 | preds_middle = mu 47 | preds_downs = preds_middle - sigma 48 | preds_up = preds_middle + sigma 49 | loss = lossQ(labels[:,:,None],preds_downs,preds_middle,preds_up) 50 | return loss 51 | 52 | 53 | def evaluate(model, x_test, tokens_tst_idx, v_tst, windows_size, sampling = False, number_of_samples = 100, scaled_param = False): 54 | """ 55 | Function to apply the model on a batch and get the evaluation 56 | model : The model to use for the evaluation 57 | x_test: 3D array, 58 | tokens_tst_idx, v_tst, windows_size, sampling = False, number_of_samples = 100, scaled_param = False 59 | 60 | """ 61 | batch_size = x_test.shape[0] 62 | # v_tst is expanded, which means it has the shape (batch_size, windows_size, 2) 63 | 64 | if sampling : 65 | # Sample from the distribution to estimate 66 | samples = torch.empty((number_of_samples,batch_size, windows_size,1)) 67 | 68 | # Distribution parameters 69 | mu, sigma = model(x = x_test, tokens = tokens_tst_idx) 70 | 71 | scaled_mu = mu[:,:,0] * v_tst[:,:,0] + v_tst[:,:,1] 72 | scaled_mu = scaled_mu[:,:,None] 73 | 74 | scaled_sigma = sigma[:,:,0] * v_tst[:,:,0] 75 | scaled_sigma = scaled_sigma[:,:,None] 76 | 77 | # Sampling loop 78 | distributions = torch.distributions.log_normal.LogNormal(mu, sigma) 79 | for k in range(number_of_samples): 80 | sample = distributions.sample() 81 | if scaled_param : 82 | sample = sample[:,:,0]* v_tst[:,:,0] + v_tst[:,:,1] 83 | sample = sample[:,:,None] 84 | samples[k,:,:] = sample 85 | 86 | estimation = samples.median(dim=0)[0] * v_tst[:,:,:1] + v_tst[:,:,1:] 87 | 88 | if scaled_param : 89 | return samples, estimation, scaled_mu,scaled_sigma 90 | else : 91 | return samples, estimation, mu, sigma 92 | 93 | else: 94 | # Distribution parameters 95 | mu, sigma = model(x = x_test, tokens = tokens_tst_idx) 96 | 97 | scaled_mu = mu[:,:,0] * v_tst[:,:,0] + v_tst[:,:,1] 98 | scaled_mu = scaled_mu[:,:,None] 99 | 100 | scaled_sigma = sigma[:,:,0] * v_tst[:,:,0] 101 | scaled_sigma = scaled_sigma[:,:,None] 102 | 103 | if scaled_param : 104 | estimation = scaled_mu 105 | else: 106 | estimation = mu 107 | return estimation, scaled_mu, scaled_sigma 108 | 109 | 110 | def test_evaluation(model,X_tst_check, tokens_tst_idx, v_tst, labels_tst, windows_size, first_affect=False, sampling=True, number_of_samples=25,loss_kind = 'nll_dist_normal'): 111 | """ 112 | Function to compute the predictions and the losses for the test batchs 113 | # X_tst_check : (batch, global_window, vars) : the batch to manage 114 | # tokens_tst_idx : (batch,) : The list of tokens of the batchs 115 | # v_tst : (batch,2) : The scale of serie for each batch 116 | # labels_tst : The serie to predict 117 | # windows_size: The model window size 118 | # first_affect : {True, False} : To make the next prediction based on the first window predicted values or not (using the authentic ones) 119 | # sampling : {True, False} : To make predictions based on samling or not (the mean of the distribution is used) 120 | # number_of_samples : If using sampling, the number of samples 121 | 122 | returns : 123 | # predicted_serie : (batch, global_window, 1) : The predicted serie 124 | # v_tst : (batch,2) : The scale of serie for each batch 125 | # labels_tst : (batch, global_window) : The true value of the serie 126 | # tst_loss : the mean loss 127 | # mus : (windows_total,batch,window_size,1) : The predicted unscaled mean predicted by the model 128 | # sigmas : (windows_total,batch,window_size,1) : The predicted unscaled sigma predicted by the model 129 | """ 130 | 131 | # Parameters : 132 | tst_global_window_size = X_tst_check.shape[1] # which is the train_windows_size that come in the batch == 62 133 | batch_size = X_tst_check.shape[0] 134 | stride = 1 # it's 1 to predict next step, if not (e.g 4), there would be stride-1 (e.g 4-1) instant not predicted each time 135 | windows_total = (tst_global_window_size - (windows_size-stride))//stride # total number of windows 136 | 137 | # Evaluation : 138 | sampling = sampling 139 | number_of_samples = number_of_samples 140 | 141 | st = 0 142 | nd = st + windows_size 143 | 144 | mus = torch.empty((int(windows_total),batch_size, windows_size,1)) 145 | sigmas= torch.empty((int(windows_total),batch_size, windows_size,1)) 146 | 147 | predicted_serie = torch.empty((batch_size, tst_global_window_size,1)) 148 | 149 | 150 | for i in range(int(windows_total)): 151 | 152 | # Xtest extraction 153 | x_test = X_tst_check[:,st:nd,:] 154 | 155 | # Model application : 156 | 157 | ### Expand the v_tst to the shape (batch, windows_size,2), since the scale is the same for all instants in the window (due to the small size difference between the learning window and the test window) 158 | v_tst_expanded = v_tst.expand(windows_size,batch_size,2).transpose(1,0) 159 | if sampling: 160 | samples, estimation, mu, sigma = evaluate(model, x_test, tokens_tst_idx, v_tst_expanded, windows_size, 161 | sampling, number_of_samples) 162 | else: 163 | estimation, mu, sigma = evaluate(model, x_test, tokens_tst_idx, v_tst_expanded, windows_size, 164 | sampling, number_of_samples) 165 | # mu & sigma history 166 | mus[i,:,:,:] = mu 167 | sigmas[i,:,:,:] = sigma 168 | 169 | # Changing the value in the X_tst to the predicted next value to make next prediction: 170 | 171 | if i == 0 and first_affect: # the first window, we affect all the predicted elements, so next prediction would be based on those 172 | X_tst_check[:,st:nd,0] = estimation[:,st:nd,0] 173 | 174 | if nd < tst_global_window_size : 175 | X_tst_check[:,nd,0] = estimation[:,-1,0] 176 | 177 | # Predicted elements : 178 | 179 | if i == 0 : 180 | predicted_serie[:,st:nd,0] = estimation[:,st:nd,0] # predicted affected 181 | 182 | if nd < tst_global_window_size : 183 | predicted_serie[:,nd,0] = estimation[:,-1,0] 184 | 185 | # Loss calculation : 186 | tst_loss = loss_fct(mu, sigma, labels_tst[:,st:nd],loss_kind) 187 | st+=stride 188 | nd+=stride 189 | 190 | tst_loss = tst_loss/int(windows_total) 191 | return predicted_serie, v_tst, labels_tst, tst_loss, mus, sigmas -------------------------------------------------------------------------------- /models/elect/ConvTransformer_elect2020-05-10 11:13:36.815714.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/models/elect/ConvTransformer_elect2020-05-10 11:13:36.815714.pth -------------------------------------------------------------------------------- /models/elect/ConvTransformer_elect2020-05-10 11:52:33.147730.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/models/elect/ConvTransformer_elect2020-05-10 11:52:33.147730.pth -------------------------------------------------------------------------------- /models/elect/model_config.json: -------------------------------------------------------------------------------- 1 | {"k": 4, "headers": 3, "depth": 3, "seq_length": 100, "kernel_size": 6, "num_tokens": 370, "path": "models//elect//ConvTransformer_elect2020-05-10 11:52:33.147730.pth"} -------------------------------------------------------------------------------- /models/elect/train_config.json: -------------------------------------------------------------------------------- 1 | {"num_epochs": 5, "lr_warmup": 1000, "learning_rate": 0.001, "default": true, "batch_size": 32, "predict_batch_size": 32} -------------------------------------------------------------------------------- /models/electricity_dt_1/ConvTransformer_electricity_dt_1_kernel6_start_3heads.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/models/electricity_dt_1/ConvTransformer_electricity_dt_1_kernel6_start_3heads.pth -------------------------------------------------------------------------------- /models/electricity_dt_1/model_config.json: -------------------------------------------------------------------------------- 1 | {"k": 4, "headers": 3, "depth": 3, "seq_length": 192, "kernel_size": 6, "num_tokens": 370, "path" : "/home/lasngd/Projects/Pytorch/ConvTransformer/models/electricity_dt_1/ConvTransformer_electricity_dt_1_kernel6_start_3heads.pth"} 2 | -------------------------------------------------------------------------------- /models/electricity_dt_1/train_config.json: -------------------------------------------------------------------------------- 1 | {"num_epochs": 5, "lr_warmup": 1000, "learning_rate": 0.001, "default": true, "batch_size": 32, "predict_batch_size": 32} -------------------------------------------------------------------------------- /net.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | import torch.nn.functional as F 4 | 5 | import numpy as np 6 | from torch.autograd import Variable 7 | from datetime import datetime 8 | 9 | import math 10 | import random 11 | #import tqdm 12 | from torchtext import data, datasets, vocab 13 | 14 | import os 15 | from torch.utils.data import DataLoader, Dataset, Sampler 16 | from torch.utils.data.sampler import RandomSampler 17 | 18 | 19 | 20 | # Self Attention Class 21 | class SelfAttentionConv(nn.Module): 22 | def __init__(self, k, headers = 8, kernel_size = 5, mask_next = True, mask_diag = False): 23 | super().__init__() 24 | 25 | self.k, self.headers, self.kernel_size = k, headers, kernel_size 26 | self.mask_next = mask_next 27 | self.mask_diag = mask_diag 28 | 29 | h = headers 30 | 31 | # Query, Key and Value Transformations 32 | 33 | padding = (kernel_size-1) 34 | self.padding_opertor = nn.ConstantPad1d((padding,0), 0) 35 | 36 | self.toqueries = nn.Conv1d(k, k*h, kernel_size, padding=0 ,bias=True) 37 | self.tokeys = nn.Conv1d(k, k*h, kernel_size, padding=0 ,bias=True) 38 | self.tovalues = nn.Conv1d(k, k*h, kernel_size = 1 , padding=0 ,bias=False) # No convolution operated 39 | 40 | # Heads unifier 41 | self.unifyheads = nn.Linear(k*h, k) 42 | def forward(self, x): 43 | 44 | # Extraction dimensions 45 | b, t, k = x.size() # batch_size, number_of_timesteps, number_of_time_series 46 | 47 | 48 | # Checking Embedding dimension 49 | assert self.k == k, 'Number of time series '+str(k)+' didn t much the number of k '+str(self.k)+' in the initiaalization of the attention layer.' 50 | h = self.headers 51 | 52 | # Transpose to see the different time series as different channels 53 | x = x.transpose(1,2) 54 | x_padded = self.padding_opertor(x) 55 | 56 | # Query, Key and Value Transformations 57 | queries = self.toqueries(x_padded).view(b,k,h,t) 58 | keys = self.tokeys(x_padded).view(b,k,h,t) 59 | values = self.tovalues(x).view(b,k,h,t) 60 | 61 | # Transposition to return the canonical format 62 | queries = queries.transpose(1,2) # batch, header, time serie, time step (b, h, k, t) 63 | queries = queries.transpose(2,3) # batch, header, time step, time serie (b, h, t, k) 64 | 65 | values = values.transpose(1,2) # batch, header, time serie, time step (b, h, k, t) 66 | values = values.transpose(2,3) # batch, header, time step, time serie (b, h, t, k) 67 | 68 | keys = keys.transpose(1,2) # batch, header, time serie, time step (b, h, k, t) 69 | keys = keys.transpose(2,3) # batch, header, time step, time serie (b, h, t, k) 70 | 71 | 72 | # Weights 73 | queries = queries/(k**(.25)) 74 | keys = keys/(k**(.25)) 75 | 76 | queries = queries.transpose(1,2).contiguous().view(b*h, t, k) 77 | keys = keys.transpose(1,2).contiguous().view(b*h, t, k) 78 | values = values.transpose(1,2).contiguous().view(b*h, t, k) 79 | 80 | 81 | weights = torch.bmm(queries, keys.transpose(1,2)) 82 | 83 | 84 | ## Mask the upper & diag of the attention matrix 85 | if self.mask_next : 86 | if self.mask_diag : 87 | indices = torch.triu_indices(t ,t , offset=0) 88 | weights[:, indices[0], indices[1]] = float('-inf') 89 | else : 90 | indices = torch.triu_indices(t ,t , offset=1) 91 | weights[:, indices[0], indices[1]] = float('-inf') 92 | 93 | # Softmax 94 | weights = F.softmax(weights, dim=2) 95 | 96 | # Output 97 | output = torch.bmm(weights, values) 98 | output = output.view(b,h,t,k) 99 | output = output.transpose(1,2).contiguous().view(b,t, k*h) 100 | 101 | return self.unifyheads(output) # shape (b,t,k) 102 | 103 | 104 | # Conv Transforme Block 105 | 106 | class ConvTransformerBLock(nn.Module): 107 | def __init__(self, k, headers, kernel_size = 5, mask_next = True, mask_diag = False, dropout_proba = 0.2): 108 | super().__init__() 109 | 110 | # Self attention 111 | self.attention = SelfAttentionConv(k, headers, kernel_size, mask_next, mask_diag) 112 | 113 | # First & Second Norm 114 | self.norm1 = nn.LayerNorm(k) 115 | self.norm2 = nn.LayerNorm(k) 116 | 117 | # Feed Forward Network 118 | self.feedforward = nn.Sequential( 119 | nn.Linear(k, 4*k), 120 | nn.ReLU(), 121 | nn.Linear(4*k, k) 122 | ) 123 | # Dropout funtcion & Relu: 124 | self.dropout = nn.Dropout(p = dropout_proba) 125 | self.activation = nn.ReLU() 126 | 127 | def forward(self, x, train=False): 128 | 129 | # Self attention + Residual 130 | x = self.attention(x) + x 131 | 132 | # Dropout attention 133 | if train : 134 | x = self.dropout(x) 135 | 136 | # First Normalization 137 | x = self.norm1(x) 138 | 139 | # Feed Froward network + residual 140 | x = self.feedforward(x) + x 141 | 142 | # Second Normalization 143 | x = self.norm2(x) 144 | 145 | return x 146 | 147 | 148 | # Forcasting Conv Transformer : 149 | class ForcastConvTransformer(nn.Module): 150 | def __init__(self, k, headers, depth, seq_length, kernel_size = 5, mask_next = True, mask_diag = False, dropout_proba = 0.2, num_tokens = None): 151 | super().__init__() 152 | # Embedding 153 | self.tokens_in_count = False 154 | if num_tokens : 155 | self.tokens_in_count = True 156 | self.token_embedding = nn.Embedding(num_tokens, k) 157 | 158 | # Embedding the position 159 | self.position_embedding = nn.Embedding(seq_length, k) 160 | 161 | # Number of time series 162 | self.k = k 163 | self.seq_length = seq_length 164 | 165 | # Transformer blocks 166 | tblocks = [] 167 | for t in range(depth): 168 | tblocks.append(ConvTransformerBLock(k, headers, kernel_size, mask_next, mask_diag, dropout_proba)) 169 | self.TransformerBlocks = nn.Sequential(*tblocks) 170 | 171 | # Transformation from k dimension to numClasses 172 | self.topreSigma = nn.Linear(k, 1) 173 | self.tomu = nn.Linear(k, 1) 174 | self.plus = nn.Softplus() 175 | 176 | def forward(self, x, tokens = None): 177 | b ,t ,k = x.size() 178 | 179 | # checking that the given batch had same number of time series as the BLock had 180 | assert k == self.k, 'The k :'+str(self.k)+' number of timeseries given in the initialization is different than what given in the x :'+str(k) 181 | assert t == self.seq_length, 'The lenght of the timeseries given t '+str(t)+' miss much with the lenght sequence given in the Tranformers initialisation self.seq_length: '+str(self.seq_length) 182 | 183 | # Position embedding 184 | pos = torch.arange(t) 185 | self.pos_emb = self.position_embedding(pos).expand(b,t,k) 186 | 187 | # Checking token embedding 188 | assert self.tokens_in_count == (not (tokens is None)), 'self.tokens_in_count = '+str(self.tokens_in_count)+' should be equal to (not (tokens is None)) = '+str((not (tokens is None))) 189 | if not (tokens is None) : 190 | ## checking that the number of tockens corresponde to the number of batch elements 191 | assert tokens.size(0) == b 192 | self.tok_emb = self.token_embedding(tokens) 193 | self.tok_emb = self.tok_emb.expand(t,b,k).transpose(0,1) 194 | 195 | # Adding Pos Embedding and token Embedding to the variable 196 | if not (tokens is None): 197 | x = self.pos_emb + self.tok_emb + x 198 | else: 199 | x = self.pos_emb + x 200 | 201 | # Transformer : 202 | x = self.TransformerBlocks(x) 203 | mu = self.tomu(x) 204 | presigma = self.topreSigma(x) 205 | sigma = self.plus(presigma) 206 | 207 | return mu, sigma 208 | 209 | # Loading data 210 | class TrainDataset(Dataset): 211 | def __init__(self, data_path, data_name): 212 | print('data_name :', data_path) 213 | self.data = np.load(os.path.join(data_path, f'train_data_{data_name}.npy')) 214 | self.v = np.load(os.path.join(data_path, f'train_v_{data_name}.npy')) 215 | self.label = np.load(os.path.join(data_path, f'train_label_{data_name}.npy')) 216 | self.train_len = self.data.shape[0] 217 | def __len__(self): 218 | return self.train_len 219 | 220 | def __getitem__(self, index): 221 | return (self.data[index,:,:-1],int(self.data[index,0,-1]), self.v[index], self.label[index]) 222 | 223 | class TestDataset(Dataset): 224 | def __init__(self, data_path, data_name): 225 | self.data = np.load(os.path.join(data_path, f'test_data_{data_name}.npy')) 226 | self.v = np.load(os.path.join(data_path, f'test_v_{data_name}.npy')) 227 | self.label = np.load(os.path.join(data_path, f'test_label_{data_name}.npy')) 228 | self.test_len = self.data.shape[0] 229 | def __len__(self): 230 | return self.test_len 231 | 232 | def __getitem__(self, index): 233 | return (self.data[index,:,:-1],int(self.data[index,0,-1]),self.v[index],self.label[index]) 234 | 235 | class WeightedSampler(Sampler): 236 | def __init__(self, data_path, data_name, replacement=True): 237 | v = np.load(os.path.join(data_path, f'train_v_{data_name}.npy')) 238 | self.weights = torch.as_tensor(np.abs(v[:,0])/np.sum(np.abs(v[:,0])), dtype=torch.double) 239 | self.num_samples = self.weights.shape[0] 240 | self.replacement = replacement 241 | 242 | def __iter__(self): 243 | return iter(torch.multinomial(self.weights, self.num_samples, self.replacement).tolist()) 244 | 245 | def __len__(self): 246 | return self.num_samples -------------------------------------------------------------------------------- /prediction.py: -------------------------------------------------------------------------------- 1 | from data_prep import * 2 | from net import * 3 | import pickle 4 | from datetime import datetime, timedelta 5 | from pandas.tseries.frequencies import to_offset 6 | import numpy as np 7 | import pandas as pd 8 | 9 | def main(data_path, data_covariates_path, date_column, save_path, prediction_name, model_config_path, meta_information_path, steps, flag_real_plot) : 10 | # model config : 11 | with open(model_config_path, 'r') as file: 12 | model_config = json.load(file) 13 | 14 | # Metainformation : 15 | with open(meta_information_path, 'r') as file: 16 | meta_information = json.load(file) 17 | 18 | # Covariates and Covariates_var 19 | with open(meta_information['transformation_covariates_var_path'], 'rb') as file: 20 | transformation_covariates_var = pickle.load(file) 21 | 22 | with open(meta_information['transformation_covariates'], 'rb') as file: 23 | transformation_covariates = pickle.load(file) 24 | 25 | flag_covariate = not (transformation_covariates is None) 26 | flag_covariate_var = not (transformation_covariates_var is None) 27 | 28 | if (flag_covariate_var == True ) and (data_covariates_path is None) : 29 | raise ValueError('Data have a covariates_var and no path to it is gived !') 30 | 31 | 32 | # Model Definition : 33 | window_size = model_config['seq_length'] 34 | number_of_vars = model_config['k'] 35 | num_timeseries_kinds = model_config['num_tokens'] 36 | 37 | headers = model_config['headers'] 38 | depth = model_config['depth'] 39 | kernel_size = model_config['kernel_size'] 40 | PATH = model_config['path'] 41 | 42 | train_end = meta_information['train_end'] 43 | 44 | ############################################################################# 45 | 46 | # Data loading 47 | 48 | print('## Data reading :') 49 | if date_column : 50 | Table = pd.read_csv(data_path) 51 | Table[date_column] = pd.to_datetime(Table[date_column]) 52 | Table.set_index(date_column,inplace=True) 53 | else : 54 | Table = pd.read_csv(data_path, index_col=0, parse_dates=True) 55 | inf_index = Table.index[np.isinf(Table).any(1)] 56 | inf_col = Table.columns.to_series()[np.isinf(Table).any()] 57 | 58 | if data_covariates_path: 59 | print('## Data covariates reading :') 60 | covariates_var = [] 61 | for file in os.listdir(data_covariates_path): 62 | if file.endswith(".csv"): 63 | if date_column : 64 | var = pd.read_csv(data_covariates_path+'/'+file) 65 | var[date_column] = pd.to_datetime(var[date_column]) 66 | var.set_index(date_column, inplace=True) 67 | else : 68 | var = pd.read_csv(data_covariates_path+'/'+file, index_col = 0, parse_date=True) 69 | covariates_var.append(var) 70 | else : 71 | covariates_var = None 72 | 73 | print('# Preparing the data :') 74 | # Train and Test splitfrom pandas.tseries.frequencies import to_offset 75 | 76 | freq = pd.infer_freq(Table.index) 77 | freq = to_offset(freq) 78 | 79 | #predict starting 80 | 81 | test_start = Table.index[0] 82 | test_end = Table.index[-1] 83 | if flag_real_plot: 84 | test_end = Table.index[-2] 85 | assert steps == 1 86 | 87 | 88 | # List of time series 89 | timeseries_list = np.array(list(Table.columns)) 90 | 91 | # covariate variables preparation 92 | test_data = Table[test_start:test_end].values 93 | tst_index = Table[test_start:test_end].index 94 | 95 | if not covariates_var is None : 96 | test_covariates_var = [] 97 | for var in covariates_var : 98 | assert (tst_index.union([tst_index[-1] + i*freq for i in range(steps+1)]) == var[test_start:test_end+steps*freq].index).all(), 'The covariate var '+str(var)+' didn t much steps for prediction : '+str(tst_index.union([tst_index[-1] + i*freq for i in range(steps+1)]))+' != '+str(var.index[test_start:test_end+steps*freq]) 99 | test_covariates_var.append(var[test_start:test_end+steps*freq].values[:,:,None]) 100 | test_covariates_var = np.concatenate(test_covariates_var,axis=2) 101 | else : 102 | test_covariates_var = None 103 | 104 | ## Test 105 | 106 | freq = pd.infer_freq(tst_index) 107 | freq = to_offset(freq) 108 | 109 | if flag_covariate : 110 | covariates_test = gen_covariates(tst_index.union([tst_index[-1] + i*freq for i in range(steps+1)])) 111 | else : 112 | covariates_test = False 113 | 114 | # data_start 115 | data_start_test = (test_data!=0).argmax(axis=0) #find first nonzero value in each time series 116 | 117 | # Apply the transformation 118 | print('# Transforming the data :') 119 | window_size_test = window_size 120 | 121 | device = 'cpu' 122 | if torch.cuda.is_available(): 123 | torch.set_default_tensor_type(torch.cuda.FloatTensor) 124 | device = 'cuda' 125 | 126 | ################### 127 | 128 | # Model 129 | model = ForcastConvTransformer(k = number_of_vars, headers=headers, depth=depth, seq_length=window_size, kernel_size=kernel_size, num_tokens=num_timeseries_kinds) 130 | ##################### 131 | print(PATH) 132 | checkpoint = torch.load(PATH) 133 | model.load_state_dict(checkpoint['model_state_dict']) 134 | model.eval() 135 | 136 | estimation, sigma = prep_data_for_prediction(model, test_data, window_size_test, steps=steps, data_start=data_start_test, timeseries_list=timeseries_list,device = 'cuda', covariates=covariates_test, covariates_var=test_covariates_var,transformation_covariates_used = transformation_covariates, transformation_covariates_var_used = transformation_covariates_var, sampling = False) 137 | 138 | pt = os.path.join(save_path, prediction_name) 139 | os.makedirs(pt, exist_ok = True) 140 | 141 | np.save(pt+'/estimation.npy',estimation) 142 | np.save(pt+'/sigma.npy',sigma) 143 | 144 | 145 | # show a plot 146 | 147 | m= np.random.randint(0,estimation.shape[1]) 148 | if flag_real_plot: 149 | plt.plot(Table[test_start:].iloc[4:,m].values,label='real', c='red') 150 | plt.plot(estimation[4:,m], label='prediction', c='b') 151 | 152 | under_line = estimation[4:,m] - sigma[4:,m] 153 | over_line = estimation[4:,m] + sigma[4:,m] 154 | plt.fill_between(range(len(estimation[4:,m])),under_line, over_line, color='b', alpha=0.2) #std curves. 155 | 156 | if train_end in Table.index : 157 | plt.axvline(Table.index.get_loc(train_end) - 4) 158 | 159 | plt.legend() 160 | plt.title(m) 161 | plt.show() 162 | 163 | # save all the plots 164 | for m in range(estimation.shape[1]): 165 | plt.close() 166 | if flag_real_plot: 167 | plt.plot(Table[test_start:].iloc[4:,m].values,label='real', c='red') 168 | plt.plot(estimation[4:,m], label='prediction', c='b') 169 | 170 | under_line = estimation[4:,m] - sigma[4:,m] 171 | over_line = estimation[4:,m] + sigma[4:,m] 172 | plt.fill_between(range(len(estimation[4:,m])),under_line, over_line, color='b', alpha=0.2) #std curves. 173 | 174 | if train_end in Table.index : 175 | plt.axvline(Table.index.get_loc(train_end) - 4) 176 | 177 | plt.legend() 178 | plt.title(m) 179 | plt.savefig(pt+'/'+str(m)+'.png') 180 | 181 | 182 | if __name__ == "__main__": 183 | # Parsing the arguments 184 | parser = argparse.ArgumentParser() 185 | parser.add_argument('--data-path', type=str, dest='data_path', help='data csv path', default=None) 186 | parser.add_argument('--data-covariates-path', type=str, dest='data_covariates_path', help='data csv path', default=None) 187 | parser.add_argument('--time-column-name', type=str, dest='date_column', help='timme column name', default=None) 188 | parser.add_argument('--save-path', type=str, dest='save_path', help='Save result path', default='data_result') 189 | parser.add_argument('--prediction-name', type=str, dest='prediction_name', help='prediction_name result path', default='data_result') 190 | 191 | parser.add_argument('--steps', type=int, dest='steps', help='window_size', default=1) 192 | 193 | parser.add_argument('--model-config', type=str, dest='model_config', help='model config') 194 | parser.add_argument('--meta-information', type=str, dest='meta_information', help='meta information') 195 | parser.add_argument('--flag-real-plot', type=bool, dest='flag_real_plot', help='flag-real-plot') 196 | 197 | args = parser.parse_args() 198 | 199 | data_path = args.data_path 200 | data_covariates_path = args.data_covariates_path 201 | date_column = args.date_column 202 | save_path = args.save_path 203 | model_config_path = args.model_config 204 | meta_information_path = args.meta_information 205 | prediction_name = args.prediction_name 206 | flag_real_plot = args.flag_real_plot 207 | 208 | 209 | steps = args.steps 210 | main(data_path, data_covariates_path, date_column, save_path, prediction_name, model_config_path, meta_information_path, steps, flag_real_plot) -------------------------------------------------------------------------------- /predictions/.placeholder: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/.placeholder -------------------------------------------------------------------------------- /predictions/elect_01/0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/0.png -------------------------------------------------------------------------------- /predictions/elect_01/1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/1.png -------------------------------------------------------------------------------- /predictions/elect_01/10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/10.png -------------------------------------------------------------------------------- /predictions/elect_01/11.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/11.png -------------------------------------------------------------------------------- /predictions/elect_01/12.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/12.png -------------------------------------------------------------------------------- /predictions/elect_01/13.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/13.png -------------------------------------------------------------------------------- /predictions/elect_01/14.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/14.png -------------------------------------------------------------------------------- /predictions/elect_01/15.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/15.png -------------------------------------------------------------------------------- /predictions/elect_01/16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/16.png -------------------------------------------------------------------------------- /predictions/elect_01/17.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/17.png -------------------------------------------------------------------------------- /predictions/elect_01/18.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/18.png -------------------------------------------------------------------------------- /predictions/elect_01/19.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/19.png -------------------------------------------------------------------------------- /predictions/elect_01/2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/2.png -------------------------------------------------------------------------------- /predictions/elect_01/20.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/20.png -------------------------------------------------------------------------------- /predictions/elect_01/21.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/21.png -------------------------------------------------------------------------------- /predictions/elect_01/22.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/22.png -------------------------------------------------------------------------------- /predictions/elect_01/23.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/23.png -------------------------------------------------------------------------------- /predictions/elect_01/24.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/24.png -------------------------------------------------------------------------------- /predictions/elect_01/25.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/25.png -------------------------------------------------------------------------------- /predictions/elect_01/26.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/26.png -------------------------------------------------------------------------------- /predictions/elect_01/27.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/27.png -------------------------------------------------------------------------------- /predictions/elect_01/28.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/28.png -------------------------------------------------------------------------------- /predictions/elect_01/29.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/29.png -------------------------------------------------------------------------------- /predictions/elect_01/3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/3.png -------------------------------------------------------------------------------- /predictions/elect_01/30.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/30.png -------------------------------------------------------------------------------- /predictions/elect_01/31.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/31.png -------------------------------------------------------------------------------- /predictions/elect_01/32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/32.png -------------------------------------------------------------------------------- /predictions/elect_01/33.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/33.png -------------------------------------------------------------------------------- /predictions/elect_01/34.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/34.png -------------------------------------------------------------------------------- /predictions/elect_01/35.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/35.png -------------------------------------------------------------------------------- /predictions/elect_01/36.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/36.png -------------------------------------------------------------------------------- /predictions/elect_01/37.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/37.png -------------------------------------------------------------------------------- /predictions/elect_01/38.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/38.png -------------------------------------------------------------------------------- /predictions/elect_01/39.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/39.png -------------------------------------------------------------------------------- /predictions/elect_01/4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/4.png -------------------------------------------------------------------------------- /predictions/elect_01/40.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/40.png -------------------------------------------------------------------------------- /predictions/elect_01/41.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/41.png -------------------------------------------------------------------------------- /predictions/elect_01/42.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/42.png -------------------------------------------------------------------------------- /predictions/elect_01/43.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/43.png -------------------------------------------------------------------------------- /predictions/elect_01/44.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/44.png -------------------------------------------------------------------------------- /predictions/elect_01/45.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/45.png -------------------------------------------------------------------------------- /predictions/elect_01/46.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/46.png -------------------------------------------------------------------------------- /predictions/elect_01/47.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/47.png -------------------------------------------------------------------------------- /predictions/elect_01/48.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/48.png -------------------------------------------------------------------------------- /predictions/elect_01/49.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/49.png -------------------------------------------------------------------------------- /predictions/elect_01/5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/5.png -------------------------------------------------------------------------------- /predictions/elect_01/50.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/50.png -------------------------------------------------------------------------------- /predictions/elect_01/51.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/51.png -------------------------------------------------------------------------------- /predictions/elect_01/52.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/52.png -------------------------------------------------------------------------------- /predictions/elect_01/53.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/53.png -------------------------------------------------------------------------------- /predictions/elect_01/54.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/54.png -------------------------------------------------------------------------------- /predictions/elect_01/55.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/55.png -------------------------------------------------------------------------------- /predictions/elect_01/56.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/56.png -------------------------------------------------------------------------------- /predictions/elect_01/57.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/57.png -------------------------------------------------------------------------------- /predictions/elect_01/58.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/58.png -------------------------------------------------------------------------------- /predictions/elect_01/59.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/59.png -------------------------------------------------------------------------------- /predictions/elect_01/6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/6.png -------------------------------------------------------------------------------- /predictions/elect_01/60.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/60.png -------------------------------------------------------------------------------- /predictions/elect_01/61.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/61.png -------------------------------------------------------------------------------- /predictions/elect_01/62.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/62.png -------------------------------------------------------------------------------- /predictions/elect_01/63.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/63.png -------------------------------------------------------------------------------- /predictions/elect_01/64.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/64.png -------------------------------------------------------------------------------- /predictions/elect_01/65.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/65.png -------------------------------------------------------------------------------- /predictions/elect_01/66.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/66.png -------------------------------------------------------------------------------- /predictions/elect_01/67.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/67.png -------------------------------------------------------------------------------- /predictions/elect_01/68.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/68.png -------------------------------------------------------------------------------- /predictions/elect_01/69.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/69.png -------------------------------------------------------------------------------- /predictions/elect_01/7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/7.png -------------------------------------------------------------------------------- /predictions/elect_01/70.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/70.png -------------------------------------------------------------------------------- /predictions/elect_01/71.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/71.png -------------------------------------------------------------------------------- /predictions/elect_01/72.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/72.png -------------------------------------------------------------------------------- /predictions/elect_01/73.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/73.png -------------------------------------------------------------------------------- /predictions/elect_01/74.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/74.png -------------------------------------------------------------------------------- /predictions/elect_01/75.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/75.png -------------------------------------------------------------------------------- /predictions/elect_01/76.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/76.png -------------------------------------------------------------------------------- /predictions/elect_01/77.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/77.png -------------------------------------------------------------------------------- /predictions/elect_01/78.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/78.png -------------------------------------------------------------------------------- /predictions/elect_01/79.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/79.png -------------------------------------------------------------------------------- /predictions/elect_01/8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/8.png -------------------------------------------------------------------------------- /predictions/elect_01/80.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/80.png -------------------------------------------------------------------------------- /predictions/elect_01/81.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/81.png -------------------------------------------------------------------------------- /predictions/elect_01/82.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/82.png -------------------------------------------------------------------------------- /predictions/elect_01/83.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/83.png -------------------------------------------------------------------------------- /predictions/elect_01/84.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/84.png -------------------------------------------------------------------------------- /predictions/elect_01/85.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/85.png -------------------------------------------------------------------------------- /predictions/elect_01/86.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/86.png -------------------------------------------------------------------------------- /predictions/elect_01/87.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/87.png -------------------------------------------------------------------------------- /predictions/elect_01/88.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/88.png -------------------------------------------------------------------------------- /predictions/elect_01/89.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/89.png -------------------------------------------------------------------------------- /predictions/elect_01/9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/9.png -------------------------------------------------------------------------------- /predictions/elect_01/90.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/90.png -------------------------------------------------------------------------------- /predictions/elect_01/91.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/91.png -------------------------------------------------------------------------------- /predictions/elect_01/92.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/92.png -------------------------------------------------------------------------------- /predictions/elect_01/93.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/93.png -------------------------------------------------------------------------------- /predictions/elect_01/94.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/94.png -------------------------------------------------------------------------------- /predictions/elect_01/95.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/95.png -------------------------------------------------------------------------------- /predictions/elect_01/96.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/96.png -------------------------------------------------------------------------------- /predictions/elect_01/97.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/97.png -------------------------------------------------------------------------------- /predictions/elect_01/98.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/98.png -------------------------------------------------------------------------------- /predictions/elect_01/99.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/99.png -------------------------------------------------------------------------------- /predictions/elect_01/estimation.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/estimation.npy -------------------------------------------------------------------------------- /predictions/elect_01/sigma.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghsama/ConvTransformerTimeSeries/4813ee1827c32498b616a3b93abbed96b411d32c/predictions/elect_01/sigma.npy -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # Convolution Transformer : 2 | This is first implimentation of 1D convolutional transformer for time series, it is inspired from the article [Enhancing the Locality and Breaking the MemoryBottleneck of Transformer on Time Series Forecasting](https://arxiv.org/pdf/1907.00235.pdf) . 3 | 4 | The model consists of a stack of transformers which takes as input the a window of instance and predict the distribution of the next value. 5 | 6 | The 1D convolution aims to have a better similarity measure in the attention layer. 7 | 8 | The used loss is a Quantile loss or a KL loss (depends on the choice you made). 9 | 10 | # The framwork : 11 | 12 | Giving a list of time series (e.g : electricity consumation in many areas) and a corresponding covariate variables (e.g : weather temperature, precepitation, holidays ..) we aim to predict the next instance distribution (the mean and the standard deviation of a Log Normal). 13 | 14 | # Usage : 15 | Applying the models on the electricity dataset used in the article. 16 | Downloading the dataset : 17 | python elect_preprocessing.py 18 | 19 | Preprocessing the data : 20 | 21 | python data_prep.py --data-path data_raw/elect/LD2011_2014.txt --save-name elect --window-size 100 --stride-size 500 --test-ratio 0.3 22 | 23 | Launching the training : 24 | 25 | python train.py --data-directory data_prepared/elect/ --dataset-name elect --meta-information data_prepared/elect/meta_information_elect.json --model-config models/elect/model_config.json --train-config models/elect/train_config.json --model-save-path models/ 26 | 27 | Extracting a portion of the test data to infeer : 28 | 29 | python elect_test.py 30 | 31 | Applying the prediction : 32 | 33 | python prediction.py --data-path data_raw/elect_prediction/500_LD2011_2014.txt --save-path predictions --steps 1 --model-config models/elect/model_config.json --meta-information data_prepared/elect/meta_information.json --prediction-name elect_01 34 | 35 | Resulats : 36 | 37 | The vertical line is the limit between train and test . 38 | 39 | ![alt text](predictions/elect_01/11.png?raw=true) 40 | ![alt text](predictions/elect_01/20.png?raw=true) 41 | ![alt text](predictions/elect_01/15.png?raw=true) 42 | ![alt text](predictions/elect_01/44.png?raw=true) 43 | 44 | # Steps Explanation : 45 | 46 | **Preprocessing:** 47 | *data_prep.py* : 48 | Preprocessing time series data and covariates and transforming them in the appropriate form. 49 | 50 | - data.csv file : a file containing in columns all the time series kinds and in the lines the time steps + a time column (The instance before the start oof the time series may be filled with 0 ) 51 | - covariate_folder : a folder containing as many CSVs as the covariate variables we have, with eath one having the same structure as the data.csv 52 | 53 | + time series independent covariates : generated automaticlly (e.g : the mounth/ the day corresponding to the timestep) 54 | Arguments : 55 | 56 | --data-path : path to the csv file : it may contain many columns, each for a kind of time series + a time column 57 | --data-covariates-path : path to the directory containing the covariate variables dataframes, it needs to have the same number of time series (columns) as in the data-path csv. 58 | --time-column-name : the column which contains the timesteps 59 | --save-name : the name used for the preprocessed data saving 60 | 61 | --window-size : the window size used in the training and the prediction 62 | --stride-size : stride between the generated windows for the training 63 | --test-ratio : the ratio of the test data 64 | 65 | Outputs : In */data_prepared/save_name/* directory 66 | 67 | - train and test npy arrays preprocessed 68 | - meta_information json : number of time series, window_size and number of covariates 69 | - covariate normalisation transformations 70 | 71 | **Training:** 72 | *train.py* : 73 | Creating and training a Transformers blocks model with a Quantile/KL loss on a LogNormal/Binomial 74 | 75 | - directory of the preprocessed train and test data 76 | - model configuration and training configuration 77 | 78 | Arguments : 79 | 80 | --data-directory : the path to the directory of the saved preprocesed data (*/data_prepared/save_name/* ) 81 | --dataset-name : the save name used during the preprocessing (*save_name*) 82 | --meta-information : the path to the jason of the meta_information (*/data_prepared/save_name/meta_information_save_name.json* ) 83 | --model-config : path to json which contains the model architecture : 84 | Number of heads in the transformer 85 | Number of the transformers blocks 86 | 1d convolution kernel size 87 | e.g : {"headers": 3, "depth": 3, "kernel_size": 6} 88 | --train-config : path to json which contains the training parameters : 89 | batch_size : the batch size during training 90 | predict_batch_size : the batch size during prediction test 91 | num_epochs : the number of epochs 92 | lr_warmup : the number of warmup 93 | learning_rate : the learning rate 94 | e.g : {"batch_size": 32, "predict_batch_size": 32, "num_epochs": 7, "lr_warmup":1000, "learning_rate":0.001} 95 | --model-save-path : path to directory where to save the model and the configuration json. 96 | 97 | Outputs : In *save_path/* directory 98 | 99 | - the configuration of the created model json file: 100 | k : Number of time series + number of covriates (the static ones -e.g : day/hour- and dynamic ones -e.g : temperature) 101 | headers : Number of heads in the transformers 102 | depth : Number of the transformers blocks 103 | seq_length : window size 104 | kernel_size : 1d convolution kernel size used in the attention layers inside the transformers blocks 105 | num_tokens : Number of time series kinds used in the train (the number of columns in the data.csv ) 106 | path : path to the model inside *save_path/* 107 | - best trained model : checkpoint of the model at the lowest test loss 108 | **Inference:** 109 | *prediction.py* : 110 | Making inference for future steps 111 | 112 | - trained model with its own configuration 113 | - data transformers 114 | - time series to infeer + their coressponding covariate variables 115 | 116 | Arguments : 117 | 118 | --data-path : path to the prediction table with same columns as in the training 119 | --data-covariates-path : path to the directory that contains the covariates tables, having *steps* more timestamps than the data Table 120 | --time-column-name : the column which contains the timesteps 121 | --save-path : directory path to save in 122 | --steps : number od steps to predict (!!! NORMALY need to be 1, but you can inferre for more with the hypothesis that the prediction you make are 100% accurate) 123 | --model-config : path to model config path, used in the training 124 | --meta-information : path to the meta data used in the training 125 | --prediction-name : folder name used to save the predictions 126 | --flag-real-plot : plot the real values (need to have steps == 1) 127 | 128 | 129 | Outputs : In *save_path/prediction_name/* directory 130 | 131 | - estimation : which is the model estimations 132 | - sigma : the standard deviation of the predictions 133 | - plots of the predictions 134 | 135 | # Sepecial thanks to : 136 | - Yunkai Zhang for the implimentation of [DeepAR](https://github.com/zhykoties/TimeSeries), which i used to get some inspiration. 137 | -------------------------------------------------------------------------------- /train.py: -------------------------------------------------------------------------------- 1 | from data_prep import * 2 | from net import * 3 | from evaluate import * 4 | from torch import nn 5 | 6 | import torch 7 | 8 | import os 9 | import argparse 10 | import json 11 | 12 | import datetime 13 | 14 | def main_train(data_dir,dataset,meta_information_path,model_config_path,train_config,save_path): 15 | 16 | print('\n\n# Loading the configurations :') 17 | # Metainformation : 18 | with open(meta_information_path, 'r') as file: 19 | meta_information = json.load(file) 20 | 21 | # model config : 22 | with open(model_config_path, 'r') as file: 23 | model_config = json.load(file) 24 | 25 | print('# Used model_config :', model_config) 26 | 27 | # train config 28 | train_config_path = train_config 29 | with open(train_config_path, 'r') as file: 30 | train_config = json.load(file) 31 | 32 | print('# Used train_config :', train_config) 33 | 34 | # Checking if it is the default configuration 35 | try : 36 | if model_config['default'] == True : 37 | print('\n\t# WARNING : YOU ARE USING THE DEFAULT model_config.json GENERATED DURING DATA PREPARATION, YOU CAN MODIFY IT AT :',model_config_path) 38 | except : 39 | pass 40 | try : 41 | if model_config['default'] == True : 42 | print('\n\t# WARNING : YOU ARE USING THE DEFAULT train_config.json GENERATED DURING DATA PREPARATION, YOU CAN MODIFY IT AT :',train_config_path) 43 | except: 44 | pass 45 | 46 | # Saving path : 47 | os.makedirs(save_path+'/'+dataset+'/', exist_ok = True) 48 | save_path = save_path+'/'+dataset+'/' 49 | 50 | # Datasets Loading : 51 | 52 | batch_size = train_config['batch_size'] #32 53 | predict_batch_size = train_config['predict_batch_size']#31 54 | 55 | train_set = TrainDataset(data_dir, dataset) 56 | test_set = TestDataset(data_dir, dataset) 57 | 58 | # Datasets Sampling 59 | sampler = WeightedSampler(data_dir, dataset) # Use weighted sampler instead of random sampler 60 | train_loader = DataLoader(train_set, batch_size=batch_size, sampler=sampler, num_workers=0) 61 | test_loader = DataLoader(test_set, batch_size=predict_batch_size, sampler=RandomSampler(test_set), num_workers=0) 62 | 63 | # timeseries_list : 64 | train_timeseries_list = np.load(os.path.join(data_dir, f'train_timeseries_dict_{dataset}.npy')) 65 | test_timeseries_list = np.load(os.path.join(data_dir, f'test_timeseries_dict_{dataset}.npy')) 66 | 67 | # Device : 68 | device = 'cpu' 69 | if torch.cuda.is_available(): 70 | torch.set_default_tensor_type(torch.cuda.FloatTensor) 71 | device = 'cuda' 72 | 73 | # Model Definition : 74 | windows_size = meta_information['window_size'] #args.window_size # window learning size 75 | number_of_vars = meta_information['number_of_vars']#4 # the covariates + the time serie to learn 76 | num_timeseries_kinds = meta_information['num_timeseries_kinds']#1 77 | 78 | headers = model_config['headers']#3 79 | depth = model_config['depth']#3 80 | kernel_size = model_config['kernel_size']#6 81 | 82 | # Model 83 | model = ForcastConvTransformer(k = number_of_vars, headers=headers, depth=depth, seq_length=windows_size, kernel_size=kernel_size, num_tokens=num_timeseries_kinds) 84 | 85 | # Global window is the size of the 86 | global_window_size = windows_size 87 | stride = 1 88 | 89 | # Global Parametrers 90 | windows_total = (global_window_size - (windows_size-stride))//stride # total number of windows train by global_size window 91 | 92 | # Learning params : 93 | 94 | # Saving parh 95 | PATH = save_path+'/ConvTransformer_'+dataset+str(datetime.datetime.now())+'.pth' 96 | 97 | created_model_config = dict() 98 | created_model_config['k'] = number_of_vars 99 | created_model_config['headers'] = headers 100 | created_model_config['depth'] = depth 101 | created_model_config['seq_length'] = windows_size 102 | created_model_config['kernel_size'] = kernel_size 103 | created_model_config['num_tokens'] = num_timeseries_kinds 104 | created_model_config['path'] = PATH 105 | 106 | with open(model_config_path, 'w') as file: 107 | json.dump(created_model_config, file) 108 | 109 | # Training 110 | 111 | num_epochs = train_config['num_epochs'] 112 | lr_warmup = train_config['lr_warmup'] 113 | learning_rate = train_config['learning_rate'] 114 | 115 | 116 | # Optimizer 117 | opt = torch.optim.Adam(lr=learning_rate, params=model.parameters()) 118 | 119 | # Training 120 | seen = 1 121 | best_test_loss = -1 122 | train_loss = [] 123 | test_loss = [] 124 | for epoch in range(num_epochs) : 125 | model.train(True) 126 | count = 0 127 | epoch_loss = 0 128 | print('Epoch',epoch) 129 | for i, (train_batch, tokens_idx, v_batch, labels_batch) in enumerate(train_loader): 130 | # Warm up learning rate : 131 | lr = max((learning_rate / lr_warmup) * seen, 1e-5) 132 | opt.lr = lr 133 | opt.zero_grad() 134 | 135 | # Transfer data to GPU 136 | train_batch = train_batch.to(torch.float32).to(device) # not scaled 137 | labels_batch = labels_batch.to(torch.float32).to(device) # not scaled 138 | v_batch = v_batch.to(torch.float32).to(device) # scale 139 | tokens_idx = tokens_idx.to(device) 140 | 141 | # The loop on the windows on the global window 142 | st = 0 143 | nd = st + windows_size 144 | loss = 0 145 | for i in range(int(windows_total)): 146 | # the window extraction : 147 | x_train = train_batch[:,st:nd,:] 148 | v_train = v_batch 149 | batch_size = v_train.shape[0] 150 | 151 | # Model application : 152 | v_train_expanded = v_train.expand(windows_size,batch_size,2).transpose(1,0) 153 | estimation, mu, sigma = evaluate(model, x_train, tokens_idx, v_train_expanded, windows_size, 154 | sampling = False) 155 | 156 | # Loss calculation: 157 | loss += loss_fct(mu, sigma, labels_batch[:,st:nd], 'quantileloss_dist_normal') 158 | st+=1 159 | nd+=1 160 | # The mean loss 161 | loss = loss/int(windows_total) 162 | epoch_loss = loss 163 | loss.backward() 164 | 165 | # CLip gradient if > 1 .. 166 | nn.utils.clip_grad_norm_(model.parameters(), 1) 167 | opt.step() 168 | 169 | seen += train_batch.size(0)*int(windows_total) 170 | if num_epochs > 100 : 171 | if (epoch % 100 == 0 and epoch !=0): 172 | print('Epoch number : ', epoch) 173 | print(f'-- "train" loss {epoch_loss:.4}') 174 | else : 175 | print(f'-- "train" loss {epoch_loss:.4}') 176 | 177 | # evaluation on test data 178 | with torch.no_grad(): 179 | model.train(False) 180 | test_loss = 0.0 181 | count = 0 182 | for i, (test_batch, tokens_tst_idx, v_tst, labels_tst) in enumerate(test_loader): 183 | # Transfer data to GPU 184 | test_batch = test_batch.to(torch.float32).to(device) 185 | labels_tst = labels_tst.to(torch.float32).to(device) 186 | v_tst = v_tst.to(torch.float32).to(device) 187 | tokens_tst_idx = tokens_tst_idx.to(device) 188 | 189 | predicted_serie, _, labels_tst, tst_loss, _, _ = test_evaluation(model,test_batch, tokens_tst_idx, v_tst, labels_tst, windows_size, first_affect=False, sampling=False, number_of_samples=25, loss_kind='quantileloss_dist_normal') 190 | test_loss += tst_loss 191 | count+=1 192 | test_loss = test_loss / count 193 | if num_epochs > 100 : 194 | if (epoch % 100 == 0 and epoch !=0): 195 | print(f'-- "test" loss {test_loss:.4}') 196 | print("------------------------") 197 | else : 198 | print(f'-- "test" loss {test_loss:.4}') 199 | print("------------------------") 200 | if test_loss < best_test_loss or epoch == 0: 201 | best_test_loss = test_loss 202 | torch.save({ 203 | 'epoch': epoch, 204 | 'model_state_dict': model.state_dict(), 205 | 'optimizer_state_dict': opt.state_dict(), 206 | 'loss': loss, 207 | }, PATH) 208 | 209 | if __name__ == '__main__': 210 | parser = argparse.ArgumentParser() 211 | parser.add_argument('--data-directory', type=str, dest='data_directory', help='data directory', default='data_prepared/') 212 | parser.add_argument('--dataset-name', type=str, dest='dataset_name', help='Dataset name', default='data_prepared') 213 | parser.add_argument('--meta-information', type=str, dest='meta_information', help='meta information') 214 | parser.add_argument('--model-config', type=str, dest='model_config', help='model config') 215 | parser.add_argument('--train-config', type =str, dest = 'train_config', help ='train config') 216 | parser.add_argument('--model-save-path', type=str, dest='model_save_path', help='model save path') 217 | 218 | 219 | args = parser.parse_args() 220 | # Data LOading 221 | # Directory and dataset name 222 | data_dir = args.data_directory 223 | dataset = args.dataset_name 224 | meta_information_path = args.meta_information 225 | model_config_path = args.model_config 226 | train_config = args.train_config 227 | save_path = args.model_save_path 228 | 229 | main_train(data_dir,dataset,meta_information_path,model_config_path,train_config,save_path) --------------------------------------------------------------------------------