51 | 
52 |
53 |
54 | 52 |
55 | 
56 |
57 |
58 | 56 |
59 | 
60 |
61 |
62 | 60 |
63 | 
64 |
65 |
--------------------------------------------------------------------------------
/conda_environment.yml:
--------------------------------------------------------------------------------
1 | name: deep-forecast-pytorch
2 | channels:
3 | - pytorch
4 | - defaults
5 | dependencies:
6 | - backports=1.0=py27h63c9359_1
7 | - backports.functools_lru_cache=1.4=py27he8db605_1
8 | - backports.shutil_get_terminal_size=1.0.0=py27h5bc021e_2
9 | - backports_abc=0.5=py27h7b3c97b_0
10 | - ca-certificates=2017.08.26=h1d4fec5_0
11 | - certifi=2017.11.5=py27h71e7faf_0
12 | - cffi=1.11.2=py27ha7929c6_0
13 | - cudatoolkit=8.0=3
14 | - cycler=0.10.0=py27hc7354d3_0
15 | - dbus=1.10.22=h3b5a359_0
16 | - decorator=4.1.2=py27h1544723_0
17 | - enum34=1.1.6=py27h99a27e9_1
18 | - expat=2.2.5=he0dffb1_0
19 | - fontconfig=2.12.4=h88586e7_1
20 | - freetype=2.8=hab7d2ae_1
21 | - functools32=3.2.3.2=py27h4ead58f_1
22 | - glib=2.53.6=h5d9569c_2
23 | - gst-plugins-base=1.12.2=he3457e5_0
24 | - gstreamer=1.12.2=h4f93127_0
25 | - icu=58.2=h9c2bf20_1
26 | - intel-openmp=2018.0.0=hc7b2577_8
27 | - ipython=5.4.1=py27h36c99b6_1
28 | - ipython_genutils=0.2.0=py27h89fb69b_0
29 | - jpeg=9b=h024ee3a_2
30 | - libedit=3.1=heed3624_0
31 | - libffi=3.2.1=hd88cf55_4
32 | - libgcc-ng=7.2.0=h7cc24e2_2
33 | - libgfortran-ng=7.2.0=h9f7466a_2
34 | - libpng=1.6.32=hbd3595f_4
35 | - libstdcxx-ng=7.2.0=h7a57d05_2
36 | - libtiff=4.0.9=h28f6b97_0
37 | - libxcb=1.12=hcd93eb1_4
38 | - libxml2=2.9.4=h2e8b1d7_6
39 | - matplotlib=2.1.1=py27h0128e01_0
40 | - mkl=2018.0.1=h19d6760_4
41 | - ncurses=6.0=h9df7e31_2
42 | - numpy=1.13.3=py27hbcc08e0_0
43 | - olefile=0.44=py27h4bd3e3c_0
44 | - openssl=1.0.2n=hb7f436b_0
45 | - pathlib2=2.3.0=py27h6e9d198_0
46 | - pcre=8.41=hc27e229_1
47 | - pexpect=4.3.1=py27_0
48 | - pickleshare=0.7.4=py27h09770e1_0
49 | - pillow=4.3.0=py27h353bd0c_1
50 | - pip=9.0.1=py27ha730c48_4
51 | - prompt_toolkit=1.0.15=py27h1b593e1_0
52 | - ptyprocess=0.5.2=py27h4ccb14c_0
53 | - pycparser=2.18=py27hefa08c5_1
54 | - pygments=2.2.0=py27h4a8b6f5_0
55 | - pyparsing=2.2.0=py27hf1513f8_1
56 | - pyqt=5.6.0=py27h4b1e83c_5
57 | - python=2.7.14=h1571d57_29
58 | - python-dateutil=2.6.1=py27h4ca5741_1
59 | - pytz=2017.3=py27h001bace_0
60 | - qt=5.6.2=h974d657_12
61 | - readline=7.0=ha6073c6_4
62 | - scandir=1.6=py27hf7388dc_0
63 | - scikit-learn=0.19.1=py27h445a80a_0
64 | - scipy=1.0.0=py27hf5f0f52_0
65 | - setuptools=36.5.0=py27h68b189e_0
66 | - simplegeneric=0.8.1=py27h19e43cd_0
67 | - singledispatch=3.4.0.3=py27h9bcb476_0
68 | - sip=4.18.1=py27he9ba0ab_2
69 | - six=1.11.0=py27h5f960f1_1
70 | - sqlite=3.20.1=hb898158_2
71 | - ssl_match_hostname=3.5.0.1=py27h4ec10b9_2
72 | - subprocess32=3.2.7=py27h373dbce_0
73 | - tk=8.6.7=hc745277_3
74 | - tornado=4.5.3=py27_0
75 | - traitlets=4.3.2=py27hd6ce930_0
76 | - wcwidth=0.1.7=py27h9e3e1ab_0
77 | - wheel=0.30.0=py27h2bc6bb2_1
78 | - xz=5.2.3=h55aa19d_2
79 | - zlib=1.2.11=ha838bed_2
80 | - pytorch=0.3.0=py27_cuda8.0.61_cudnn7.0.3hf383a3f_4
81 | - torchvision=0.2.0=py27hfb27419_1
82 | - pip:
83 | - altgraph==0.15
84 | - backports.ssl-match-hostname==3.5.0.1
85 | - convertdate==2.1.1
86 | - dateparser==0.6.0
87 | - dis3==0.1.2
88 | - ephem==3.7.6.0
89 | - future==0.16.0
90 | - iso3166==0.8
91 | - lmdb==0.93
92 | - macholib==1.9
93 | - pathlib==1.0.1
94 | - pefile==2017.11.5
95 | - pyinstaller==3.3.1
96 | - pylibdmtx==0.1.7
97 | - pyyaml==3.12
98 | - regex==2017.12.12
99 | - ruamel.ordereddict==0.4.13
100 | - ruamel.yaml==0.15.26
101 | - torch==0.3.0.post4
102 | - tzlocal==1.4
103 | - umalqurra==0.2
104 | - xlrd==1.0.0
105 | prefix: /home/jcmaxwell/miniconda2/envs/deep-forecast-pytorch
106 |
107 |
--------------------------------------------------------------------------------
/data/scripts/1-process.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import datetime, argparse
3 |
4 | parser = argparse.ArgumentParser()
5 | parser.add_argument('--input', help='input file')
6 | parser.add_argument('--output', help='output file')
7 | args = parser.parse_args()
8 |
9 |
10 | data = np.loadtxt(args.input, dtype='str', delimiter=',')
11 | header = data[0]
12 | data = data[1:]
13 |
14 | def convert2datetime(txt):
15 | return datetime.datetime.strptime(txt, '%Y-%m-%d %H:%M')
16 |
17 | dates = np.array(map(convert2datetime, data[:,1]))
18 | data = data[:, [0, 2]]
19 |
20 | #discarded_stations = ['XNT', '6B9', 'OLE', 'MTP', 'N03', 'NY0', 'OGS']
21 | discarded_stations = []
22 |
23 | stations = list(set(data[:, 0]).difference(discarded_stations))
24 |
25 | all_dates = set()
26 | all_data = []
27 | for station in stations:
28 | station_filter = data[:, 0] == station
29 |
30 | station_winds = data[station_filter][:, 1]
31 | station_dates = dates[station_filter]
32 |
33 | sort_idxes = station_dates.argsort()
34 | station_winds = station_winds[sort_idxes]
35 | station_dates = station_dates[sort_idxes]
36 |
37 | select_idxes = np.array([i for i in xrange(len(station_dates)) if station_dates[i].minute == 50]) #TODO: 0
38 |
39 | if len(select_idxes) == 0:
40 | print station
41 | continue
42 |
43 | station_winds = station_winds[select_idxes]
44 | station_dates = station_dates[select_idxes]
45 |
46 | #print '##############'
47 | #print station
48 | #print station_winds, station_dates
49 |
50 | all_data.append([station, station_dates, station_winds])
51 |
52 | for s in station_dates:
53 | all_dates.add(s)
54 |
55 | all_dates = list(all_dates)
56 | all_dates.sort()
57 |
58 | last_date = max(all_dates)
59 | first_date = min(all_dates)
60 | diff = last_date - first_date
61 |
62 | n_hours = int(diff.total_seconds() / (60. * 60.))
63 | date_axis = np.array([first_date + datetime.timedelta(hours=i) for i in range(n_hours + 1)])
64 |
65 | data_axis = []
66 | for data in all_data:
67 | station = data[0]
68 | dates = np.array(data[1])
69 | winds = np.array(data[2])
70 |
71 | winds_axis = []
72 | for date in date_axis:
73 | idxes = np.where(dates == date)[0]
74 | assert (len(idxes) in [0, 1])
75 | if len(idxes) == 0:
76 | winds_axis.append(None)
77 | else:
78 | idx = idxes[0]
79 | winds_axis.append(winds[idx])
80 | data_axis.append([station, winds_axis])
81 |
82 | new_data = []
83 | for station, winds in data_axis[1:]:
84 | _data = [station, ] + winds
85 | new_data.append(_data)
86 |
87 | out = np.concatenate((np.expand_dims(['date', ] + list(date_axis.astype('str')), axis=1), np.stack(new_data, axis=1)), axis=1)
88 |
89 | np.savetxt(args.output, out, fmt='%s', delimiter=',')
90 |
--------------------------------------------------------------------------------
/data/scripts/2-impute.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import datetime, argparse
3 | from sklearn.preprocessing import Imputer
4 |
5 | parser = argparse.ArgumentParser()
6 | parser.add_argument('--input', help='input file')
7 | parser.add_argument('--processed', help='processed file')
8 | parser.add_argument('--output', help='output file')
9 | args = parser.parse_args()
10 |
11 |
12 | data = np.loadtxt(args.processed, dtype='str', delimiter=',')
13 | org_data = np.loadtxt(args.input, dtype='str', delimiter=',')
14 |
15 | header = data[0]
16 | data = data[1:]
17 | org_data = org_data[1:]
18 |
19 | def convert2datetime(txt):
20 | return datetime.datetime.strptime(txt, '%Y-%m-%d %H:%M')
21 |
22 | org_data_dates = np.array(map(convert2datetime, org_data[:, 1]))
23 |
24 | n_hours = len(data)
25 |
26 | for i in xrange(data.shape[1] - 1):
27 | station_name = header[i + 1] # Station Names
28 |
29 | print '#### ', i, station_name
30 |
31 | wind_speeds = data[:, i + 1] # Wind Speeds for that station
32 |
33 | empties = np.where(wind_speeds == 'None')[0]
34 | ms = np.where(wind_speeds == 'M')[0]
35 | missing_indexes = list(empties) + list(ms)
36 | missing_indexes = list(set(missing_indexes)) # Missing value indexes
37 |
38 | missing_ratio = float(len(missing_indexes)) / n_hours
39 |
40 | print missing_ratio
41 |
42 | if missing_ratio > 0.02:
43 | continue
44 |
45 | for idx in missing_indexes:
46 | date = datetime.datetime.strptime(data[idx, 0], '%Y-%m-%d %H:%M:%S')
47 |
48 | org_station_indexes = org_data[:, 0] == station_name
49 | org_station_data = org_data[org_station_indexes]
50 | org_station_dates = org_data_dates[org_station_indexes]
51 |
52 | found = False
53 | for cntr in range(20, 30 + 1, 10): #5, 30 + 1, 5
54 | below_range = date - datetime.timedelta(minutes=cntr)
55 | upper_range = date + datetime.timedelta(minutes=cntr)
56 |
57 | for j, d in enumerate(org_station_dates):
58 | if (below_range < d < upper_range) and (d != date) and (org_station_data[j][-1] != 'M'):
59 | replacement_date = d
60 | found = True
61 | break
62 |
63 | if found:
64 | break
65 |
66 | if not found:
67 | continue
68 |
69 | data[idx, i + 1] = org_station_data[org_station_dates == replacement_date][0][-1]
70 |
71 |
72 |
73 | new_header = []
74 | new_data = []
75 |
76 | n_hours = len(data)
77 | for i in xrange(data.shape[1] - 1):
78 | empties = np.where(data[:, i + 1] == 'None')[0]
79 | ms = np.where(data[:, i + 1] == 'M')[0]
80 | missing_indexes = list(empties) + list(ms)
81 | missing_indexes = list(set(missing_indexes))
82 | missing_ratio = float(len(missing_indexes)) / n_hours
83 | if missing_ratio <= 0.005:
84 | print i, missing_ratio
85 | new_header.append(header[i + 1])
86 | new_data.append(data[:, i + 1])
87 |
88 | new_data = np.stack(new_data, axis=1)
89 | #new_data = np.concatenate((np.expand_dims(new_header, axis=0), new_data), axis=0)
90 |
91 | new_data[new_data == 'None'] = '-1'
92 | new_data[new_data == 'M'] = '-1'
93 |
94 | new_data = new_data.astype('float')
95 | imp = Imputer(missing_values=-1.0, axis=0)
96 | new_data = imp.fit_transform(new_data)
97 |
98 | np.savetxt(args.output, new_data, fmt='%s', delimiter=',')
99 |
--------------------------------------------------------------------------------
/figures/0.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Wizaron/deep-forecast-pytorch/deeaa075b71371b9ea99b723df5c51255f018863/figures/0.png
--------------------------------------------------------------------------------
/figures/1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Wizaron/deep-forecast-pytorch/deeaa075b71371b9ea99b723df5c51255f018863/figures/1.png
--------------------------------------------------------------------------------
/figures/2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Wizaron/deep-forecast-pytorch/deeaa075b71371b9ea99b723df5c51255f018863/figures/2.png
--------------------------------------------------------------------------------
/figures/3.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Wizaron/deep-forecast-pytorch/deeaa075b71371b9ea99b723df5c51255f018863/figures/3.png
--------------------------------------------------------------------------------
/lib/__init__.py:
--------------------------------------------------------------------------------
1 | from data import Data
2 | from model import Model
3 | from loader import Loader
4 | from prediction import draw_graph_all_stations
5 |
--------------------------------------------------------------------------------
/lib/arch.py:
--------------------------------------------------------------------------------
1 | import torch.nn as nn
2 |
3 | class MultiLSTM(nn.Module):
4 |
5 | def __init__(self, in_out_vec_dim, moving_horizon, activation, usegpu=True):
6 | super(MultiLSTM, self).__init__()
7 |
8 | assert activation in ['relu', 'sigmoid', 'elu']
9 |
10 | self.n_inputs = in_out_vec_dim
11 | self.n_outputs = in_out_vec_dim
12 | self.moving_horizon = moving_horizon
13 | self.activation = activation
14 | self.usegpu = usegpu
15 |
16 | self.rnn_models = nn.ModuleList()
17 | for model_idx in range(1, self.moving_horizon + 1):
18 | self.rnn_models.append(nn.ModuleList(self.build_rnn(model_idx)))
19 |
20 | def build_single_layer_rnn(self, n_units):
21 |
22 | rnn = nn.LSTM(self.n_inputs, n_units, batch_first=True) #TODO: Bidi + attention
23 | output = nn.Linear(n_units, self.n_outputs)
24 | if self.activation == 'relu':
25 | act = nn.ReLU()
26 | elif self.activation == 'sigmoid':
27 | act = nn.Sigmoid()
28 | elif self.activation == 'elu':
29 | act = nn.ELU()
30 |
31 | return [rnn, output, act]
32 |
33 | def build_two_layer_rnn(self, n_units_1, n_units_2):
34 |
35 | rnn_1 = nn.LSTM(self.n_inputs, n_units_1, batch_first=True)
36 | rnn_2 = nn.LSTM(n_units_1, n_units_2, batch_first=True) #TODO: Bidi + attention
37 | output = nn.Linear(n_units_2, self.n_outputs)
38 | if self.activation == 'relu':
39 | act = nn.ReLU()
40 | elif self.activation == 'sigmoid':
41 | act = nn.Sigmoid()
42 | elif self.activation == 'elu':
43 | act = nn.ELU()
44 |
45 | return [rnn_1, rnn_2, output, act]
46 |
47 | def build_rnn_1(self):
48 |
49 | return self.build_single_layer_rnn(self.n_inputs * 2)
50 |
51 | def build_rnn_2(self):
52 |
53 | #return self.build_single_layer_rnn(10)
54 | return self.build_single_layer_rnn(self.n_inputs * 2)
55 |
56 | def build_rnn_3(self):
57 |
58 | #return self.build_single_layer_rnn(self.n_inputs)
59 | return self.build_single_layer_rnn(self.n_inputs * 2)
60 |
61 | def build_rnn_4(self):
62 |
63 | #return self.build_two_layer_rnn(self.n_inputs, self.n_inputs * 2)
64 | return self.build_single_layer_rnn(self.n_inputs * 2)
65 |
66 | def build_rnn_5(self):
67 |
68 | #return self.build_single_layer_rnn(30)
69 | return self.build_single_layer_rnn(self.n_inputs * 2)
70 |
71 | def build_rnn_6(self):
72 |
73 | #return self.build_two_layer_rnn(self.n_inputs * 2, self.n_inputs * 2)
74 | return self.build_single_layer_rnn(self.n_inputs * 2)
75 |
76 | def build_rnn(self, rnn_model_num):
77 |
78 | if rnn_model_num == 1:
79 | return self.build_rnn_1()
80 | elif rnn_model_num == 2:
81 | return self.build_rnn_2()
82 | elif rnn_model_num == 3:
83 | return self.build_rnn_3()
84 | elif rnn_model_num == 4:
85 | return self.build_rnn_4()
86 | elif rnn_model_num == 5:
87 | return self.build_rnn_5()
88 | elif rnn_model_num == 6:
89 | return self.build_rnn_6()
90 | else:
91 | return self.build_rnn_6()
92 |
93 | def forward(self, (x, rnn_model_num)):
94 |
95 | rnn_model = self.rnn_models[rnn_model_num - 1]
96 |
97 | if len(rnn_model) == 3:
98 | rnn_layer, output_layer, activation = rnn_model
99 | _, x = rnn_layer(x)
100 | x = x[0]
101 | x = x.squeeze(0)
102 | x = output_layer(x)
103 | x = activation(x)
104 |
105 | elif len(rnn_model) == 4:
106 | rnn_layer_1, rnn_layer_2, output_layer, activation = rnn_model
107 | x, _ = rnn_layer_1(x)
108 | _, x = rnn_layer_2(x)
109 | x = x[0]
110 | x = x.squeeze(0)
111 | x = output_layer(x)
112 | x = activation(x)
113 | else:
114 | NotImplementedError
115 |
116 | return x
117 |
--------------------------------------------------------------------------------
/lib/data.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import csv
3 |
4 | class Data(object):
5 |
6 | def __init__(self, data_file='data/data_imputation_turkey.csv', input_horizon=12, n_stations=27,
7 | train_ratio=0.985, val_ratio=0.05, debug=False):
8 | """
9 | Inputs
10 | ======
11 | :param data_file: input csv filepath.
12 | :param input_horizon:
13 | :param n_stations:
14 | :param train_ratio:
15 | :param debug:"""
16 |
17 | self.data_file = data_file
18 | self.input_horizon = input_horizon
19 | self.n_stations = n_stations
20 | self.train_ratio = train_ratio
21 | self.val_ratio = val_ratio
22 |
23 | self.trainDataRate = 1 if not debug else 0.05 # percentage of data used for training (saving time for debuging)
24 |
25 | self.winds = Data.read_data(self.data_file, self.n_stations)
26 | self.winds, self.means_stds = Data.normalize_data(self.winds)
27 |
28 | @staticmethod
29 | def read_data(data_file, n_stations):
30 | with open(data_file) as f:
31 | data = csv.reader(f, delimiter=',')
32 |
33 | winds = [line for line in data]
34 | winds = np.array(winds).astype(np.float32)
35 | winds = winds[:, : n_stations]
36 |
37 | return winds
38 |
39 | @staticmethod
40 | def normalize_data(winds):
41 | wind_min = winds.min()
42 | wind_max = winds.max() - wind_min
43 |
44 | normal_winds = (winds - wind_min) / wind_max
45 | mins_maxs = [wind_min, wind_max]
46 |
47 | return normal_winds, mins_maxs
48 |
49 | def denormalize_data(self, vec):
50 | wind_min, wind_max = self.means_stds
51 | res = vec * wind_max + wind_min
52 | return res
53 |
54 | def load_data_lstm_1(self): # For LSTM 1
55 |
56 | samples = []
57 | for index in range(self.winds.shape[0] - self.input_horizon): # Last one is reserved for label
58 | samples.append(self.winds[index : index + self.input_horizon])
59 | samples = np.array(samples)
60 |
61 | n_samples = samples.shape[0]
62 | n_train_samples = int(round(n_samples * self.train_ratio))
63 | n_train_samples = int(round(n_train_samples * self.trainDataRate))
64 |
65 | X_train = samples[: n_train_samples, :]
66 | y_train = self.winds[self.input_horizon : n_train_samples + self.input_horizon] # Shifted by self.input_horizon
67 |
68 | X_test = samples[n_train_samples : n_samples]
69 | y_test = self.winds[n_train_samples + self.input_horizon : n_samples + self.input_horizon]
70 |
71 | n_val_samples = int(np.ceil(n_train_samples * self.val_ratio))
72 | X_val = X_train[: n_val_samples]
73 | y_val = y_train[: n_val_samples]
74 |
75 | X_train = X_train[n_val_samples :]
76 | y_train = y_train[n_val_samples :]
77 |
78 | return [X_train, y_train], [X_val, y_val], [X_test, y_test]
79 |
80 | def load_data(self, pre_x_train_val, pre_y_train_val, model, rnn_model_num): # pre_x_train_val and pre_y_train_val from load_data_lstm_1
81 |
82 | X_train_val, y_train_val = np.ones_like(pre_x_train_val), np.zeros_like(pre_y_train_val)
83 |
84 | for ind in range(len(pre_x_train_val) - 1):
85 | tempInput = pre_x_train_val[ind]
86 | temp_shape = tempInput.shape
87 | tempInput = np.reshape(tempInput, (1, temp_shape[0], temp_shape[1]))
88 |
89 | output = model.predict(rnn_model_num, tempInput)
90 |
91 | tInput = np.reshape(tempInput, temp_shape)
92 | tempInput = np.vstack((tInput, output))
93 | tempInput = np.delete(tempInput, 0, axis=0)
94 |
95 | X_train_val[ind] = tempInput
96 | y_train_val[ind] = pre_y_train_val[ind + 1]
97 |
98 | X_train_val = X_train_val[:-1]
99 | y_train_val = y_train_val[:-1]
100 |
101 | return [X_train_val, y_train_val]
102 |
--------------------------------------------------------------------------------
/lib/loader.py:
--------------------------------------------------------------------------------
1 | from torch.utils.data import Dataset
2 |
3 | class Loader(Dataset):
4 | """Dataset Reader"""
5 |
6 | def __init__(self, (X, y)):
7 |
8 | self.features = X
9 | self.targets = y
10 |
11 | self.n_samples = len(self.features)
12 |
13 | def __getitem__(self, index):
14 |
15 | assert index <= len(self), 'index range error'
16 |
17 | features = self.features[index]
18 | target = self.targets[index]
19 |
20 | return features, target
21 |
22 | def __len__(self):
23 | return self.n_samples
24 |
--------------------------------------------------------------------------------
/lib/model.py:
--------------------------------------------------------------------------------
1 | import os, time
2 | import torch
3 | import torch.optim as optim
4 | from torch.optim.lr_scheduler import ReduceLROnPlateau
5 | from torch.autograd import Variable
6 | import torch.backends.cudnn as cudnn
7 | import numpy as np
8 | from itertools import ifilter
9 |
10 | from arch import MultiLSTM
11 |
12 | class Model(object):
13 |
14 | def __init__(self, in_out_vec_dim, moving_horizon, activation, criterion, load_model_path='', usegpu=True):
15 |
16 | self.in_out_vec_dim = in_out_vec_dim
17 | self.moving_horizon = moving_horizon
18 | self.activation = activation
19 | self.load_model_path = load_model_path
20 | self.usegpu = usegpu
21 |
22 | self.model = MultiLSTM(self.in_out_vec_dim, self.moving_horizon, self.activation, usegpu=self.usegpu)
23 | self.__load_weights(load_model_path)
24 | self.__define_criterion(criterion)
25 |
26 | if self.usegpu:
27 | cudnn.benchmark = True
28 | self.model.cuda()
29 | #self.model = torch.nn.DataParallel(self.model, device_ids=range(self.ngpus))
30 |
31 | print self.model
32 |
33 | def __load_weights(self, load_model_path):
34 |
35 | #def weights_initializer(m):
36 | # """Custom weights initialization"""
37 | # classname = m.__class__.__name__
38 | # if classname.find('Linear') != -1:
39 | # m.weight.data.normal_(0.0, 0.001)
40 | # m.bias.data.zero_()
41 |
42 | if load_model_path != '':
43 | assert os.path.isfile(load_model_path), 'Model : {} does not exists!'.format(load_model_path)
44 | print 'Loading model from {}'.format(load_model_path)
45 |
46 | if self.usegpu:
47 | self.model.load_state_dict(torch.load(load_model_path))
48 | else:
49 | self.model.load_state_dict(torch.load(load_model_path, map_location=lambda storage, loc: storage))
50 |
51 | #else:
52 | # self.model.apply(weights_initializer)
53 |
54 | def __define_variable(self, tensor, volatile=False):
55 | return Variable(tensor, volatile=volatile)
56 |
57 | def __define_input_variables(self, features, targets, volatile=False):
58 | features_var = self.__define_variable(features, volatile=volatile)
59 | targets_var = self.__define_variable(targets, volatile=volatile)
60 |
61 | return features_var, targets_var
62 |
63 | def __define_criterion(self, criterion):
64 |
65 | assert criterion in ['MSE', 'L1Loss', 'SmoothL1Loss']
66 |
67 | if criterion == 'MSE':
68 | self.criterion = torch.nn.MSELoss()
69 | elif criterion == 'L1Loss':
70 | self.criterion = torch.nn.L1Loss()
71 | elif criterion == 'SmoothL1Loss':
72 | self.criterion = torch.nn.SmoothL1Loss()
73 |
74 | def __define_optimizer(self, learning_rate, weight_decay, lr_drop_factor, lr_drop_patience, rnn_model_num, optimizer='Adam'):
75 | assert optimizer in ['RMSprop', 'Adam', 'Adadelta', 'SGD']
76 |
77 | for rnn_model_num_counter in range(1, self.moving_horizon + 1):
78 | if rnn_model_num_counter == rnn_model_num:
79 | for param in self.model.rnn_models[rnn_model_num_counter - 1].parameters():
80 | param.requires_grad = True
81 | else:
82 | for param in self.model.rnn_models[rnn_model_num_counter - 1].parameters():
83 | param.requires_grad = False
84 |
85 | parameters = ifilter(lambda p: p.requires_grad, self.model.rnn_models[rnn_model_num - 1].parameters())
86 |
87 | if optimizer == 'RMSprop':
88 | self.optimizer = optim.RMSprop(parameters, lr=learning_rate, weight_decay=weight_decay)
89 | elif optimizer == 'Adadelta':
90 | self.optimizer = optim.Adadelta(parameters, lr=learning_rate, weight_decay=weight_decay)
91 | elif optimizer == 'Adam':
92 | self.optimizer = optim.Adam(parameters, lr=learning_rate, weight_decay=weight_decay)
93 | elif optimizer == 'SGD':
94 | self.optimizer = optim.SGD(parameters, lr=learning_rate, momentum=0.9, weight_decay=weight_decay)
95 |
96 | self.lr_scheduler = ReduceLROnPlateau(self.optimizer, mode='min', factor=lr_drop_factor, patience=lr_drop_patience, verbose=True)
97 |
98 | @staticmethod
99 | def __get_loss_averager():
100 | return averager()
101 |
102 | def __minibatch(self, train_test_iter, rnn_model_num, clip_grad_norm, mode='training'):
103 | assert mode in ['training', 'test'], 'Mode must be either "training" or "test"'
104 |
105 | if mode == 'training':
106 | for rnn_model_num_counter in range(1, self.moving_horizon + 1):
107 | if rnn_model_num_counter == rnn_model_num:
108 | for param in self.model.rnn_models[rnn_model_num_counter - 1].parameters():
109 | param.requires_grad = True
110 | else:
111 | for param in self.model.rnn_models[rnn_model_num_counter - 1].parameters():
112 | param.requires_grad = False
113 | self.model.train()
114 | else:
115 | for param in self.model.parameters():
116 | param.requires_grad = False
117 | self.model.eval()
118 |
119 | cpu_features, cpu_targets = train_test_iter.next()
120 | cpu_features = cpu_features.contiguous()
121 | cpu_targets = cpu_targets.contiguous()
122 |
123 | if self.usegpu:
124 | gpu_features = cpu_features.cuda(async=True)
125 | gpu_targets = cpu_targets.cuda(async=True)
126 | else:
127 | gpu_features = cpu_features
128 | gpu_targets = cpu_targets
129 |
130 | if mode == 'training':
131 | gpu_features, gpu_targets = self.__define_input_variables(gpu_features, gpu_targets)
132 | else:
133 | gpu_features, gpu_targets = self.__define_input_variables(gpu_features, gpu_targets, volatile=True)
134 |
135 | predictions = self.model((gpu_features, rnn_model_num))
136 |
137 | cost = self.criterion(predictions, gpu_targets)
138 |
139 | if mode == 'training':
140 | self.model.zero_grad()
141 | cost.backward()
142 | if clip_grad_norm != 0:
143 | torch.nn.utils.clip_grad_norm(self.model.parameters(), clip_grad_norm)
144 | self.optimizer.step()
145 |
146 | return cost, predictions, cpu_targets
147 |
148 | def __test(self, rnn_model_num, test_loader):
149 |
150 | print '***** Testing *****'
151 |
152 | n_minibatches = len(test_loader)
153 | test_iter = iter(test_loader)
154 |
155 | test_loss_averager = Model.__get_loss_averager()
156 |
157 | for minibatch_index in range(n_minibatches):
158 | loss, predictions, cpu_targets = self.__minibatch(test_iter, rnn_model_num, 0.0, mode='test')
159 | test_loss_averager.add(loss)
160 |
161 | test_loss = test_loss_averager.val()
162 |
163 | print 'Loss : {}'.format(test_loss)
164 |
165 | return test_loss
166 |
167 | def fit(self, rnn_model_num, learning_rate, weight_decay, clip_grad_norm, lr_drop_factor, lr_drop_patience, patience, optimizer,
168 | n_epochs, train_loader, test_loader, model_save_path):
169 |
170 | model_path = os.path.join(model_save_path, 'model_best.pth')
171 |
172 | training_log_file = open(os.path.join(model_save_path, 'training.log'), 'w')
173 | validation_log_file = open(os.path.join(model_save_path, 'validation.log'), 'w')
174 |
175 | training_log_file.write('Epoch,Loss\n')
176 | validation_log_file.write('Epoch,Loss\n')
177 |
178 | train_loss_averager = Model.__get_loss_averager()
179 |
180 | self.__define_optimizer(learning_rate, weight_decay, lr_drop_factor, lr_drop_patience, rnn_model_num, optimizer=optimizer)
181 |
182 | self.__test(rnn_model_num, test_loader)
183 |
184 | best_val_loss = np.Inf
185 | n_epochs_wo_best_model = 0
186 | for epoch in range(n_epochs):
187 | epoch_start = time.time()
188 |
189 | train_iter = iter(train_loader)
190 | n_minibatches = len(train_loader)
191 |
192 | minibatch_index = 0
193 | while minibatch_index < n_minibatches:
194 | minibatch_loss, minibatch_predictions, minibatch_cpu_targets = self.__minibatch(train_iter, rnn_model_num, clip_grad_norm,
195 | mode='training')
196 |
197 | train_loss_averager.add(minibatch_loss)
198 | minibatch_index += 1
199 |
200 | train_loss = train_loss_averager.val()
201 |
202 | epoch_end = time.time()
203 | epoch_duration = epoch_end - epoch_start
204 |
205 | print '[{}] [{}/{}] Loss : {}'.format(epoch_duration, epoch, n_epochs, train_loss)
206 |
207 | val_loss = self.__test(rnn_model_num, test_loader)
208 |
209 | self.lr_scheduler.step(val_loss)
210 |
211 | is_best_model = val_loss <= best_val_loss
212 |
213 | if is_best_model:
214 | best_val_loss = val_loss
215 | n_epochs_wo_best_model = 0
216 | torch.save(self.model.state_dict(), model_path)
217 | else:
218 | n_epochs_wo_best_model += 1
219 |
220 | training_log_file.write('{},{}\n'.format(epoch, train_loss))
221 | validation_log_file.write('{},{}\n'.format(epoch, val_loss))
222 | training_log_file.flush()
223 | validation_log_file.flush()
224 |
225 | train_loss_averager.reset()
226 |
227 | if n_epochs_wo_best_model == patience:
228 | break
229 |
230 | training_log_file.close()
231 | validation_log_file.close()
232 |
233 | # Load best model
234 | self.__load_weights(model_path)
235 |
236 | def test(self, (X_test, y_test)):
237 |
238 | predicted = np.zeros_like(y_test)
239 |
240 | for ind in range(len(X_test)):
241 | model_ind = ind % self.moving_horizon
242 | rnn_model_num = model_ind + 1
243 | if model_ind == 0:
244 | test_input_raw = X_test[ind]
245 | #test_input_shape = test_input_raw.shape
246 | #test_input = np.reshape(test_input_raw, [1, test_input_shape[0], test_input_shape[1]])
247 | test_input = np.expand_dims(test_input_raw, axis=0)
248 | else:
249 | test_input_raw = np.vstack((test_input_raw, predicted[ind - 1]))
250 | test_input_raw = np.delete(test_input_raw, 0, axis=0)
251 | #test_input_shape = test_input_raw.shape
252 | #test_input = np.reshape(test_input_raw, [1, test_input_shape[0], test_input_shape[1]])
253 | test_input = np.expand_dims(test_input_raw, axis=0)
254 |
255 | predicted[ind] = self.predict(rnn_model_num, test_input)[0]
256 |
257 | return predicted
258 |
259 | def predict(self, rnn_model_num, features):
260 |
261 | assert len(features.shape) == 3 #b, t, feats
262 |
263 | for param in self.model.parameters():
264 | param.requires_grad = False
265 | self.model.eval()
266 |
267 | features = torch.from_numpy(features.astype(np.float32))
268 |
269 | features = features.contiguous()
270 | if self.usegpu:
271 | features = features.cuda(async=True)
272 |
273 | features = self.__define_variable(features, volatile=True)
274 |
275 | predictions = self.model((features, rnn_model_num))
276 | predictions = predictions.data.cpu().numpy().astype(np.float32)
277 |
278 | return predictions
279 |
280 | class averager(object):
281 | """Compute average for `torch.Variable` and `torch.Tensor`."""
282 |
283 | def __init__(self):
284 | self.reset()
285 |
286 | def add(self, v):
287 | if isinstance(v, Variable):
288 | count = v.data.numel()
289 | v = v.data.sum()
290 | elif isinstance(v, torch.Tensor):
291 | count = v.numel()
292 | v = v.sum()
293 |
294 | self.n_count += count
295 | self.sum += v
296 |
297 | def reset(self):
298 | self.n_count = 0
299 | self.sum = 0
300 |
301 | def val(self):
302 | res = 0
303 | if self.n_count != 0:
304 | res = self.sum / float(self.n_count)
305 | return res
306 |
--------------------------------------------------------------------------------
/lib/prediction.py:
--------------------------------------------------------------------------------
1 | import matplotlib
2 | matplotlib.use('Agg')
3 | import matplotlib.pyplot as plt
4 | import numpy as np
5 |
6 | def get_error_measures(denormal_y, denormal_predicted):
7 |
8 | mae = np.mean(np.absolute(denormal_y - denormal_predicted))
9 | rmse = np.sqrt((np.mean((np.absolute(denormal_y - denormal_predicted)) ** 2)))
10 | nrsme_max_min = 100 * rmse / (denormal_y.max() - denormal_y.min())
11 | nrsme_mean = 100 * rmse / (denormal_y.mean())
12 |
13 | return mae, rmse, nrsme_max_min, nrsme_mean
14 |
15 | def draw_graph_station(dataset, yTest, yTestPred, station, visualise=1, ax=None):
16 |
17 | yTest = yTest[:, station]
18 | denormalYTest = dataset.denormalize_data(yTest)
19 | denormalPredicted = dataset.denormalize_data(yTestPred[:, station])
20 |
21 | mae, rmse, nrmse_maxMin, nrmse_mean = get_error_measures(denormalYTest, denormalPredicted)
22 | print 'Station %s : MAE = %7.7s - RMSE = %7.7s - nrmse_maxMin = %7.7s - nrmse_mean = %7.7s'%(station + 1, mae, rmse, nrmse_maxMin, nrmse_mean)
23 |
24 | if visualise:
25 | if ax is None:
26 | fig = plt.figure()
27 | ax = fig.add_subplot(111)
28 |
29 | ax.plot(denormalYTest, label='Real', color='blue')
30 | ax.plot(denormalPredicted, label='Predicted', color='red')
31 | ax.set_xticklabels(range(0, len(yTest), 100), rotation=40)
32 |
33 | return mae, rmse, nrmse_maxMin, nrmse_mean
34 |
35 | def draw_graph_all_stations(output_dir, dataset, n_stations, yTest, yTestPred):
36 | maeRmse = np.zeros((n_stations, 4))
37 |
38 | for staInd in range(n_stations):
39 | fig, ax = plt.subplots(figsize=(20, 10))
40 | maeRmse[staInd] = draw_graph_station(dataset, yTest, yTestPred, staInd, visualise=1, ax=ax)
41 | plt.xticks(range(0, len(yTest), 100))
42 | filename = '{}/finalEpoch_{}'.format(output_dir, staInd)
43 | plt.savefig('{}.png'.format(filename))
44 |
45 | errMean = maeRmse.mean(axis=0)
46 | print 'OUTPUT : ', maeRmse.mean(axis=0)
47 |
--------------------------------------------------------------------------------
/settings.py:
--------------------------------------------------------------------------------
1 | class Settings(object):
2 |
3 | def __init__(self, debug=False):
4 |
5 | self.INPUT_HORIZON = 12
6 | self.MOVING_HORIZON = 6
7 |
8 | self.ACTIVATION = 'relu' # One of 'sigmoid', 'relu', 'elu'
9 | self.CRITERION = 'L1Loss' # One of 'L1Loss', 'MSE', 'SmoothL1Loss'
10 | self.LEARNING_RATE = 0.01
11 | self.WEIGHT_DECAY = 1e-3
12 | self.CLIP_GRAD_NORM = 40
13 | self.LR_DROP_FACTOR = 0.1
14 | self.LR_DROP_PATIENCE = 10
15 | self.PATIENCE = 25
16 | self.OPTIMIZER = 'RMSprop' # One of 'RMSprop', 'Adam', 'Adadelta', 'SGD'
17 | self.N_EPOCHS = [100, 100, 100, 100, 100, 100] if not debug else [1, 1, 1, 1, 1, 1]
18 |
19 | self.TRAIN_RATIO = 0.98
20 | self.VAL_RATIO = 0.1
21 |
22 | self.SEED = 73
23 |
24 | assert len(self.N_EPOCHS) == self.MOVING_HORIZON
25 |
--------------------------------------------------------------------------------
/test.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import random, os, argparse
3 | import numpy as np
4 | from lib import Data, Model, Loader, draw_graph_all_stations
5 | from settings import Settings
6 |
7 | s = Settings()
8 |
9 | parser = argparse.ArgumentParser()
10 | parser.add_argument('--data', help='Path of the processed dataset', type=str, required=True)
11 | parser.add_argument('--model', help='Path of the model', type=str, required=True)
12 | parser.add_argument('--n_stations', help='Number of stations to use', type=int, default=5)
13 | parser.add_argument('--usegpu', action='store_true', help='Enable cuda to train on gpu')
14 | args = parser.parse_args()
15 |
16 | if torch.cuda.is_available() and not args.usegpu:
17 | print 'WARNING: You have a CUDA device, so you should probably run with --usegpu'
18 |
19 | model_dir = os.path.dirname(args.model)
20 |
21 | # Load Seeds
22 | random.seed(s.SEED)
23 | np.random.seed(s.SEED)
24 | torch.manual_seed(s.SEED)
25 |
26 | # Load Data
27 | data = Data(data_file=args.data, input_horizon=s.INPUT_HORIZON,
28 | n_stations=args.n_stations, train_ratio=s.TRAIN_RATIO,
29 | val_ratio=s.VAL_RATIO, debug=False)
30 |
31 | # Load Model
32 | model = Model(args.n_stations, s.MOVING_HORIZON, s.ACTIVATION, s.CRITERION, load_model_path=args.model, usegpu=args.usegpu)
33 |
34 | # Train First RNN
35 | _, _, [X_test, y_test] = data.load_data_lstm_1()
36 |
37 | print '\n\n' + '#' * 10 + ' TESTING ' + '#' * 10
38 | prediction_test = model.test([X_test, y_test])
39 | draw_graph_all_stations(model_dir, data, args.n_stations, y_test, prediction_test)
40 |
--------------------------------------------------------------------------------
/train.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import random, getpass, datetime, shutil, os, argparse
3 | import numpy as np
4 | from lib import Data, Model, Loader, draw_graph_all_stations
5 | from settings import Settings
6 |
7 | s = Settings()
8 |
9 | parser = argparse.ArgumentParser()
10 | parser.add_argument('--data', help='Path of the processed dataset', type=str, required=True)
11 | parser.add_argument('--n_stations', help='Number of stations to use', type=int, default=5)
12 | parser.add_argument('--batch_size', help='Input minibatch size', type=int, default=256)
13 | parser.add_argument('--n_workers', help='Number of data loading workers [0 to do it using main process]', type=int, default=4)
14 | parser.add_argument('--usegpu', action='store_true', help='Enable cuda to train on gpu')
15 | parser.add_argument('--debug', action='store_true', help='Enable debug mode')
16 | args = parser.parse_args()
17 |
18 | if torch.cuda.is_available() and not args.usegpu:
19 | print 'WARNING: You have a CUDA device, so you should probably run with --usegpu'
20 |
21 | def generate_run_id():
22 |
23 | username = getpass.getuser()
24 |
25 | now = datetime.datetime.now()
26 | date = map(str, [now.year, now.month, now.day])
27 | coarse_time = map(str, [now.hour, now.minute])
28 | fine_time = map(str, [now.second, now.microsecond])
29 |
30 | run_id = '_'.join(['-'.join(date), '-'.join(coarse_time), username, '-'.join(fine_time)])
31 | return run_id
32 |
33 | RUN_ID = generate_run_id()
34 | model_save_path = os.path.join('models', RUN_ID, 'models_{}')
35 | output_dir = os.path.join('outputs', RUN_ID)
36 |
37 | for rnn_model_num in range(1, s.MOVING_HORIZON + 1):
38 | try:
39 | os.makedirs(model_save_path.format(rnn_model_num))
40 | except:
41 | pass
42 |
43 | try:
44 | os.makedirs(output_dir)
45 | except:
46 | pass
47 |
48 | pin_memory = False
49 | if args.usegpu:
50 | pin_memory = True
51 |
52 | # Load Seeds
53 | random.seed(s.SEED)
54 | np.random.seed(s.SEED)
55 | torch.manual_seed(s.SEED)
56 |
57 | # Load Data
58 | data = Data(data_file=args.data, input_horizon=s.INPUT_HORIZON,
59 | n_stations=args.n_stations, train_ratio=s.TRAIN_RATIO,
60 | val_ratio=s.VAL_RATIO, debug=args.debug)
61 |
62 | # Load Model
63 | model = Model(args.n_stations, s.MOVING_HORIZON, s.ACTIVATION, s.CRITERION, usegpu=args.usegpu)
64 |
65 | # Train First RNN
66 | [X_train, y_train], [X_val, y_val], [X_test, y_test] = data.load_data_lstm_1()
67 |
68 | rnn_model_num = 1
69 | print '#' * 10 + ' RNN 1 ' + '#' * 10
70 |
71 | train_loader = torch.utils.data.DataLoader(Loader((X_train, y_train)), batch_size=args.batch_size, shuffle=True,
72 | num_workers=args.n_workers, pin_memory=pin_memory)
73 |
74 | val_loader = torch.utils.data.DataLoader(Loader((X_val, y_val)), batch_size=args.batch_size, shuffle=False,
75 | num_workers=args.n_workers, pin_memory=pin_memory)
76 |
77 | model.fit(rnn_model_num, s.LEARNING_RATE, s.WEIGHT_DECAY, s.CLIP_GRAD_NORM, s.LR_DROP_FACTOR, s.LR_DROP_PATIENCE, s.PATIENCE,
78 | s.OPTIMIZER, s.N_EPOCHS[rnn_model_num - 1],
79 | train_loader, val_loader, model_save_path.format(rnn_model_num))
80 |
81 | # Train Other RNNs
82 | for rnn_model_num in range(2, s.MOVING_HORIZON + 1):
83 | X_train, y_train = data.load_data(X_train, y_train, model, rnn_model_num - 1)
84 | X_val, y_val = data.load_data(X_val, y_val, model, rnn_model_num - 1)
85 | print '#' * 10 + ' RNN {} '.format(rnn_model_num) + '#' * 10
86 | train_loader = torch.utils.data.DataLoader(Loader((X_train, y_train)), batch_size=args.batch_size, shuffle=True,
87 | num_workers=args.n_workers, pin_memory=pin_memory)
88 |
89 | val_loader = torch.utils.data.DataLoader(Loader((X_val, y_val)), batch_size=args.batch_size, shuffle=False,
90 | num_workers=args.n_workers, pin_memory=pin_memory)
91 |
92 | model.fit(rnn_model_num, s.LEARNING_RATE, s.WEIGHT_DECAY, s.CLIP_GRAD_NORM, s.LR_DROP_FACTOR, s.LR_DROP_PATIENCE, s.PATIENCE,
93 | s.OPTIMIZER, s.N_EPOCHS[rnn_model_num - 1],
94 | train_loader, val_loader, model_save_path.format(rnn_model_num))
95 |
96 |
97 | print '\n\n' + '#' * 10 + ' TESTING ' + '#' * 10
98 | prediction_test = model.test([X_test, y_test])
99 | draw_graph_all_stations(output_dir, data, args.n_stations, y_test, prediction_test)
100 |
--------------------------------------------------------------------------------
64 |