├── README.md ├── basic_attention.py ├── basic_rnn_lstm_gru.py ├── data └── google.csv ├── data_loader.py ├── main_time_series_prediction.py └── utils.py /README.md: -------------------------------------------------------------------------------- 1 | # Codebase for "Time-series prediction" with RNN, GRU, LSTM and Attention 2 | 3 | Authors: Jinsung Yoon 4 | Contact: jsyoon0823@gmail.com 5 | 6 | This directory contains implementations of basic time-series prediction 7 | using RNN, GRU, LSTM or Attention methods. 8 | To run the pipeline, simply run python3 -m main_time_series_prediction.py. 9 | 10 | Note: We recommend to do MinMax normalization on both input and output. 11 | 12 | ## Stages of time-series prediction framework: 13 | 14 | - Load dataset (Google stocks data) 15 | - Train model: 16 | (1) RNN based: Simple RNN, GRU, LSTM 17 | (2) Attention based 18 | - Evaluate the performance: MAE or MSE metrics 19 | 20 | ### Command inputs: 21 | 22 | - train_rate: training data ratio 23 | - seq_len: sequence length 24 | - task: classification or regression 25 | - model_type: rnn, lstm, gru, or attention 26 | - h_dim: hidden state dimensions 27 | - n_layer: number of layers 28 | - batch_size: the number of samples in each mini-batch 29 | - epoch: the number of iterations 30 | - learning_rate: learning rates 31 | - metric_name: mse or mae 32 | 33 | ### Example command 34 | 35 | ```shell 36 | $ python3 main_time_series_prediction.py 37 | --train_rate 0.8 --seq_len 7 --task regression --model_type lstm 38 | --h_dim 10 --n_layer 3 --batch_size 32 --epoch 100 --learning_rate 0.01 39 | --metric_name mae 40 | ``` 41 | 42 | ### Outputs 43 | 44 | - MAE or MSE performance of trained model 45 | -------------------------------------------------------------------------------- /basic_attention.py: -------------------------------------------------------------------------------- 1 | """Basic Attention core functions for time-series prediction. 2 | 3 | Author: Jinsung Yoon 4 | Contact: jsyoon0823@gmail.com 5 | """ 6 | 7 | # Necessary packages 8 | import tensorflow as tf 9 | import numpy as np 10 | import os 11 | import shutil 12 | 13 | 14 | class Attention(): 15 | """Attention class. 16 | 17 | Attributes: 18 | - model_parameters: 19 | - task: classificiation or regression 20 | - h_dim: hidden state dimensions 21 | - batch_size: the number of samples in each mini-batch 22 | - epoch: the number of iterations 23 | - learning_rate: learning rate of training 24 | """ 25 | def __init__(self, model_parameters): 26 | 27 | tf.compat.v1.reset_default_graph() 28 | self.task = model_parameters['task'] 29 | self.h_dim = model_parameters['h_dim'] 30 | self.batch_size = model_parameters['batch_size'] 31 | self.epoch = model_parameters['epoch'] 32 | self.learning_rate = model_parameters['learning_rate'] 33 | 34 | self.save_file_directory = 'tmp/attention/' 35 | 36 | 37 | def process_batch_input_for_RNN(self, batch_input): 38 | """Function to convert batch input data to use scan ops of tensorflow. 39 | 40 | Args: 41 | - batch_input: original batch input 42 | 43 | Returns: 44 | - x: batch_input for RNN 45 | """ 46 | batch_input_ = tf.transpose(batch_input, perm=[2, 0, 1]) 47 | x = tf.transpose(batch_input_) 48 | return x 49 | 50 | 51 | def sample_X(self, m, n): 52 | """Sample from the real data (Mini-batch index sampling). 53 | """ 54 | return np.random.permutation(m)[:n] 55 | 56 | 57 | def fit(self, x, y): 58 | """Train the model. 59 | 60 | Args: 61 | - x: training feature 62 | - y: training label 63 | """ 64 | 65 | # Basic parameters 66 | no, seq_len, x_dim = x.shape 67 | y_dim = len(y[0, :]) 68 | 69 | # Weights for GRU 70 | Wr = tf.Variable(tf.zeros([x_dim, self.h_dim])) 71 | Ur = tf.Variable(tf.zeros([self.h_dim, self.h_dim])) 72 | br = tf.Variable(tf.zeros([self.h_dim])) 73 | 74 | Wu = tf.Variable(tf.zeros([x_dim, self.h_dim])) 75 | Uu = tf.Variable(tf.zeros([self.h_dim, self.h_dim])) 76 | bu = tf.Variable(tf.zeros([self.h_dim])) 77 | 78 | Wh = tf.Variable(tf.zeros([x_dim, self.h_dim])) 79 | Uh = tf.Variable(tf.zeros([self.h_dim, self.h_dim])) 80 | bh = tf.Variable(tf.zeros([self.h_dim])) 81 | 82 | # Weights for attention mechanism 83 | Wa1 = tf.Variable(tf.random.truncated_normal([self.h_dim + x_dim, 84 | self.h_dim], 85 | mean=0, stddev=.01)) 86 | Wa2 = tf.Variable(tf.random.truncated_normal([self.h_dim, y_dim], 87 | mean=0, stddev=.01)) 88 | ba1 = tf.Variable(tf.random.truncated_normal([self.h_dim], 89 | mean=0, stddev=.01)) 90 | ba2 = tf.Variable(tf.random.truncated_normal([y_dim], mean=0, stddev=.01)) 91 | 92 | # Weights for output layers 93 | Wo = tf.Variable(tf.random.truncated_normal([self.h_dim, y_dim], 94 | mean=0, stddev=.01)) 95 | bo = tf.Variable(tf.random.truncated_normal([y_dim], mean=0, stddev=.01)) 96 | 97 | # Target 98 | Y = tf.compat.v1.placeholder(tf.float32, [None,1]) 99 | # Input vector with shape[batch, seq, embeddings] 100 | _inputs = tf.compat.v1.placeholder(tf.float32, shape=[None, None, x_dim], 101 | name='inputs') 102 | 103 | # Processing inputs to work with scan function 104 | processed_input = self.process_batch_input_for_RNN(_inputs) 105 | 106 | # Initial Hidden States 107 | initial_hidden = _inputs[:, 0, :] 108 | initial_hidden = tf.matmul(initial_hidden, tf.zeros([x_dim, self.h_dim])) 109 | 110 | 111 | def GRU(previous_hidden_state, x): 112 | """Function for Forward GRU cell. 113 | 114 | Args: 115 | - previous_hidden_state 116 | - x: current input 117 | 118 | Returns: 119 | - current_hidden_state 120 | """ 121 | # R Gate 122 | r = tf.sigmoid(tf.matmul(x, Wr) + \ 123 | tf.matmul(previous_hidden_state, Ur) + br) 124 | # U Gate 125 | u = tf.sigmoid(tf.matmul(x, Wu) + \ 126 | tf.matmul(previous_hidden_state, Uu) + bu) 127 | # Final Memory cell 128 | c = tf.tanh(tf.matmul(x, Wh) + \ 129 | tf.matmul( tf.multiply(r, previous_hidden_state), Uh) + bh) 130 | # Current Hidden state 131 | current_hidden_state = tf.multiply( (1 - u), previous_hidden_state ) + \ 132 | tf.multiply( u, c ) 133 | return current_hidden_state 134 | 135 | 136 | def get_states(): 137 | """Function to get the hidden and memory cells after forward pass. 138 | 139 | Returns: 140 | - all_hidden_states 141 | """ 142 | # Getting all hidden state through time 143 | all_hidden_states = tf.scan(GRU, processed_input, 144 | initializer=initial_hidden, name='states') 145 | return all_hidden_states 146 | 147 | 148 | def get_attention(hidden_state): 149 | """Function to get attention with the last input. 150 | 151 | Args: 152 | - hidden_states 153 | 154 | Returns: 155 | - e_values 156 | """ 157 | inputs = tf.concat((hidden_state, processed_input[-1]), axis = 1) 158 | hidden_values = tf.nn.tanh(tf.matmul(inputs, Wa1) + ba1) 159 | e_values = (tf.matmul(hidden_values, Wa2) + ba2) 160 | return e_values 161 | 162 | 163 | def get_outputs(): 164 | """Function for getting output and attention coefficient. 165 | 166 | Returns: 167 | - output: final outputs 168 | - a_values: attention values 169 | """ 170 | all_hidden_states = get_states() 171 | all_attention = tf.map_fn(get_attention, all_hidden_states) 172 | a_values = tf.nn.softmax(all_attention, axis = 0) 173 | final_hidden_state = tf.einsum('ijk,ijl->jkl', a_values, 174 | all_hidden_states) 175 | output = tf.nn.sigmoid(tf.matmul(final_hidden_state[:,0,:], Wo) + bo, 176 | name='outputs') 177 | return output, a_values 178 | 179 | # Getting all outputs from rnn 180 | outputs, attention_values = get_outputs() 181 | 182 | # reshape out for sequence_loss 183 | if self.task == 'classification': 184 | loss = tf.reduce_mean(Y * tf.log(outputs + 1e-8) + \ 185 | (1-Y) * tf.log(1-outputs + 1e-8)) 186 | elif self.task == 'regression': 187 | loss = tf.sqrt(tf.reduce_mean(tf.square(outputs - Y))) 188 | 189 | # Optimization 190 | optimizer = tf.compat.v1.train.AdamOptimizer(self.learning_rate) 191 | train = optimizer.minimize(loss) 192 | 193 | # Sessions 194 | sess = tf.compat.v1.Session() 195 | sess.run(tf.compat.v1.global_variables_initializer()) 196 | 197 | # Training 198 | iteration_per_epoch = int(no/self.batch_size) 199 | iterations = int((self.epoch * no) / self.batch_size) 200 | 201 | for i in range(iterations): 202 | 203 | idx = self.sample_X(no, self.batch_size) 204 | Input = x[idx,:,:] 205 | _, step_loss = sess.run([train, loss], 206 | feed_dict={Y: y[idx], _inputs: Input}) 207 | 208 | # Print intermediate results 209 | if i % iteration_per_epoch == iteration_per_epoch-1: 210 | print('Epoch: ' + str(int(i/iteration_per_epoch)) + 211 | ', Loss: ' + str(np.round(step_loss, 4))) 212 | 213 | # Reset the directory for saving 214 | if not os.path.exists(self.save_file_directory): 215 | os.makedirs(self.save_file_directory) 216 | else: 217 | shutil.rmtree(self.save_file_directory) 218 | 219 | # Save model 220 | inputs = {'inputs': _inputs} 221 | outputs = {'outputs': outputs} 222 | tf.compat.v1.saved_model.simple_save(sess, self.save_file_directory, 223 | inputs, outputs) 224 | 225 | 226 | def predict(self, test_x): 227 | """Prediction with trained model. 228 | 229 | Args: 230 | - test_x: testing features 231 | 232 | Returns: 233 | - test_y_hat: predictions on testing set 234 | """ 235 | 236 | graph = tf.Graph() 237 | with graph.as_default(): 238 | with tf.compat.v1.Session() as sess: 239 | tf.compat.v1.saved_model.loader.load(sess, [tf.saved_model.SERVING], 240 | self.save_file_directory) 241 | x = graph.get_tensor_by_name('inputs:0') 242 | outputs = graph.get_tensor_by_name('outputs:0') 243 | 244 | test_y_hat = sess.run(outputs, feed_dict={x: test_x}) 245 | 246 | return test_y_hat 247 | -------------------------------------------------------------------------------- /basic_rnn_lstm_gru.py: -------------------------------------------------------------------------------- 1 | """General RNN core functions for time-series prediction. 2 | 3 | Author: Jinsung Yoon 4 | Contact: jsyoon0823@gmail.com 5 | """ 6 | 7 | # Necessary packages 8 | import os 9 | import tensorflow as tf 10 | import numpy as np 11 | from datetime import datetime 12 | from tensorflow.keras import layers 13 | from keras.callbacks import ModelCheckpoint 14 | from utils import binary_cross_entropy_loss, mse_loss, rnn_sequential 15 | 16 | 17 | class GeneralRNN(): 18 | """RNN predictive model to time-series. 19 | 20 | Attributes: 21 | - model_parameters: 22 | - task: classification or regression 23 | - model_type: 'rnn', 'lstm', or 'gru' 24 | - h_dim: hidden dimensions 25 | - n_layer: the number of layers 26 | - batch_size: the number of samples in each batch 27 | - epoch: the number of iteration epochs 28 | - learning_rate: the learning rate of model training 29 | """ 30 | 31 | def __init__(self, model_parameters): 32 | 33 | self.task = model_parameters['task'] 34 | self.model_type = model_parameters['model_type'] 35 | self.h_dim = model_parameters['h_dim'] 36 | self.n_layer = model_parameters['n_layer'] 37 | self.batch_size = model_parameters['batch_size'] 38 | self.epoch = model_parameters['epoch'] 39 | self.learning_rate = model_parameters['learning_rate'] 40 | 41 | assert self.model_type in ['rnn', 'lstm', 'gru'] 42 | 43 | # Predictor model define 44 | self.predictor_model = None 45 | 46 | # Set path for model saving 47 | model_path = 'tmp' 48 | if not os.path.exists(model_path): 49 | os.makedirs(model_path) 50 | self.save_file_name = '{}'.format(model_path) + \ 51 | datetime.now().strftime('%H%M%S') + '.hdf5' 52 | 53 | 54 | def _build_model(self, x, y): 55 | """Construct the model using feature and label statistics. 56 | 57 | Args: 58 | - x: features 59 | - y: labels 60 | 61 | Returns: 62 | - model: predictor model 63 | """ 64 | # Parameters 65 | h_dim = self.h_dim 66 | n_layer = self.n_layer 67 | dim = len(x[0, 0, :]) 68 | max_seq_len = len(x[0, :, 0]) 69 | 70 | model = tf.keras.Sequential() 71 | model.add(layers.Masking(mask_value=0., input_shape=(max_seq_len, dim))) 72 | 73 | for _ in range(n_layer - 1): 74 | model = rnn_sequential(model, self.model_type, h_dim, return_seq=True) 75 | 76 | model = rnn_sequential(model, self.model_type, h_dim, 77 | return_seq=False) 78 | adam = tf.keras.optimizers.Adam(learning_rate=self.learning_rate, 79 | beta_1=0.9, beta_2=0.999, amsgrad=False) 80 | 81 | if self.task == 'classification': 82 | model.add(layers.Dense(y.shape[-1], activation='sigmoid')) 83 | model.compile(loss=binary_cross_entropy_loss, optimizer=adam) 84 | 85 | elif self.task == 'regression': 86 | model.add(layers.Dense(y.shape[-1], activation='linear')) 87 | model.compile(loss=mse_loss, optimizer=adam, metrics=['mse']) 88 | 89 | return model 90 | 91 | 92 | def fit(self, x, y): 93 | """Fit the predictor model. 94 | 95 | Args: 96 | - x: training features 97 | - y: training labels 98 | 99 | Returns: 100 | - self.predictor_model: trained predictor model 101 | """ 102 | idx = np.random.permutation(len(x)) 103 | train_idx = idx[:int(len(idx)*0.8)] 104 | valid_idx = idx[int(len(idx)*0.8):] 105 | 106 | train_x, train_y = x[train_idx], y[train_idx] 107 | valid_x, valid_y = x[valid_idx], y[valid_idx] 108 | 109 | self.predictor_model = self._build_model(train_x, train_y) 110 | 111 | # Callback for the best model saving 112 | save_best = ModelCheckpoint(self.save_file_name, monitor='val_loss', 113 | mode='min', verbose=False, 114 | save_best_only=True) 115 | 116 | # Train the model 117 | self.predictor_model.fit(train_x, train_y, 118 | batch_size=self.batch_size, epochs=self.epoch, 119 | validation_data=(valid_x, valid_y), 120 | callbacks=[save_best], verbose=True) 121 | 122 | self.predictor_model.load_weights(self.save_file_name) 123 | os.remove(self.save_file_name) 124 | 125 | return self.predictor_model 126 | 127 | 128 | def predict(self, test_x): 129 | """Return the temporal and feature importance. 130 | 131 | Args: 132 | - test_x: testing features 133 | 134 | Returns: 135 | - test_y_hat: predictions on testing set 136 | """ 137 | test_y_hat = self.predictor_model.predict(test_x) 138 | return test_y_hat 139 | -------------------------------------------------------------------------------- /data/google.csv: -------------------------------------------------------------------------------- 1 | Open,High,Low,Volume,Close 2 | 828.659973,833.450012,828.349976,1247700,831.659973 3 | 823.02002,828.070007,821.655029,1597800,828.070007 4 | 819.929993,824.400024,818.97998,1281700,824.159973 5 | 819.359985,823,818.469971,1304000,818.97998 6 | 819,823,816,1053600,820.450012 7 | 816,820.958984,815.48999,1198100,819.23999 8 | 811.700012,815.25,809.780029,1129100,813.669983 9 | 809.51001,810.659973,804.539978,989700,809.559998 10 | 807,811.840027,803.190002,1155300,808.380005 11 | 803.98999,810.5,801.780029,1235200,806.969971 12 | 799.700012,801.669983,795.25,1174200,801.340027 13 | 802.98999,806,800.369995,1460400,801.48999 14 | 793.799988,802.700012,792,1525800,798.530029 15 | 799.679993,801.190002,791.190002,2023300,795.695007 16 | 796.859985,801.25,790.52002,2143500,796.789978 17 | 814.659973,815.840027,799.799988,3228900,802.320007 18 | 834.710022,841.950012,820.440002,2951800,823.309998 19 | 837.809998,838,827.01001,2734400,832.150024 20 | 829.619995,835.77002,825.059998,1494500,835.669983 21 | 822.299988,825.900024,817.820984,1461000,823.869995 22 | 807.25,820.869995,803.73999,1901600,819.309998 23 | 806.909973,806.909973,801.690002,1645000,805.02002 24 | 805.119995,809.47998,801.799988,912000,802.174988 25 | 805.809998,806.205017,800.98999,1293300,806.070007 26 | 807.080017,807.140015,800.369995,1355800,804.609985 27 | 807.47998,811.223999,806.690002,1090100,807.880005 28 | 807.140015,807.390015,799.169983,1351000,806.359985 29 | 805,808.150024,801.369995,1057900,807.909973 30 | 807.859985,809.130005,803.51001,1174300,804.789978 31 | 806.400024,809.966003,802.830017,1272400,806.650024 32 | 795.26001,807.900024,792.203979,1620500,806.150024 33 | 786.080017,794.47998,785.02002,1315400,794.02002 34 | 788.359985,791.340027,783.159973,1065400,786.900024 35 | 778.809998,789.630005,775.799988,1643100,786.140015 36 | 782.75,782.780029,770.409973,1760200,771.820007 37 | 783.330017,785.929993,778.919983,742200,782.789978 38 | 793.700012,794.22998,783.200012,1132700,785.049988 39 | 790.679993,797.859985,787.656982,789100,791.549988 40 | 790.900024,792.73999,787.280029,623400,789.909973 41 | 792.359985,793.320007,788.580017,969100,791.26001 42 | 795.840027,796.676025,787.099976,1208700,794.559998 43 | 796.76001,798.650024,793.27002,925100,796.419983 44 | 790.219971,797.659973,786.27002,1225900,794.200012 45 | 800.400024,800.856018,790.289978,2428300,790.799988 46 | 797.340027,803,792.919983,1623700,797.849976 47 | 797.400024,804,794.01001,1664500,797.070007 48 | 793.900024,804.380005,793.340027,2119800,796.099976 49 | 785.039978,791.25,784.35498,2097100,789.27002 50 | 780,789.429993,779.020996,1780500,789.289978 51 | 772.47998,778.179993,767.22998,1448200,776.419983 52 | 761,771.359985,755.799988,1750800,771.190002 53 | 764.72998,768.830017,757.340027,1632800,759.109985 54 | 757.710022,763.900024,752.900024,1382900,762.52002 55 | 744.590027,754,743.099976,1448700,750.5 56 | 757.440002,759.849976,737.025024,2996900,747.919983 57 | 770.070007,772.98999,754.830017,2365800,758.039978 58 | 771.530029,778.5,768.23999,1604500,770.840027 59 | 760,779.530029,759.799988,2172200,768.23999 60 | 764.26001,765,760.52002,586000,761.679993 61 | 767.72998,768.28302,755.25,1477400,760.98999 62 | 772.630005,776.960022,767,1591800,768.27002 63 | 762.609985,769.700012,760.599976,1323000,769.200012 64 | 771.369995,775,760,1526200,760.539978 65 | 766.919983,772.700012,764.22998,1287000,771.22998 66 | 755.200012,766.359985,750.51001,1465200,764.47998 67 | 746.969971,764.416016,746.969971,2341200,758.48999 68 | 755.599976,757.849976,727.539978,3631700,736.080017 69 | 756.539978,760.780029,750.380005,2394000,754.02002 70 | 791.169983,791.169983,752.179993,4740100,762.559998 71 | 779.940002,791.22699,771.669983,2602700,785.309998 72 | 783.400024,795.632996,780.190002,1350800,790.51001 73 | 774.5,785.190002,772.549988,1585100,782.52002 74 | 750.659973,770.359985,750.560974,2134800,762.02002 75 | 767.25,769.950012,759.030029,1943200,762.130005 76 | 778.200012,781.650024,763.450012,1872400,768.700012 77 | 782.890015,789.48999,775.539978,2406400,783.609985 78 | 795.469971,796.859985,784,2427300,784.539978 79 | 808.349976,815.48999,793.590027,4269900,795.369995 80 | 801,803.48999,791.5,2749200,795.349976 81 | 806.340027,806.97998,796.320007,1647700,799.070007 82 | 816.679993,816.679993,805.140015,1576400,807.669983 83 | 804.900024,815.179993,804.820007,1697500,813.109985 84 | 795,799.5,794,1266200,799.369995 85 | 803.299988,803.969971,796.030029,1757500,796.969971 86 | 798.23999,804.599976,798.030029,1766800,801.5 87 | 787.849976,801.609985,785.565002,2056900,795.26001 88 | 779.799988,785.849976,777.5,1093000,779.960022 89 | 781.650024,783.950012,776,852500,778.530029 90 | 781.219971,781.219971,773,1365300,778.190002 91 | 783.76001,788.130005,782.059998,937400,786.140015 92 | 786.659973,792.280029,780.580017,1372500,783.070007 93 | 777.710022,789.380005,775.869995,1174900,785.940002 94 | 779.659973,779.659973,770.75,933200,775.080017 95 | 779,780.47998,775.539978,1070700,776.859985 96 | 779.309998,782.070007,775.650024,1461200,776.469971 97 | 776.030029,778.710022,772.890015,1201400,776.429993 98 | 774.25,776.065002,769.5,1278800,772.559998 99 | 776.330017,780.940002,774.090027,1585300,777.289978 100 | 781.440002,785.799988,774.231995,1314700,775.01001 101 | 777.849976,781.809998,774.969971,1109800,781.559998 102 | 775.5,785.98999,774.307983,1153200,783.01001 103 | 782.73999,782.73999,773.070007,1533200,774.210022 104 | 786.590027,788.929993,784.150024,1411900,786.900024 105 | 780,789.849976,778.440002,1486200,787.210022 106 | 772.659973,777.159973,768.301025,1167800,776.219971 107 | 769,773.330017,768.530029,978600,771.409973 108 | 772.419983,774,764.440979,1172800,765.700012 109 | 769.75,769.75,764.659973,2049300,768.880005 110 | 762.890015,773.799988,759.960022,1305100,771.76001 111 | 759.609985,767.679993,759.109985,1087400,762.48999 112 | 764.47998,766.219971,755.799988,1395000,759.690002 113 | 755.130005,770.289978,754,1311000,769.02002 114 | 770.099976,773.244995,759.659973,1885500,759.659973 115 | 778.590027,780.349976,773.580017,1270300,775.320007 116 | 780,782.72998,776.200012,893700,780.349976 117 | 773.450012,782,771,1442800,780.080017 118 | 773.01001,773.919983,768.409973,1072700,771.460022 119 | 769.25,771.02002,764.299988,925100,768.780029 120 | 767.01001,769.090027,765.380005,1248600,767.049988 121 | 769.330017,774.466003,766.840027,1130000,769.090027 122 | 768.73999,774.98999,766.61499,847600,772.150024 123 | 769,776.080017,765.849976,1166700,769.539978 124 | 767,771.890015,763.184998,926900,769.409973 125 | 770.580017,774.5,767.070007,1072000,769.640015 126 | 775.47998,776.440002,771.784973,928200,772.080017 127 | 773.27002,774.539978,770.049988,951400,772.150024 128 | 775,777.099976,773.130005,861500,775.419983 129 | 780.01001,782.859985,777,719400,777.5 130 | 777.320007,780.809998,773.530029,924200,779.909973 131 | 780.299988,780.97998,773.44397,1028000,777.140015 132 | 783.75,787.48999,780.109985,938200,782.440002 133 | 781.5,783.39502,780.400024,740500,783.219971 134 | 785,789.75,782.969971,975100,784.849976 135 | 783.75,786.812012,782.778015,786400,784.679993 136 | 781.099976,788.940002,780.570007,1318900,784.26001 137 | 782,782.630005,778.091003,1107900,781.76001 138 | 773.780029,783.039978,772.340027,1801200,782.219971 139 | 772.219971,774.070007,768.794983,1140300,771.609985 140 | 767.179993,773.210022,766.820007,1287400,773.179993 141 | 768.690002,775.840027,767.849976,1784500,771.070007 142 | 761.090027,780.429993,761.090027,2700500,772.880005 143 | 772.710022,778.549988,766.77002,3841500,768.789978 144 | 747.039978,748.650024,739.299988,3530200,745.909973 145 | 738.280029,744.460022,737,1512500,741.77002 146 | 739.039978,741.690002,734.27002,1186700,738.419983 147 | 740.669983,742.609985,737.5,1032400,739.77002 148 | 741.859985,743.23999,736.559998,1259800,742.73999 149 | 740.359985,741.690002,735.830994,1026300,738.630005 150 | 737.330017,742.130005,737.099976,1289700,741.190002 151 | 729.890015,736.98999,729,1227500,736.960022 152 | 722.710022,736.130005,721.190002,1295500,733.780029 153 | 725.72998,725.73999,719.054993,1279300,719.849976 154 | 721.580017,722.210022,718.030029,950200,720.950012 155 | 723.619995,724,716.849976,935900,716.97998 156 | 719.119995,722.940002,715.909973,1336900,720.640015 157 | 708.049988,716.51001,707.23999,1111800,715.090027 158 | 699.5,705.710022,696.434998,1575200,705.630005 159 | 698.080017,698.200012,688.215027,1304200,695.359985 160 | 689.97998,701.679993,689.090027,1411900,697.77002 161 | 696.059998,696.940002,688.880005,1462600,694.950012 162 | 692.200012,700.650024,692.130005,1344700,699.210022 163 | 685.469971,692.320007,683.650024,1597700,692.099976 164 | 683,687.429016,681.409973,1932600,684.109985 165 | 678.969971,680.330017,673,2173800,680.039978 166 | 671,672.299988,663.283997,2629000,668.26001 167 | 675.169983,689.400024,673.450012,4449000,675.219971 168 | 697.450012,701.950012,687,2171400,701.869995 169 | 699.059998,700.859985,693.08197,1184300,697.460022 170 | 698.400024,702.77002,692.01001,1465600,695.940002 171 | 698.77002,702.47998,693.409973,2082500,693.710022 172 | 708.650024,708.820007,688.452026,3402400,691.719971 173 | 714.909973,716.650024,703.26001,1982500,710.359985 174 | 719,722.97998,717.309998,1214500,718.919983 175 | 716.47998,722.469971,713.119995,1306100,718.27002 176 | 716.51001,725.440002,716.51001,1258900,718.359985 177 | 719.469971,725.890015,716.429993,1216400,719.409973 178 | 722.869995,729.539978,722.335999,988900,728.580017 179 | 723.960022,728.570007,720.580017,1583700,728.280029 180 | 719.840027,721.97998,716.549988,1336200,716.650024 181 | 724.909973,724.909973,714.609985,1565300,716.549988 182 | 729.27002,729.48999,720.559998,1226300,722.340027 183 | 732.5,733.02002,724.169983,1341800,730.400024 184 | 734.530029,737.210022,730.659973,1253600,734.150024 185 | 731.73999,739.72998,731.26001,2129500,735.719971 186 | 724.01001,733.935974,724,1974000,732.659973 187 | 722.869995,728.330017,720.280029,1576300,724.119995 188 | 720.76001,727.51001,719.705017,1629200,725.27002 189 | 706.859985,720.969971,706.859985,1929500,720.090027 190 | 706.530029,711.478027,704.179993,1330700,704.23999 191 | 701.619995,714.580017,700.52002,1828400,709.73999 192 | 702.359985,706,696.799988,1670200,700.320007 193 | 703.669983,711.599976,700.630005,1766800,706.630005 194 | 715.98999,721.52002,704.109985,2001200,706.22998 195 | 709.130005,718.47998,705.650024,1317100,716.48999 196 | 711.929993,716.661987,709.26001,1314500,710.830017 197 | 717.059998,719.25,709,1360700,713.309998 198 | 723.409973,724.47998,712.799988,1692100,715.289978 199 | 716.75,723.5,715.719971,1569600,723.179993 200 | 712,718.710022,710,1510300,712.900024 201 | 698.380005,711.859985,698.106995,1826100,711.119995 202 | 697.700012,702.320007,695.719971,1683500,701.429993 203 | 690.48999,699.75,689.01001,1693500,695.700012 204 | 696.869995,697.840027,692,1543800,692.359985 205 | 697.630005,700.640015,691,1645300,698.210022 206 | 690.700012,697.619995,689,2487700,693.01001 207 | 708.26001,714.169983,689.549988,2867300,691.02002 208 | 707.289978,708.97998,692.36499,3098600,705.840027 209 | 725.419983,725.765991,703.026001,2744600,708.140015 210 | 716.099976,723.929993,715.590027,1959200,723.150024 211 | 726.299988,736.119995,713.609985,5951900,718.77002 212 | 755.380005,760.450012,749.549988,3060500,759.140015 213 | 758,758.132019,750.01001,1529200,752.669983 214 | 769.51001,769.900024,749.330017,2030500,753.929993 215 | 760.460022,768.049988,757.299988,1556000,766.609985 216 | 753.97998,761,752.69397,1809300,759 217 | 754.01001,757.309998,752.705017,1135300,753.200012 218 | 749.159973,754.380005,744.260986,1707100,751.719971 219 | 738,743.830017,731.01001,1353000,743.090027 220 | 743.02002,745,736.049988,1220100,736.099976 221 | 743.969971,745.450012,735.549988,1290800,739.150024 222 | 745.369995,747,736.280029,1453200,740.280029 223 | 735.77002,746.23999,735.559998,1053700,745.690002 224 | 738,742.799988,735.369995,1132300,737.799988 225 | 750.059998,752.799988,742.429993,1134200,745.289978 226 | 738.599976,750.340027,737,1576700,749.909973 227 | 749.25,750.849976,740.940002,1718800,744.950012 228 | 750.099976,757.880005,748.73999,1782400,750.530029 229 | 734.590027,747.25,728.76001,1903800,744.77002 230 | 736.789978,738.98999,732.5,1301300,733.530029 231 | 732.01001,737.747009,731,1594900,735.299988 232 | 742.359985,745.719971,736.150024,1432100,738.059998 233 | 737.460022,745,737.460022,1269700,740.75 234 | 736.5,742.5,733.515991,1836500,742.090027 235 | 741.859985,742,731.830017,2980700,737.599976 236 | 736.450012,743.070007,736,1860800,737.780029 237 | 726.369995,737.469971,724.51001,1624400,736.090027 238 | 726.919983,732.289978,724.77002,1721000,728.330017 239 | 726.809998,735.5,725.150024,1718300,730.48999 240 | 720,726.919983,717.125,1970800,726.820007 241 | 708.119995,716.440002,703.359985,2833500,712.820007 242 | 698.469971,705.679993,694,1421500,705.23999 243 | 688.590027,703.789978,685.340027,2076300,693.969971 244 | 706.900024,708.091003,686.900024,2985100,695.159973 245 | 714.98999,716.48999,706.02002,1972100,710.890015 246 | 718.679993,719.450012,706.02002,1958000,712.419983 247 | 719,720,712,1629000,718.849976 248 | 703.619995,718.809998,699.77002,2151400,718.809998 249 | 700.320007,710.890015,697.679993,2481100,697.77002 250 | 708.580017,713.429993,700.859985,2243500,705.070007 251 | 700.01001,705.97998,690.585022,1642200,705.75 252 | 688.919983,700,680.780029,1963600,699.559998 253 | 701.450012,708.400024,693.580017,2009300,695.849976 254 | 707.450012,713.23999,702.51001,1949800,706.460022 255 | 695.030029,703.080994,694.049988,1589300,700.909973 256 | 710,712.349976,696.030029,1883200,697.349976 257 | 698.090027,709.75,691.380005,2492600,708.400024 258 | 692.97998,698,685.049988,2520000,691 259 | 690.26001,693.75,678.599976,2141400,682.400024 260 | 675,689.349976,668.867981,3024000,683.109985 261 | 686.859985,701.309998,682.130005,2638000,684.119995 262 | 672.320007,699.900024,668.77002,3608900,678.109985 263 | 667.849976,684.030029,663.059998,4247400,682.73999 264 | 703.869995,703.98999,680.150024,5105700,683.570007 265 | 722.809998,727,701.859985,5168700,708.01001 266 | 770.219971,774.5,720.5,6171000,726.950012 267 | 784.5,789.869995,764.650024,6348100,764.650024 268 | 750.460022,757.859985,743.27002,5139200,752 269 | 731.530029,744.98999,726.799988,3474300,742.950012 270 | 722.219971,733.690002,712.349976,2676400,730.960022 271 | 713.669983,718.234985,694.390015,2194200,699.98999 272 | 713.849976,718.280029,706.47998,1331700,713.039978 273 | 723.580017,729.679993,710.01001,1711700,711.669983 274 | 723.599976,728.130005,720.120972,2011800,725.25 275 | 702.179993,719.190002,694.460022,2412200,706.590027 276 | 688.609985,706.849976,673.26001,3445000,698.450012 277 | 703.299988,709.97998,693.409973,2268100,701.789978 278 | 692.289978,706.73999,685.369995,3592400,694.450012 279 | 705.380005,721.924988,689.099976,2225800,714.719971 280 | 730.849976,734.73999,698.609985,2501700,700.559998 281 | 721.679993,728.75,717.317017,2024500,726.070007 282 | 716.609985,718.85498,703.539978,2090600,716.030029 283 | 731.450012,733.22998,713,2450900,714.469971 284 | 730.309998,738.5,719.059998,2963700,726.390015 285 | 730,747.179993,728.919983,1947000,743.619995 286 | 746.450012,752,738.640015,1950700,742.580017 287 | 743,744.059998,731.257996,3272800,741.840027 288 | 769.5,769.5,758.340027,1489600,758.880005 289 | 776.599976,777.599976,766.900024,1293300,771 290 | 766.690002,779.97998,766.429993,1765000,776.599976 291 | 752.919983,762.98999,749.52002,1515300,762.51001 292 | 749.549988,751.349976,746.619995,527200,748.400024 293 | 753.469971,754.210022,744,1565900,750.309998 294 | 751.650024,754.849976,745.530029,1365400,750 295 | 746.130005,750,740,1525700,747.77002 296 | 746.51001,754.130005,738.150024,3148700,739.309998 297 | 762.419983,762.679993,749,1553400,749.429993 298 | 750,760.590027,739.434998,1993300,758.090027 299 | 753,758.080017,743.01001,2666200,743.400024 300 | 741.789978,748.72998,724.169983,2412500,747.77002 301 | 741.159973,745.710022,736.75,2224400,738.869995 302 | 752.849976,755.849976,743.830017,1984900,749.460022 303 | 759.169983,764.22998,737.000977,2700000,751.609985 304 | 757.890015,764.799988,754.200012,1829500,762.369995 305 | 767.77002,768.72998,755.090027,1812300,763.25 306 | 753.099976,768.48999,750,2757300,766.809998 307 | 766.01001,768.994995,745.630005,2590600,752.539978 308 | 768.900024,775.955017,758.960022,2230400,762.380005 309 | 747.109985,768.950012,746.700012,2134600,767.039978 310 | 748.809998,754.929993,741.27002,2097600,742.599976 311 | 748.460022,753.409973,747.48999,838500,750.26001 312 | 748.140015,752,746.059998,1122100,748.150024 313 | 752,755.278992,737.630005,2333100,748.280029 314 | 757.450012,762.708008,751.820007,1414500,755.97998 315 | 746.530029,757.919983,743,2212300,756.599976 316 | 738.73999,742,737.429993,1327100,738.409973 317 | 727.580017,741.409973,727,1684300,740 318 | 729.289978,731.844971,723.026978,1510900,725.299988 319 | 715.599976,729.48999,711.330017,1905900,728.960022 320 | 729.169983,731.150024,716.72998,2075500,717 321 | 731,737.799988,728.64502,1837200,731.22998 322 | 732.460022,741,730.22998,1366400,735.400024 323 | 724.400024,730.590027,718.5,1608000,728.320007 324 | 730.200012,734.710022,719.429993,2069800,724.890015 325 | 731.5,735.409973,727.01001,1511600,733.76001 326 | 729.469971,739.47998,729.469971,1861600,731.25 327 | 722,733.099976,721.900024,1706700,728.109985 328 | 718.859985,724.650024,714.719971,1565400,722.159973 329 | 711.059998,721.619995,705.849976,1886300,721.109985 330 | 715.72998,718,710.049988,1908800,710.809998 331 | 710.5,718.26001,710.01001,1456000,716.919983 332 | 707.330017,712.97998,703.080017,2178900,712.950012 333 | 707.380005,713.619995,704.549988,2245800,708.48999 334 | 701.549988,719.150024,701.26001,2716600,712.780029 335 | 727.5,730,701.5,6653900,702 336 | 646.700012,657.799988,644.01001,4071000,651.789978 337 | 654.150024,655.869995,641.72998,1791100,642.609985 338 | 664.039978,664.719971,644.195007,2490000,650.280029 339 | 661.179993,666.820007,659.580017,1477300,666.099976 340 | 664.109985,664.969971,657.200012,1611100,662.200012 341 | 654.659973,663.130005,654.460022,1885700,661.73999 342 | 653.210022,659.390015,648.849976,1415500,651.159973 343 | 643.150024,657.812012,643.150024,1807700,652.299988 344 | 642.090027,648.5,639.01001,1275200,646.669983 345 | 640,645.98999,635.317993,1648700,643.609985 346 | 641.359985,644.450012,625.559998,2182100,639.159973 347 | 649.23999,650.609009,632.150024,2092700,642.359985 348 | 638.840027,649.25,636.530029,2166300,645.440002 349 | 632,643.01001,627,1803600,641.469971 350 | 607.200012,627.340027,603.130005,2684800,626.909973 351 | 608.369995,612.090027,599.849976,1867600,611.289978 352 | 603.280029,608.76001,600.72998,2413400,608.419983 353 | 597.280029,605,590.219971,2309500,594.969971 354 | 610.340027,614.60498,589.380005,3127700,594.890015 355 | 629.77002,629.77002,611,2174000,611.969971 356 | 616.640015,627.320007,612.400024,2240100,625.799988 357 | 622.049988,628.929993,620,1470900,622.359985 358 | 627,627.549988,615.429993,2562900,622.690002 359 | 634.400024,636.48999,625.940002,1788500,635.440002 360 | 636.789978,640,627.02002,5133400,629.25 361 | 637.789978,650.900024,635.02002,2274700,642.900024 362 | 635.469971,637.950012,632.320007,1286500,635.97998 363 | 626.700012,638.700012,623.780029,2084400,635.140015 364 | 625.700012,625.859985,619.429993,1702300,623.23999 365 | 619.75,625.780029,617.419983,1373500,625.77002 366 | 613.099976,624.159973,611.429993,1905300,621.349976 367 | 621.219971,626.52002,609.599976,1702100,612.719971 368 | 612.48999,616.309998,604.119995,2279500,614.659973 369 | 600,603.469971,595.25,2089000,600.700012 370 | 617,619.710022,602.820984,1759600,606.25 371 | 605.590027,614.340027,599.710022,2575600,614.340027 372 | 602.359985,612.859985,594.099976,3702100,597.789978 373 | 627.539978,635.799988,617.679993,2176700,618.25 374 | 632.820007,636.880005,624.559998,1978700,630.380005 375 | 639.400024,643.590027,622,3491300,637.609985 376 | 610.349976,631.710022,599.049988,4235900,628.619995 377 | 614.909973,617.450012,581.109985,3538000,582.059998 378 | 573,599.330017,565.049988,5770300,589.609985 379 | 639.780029,640.049988,612.330017,4265200,612.47998 380 | 655.460022,662.98999,642.900024,2855300,646.830017 381 | 656.599976,667,654.190002,2131600,660.900024 382 | 661.900024,664,653.460022,1456100,656.130005 383 | 656.799988,661.380005,651.23999,1051700,660.869995 384 | 655.01001,659.85498,652.659973,1072100,657.119995 385 | 659.322021,664.5,651.661011,1810700,656.450012 386 | 663.080017,665,652.289978,2936700,659.559998 387 | 669.200012,674.900024,654.27002,5029200,660.780029 388 | 639.47998,643.440002,631.249023,1809200,633.72998 389 | 640.22998,642.679993,629.710022,1403900,635.299988 390 | 645,645.379028,632.25,1572600,642.679993 391 | 634.330017,647.859985,633.159973,2334300,643.780029 392 | 628.419983,634.809998,627.159973,1490900,629.25 393 | 625.340027,633.05603,625.340027,1304500,631.210022 394 | 631.380005,632.909973,625.5,1706100,625.609985 395 | 630,635.219971,622.049988,1474200,632.590027 396 | 628.799988,633.359985,622.650024,1575100,631.929993 397 | 632.830017,632.830017,623.309998,1727300,628 398 | 621,634.299988,620.5,2675400,627.26001 399 | 647,648.169983,622.52002,3625700,623.559998 400 | 661.27002,663.630005,641,3029100,644.280029 401 | 660.890015,678.640015,659,3929300,662.099976 402 | 655.210022,673,654.299988,3377200,662.299988 403 | 659.23999,668.880005,653.01001,5860900,663.02002 404 | 649,674.468018,645,11164900,672.929993 405 | 565.119995,580.679993,565,4768300,579.849976 406 | 560.130005,566.502991,556.789978,1784600,560.219971 407 | 546.76001,565.848999,546.710022,3244100,561.099976 408 | 532.880005,547.109985,532.400024,2206500,546.549988 409 | 526.289978,532.559998,525.549988,1956700,530.130005 410 | 523.119995,523.77002,520.349976,1839400,520.679993 411 | 521.049988,522.734009,516.109985,1296700,516.830017 412 | 523.130005,526.179993,515.179993,1597200,525.02002 413 | 519.5,525.25,519,1280500,522.859985 414 | 521.080017,524.650024,521.080017,1235900,523.400024 415 | 524.72998,525.690002,518.22998,1961000,521.840027 416 | 526.02002,526.25,520.5,2217200,520.51001 417 | 525.01001,528.609985,520.539978,1937800,521.52002 418 | 537.26001,537.76001,531.349976,2109100,531.690002 419 | 538.869995,540.900024,535.22998,1335700,535.22998 420 | 540,540,535.659973,1286600,537.840027 421 | 539.640015,541.499023,535.25,1197500,540.47998 422 | 539.590027,543.73999,537.530029,1250300,538.190002 423 | 537.210022,538.25,533.01001,1893500,536.690002 424 | 531,538.150024,530.789978,1833100,536.72998 425 | 529.369995,530.97998,525.099976,1294200,529.26001 426 | 528.400024,529.640015,525.559998,1071800,528.150024 427 | 528,528.299988,524,1632700,527.200012 428 | 531.599976,533.119995,530.159973,955800,532.330017 429 | 538.424988,538.97998,533.02002,1217500,534.609985 430 | 529.359985,538.359985,529.349976,1815000,536.690002 431 | 527.559998,529.200012,523.01001,1455300,526.690002 432 | 533.309998,534.119995,526.23999,1520600,526.830017 433 | 536.349976,537.200012,532.52002,1388200,533.330017 434 | 537.76001,540.590027,534.320007,1348300,536.700012 435 | 539.909973,543.5,537.109985,1714500,540.309998 436 | 532.929993,543,531.330017,1939000,539.179993 437 | 536.789978,536.789978,529.76001,1904300,533.98999 438 | 537.369995,538.630005,531.450012,2597400,532.109985 439 | 538.01001,540.609985,536.25,1029800,539.780029 440 | 532.799988,540.549988,531.710022,1525000,539.789978 441 | 538.119995,539,529.880005,2406500,532.320007 442 | 540.150024,544.190002,539.51001,1176200,540.109985 443 | 537.950012,543.840027,535.97998,1462700,542.51001 444 | 538.48999,542.919983,532.971985,1430800,539.27002 445 | 533.97998,540.659973,533.039978,1966900,537.359985 446 | 532.01001,534.820007,528.849976,2003400,532.299988 447 | 539.179993,539.273987,530.380005,1971300,533.849976 448 | 533.77002,539,532.409973,1403900,538.400024 449 | 530.559998,534.322021,528.655029,1252300,529.619995 450 | 531.599976,533.208984,525.26001,1634200,529.039978 451 | 538.369995,541.97998,535.400024,905300,535.700012 452 | 536.650024,541.150024,525,1527600,538.219971 453 | 523.98999,533.460022,521.75,1546300,530.700012 454 | 531.23999,532.380005,521.085022,1567000,524.219971 455 | 538.210022,539.73999,530.390991,1383100,530.799988 456 | 538.530029,544.070007,535.059998,1308000,540.780029 457 | 538.429993,539.539978,532.099976,1768200,537.900024 458 | 547.869995,548.590027,535.049988,2082200,537.340027 459 | 550.469971,553.679993,546.905029,1698800,549.080017 460 | 554.640015,556.02002,550.366028,1491000,553.679993 461 | 563.390015,565.950012,553.200012,2398000,555.369995 462 | 566.102584,571.14259,557.252507,4932500,565.062561 463 | 541.002435,550.96249,540.23244,4184800,547.002472 464 | 534.402426,541.082428,531.752397,1593500,539.367458 465 | 537.512456,539.392429,533.677415,1844700,533.972413 466 | 525.602352,536.092424,524.50235,1679200,535.382408 467 | 528.662379,529.842373,521.012371,2151800,524.052386 468 | 529.902414,535.592396,529.612373,1299800,533.802391 469 | 528.702406,534.732432,523.22235,2318800,532.532429 470 | 536.252409,537.572435,528.094416,2604100,530.392405 471 | 538.412447,544.062463,537.312445,1645300,539.172466 472 | 542.292472,542.292472,537.312445,1409400,540.012416 473 | 541.032425,541.952428,535.492451,1557800,540.782472 474 | 538.382457,543.852476,538.382457,1178500,541.612446 475 | 538.08244,542.692434,536.002456,1302800,537.022404 476 | 532.222436,538.412447,529.572407,1324400,536.767432 477 | 540.852427,540.852427,533.849395,1716300,535.532417 478 | 548.602502,551.142488,539.502472,1963000,542.562439 479 | 550.00246,554.712521,546.722468,1588000,548.002468 480 | 551.622503,553.472487,548.17249,1287500,552.032502 481 | 553.002509,555.282504,548.132463,1897400,548.342512 482 | 557.59255,558.90254,550.652497,1572600,555.172522 483 | 570.50259,572.262605,558.742555,2152200,558.787539 484 | 562.562541,574.592603,561.212525,2583200,570.192597 485 | 560.432554,562.362529,555.832536,1643800,558.81251 486 | 561.652513,561.722529,559.052548,2616800,560.362537 487 | 559.392531,560.802526,556.147548,1197200,557.992512 488 | 552.50248,559.782516,547.002472,2134500,559.502513 489 | 551.712472,553.802493,548.002468,1805500,550.842471 490 | 550.952514,556.852545,546.002476,1640900,554.512509 491 | 553.502476,558.402511,544.222448,1703500,547.322503 492 | 553.512513,556.37253,550.462462,1389600,555.512505 493 | 555.142533,558.142521,550.682486,1820700,551.182515 494 | 564.252539,564.852573,554.732534,1792300,555.012538 495 | 566.862541,570.272589,563.537566,1062100,568.852557 496 | 574.882583,576.682625,566.762536,1659100,567.687558 497 | 575.022616,577.912621,573.412609,1389600,575.332609 498 | 571.872619,577.112637,568.012546,1876800,573.372583 499 | 570.452587,575.392588,566.522559,1704700,573.64261 500 | 560.532559,572.152562,558.752531,2129600,571.342601 501 | 554.242482,564.712541,552.902503,2410100,558.402511 502 | 543.212476,556.142529,541.502464,2311500,555.482516 503 | 535.90245,546.222501,535.447406,1825900,543.872489 504 | 530.002419,536.792403,528.252381,1005000,536.092424 505 | 536.052398,536.441404,529.412422,1457800,531.912381 506 | 543.132484,543.75247,535.802444,1444300,538.952441 507 | 538.042413,543.11247,538.012424,989100,542.872432 508 | 541.402458,545.492471,537.512456,1453000,539.702422 509 | 546.83245,550.00246,541.092465,1616800,542.842443 510 | 543.352447,549.912491,543.132484,1900300,549.012501 511 | 537.252405,544.822482,534.675391,1620200,542.932472 512 | 535.302416,538.452412,533.380397,1377700,535.972405 513 | 529.302379,537.702431,526.922378,1749800,536.942412 514 | 528.002366,532.002411,526.022388,1267700,527.832406 515 | 527.64237,537.202402,526.412373,1763500,531.002415 516 | 523.792395,528.502395,522.092359,1849700,527.582391 517 | 529.2424,532.67442,521.272361,1663600,522.762349 518 | 528.002366,533.40243,523.262377,2038600,529.2424 519 | 531.732383,533.002407,518.552377,2849800,528.482381 520 | 515.862322,539.872444,515.522339,5606300,534.522445 521 | 511.002313,511.092313,501.202274,4186300,510.6623 522 | 522.782362,522.992349,510.002318,1683800,510.002318 523 | 529.972369,530.702398,518.19232,1904000,518.63237 524 | 538.532466,539.002444,529.672413,1543700,535.212448 525 | 535.592396,542.172453,533.002407,2281700,539.952437 526 | 521.482349,536.332401,519.702382,2676900,534.392388 527 | 507.252283,519.282346,506.202284,2268700,518.042373 528 | 511.002313,512.502307,506.018277,2232000,506.902294 529 | 500.012273,508.1923,500.002267,2298200,508.082288 530 | 505.572291,505.682303,497.762267,2715800,501.792271 531 | 494.652237,503.232286,493.002234,2215500,500.872267 532 | 498.842256,502.982272,492.392224,2370400,496.182251 533 | 494.942247,495.97823,487.562205,2326700,492.552239 534 | 504.7623,504.922285,494.792239,2071300,496.172244 535 | 497.992268,503.48227,491.002212,3353500,502.682285 536 | 507.002299,507.246285,499.652247,2065000,501.102268 537 | 515.002358,516.177334,501.052266,2899900,501.962262 538 | 523.262377,524.332389,513.062315,2059800,513.872306 539 | 529.012399,531.272382,524.102388,1447500,524.812404 540 | 531.252429,532.602384,525.802363,1368200,526.402397 541 | 528.092396,531.152424,527.132366,876200,530.422394 542 | 532.192385,535.482414,530.013375,2278500,530.332426 543 | 528.772422,534.252417,527.312364,1040500,534.032392 544 | 530.512424,531.761394,527.022384,705900,528.772422 545 | 527.00237,534.56241,526.292354,2197600,530.592416 546 | 516.082347,526.462376,516.082347,2723700,524.872383 547 | 511.512318,517.722342,506.91328,3690200,516.352313 548 | 512.952333,513.872306,504.70229,2926700,511.102319 549 | 497.002248,507.002299,496.812244,2883100,504.892295 550 | 511.562321,513.052308,489.00222,3964200,495.392242 551 | 522.742396,523.102392,513.272333,2813400,513.802351 552 | 523.512391,528.502395,518.662359,1994500,518.662359 553 | 527.802416,533.922411,527.102376,1610800,528.34241 554 | 533.082399,536.332401,525.562386,1712200,526.062353 555 | 522.142362,534.192438,520.502366,1871300,533.37244 556 | 527.132366,531.002415,523.792395,2329300,526.982357 557 | 531.002415,532.892425,524.282386,2565600,525.262369 558 | 531.1624,537.342434,528.592363,1392100,537.312445 559 | 531.442404,535.998416,529.262414,1277900,531.322384 560 | 533.512412,535.502427,529.802408,1526600,533.752389 561 | 538.902438,541.412434,531.862379,2115300,533.802391 562 | 540.622426,542.002431,536.602429,1148300,541.832471 563 | 540.882478,541.552467,537.044437,1522900,540.372473 564 | 539.002444,543.982471,538.60444,1789900,541.082428 565 | 537.652428,542.702471,535.622446,1706300,539.272471 566 | 541.612446,542.142464,536.562402,2224200,537.502419 567 | 531.252429,535.112442,531.082408,1563300,534.832438 568 | 535.002399,538.242425,530.082412,1392200,536.992415 569 | 537.502419,541.942452,534.172425,1962600,535.032449 570 | 543.582448,543.792436,534.063422,1726000,536.512399 571 | 546.682503,546.682503,542.15244,1289500,544.402446 572 | 549.80251,549.80251,543.482442,1339300,545.38249 573 | 550.392507,550.462462,545.172441,1129600,547.312465 574 | 548.492459,551.942473,546.302493,965500,550.292501 575 | 541.462437,549.592461,541.022449,1134600,547.492463 576 | 546.212464,546.212464,538.672437,1633700,541.012473 577 | 545.502448,546.887472,540.972446,1333200,542.042458 578 | 556.802542,556.802542,544.052487,2032200,545.922484 579 | 553.002509,555.502529,549.302481,1244200,554.112486 580 | 555.502529,557.902544,553.23251,1382200,555.222525 581 | 559.352504,559.572529,554.752486,2035000,559.082538 582 | 548.952461,552.802497,543.512432,1455600,550.312514 583 | 550.00246,554.192479,546.982459,1770500,549.33247 584 | 543.002488,548.982512,541.622422,1270900,548.902458 585 | 537.032441,544.412483,537.032441,1185300,540.772435 586 | 544.36248,544.882461,535.792407,1973100,539.782415 587 | 539.322413,547.222497,535.852447,2348800,543.982471 588 | 529.892376,539.802428,528.802412,2919200,532.712427 589 | 525.192353,526.792383,519.112324,2336200,526.542369 590 | 509.452317,521.762353,508.102301,2607500,520.842349 591 | 527.252385,530.982402,508.532313,5539300,511.172305 592 | 519.002342,529.432374,515.002358,3708500,524.512387 593 | 531.012391,532.802396,518.302363,3719300,530.032409 594 | 538.902438,547.192507,533.172429,2222500,537.942408 595 | 544.992443,549.502492,533.102413,2581600,533.212394 596 | 557.722546,565.132577,544.052487,3081900,544.492476 597 | 571.182555,571.49261,559.062524,2524800,560.882518 598 | 565.572566,573.882587,557.492545,1990800,572.502582 599 | 574.402629,575.27263,563.742534,1911300,563.742534 600 | 578.802636,581.002639,574.442595,1214600,577.352614 601 | 573.052613,577.227637,572.502582,1141700,575.282606 602 | 567.312567,571.912585,563.322559,1178400,570.082554 603 | 576.012635,577.582615,567.01255,1445400,568.272597 604 | 576.932639,579.852634,572.852602,1621700,577.36259 605 | 571.7526,578.192625,571.172579,1282400,576.362594 606 | 576.062638,579.2526,574.662619,1443600,577.1026 607 | 587.552646,587.982658,574.182604,1925900,575.062581 608 | 581.462641,589.632691,580.522624,1728100,587.992634 609 | 586.852667,586.852667,581.002639,1471400,581.132634 610 | 593.82271,593.951665,583.462632,1689500,587.372648 611 | 591.502688,596.482715,589.502696,3736600,596.082692 612 | 587.002675,589.542661,585.002622,1444500,589.272695 613 | 580.012619,587.522656,578.777604,1692800,584.772622 614 | 572.762572,581.502606,572.662566,1480300,579.95264 615 | 572.94257,574.952599,568.212557,1597500,573.102616 616 | 581.002639,581.642639,574.462608,1601600,575.622589 617 | 580.362639,581.812661,576.262588,1221000,581.352659 618 | 581.502606,583.502659,576.942615,977300,583.102636 619 | 588.902662,589.002667,580.002643,1287200,581.012615 620 | 586.602653,591.772654,586.302635,1431000,589.722659 621 | 583.982613,586.55265,581.952632,1632300,586.082672 622 | 580.002643,586.00268,579.222611,1458100,581.982621 623 | 580.002643,582.992655,575.002602,1215100,577.942611 624 | 571.852606,577.832629,571.192593,1578400,577.332601 625 | 571.332564,572.04258,567.07155,1083700,571.602592 626 | 569.562573,573.252625,567.102579,1292900,569.202577 627 | 577.272622,578.492642,570.105566,1703300,571.002557 628 | 581.262629,581.802623,576.582619,1639600,577.862619 629 | 584.722619,585.002622,579.002647,1361400,580.202654 630 | 583.592628,585.238621,580.642643,789000,582.562642 631 | 583.822628,584.502655,581.14261,914800,583.372664 632 | 585.88266,586.702658,582.572618,1036700,584.492618 633 | 585.002622,587.342658,584.002627,978600,586.862643 634 | 576.11258,584.512631,576.002598,1284100,582.162619 635 | 577.862619,579.382595,570.522603,1519100,573.482626 636 | 576.182596,577.902645,570.882599,985400,574.652582 637 | 567.312567,575.002602,565.752564,1439200,574.782577 638 | 564.522567,565.902572,560.882518,1542000,562.732562 639 | 569.992585,570.492553,566.002578,1214700,567.882551 640 | 563.562536,570.252576,560.352561,1494700,568.772565 641 | 568.00257,569.89258,561.102543,1110900,563.362525 642 | 561.782569,570.702601,560.002541,1334300,566.376589 643 | 570.052564,571.982601,562.612543,1551200,565.072537 644 | 569.042592,575.352622,564.102531,1427300,573.152619 645 | 570.402584,575.962633,562.85252,1955200,566.072533 646 | 580.602616,583.652668,570.002561,2102700,571.602592 647 | 586.55265,589.502696,584.002627,1016400,587.42265 648 | 588.752653,589.702646,583.517654,1349800,585.612633 649 | 588.072688,592.502683,584.755668,986800,590.602697 650 | 590.402686,591.862684,587.032665,932400,589.022681 651 | 596.452725,599.502716,591.772654,1035100,593.352671 652 | 593.232713,597.852683,592.502683,1233100,595.982686 653 | 590.722655,599.652725,590.602697,1699100,594.742713 654 | 591.752702,594.40267,585.235622,2062100,589.472645 655 | 593.002712,596.802684,582.002635,4014100,595.082696 656 | 579.532604,580.992601,568.61258,3016500,573.732579 657 | 588.002671,588.402633,582.202646,1397100,582.662648 658 | 585.742628,585.807626,576.562606,1623000,584.782659 659 | 582.602608,585.212671,578.032641,1854000,584.872627 660 | 571.912585,580.85263,571.422594,1621700,579.182645 661 | 565.912548,576.592595,565.012558,1356700,571.102563 662 | 571.582578,576.72259,569.378597,1116700,576.08259 663 | 577.662607,579.530645,566.137592,1909500,571.092587 664 | 583.76265,586.432631,579.592644,1064600,582.252649 665 | 583.352651,585.01266,580.922646,714100,584.732656 666 | 583.352651,585.442672,580.392628,1056300,582.33766 667 | 578.32262,584.402649,576.652635,1447900,582.672624 668 | 578.662603,579.572631,574.752588,1313700,575.282606 669 | 577.182592,579.872648,573.802595,2236900,577.242632 670 | 581.002639,582.45266,571.852606,1741900,576.002598 671 | 565.262572,579.962616,565.222546,1969300,578.652627 672 | 565.192556,572.648612,561.012513,2207100,564.622572 673 | 555.152509,565.002582,554.252519,1536800,564.952579 674 | 556.852545,557.582513,550.394465,4508300,556.362492 675 | 554.242482,555.0025,548.512473,2456800,554.902495 676 | 544.862448,553.562516,544.002484,1741700,553.372481 677 | 544.202435,545.32245,539.33245,1444500,543.012465 678 | 549.262515,549.622511,541.522477,1702500,544.282488 679 | 552.262503,552.30253,545.562488,1220500,551.762475 680 | 557.302509,557.992512,548.46247,1458400,551.352476 681 | 558.002549,559.882522,555.022514,1100100,558.842499 682 | 560.512546,563.602563,557.902544,1351700,560.552511 683 | 557.152501,562.902523,556.042523,1467500,562.122552 684 | 558.062528,558.062528,548.932509,1736700,556.332503 685 | 546.402499,554.952498,544.452449,1689100,553.902499 686 | 541.502464,548.612478,538.752429,1816400,544.662436 687 | 550.99248,552.342495,542.552463,1866600,544.94244 688 | 560.70252,560.902531,545.732448,1435000,553.932488 689 | 560.802526,561.352557,555.912528,1771100,559.892559 690 | 563.352549,564.002525,558.712504,1354100,560.082534 691 | 564.57257,567.842585,561.002537,1652000,561.682564 692 | 556.002496,566.002578,554.352525,2104100,565.952575 693 | 547.262462,553.642508,543.702467,1932100,552.702492 694 | 541.132431,547.602506,540.782472,1615800,545.062459 695 | 532.902401,539.185441,531.912381,1196200,538.942465 696 | 529.742368,536.232457,526.302392,1784700,529.772418 697 | 519.702382,529.782394,517.58537,1277700,528.862391 698 | 521.39238,521.802379,515.442347,1485200,520.632362 699 | 525.702357,525.872379,517.422325,1704300,519.982324 700 | 533.002407,533.002407,525.292358,1191700,526.652412 701 | 530.892433,536.072411,529.512367,1653400,533.092437 702 | 523.512391,530.192393,519.012379,1912500,529.922427 703 | 510.752299,519.902332,504.202292,2439400,518.732375 704 | 508.462297,517.232351,506.452298,2021300,511.002313 705 | 515.792305,516.68232,503.302272,3224300,509.962321 706 | 525.232379,526.812396,515.062337,1689000,515.14233 707 | 524.822381,528.902418,521.322364,1024100,527.812392 708 | 533.762426,534.002403,525.612389,1688500,527.932411 709 | 527.112413,532.932391,523.882364,1905500,531.352435 710 | 527.602405,528.002366,522.522372,1751100,526.662388 711 | 516.902344,529.462425,516.322324,2699000,527.70241 712 | 517.182348,518.602319,502.802274,3335500,517.152359 713 | 522.512395,524.702361,515.422333,2100300,516.182352 714 | 530.072374,531.652391,522.122349,1883100,525.162363 715 | 533.792415,533.872408,526.252389,2052300,526.942391 716 | 528.642366,537.232453,527.512375,2365300,534.812425 717 | 536.1024,536.702435,525.602352,2566700,528.622414 718 | 548.81249,549.502492,531.152424,6809500,536.1024 719 | 543.002488,557.002492,540.00244,4893200,556.54249 720 | 536.822454,538.452412,518.462348,3855000,536.442444 721 | 538.252462,544.10249,529.56237,2575000,532.522392 722 | 532.552381,540.00244,526.532392,3924800,530.602392 723 | 565.002582,565.002582,539.902434,4036800,540.952433 724 | 559.622532,565.372554,552.952506,3330800,564.142557 725 | 542.602466,555.0025,541.612446,3151200,554.902495 726 | 540.742445,548.482483,527.152379,4401600,538.152456 727 | 574.652582,577.772589,543.002488,6369300,543.14246 728 | 569.852553,587.282679,564.132581,5099100,569.742571 729 | 599.992707,604.832763,562.192568,147100,567.002574 730 | 558.712504,568.452595,558.712504,7900,567.162558 731 | 566.892592,567.002574,556.932537,10800,556.972503 732 | 561.202549,566.43259,558.672539,41200,559.992565 733 | 568.00257,568.00257,552.922516,13100,558.462551 734 | -------------------------------------------------------------------------------- /data_loader.py: -------------------------------------------------------------------------------- 1 | """Data loader. 2 | 3 | Author: Jinsung Yoon 4 | Contact: jsyoon0823@gmail.com 5 | ---------------------------------------- 6 | Loads Google stock dataset with MinMax normalization. 7 | Reference: https://finance.yahoo.com/quote/GOOGL/history?p=GOOGL 8 | """ 9 | 10 | # Necessary Packages 11 | import numpy as np 12 | from utils import MinMaxScaler 13 | 14 | 15 | def data_loader(train_rate = 0.8, seq_len = 7): 16 | """Loads Google stock data. 17 | 18 | Args: 19 | - train_rate: the ratio between training and testing sets 20 | - seq_len: sequence length 21 | 22 | Returns: 23 | - train_x: training feature 24 | - train_y: training labels 25 | - test_x: testing features 26 | - test_y: testing labels 27 | """ 28 | 29 | # Load data 30 | ori_data = np.loadtxt('data/google.csv', delimiter=',', skiprows = 1) 31 | # Reverse the time order 32 | reverse_data = ori_data[::-1] 33 | # Normalization 34 | norm_data = MinMaxScaler(reverse_data) 35 | 36 | # Build dataset 37 | data_x = [] 38 | data_y = [] 39 | 40 | for i in range(0, len(norm_data[:,0]) - seq_len): 41 | # Previous seq_len data as features 42 | temp_x = norm_data[i:i + seq_len,:] 43 | # Values at next time point as labels 44 | temp_y = norm_data[i + seq_len, [-1]] 45 | data_x = data_x + [temp_x] 46 | data_y = data_y + [temp_y] 47 | 48 | data_x = np.asarray(data_x) 49 | data_y = np.asarray(data_y) 50 | 51 | # Train / test Division 52 | idx = np.random.permutation(len(data_x)) 53 | train_idx = idx[:int(train_rate * len(data_x))] 54 | test_idx = idx[int(train_rate * len(data_x)):] 55 | 56 | train_x, test_x = data_x[train_idx, :, :], data_x[test_idx, :, :] 57 | train_y, test_y = data_y[train_idx, :], data_y[test_idx, :] 58 | 59 | return train_x, train_y, test_x, test_y 60 | -------------------------------------------------------------------------------- /main_time_series_prediction.py: -------------------------------------------------------------------------------- 1 | """Time-series prediction main function 2 | 3 | Author: Jinsung Yoon 4 | Contact: jsyoon0823@gmail.com 5 | ------------------------------------ 6 | (1) Load data 7 | (2) Train model (RNN, GRU, LSTM, Attention) 8 | (3) Evaluate the trained model 9 | """ 10 | 11 | # Necessary packages 12 | from __future__ import absolute_import 13 | from __future__ import division 14 | from __future__ import print_function 15 | 16 | import argparse 17 | import warnings 18 | warnings.filterwarnings("ignore") 19 | 20 | from data_loader import data_loader 21 | from basic_rnn_lstm_gru import GeneralRNN 22 | from basic_attention import Attention 23 | from utils import performance 24 | 25 | 26 | def main (args): 27 | """Time-series prediction main function. 28 | 29 | Args: 30 | - train_rate: training data ratio 31 | - seq_len: sequence length 32 | - task: classification or regression 33 | - model_type: rnn, lstm, gru, or attention 34 | - h_dim: hidden state dimensions 35 | - n_layer: number of layers 36 | - batch_size: the number of samples in each mini-batch 37 | - epoch: the number of iterations 38 | - learning_rate: learning rates 39 | - metric_name: mse or mae 40 | """ 41 | # Load data 42 | train_x, train_y, test_x, test_y = data_loader(args.train_rate, 43 | args.seq_len) 44 | 45 | # Model traininig / testing 46 | model_parameters = {'task': args.task, 47 | 'model_type': args.model_type, 48 | 'h_dim': args.h_dim, 49 | 'n_layer': args.n_layer, 50 | 'batch_size': args.batch_size, 51 | 'epoch': args.epoch, 52 | 'learning_rate': args.learning_rate} 53 | 54 | if args.model_type in ['rnn','lstm','gru']: 55 | general_rnn = GeneralRNN(model_parameters) 56 | general_rnn.fit(train_x, train_y) 57 | test_y_hat = general_rnn.predict(test_x) 58 | elif args.model_type == 'attention': 59 | basic_attention = Attention(model_parameters) 60 | basic_attention.fit(train_x, train_y) 61 | test_y_hat = basic_attention.predict(test_x) 62 | 63 | # Evaluation 64 | result = performance(test_y, test_y_hat, args.metric_name) 65 | print('Performance (' + args.metric_name + '): ' + str(result)) 66 | 67 | 68 | ## 69 | if __name__ == '__main__': 70 | 71 | # Inputs for the main function 72 | parser = argparse.ArgumentParser() 73 | parser.add_argument( 74 | '--train_rate', 75 | help='training data ratio', 76 | default=0.8, 77 | type=str) 78 | parser.add_argument( 79 | '--seq_len', 80 | help='sequence length', 81 | default=7, 82 | type=int) 83 | parser.add_argument( 84 | '--model_type', 85 | choices=['rnn','gru','lstm','attention'], 86 | default='attention', 87 | type=str) 88 | parser.add_argument( 89 | '--h_dim', 90 | default=10, 91 | type=int) 92 | parser.add_argument( 93 | '--n_layer', 94 | default=3, 95 | type=int) 96 | parser.add_argument( 97 | '--batch_size', 98 | default=32, 99 | type=int) 100 | parser.add_argument( 101 | '--epoch', 102 | default=100, 103 | type=int) 104 | parser.add_argument( 105 | '--learning_rate', 106 | default=0.01, 107 | type=float) 108 | parser.add_argument( 109 | '--task', 110 | choices=['classification','regression'], 111 | default='regression', 112 | type=str) 113 | parser.add_argument( 114 | '--metric_name', 115 | choices=['mse','mae'], 116 | default='mae', 117 | type=str) 118 | 119 | args = parser.parse_args() 120 | 121 | # Call main function 122 | main(args) -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | """Utility functions for time-series prediction. 2 | 3 | Author: Jinsung Yoon 4 | Contact: jsyoon0823@gmail.com 5 | ------------------------------------ 6 | (1) MinMaxScaler: MinMax normalizer 7 | (2) performance: performance evaluator 8 | (3) binary_cross_entropy_loss: loss for RNN on classification 9 | (4) mse_loss: loss for RNN on regression 10 | (5) rnn_sequential: Architecture 11 | """ 12 | 13 | # Necessary packages 14 | import numpy as np 15 | from sklearn.metrics import mean_squared_error, mean_absolute_error 16 | import tensorflow as tf 17 | from tensorflow.keras import layers 18 | 19 | 20 | def MinMaxScaler(data): 21 | """Normalizer (MinMax criteria). 22 | 23 | Args: 24 | - data: original data 25 | 26 | Returns: 27 | - norm_data: normalized data 28 | """ 29 | numerator = data - np.min(data, 0) 30 | denominator = np.max(data, 0) - np.min(data, 0) 31 | norm_data = numerator / (denominator + 1e-8) 32 | return norm_data 33 | 34 | 35 | def performance(test_y, test_y_hat, metric_name): 36 | """Evaluate predictive model performance. 37 | 38 | Args: 39 | - test_y: original testing labels 40 | - test_y_hat: prediction on testing data 41 | - metric_name: 'mse' or 'mae' 42 | 43 | Returns: 44 | - score: performance of the predictive model 45 | """ 46 | assert metric_name in ['mse', 'mae'] 47 | 48 | if metric_name == 'mse': 49 | score = mean_squared_error(test_y, test_y_hat) 50 | elif metric_name == 'mae': 51 | score = mean_absolute_error(test_y, test_y_hat) 52 | 53 | score = np.round(score, 4) 54 | 55 | return score 56 | 57 | 58 | def binary_cross_entropy_loss (y_true, y_pred): 59 | """User defined cross entropy loss. 60 | 61 | Args: 62 | - y_true: true labels 63 | - y_pred: predictions 64 | 65 | Returns: 66 | - loss: computed loss 67 | """ 68 | # Exclude masked labels 69 | idx = tf.cast((y_true >= 0), float) 70 | # Cross entropy loss excluding masked labels 71 | loss = -(idx * y_true * tf.math.log(y_pred) + \ 72 | idx * (1-y_true) * tf.math.log(1-y_pred)) 73 | return loss 74 | 75 | 76 | def mse_loss (y_true, y_pred): 77 | """User defined mean squared loss. 78 | 79 | Args: 80 | - y_true: true labels 81 | - y_pred: predictions 82 | 83 | Returns: 84 | - loss: computed loss 85 | """ 86 | # Exclude masked labels 87 | idx = tf.cast((y_true >= 0), float) 88 | # Mean squared loss excluding masked labels 89 | loss = idx * tf.pow(y_true - y_pred, 2) 90 | return loss 91 | 92 | 93 | def rnn_sequential (model, model_name, h_dim, return_seq): 94 | """Add one rnn layer in sequential model. 95 | 96 | Args: 97 | - model: sequential rnn model 98 | - model_name: rnn, lstm, or gru 99 | - h_dim: hidden state dimensions 100 | - return_seq: True or False 101 | 102 | Returns: 103 | - model: sequential rnn model 104 | """ 105 | 106 | if model_name == 'rnn': 107 | model.add(layers.SimpleRNN(h_dim, return_sequences=return_seq)) 108 | elif model_name == 'lstm': 109 | model.add(layers.LSTM(h_dim, return_sequences=return_seq)) 110 | elif model_name == 'gru': 111 | model.add(layers.GRU(h_dim, return_sequences=return_seq)) 112 | 113 | return model --------------------------------------------------------------------------------