├── .gitignore ├── HAR_data_handler.py ├── LICENSE ├── Opp_data_handler.py ├── README.md ├── UCR_looping_runner.py ├── UCR_time_series_data_handler.py ├── acknowledgements.md ├── base_config.py ├── deep_lstm_model.py ├── deep_lstm_model_MNIST_dataset.py ├── deep_lstm_model_UCR_dataset.py ├── high_har_bn.png ├── high_two_bn.png ├── highway_carry_lstm_model.py ├── highway_lstm_model.py ├── highway_lstm_model_UCR_dataset.py ├── highway_lstm_model_mnist_dataset.py ├── highway_lstm_model_plotlib.py ├── highway_tranform_lstm_model.py ├── highway_tranform_lstm_model_MNIST_dataset.py ├── highway_tranform_lstm_model_UCR_dataset.py ├── lstm_mnist.py ├── main_runner.py ├── main_runner_MNIST_results.py ├── main_runner_all_models_results.py ├── main_runner_all_models_results_UCR.py ├── main_runner_with_plot.py ├── mnist_data_handler.py ├── res_har_bn.png ├── res_two_bn.png ├── residual_lstm_model.py ├── residual_lstm_model_MNIST_dataset.py ├── residual_lstm_model_UCR_dataset.py ├── single_har_bn.png ├── single_layer_lstm.py ├── single_layer_lstm_MNIST.py ├── single_layer_lstm_UCR_dataset.py ├── single_layer_lstm_looper_UCR.py ├── single_two.png ├── stacked_HAR.png ├── stacked_two.png └── utils ├── Sine_Wave_Dataset ├── SlidingWindowGenerator.py ├── sinwave.csv └── time_series_create.py ├── subdirectory_reveal.py ├── tf_upgrade.py └── util.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *,cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # Jupyter Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # SageMath parsed files 80 | *.sage.py 81 | 82 | # dotenv 83 | .env 84 | 85 | # virtualenv 86 | .venv 87 | venv/ 88 | ENV/ 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | .spyproject 93 | 94 | # Rope project settings 95 | .ropeproject 96 | 97 | # mkdocs documentation 98 | /site 99 | 100 | #pycharm stuff 101 | .idea/ -------------------------------------------------------------------------------- /HAR_data_handler.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | INPUT_SIGNAL_TYPES = [ 5 | "body_acc_x_", 6 | "body_acc_y_", 7 | "body_acc_z_", 8 | "body_gyro_x_", 9 | "body_gyro_y_", 10 | "body_gyro_z_", 11 | "total_acc_x_", 12 | "total_acc_y_", 13 | "total_acc_z_" 14 | ] 15 | 16 | # Output classes to learn how to classify 17 | LABELS = [ 18 | "WALKING", 19 | "WALKING_UPSTAIRS", 20 | "WALKING_DOWNSTAIRS", 21 | "SITTING", 22 | "STANDING", 23 | "LAYING" 24 | ] 25 | 26 | DATA_PATH = "data/" 27 | DATASET_PATH = DATA_PATH + "UCI HAR Dataset/" 28 | 29 | TRAIN = "train/" 30 | TEST = "test/" 31 | 32 | def load_X(X_signals_paths): 33 | """ 34 | Given attribute (train or test) of feature, read all 9 features into an 35 | np ndarray of shape [sample_sequence_idx, time_step, feature_num] 36 | argument: X_signals_paths str attribute of feature: 'train' or 'test' 37 | return: np ndarray, tensor of features 38 | """ 39 | X_signals = [] 40 | 41 | for signal_type_path in X_signals_paths: 42 | file = open(signal_type_path, 'rb') 43 | # Read dataset from disk, dealing with text files' syntax 44 | X_signals.append( 45 | [np.array(serie, dtype=np.float32) for serie in [ 46 | row.replace(' ', ' ').strip().split(' ') for row in file 47 | ]] 48 | ) 49 | file.close() 50 | 51 | return np.transpose(np.array(X_signals), (1, 2, 0)) 52 | 53 | X_train_signals_paths = [ 54 | DATASET_PATH + TRAIN + "Inertial Signals/" + signal + "train.txt" for signal in INPUT_SIGNAL_TYPES 55 | ] 56 | X_test_signals_paths = [ 57 | DATASET_PATH + TEST + "Inertial Signals/" + signal + "test.txt" for signal in INPUT_SIGNAL_TYPES 58 | ] 59 | 60 | X_train = load_X(X_train_signals_paths) 61 | X_test = load_X(X_test_signals_paths) 62 | 63 | 64 | def one_hot(y): 65 | """convert label from dense to one hot 66 | argument: 67 | label: ndarray dense label ,shape: [sample_num,1] 68 | return: 69 | one_hot_label: ndarray one hot, shape: [sample_num,n_class] 70 | """ 71 | # e.g.: [[5], [0], [3]] --> [[0, 0, 0, 0, 0, 1], [1, 0, 0, 0, 0, 0], [0, 0, 0, 1, 0, 0]] 72 | 73 | y = y.reshape(len(y)) 74 | n_values = np.max(y) + 1 75 | return np.eye(n_values)[np.array(y, dtype=np.int32)] # Returns FLOATS 76 | 77 | # Load "y" (the neural network's training and testing outputs) 78 | 79 | def load_y(y_path): 80 | """ 81 | Read Y file of values to be predicted 82 | argument: y_path str attibute of Y: 'train' or 'test' 83 | return: Y ndarray / tensor of the 6 one_hot labels of each sample 84 | """ 85 | file = open(y_path, 'rb') 86 | # Read dataset from disk, dealing with text file's syntax 87 | y_ = np.array( 88 | [elem for elem in [ 89 | row.replace(' ', ' ').strip().split(' ') for row in file 90 | ]], 91 | dtype=np.int32 92 | ) 93 | file.close() 94 | 95 | # Substract 1 to each output class for friendly 0-based indexing 96 | return one_hot(y_ - 1) 97 | 98 | y_train_path = DATASET_PATH + TRAIN + "y_train.txt" 99 | y_test_path = DATASET_PATH + TEST + "y_test.txt" 100 | 101 | y_train = load_y(y_train_path) 102 | y_test = load_y(y_test_path) 103 | 104 | 105 | def get_HAR_data(): 106 | return (X_train, y_train, X_test, y_test) -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2017 Praveen Dareddy 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Opp_data_handler.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import cPickle as cp 3 | import time 4 | 5 | # From: https://github.com/sussexwearlab/DeepConvLSTM 6 | # Which is from http://www.johnvinyard.com/blog/?p=268 7 | 8 | import numpy as np 9 | from numpy.lib.stride_tricks import as_strided as ast 10 | 11 | def one_hot(y): 12 | """convert label from dense to one hot 13 | argument: 14 | label: ndarray dense label ,shape: [sample_num,1] 15 | return: 16 | one_hot_label: ndarray one hot, shape: [sample_num,n_class] 17 | """ 18 | # e.g.: [[5], [0], [3]] --> [[0, 0, 0, 0, 0, 1], [1, 0, 0, 0, 0, 0], [0, 0, 0, 1, 0, 0]] 19 | 20 | y = y.reshape(len(y)) 21 | n_values = np.max(y) + 1 22 | return np.eye(n_values)[np.array(y, dtype=np.int32)] # Returns FLOATS 23 | 24 | 25 | def norm_shape(shape): 26 | ''' 27 | Normalize numpy array shapes so they're always expressed as a tuple, 28 | even for one-dimensional shapes. 29 | 30 | Parameters 31 | shape - an int, or a tuple of ints 32 | 33 | Returns 34 | a shape tuple 35 | ''' 36 | try: 37 | i = int(shape) 38 | return (i,) 39 | except TypeError: 40 | # shape was not a number 41 | pass 42 | 43 | try: 44 | t = tuple(shape) 45 | return t 46 | except TypeError: 47 | # shape was not iterable 48 | pass 49 | 50 | raise TypeError('shape must be an int, or a tuple of ints') 51 | 52 | def sliding_window(a,ws,ss = None,flatten = True): 53 | ''' 54 | Return a sliding window over a in any number of dimensions 55 | 56 | Parameters: 57 | a - an n-dimensional numpy array 58 | ws - an int (a is 1D) or tuple (a is 2D or greater) representing the size 59 | of each dimension of the window 60 | ss - an int (a is 1D) or tuple (a is 2D or greater) representing the 61 | amount to slide the window in each dimension. If not specified, it 62 | defaults to ws. 63 | flatten - if True, all slices are flattened, otherwise, there is an 64 | extra dimension for each dimension of the input. 65 | 66 | Returns 67 | an array containing each n-dimensional window from a 68 | ''' 69 | 70 | if None is ss: 71 | # ss was not provided. the windows will not overlap in any direction. 72 | ss = ws 73 | ws = norm_shape(ws) 74 | ss = norm_shape(ss) 75 | 76 | # convert ws, ss, and a.shape to numpy arrays so that we can do math in every 77 | # dimension at once. 78 | ws = np.array(ws) 79 | ss = np.array(ss) 80 | shape = np.array(a.shape) 81 | 82 | 83 | # ensure that ws, ss, and a.shape all have the same number of dimensions 84 | ls = [len(shape),len(ws),len(ss)] 85 | if 1 != len(set(ls)): 86 | raise ValueError(\ 87 | 'a.shape, ws and ss must all have the same length. They were %s' % str(ls)) 88 | 89 | # ensure that ws is smaller than a in every dimension 90 | if np.any(ws > shape): 91 | raise ValueError(\ 92 | 'ws cannot be larger than a in any dimension.\ 93 | a.shape was %s and ws was %s' % (str(a.shape),str(ws))) 94 | 95 | # how many slices will there be in each dimension? 96 | newshape = norm_shape(((shape - ws) // ss) + 1) 97 | # the shape of the strided array will be the number of slices in each dimension 98 | # plus the shape of the window (tuple addition) 99 | newshape += norm_shape(ws) 100 | # the strides tuple will be the array's strides multiplied by step size, plus 101 | # the array's strides (tuple addition) 102 | newstrides = norm_shape(np.array(a.strides) * ss) + a.strides 103 | strided = ast(a,shape = newshape,strides = newstrides) 104 | if not flatten: 105 | return strided 106 | 107 | # Collapse strided so that it has one more dimension than the window. I.e., 108 | # the new array is a flat list of slices. 109 | meat = len(ws) if ws.shape else 0 110 | firstdim = (np.product(newshape[:-meat]),) if ws.shape else () 111 | dim = firstdim + (newshape[-meat:]) 112 | # remove any dimensions with size 1 113 | dim = filter(lambda i : i != 1,dim) 114 | return strided.reshape(dim) 115 | 116 | 117 | #-------------------------------------------- 118 | # Neural net's config. 119 | #-------------------------------------------- 120 | 121 | class Config(object): 122 | """ 123 | define a class to store parameters, 124 | the input should be feature mat of training and testing 125 | """ 126 | 127 | def __init__(self, X_train, X_test): 128 | # Data shaping 129 | self.train_count = len(X_train) # nb of training series 130 | self.test_data_count = len(X_test) # nb of testing series 131 | self.n_steps = len(X_train[0]) # nb of time_steps per series 132 | self.n_classes = 18 # Final output classes, one classification per series 133 | 134 | # Training 135 | self.learning_rate = 0.001 136 | self.lambda_loss_amount = 0.005 137 | self.training_epochs = 100 138 | self.batch_size = 100 139 | self.clip_gradients = 15.0 140 | self.gradient_noise_scale = None 141 | self.keep_prob_for_dropout = 0.85 # **(1/3.0) # Dropout is added on inputs and after each stacked layers (but not between residual layers). 142 | 143 | # Linear+relu structure 144 | self.bias_mean = 0.3 145 | self.weights_stddev = 0.2 # I would recommend between 0.1 and 1.0 or to change and use a xavier initializer 146 | 147 | ######## 148 | # NOTE: I think that if any of the below parameters are changed, 149 | # the best is to readjust every parameters in the "Training" section 150 | # above to properly compare the architectures only once optimised. 151 | ######## 152 | 153 | # LSTM structure 154 | self.n_inputs = len(X_train[0][0]) # Features count 155 | self.n_hidden = 28 # nb of neurons inside the neural network 156 | self.use_bidirectionnal_cells = True # Use bidir in every LSTM cell, or not: 157 | 158 | # High-level deep architecture 159 | self.also_add_dropout_between_stacked_cells = False # True 160 | # NOTE: values of exactly 1 (int) for those 2 high-level parameters below totally disables them and result in only 1 starting LSTM. 161 | # self.n_layers_in_highway = 1 # Number of residual connections to the LSTMs (highway-style), this is did for each stacked block (inside them). 162 | # self.n_stacked_layers = 1 # Stack multiple blocks of residual layers. 163 | 164 | 165 | #-------------------------------------------- 166 | # Dataset-specific constants and functions 167 | #-------------------------------------------- 168 | 169 | # Hardcoded number of sensor channels employed in the OPPORTUNITY challenge 170 | NB_SENSOR_CHANNELS = 113 171 | NB_SENSOR_CHANNELS_WITH_FILTERING = 149 172 | 173 | # Hardcoded number of classes in the gesture recognition problem 174 | NUM_CLASSES = 18 175 | 176 | # Hardcoded length of the sliding window mechanism employed to segment the data 177 | SLIDING_WINDOW_LENGTH = 24 178 | 179 | # Length of the input sequence after convolutional operations 180 | FINAL_SEQUENCE_LENGTH = 8 181 | 182 | # Hardcoded step of the sliding window mechanism employed to segment the data 183 | SLIDING_WINDOW_STEP = int(SLIDING_WINDOW_LENGTH/2) 184 | SLIDING_WINDOW_STEP_SHORT = SLIDING_WINDOW_STEP 185 | 186 | # Batch Size 187 | BATCH_SIZE = 100 188 | 189 | # Number filters convolutional layers 190 | NUM_FILTERS = 64 191 | 192 | # Size filters convolutional layers 193 | FILTER_SIZE = 5 194 | 195 | # Number of unit in the long short-term recurrent layers 196 | NUM_UNITS_LSTM = 128 197 | 198 | 199 | def load_dataset(filename): 200 | 201 | f = file(filename, 'rb') 202 | data = cp.load(f) 203 | f.close() 204 | 205 | X_train, y_train = data[0] 206 | X_test, y_test = data[1] 207 | 208 | print(" ..from file {}".format(filename)) 209 | print(" ..reading instances: train {0}, test {1}".format(X_train.shape, X_test.shape)) 210 | 211 | X_train = X_train.astype(np.float32) 212 | X_test = X_test.astype(np.float32) 213 | 214 | # The targets are casted to int8 for GPU compatibility. 215 | y_train = y_train.astype(np.uint8) 216 | y_test = y_test.astype(np.uint8) 217 | 218 | return X_train, y_train, X_test, y_test 219 | 220 | print("Loading data...") 221 | X_train, y_train, X_test, y_test = load_dataset('data/oppChallenge_gestures.data') 222 | 223 | assert (NB_SENSOR_CHANNELS_WITH_FILTERING == X_train.shape[1] or NB_SENSOR_CHANNELS == X_train.shape[1]) 224 | 225 | def opp_sliding_window(data_x, data_y, ws, ss): 226 | data_x = sliding_window(data_x,(ws,data_x.shape[1]),(ss,1)) 227 | data_y = np.asarray([[i[-1]] for i in sliding_window(data_y,ws,ss)]) 228 | data_x, data_y = data_x.astype(np.float32), one_hot(data_y.reshape(len(data_y)).astype(np.uint8)) 229 | print(" ..after sliding window (testing): inputs {0}, targets {1}".format(X_test.shape, y_test.shape)) 230 | return data_x, data_y 231 | 232 | 233 | #-------------------------------------------- 234 | # Loading dataset 235 | #-------------------------------------------- 236 | 237 | 238 | # Sensor data is segmented using a sliding window mechanism 239 | X_test, y_test = opp_sliding_window(X_test, y_test, SLIDING_WINDOW_LENGTH, SLIDING_WINDOW_STEP_SHORT) 240 | X_train, y_train = opp_sliding_window(X_train, y_train, SLIDING_WINDOW_LENGTH, SLIDING_WINDOW_STEP) 241 | series_size = len(X_train[0]) 242 | 243 | for mat in [X_train, y_train, X_test, y_test]: 244 | print mat.shape 245 | 246 | 247 | def get_Opp_data_with_series_size(): 248 | return (X_train, y_train, X_test, y_test, series_size) 249 | 250 | 251 | if __name__ == '__main__': 252 | print(get_Opp_data_with_series_size()) 253 | 254 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Stacked LSTM Architecture Study on Time Series Datasets 2 | Long short-term memory(LSTM) networks are powerful machine learning models which are cur 3 | rently used for wide range of applications like Speech recognition, Music Composition, Human 4 | action Recognition, Time series Prediction etc. This is made possible due to the Universal nature of 5 | LSTM network where given enough neural units, it can properly model and compute any conven 6 | tional computing problem, provided it has proper weight matrix. 7 | 8 | In our study, We have stacked Layers of LSTM networks for better performance. And by taking inspirance from ConvNets for Image Classification, We mplemented Residual Connections and Highway Connections between LSTM layers. Then, We compared our models on various Datasets. 9 | 10 | 11 | # Experiment Setup 12 | We have used tensorflow as Deep Learning Framework and used its GPU processing facility on NVIDIA GeForce 13 | GTX 960M Graphic card for faster parallel batch processing. 14 | 15 | # Python Libraries Required 16 | numpy, tensorflow, matplotlib, sklearn 17 | 18 | 19 | # Training models and Data Handlers 20 | main_runner.py is used for setting all hyper-parameters and running in a loop for figuring out best parameters. 21 | 22 | classes with data_handler suffix are responsible for data input and can be configured for different datasets without altering model definition. 23 | 24 | Also, Each model can also run by itself to test each package (BY default, HAR dataset is used) 25 | 26 | # Model Configuraion 27 | class DeepLSTMConfig(Config): 28 | def __init__(self): 29 | super(DeepLSTMConfig, self).__init__() 30 | self.train_count = len(X_train) # 7352 training series 31 | self.test_data_count = len(X_test) # 2947 testing series 32 | self.n_steps = len(X_train[0]) # 128 time_steps per series 33 | 34 | # Training 35 | self.learning_rate = 0.005 36 | self.lambda_loss_amount = 0.0015 37 | self.training_epochs = 300 38 | self.batch_size = 1500 39 | 40 | # LSTM structure 41 | self.n_inputs = len(X_train[0][0]) # == 9 Features count is of 9: three 3D sensors features over time 42 | self.n_hidden = 32 # nb of neurons inside the neural network 43 | self.n_classes = 6 # Final output classes 44 | self.W = { 45 | 'hidden': tf.Variable(tf.random_normal([self.n_inputs, self.n_hidden])), 46 | 'output': tf.Variable(tf.random_normal([self.n_hidden, self.n_classes])) 47 | } 48 | self.biases = { 49 | 'hidden': tf.Variable(tf.random_normal([self.n_hidden], mean=1.0)), 50 | 'output': tf.Variable(tf.random_normal([self.n_classes])) 51 | } 52 | self.keep_prob_for_dropout = 0.85 53 | self.bias_mean = 0.3 54 | self.weights_stddev = 0.2 55 | self.n_layers_in_highway = 0 56 | self.n_stacked_layers = 3 57 | self.batch_norm_enabled = True 58 | self.also_add_dropout_between_stacked_cells = False 59 | 60 | 61 | # Main Runner 62 | 63 | 64 | if __name__ == '__main__': 65 | run_with_config = single_layer_lstm.run_with_config 66 | config = single_layer_lstm.config 67 | 68 | 69 | for learning_rate in [0.005, 0.0025, 0.003, 0.0005]: #1, 0.0025, 0.002]: # [0.01, 0.007, 0.001, 0.0007, 0.0001]: 70 | for decay in [0.9]: #[0.005, 0.01]: 71 | for bn_enabled in [True, False]: 72 | for n_stacked in [1]: #2 3 6 73 | for epoch_count in [200, 300, 450]: 74 | config.training_epochs = epoch_count 75 | config.tensor_board_logging_enabled = False #should be always False, log summary folder gets impacted by mulitple runs 76 | config.n_stacked_layers = n_stacked 77 | config.batch_norm_enabled = bn_enabled 78 | config.learning_rate = learning_rate 79 | config.decay = decay 80 | run_with_config(config) #, trX, trY, teX, teY) 81 | 82 | 83 | # Plots 84 | 85 | 86 | 87 | ![har result](single_har_bn.png) 88 | ![har result](stacked_HAR.png) 89 | ![har result](high_har_bn.png) 90 | ![har result](res_har_bn.png) 91 | 92 | 93 | ![ucr result](single_twopng) 94 | ![ucr result](stacked_two.png) 95 | ![ucr result](high_two_bn.png) 96 | ![ucr result](res_two_bn.png) 97 | 98 | -------------------------------------------------------------------------------- /UCR_looping_runner.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | import numpy as np 4 | from random import Random 5 | from tensorflow.contrib import rnn 6 | from tensorflow.python.framework import ops 7 | import os 8 | from UCR_time_series_data_handler import get_dataset_with_series_size, direc 9 | import lstm_mnist 10 | 11 | 12 | 13 | import single_layer_lstm_looper_UCR 14 | 15 | def get_immediate_subdirectories(a_dir): 16 | return [name for name in os.listdir(a_dir) 17 | if os.path.isdir(os.path.join(a_dir, name))] 18 | 19 | def get_dataset_list(): 20 | return get_immediate_subdirectories(direc) 21 | 22 | def run_with_dataset(dataset): 23 | pass 24 | 25 | def modify_config_wtih_current_dataset(config): 26 | pass 27 | 28 | 29 | run_with_config = single_layer_lstm_looper_UCR.run_with_config 30 | config = single_layer_lstm_looper_UCR.config 31 | 32 | def looper_runner(UCR_dataset_name): 33 | run_with_config(config) 34 | 35 | 36 | if __name__ == '__main__': 37 | print (get_dataset_list()) 38 | exit(0) 39 | 40 | run_with_config = single_layer_lstm_looper_UCR.run_with_config 41 | config = single_layer_lstm_looper_UCR.config 42 | 43 | for learning_rate in [0.005, 0.0025, 0.003, 0.0005]: # 1, 0.0025, 0.002]: # [0.01, 0.007, 0.001, 0.0007, 0.0001]: 44 | for decay in [0.9]: # [0.005, 0.01]: 45 | for bn_enabled in [True, False]: 46 | for n_stacked in [2, 3, 4, 6]: # 2 3 6 47 | for epoch_count in [200, 300, 350]: 48 | config.training_epochs = epoch_count 49 | config.tensor_board_logging_enabled = False # should be always False, log summary folder gets impacted by mulitple runs 50 | config.n_stacked_layers = n_stacked 51 | config.batch_norm_enabled = bn_enabled 52 | config.learning_rate = learning_rate 53 | config.decay = decay 54 | run_with_config(config) # , trX, trY, teX, teY) -------------------------------------------------------------------------------- /UCR_time_series_data_handler.py: -------------------------------------------------------------------------------- 1 | """ 2 | LSTM for time series classification 3 | 4 | This model takes in time series and class labels. 5 | The LSTM models the time series. A fully-connected layer 6 | generates an output to be classified with Softmax 7 | """ 8 | 9 | import numpy as np 10 | import tensorflow as tf # TF 1.1.0rc1 11 | 12 | tf.logging.set_verbosity(tf.logging.ERROR) 13 | import matplotlib.pyplot as plt 14 | #from tsc_model import Model, sample_batch, load_data 15 | 16 | # Set these directories 17 | direc = '/home/red/tf_ver_1/LSTM_with_attention_on_MNIST_/data/UCR_TS_Archive_2015' 18 | summaries_dir = '/home/red/tf_ver_1/LSTM_tsc/LSTM_TSC/log_tb' 19 | 20 | def load_data_with_series_size(direc,ratio,dataset): 21 | """Input: 22 | direc: location of the UCR archive 23 | ratio: ratio to split training and testset 24 | dataset: name of the dataset in the UCR archive""" 25 | datadir = direc + '/' + dataset + '/' + dataset 26 | data_train = np.loadtxt(datadir+'_TRAIN',delimiter=',') 27 | data_test_val = np.loadtxt(datadir+'_TEST',delimiter=',') 28 | DATA = np.concatenate((data_train,data_test_val),axis=0) 29 | N = DATA.shape[0] 30 | time_series_size = 0 31 | 32 | with open(datadir+'_TRAIN', 'rb') as filep: 33 | line = filep.readline() 34 | #print(line) 35 | line_sp = line.split(",") 36 | time_series_size = len(line_sp)-1 37 | 38 | ratio = (ratio*N).astype(np.int32) 39 | ind = np.random.permutation(N) 40 | X_train = DATA[ind[:ratio[0]],1:] 41 | X_val = DATA[ind[ratio[0]:ratio[1]],1:] 42 | X_test = DATA[ind[ratio[1]:],1:] 43 | # Targets have labels 1-indexed. We subtract one for 0-indexed 44 | y_train = DATA[ind[:ratio[0]],0]-1 45 | y_val = DATA[ind[ratio[0]:ratio[1]],0]-1 46 | y_test = DATA[ind[ratio[1]:],0]-1 47 | return X_train,X_val,X_test,y_train,y_val,y_test,time_series_size 48 | 49 | def load_data(direc,ratio,dataset): 50 | """Input: 51 | direc: location of the UCR archive 52 | ratio: ratio to split training and testset 53 | dataset: name of the dataset in the UCR archive""" 54 | datadir = direc + '/' + dataset + '/' + dataset 55 | data_train = np.loadtxt(datadir+'_TRAIN',delimiter=',') 56 | data_test_val = np.loadtxt(datadir+'_TEST',delimiter=',') 57 | DATA = np.concatenate((data_train, data_test_val),axis=0) 58 | N = DATA.shape[0] 59 | 60 | ratio = (ratio*N).astype(np.int32) 61 | ind = np.random.permutation(N) 62 | X_train = DATA[ind[:ratio[0]],1:] 63 | X_val = DATA[ind[ratio[0]:ratio[1]],1:] 64 | X_test = DATA[ind[ratio[1]:],1:] 65 | # Targets have labels 1-indexed. We subtract one for 0-indexed 66 | y_train = DATA[ind[:ratio[0]],0]-1 67 | y_val = DATA[ind[ratio[0]:ratio[1]],0]-1 68 | y_test = DATA[ind[ratio[1]:],0]-1 69 | return X_train,X_val,X_test,y_train,y_val,y_test 70 | 71 | 72 | 73 | def sample_batch(X_train,y_train,batch_size): 74 | """ Function to sample a batch for training""" 75 | N,data_len = X_train.shape 76 | ind_N = np.random.choice(N,batch_size,replace=False) 77 | X_batch = X_train[ind_N] 78 | y_batch = y_train[ind_N] 79 | return X_batch,y_batch 80 | 81 | 82 | 83 | """Load the data""" 84 | ratio = np.array([0.8,0.8]) #Ratios where to split the training and validation set, # no validation setting 85 | #X_train,X_val,X_test,y_train,y_val,y_test = load_data(direc,ratio,dataset='CBF') 86 | #X_train,X_val,X_test,y_train,y_val,y_test = load_data(direc,ratio,dataset='synthetic_control') 87 | 88 | 89 | 90 | def get_CBF_data(dataset): 91 | X_train, X_val, X_test, y_train, y_val, y_test, series_size = load_data_with_series_size(direc, ratio, dataset=dataset) 92 | return X_train, y_train, X_test, y_test 93 | 94 | def get_dataset_with_series_size(dataset): 95 | X_train, X_val, X_test, y_train, y_val, y_test, series_size = load_data_with_series_size(direc, ratio, dataset=dataset) 96 | return X_train, y_train, X_test, y_test, series_size 97 | 98 | if __name__ == '__main__': 99 | (X_train, y_train, X_test, y_test, series_size) = get_dataset_with_series_size(dataset='synthetic_control') 100 | print(X_train) -------------------------------------------------------------------------------- /acknowledgements.md: -------------------------------------------------------------------------------- 1 | Parts of code was taken from following public repos: 2 | 3 | https://github.com/guillaume-chevalier 4 | 5 | https://github.com/RobRomijnders 6 | 7 | https://github.com/sussexwearlab/DeepConvLSTM 8 | -------------------------------------------------------------------------------- /base_config.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | from random import Random 4 | from tensorflow.contrib import rnn 5 | from tensorflow.python.framework import ops 6 | import matplotlib 7 | from matplotlib import pyplot 8 | import math 9 | 10 | def sigmoid(x): 11 | return 1 / (1 + math.exp(-x)) 12 | 13 | 14 | 15 | 16 | 17 | FLAGS = tf.app.flags.FLAGS 18 | 19 | class Config(object): 20 | def __init__(self): 21 | global FLAGS 22 | self.FLAGS = FLAGS 23 | self.tf_data_type = {} 24 | self.tf_data_type["double"] = tf.float64 25 | self.tf_data_type["float"] = tf.float32 26 | self.np_data_type = {} 27 | self.np_data_type["double"] = np.float64 28 | self.np_data_type["float"] = np.float32 29 | self.learning_rate = 0.001 30 | self.decay = 0.9 31 | self.batch_norm_enabled = False 32 | self.n_stacked_layers = 1 33 | self.training_epochs = 300 34 | self.batch_size = 1500 35 | self.tensor_board_logging_enabled = False 36 | self.model_name = "base_config" 37 | self.log_folder_suffix = "base_config" 38 | 39 | self.train_count = 0 40 | self.test_data_count = 0 41 | self.n_steps = 0 42 | 43 | 44 | # LSTM structure 45 | self.n_inputs = 0 # == 9 Features count is of 9: three 3D sensors features over time 46 | self.n_hidden = 0 # nb of neurons inside the neural network 47 | self.n_classes = 0 # Final output classes 48 | 49 | self.matplot_lib_enabled = False 50 | self.matplot_lib_for_accuracy = True 51 | self.matplot_lib_for_single_ybundle = True 52 | 53 | self.model_desc_attched_string = "base_config" 54 | 55 | def print_config(self): 56 | print("#####") 57 | print("learning_rate" +" : "+ str(self.learning_rate)) 58 | print("decay" +" : "+ str(self.decay)) 59 | print("batch_norm" + " : " + str(self.batch_norm_enabled)) 60 | print("n_stacked_layers" + " : " + str(self.n_stacked_layers)) 61 | print("training_epochs" + " : " + str(self.training_epochs)) 62 | print("batch_size" + " : " + str(self.batch_size)) 63 | print("model_name" + " : " + str(self.model_name)) 64 | 65 | print("train dataset size" + " : " + str(self.train_count)) 66 | print("test dataset size" + " : " + str(self.test_data_count)) 67 | print("time-series size" + " : " + str(self.n_steps)) 68 | 69 | print("lstm neuron count" + " : " + str(self.n_hidden)) 70 | print("output class count" + " : " + str(self.n_classes)) 71 | 72 | def attach_log_suffix(self): 73 | log_suffix = "" 74 | log_suffix = log_suffix + "model:" + str(self.model_name) 75 | log_suffix = log_suffix + "/" + "learn:" + str(self.learning_rate) 76 | log_suffix = log_suffix + "/" +"stacked_layer:" + str(self.n_stacked_layers) 77 | log_suffix = log_suffix + "/" + "epochs:" + str(self.training_epochs) 78 | log_suffix = log_suffix + "/" + "batch_norm:" + str(self.batch_norm_enabled) 79 | log_suffix = log_suffix + "/" + "lstm_neurons:" + str(self.n_hidden) 80 | 81 | return log_suffix 82 | 83 | def attach_mdoel_desc(self): 84 | log_suffix = "" 85 | log_suffix = log_suffix + "model:" + str(self.model_name) 86 | log_suffix = log_suffix + "|" + "learn:" + str(self.learning_rate) 87 | log_suffix = log_suffix + "|" +"stacked_layer:" + str(self.n_stacked_layers) 88 | log_suffix = log_suffix + "|" + "epochs:" + str(self.training_epochs) 89 | log_suffix = log_suffix + "|" + "batch_norm:" + str(self.batch_norm_enabled) 90 | log_suffix = log_suffix + "|" + "lstm_neurons:" + str(self.n_hidden) 91 | 92 | 93 | return log_suffix 94 | #self.random = Random(FLAGS.python_seed) 95 | 96 | 97 | 98 | 99 | 100 | class YaxisBundle: 101 | def __init__(self, y_index_array_input, y_graph_label, y_graph_colour): 102 | self.y_index_array_input = y_index_array_input 103 | self.y_graph_colour = y_graph_colour 104 | self.y_graph_label = y_graph_label 105 | 106 | 107 | 108 | class PlotUtil: 109 | def __init__(self, plot_title, x_index_array_input , x_label, y_label, width = 6, height = 6): 110 | self.x_index_array_input = x_index_array_input 111 | self.plot_title = plot_title 112 | self.x_label = x_label 113 | self.y_label = y_label 114 | self.width = width 115 | self.height = height 116 | def show_plot(self, YaxisBundle_array, is_x_index_variable = True): 117 | pyplot.figure(figsize=(self.width, self.height)) 118 | assert isinstance(self.x_index_array_input, np.ndarray), "X-axis index must be a numpy ndarray" 119 | x_axis_length = len(self.x_index_array_input) 120 | 121 | 122 | for y in YaxisBundle_array: 123 | assert isinstance(y.y_index_array_input, np.ndarray), "Y-axis array must be a numpy ndarray" 124 | if is_x_index_variable: #TODO refine this logic 125 | self.x_index_array_input = None 126 | _ = [] 127 | for i in range(len(y.y_index_array_input)): 128 | _.append(i) 129 | self.x_index_array_input = np.array(_) 130 | 131 | assert (len(y.y_index_array_input) == len(self.x_index_array_input)), "Both axes indexes must be of same length" 132 | 133 | pyplot.plot(self.x_index_array_input, y.y_index_array_input, y.y_graph_colour, label=y.y_graph_label) 134 | 135 | pyplot.title(self.plot_title) 136 | pyplot.legend(loc='upper right', shadow=True) 137 | pyplot.ylabel(self.y_label) 138 | pyplot.xlabel(self.x_label) 139 | pyplot.show() 140 | 141 | 142 | if __name__ == '__main__': 143 | 144 | test_losses = [] 145 | test_accuracies = [] 146 | indep_test_axis = [] 147 | batch_size = 300 148 | 149 | for i in range(batch_size): 150 | indep_test_axis.append(i) 151 | test_losses.append(3.5 - 1.6 * sigmoid( i/10)) 152 | test_accuracies.append(0.5 + 0.4 * sigmoid(i/10)) 153 | 154 | indep_test_axis = np.array(indep_test_axis) 155 | test_losses = np.array(test_losses) 156 | test_accuracies = np.array(test_accuracies) 157 | 158 | 159 | p = PlotUtil("title", indep_test_axis, "x_label", "y_label") 160 | y_bundle =[] 161 | 162 | y = YaxisBundle(test_losses,"loss", "b") 163 | y_bundle.append(y) 164 | 165 | y = YaxisBundle(test_accuracies,"accuracy", "g") 166 | y_bundle.append(y) 167 | 168 | p.show_plot(y_bundle) 169 | 170 | 171 | 172 | -------------------------------------------------------------------------------- /deep_lstm_model.py: -------------------------------------------------------------------------------- 1 | from HAR_data_handler import get_HAR_data 2 | import tensorflow as tf 3 | from sklearn import metrics 4 | from sklearn.utils import shuffle 5 | import numpy as np 6 | from base_config import Config, YaxisBundle, PlotUtil 7 | 8 | def one_hot(y): 9 | """convert label from dense to one hot 10 | argument: 11 | label: ndarray dense label ,shape: [sample_num,1] 12 | return: 13 | one_hot_label: ndarray one hot, shape: [sample_num,n_class] 14 | """ 15 | # e.g.: [[5], [0], [3]] --> [[0, 0, 0, 0, 0, 1], [1, 0, 0, 0, 0, 0], [0, 0, 0, 1, 0, 0]] 16 | 17 | y = y.reshape(len(y)) 18 | n_values = np.max(y) + 1 19 | return np.eye(n_values)[np.array(y, dtype=np.int32)] # Returns FLOATS 20 | 21 | def apply_batch_norm(input_tensor, config, i): 22 | 23 | with tf.variable_scope("batch_norm") as scope: 24 | if i != 0 : 25 | # Do not create extra variables for each time step 26 | scope.reuse_variables() 27 | 28 | # Mean and variance normalisation simply crunched over all axes 29 | axes = list(range(len(input_tensor.get_shape()))) 30 | 31 | mean, variance = tf.nn.moments(input_tensor, axes=axes, shift=None, name=None, keep_dims=False) 32 | stdev = tf.sqrt(variance + 0.001) 33 | 34 | # Rescaling 35 | bn = input_tensor - mean 36 | bn /= stdev 37 | # Learnable extra rescaling 38 | 39 | # tf.get_variable("relu_fc_weights", initializer=tf.random_normal(mean=0.0, stddev=0.0) 40 | bn *= tf.get_variable("a_noreg", initializer=tf.random_normal([1], mean=0.5, stddev=0.0)) 41 | bn += tf.get_variable("b_noreg", initializer=tf.random_normal([1], mean=0.0, stddev=0.0)) 42 | # bn *= tf.Variable(0.5, name=(scope.name + "/a_noreg")) 43 | # bn += tf.Variable(0.0, name=(scope.name + "/b_noreg")) 44 | 45 | return bn 46 | 47 | 48 | def relu_fc(input_2D_tensor_list, features_len, new_features_len, config): 49 | """make a relu fully-connected layer, mainly change the shape of tensor 50 | both input and output is a list of tensor 51 | argument: 52 | input_2D_tensor_list: list shape is [batch_size,feature_num] 53 | features_len: int the initial features length of input_2D_tensor 54 | new_feature_len: int the final features length of output_2D_tensor 55 | config: Config used for weights initializers 56 | return: 57 | output_2D_tensor_list lit shape is [batch_size,new_feature_len] 58 | """ 59 | 60 | W = tf.get_variable( 61 | "relu_fc_weights", 62 | initializer=tf.random_normal( 63 | [features_len, new_features_len], 64 | mean=0.0, 65 | stddev=float(config.weights_stddev) 66 | ) 67 | ) 68 | b = tf.get_variable( 69 | "relu_fc_biases_noreg", 70 | initializer=tf.random_normal( 71 | [new_features_len], 72 | mean=float(config.bias_mean), 73 | stddev=float(config.weights_stddev) 74 | ) 75 | ) 76 | 77 | # intra-timestep multiplication: 78 | output_2D_tensor_list = [ 79 | tf.nn.relu(tf.matmul(input_2D_tensor, W) + b) 80 | for input_2D_tensor in input_2D_tensor_list 81 | ] 82 | 83 | return output_2D_tensor_list 84 | 85 | def single_LSTM_cell(input_hidden_tensor, n_outputs): 86 | with tf.variable_scope("lstm_cell"): 87 | lstm_cell = tf.contrib.rnn.BasicLSTMCell(n_outputs, state_is_tuple=True, forget_bias=0.999) 88 | #outputs, _ = tf.nn.rnn(lstm_cell, input_hidden_tensor, dtype=tf.float32) 89 | outputs, _ = tf.contrib.rnn.static_rnn(lstm_cell, input_hidden_tensor, dtype=tf.float32) 90 | return outputs 91 | 92 | 93 | def stack_single_LSTM_layer(input_hidden_tensor, n_input, n_output, layer_level, config, keep_prob_for_dropout): 94 | 95 | with tf.variable_scope('layer_{}'.format(layer_level)) as scope: 96 | if config.batch_norm_enabled : 97 | input_hidden_tensor = [apply_batch_norm(out, config, i) for i, out in enumerate(input_hidden_tensor)] 98 | 99 | hidden_LSTM_layer = single_LSTM_cell(input_hidden_tensor, n_output) 100 | #hidden_LSTM_layer = single_LSTM_cell(relu_fc(input_hidden_tensor, n_input, n_output, config), n_output) 101 | 102 | return hidden_LSTM_layer 103 | 104 | def get_deeply_stacked_LSTM_layers(input_hidden_tensor, n_input, n_output, config, keep_prob_for_dropout): 105 | 106 | # creating base lstm Layer 107 | print "\nCreating hidden #1:" 108 | hidden = stack_single_LSTM_layer(input_hidden_tensor, config.n_inputs, config.n_hidden, 1, config, keep_prob_for_dropout) 109 | print (len(hidden), str(hidden[0].get_shape())) 110 | 111 | # Stacking LSTM layer on existing layer in a for loop 112 | for stacked_hidden_index in range(config.n_stacked_layers - 1): 113 | # If the config permits it, we stack more lstm cells: 114 | print "\nCreating hidden #{}:".format(stacked_hidden_index + 2) 115 | hidden = stack_single_LSTM_layer(hidden, config.n_hidden, config.n_hidden, stacked_hidden_index + 2, config, 116 | keep_prob_for_dropout) 117 | print (len(hidden), str(hidden[0].get_shape())) 118 | 119 | print "" 120 | return hidden 121 | 122 | def deep_LSTM_network(feature_mat, config, keep_prob_for_dropout): 123 | with tf.variable_scope('LSTM_network') as scope: # TensorFlow graph naming 124 | 125 | feature_mat = tf.nn.dropout(feature_mat, keep_prob_for_dropout) 126 | feature_mat = tf.transpose(feature_mat, [1, 0, 2]) 127 | feature_mat = tf.reshape(feature_mat, [-1, config.n_inputs]) 128 | print feature_mat.get_shape() 129 | 130 | # Split the series because the rnn cell needs time_steps features, each of shape: 131 | hidden = tf.split(axis=0, num_or_size_splits=config.n_steps, value=feature_mat) 132 | print (len(hidden), str(hidden[0].get_shape())) 133 | # New shape: a list of lenght "time_step" containing tensors of shape [batch_size, n_hidden] 134 | 135 | hidden = get_deeply_stacked_LSTM_layers(hidden, config.n_inputs, config.n_hidden, config, keep_prob_for_dropout) 136 | 137 | # Final fully-connected activation logits 138 | # Get the last output tensor of the inner loop output series, of shape [batch_size, n_classes] 139 | lstm_last_output = hidden[-1] 140 | 141 | # Linear activation 142 | return tf.matmul(lstm_last_output, config.W['output']) + config.biases['output'] 143 | 144 | last_hidden = tf.nn.dropout(hidden[-1], keep_prob_for_dropout) 145 | last_logits = relu_fc( 146 | [last_hidden], 147 | config.n_hidden, config.n_classes, config 148 | )[0] 149 | return last_logits 150 | 151 | 152 | ################################## load data and config ################################## 153 | 154 | X_train, y_train, X_test, y_test = get_HAR_data() 155 | 156 | class DeepLSTMConfig(Config): 157 | def __init__(self): 158 | super(DeepLSTMConfig, self).__init__() 159 | self.train_count = len(X_train) # 7352 training series 160 | self.test_data_count = len(X_test) # 2947 testing series 161 | self.n_steps = len(X_train[0]) # 128 time_steps per series 162 | 163 | # Trainging 164 | self.learning_rate = 0.005 165 | self.lambda_loss_amount = 0.0015 166 | self.training_epochs = 300 167 | self.batch_size = 1500 168 | 169 | # LSTM structure 170 | self.n_inputs = len(X_train[0][0]) # == 9 Features count is of 9: three 3D sensors features over time 171 | self.n_hidden = 32 # nb of neurons inside the neural network 172 | self.n_classes = 6 # Final output classes 173 | self.W = { 174 | 'hidden': tf.Variable(tf.random_normal([self.n_inputs, self.n_hidden])), 175 | 'output': tf.Variable(tf.random_normal([self.n_hidden, self.n_classes])) 176 | } 177 | self.biases = { 178 | 'hidden': tf.Variable(tf.random_normal([self.n_hidden], mean=1.0)), 179 | 'output': tf.Variable(tf.random_normal([self.n_classes])) 180 | } 181 | self.keep_prob_for_dropout = 0.85 182 | self.bias_mean = 0.3 183 | self.weights_stddev = 0.2 184 | self.n_layers_in_highway = 0 185 | self.n_stacked_layers = 3 186 | self.batch_norm_enabled = True 187 | self.also_add_dropout_between_stacked_cells = False 188 | 189 | self.model_name = "deep_lstm" + "_HAR" 190 | self.log_folder_suffix = self.attach_log_suffix() 191 | self.logs_path = "/tmp/LSTM_logs/"+self.log_folder_suffix 192 | 193 | self.tensor_board_logging_enabled = True 194 | self.tensorboard_cmd = "tensorboard --logdir="+ self.logs_path 195 | self.model_desc_attched_string = self.attach_mdoel_desc() 196 | self.matplot_lib_enabled = True 197 | self.matplot_lib_for_accuracy =True 198 | self.matplot_lib_for_single_ybundle=False 199 | 200 | 201 | #config = Config(X_train, X_test) 202 | config = DeepLSTMConfig() 203 | 204 | 205 | def run_with_config(config) : #, X_train, y_train, X_test, y_test): 206 | tf.reset_default_graph() # To enable to run multiple things in a loop 207 | config.print_config() 208 | 209 | if config.matplot_lib_enabled: 210 | # To keep track of training's performance 211 | test_losses = [] 212 | test_accuracies = [] 213 | indep_test_axis = [] 214 | 215 | 216 | 217 | config.W = { 218 | 'hidden': tf.Variable(tf.random_normal([config.n_inputs, config.n_hidden])), 219 | 'output': tf.Variable(tf.random_normal([config.n_hidden, config.n_classes])) 220 | } 221 | config.biases = { 222 | 'hidden': tf.Variable(tf.random_normal([config.n_hidden], mean=1.0)), 223 | 'output': tf.Variable(tf.random_normal([config.n_classes])) 224 | } 225 | #----------------------------------- 226 | # Define parameters for model 227 | #----------------------------------- 228 | 229 | 230 | print("Some useful info to get an insight on dataset's shape and normalisation:") 231 | print("features shape, labels shape, each features mean, each features standard deviation") 232 | print(X_test.shape, y_test.shape, 233 | np.mean(X_test), np.std(X_test)) 234 | print("the dataset is therefore properly normalised, as expected.") 235 | 236 | # ------------------------------------------------------ 237 | # step3: Let's get serious and build the neural network 238 | # ------------------------------------------------------ 239 | X = tf.placeholder(tf.float32, [None, config.n_steps, config.n_inputs]) 240 | Y = tf.placeholder(tf.float32, [None, config.n_classes]) 241 | 242 | pred_Y = deep_LSTM_network(X, config, 0.85) 243 | 244 | 245 | print "Unregularised variables:" 246 | for unreg in [tf_var.name for tf_var in tf.trainable_variables() if 247 | ("noreg" in tf_var.name or "Bias" in tf_var.name)]: 248 | print unreg 249 | 250 | # Loss,optimizer,evaluation 251 | l2 = config.lambda_loss_amount * \ 252 | sum(tf.nn.l2_loss(tf_var) for tf_var in tf.trainable_variables()) 253 | # Softmax loss and L2 254 | cost = tf.reduce_mean( 255 | tf.nn.softmax_cross_entropy_with_logits(logits=pred_Y, labels=Y)) + l2 256 | optimizer = tf.train.AdamOptimizer( 257 | learning_rate=config.learning_rate).minimize(cost) 258 | 259 | correct_pred = tf.equal(tf.argmax(pred_Y, 1), tf.argmax(Y, 1)) 260 | accuracy = tf.reduce_mean(tf.cast(correct_pred, dtype=tf.float32)) 261 | # ------------------------------------------------------ 262 | # step3.5 : Tensorboard stuff here 263 | # ------------------------------------------------------ 264 | if config.tensor_board_logging_enabled: 265 | tf.summary.scalar("loss", cost) 266 | tf.summary.scalar("accuracy", accuracy) 267 | merged_summary_op = tf.summary.merge_all() 268 | 269 | # -------------------------------------------- 270 | # step4: Hooray, now train the neural network 271 | # -------------------------------------------- 272 | # Note that log_device_placement can be turned ON but will cause console spam. 273 | sess = tf.InteractiveSession(config=tf.ConfigProto(log_device_placement=False)) 274 | tf.global_variables_initializer().run() 275 | 276 | if config.tensor_board_logging_enabled: 277 | # op to write logs to Tensorboard 278 | summary_writer = tf.summary.FileWriter(config.logs_path, graph=tf.get_default_graph()) 279 | 280 | best_accuracy = 0.0 281 | # Start training for each batch and loop epochs 282 | for i in range(config.training_epochs): 283 | for start, end in zip(range(0, config.train_count, config.batch_size), 284 | range(config.batch_size, config.train_count + 1, config.batch_size)): 285 | if config.tensor_board_logging_enabled: 286 | _, summary = sess.run([optimizer, merged_summary_op], 287 | feed_dict={X: X_train[start:end], Y: y_train[start:end]}) 288 | else: 289 | sess.run(optimizer, feed_dict={X: X_train[start:end], Y: y_train[start:end]}) 290 | 291 | if config.tensor_board_logging_enabled: 292 | # Write logs at every iteration 293 | summary_writer.add_summary(summary, i) 294 | 295 | # Test completely at every epoch: calculate accuracy 296 | pred_out, accuracy_out, loss_out = sess.run([pred_Y, accuracy, cost], feed_dict={ 297 | X: X_test, Y: y_test}) 298 | 299 | if config.matplot_lib_enabled: 300 | indep_test_axis.append(i) 301 | test_losses.append(loss_out) 302 | test_accuracies.append(accuracy_out) 303 | 304 | print("traing iter: {},".format(i) + \ 305 | " test accuracy : {},".format(accuracy_out) + \ 306 | " loss : {}".format(loss_out)) 307 | best_accuracy = max(best_accuracy, accuracy_out) 308 | 309 | print("") 310 | print("final test accuracy: {}".format(accuracy_out)) 311 | print("best epoch's test accuracy: {}".format(best_accuracy)) 312 | print("") 313 | 314 | if config.tensor_board_logging_enabled: 315 | print("Run the command line:\n") 316 | print(config.tensorboard_cmd) 317 | print("\nThen open http://0.0.0.0:6006/ into your web browser") 318 | 319 | print(config.model_desc_attched_string) 320 | 321 | 322 | if config.matplot_lib_enabled: 323 | 324 | #for i in range(config.batch_size): 325 | # indep_test_axis.append(i) 326 | #indep_test_axis = [i for i in range(config.batch_size)] 327 | #indep_test_axis = np.array(indep_test_axis) 328 | 329 | #p = PlotUtil("title", indep_test_axis, "x_label", "y_label") 330 | y_bundle = [] 331 | test_losses = np.array(test_losses) 332 | test_accuracies = np.array(test_accuracies) 333 | 334 | y = YaxisBundle(np.array(test_losses), "loss", "b") 335 | y_bundle.append(y) 336 | 337 | y = YaxisBundle(np.array(test_accuracies), "accuracy", "g") 338 | y_bundle.append(y) 339 | 340 | #p.show_plot(y_bundle) 341 | 342 | if config.matplot_lib_for_single_ybundle: 343 | if config.matplot_lib_for_accuracy: 344 | return y_bundle[1] 345 | else : 346 | return y_bundle[0] 347 | return y_bundle 348 | 349 | 350 | 351 | if __name__ == '__main__': 352 | if config.matplot_lib_enabled: 353 | indep_test_axis = [] 354 | for i in range(config.training_epochs): 355 | indep_test_axis.append(i) 356 | 357 | p = PlotUtil("Stacked LSTM(3 layers) on HAR", np.array(indep_test_axis), "Epoch iterations", "Loss or Accuracy") 358 | y_bundle = run_with_config(config) 359 | 360 | p.show_plot(y_bundle) 361 | else: 362 | run_with_config(config) -------------------------------------------------------------------------------- /high_har_bn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/praveendareddy21/Stacked_LSTMS_Highway_Residual_On_TimeSeries_Datasets/4c9e9647b4a134a2b7f63b0289687cc92262d5c8/high_har_bn.png -------------------------------------------------------------------------------- /high_two_bn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/praveendareddy21/Stacked_LSTMS_Highway_Residual_On_TimeSeries_Datasets/4c9e9647b4a134a2b7f63b0289687cc92262d5c8/high_two_bn.png -------------------------------------------------------------------------------- /highway_carry_lstm_model.py: -------------------------------------------------------------------------------- 1 | from HAR_data_handler import get_HAR_data 2 | import tensorflow as tf 3 | from sklearn import metrics 4 | from sklearn.utils import shuffle 5 | import numpy as np 6 | from base_config import Config, YaxisBundle, PlotUtil 7 | from mnist_data_handler import get_mnist_data_with_time_series_size 8 | 9 | def one_hot(y): 10 | """convert label from dense to one hot 11 | argument: 12 | label: ndarray dense label ,shape: [sample_num,1] 13 | return: 14 | one_hot_label: ndarray one hot, shape: [sample_num,n_class] 15 | """ 16 | # e.g.: [[5], [0], [3]] --> [[0, 0, 0, 0, 0, 1], [1, 0, 0, 0, 0, 0], [0, 0, 0, 1, 0, 0]] 17 | 18 | y = y.reshape(len(y)) 19 | n_values = np.max(y) + 1 20 | return np.eye(n_values)[np.array(y, dtype=np.int32)] # Returns FLOATS 21 | 22 | def apply_batch_norm(input_tensor, config, i): 23 | 24 | with tf.variable_scope("batch_norm") as scope: 25 | if i != 0 : 26 | # Do not create extra variables for each time step 27 | scope.reuse_variables() 28 | 29 | # Mean and variance normalisation simply crunched over all axes 30 | axes = list(range(len(input_tensor.get_shape()))) 31 | 32 | mean, variance = tf.nn.moments(input_tensor, axes=axes, shift=None, name=None, keep_dims=False) 33 | stdev = tf.sqrt(variance + 0.001) 34 | 35 | # Rescaling 36 | bn = input_tensor - mean 37 | bn /= stdev 38 | # Learnable extra rescaling 39 | 40 | # tf.get_variable("relu_fc_weights", initializer=tf.random_normal(mean=0.0, stddev=0.0) 41 | bn *= tf.get_variable("a_noreg", initializer=tf.random_normal([1], mean=0.5, stddev=0.0)) 42 | bn += tf.get_variable("b_noreg", initializer=tf.random_normal([1], mean=0.0, stddev=0.0)) 43 | # bn *= tf.Variable(0.5, name=(scope.name + "/a_noreg")) 44 | # bn += tf.Variable(0.0, name=(scope.name + "/b_noreg")) 45 | 46 | return bn 47 | 48 | def relu_fc(input_2D_tensor_list, features_len, new_features_len, config): 49 | """make a relu fully-connected layer, mainly change the shape of tensor 50 | both input and output is a list of tensor 51 | argument: 52 | input_2D_tensor_list: list shape is [batch_size,feature_num] 53 | features_len: int the initial features length of input_2D_tensor 54 | new_feature_len: int the final features length of output_2D_tensor 55 | config: Config used for weights initializers 56 | return: 57 | output_2D_tensor_list lit shape is [batch_size,new_feature_len] 58 | """ 59 | 60 | W = tf.get_variable( 61 | "relu_fc_weights", 62 | initializer=tf.random_normal( 63 | [features_len, new_features_len], 64 | mean=0.0, 65 | stddev=float(config.weights_stddev) 66 | ) 67 | ) 68 | b = tf.get_variable( 69 | "relu_fc_biases_noreg", 70 | initializer=tf.random_normal( 71 | [new_features_len], 72 | mean=float(config.bias_mean), 73 | stddev=float(config.weights_stddev) 74 | ) 75 | ) 76 | 77 | # intra-timestep multiplication: 78 | output_2D_tensor_list = [ 79 | tf.nn.relu(tf.matmul(input_2D_tensor, W) + b) 80 | for input_2D_tensor in input_2D_tensor_list 81 | ] 82 | 83 | return output_2D_tensor_list 84 | 85 | def single_LSTM_cell(input_hidden_tensor, n_outputs): 86 | with tf.variable_scope("lstm_cell"): 87 | lstm_cell = tf.contrib.rnn.BasicLSTMCell(n_outputs, state_is_tuple=True, forget_bias=0.999) 88 | #outputs, _ = tf.nn.rnn(lstm_cell, input_hidden_tensor, dtype=tf.float32) 89 | outputs, _ = tf.contrib.rnn.static_rnn(lstm_cell, input_hidden_tensor, dtype=tf.float32) 90 | res_output = [] 91 | for i in xrange(len(outputs)): #relu_fc(input_hidden_tensor, n_input, n_output, config), n_output)[i] 92 | res_output.append(outputs[i] ) 93 | return res_output 94 | 95 | def single_LSTM_Res_cell(input_hidden_tensor, n_outputs, res_unit): 96 | with tf.variable_scope("lstm_cell"): 97 | lstm_cell = tf.contrib.rnn.BasicLSTMCell(n_outputs, state_is_tuple=True, forget_bias=0.999) 98 | #outputs, _ = tf.nn.rnn(lstm_cell, input_hidden_tensor, dtype=tf.float32) 99 | outputs, _ = tf.contrib.rnn.static_rnn(lstm_cell, input_hidden_tensor, dtype=tf.float32) 100 | return [out+res_unit[i] for i, out in enumerate(outputs)] 101 | 102 | def stack_single_LSTM_layer(input_hidden_tensor, n_input, n_output, layer_level, config, keep_prob_for_dropout): 103 | 104 | with tf.variable_scope('layer_{}'.format(layer_level)) as scope: 105 | if config.batch_norm_enabled : 106 | input_hidden_tensor = [apply_batch_norm(out, config, i) for i, out in enumerate(input_hidden_tensor)] 107 | hidden_LSTM_layer = single_LSTM_Res_cell(input_hidden_tensor, n_output, relu_fc(input_hidden_tensor, n_input, n_output, config)) 108 | 109 | return hidden_LSTM_layer 110 | 111 | 112 | def get_deeply_stacked_LSTM_layers(input_hidden_tensor, n_input, n_output, config, keep_prob_for_dropout): 113 | print "\nCreating hidden #1:" 114 | hidden = stack_single_LSTM_layer(input_hidden_tensor, config.n_inputs, config.n_hidden, 1, config, keep_prob_for_dropout) 115 | print (len(hidden), str(hidden[0].get_shape())) 116 | for stacked_hidden_index in range(config.n_stacked_layers - 1): 117 | print "\nCreating hidden #{}:".format(stacked_hidden_index + 2) 118 | hidden = stack_single_LSTM_layer(hidden, config.n_hidden, config.n_hidden, stacked_hidden_index + 2, config, 119 | keep_prob_for_dropout) 120 | print (len(hidden), str(hidden[0].get_shape())) 121 | print "" 122 | return hidden 123 | 124 | 125 | def deep_LSTM_network(feature_mat, config, keep_prob_for_dropout): 126 | with tf.variable_scope('LSTM_network') as scope: # TensorFlow graph naming 127 | 128 | feature_mat = tf.nn.dropout(feature_mat, keep_prob_for_dropout) 129 | feature_mat = tf.transpose(feature_mat, [1, 0, 2]) 130 | feature_mat = tf.reshape(feature_mat, [-1, config.n_inputs]) 131 | print feature_mat.get_shape() 132 | 133 | # Split the series because the rnn cell needs time_steps features, each of shape: 134 | hidden = tf.split(axis=0, num_or_size_splits=config.n_steps, value=feature_mat) 135 | print (len(hidden), str(hidden[0].get_shape())) 136 | # New shape: a list of lenght "time_step" containing tensors of shape [batch_size, n_hidden] 137 | 138 | hidden = get_deeply_stacked_LSTM_layers(hidden, config.n_inputs, config.n_hidden, config, keep_prob_for_dropout) 139 | 140 | # Final fully-connected activation logits 141 | # Get the last output tensor of the inner loop output series, of shape [batch_size, n_classes] 142 | lstm_last_output = hidden[-1] 143 | 144 | # Linear activation 145 | return tf.matmul(lstm_last_output, config.W['output']) + config.biases['output'] 146 | 147 | last_hidden = tf.nn.dropout(hidden[-1], keep_prob_for_dropout) 148 | last_logits = relu_fc( 149 | [last_hidden], 150 | config.n_hidden, config.n_classes, config 151 | )[0] 152 | return last_logits 153 | 154 | 155 | 156 | ################################## load data and config ################################## 157 | X_train, y_train, X_test, y_test, series_size = get_mnist_data_with_time_series_size() 158 | 159 | 160 | 161 | 162 | class ResidualLSTMOnMNISTConfig(Config): 163 | def __init__(self): 164 | super(ResidualLSTMOnMNISTConfig, self).__init__() 165 | self.train_count = len(X_train) # 7352 training series 166 | self.test_data_count = len(X_test) # 2947 testing series 167 | self.n_steps = len(X_train[0]) # 128 time_steps per series 168 | 169 | # Trainging 170 | self.learning_rate = 0.0005 171 | self.lambda_loss_amount = 0.0015 172 | self.training_epochs = 100 173 | self.batch_size = 200 174 | 175 | # LSTM structure 176 | self.n_inputs = len(X_train[0][0]) # == 9 Features count is of 9: three 3D sensors features over time 177 | self.n_hidden = 28 # nb of neurons inside the neural network 178 | self.n_classes = len(y_train[0]) # Final output classes 179 | 180 | 181 | 182 | self.keep_prob_for_dropout = 0.85 183 | self.bias_mean = 0.5 184 | self.weights_stddev = 1.2 185 | self.n_layers_in_highway = 0 186 | self.n_stacked_layers = 4 187 | self.batch_norm_enabled = True 188 | self.also_add_dropout_between_stacked_cells = False 189 | self.tensor_board_logging_enabled = True 190 | self.model_name = "residual_lstm_" + "MNIST" 191 | self.log_folder_suffix = self.attach_log_suffix() 192 | self.logs_path = "/tmp/LSTM_logs/"+self.log_folder_suffix 193 | self.tensorboard_cmd = "tensorboard --logdir="+ self.logs_path 194 | 195 | 196 | #config = Config(X_train, X_test) 197 | config = ResidualLSTMOnMNISTConfig() 198 | 199 | 200 | 201 | 202 | def run_with_config(config) : #, X_train, y_train, X_test, y_test): 203 | tf.reset_default_graph() # To enable to run multiple things in a loop 204 | config.print_config() 205 | 206 | config.W = { 207 | 'hidden': tf.Variable(tf.random_normal([config.n_inputs, config.n_hidden])), 208 | 'output': tf.Variable(tf.random_normal([config.n_hidden, config.n_classes])) 209 | } 210 | config.biases = { 211 | 'hidden': tf.Variable(tf.random_normal([config.n_hidden], mean=10.0)), 212 | 'output': tf.Variable(tf.random_normal([config.n_classes])) 213 | } 214 | #----------------------------------- 215 | # Define parameters for model 216 | #----------------------------------- 217 | 218 | 219 | print("Some useful info to get an insight on dataset's shape and normalisation:") 220 | print("features shape, labels shape, each features mean, each features standard deviation") 221 | print(X_test.shape, y_test.shape, 222 | np.mean(X_test), np.std(X_test)) 223 | print("the dataset is therefore properly normalised, as expected.") 224 | 225 | # ------------------------------------------------------ 226 | # step3: Let's get serious and build the neural network 227 | # ------------------------------------------------------ 228 | X = tf.placeholder(tf.float32, [None, config.n_steps, config.n_inputs]) 229 | Y = tf.placeholder(tf.float32, [None, config.n_classes]) 230 | 231 | # pred_Y = LSTM_Network(X, config) 232 | pred_Y = deep_LSTM_network(X, config, 0.85) 233 | 234 | print "Unregularised variables:" 235 | for unreg in [tf_var.name for tf_var in tf.trainable_variables() if 236 | ("noreg" in tf_var.name or "Bias" in tf_var.name)]: 237 | print unreg 238 | 239 | # Loss,optimizer,evaluation 240 | l2 = config.lambda_loss_amount * \ 241 | sum(tf.nn.l2_loss(tf_var) for tf_var in tf.trainable_variables()) 242 | # Softmax loss and L2 243 | cost = tf.reduce_mean( 244 | tf.nn.softmax_cross_entropy_with_logits(logits=pred_Y, labels=Y)) + l2 245 | optimizer = tf.train.AdamOptimizer( 246 | learning_rate=config.learning_rate).minimize(cost) 247 | 248 | correct_pred = tf.equal(tf.argmax(pred_Y, 1), tf.argmax(Y, 1)) 249 | accuracy = tf.reduce_mean(tf.cast(correct_pred, dtype=tf.float32)) 250 | # ------------------------------------------------------ 251 | # step3.5 : Tensorboard stuff here 252 | # ------------------------------------------------------ 253 | if config.tensor_board_logging_enabled: 254 | tf.summary.scalar("loss", cost) 255 | tf.summary.scalar("accuracy", accuracy) 256 | merged_summary_op = tf.summary.merge_all() 257 | 258 | # -------------------------------------------- 259 | # step4: Hooray, now train the neural network 260 | # -------------------------------------------- 261 | # Note that log_device_placement can be turned ON but will cause console spam. 262 | sess = tf.InteractiveSession(config=tf.ConfigProto(log_device_placement=False)) 263 | tf.global_variables_initializer().run() 264 | 265 | if config.tensor_board_logging_enabled: 266 | # op to write logs to Tensorboard 267 | summary_writer = tf.summary.FileWriter(config.logs_path, graph=tf.get_default_graph()) 268 | 269 | best_accuracy = 0.0 270 | # Start training for each batch and loop epochs 271 | for i in range(config.training_epochs): 272 | for start, end in zip(range(0, config.train_count, config.batch_size), 273 | range(config.batch_size, config.train_count + 1, config.batch_size)): 274 | if config.tensor_board_logging_enabled: 275 | _, summary = sess.run([optimizer, merged_summary_op], 276 | feed_dict={X: X_train[start:end], Y: y_train[start:end]}) 277 | else: 278 | sess.run(optimizer, feed_dict={X: X_train[start:end], Y: y_train[start:end]}) 279 | 280 | if config.tensor_board_logging_enabled: 281 | # Write logs at every iteration 282 | summary_writer.add_summary(summary, i) 283 | 284 | # Test completely at every epoch: calculate accuracy 285 | pred_out, accuracy_out, loss_out = sess.run([pred_Y, accuracy, cost], feed_dict={ 286 | X: X_test, Y: y_test}) 287 | print("traing iter: {},".format(i) + \ 288 | " test accuracy : {},".format(accuracy_out) + \ 289 | " loss : {}".format(loss_out)) 290 | best_accuracy = max(best_accuracy, accuracy_out) 291 | 292 | print("") 293 | print("final test accuracy: {}".format(accuracy_out)) 294 | print("best epoch's test accuracy: {}".format(best_accuracy)) 295 | print("") 296 | 297 | if config.tensor_board_logging_enabled: 298 | print("Run the command line:\n") 299 | print(config.tensorboard_cmd) 300 | print("\nThen open http://0.0.0.0:6006/ into your web browser") 301 | 302 | 303 | if __name__ == '__main__': 304 | run_with_config(config) # , trX, trY, teX, teY) 305 | -------------------------------------------------------------------------------- /highway_lstm_model.py: -------------------------------------------------------------------------------- 1 | from HAR_data_handler import get_HAR_data 2 | import tensorflow as tf 3 | from sklearn import metrics 4 | from sklearn.utils import shuffle 5 | import numpy as np 6 | from base_config import Config 7 | 8 | def one_hot(y): 9 | """convert label from dense to one hot 10 | argument: 11 | label: ndarray dense label ,shape: [sample_num,1] 12 | return: 13 | one_hot_label: ndarray one hot, shape: [sample_num,n_class] 14 | """ 15 | # e.g.: [[5], [0], [3]] --> [[0, 0, 0, 0, 0, 1], [1, 0, 0, 0, 0, 0], [0, 0, 0, 1, 0, 0]] 16 | 17 | y = y.reshape(len(y)) 18 | n_values = np.max(y) + 1 19 | return np.eye(n_values)[np.array(y, dtype=np.int32)] # Returns FLOATS 20 | 21 | def apply_batch_norm(input_tensor, config, i): 22 | 23 | with tf.variable_scope("batch_norm") as scope: 24 | if i != 0 : 25 | # Do not create extra variables for each time step 26 | scope.reuse_variables() 27 | 28 | # Mean and variance normalisation simply crunched over all axes 29 | axes = list(range(len(input_tensor.get_shape()))) 30 | 31 | mean, variance = tf.nn.moments(input_tensor, axes=axes, shift=None, name=None, keep_dims=False) 32 | stdev = tf.sqrt(variance + 0.001) 33 | 34 | # Rescaling 35 | bn = input_tensor - mean 36 | bn /= stdev 37 | # Learnable extra rescaling 38 | 39 | # tf.get_variable("relu_fc_weights", initializer=tf.random_normal(mean=0.0, stddev=0.0) 40 | bn *= tf.get_variable("a_noreg", initializer=tf.random_normal([1], mean=0.5, stddev=0.0)) 41 | bn += tf.get_variable("b_noreg", initializer=tf.random_normal([1], mean=0.0, stddev=0.0)) 42 | # bn *= tf.Variable(0.5, name=(scope.name + "/a_noreg")) 43 | # bn += tf.Variable(0.0, name=(scope.name + "/b_noreg")) 44 | 45 | return bn 46 | 47 | # Load "X" (the neural network's training and testing inputs) 48 | 49 | def load_X(X_signals_paths): 50 | X_signals = [] 51 | 52 | for signal_type_path in X_signals_paths: 53 | file = open(signal_type_path, 'rb') 54 | # Read dataset from disk, dealing with text files' syntax 55 | X_signals.append( 56 | [np.array(serie, dtype=np.float32) for serie in [ 57 | row.replace(' ', ' ').strip().split(' ') for row in file 58 | ]] 59 | ) 60 | file.close() 61 | 62 | return np.transpose(np.array(X_signals), (1, 2, 0)) 63 | 64 | 65 | # Load "y" (the neural network's training and testing outputs) 66 | 67 | def load_y(y_path): 68 | file = open(y_path, 'rb') 69 | # Read dataset from disk, dealing with text file's syntax 70 | y_ = np.array( 71 | [elem for elem in [ 72 | row.replace(' ', ' ').strip().split(' ') for row in file 73 | ]], 74 | dtype=np.int32 75 | ) 76 | file.close() 77 | # Substract 1 to each output class for friendly 0-based indexing 78 | return y_ - 1 79 | 80 | 81 | 82 | def relu_fc(input_2D_tensor_list, features_len, new_features_len, config): 83 | """make a relu fully-connected layer, mainly change the shape of tensor 84 | both input and output is a list of tensor 85 | argument: 86 | input_2D_tensor_list: list shape is [batch_size,feature_num] 87 | features_len: int the initial features length of input_2D_tensor 88 | new_feature_len: int the final features length of output_2D_tensor 89 | config: Config used for weights initializers 90 | return: 91 | output_2D_tensor_list lit shape is [batch_size,new_feature_len] 92 | """ 93 | 94 | W = tf.get_variable( 95 | "relu_fc_weights", 96 | initializer=tf.random_normal( 97 | [features_len, new_features_len], 98 | mean=0.0, 99 | stddev=float(config.weights_stddev) 100 | ) 101 | ) 102 | b = tf.get_variable( 103 | "relu_fc_biases_noreg", 104 | initializer=tf.random_normal( 105 | [new_features_len], 106 | mean=float(config.bias_mean), 107 | stddev=float(config.weights_stddev) 108 | ) 109 | ) 110 | 111 | # intra-timestep multiplication: 112 | output_2D_tensor_list = [ 113 | tf.nn.relu(tf.matmul(input_2D_tensor, W) + b) 114 | for input_2D_tensor in input_2D_tensor_list 115 | ] 116 | 117 | return output_2D_tensor_list 118 | 119 | def single_LSTM_cell(input_hidden_tensor, n_outputs): 120 | with tf.variable_scope("lstm_cell"): 121 | lstm_cell = tf.contrib.rnn.BasicLSTMCell(n_outputs, state_is_tuple=True, forget_bias=0.999) 122 | #print(lstm_cell) 123 | #outputs, _ = tf.nn.rnn(lstm_cell, input_hidden_tensor, dtype=tf.float32) 124 | outputs, states = tf.contrib.rnn.static_rnn(lstm_cell, input_hidden_tensor, dtype=tf.float32) 125 | #print(states) 126 | 127 | return outputs 128 | 129 | 130 | def stack_single_highway_LSTM_layer(input_hidden_tensor, n_input, n_output, layer_level, config, keep_prob_for_dropout): 131 | with tf.variable_scope('baselayer_{}'.format(layer_level)) as scope: 132 | if config.batch_norm_enabled : 133 | input_hidden_tensor = [apply_batch_norm(out, config, i) for i, out in enumerate(input_hidden_tensor)] 134 | hidden_LSTM_layer = single_LSTM_cell(input_hidden_tensor, n_output) 135 | with tf.variable_scope('residuallayer_{}'.format(layer_level)) as scope: 136 | upper_LSTM_layer = single_LSTM_cell(hidden_LSTM_layer, n_output) 137 | hidden_LSTM_layer = [a + b for a, b in zip(hidden_LSTM_layer, upper_LSTM_layer)] 138 | return hidden_LSTM_layer 139 | 140 | 141 | def stack_single_highway_LSTM_layer_(input_hidden_tensor, n_input, n_output, layer_level, config, keep_prob_for_dropout): 142 | with tf.variable_scope('layer_{}'.format(layer_level)) as scope: 143 | upper_LSTM_layer = single_LSTM_cell(input_hidden_tensor, n_output) 144 | hidden_LSTM_layer = [ a + b for a,b in zip(input_hidden_tensor , upper_LSTM_layer)] 145 | return hidden_LSTM_layer 146 | 147 | 148 | 149 | def get_stacked_highway_LSTM_layers(input_hidden_tensor, n_input, n_output, config, keep_prob_for_dropout): 150 | 151 | # creating base lstm Layer 152 | print "\nCreating hidden #1:" 153 | hidden = stack_single_highway_LSTM_layer(input_hidden_tensor, config.n_inputs, config.n_hidden, 1, config, keep_prob_for_dropout) 154 | print (len(hidden), str(hidden[0].get_shape())) 155 | 156 | 157 | 158 | 159 | # Stacking LSTM layer on existing layer in a for loop 160 | for stacked_hidden_index in range(config.n_stacked_layers - 1): 161 | # If the config permits it, we stack more lstm cells: 162 | print "\nCreating hidden #{}:".format(stacked_hidden_index + 2) 163 | hidden = stack_single_highway_LSTM_layer(hidden, config.n_hidden, config.n_hidden, stacked_hidden_index + 2, config, 164 | keep_prob_for_dropout) 165 | print (len(hidden), str(hidden[0].get_shape())) 166 | 167 | print "" 168 | return hidden 169 | 170 | 171 | 172 | 173 | def deep_LSTM_network(feature_mat, config, keep_prob_for_dropout): 174 | with tf.variable_scope('LSTM_network') as scope: # TensorFlow graph naming 175 | 176 | feature_mat = tf.nn.dropout(feature_mat, keep_prob_for_dropout) 177 | feature_mat = tf.transpose(feature_mat, [1, 0, 2]) 178 | feature_mat = tf.reshape(feature_mat, [-1, config.n_inputs]) 179 | print feature_mat.get_shape() 180 | 181 | # Split the series because the rnn cell needs time_steps features, each of shape: 182 | hidden = tf.split(axis=0, num_or_size_splits=config.n_steps, value=feature_mat) 183 | print (len(hidden), str(hidden[0].get_shape())) 184 | # New shape: a list of lenght "time_step" containing tensors of shape [batch_size, n_hidden] 185 | 186 | hidden = get_stacked_highway_LSTM_layers(hidden, config.n_inputs, config.n_hidden, config, keep_prob_for_dropout) 187 | 188 | # Final fully-connected activation logits 189 | # Get the last output tensor of the inner loop output series, of shape [batch_size, n_classes] 190 | last_hidden = tf.nn.dropout(hidden[-1], keep_prob_for_dropout) 191 | last_logits = relu_fc( 192 | [last_hidden], 193 | config.n_hidden, config.n_classes, config 194 | )[0] 195 | return last_logits 196 | 197 | ################################## load data and config ################################## 198 | 199 | X_train, y_train, X_test, y_test = get_HAR_data() 200 | 201 | class HighwayConfig(Config): 202 | def __init__(self): 203 | super(HighwayConfig, self).__init__() 204 | self.train_count = len(X_train) # 7352 training series 205 | self.test_data_count = len(X_test) # 2947 testing series 206 | self.n_steps = len(X_train[0]) # 128 time_steps per series 207 | 208 | # Trainging 209 | self.learning_rate = 0.0005 210 | self.lambda_loss_amount = 0.0015 211 | self.training_epochs = 450 212 | self.batch_size = 1500 213 | 214 | # LSTM structure 215 | self.n_inputs = len(X_train[0][0]) # == 9 Features count is of 9: three 3D sensors features over time 216 | self.n_hidden = 32 # nb of neurons inside the neural network 217 | self.n_classes = 6 # Final output classes 218 | 219 | self.keep_prob_for_dropout = 0.85 220 | self.bias_mean = 0.3 221 | self.weights_stddev = 0.2 222 | self.n_layers_in_highway = 0 223 | self.n_stacked_layers = 4 224 | self.also_add_dropout_between_stacked_cells = False 225 | self.batch_norm_enabled = True 226 | self.tensor_board_logging_enabled = True 227 | self.logs_path = "/tmp/LSTM_logs/highway_lstm/" 228 | self.tensorboard_cmd = "tensorboard --logdir="+ self.logs_path 229 | 230 | 231 | #config = Config(X_train, X_test) 232 | config = HighwayConfig() 233 | 234 | 235 | 236 | def run_with_config(config) : #, X_train, y_train, X_test, y_test): 237 | tf.reset_default_graph() # To enable to run multiple things in a loop 238 | config.print_config() 239 | 240 | config.W = { 241 | 'hidden': tf.Variable(tf.random_normal([config.n_inputs, config.n_hidden])), 242 | 'output': tf.Variable(tf.random_normal([config.n_hidden, config.n_classes])) 243 | } 244 | config.biases = { 245 | 'hidden': tf.Variable(tf.random_normal([config.n_hidden], mean=1.0)), 246 | 'output': tf.Variable(tf.random_normal([config.n_classes])) 247 | } 248 | #----------------------------------- 249 | # Define parameters for model 250 | #----------------------------------- 251 | 252 | 253 | print("Some useful info to get an insight on dataset's shape and normalisation:") 254 | print("features shape, labels shape, each features mean, each features standard deviation") 255 | print(X_test.shape, y_test.shape, 256 | np.mean(X_test), np.std(X_test)) 257 | print("the dataset is therefore properly normalised, as expected.") 258 | 259 | # ------------------------------------------------------ 260 | # step3: Let's get serious and build the neural network 261 | # ------------------------------------------------------ 262 | X = tf.placeholder(tf.float32, [None, config.n_steps, config.n_inputs]) 263 | Y = tf.placeholder(tf.float32, [None, config.n_classes]) 264 | 265 | # pred_Y = LSTM_Network(X, config) 266 | pred_Y = deep_LSTM_network(X, config, 0.85) 267 | 268 | print "Unregularised variables:" 269 | for unreg in [tf_var.name for tf_var in tf.trainable_variables() if 270 | ("noreg" in tf_var.name or "Bias" in tf_var.name)]: 271 | print unreg 272 | 273 | # Loss,optimizer,evaluation 274 | l2 = config.lambda_loss_amount * \ 275 | sum(tf.nn.l2_loss(tf_var) for tf_var in tf.trainable_variables()) 276 | # Softmax loss and L2 277 | cost = tf.reduce_mean( 278 | tf.nn.softmax_cross_entropy_with_logits(logits=pred_Y, labels=Y)) + l2 279 | optimizer = tf.train.AdamOptimizer( 280 | learning_rate=config.learning_rate).minimize(cost) 281 | 282 | correct_pred = tf.equal(tf.argmax(pred_Y, 1), tf.argmax(Y, 1)) 283 | accuracy = tf.reduce_mean(tf.cast(correct_pred, dtype=tf.float32)) 284 | # ------------------------------------------------------ 285 | # step3.5 : Tensorboard stuff here 286 | # ------------------------------------------------------ 287 | if config.tensor_board_logging_enabled: 288 | tf.summary.scalar("loss", cost) 289 | tf.summary.scalar("accuracy", accuracy) 290 | merged_summary_op = tf.summary.merge_all() 291 | 292 | # -------------------------------------------- 293 | # step4: Hooray, now train the neural network 294 | # -------------------------------------------- 295 | # Note that log_device_placement can be turned ON but will cause console spam. 296 | sess = tf.InteractiveSession(config=tf.ConfigProto(log_device_placement=False)) 297 | tf.global_variables_initializer().run() 298 | 299 | if config.tensor_board_logging_enabled: 300 | # op to write logs to Tensorboard 301 | summary_writer = tf.summary.FileWriter(config.logs_path, graph=tf.get_default_graph()) 302 | 303 | best_accuracy = 0.0 304 | # Start training for each batch and loop epochs 305 | for i in range(config.training_epochs): 306 | for start, end in zip(range(0, config.train_count, config.batch_size), 307 | range(config.batch_size, config.train_count + 1, config.batch_size)): 308 | if config.tensor_board_logging_enabled: 309 | _, summary = sess.run([optimizer, merged_summary_op], 310 | feed_dict={X: X_train[start:end], Y: y_train[start:end]}) 311 | else: 312 | sess.run(optimizer, feed_dict={X: X_train[start:end], Y: y_train[start:end]}) 313 | 314 | if config.tensor_board_logging_enabled: 315 | # Write logs at every iteration 316 | summary_writer.add_summary(summary, i) 317 | 318 | # Test completely at every epoch: calculate accuracy 319 | pred_out, accuracy_out, loss_out = sess.run([pred_Y, accuracy, cost], feed_dict={ 320 | X: X_test, Y: y_test}) 321 | print("traing iter: {},".format(i) + \ 322 | " test accuracy : {},".format(accuracy_out) + \ 323 | " loss : {}".format(loss_out)) 324 | best_accuracy = max(best_accuracy, accuracy_out) 325 | 326 | print("") 327 | print("final test accuracy: {}".format(accuracy_out)) 328 | print("best epoch's test accuracy: {}".format(best_accuracy)) 329 | print("") 330 | 331 | if config.tensor_board_logging_enabled: 332 | print("Run the command line:\n") 333 | print(config.tensorboard_cmd) 334 | print("\nThen open http://0.0.0.0:6006/ into your web browser") 335 | 336 | 337 | if __name__ == '__main__': 338 | run_with_config(config) # , trX, trY, teX, teY) 339 | 340 | 341 | -------------------------------------------------------------------------------- /highway_tranform_lstm_model.py: -------------------------------------------------------------------------------- 1 | from HAR_data_handler import get_HAR_data 2 | import tensorflow as tf 3 | from sklearn import metrics 4 | from sklearn.utils import shuffle 5 | import numpy as np 6 | from base_config import Config, YaxisBundle, PlotUtil 7 | 8 | def one_hot(y): 9 | """convert label from dense to one hot 10 | argument: 11 | label: ndarray dense label ,shape: [sample_num,1] 12 | return: 13 | one_hot_label: ndarray one hot, shape: [sample_num,n_class] 14 | """ 15 | # e.g.: [[5], [0], [3]] --> [[0, 0, 0, 0, 0, 1], [1, 0, 0, 0, 0, 0], [0, 0, 0, 1, 0, 0]] 16 | 17 | y = y.reshape(len(y)) 18 | n_values = np.max(y) + 1 19 | return np.eye(n_values)[np.array(y, dtype=np.int32)] # Returns FLOATS 20 | 21 | def apply_batch_norm(input_tensor, config, i): 22 | 23 | with tf.variable_scope("batch_norm") as scope: 24 | if i != 0 : 25 | # Do not create extra variables for each time step 26 | scope.reuse_variables() 27 | 28 | # Mean and variance normalisation simply crunched over all axes 29 | axes = list(range(len(input_tensor.get_shape()))) 30 | 31 | mean, variance = tf.nn.moments(input_tensor, axes=axes, shift=None, name=None, keep_dims=False) 32 | stdev = tf.sqrt(variance + 0.001) 33 | 34 | # Rescaling 35 | bn = input_tensor - mean 36 | bn /= stdev 37 | # Learnable extra rescaling 38 | 39 | # tf.get_variable("relu_fc_weights", initializer=tf.random_normal(mean=0.0, stddev=0.0) 40 | bn *= tf.get_variable("a_noreg", initializer=tf.random_normal([1], mean=0.5, stddev=0.0)) 41 | bn += tf.get_variable("b_noreg", initializer=tf.random_normal([1], mean=0.0, stddev=0.0)) 42 | # bn *= tf.Variable(0.5, name=(scope.name + "/a_noreg")) 43 | # bn += tf.Variable(0.0, name=(scope.name + "/b_noreg")) 44 | 45 | return bn 46 | 47 | def relu_fc(input_2D_tensor_list, features_len, new_features_len, config): 48 | """make a relu fully-connected layer, mainly change the shape of tensor 49 | both input and output is a list of tensor 50 | argument: 51 | input_2D_tensor_list: list shape is [batch_size,feature_num] 52 | features_len: int the initial features length of input_2D_tensor 53 | new_feature_len: int the final features length of output_2D_tensor 54 | config: Config used for weights initializers 55 | return: 56 | output_2D_tensor_list lit shape is [batch_size,new_feature_len] 57 | """ 58 | 59 | W = tf.get_variable( 60 | "relu_fc_weights", 61 | initializer=tf.random_normal( 62 | [features_len, new_features_len], 63 | mean=0.0, 64 | stddev=float(config.weights_stddev) 65 | ) 66 | ) 67 | b = tf.get_variable( 68 | "relu_fc_biases_noreg", 69 | initializer=tf.random_normal( 70 | [new_features_len], 71 | mean=float(config.bias_mean), 72 | stddev=float(config.weights_stddev) 73 | ) 74 | ) 75 | 76 | # intra-timestep multiplication: 77 | output_2D_tensor_list = [ 78 | tf.nn.relu(tf.matmul(input_2D_tensor, W) + b) 79 | for input_2D_tensor in input_2D_tensor_list 80 | ] 81 | 82 | return output_2D_tensor_list 83 | 84 | def single_LSTM_cell(input_hidden_tensor, n_outputs): 85 | with tf.variable_scope("lstm_cell"): 86 | lstm_cell = tf.contrib.rnn.BasicLSTMCell(n_outputs, state_is_tuple=True, forget_bias=0.999) 87 | #outputs, _ = tf.nn.rnn(lstm_cell, input_hidden_tensor, dtype=tf.float32) 88 | outputs, _ = tf.contrib.rnn.static_rnn(lstm_cell, input_hidden_tensor, dtype=tf.float32) 89 | res_output = [] 90 | for i in xrange(len(outputs)): #relu_fc(input_hidden_tensor, n_input, n_output, config), n_output)[i] 91 | res_output.append(outputs[i] ) 92 | return res_output 93 | 94 | def single_LSTM_cell_with_Highway(input_hidden_tensor, n_outputs, res_unit): 95 | with tf.variable_scope("lstm_cell"): 96 | lstm_cell = tf.contrib.rnn.BasicLSTMCell(n_outputs, state_is_tuple=True, forget_bias=0.999) 97 | outputs, _ = tf.contrib.rnn.static_rnn(lstm_cell, input_hidden_tensor, dtype=tf.float32) 98 | 99 | residual_out = [] 100 | for i, H in enumerate(outputs) : 101 | T = tf.sigmoid(config.biases['transform'][i]) 102 | residual_out.append(H* T + res_unit[i]*(1-T)) 103 | return residual_out 104 | 105 | def stack_single_LSTM_layer(input_hidden_tensor, n_input, n_output, layer_level, config, keep_prob_for_dropout): 106 | 107 | with tf.variable_scope('layer_{}'.format(layer_level)) as scope: 108 | if config.batch_norm_enabled : 109 | input_hidden_tensor = [apply_batch_norm(out, config, i) for i, out in enumerate(input_hidden_tensor)] 110 | hidden_LSTM_layer = single_LSTM_cell_with_Highway(input_hidden_tensor, n_output, relu_fc(input_hidden_tensor, n_input, n_output, config)) 111 | 112 | return hidden_LSTM_layer 113 | 114 | 115 | def get_deeply_stacked_LSTM_layers(input_hidden_tensor, n_input, n_output, config, keep_prob_for_dropout): 116 | print "\nCreating hidden #1:" 117 | hidden = stack_single_LSTM_layer(input_hidden_tensor, config.n_inputs, config.n_hidden, 1, config, keep_prob_for_dropout) 118 | print (len(hidden), str(hidden[0].get_shape())) 119 | for stacked_hidden_index in range(config.n_stacked_layers - 1): 120 | print "\nCreating hidden #{}:".format(stacked_hidden_index + 2) 121 | hidden = stack_single_LSTM_layer(hidden, config.n_hidden, config.n_hidden, stacked_hidden_index + 2, config, 122 | keep_prob_for_dropout) 123 | print (len(hidden), str(hidden[0].get_shape())) 124 | print "" 125 | return hidden 126 | 127 | 128 | def deep_LSTM_network(feature_mat, config, keep_prob_for_dropout): 129 | with tf.variable_scope('LSTM_network') as scope: # TensorFlow graph naming 130 | 131 | feature_mat = tf.nn.dropout(feature_mat, keep_prob_for_dropout) 132 | feature_mat = tf.transpose(feature_mat, [1, 0, 2]) 133 | feature_mat = tf.reshape(feature_mat, [-1, config.n_inputs]) 134 | print feature_mat.get_shape() 135 | 136 | # Split the series because the rnn cell needs time_steps features, each of shape: 137 | hidden = tf.split(axis=0, num_or_size_splits=config.n_steps, value=feature_mat) 138 | print (len(hidden), str(hidden[0].get_shape())) 139 | # New shape: a list of lenght "time_step" containing tensors of shape [batch_size, n_hidden] 140 | 141 | hidden = get_deeply_stacked_LSTM_layers(hidden, config.n_inputs, config.n_hidden, config, keep_prob_for_dropout) 142 | 143 | # Final fully-connected activation logits 144 | # Get the last output tensor of the inner loop output series, of shape [batch_size, n_classes] 145 | lstm_last_output = hidden[-1] 146 | 147 | # Linear activation 148 | return tf.matmul(lstm_last_output, config.W['output']) + config.biases['output'] 149 | 150 | last_hidden = tf.nn.dropout(hidden[-1], keep_prob_for_dropout) 151 | last_logits = relu_fc( 152 | [last_hidden], 153 | config.n_hidden, config.n_classes, config 154 | )[0] 155 | return last_logits 156 | 157 | ################################## load data and config ################################## 158 | 159 | X_train, y_train, X_test, y_test = get_HAR_data() 160 | 161 | class HigwayTransformLstmConfig(Config): 162 | def __init__(self): 163 | super(HigwayTransformLstmConfig, self).__init__() 164 | self.train_count = len(X_train) # 7352 training series 165 | self.test_data_count = len(X_test) # 2947 testing series 166 | self.n_steps = len(X_train[0]) # 128 time_steps per series 167 | 168 | # Trainging 169 | self.learning_rate = 0.005 170 | self.lambda_loss_amount = 0.0015 171 | self.training_epochs = 300 172 | self.batch_size = 1500 173 | 174 | # LSTM structure 175 | self.n_inputs = len(X_train[0][0]) # == 9 Features count is of 9: three 3D sensors features over time 176 | self.n_hidden = 32 # nb of neurons inside the neural network 177 | self.n_classes = 6 # Final output classes 178 | 179 | self.keep_prob_for_dropout = 0.85 180 | self.bias_mean = 0.3 181 | self.weights_stddev = 0.2 182 | self.n_layers_in_highway = 0 183 | self.n_stacked_layers = 3 184 | self.also_add_dropout_between_stacked_cells = False 185 | self.batch_norm_enabled = True 186 | 187 | self.model_name = "highway_transform_lstm_" + "HAR" 188 | self.log_folder_suffix = self.attach_log_suffix() 189 | self.logs_path = "/tmp/LSTM_logs/"+self.log_folder_suffix 190 | 191 | self.tensor_board_logging_enabled = True 192 | self.tensorboard_cmd = "tensorboard --logdir="+ self.logs_path 193 | self.model_desc_attched_string = self.attach_mdoel_desc() 194 | self.matplot_lib_enabled = True 195 | self.matplot_lib_for_accuracy =True 196 | self.matplot_lib_for_single_ybundle=False 197 | 198 | 199 | #config = Config(X_train, X_test) 200 | config = HigwayTransformLstmConfig() 201 | 202 | 203 | 204 | def run_with_config(config) : #, X_train, y_train, X_test, y_test): 205 | tf.reset_default_graph() # To enable to run multiple things in a loop 206 | config.print_config() 207 | 208 | if config.matplot_lib_enabled: 209 | # To keep track of training's performance 210 | test_losses = [] 211 | test_accuracies = [] 212 | indep_test_axis = [] 213 | 214 | 215 | 216 | config.W = { 217 | 'hidden': tf.Variable(tf.random_normal([config.n_inputs, config.n_hidden])), 218 | 'output': tf.Variable(tf.random_normal([config.n_hidden, config.n_classes])) 219 | } 220 | config.biases = { 221 | 'hidden': tf.Variable(tf.random_normal([config.n_hidden], mean=1.0)), 222 | 'output': tf.Variable(tf.random_normal([config.n_classes])), 223 | 'transform': tf.Variable(tf.random_normal([config.n_steps], dtype=tf.float32)) 224 | } 225 | #----------------------------------- 226 | # Define parameters for model 227 | #----------------------------------- 228 | 229 | 230 | print("Some useful info to get an insight on dataset's shape and normalisation:") 231 | print("features shape, labels shape, each features mean, each features standard deviation") 232 | print(X_test.shape, y_test.shape, 233 | np.mean(X_test), np.std(X_test)) 234 | print("the dataset is therefore properly normalised, as expected.") 235 | 236 | # ------------------------------------------------------ 237 | # step3: Let's get serious and build the neural network 238 | # ------------------------------------------------------ 239 | X = tf.placeholder(tf.float32, [None, config.n_steps, config.n_inputs]) 240 | Y = tf.placeholder(tf.float32, [None, config.n_classes]) 241 | 242 | pred_Y = deep_LSTM_network(X, config, 0.85) 243 | 244 | 245 | print "Unregularised variables:" 246 | for unreg in [tf_var.name for tf_var in tf.trainable_variables() if 247 | ("noreg" in tf_var.name or "Bias" in tf_var.name)]: 248 | print unreg 249 | 250 | # Loss,optimizer,evaluation 251 | l2 = config.lambda_loss_amount * \ 252 | sum(tf.nn.l2_loss(tf_var) for tf_var in tf.trainable_variables()) 253 | # Softmax loss and L2 254 | cost = tf.reduce_mean( 255 | tf.nn.softmax_cross_entropy_with_logits(logits=pred_Y, labels=Y)) + l2 256 | optimizer = tf.train.AdamOptimizer( 257 | learning_rate=config.learning_rate).minimize(cost) 258 | 259 | correct_pred = tf.equal(tf.argmax(pred_Y, 1), tf.argmax(Y, 1)) 260 | accuracy = tf.reduce_mean(tf.cast(correct_pred, dtype=tf.float32)) 261 | # ------------------------------------------------------ 262 | # step3.5 : Tensorboard stuff here 263 | # ------------------------------------------------------ 264 | if config.tensor_board_logging_enabled: 265 | tf.summary.scalar("loss", cost) 266 | tf.summary.scalar("accuracy", accuracy) 267 | merged_summary_op = tf.summary.merge_all() 268 | 269 | # -------------------------------------------- 270 | # step4: Hooray, now train the neural network 271 | # -------------------------------------------- 272 | # Note that log_device_placement can be turned ON but will cause console spam. 273 | sess = tf.InteractiveSession(config=tf.ConfigProto(log_device_placement=False)) 274 | tf.global_variables_initializer().run() 275 | 276 | if config.tensor_board_logging_enabled: 277 | # op to write logs to Tensorboard 278 | summary_writer = tf.summary.FileWriter(config.logs_path, graph=tf.get_default_graph()) 279 | 280 | best_accuracy = 0.0 281 | # Start training for each batch and loop epochs 282 | for i in range(config.training_epochs): 283 | for start, end in zip(range(0, config.train_count, config.batch_size), 284 | range(config.batch_size, config.train_count + 1, config.batch_size)): 285 | if config.tensor_board_logging_enabled: 286 | _, summary = sess.run([optimizer, merged_summary_op], 287 | feed_dict={X: X_train[start:end], Y: y_train[start:end]}) 288 | else: 289 | sess.run(optimizer, feed_dict={X: X_train[start:end], Y: y_train[start:end]}) 290 | 291 | if config.tensor_board_logging_enabled: 292 | # Write logs at every iteration 293 | summary_writer.add_summary(summary, i) 294 | 295 | # Test completely at every epoch: calculate accuracy 296 | pred_out, accuracy_out, loss_out = sess.run([pred_Y, accuracy, cost], feed_dict={ 297 | X: X_test, Y: y_test}) 298 | 299 | if config.matplot_lib_enabled: 300 | indep_test_axis.append(i) 301 | test_losses.append(loss_out) 302 | test_accuracies.append(accuracy_out) 303 | 304 | print("traing iter: {},".format(i) + \ 305 | " test accuracy : {},".format(accuracy_out) + \ 306 | " loss : {}".format(loss_out)) 307 | best_accuracy = max(best_accuracy, accuracy_out) 308 | 309 | print("") 310 | print("final test accuracy: {}".format(accuracy_out)) 311 | print("best epoch's test accuracy: {}".format(best_accuracy)) 312 | print("") 313 | 314 | if config.tensor_board_logging_enabled: 315 | print("Run the command line:\n") 316 | print(config.tensorboard_cmd) 317 | print("\nThen open http://0.0.0.0:6006/ into your web browser") 318 | 319 | print(config.model_desc_attched_string) 320 | 321 | 322 | if config.matplot_lib_enabled: 323 | 324 | #for i in range(config.batch_size): 325 | # indep_test_axis.append(i) 326 | #indep_test_axis = [i for i in range(config.batch_size)] 327 | #indep_test_axis = np.array(indep_test_axis) 328 | 329 | #p = PlotUtil("title", indep_test_axis, "x_label", "y_label") 330 | y_bundle = [] 331 | test_losses = np.array(test_losses) 332 | test_accuracies = np.array(test_accuracies) 333 | 334 | y = YaxisBundle(np.array(test_losses), "loss", "b") 335 | y_bundle.append(y) 336 | 337 | y = YaxisBundle(np.array(test_accuracies), "accuracy", "g") 338 | y_bundle.append(y) 339 | 340 | #p.show_plot(y_bundle) 341 | 342 | if config.matplot_lib_for_single_ybundle: 343 | if config.matplot_lib_for_accuracy: 344 | return y_bundle[1] 345 | else : 346 | return y_bundle[0] 347 | return y_bundle 348 | 349 | 350 | 351 | if __name__ == '__main__': 352 | if config.matplot_lib_enabled: 353 | indep_test_axis = [] 354 | for i in range(config.training_epochs): 355 | indep_test_axis.append(i) 356 | 357 | p = PlotUtil("Highway LSTM(3 layers) on HAR", np.array(indep_test_axis), "Epoch iterations", "Loss or Accuracy") 358 | y_bundle = run_with_config(config) 359 | 360 | p.show_plot(y_bundle) 361 | else: 362 | run_with_config(config) 363 | -------------------------------------------------------------------------------- /highway_tranform_lstm_model_MNIST_dataset.py: -------------------------------------------------------------------------------- 1 | from HAR_data_handler import get_HAR_data 2 | import tensorflow as tf 3 | from sklearn import metrics 4 | from sklearn.utils import shuffle 5 | import numpy as np 6 | from base_config import Config, YaxisBundle, PlotUtil 7 | from mnist_data_handler import get_mnist_data_with_time_series_size 8 | 9 | def one_hot(y): 10 | """convert label from dense to one hot 11 | argument: 12 | label: ndarray dense label ,shape: [sample_num,1] 13 | return: 14 | one_hot_label: ndarray one hot, shape: [sample_num,n_class] 15 | """ 16 | # e.g.: [[5], [0], [3]] --> [[0, 0, 0, 0, 0, 1], [1, 0, 0, 0, 0, 0], [0, 0, 0, 1, 0, 0]] 17 | 18 | y = y.reshape(len(y)) 19 | n_values = np.max(y) + 1 20 | return np.eye(n_values)[np.array(y, dtype=np.int32)] # Returns FLOATS 21 | 22 | def apply_batch_norm(input_tensor, config, i): 23 | 24 | with tf.variable_scope("batch_norm") as scope: 25 | if i != 0 : 26 | # Do not create extra variables for each time step 27 | scope.reuse_variables() 28 | 29 | # Mean and variance normalisation simply crunched over all axes 30 | axes = list(range(len(input_tensor.get_shape()))) 31 | 32 | mean, variance = tf.nn.moments(input_tensor, axes=axes, shift=None, name=None, keep_dims=False) 33 | stdev = tf.sqrt(variance + 0.001) 34 | 35 | # Rescaling 36 | bn = input_tensor - mean 37 | bn /= stdev 38 | # Learnable extra rescaling 39 | 40 | # tf.get_variable("relu_fc_weights", initializer=tf.random_normal(mean=0.0, stddev=0.0) 41 | bn *= tf.get_variable("a_noreg", initializer=tf.random_normal([1], mean=0.5, stddev=0.0)) 42 | bn += tf.get_variable("b_noreg", initializer=tf.random_normal([1], mean=0.0, stddev=0.0)) 43 | # bn *= tf.Variable(0.5, name=(scope.name + "/a_noreg")) 44 | # bn += tf.Variable(0.0, name=(scope.name + "/b_noreg")) 45 | 46 | return bn 47 | 48 | def relu_fc(input_2D_tensor_list, features_len, new_features_len, config): 49 | """make a relu fully-connected layer, mainly change the shape of tensor 50 | both input and output is a list of tensor 51 | argument: 52 | input_2D_tensor_list: list shape is [batch_size,feature_num] 53 | features_len: int the initial features length of input_2D_tensor 54 | new_feature_len: int the final features length of output_2D_tensor 55 | config: Config used for weights initializers 56 | return: 57 | output_2D_tensor_list lit shape is [batch_size,new_feature_len] 58 | """ 59 | 60 | W = tf.get_variable( 61 | "relu_fc_weights", 62 | initializer=tf.random_normal( 63 | [features_len, new_features_len], 64 | mean=0.0, 65 | stddev=float(config.weights_stddev) 66 | ) 67 | ) 68 | b = tf.get_variable( 69 | "relu_fc_biases_noreg", 70 | initializer=tf.random_normal( 71 | [new_features_len], 72 | mean=float(config.bias_mean), 73 | stddev=float(config.weights_stddev) 74 | ) 75 | ) 76 | 77 | # intra-timestep multiplication: 78 | output_2D_tensor_list = [ 79 | tf.nn.relu(tf.matmul(input_2D_tensor, W) + b) 80 | for input_2D_tensor in input_2D_tensor_list 81 | ] 82 | 83 | return output_2D_tensor_list 84 | 85 | def single_LSTM_cell(input_hidden_tensor, n_outputs): 86 | with tf.variable_scope("lstm_cell"): 87 | lstm_cell = tf.contrib.rnn.BasicLSTMCell(n_outputs, state_is_tuple=True, forget_bias=0.999) 88 | #outputs, _ = tf.nn.rnn(lstm_cell, input_hidden_tensor, dtype=tf.float32) 89 | outputs, _ = tf.contrib.rnn.static_rnn(lstm_cell, input_hidden_tensor, dtype=tf.float32) 90 | res_output = [] 91 | for i in xrange(len(outputs)): #relu_fc(input_hidden_tensor, n_input, n_output, config), n_output)[i] 92 | res_output.append(outputs[i] ) 93 | return res_output 94 | 95 | def single_LSTM_cell_with_Highway(input_hidden_tensor, n_outputs, res_unit): 96 | with tf.variable_scope("lstm_cell"): 97 | lstm_cell = tf.contrib.rnn.BasicLSTMCell(n_outputs, state_is_tuple=True, forget_bias=0.999) 98 | outputs, _ = tf.contrib.rnn.static_rnn(lstm_cell, input_hidden_tensor, dtype=tf.float32) 99 | 100 | residual_out = [] 101 | for i, H in enumerate(outputs) : 102 | T = tf.sigmoid(config.biases['transform'][i]) 103 | residual_out.append(H* T + res_unit[i]*(1-T)) 104 | return residual_out 105 | 106 | def stack_single_LSTM_layer(input_hidden_tensor, n_input, n_output, layer_level, config, keep_prob_for_dropout): 107 | 108 | with tf.variable_scope('layer_{}'.format(layer_level)) as scope: 109 | if config.batch_norm_enabled : 110 | input_hidden_tensor = [apply_batch_norm(out, config, i) for i, out in enumerate(input_hidden_tensor)] 111 | hidden_LSTM_layer = single_LSTM_cell_with_Highway(input_hidden_tensor, n_output, relu_fc(input_hidden_tensor, n_input, n_output, config)) 112 | 113 | return hidden_LSTM_layer 114 | 115 | 116 | def get_deeply_stacked_LSTM_layers(input_hidden_tensor, n_input, n_output, config, keep_prob_for_dropout): 117 | print "\nCreating hidden #1:" 118 | hidden = stack_single_LSTM_layer(input_hidden_tensor, config.n_inputs, config.n_hidden, 1, config, keep_prob_for_dropout) 119 | print (len(hidden), str(hidden[0].get_shape())) 120 | for stacked_hidden_index in range(config.n_stacked_layers - 1): 121 | print "\nCreating hidden #{}:".format(stacked_hidden_index + 2) 122 | hidden = stack_single_LSTM_layer(hidden, config.n_hidden, config.n_hidden, stacked_hidden_index + 2, config, 123 | keep_prob_for_dropout) 124 | print (len(hidden), str(hidden[0].get_shape())) 125 | print "" 126 | return hidden 127 | 128 | 129 | def deep_LSTM_network(feature_mat, config, keep_prob_for_dropout): 130 | with tf.variable_scope('LSTM_network') as scope: # TensorFlow graph naming 131 | 132 | feature_mat = tf.nn.dropout(feature_mat, keep_prob_for_dropout) 133 | feature_mat = tf.transpose(feature_mat, [1, 0, 2]) 134 | feature_mat = tf.reshape(feature_mat, [-1, config.n_inputs]) 135 | print feature_mat.get_shape() 136 | 137 | # Split the series because the rnn cell needs time_steps features, each of shape: 138 | hidden = tf.split(axis=0, num_or_size_splits=config.n_steps, value=feature_mat) 139 | print (len(hidden), str(hidden[0].get_shape())) 140 | # New shape: a list of lenght "time_step" containing tensors of shape [batch_size, n_hidden] 141 | 142 | hidden = get_deeply_stacked_LSTM_layers(hidden, config.n_inputs, config.n_hidden, config, keep_prob_for_dropout) 143 | 144 | # Final fully-connected activation logits 145 | # Get the last output tensor of the inner loop output series, of shape [batch_size, n_classes] 146 | lstm_last_output = hidden[-1] 147 | 148 | # Linear activation 149 | return tf.matmul(lstm_last_output, config.W['output']) + config.biases['output'] 150 | 151 | last_hidden = tf.nn.dropout(hidden[-1], keep_prob_for_dropout) 152 | last_logits = relu_fc( 153 | [last_hidden], 154 | config.n_hidden, config.n_classes, config 155 | )[0] 156 | return last_logits 157 | 158 | ################################## load data and config ################################## 159 | 160 | X_train, y_train, X_test, y_test, series_size = get_mnist_data_with_time_series_size() 161 | 162 | class HigwayTransformLstmConfig(Config): 163 | def __init__(self): 164 | super(HigwayTransformLstmConfig, self).__init__() 165 | self.train_count = len(X_train) # 7352 training series 166 | self.test_data_count = len(X_test) # 2947 testing series 167 | self.n_steps = len(X_train[0]) # 128 time_steps per series 168 | 169 | # Trainging 170 | self.learning_rate = 0.005 171 | self.lambda_loss_amount = 0.0015 172 | self.training_epochs = 300 173 | self.batch_size = 1500 174 | 175 | # LSTM structure 176 | self.n_inputs = len(X_train[0][0]) 177 | self.n_hidden = 28 # nb of neurons inside the neural network 178 | self.n_classes = len(y_train[0]) # Final output classes 179 | 180 | self.keep_prob_for_dropout = 0.85 181 | self.bias_mean = 0.3 182 | self.weights_stddev = 0.2 183 | self.n_layers_in_highway = 0 184 | self.n_stacked_layers = 3 185 | self.also_add_dropout_between_stacked_cells = False 186 | self.batch_norm_enabled = True 187 | 188 | self.model_name = "highway_transform_lstm_" + "MNIST" 189 | self.log_folder_suffix = self.attach_log_suffix() 190 | self.logs_path = "/tmp/LSTM_logs/"+self.log_folder_suffix 191 | 192 | self.tensor_board_logging_enabled = True 193 | self.tensorboard_cmd = "tensorboard --logdir="+ self.logs_path 194 | self.model_desc_attched_string = self.attach_mdoel_desc() 195 | self.matplot_lib_enabled = True 196 | self.matplot_lib_for_accuracy =True 197 | self.matplot_lib_for_single_ybundle=False 198 | 199 | 200 | #config = Config(X_train, X_test) 201 | config = HigwayTransformLstmConfig() 202 | 203 | 204 | 205 | def run_with_config(config) : #, X_train, y_train, X_test, y_test): 206 | tf.reset_default_graph() # To enable to run multiple things in a loop 207 | config.print_config() 208 | 209 | if config.matplot_lib_enabled: 210 | # To keep track of training's performance 211 | test_losses = [] 212 | test_accuracies = [] 213 | indep_test_axis = [] 214 | 215 | 216 | 217 | config.W = { 218 | 'hidden': tf.Variable(tf.random_normal([config.n_inputs, config.n_hidden])), 219 | 'output': tf.Variable(tf.random_normal([config.n_hidden, config.n_classes])) 220 | } 221 | config.biases = { 222 | 'hidden': tf.Variable(tf.random_normal([config.n_hidden], mean=1.0)), 223 | 'output': tf.Variable(tf.random_normal([config.n_classes])), 224 | 'transform': tf.Variable(tf.random_normal([config.n_steps], dtype=tf.float32)) 225 | } 226 | #----------------------------------- 227 | # Define parameters for model 228 | #----------------------------------- 229 | 230 | 231 | print("Some useful info to get an insight on dataset's shape and normalisation:") 232 | print("features shape, labels shape, each features mean, each features standard deviation") 233 | print(X_test.shape, y_test.shape, 234 | np.mean(X_test), np.std(X_test)) 235 | print("the dataset is therefore properly normalised, as expected.") 236 | 237 | # ------------------------------------------------------ 238 | # step3: Let's get serious and build the neural network 239 | # ------------------------------------------------------ 240 | X = tf.placeholder(tf.float32, [None, config.n_steps, config.n_inputs]) 241 | Y = tf.placeholder(tf.float32, [None, config.n_classes]) 242 | 243 | pred_Y = deep_LSTM_network(X, config, 0.85) 244 | 245 | 246 | print "Unregularised variables:" 247 | for unreg in [tf_var.name for tf_var in tf.trainable_variables() if 248 | ("noreg" in tf_var.name or "Bias" in tf_var.name)]: 249 | print unreg 250 | 251 | # Loss,optimizer,evaluation 252 | l2 = config.lambda_loss_amount * \ 253 | sum(tf.nn.l2_loss(tf_var) for tf_var in tf.trainable_variables()) 254 | # Softmax loss and L2 255 | cost = tf.reduce_mean( 256 | tf.nn.softmax_cross_entropy_with_logits(logits=pred_Y, labels=Y)) + l2 257 | optimizer = tf.train.AdamOptimizer( 258 | learning_rate=config.learning_rate).minimize(cost) 259 | 260 | correct_pred = tf.equal(tf.argmax(pred_Y, 1), tf.argmax(Y, 1)) 261 | accuracy = tf.reduce_mean(tf.cast(correct_pred, dtype=tf.float32)) 262 | # ------------------------------------------------------ 263 | # step3.5 : Tensorboard stuff here 264 | # ------------------------------------------------------ 265 | if config.tensor_board_logging_enabled: 266 | tf.summary.scalar("loss", cost) 267 | tf.summary.scalar("accuracy", accuracy) 268 | merged_summary_op = tf.summary.merge_all() 269 | 270 | # -------------------------------------------- 271 | # step4: Hooray, now train the neural network 272 | # -------------------------------------------- 273 | # Note that log_device_placement can be turned ON but will cause console spam. 274 | sess = tf.InteractiveSession(config=tf.ConfigProto(log_device_placement=False)) 275 | tf.global_variables_initializer().run() 276 | 277 | if config.tensor_board_logging_enabled: 278 | # op to write logs to Tensorboard 279 | summary_writer = tf.summary.FileWriter(config.logs_path, graph=tf.get_default_graph()) 280 | 281 | best_accuracy = 0.0 282 | # Start training for each batch and loop epochs 283 | for i in range(config.training_epochs): 284 | for start, end in zip(range(0, config.train_count, config.batch_size), 285 | range(config.batch_size, config.train_count + 1, config.batch_size)): 286 | if config.tensor_board_logging_enabled: 287 | _, summary = sess.run([optimizer, merged_summary_op], 288 | feed_dict={X: X_train[start:end], Y: y_train[start:end]}) 289 | else: 290 | sess.run(optimizer, feed_dict={X: X_train[start:end], Y: y_train[start:end]}) 291 | 292 | if config.tensor_board_logging_enabled: 293 | # Write logs at every iteration 294 | summary_writer.add_summary(summary, i) 295 | 296 | # Test completely at every epoch: calculate accuracy 297 | pred_out, accuracy_out, loss_out = sess.run([pred_Y, accuracy, cost], feed_dict={ 298 | X: X_test, Y: y_test}) 299 | 300 | if config.matplot_lib_enabled: 301 | indep_test_axis.append(i) 302 | test_losses.append(loss_out) 303 | test_accuracies.append(accuracy_out) 304 | 305 | print("traing iter: {},".format(i) + \ 306 | " test accuracy : {},".format(accuracy_out) + \ 307 | " loss : {}".format(loss_out)) 308 | best_accuracy = max(best_accuracy, accuracy_out) 309 | 310 | print("") 311 | print("final test accuracy: {}".format(accuracy_out)) 312 | print("best epoch's test accuracy: {}".format(best_accuracy)) 313 | print("") 314 | 315 | if config.tensor_board_logging_enabled: 316 | print("Run the command line:\n") 317 | print(config.tensorboard_cmd) 318 | print("\nThen open http://0.0.0.0:6006/ into your web browser") 319 | 320 | print(config.model_desc_attched_string) 321 | 322 | 323 | if config.matplot_lib_enabled: 324 | 325 | #for i in range(config.batch_size): 326 | # indep_test_axis.append(i) 327 | #indep_test_axis = [i for i in range(config.batch_size)] 328 | #indep_test_axis = np.array(indep_test_axis) 329 | 330 | #p = PlotUtil("title", indep_test_axis, "x_label", "y_label") 331 | y_bundle = [] 332 | test_losses = np.array(test_losses) 333 | test_accuracies = np.array(test_accuracies) 334 | 335 | y = YaxisBundle(np.array(test_losses), "loss", "b") 336 | y_bundle.append(y) 337 | 338 | y = YaxisBundle(np.array(test_accuracies), "accuracy", "g") 339 | y_bundle.append(y) 340 | 341 | #p.show_plot(y_bundle) 342 | 343 | if config.matplot_lib_for_single_ybundle: 344 | if config.matplot_lib_for_accuracy: 345 | return y_bundle[1] 346 | else : 347 | return y_bundle[0] 348 | return y_bundle 349 | 350 | 351 | 352 | if __name__ == '__main__': 353 | if config.matplot_lib_enabled: 354 | indep_test_axis = [] 355 | for i in range(config.training_epochs): 356 | indep_test_axis.append(i) 357 | 358 | p = PlotUtil("Highway LSTM(3 layers) on MNIST", np.array(indep_test_axis), "Epoch iterations", "Loss or Accuracy") 359 | y_bundle = run_with_config(config) 360 | 361 | p.show_plot(y_bundle) 362 | else: 363 | run_with_config(config) 364 | -------------------------------------------------------------------------------- /lstm_mnist.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | from random import Random 4 | from tensorflow.contrib import rnn 5 | from mnist_data_handler import get_mnist_data 6 | 7 | 8 | 9 | 10 | # configuration 11 | # O * W + b -> 10 labels for each image, O[? 28], W[28 10], B[10] 12 | # ^ (O: output 28 vec from 28 vec input) 13 | # | 14 | # +-+ +-+ +--+ 15 | # |1|->|2|-> ... |28| time_step_size = 28 16 | # +-+ +-+ +--+ 17 | # ^ ^ ... ^ 18 | # | | | 19 | # img1:[28] [28] ... [28] 20 | # img2:[28] [28] ... [28] 21 | # img3:[28] [28] ... [28] 22 | # ... 23 | # img128 or img256 (batch_size or test_size 256) 24 | # each input size = input_vec_size=lstm_size=28 25 | 26 | 27 | 28 | ############################## model definition ###################################### 29 | 30 | input_vec_size = lstm_size = 28 31 | time_step_size = 28 32 | 33 | batch_size = 128 34 | test_size = 10 35 | 36 | def init_weights(shape): 37 | return tf.Variable(tf.random_normal(shape, stddev=0.01)) 38 | 39 | 40 | def model(X, W, B, lstm_size): 41 | # X, input shape: (batch_size, time_step_size, input_vec_size) 42 | XT = tf.transpose(X, [1, 0, 2]) # permute time_step_size and batch_size 43 | # XT shape: (time_step_size, batch_size, input_vec_size) 44 | XR = tf.reshape(XT, [-1, lstm_size]) # each row has input for each lstm cell (lstm_size=input_vec_size) 45 | # XR shape: (time_step_size * batch_size, input_vec_size) 46 | X_split = tf.split(XR, time_step_size, 0) # split them to time_step_size (28 arrays) 47 | # Each array shape: (batch_size, input_vec_size) 48 | 49 | # Make lstm with lstm_size (each input vector size) 50 | lstm = rnn.BasicLSTMCell(lstm_size, forget_bias=1.0, state_is_tuple=True) 51 | 52 | # Get lstm cell output, time_step_size (28) arrays with lstm_size output: (batch_size, lstm_size) 53 | outputs, _states = rnn.static_rnn(lstm, X_split, dtype=tf.float32) 54 | 55 | # Linear activation 56 | # Get the last output 57 | return tf.matmul(outputs[-1], W) + B, lstm.state_size # State size to initialize the stat 58 | 59 | 60 | 61 | ############################## model definition end ###################################### 62 | 63 | 64 | 65 | def run_with_config(Config):#, : 66 | Config.print_config() 67 | tf.reset_default_graph() ## enables re-running of graph with different config 68 | 69 | (trX, trY, teX, teY) = get_mnist_data(); ## dataset loading 70 | 71 | with tf.device("/cpu:0"): # Remove this line to use GPU. If you have a too small GPU, it crashes. 72 | X = tf.placeholder("float", [None, 28, 28]) 73 | Y = tf.placeholder("float", [None, 10]) 74 | 75 | # get lstm_size and output 10 labels 76 | W = init_weights([lstm_size, 10]) 77 | B = init_weights([10]) 78 | 79 | py_x, state_size = model(X, W, B, lstm_size) 80 | 81 | 82 | cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=py_x, labels=Y)) 83 | train_op = tf.train.RMSPropOptimizer(Config.learning_rate, Config.decay).minimize(cost) 84 | predict_op = tf.argmax(py_x, 1) 85 | 86 | session_conf = tf.ConfigProto() 87 | session_conf.gpu_options.allow_growth = True 88 | 89 | # Launch the graph in a session 90 | with tf.Session(config=session_conf) as sess: 91 | # you need to initialize all variables 92 | tf.global_variables_initializer().run() 93 | 94 | for i in range(100): # was 100 iter 95 | #traing done here 96 | for start, end in zip(range(0, len(trX), batch_size), range(batch_size, len(trX)+1, batch_size)): 97 | sess.run(train_op, feed_dict={X: trX[start:end], Y: trY[start:end]}) 98 | 99 | 100 | #testing from here 101 | test_indices = np.arange(len(teX)) # Get A Test Batch 102 | run_result = sess.run(predict_op, feed_dict={X: teX[test_indices]} ) 103 | res = np.argmax(teY[test_indices], axis=1) == run_result 104 | print(i, np.mean(res)) 105 | 106 | 107 | 108 | -------------------------------------------------------------------------------- /main_runner.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | import numpy as np 4 | from random import Random 5 | from tensorflow.contrib import rnn 6 | from tensorflow.python.framework import ops 7 | 8 | 9 | import lstm_mnist 10 | 11 | 12 | 13 | import deep_lstm_model 14 | import single_layer_lstm 15 | import highway_lstm_model 16 | import residual_lstm_model 17 | import deep_lstm_model_UCR_dataset 18 | import highway_lstm_model_UCR_dataset 19 | 20 | 21 | 22 | 23 | if __name__ == '__main__': 24 | """ 25 | for training_epochs_ in [10, 20]: 26 | config.tensor_board_logging_enabled = False # should be always False, log summary folder gets impacted by mulitple runs 27 | config.training_epochs = training_epochs_ 28 | run_with_config(config) 29 | exit(0) 30 | """ 31 | #run_with_config = deep_lstm_model.run_with_config 32 | #config = deep_lstm_model.config 33 | 34 | #run_with_config = deep_lstm_model_on_ucr_dataset.run_with_configp 35 | #config = deep_lstm_model_on_ucr_dataset.config 36 | 37 | run_with_config = single_layer_lstm.run_with_config 38 | config = single_layer_lstm.config 39 | 40 | 41 | for learning_rate in [0.005, 0.0025, 0.003, 0.0005]: #1, 0.0025, 0.002]: # [0.01, 0.007, 0.001, 0.0007, 0.0001]: 42 | for decay in [0.9]: #[0.005, 0.01]: 43 | for bn_enabled in [True, False]: 44 | for n_stacked in [1]: #2 3 6 45 | for epoch_count in [200, 300, 450]: 46 | config.training_epochs = epoch_count 47 | config.tensor_board_logging_enabled = False #should be always False, log summary folder gets impacted by mulitple runs 48 | config.n_stacked_layers = n_stacked 49 | config.batch_norm_enabled = bn_enabled 50 | config.learning_rate = learning_rate 51 | config.decay = decay 52 | run_with_config(config) #, trX, trY, teX, teY) 53 | -------------------------------------------------------------------------------- /main_runner_MNIST_results.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | import numpy as np 4 | from random import Random 5 | from tensorflow.contrib import rnn 6 | from tensorflow.python.framework import ops 7 | from base_config import PlotUtil, YaxisBundle 8 | 9 | 10 | import lstm_mnist 11 | 12 | 13 | 14 | import deep_lstm_model 15 | import single_layer_lstm 16 | import highway_lstm_model 17 | import residual_lstm_model 18 | import deep_lstm_model_UCR_dataset 19 | import highway_lstm_model_UCR_dataset 20 | import highway_lstm_model_plotlib 21 | 22 | import single_layer_lstm_MNIST 23 | import deep_lstm_model_MNIST_dataset 24 | import highway_tranform_lstm_model_MNIST_dataset 25 | import residual_lstm_model_MNIST_dataset 26 | 27 | import math 28 | 29 | y_bundle = [] 30 | 31 | 32 | batch_size = 300 33 | 34 | 35 | 36 | def sigmoid(x): 37 | return 1 / (1 + math.exp(-x)) 38 | 39 | 40 | def show_multi_plot(y_bundle, batch_size_): 41 | indep_test_axis = [] 42 | for i in range(batch_size_): 43 | indep_test_axis.append(i) 44 | p = PlotUtil("Accuracy Results on MNIST Dataset", np.array(indep_test_axis), "Epoch iterations", "Accuracy") 45 | p.show_plot(y_bundle) 46 | 47 | def runner_with_results_output(y_bundle): 48 | test_losses = [] 49 | test_accuracies = [] 50 | 51 | for i in range(batch_size): 52 | test_losses.append(3.5 - 1.6 * sigmoid(i / 10)) 53 | test_accuracies.append(0.5 + 0.4 * sigmoid(i / 10)) 54 | 55 | y = YaxisBundle(test_losses, "loss", "b") 56 | y_bundle.append(y) 57 | 58 | y = YaxisBundle(test_accuracies, "accuracy", "g") 59 | y_bundle.append(y) 60 | return y_bundle 61 | 62 | 63 | 64 | 65 | if __name__ == '__main__': 66 | run_with_config = single_layer_lstm_MNIST.run_with_config 67 | config = single_layer_lstm_MNIST.config 68 | 69 | config.tensor_board_logging_enabled = False 70 | config.matplot_lib_enabled = True 71 | config.matplot_lib_for_single_ybundle = True 72 | config.matplot_lib_for_accuracy = True 73 | config.training_epochs = 100 74 | config.n_stacked_layers = 1 75 | config.bias_mean = 0.5 76 | config.learning_rate = 0.0005 77 | 78 | y = run_with_config(config) 79 | y.y_graph_label = "Vanilla LSTM" 80 | y.y_graph_colour = "b" 81 | y_bundle.append(y) 82 | 83 | ##### end for model########### 84 | 85 | run_with_config = deep_lstm_model_MNIST_dataset.run_with_config 86 | config = deep_lstm_model_MNIST_dataset.config 87 | 88 | config.tensor_board_logging_enabled = False 89 | config.matplot_lib_enabled = True 90 | config.matplot_lib_for_single_ybundle = True 91 | config.matplot_lib_for_accuracy = True 92 | config.training_epochs = 100 93 | config.n_stacked_layers = 3 94 | config.bias_mean = 0.5 95 | config.learning_rate = 0.0005 96 | 97 | 98 | y = run_with_config(config) 99 | y.y_graph_label = "Stacked LSTM" 100 | y.y_graph_colour = "g" 101 | y_bundle.append(y) 102 | 103 | ##### end for model########### 104 | 105 | run_with_config = highway_tranform_lstm_model_MNIST_dataset.run_with_config 106 | config = highway_tranform_lstm_model_MNIST_dataset.config 107 | 108 | config.tensor_board_logging_enabled = False 109 | config.matplot_lib_enabled = True 110 | config.matplot_lib_for_single_ybundle = True 111 | config.matplot_lib_for_accuracy = True 112 | config.training_epochs = 100 113 | config.n_stacked_layers = 3 114 | config.bias_mean = 0.5 115 | config.learning_rate = 0.0005 116 | 117 | y = run_with_config(config) 118 | y.y_graph_label = "Highway LSTM" 119 | y.y_graph_colour = "r" 120 | y_bundle.append(y) 121 | 122 | ##### end for model########### 123 | 124 | run_with_config = residual_lstm_model_MNIST_dataset.run_with_config 125 | config = residual_lstm_model_MNIST_dataset.config 126 | 127 | config.tensor_board_logging_enabled = False 128 | config.matplot_lib_enabled = True 129 | config.matplot_lib_for_single_ybundle = True 130 | config.matplot_lib_for_accuracy = True 131 | config.training_epochs = 100 132 | config.n_stacked_layers = 3 133 | config.bias_mean = 0.5 134 | config.learning_rate = 0.0005 135 | 136 | y = run_with_config(config) 137 | y.y_graph_label = "Residual LSTM" 138 | y.y_graph_colour = "y" 139 | y_bundle.append(y) 140 | 141 | ##### end for model########### 142 | 143 | 144 | show_multi_plot(y_bundle, config.training_epochs) 145 | exit(0) 146 | -------------------------------------------------------------------------------- /main_runner_all_models_results.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | import numpy as np 4 | from random import Random 5 | from tensorflow.contrib import rnn 6 | from tensorflow.python.framework import ops 7 | from base_config import PlotUtil, YaxisBundle 8 | 9 | 10 | import lstm_mnist 11 | 12 | 13 | 14 | import deep_lstm_model 15 | import single_layer_lstm 16 | import highway_lstm_model 17 | import residual_lstm_model 18 | import deep_lstm_model_UCR_dataset 19 | import highway_lstm_model_UCR_dataset 20 | import highway_lstm_model_plotlib 21 | 22 | import single_layer_lstm_MNIST 23 | import highway_tranform_lstm_model 24 | import residual_lstm_model_MNIST_dataset 25 | 26 | import math 27 | 28 | y_bundle = [] 29 | 30 | 31 | learning_rate = 300 32 | layers = 6 33 | _matplot_lib_enabled = False 34 | 35 | _bias_mean = 0.5 36 | _learning_rate = 0.005 37 | 38 | 39 | if __name__ == '__main__': 40 | run_with_config = single_layer_lstm.run_with_config 41 | config = single_layer_lstm.config 42 | 43 | config.tensor_board_logging_enabled = False 44 | config.matplot_lib_enabled = _matplot_lib_enabled 45 | 46 | config.matplot_lib_for_single_ybundle = True 47 | config.matplot_lib_for_accuracy = True 48 | config.n_stacked_layers = 1 49 | 50 | 51 | config.bias_mean = _bias_mean 52 | config.learning_rate = _learning_rate 53 | 54 | y = run_with_config(config) 55 | 56 | ##### end for model########### 57 | 58 | run_with_config = deep_lstm_model.run_with_config 59 | config = deep_lstm_model.config 60 | 61 | config.matplot_lib_enabled = _matplot_lib_enabled 62 | 63 | 64 | config.tensor_board_logging_enabled = False 65 | config.n_stacked_layers = layers 66 | 67 | config.bias_mean = _bias_mean 68 | config.learning_rate = _learning_rate 69 | 70 | 71 | y = run_with_config(config) 72 | 73 | ##### end for model########### 74 | 75 | run_with_config = highway_tranform_lstm_model.run_with_config 76 | config = highway_tranform_lstm_model.config 77 | 78 | 79 | config.matplot_lib_enabled = _matplot_lib_enabled 80 | 81 | config.tensor_board_logging_enabled = False 82 | config.n_stacked_layers = layers 83 | config.matplot_lib_enabled = _matplot_lib_enabled 84 | 85 | config.bias_mean = _bias_mean 86 | config.learning_rate = _learning_rate 87 | 88 | y = run_with_config(config) 89 | 90 | ##### end for model########### 91 | 92 | run_with_config = residual_lstm_model.run_with_config 93 | config = residual_lstm_model.config 94 | 95 | config.tensor_board_logging_enabled = False 96 | config.n_stacked_layers = layers 97 | config.matplot_lib_enabled = _matplot_lib_enabled 98 | 99 | config.bias_mean = _bias_mean 100 | config.learning_rate = _learning_rate 101 | 102 | 103 | y = run_with_config(config) 104 | 105 | ##### end for model########### 106 | 107 | 108 | exit(0) 109 | 110 | 111 | -------------------------------------------------------------------------------- /main_runner_all_models_results_UCR.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | import numpy as np 4 | from random import Random 5 | from tensorflow.contrib import rnn 6 | from tensorflow.python.framework import ops 7 | from base_config import PlotUtil, YaxisBundle 8 | 9 | 10 | import lstm_mnist 11 | 12 | 13 | 14 | 15 | import single_layer_lstm_UCR_dataset 16 | import highway_tranform_lstm_model_UCR_dataset 17 | import residual_lstm_model_UCR_dataset 18 | import deep_lstm_model_UCR_dataset 19 | 20 | 21 | import math 22 | 23 | y_bundle = [] 24 | 25 | 26 | learning_rate = 300 27 | layers = 3 28 | _matplot_lib_enabled = False 29 | 30 | _bias_mean = 0.5 31 | _learning_rate = 0.005 32 | 33 | 34 | if __name__ == '__main__': 35 | run_with_config = single_layer_lstm_UCR_dataset.run_with_config 36 | config = single_layer_lstm_UCR_dataset.config 37 | 38 | config.tensor_board_logging_enabled = False 39 | config.matplot_lib_enabled = _matplot_lib_enabled 40 | 41 | config.matplot_lib_for_single_ybundle = True 42 | config.matplot_lib_for_accuracy = True 43 | config.n_stacked_layers = 1 44 | 45 | 46 | config.bias_mean = _bias_mean 47 | config.learning_rate = _learning_rate 48 | 49 | y = run_with_config(config) 50 | 51 | ##### end for model########### 52 | 53 | run_with_config = deep_lstm_model_UCR_dataset.run_with_config 54 | config = deep_lstm_model_UCR_dataset.config 55 | 56 | config.matplot_lib_enabled = _matplot_lib_enabled 57 | 58 | 59 | config.tensor_board_logging_enabled = False 60 | config.n_stacked_layers = layers 61 | 62 | config.bias_mean = _bias_mean 63 | config.learning_rate = _learning_rate 64 | 65 | 66 | y = run_with_config(config) 67 | 68 | ##### end for model########### 69 | 70 | run_with_config = highway_tranform_lstm_model_UCR_dataset.run_with_config 71 | config = highway_tranform_lstm_model_UCR_dataset.config 72 | 73 | 74 | config.matplot_lib_enabled = _matplot_lib_enabled 75 | 76 | config.tensor_board_logging_enabled = False 77 | config.n_stacked_layers = layers 78 | config.matplot_lib_enabled = _matplot_lib_enabled 79 | 80 | config.bias_mean = _bias_mean 81 | config.learning_rate = _learning_rate 82 | 83 | y = run_with_config(config) 84 | 85 | ##### end for model########### 86 | 87 | run_with_config = residual_lstm_model_UCR_dataset.run_with_config 88 | config = residual_lstm_model_UCR_dataset.config 89 | 90 | config.tensor_board_logging_enabled = False 91 | config.n_stacked_layers = layers 92 | config.matplot_lib_enabled = _matplot_lib_enabled 93 | 94 | config.bias_mean = _bias_mean 95 | config.learning_rate = _learning_rate 96 | 97 | 98 | y = run_with_config(config) 99 | 100 | ##### end for model########### 101 | 102 | 103 | exit(0) 104 | 105 | 106 | -------------------------------------------------------------------------------- /main_runner_with_plot.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | import numpy as np 4 | from random import Random 5 | from tensorflow.contrib import rnn 6 | from tensorflow.python.framework import ops 7 | from base_config import PlotUtil, YaxisBundle 8 | 9 | 10 | import lstm_mnist 11 | 12 | 13 | 14 | import deep_lstm_model 15 | import single_layer_lstm 16 | import highway_lstm_model 17 | import residual_lstm_model 18 | import deep_lstm_model_UCR_dataset 19 | import highway_lstm_model_UCR_dataset 20 | import highway_lstm_model_plotlib 21 | 22 | import math 23 | 24 | y_bundle = [] 25 | 26 | 27 | batch_size = 300 28 | 29 | 30 | 31 | def sigmoid(x): 32 | return 1 / (1 + math.exp(-x)) 33 | 34 | 35 | def show_multi_plot(y_bundle, batch_size_): 36 | indep_test_axis = [] 37 | for i in range(batch_size_): 38 | indep_test_axis.append(i) 39 | p = PlotUtil("title", np.array(indep_test_axis), "x_label", "y_label") 40 | p.show_plot(y_bundle) 41 | 42 | def runner_with_results_output(y_bundle): 43 | test_losses = [] 44 | test_accuracies = [] 45 | 46 | for i in range(batch_size): 47 | test_losses.append(3.5 - 1.6 * sigmoid(i / 10)) 48 | test_accuracies.append(0.5 + 0.4 * sigmoid(i / 10)) 49 | 50 | y = YaxisBundle(test_losses, "loss", "b") 51 | y_bundle.append(y) 52 | 53 | y = YaxisBundle(test_accuracies, "accuracy", "g") 54 | y_bundle.append(y) 55 | return y_bundle 56 | 57 | 58 | 59 | 60 | if __name__ == '__main__': 61 | run_with_config = highway_lstm_model_plotlib.run_with_config 62 | config = highway_lstm_model_plotlib.config 63 | 64 | config.tensor_board_logging_enabled = False 65 | config.matplot_lib_enabled = True 66 | config.matplot_lib_for_single_ybundle = True 67 | config.matplot_lib_for_accuracy = True 68 | config.training_epochs = 2 69 | config.n_stacked_layers = 1 70 | 71 | 72 | y_bundle.append(run_with_config(config)) 73 | 74 | run_with_config = highway_lstm_model_UCR_dataset.run_with_config 75 | config = highway_lstm_model_UCR_dataset.config 76 | 77 | config.tensor_board_logging_enabled = False 78 | config.matplot_lib_enabled = True 79 | config.matplot_lib_for_single_ybundle = True 80 | config.matplot_lib_for_accuracy = True 81 | config.training_epochs = 2 82 | config.n_stacked_layers = 1 83 | 84 | 85 | y_bundle.append(run_with_config(config)) 86 | 87 | show_multi_plot(y_bundle, config.training_epochs) 88 | exit(0) 89 | 90 | 91 | """ 92 | for training_epochs_ in [10, 20]: 93 | config.tensor_board_logging_enabled = False # should be always False, log summary folder gets impacted by mulitple runs 94 | config.training_epochs = training_epochs_ 95 | run_with_config(config) 96 | exit(0) 97 | """ 98 | #run_with_config = deep_lstm_model.run_with_config 99 | #config = deep_lstm_model.config 100 | 101 | #run_with_config = deep_lstm_model_on_ucr_dataset.run_with_configp 102 | #config = deep_lstm_model_on_ucr_dataset.config 103 | 104 | run_with_config = highway_lstm_model_UCR_dataset.run_with_config 105 | config = highway_lstm_model_UCR_dataset.config 106 | 107 | 108 | for learning_rate in [0.005, 0.0025, 0.003, 0.0005]: #1, 0.0025, 0.002]: # [0.01, 0.007, 0.001, 0.0007, 0.0001]: 109 | for decay in [0.9]: #[0.005, 0.01]: 110 | for bn_enabled in [True, False]: 111 | for n_stacked in [2,3, 6]: 112 | for epoch_count in [200, 300, 450]: 113 | config.training_epochs = epoch_count 114 | config.tensor_board_logging_enabled = False #should be always False, log summary folder gets impacted by mulitple runs 115 | config.n_stacked_layers = n_stacked 116 | config.batch_norm_enabled = bn_enabled 117 | config.learning_rate = learning_rate 118 | config.decay = decay 119 | run_with_config(config) #, trX, trY, teX, teY) 120 | -------------------------------------------------------------------------------- /mnist_data_handler.py: -------------------------------------------------------------------------------- 1 | from tensorflow.examples.tutorials.mnist import input_data 2 | 3 | 4 | 5 | def get_mnist_data(): 6 | mnist = input_data.read_data_sets("MNIST_data/", one_hot=True) 7 | trX, trY, teX, teY = mnist.train.images, mnist.train.labels, mnist.test.images, mnist.test.labels 8 | trX = trX.reshape(-1, 28, 28) 9 | teX = teX.reshape(-1, 28, 28) 10 | return (trX, trY, teX, teY) 11 | 12 | def get_mnist_data_with_time_series_size(): 13 | mnist = input_data.read_data_sets("MNIST_data/", one_hot=True) 14 | trX, trY, teX, teY = mnist.train.images, mnist.train.labels, mnist.test.images, mnist.test.labels 15 | trX = trX.reshape(-1, 28, 28) 16 | teX = teX.reshape(-1, 28, 28) 17 | return (trX, trY, teX, teY, 28) 18 | -------------------------------------------------------------------------------- /res_har_bn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/praveendareddy21/Stacked_LSTMS_Highway_Residual_On_TimeSeries_Datasets/4c9e9647b4a134a2b7f63b0289687cc92262d5c8/res_har_bn.png -------------------------------------------------------------------------------- /res_two_bn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/praveendareddy21/Stacked_LSTMS_Highway_Residual_On_TimeSeries_Datasets/4c9e9647b4a134a2b7f63b0289687cc92262d5c8/res_two_bn.png -------------------------------------------------------------------------------- /residual_lstm_model.py: -------------------------------------------------------------------------------- 1 | from HAR_data_handler import get_HAR_data 2 | import tensorflow as tf 3 | from sklearn import metrics 4 | from sklearn.utils import shuffle 5 | import numpy as np 6 | from base_config import Config, YaxisBundle, PlotUtil 7 | 8 | def one_hot(y): 9 | """convert label from dense to one hot 10 | argument: 11 | label: ndarray dense label ,shape: [sample_num,1] 12 | return: 13 | one_hot_label: ndarray one hot, shape: [sample_num,n_class] 14 | """ 15 | # e.g.: [[5], [0], [3]] --> [[0, 0, 0, 0, 0, 1], [1, 0, 0, 0, 0, 0], [0, 0, 0, 1, 0, 0]] 16 | 17 | y = y.reshape(len(y)) 18 | n_values = np.max(y) + 1 19 | return np.eye(n_values)[np.array(y, dtype=np.int32)] # Returns FLOATS 20 | 21 | def apply_batch_norm(input_tensor, config, i): 22 | 23 | with tf.variable_scope("batch_norm") as scope: 24 | if i != 0 : 25 | # Do not create extra variables for each time step 26 | scope.reuse_variables() 27 | 28 | # Mean and variance normalisation simply crunched over all axes 29 | axes = list(range(len(input_tensor.get_shape()))) 30 | 31 | mean, variance = tf.nn.moments(input_tensor, axes=axes, shift=None, name=None, keep_dims=False) 32 | stdev = tf.sqrt(variance + 0.001) 33 | 34 | # Rescaling 35 | bn = input_tensor - mean 36 | bn /= stdev 37 | # Learnable extra rescaling 38 | 39 | # tf.get_variable("relu_fc_weights", initializer=tf.random_normal(mean=0.0, stddev=0.0) 40 | bn *= tf.get_variable("a_noreg", initializer=tf.random_normal([1], mean=0.5, stddev=0.0)) 41 | bn += tf.get_variable("b_noreg", initializer=tf.random_normal([1], mean=0.0, stddev=0.0)) 42 | # bn *= tf.Variable(0.5, name=(scope.name + "/a_noreg")) 43 | # bn += tf.Variable(0.0, name=(scope.name + "/b_noreg")) 44 | 45 | return bn 46 | 47 | def relu_fc(input_2D_tensor_list, features_len, new_features_len, config): 48 | """make a relu fully-connected layer, mainly change the shape of tensor 49 | both input and output is a list of tensor 50 | argument: 51 | input_2D_tensor_list: list shape is [batch_size,feature_num] 52 | features_len: int the initial features length of input_2D_tensor 53 | new_feature_len: int the final features length of output_2D_tensor 54 | config: Config used for weights initializers 55 | return: 56 | output_2D_tensor_list lit shape is [batch_size,new_feature_len] 57 | """ 58 | 59 | W = tf.get_variable( 60 | "relu_fc_weights", 61 | initializer=tf.random_normal( 62 | [features_len, new_features_len], 63 | mean=0.0, 64 | stddev=float(config.weights_stddev) 65 | ) 66 | ) 67 | b = tf.get_variable( 68 | "relu_fc_biases_noreg", 69 | initializer=tf.random_normal( 70 | [new_features_len], 71 | mean=float(config.bias_mean), 72 | stddev=float(config.weights_stddev) 73 | ) 74 | ) 75 | 76 | # intra-timestep multiplication: 77 | output_2D_tensor_list = [ 78 | tf.nn.relu(tf.matmul(input_2D_tensor, W) + b) 79 | for input_2D_tensor in input_2D_tensor_list 80 | ] 81 | 82 | return output_2D_tensor_list 83 | 84 | def single_LSTM_cell(input_hidden_tensor, n_outputs): 85 | with tf.variable_scope("lstm_cell"): 86 | lstm_cell = tf.contrib.rnn.BasicLSTMCell(n_outputs, state_is_tuple=True, forget_bias=0.999) 87 | #outputs, _ = tf.nn.rnn(lstm_cell, input_hidden_tensor, dtype=tf.float32) 88 | outputs, _ = tf.contrib.rnn.static_rnn(lstm_cell, input_hidden_tensor, dtype=tf.float32) 89 | res_output = [] 90 | for i in xrange(len(outputs)): #relu_fc(input_hidden_tensor, n_input, n_output, config), n_output)[i] 91 | res_output.append(outputs[i] ) 92 | return res_output 93 | 94 | def single_LSTM_Res_cell(input_hidden_tensor, n_outputs, res_unit): 95 | with tf.variable_scope("lstm_cell"): 96 | lstm_cell = tf.contrib.rnn.BasicLSTMCell(n_outputs, state_is_tuple=True, forget_bias=0.999) 97 | #outputs, _ = tf.nn.rnn(lstm_cell, input_hidden_tensor, dtype=tf.float32) 98 | outputs, _ = tf.contrib.rnn.static_rnn(lstm_cell, input_hidden_tensor, dtype=tf.float32) 99 | return [out+res_unit[i] for i, out in enumerate(outputs)] 100 | 101 | def stack_single_LSTM_layer(input_hidden_tensor, n_input, n_output, layer_level, config, keep_prob_for_dropout): 102 | 103 | with tf.variable_scope('layer_{}'.format(layer_level)) as scope: 104 | if config.batch_norm_enabled : 105 | input_hidden_tensor = [apply_batch_norm(out, config, i) for i, out in enumerate(input_hidden_tensor)] 106 | hidden_LSTM_layer = single_LSTM_Res_cell(input_hidden_tensor, n_output, relu_fc(input_hidden_tensor, n_input, n_output, config)) 107 | 108 | return hidden_LSTM_layer 109 | 110 | 111 | def get_deeply_stacked_LSTM_layers(input_hidden_tensor, n_input, n_output, config, keep_prob_for_dropout): 112 | print "\nCreating hidden #1:" 113 | hidden = stack_single_LSTM_layer(input_hidden_tensor, config.n_inputs, config.n_hidden, 1, config, keep_prob_for_dropout) 114 | print (len(hidden), str(hidden[0].get_shape())) 115 | for stacked_hidden_index in range(config.n_stacked_layers - 1): 116 | print "\nCreating hidden #{}:".format(stacked_hidden_index + 2) 117 | hidden = stack_single_LSTM_layer(hidden, config.n_hidden, config.n_hidden, stacked_hidden_index + 2, config, 118 | keep_prob_for_dropout) 119 | print (len(hidden), str(hidden[0].get_shape())) 120 | print "" 121 | return hidden 122 | 123 | 124 | def deep_LSTM_network(feature_mat, config, keep_prob_for_dropout): 125 | with tf.variable_scope('LSTM_network') as scope: # TensorFlow graph naming 126 | 127 | feature_mat = tf.nn.dropout(feature_mat, keep_prob_for_dropout) 128 | feature_mat = tf.transpose(feature_mat, [1, 0, 2]) 129 | feature_mat = tf.reshape(feature_mat, [-1, config.n_inputs]) 130 | print feature_mat.get_shape() 131 | 132 | # Split the series because the rnn cell needs time_steps features, each of shape: 133 | hidden = tf.split(axis=0, num_or_size_splits=config.n_steps, value=feature_mat) 134 | print (len(hidden), str(hidden[0].get_shape())) 135 | # New shape: a list of lenght "time_step" containing tensors of shape [batch_size, n_hidden] 136 | 137 | hidden = get_deeply_stacked_LSTM_layers(hidden, config.n_inputs, config.n_hidden, config, keep_prob_for_dropout) 138 | 139 | # Final fully-connected activation logits 140 | # Get the last output tensor of the inner loop output series, of shape [batch_size, n_classes] 141 | lstm_last_output = hidden[-1] 142 | 143 | # Linear activation 144 | return tf.matmul(lstm_last_output, config.W['output']) + config.biases['output'] 145 | 146 | last_hidden = tf.nn.dropout(hidden[-1], keep_prob_for_dropout) 147 | last_logits = relu_fc( 148 | [last_hidden], 149 | config.n_hidden, config.n_classes, config 150 | )[0] 151 | return last_logits 152 | 153 | ################################## load data and config ################################## 154 | 155 | X_train, y_train, X_test, y_test = get_HAR_data() 156 | 157 | class ResLstmConfig(Config): 158 | def __init__(self): 159 | super(ResLstmConfig, self).__init__() 160 | self.train_count = len(X_train) # 7352 training series 161 | self.test_data_count = len(X_test) # 2947 testing series 162 | self.n_steps = len(X_train[0]) # 128 time_steps per series 163 | 164 | # Trainging 165 | self.learning_rate = 0.005 166 | self.lambda_loss_amount = 0.0015 167 | self.training_epochs = 300 168 | self.batch_size = 1500 169 | 170 | # LSTM structure 171 | self.n_inputs = len(X_train[0][0]) # == 9 Features count is of 9: three 3D sensors features over time 172 | self.n_hidden = 32 # nb of neurons inside the neural network 173 | self.n_classes = 6 # Final output classes 174 | 175 | self.keep_prob_for_dropout = 0.85 176 | self.bias_mean = 0.3 177 | self.weights_stddev = 0.2 178 | self.n_layers_in_highway = 0 179 | self.n_stacked_layers = 4 180 | self.also_add_dropout_between_stacked_cells = False 181 | self.batch_norm_enabled = True 182 | 183 | self.model_name = "residual_lstm_" + "HAR" 184 | self.log_folder_suffix = self.attach_log_suffix() 185 | self.logs_path = "/tmp/LSTM_logs/"+self.log_folder_suffix 186 | 187 | self.tensor_board_logging_enabled = True 188 | self.tensorboard_cmd = "tensorboard --logdir="+ self.logs_path 189 | self.model_desc_attched_string = self.attach_mdoel_desc() 190 | self.matplot_lib_enabled = True 191 | self.matplot_lib_for_accuracy =True 192 | self.matplot_lib_for_single_ybundle=False 193 | 194 | 195 | #config = Config(X_train, X_test) 196 | config = ResLstmConfig() 197 | 198 | 199 | 200 | def run_with_config(config) : #, X_train, y_train, X_test, y_test): 201 | tf.reset_default_graph() # To enable to run multiple things in a loop 202 | config.print_config() 203 | 204 | if config.matplot_lib_enabled: 205 | # To keep track of training's performance 206 | test_losses = [] 207 | test_accuracies = [] 208 | indep_test_axis = [] 209 | 210 | 211 | 212 | config.W = { 213 | 'hidden': tf.Variable(tf.random_normal([config.n_inputs, config.n_hidden])), 214 | 'output': tf.Variable(tf.random_normal([config.n_hidden, config.n_classes])) 215 | } 216 | config.biases = { 217 | 'hidden': tf.Variable(tf.random_normal([config.n_hidden], mean=1.0)), 218 | 'output': tf.Variable(tf.random_normal([config.n_classes])) 219 | } 220 | #----------------------------------- 221 | # Define parameters for model 222 | #----------------------------------- 223 | 224 | 225 | print("Some useful info to get an insight on dataset's shape and normalisation:") 226 | print("features shape, labels shape, each features mean, each features standard deviation") 227 | print(X_test.shape, y_test.shape, 228 | np.mean(X_test), np.std(X_test)) 229 | print("the dataset is therefore properly normalised, as expected.") 230 | 231 | # ------------------------------------------------------ 232 | # step3: Let's get serious and build the neural network 233 | # ------------------------------------------------------ 234 | X = tf.placeholder(tf.float32, [None, config.n_steps, config.n_inputs]) 235 | Y = tf.placeholder(tf.float32, [None, config.n_classes]) 236 | 237 | pred_Y = deep_LSTM_network(X, config, 0.85) 238 | 239 | 240 | print "Unregularised variables:" 241 | for unreg in [tf_var.name for tf_var in tf.trainable_variables() if 242 | ("noreg" in tf_var.name or "Bias" in tf_var.name)]: 243 | print unreg 244 | 245 | # Loss,optimizer,evaluation 246 | l2 = config.lambda_loss_amount * \ 247 | sum(tf.nn.l2_loss(tf_var) for tf_var in tf.trainable_variables()) 248 | # Softmax loss and L2 249 | cost = tf.reduce_mean( 250 | tf.nn.softmax_cross_entropy_with_logits(logits=pred_Y, labels=Y)) + l2 251 | optimizer = tf.train.AdamOptimizer( 252 | learning_rate=config.learning_rate).minimize(cost) 253 | 254 | correct_pred = tf.equal(tf.argmax(pred_Y, 1), tf.argmax(Y, 1)) 255 | accuracy = tf.reduce_mean(tf.cast(correct_pred, dtype=tf.float32)) 256 | # ------------------------------------------------------ 257 | # step3.5 : Tensorboard stuff here 258 | # ------------------------------------------------------ 259 | if config.tensor_board_logging_enabled: 260 | tf.summary.scalar("loss", cost) 261 | tf.summary.scalar("accuracy", accuracy) 262 | merged_summary_op = tf.summary.merge_all() 263 | 264 | # -------------------------------------------- 265 | # step4: Hooray, now train the neural network 266 | # -------------------------------------------- 267 | # Note that log_device_placement can be turned ON but will cause console spam. 268 | sess = tf.InteractiveSession(config=tf.ConfigProto(log_device_placement=False)) 269 | tf.global_variables_initializer().run() 270 | 271 | if config.tensor_board_logging_enabled: 272 | # op to write logs to Tensorboard 273 | summary_writer = tf.summary.FileWriter(config.logs_path, graph=tf.get_default_graph()) 274 | 275 | best_accuracy = 0.0 276 | # Start training for each batch and loop epochs 277 | for i in range(config.training_epochs): 278 | for start, end in zip(range(0, config.train_count, config.batch_size), 279 | range(config.batch_size, config.train_count + 1, config.batch_size)): 280 | if config.tensor_board_logging_enabled: 281 | _, summary = sess.run([optimizer, merged_summary_op], 282 | feed_dict={X: X_train[start:end], Y: y_train[start:end]}) 283 | else: 284 | sess.run(optimizer, feed_dict={X: X_train[start:end], Y: y_train[start:end]}) 285 | 286 | if config.tensor_board_logging_enabled: 287 | # Write logs at every iteration 288 | summary_writer.add_summary(summary, i) 289 | 290 | # Test completely at every epoch: calculate accuracy 291 | pred_out, accuracy_out, loss_out = sess.run([pred_Y, accuracy, cost], feed_dict={ 292 | X: X_test, Y: y_test}) 293 | 294 | if config.matplot_lib_enabled: 295 | indep_test_axis.append(i) 296 | test_losses.append(loss_out) 297 | test_accuracies.append(accuracy_out) 298 | 299 | print("traing iter: {},".format(i) + \ 300 | " test accuracy : {},".format(accuracy_out) + \ 301 | " loss : {}".format(loss_out)) 302 | best_accuracy = max(best_accuracy, accuracy_out) 303 | 304 | print("") 305 | print("final test accuracy: {}".format(accuracy_out)) 306 | print("best epoch's test accuracy: {}".format(best_accuracy)) 307 | print("") 308 | 309 | if config.tensor_board_logging_enabled: 310 | print("Run the command line:\n") 311 | print(config.tensorboard_cmd) 312 | print("\nThen open http://0.0.0.0:6006/ into your web browser") 313 | 314 | print(config.model_desc_attched_string) 315 | 316 | 317 | if config.matplot_lib_enabled: 318 | 319 | #for i in range(config.batch_size): 320 | # indep_test_axis.append(i) 321 | #indep_test_axis = [i for i in range(config.batch_size)] 322 | #indep_test_axis = np.array(indep_test_axis) 323 | 324 | #p = PlotUtil("title", indep_test_axis, "x_label", "y_label") 325 | y_bundle = [] 326 | test_losses = np.array(test_losses) 327 | test_accuracies = np.array(test_accuracies) 328 | 329 | y = YaxisBundle(np.array(test_losses), "loss", "b") 330 | y_bundle.append(y) 331 | 332 | y = YaxisBundle(np.array(test_accuracies), "accuracy", "g") 333 | y_bundle.append(y) 334 | 335 | #p.show_plot(y_bundle) 336 | 337 | if config.matplot_lib_for_single_ybundle: 338 | if config.matplot_lib_for_accuracy: 339 | return y_bundle[1] 340 | else : 341 | return y_bundle[0] 342 | return y_bundle 343 | 344 | 345 | 346 | if __name__ == '__main__': 347 | if config.matplot_lib_enabled: 348 | indep_test_axis = [] 349 | for i in range(config.training_epochs): 350 | indep_test_axis.append(i) 351 | 352 | p = PlotUtil("Residual LSTM(3 layers) on HAR", np.array(indep_test_axis), "Epoch iterations", "Loss or Accuracy") 353 | y_bundle = run_with_config(config) 354 | 355 | p.show_plot(y_bundle) 356 | else: 357 | run_with_config(config) 358 | -------------------------------------------------------------------------------- /residual_lstm_model_MNIST_dataset.py: -------------------------------------------------------------------------------- 1 | from HAR_data_handler import get_HAR_data 2 | import tensorflow as tf 3 | from sklearn import metrics 4 | from sklearn.utils import shuffle 5 | import numpy as np 6 | from base_config import Config, YaxisBundle, PlotUtil 7 | from mnist_data_handler import get_mnist_data_with_time_series_size 8 | 9 | def one_hot(y): 10 | """convert label from dense to one hot 11 | argument: 12 | label: ndarray dense label ,shape: [sample_num,1] 13 | return: 14 | one_hot_label: ndarray one hot, shape: [sample_num,n_class] 15 | """ 16 | # e.g.: [[5], [0], [3]] --> [[0, 0, 0, 0, 0, 1], [1, 0, 0, 0, 0, 0], [0, 0, 0, 1, 0, 0]] 17 | 18 | y = y.reshape(len(y)) 19 | n_values = np.max(y) + 1 20 | return np.eye(n_values)[np.array(y, dtype=np.int32)] # Returns FLOATS 21 | 22 | def apply_batch_norm(input_tensor, config, i): 23 | 24 | with tf.variable_scope("batch_norm") as scope: 25 | if i != 0 : 26 | # Do not create extra variables for each time step 27 | scope.reuse_variables() 28 | 29 | # Mean and variance normalisation simply crunched over all axes 30 | axes = list(range(len(input_tensor.get_shape()))) 31 | 32 | mean, variance = tf.nn.moments(input_tensor, axes=axes, shift=None, name=None, keep_dims=False) 33 | stdev = tf.sqrt(variance + 0.001) 34 | 35 | # Rescaling 36 | bn = input_tensor - mean 37 | bn /= stdev 38 | # Learnable extra rescaling 39 | 40 | # tf.get_variable("relu_fc_weights", initializer=tf.random_normal(mean=0.0, stddev=0.0) 41 | bn *= tf.get_variable("a_noreg", initializer=tf.random_normal([1], mean=0.5, stddev=0.0)) 42 | bn += tf.get_variable("b_noreg", initializer=tf.random_normal([1], mean=0.0, stddev=0.0)) 43 | # bn *= tf.Variable(0.5, name=(scope.name + "/a_noreg")) 44 | # bn += tf.Variable(0.0, name=(scope.name + "/b_noreg")) 45 | 46 | return bn 47 | 48 | def relu_fc(input_2D_tensor_list, features_len, new_features_len, config): 49 | """make a relu fully-connected layer, mainly change the shape of tensor 50 | both input and output is a list of tensor 51 | argument: 52 | input_2D_tensor_list: list shape is [batch_size,feature_num] 53 | features_len: int the initial features length of input_2D_tensor 54 | new_feature_len: int the final features length of output_2D_tensor 55 | config: Config used for weights initializers 56 | return: 57 | output_2D_tensor_list lit shape is [batch_size,new_feature_len] 58 | """ 59 | 60 | W = tf.get_variable( 61 | "relu_fc_weights", 62 | initializer=tf.random_normal( 63 | [features_len, new_features_len], 64 | mean=0.0, 65 | stddev=float(config.weights_stddev) 66 | ) 67 | ) 68 | b = tf.get_variable( 69 | "relu_fc_biases_noreg", 70 | initializer=tf.random_normal( 71 | [new_features_len], 72 | mean=float(config.bias_mean), 73 | stddev=float(config.weights_stddev) 74 | ) 75 | ) 76 | 77 | # intra-timestep multiplication: 78 | output_2D_tensor_list = [ 79 | tf.nn.relu(tf.matmul(input_2D_tensor, W) + b) 80 | for input_2D_tensor in input_2D_tensor_list 81 | ] 82 | 83 | return output_2D_tensor_list 84 | 85 | def single_LSTM_cell(input_hidden_tensor, n_outputs): 86 | with tf.variable_scope("lstm_cell"): 87 | lstm_cell = tf.contrib.rnn.BasicLSTMCell(n_outputs, state_is_tuple=True, forget_bias=0.999) 88 | #outputs, _ = tf.nn.rnn(lstm_cell, input_hidden_tensor, dtype=tf.float32) 89 | outputs, _ = tf.contrib.rnn.static_rnn(lstm_cell, input_hidden_tensor, dtype=tf.float32) 90 | res_output = [] 91 | for i in xrange(len(outputs)): #relu_fc(input_hidden_tensor, n_input, n_output, config), n_output)[i] 92 | res_output.append(outputs[i] ) 93 | return res_output 94 | 95 | def single_LSTM_Res_cell(input_hidden_tensor, n_outputs, res_unit): 96 | with tf.variable_scope("lstm_cell"): 97 | lstm_cell = tf.contrib.rnn.BasicLSTMCell(n_outputs, state_is_tuple=True, forget_bias=0.999) 98 | #outputs, _ = tf.nn.rnn(lstm_cell, input_hidden_tensor, dtype=tf.float32) 99 | outputs, _ = tf.contrib.rnn.static_rnn(lstm_cell, input_hidden_tensor, dtype=tf.float32) 100 | return [out+res_unit[i] for i, out in enumerate(outputs)] 101 | 102 | def stack_single_LSTM_layer(input_hidden_tensor, n_input, n_output, layer_level, config, keep_prob_for_dropout): 103 | 104 | with tf.variable_scope('layer_{}'.format(layer_level)) as scope: 105 | if config.batch_norm_enabled : 106 | input_hidden_tensor = [apply_batch_norm(out, config, i) for i, out in enumerate(input_hidden_tensor)] 107 | hidden_LSTM_layer = single_LSTM_Res_cell(input_hidden_tensor, n_output, relu_fc(input_hidden_tensor, n_input, n_output, config)) 108 | 109 | return hidden_LSTM_layer 110 | 111 | 112 | def get_deeply_stacked_LSTM_layers(input_hidden_tensor, n_input, n_output, config, keep_prob_for_dropout): 113 | print "\nCreating hidden #1:" 114 | hidden = stack_single_LSTM_layer(input_hidden_tensor, config.n_inputs, config.n_hidden, 1, config, keep_prob_for_dropout) 115 | print (len(hidden), str(hidden[0].get_shape())) 116 | for stacked_hidden_index in range(config.n_stacked_layers - 1): 117 | print "\nCreating hidden #{}:".format(stacked_hidden_index + 2) 118 | hidden = stack_single_LSTM_layer(hidden, config.n_hidden, config.n_hidden, stacked_hidden_index + 2, config, 119 | keep_prob_for_dropout) 120 | print (len(hidden), str(hidden[0].get_shape())) 121 | print "" 122 | return hidden 123 | 124 | 125 | def deep_LSTM_network(feature_mat, config, keep_prob_for_dropout): 126 | with tf.variable_scope('LSTM_network') as scope: # TensorFlow graph naming 127 | 128 | feature_mat = tf.nn.dropout(feature_mat, keep_prob_for_dropout) 129 | feature_mat = tf.transpose(feature_mat, [1, 0, 2]) 130 | feature_mat = tf.reshape(feature_mat, [-1, config.n_inputs]) 131 | print feature_mat.get_shape() 132 | 133 | # Split the series because the rnn cell needs time_steps features, each of shape: 134 | hidden = tf.split(axis=0, num_or_size_splits=config.n_steps, value=feature_mat) 135 | print (len(hidden), str(hidden[0].get_shape())) 136 | # New shape: a list of lenght "time_step" containing tensors of shape [batch_size, n_hidden] 137 | 138 | hidden = get_deeply_stacked_LSTM_layers(hidden, config.n_inputs, config.n_hidden, config, keep_prob_for_dropout) 139 | 140 | # Final fully-connected activation logits 141 | # Get the last output tensor of the inner loop output series, of shape [batch_size, n_classes] 142 | lstm_last_output = hidden[-1] 143 | 144 | # Linear activation 145 | return tf.matmul(lstm_last_output, config.W['output']) + config.biases['output'] 146 | 147 | last_hidden = tf.nn.dropout(hidden[-1], keep_prob_for_dropout) 148 | last_logits = relu_fc( 149 | [last_hidden], 150 | config.n_hidden, config.n_classes, config 151 | )[0] 152 | return last_logits 153 | 154 | ################################## load data and config ################################## 155 | X_train, y_train, X_test, y_test, series_size = get_mnist_data_with_time_series_size() 156 | 157 | class ResLstmConfig(Config): 158 | def __init__(self): 159 | super(ResLstmConfig, self).__init__() 160 | self.train_count = len(X_train) # 7352 training series 161 | self.test_data_count = len(X_test) # 2947 testing series 162 | self.n_steps = len(X_train[0]) # 128 time_steps per series 163 | 164 | # Trainging 165 | self.learning_rate = 0.005 166 | self.lambda_loss_amount = 0.0015 167 | self.training_epochs = 100 168 | self.batch_size = 1500 169 | 170 | # LSTM structure 171 | self.n_inputs = len(X_train[0][0]) 172 | self.n_hidden = 28 # nb of neurons inside the neural network 173 | self.n_classes = len(y_train[0]) # Final output classes 174 | 175 | self.keep_prob_for_dropout = 0.85 176 | self.bias_mean = 0.3 177 | self.weights_stddev = 0.9 178 | self.n_layers_in_highway = 0 179 | self.n_stacked_layers = 4 180 | self.also_add_dropout_between_stacked_cells = False 181 | self.batch_norm_enabled = True 182 | 183 | self.model_name = "residual_lstm_" + "MNIST" 184 | self.log_folder_suffix = self.attach_log_suffix() 185 | self.logs_path = "/tmp/LSTM_logs/"+self.log_folder_suffix 186 | 187 | self.tensor_board_logging_enabled = True 188 | self.tensorboard_cmd = "tensorboard --logdir="+ self.logs_path 189 | self.model_desc_attched_string = self.attach_mdoel_desc() 190 | self.matplot_lib_enabled = True 191 | self.matplot_lib_for_accuracy =True 192 | self.matplot_lib_for_single_ybundle=False 193 | 194 | 195 | #config = Config(X_train, X_test) 196 | config = ResLstmConfig() 197 | 198 | 199 | 200 | def run_with_config(config) : #, X_train, y_train, X_test, y_test): 201 | tf.reset_default_graph() # To enable to run multiple things in a loop 202 | config.print_config() 203 | 204 | if config.matplot_lib_enabled: 205 | # To keep track of training's performance 206 | test_losses = [] 207 | test_accuracies = [] 208 | indep_test_axis = [] 209 | 210 | 211 | 212 | config.W = { 213 | 'hidden': tf.Variable(tf.random_normal([config.n_inputs, config.n_hidden])), 214 | 'output': tf.Variable(tf.random_normal([config.n_hidden, config.n_classes])) 215 | } 216 | config.biases = { 217 | 'hidden': tf.Variable(tf.random_normal([config.n_hidden], mean=1.0)), 218 | 'output': tf.Variable(tf.random_normal([config.n_classes])) 219 | } 220 | #----------------------------------- 221 | # Define parameters for model 222 | #----------------------------------- 223 | 224 | 225 | print("Some useful info to get an insight on dataset's shape and normalisation:") 226 | print("features shape, labels shape, each features mean, each features standard deviation") 227 | print(X_test.shape, y_test.shape, 228 | np.mean(X_test), np.std(X_test)) 229 | print("the dataset is therefore properly normalised, as expected.") 230 | 231 | # ------------------------------------------------------ 232 | # step3: Let's get serious and build the neural network 233 | # ------------------------------------------------------ 234 | X = tf.placeholder(tf.float32, [None, config.n_steps, config.n_inputs]) 235 | Y = tf.placeholder(tf.float32, [None, config.n_classes]) 236 | 237 | pred_Y = deep_LSTM_network(X, config, 0.85) 238 | 239 | 240 | print "Unregularised variables:" 241 | for unreg in [tf_var.name for tf_var in tf.trainable_variables() if 242 | ("noreg" in tf_var.name or "Bias" in tf_var.name)]: 243 | print unreg 244 | 245 | # Loss,optimizer,evaluation 246 | l2 = config.lambda_loss_amount * \ 247 | sum(tf.nn.l2_loss(tf_var) for tf_var in tf.trainable_variables()) 248 | # Softmax loss and L2 249 | cost = tf.reduce_mean( 250 | tf.nn.softmax_cross_entropy_with_logits(logits=pred_Y, labels=Y)) + l2 251 | optimizer = tf.train.AdamOptimizer( 252 | learning_rate=config.learning_rate).minimize(cost) 253 | 254 | correct_pred = tf.equal(tf.argmax(pred_Y, 1), tf.argmax(Y, 1)) 255 | accuracy = tf.reduce_mean(tf.cast(correct_pred, dtype=tf.float32)) 256 | # ------------------------------------------------------ 257 | # step3.5 : Tensorboard stuff here 258 | # ------------------------------------------------------ 259 | if config.tensor_board_logging_enabled: 260 | tf.summary.scalar("loss", cost) 261 | tf.summary.scalar("accuracy", accuracy) 262 | merged_summary_op = tf.summary.merge_all() 263 | 264 | # -------------------------------------------- 265 | # step4: Hooray, now train the neural network 266 | # -------------------------------------------- 267 | # Note that log_device_placement can be turned ON but will cause console spam. 268 | sess = tf.InteractiveSession(config=tf.ConfigProto(log_device_placement=False)) 269 | tf.global_variables_initializer().run() 270 | 271 | if config.tensor_board_logging_enabled: 272 | # op to write logs to Tensorboard 273 | summary_writer = tf.summary.FileWriter(config.logs_path, graph=tf.get_default_graph()) 274 | 275 | best_accuracy = 0.0 276 | # Start training for each batch and loop epochs 277 | for i in range(config.training_epochs): 278 | for start, end in zip(range(0, config.train_count, config.batch_size), 279 | range(config.batch_size, config.train_count + 1, config.batch_size)): 280 | if config.tensor_board_logging_enabled: 281 | _, summary = sess.run([optimizer, merged_summary_op], 282 | feed_dict={X: X_train[start:end], Y: y_train[start:end]}) 283 | else: 284 | sess.run(optimizer, feed_dict={X: X_train[start:end], Y: y_train[start:end]}) 285 | 286 | if config.tensor_board_logging_enabled: 287 | # Write logs at every iteration 288 | summary_writer.add_summary(summary, i) 289 | 290 | # Test completely at every epoch: calculate accuracy 291 | pred_out, accuracy_out, loss_out = sess.run([pred_Y, accuracy, cost], feed_dict={ 292 | X: X_test, Y: y_test}) 293 | 294 | if config.matplot_lib_enabled: 295 | indep_test_axis.append(i) 296 | test_losses.append(loss_out) 297 | test_accuracies.append(accuracy_out) 298 | 299 | print("traing iter: {},".format(i) + \ 300 | " test accuracy : {},".format(accuracy_out) + \ 301 | " loss : {}".format(loss_out)) 302 | best_accuracy = max(best_accuracy, accuracy_out) 303 | 304 | print("") 305 | print("final test accuracy: {}".format(accuracy_out)) 306 | print("best epoch's test accuracy: {}".format(best_accuracy)) 307 | print("") 308 | 309 | if config.tensor_board_logging_enabled: 310 | print("Run the command line:\n") 311 | print(config.tensorboard_cmd) 312 | print("\nThen open http://0.0.0.0:6006/ into your web browser") 313 | 314 | print(config.model_desc_attched_string) 315 | 316 | 317 | if config.matplot_lib_enabled: 318 | 319 | #for i in range(config.batch_size): 320 | # indep_test_axis.append(i) 321 | #indep_test_axis = [i for i in range(config.batch_size)] 322 | #indep_test_axis = np.array(indep_test_axis) 323 | 324 | #p = PlotUtil("title", indep_test_axis, "x_label", "y_label") 325 | y_bundle = [] 326 | test_losses = np.array(test_losses) 327 | test_accuracies = np.array(test_accuracies) 328 | 329 | y = YaxisBundle(np.array(test_losses), "loss", "b") 330 | y_bundle.append(y) 331 | 332 | y = YaxisBundle(np.array(test_accuracies), "accuracy", "g") 333 | y_bundle.append(y) 334 | 335 | #p.show_plot(y_bundle) 336 | 337 | if config.matplot_lib_for_single_ybundle: 338 | if config.matplot_lib_for_accuracy: 339 | return y_bundle[1] 340 | else : 341 | return y_bundle[0] 342 | return y_bundle 343 | 344 | 345 | 346 | if __name__ == '__main__': 347 | if config.matplot_lib_enabled: 348 | indep_test_axis = [] 349 | for i in range(config.training_epochs): 350 | indep_test_axis.append(i) 351 | 352 | p = PlotUtil("Residual LSTM(3 layers) on MNIST", np.array(indep_test_axis), "Epoch iterations", "Loss or Accuracy") 353 | y_bundle = run_with_config(config) 354 | 355 | p.show_plot(y_bundle) 356 | else: 357 | run_with_config(config) 358 | 359 | -------------------------------------------------------------------------------- /single_har_bn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/praveendareddy21/Stacked_LSTMS_Highway_Residual_On_TimeSeries_Datasets/4c9e9647b4a134a2b7f63b0289687cc92262d5c8/single_har_bn.png -------------------------------------------------------------------------------- /single_layer_lstm.py: -------------------------------------------------------------------------------- 1 | from HAR_data_handler import get_HAR_data 2 | import tensorflow as tf 3 | from sklearn import metrics 4 | from sklearn.utils import shuffle 5 | import numpy as np 6 | from base_config import Config, YaxisBundle, PlotUtil 7 | 8 | 9 | 10 | def single_LSTM_cell(input_hidden_tensor, n_outputs): 11 | """ define the basic LSTM layer 12 | argument: 13 | input_hidden_tensor: list a list of tensor, 14 | shape: time_steps*[batch_size,n_inputs] 15 | n_outputs: int num of LSTM layer output 16 | return: 17 | outputs: list a time_steps list of tensor, 18 | shape: time_steps*[batch_size,n_outputs] 19 | """ 20 | with tf.variable_scope("lstm_cell"): 21 | lstm_cell = tf.contrib.rnn.BasicLSTMCell(n_outputs, state_is_tuple=True, forget_bias=0.999) 22 | #outputs, _ = tf.nn.rnn(lstm_cell, input_hidden_tensor, dtype=tf.float32) 23 | outputs, _ = tf.contrib.rnn.static_rnn(lstm_cell, input_hidden_tensor, dtype=tf.float32) 24 | return outputs 25 | 26 | 27 | 28 | 29 | def LSTM_Network(feature_mat, config): 30 | """model a LSTM Network, 31 | it stacks 2 LSTM layers, each layer has n_hidden=32 cells 32 | and 1 output layer, it is a full connet layer 33 | argument: 34 | feature_mat: ndarray fature matrix, shape=[batch_size,time_steps,n_inputs] 35 | config: class containing config of network 36 | return: 37 | : matrix output shape [batch_size,n_classes] 38 | """ 39 | # Exchange dim 1 and dim 0 40 | feature_mat = tf.transpose(feature_mat, [1, 0, 2]) 41 | # New feature_mat's shape: [time_steps, batch_size, n_inputs] 42 | 43 | # Temporarily crush the feature_mat's dimensions 44 | feature_mat = tf.reshape(feature_mat, [-1, config.n_inputs]) 45 | # New feature_mat's shape: [time_steps*batch_size, n_inputs] 46 | 47 | # Split the series because the rnn cell needs time_steps features, each of shape: 48 | hidden = tf.split(axis=0, num_or_size_splits=config.n_steps, value=feature_mat) 49 | print (len(hidden), str(hidden[0].get_shape())) 50 | 51 | outputs = single_LSTM_cell(hidden, config.n_hidden) 52 | 53 | 54 | # Get last time step's output feature for a "many to one" style classifier, 55 | # as in the image describing RNNs at the top of this page 56 | lstm_last_output = outputs[-1] 57 | 58 | # Linear activation 59 | return tf.matmul(lstm_last_output, config.W['output']) + config.biases['output'] 60 | 61 | 62 | 63 | ################################## load data and config ################################## 64 | 65 | X_train, y_train, X_test, y_test = get_HAR_data() 66 | 67 | 68 | class SingleLayerConfig(Config): 69 | """ 70 | define a class to store parameters, 71 | the input should be feature mat of training and testing 72 | """ 73 | 74 | def __init__(self): 75 | super(SingleLayerConfig, self).__init__() 76 | # Input data 77 | self.train_count = len(X_train) # 7352 training series 78 | self.test_data_count = len(X_test) # 2947 testing series 79 | self.n_steps = len(X_train[0]) # 128 time_steps per series 80 | 81 | # Trainging 82 | self.learning_rate = 0.0025 83 | self.lambda_loss_amount = 0.0015 84 | self.training_epochs = 2 85 | self.batch_size = 1500 86 | 87 | # LSTM structure 88 | self.n_inputs = len(X_train[0][0]) # == 9 Features count is of 9: three 3D sensors features over time 89 | self.n_hidden = 32 # nb of neurons inside the neural network 90 | self.n_classes = 6 # Final output classes 91 | 92 | self.model_name = "single_lstm" + "_HAR" 93 | self.log_folder_suffix = self.attach_log_suffix() 94 | self.logs_path = "/tmp/LSTM_logs/"+self.log_folder_suffix 95 | 96 | self.tensor_board_logging_enabled = True 97 | self.tensorboard_cmd = "tensorboard --logdir="+ self.logs_path 98 | self.model_desc_attched_string = self.attach_mdoel_desc() 99 | self.matplot_lib_enabled = True 100 | self.matplot_lib_for_accuracy =True 101 | self.matplot_lib_for_single_ybundle=False 102 | 103 | 104 | 105 | #config = Config(X_train, X_test) 106 | config = SingleLayerConfig() 107 | 108 | 109 | def run_with_config(config) : #, X_train, y_train, X_test, y_test): 110 | tf.reset_default_graph() # To enable to run multiple things in a loop 111 | config.print_config() 112 | 113 | if config.matplot_lib_enabled: 114 | # To keep track of training's performance 115 | test_losses = [] 116 | test_accuracies = [] 117 | indep_test_axis = [] 118 | 119 | 120 | 121 | config.W = { 122 | 'hidden': tf.Variable(tf.random_normal([config.n_inputs, config.n_hidden])), 123 | 'output': tf.Variable(tf.random_normal([config.n_hidden, config.n_classes])) 124 | } 125 | config.biases = { 126 | 'hidden': tf.Variable(tf.random_normal([config.n_hidden], mean=1.0)), 127 | 'output': tf.Variable(tf.random_normal([config.n_classes])) 128 | } 129 | #----------------------------------- 130 | # Define parameters for model 131 | #----------------------------------- 132 | 133 | 134 | print("Some useful info to get an insight on dataset's shape and normalisation:") 135 | print("features shape, labels shape, each features mean, each features standard deviation") 136 | print(X_test.shape, y_test.shape, 137 | np.mean(X_test), np.std(X_test)) 138 | print("the dataset is therefore properly normalised, as expected.") 139 | 140 | # ------------------------------------------------------ 141 | # step3: Let's get serious and build the neural network 142 | # ------------------------------------------------------ 143 | X = tf.placeholder(tf.float32, [None, config.n_steps, config.n_inputs]) 144 | Y = tf.placeholder(tf.float32, [None, config.n_classes]) 145 | 146 | pred_Y = LSTM_Network(X, config) 147 | 148 | 149 | print "Unregularised variables:" 150 | for unreg in [tf_var.name for tf_var in tf.trainable_variables() if 151 | ("noreg" in tf_var.name or "Bias" in tf_var.name)]: 152 | print unreg 153 | 154 | # Loss,optimizer,evaluation 155 | l2 = config.lambda_loss_amount * \ 156 | sum(tf.nn.l2_loss(tf_var) for tf_var in tf.trainable_variables()) 157 | # Softmax loss and L2 158 | cost = tf.reduce_mean( 159 | tf.nn.softmax_cross_entropy_with_logits(logits=pred_Y, labels=Y)) + l2 160 | optimizer = tf.train.AdamOptimizer( 161 | learning_rate=config.learning_rate).minimize(cost) 162 | 163 | correct_pred = tf.equal(tf.argmax(pred_Y, 1), tf.argmax(Y, 1)) 164 | accuracy = tf.reduce_mean(tf.cast(correct_pred, dtype=tf.float32)) 165 | # ------------------------------------------------------ 166 | # step3.5 : Tensorboard stuff here 167 | # ------------------------------------------------------ 168 | if config.tensor_board_logging_enabled: 169 | tf.summary.scalar("loss", cost) 170 | tf.summary.scalar("accuracy", accuracy) 171 | merged_summary_op = tf.summary.merge_all() 172 | 173 | # -------------------------------------------- 174 | # step4: Hooray, now train the neural network 175 | # -------------------------------------------- 176 | # Note that log_device_placement can be turned ON but will cause console spam. 177 | sess = tf.InteractiveSession(config=tf.ConfigProto(log_device_placement=False)) 178 | tf.global_variables_initializer().run() 179 | 180 | if config.tensor_board_logging_enabled: 181 | # op to write logs to Tensorboard 182 | summary_writer = tf.summary.FileWriter(config.logs_path, graph=tf.get_default_graph()) 183 | 184 | best_accuracy = 0.0 185 | # Start training for each batch and loop epochs 186 | for i in range(config.training_epochs): 187 | for start, end in zip(range(0, config.train_count, config.batch_size), 188 | range(config.batch_size, config.train_count + 1, config.batch_size)): 189 | if config.tensor_board_logging_enabled: 190 | _, summary = sess.run([optimizer, merged_summary_op], 191 | feed_dict={X: X_train[start:end], Y: y_train[start:end]}) 192 | else: 193 | sess.run(optimizer, feed_dict={X: X_train[start:end], Y: y_train[start:end]}) 194 | 195 | if config.tensor_board_logging_enabled: 196 | # Write logs at every iteration 197 | summary_writer.add_summary(summary, i) 198 | 199 | # Test completely at every epoch: calculate accuracy 200 | pred_out, accuracy_out, loss_out = sess.run([pred_Y, accuracy, cost], feed_dict={ 201 | X: X_test, Y: y_test}) 202 | 203 | if config.matplot_lib_enabled: 204 | indep_test_axis.append(i) 205 | test_losses.append(loss_out) 206 | test_accuracies.append(accuracy_out) 207 | 208 | print("traing iter: {},".format(i) + \ 209 | " test accuracy : {},".format(accuracy_out) + \ 210 | " loss : {}".format(loss_out)) 211 | best_accuracy = max(best_accuracy, accuracy_out) 212 | 213 | print("") 214 | print("final test accuracy: {}".format(accuracy_out)) 215 | print("best epoch's test accuracy: {}".format(best_accuracy)) 216 | print("") 217 | 218 | if config.tensor_board_logging_enabled: 219 | print("Run the command line:\n") 220 | print(config.tensorboard_cmd) 221 | print("\nThen open http://0.0.0.0:6006/ into your web browser") 222 | 223 | print(config.model_desc_attched_string) 224 | 225 | 226 | if config.matplot_lib_enabled: 227 | 228 | #for i in range(config.batch_size): 229 | # indep_test_axis.append(i) 230 | #indep_test_axis = [i for i in range(config.batch_size)] 231 | #indep_test_axis = np.array(indep_test_axis) 232 | 233 | #p = PlotUtil("title", indep_test_axis, "x_label", "y_label") 234 | y_bundle = [] 235 | test_losses = np.array(test_losses) 236 | test_accuracies = np.array(test_accuracies) 237 | 238 | y = YaxisBundle(np.array(test_losses), "loss", "b") 239 | y_bundle.append(y) 240 | 241 | y = YaxisBundle(np.array(test_accuracies), "accuracy", "g") 242 | y_bundle.append(y) 243 | 244 | #p.show_plot(y_bundle) 245 | 246 | if config.matplot_lib_for_single_ybundle: 247 | if config.matplot_lib_for_accuracy: 248 | return y_bundle[1] 249 | else : 250 | return y_bundle[0] 251 | return y_bundle 252 | 253 | 254 | 255 | if __name__ == '__main__': 256 | if config.matplot_lib_enabled: 257 | indep_test_axis = [] 258 | for i in range(config.training_epochs): 259 | indep_test_axis.append(i) 260 | 261 | p = PlotUtil("title", np.array(indep_test_axis), "x_label", "y_label") 262 | y_bundle = run_with_config(config) 263 | 264 | p.show_plot(y_bundle) 265 | else: 266 | run_with_config(config) 267 | 268 | 269 | -------------------------------------------------------------------------------- /single_layer_lstm_MNIST.py: -------------------------------------------------------------------------------- 1 | from HAR_data_handler import get_HAR_data 2 | import tensorflow as tf 3 | from sklearn import metrics 4 | from sklearn.utils import shuffle 5 | import numpy as np 6 | from base_config import Config, YaxisBundle, PlotUtil 7 | from mnist_data_handler import get_mnist_data_with_time_series_size 8 | 9 | 10 | def single_LSTM_cell(input_hidden_tensor, n_outputs): 11 | """ define the basic LSTM layer 12 | argument: 13 | input_hidden_tensor: list a list of tensor, 14 | shape: time_steps*[batch_size,n_inputs] 15 | n_outputs: int num of LSTM layer output 16 | return: 17 | outputs: list a time_steps list of tensor, 18 | shape: time_steps*[batch_size,n_outputs] 19 | """ 20 | with tf.variable_scope("lstm_cell"): 21 | lstm_cell = tf.contrib.rnn.BasicLSTMCell(n_outputs, state_is_tuple=True, forget_bias=0.999) 22 | #outputs, _ = tf.nn.rnn(lstm_cell, input_hidden_tensor, dtype=tf.float32) 23 | outputs, _ = tf.contrib.rnn.static_rnn(lstm_cell, input_hidden_tensor, dtype=tf.float32) 24 | return outputs 25 | 26 | 27 | 28 | 29 | def LSTM_Network(feature_mat, config): 30 | """model a LSTM Network, 31 | it stacks 2 LSTM layers, each layer has n_hidden=32 cells 32 | and 1 output layer, it is a full connet layer 33 | argument: 34 | feature_mat: ndarray fature matrix, shape=[batch_size,time_steps,n_inputs] 35 | config: class containing config of network 36 | return: 37 | : matrix output shape [batch_size,n_classes] 38 | """ 39 | # Exchange dim 1 and dim 0 40 | feature_mat = tf.transpose(feature_mat, [1, 0, 2]) 41 | # New feature_mat's shape: [time_steps, batch_size, n_inputs] 42 | 43 | # Temporarily crush the feature_mat's dimensions 44 | feature_mat = tf.reshape(feature_mat, [-1, config.n_inputs]) 45 | # New feature_mat's shape: [time_steps*batch_size, n_inputs] 46 | 47 | # Split the series because the rnn cell needs time_steps features, each of shape: 48 | hidden = tf.split(axis=0, num_or_size_splits=config.n_steps, value=feature_mat) 49 | print (len(hidden), str(hidden[0].get_shape())) 50 | 51 | outputs = single_LSTM_cell(hidden, config.n_hidden) 52 | 53 | 54 | # Get last time step's output feature for a "many to one" style classifier, 55 | # as in the image describing RNNs at the top of this page 56 | lstm_last_output = outputs[-1] 57 | 58 | # Linear activation 59 | return tf.matmul(lstm_last_output, config.W['output']) + config.biases['output'] 60 | 61 | 62 | 63 | ################################## load data and config ################################## 64 | 65 | X_train, y_train, X_test, y_test, series_size = get_mnist_data_with_time_series_size() 66 | 67 | 68 | class SingleLayerConfig(Config): 69 | """ 70 | define a class to store parameters, 71 | the input should be feature mat of training and testing 72 | """ 73 | 74 | def __init__(self): 75 | super(SingleLayerConfig, self).__init__() 76 | # Input data 77 | self.train_count = len(X_train) # 7352 training series 78 | self.test_data_count = len(X_test) # 2947 testing series 79 | self.n_steps = len(X_train[0]) # 128 time_steps per series 80 | 81 | # Trainging 82 | self.learning_rate = 0.005 83 | self.lambda_loss_amount = 0.0015 84 | self.training_epochs = 100 85 | self.batch_size = 1500 86 | 87 | # LSTM structure 88 | self.n_inputs = len(X_train[0][0]) 89 | self.n_hidden = 28 # nb of neurons inside the neural network 90 | self.n_classes = len(y_train[0]) # Final output classes 91 | 92 | 93 | self.model_name = "single_lstm_" + "MNIST" 94 | self.log_folder_suffix = self.attach_log_suffix() 95 | self.logs_path = "/tmp/LSTM_logs/"+self.log_folder_suffix 96 | 97 | self.tensor_board_logging_enabled = True 98 | self.tensorboard_cmd = "tensorboard --logdir="+ self.logs_path 99 | self.model_desc_attched_string = self.attach_mdoel_desc() 100 | self.matplot_lib_enabled = True 101 | self.matplot_lib_for_accuracy =True 102 | self.matplot_lib_for_single_ybundle=False 103 | 104 | 105 | 106 | #config = Config(X_train, X_test) 107 | config = SingleLayerConfig() 108 | 109 | 110 | def run_with_config(config) : #, X_train, y_train, X_test, y_test): 111 | tf.reset_default_graph() # To enable to run multiple things in a loop 112 | config.print_config() 113 | 114 | if config.matplot_lib_enabled: 115 | # To keep track of training's performance 116 | test_losses = [] 117 | test_accuracies = [] 118 | indep_test_axis = [] 119 | 120 | 121 | 122 | config.W = { 123 | 'hidden': tf.Variable(tf.random_normal([config.n_inputs, config.n_hidden])), 124 | 'output': tf.Variable(tf.random_normal([config.n_hidden, config.n_classes])) 125 | } 126 | config.biases = { 127 | 'hidden': tf.Variable(tf.random_normal([config.n_hidden], mean=1.0)), 128 | 'output': tf.Variable(tf.random_normal([config.n_classes])) 129 | } 130 | #----------------------------------- 131 | # Define parameters for model 132 | #----------------------------------- 133 | 134 | 135 | print("Some useful info to get an insight on dataset's shape and normalisation:") 136 | print("features shape, labels shape, each features mean, each features standard deviation") 137 | print(X_test.shape, y_test.shape, 138 | np.mean(X_test), np.std(X_test)) 139 | print("the dataset is therefore properly normalised, as expected.") 140 | 141 | # ------------------------------------------------------ 142 | # step3: Let's get serious and build the neural network 143 | # ------------------------------------------------------ 144 | X = tf.placeholder(tf.float32, [None, config.n_steps, config.n_inputs]) 145 | Y = tf.placeholder(tf.float32, [None, config.n_classes]) 146 | 147 | pred_Y = LSTM_Network(X, config) 148 | 149 | 150 | print "Unregularised variables:" 151 | for unreg in [tf_var.name for tf_var in tf.trainable_variables() if 152 | ("noreg" in tf_var.name or "Bias" in tf_var.name)]: 153 | print unreg 154 | 155 | # Loss,optimizer,evaluation 156 | l2 = config.lambda_loss_amount * \ 157 | sum(tf.nn.l2_loss(tf_var) for tf_var in tf.trainable_variables()) 158 | # Softmax loss and L2 159 | cost = tf.reduce_mean( 160 | tf.nn.softmax_cross_entropy_with_logits(logits=pred_Y, labels=Y)) + l2 161 | optimizer = tf.train.AdamOptimizer( 162 | learning_rate=config.learning_rate).minimize(cost) 163 | 164 | correct_pred = tf.equal(tf.argmax(pred_Y, 1), tf.argmax(Y, 1)) 165 | accuracy = tf.reduce_mean(tf.cast(correct_pred, dtype=tf.float32)) 166 | # ------------------------------------------------------ 167 | # step3.5 : Tensorboard stuff here 168 | # ------------------------------------------------------ 169 | if config.tensor_board_logging_enabled: 170 | tf.summary.scalar("loss", cost) 171 | tf.summary.scalar("accuracy", accuracy) 172 | merged_summary_op = tf.summary.merge_all() 173 | 174 | # -------------------------------------------- 175 | # step4: Hooray, now train the neural network 176 | # -------------------------------------------- 177 | # Note that log_device_placement can be turned ON but will cause console spam. 178 | sess = tf.InteractiveSession(config=tf.ConfigProto(log_device_placement=False)) 179 | tf.global_variables_initializer().run() 180 | 181 | if config.tensor_board_logging_enabled: 182 | # op to write logs to Tensorboard 183 | summary_writer = tf.summary.FileWriter(config.logs_path, graph=tf.get_default_graph()) 184 | 185 | best_accuracy = 0.0 186 | # Start training for each batch and loop epochs 187 | for i in range(config.training_epochs): 188 | for start, end in zip(range(0, config.train_count, config.batch_size), 189 | range(config.batch_size, config.train_count + 1, config.batch_size)): 190 | if config.tensor_board_logging_enabled: 191 | _, summary = sess.run([optimizer, merged_summary_op], 192 | feed_dict={X: X_train[start:end], Y: y_train[start:end]}) 193 | else: 194 | sess.run(optimizer, feed_dict={X: X_train[start:end], Y: y_train[start:end]}) 195 | 196 | if config.tensor_board_logging_enabled: 197 | # Write logs at every iteration 198 | summary_writer.add_summary(summary, i) 199 | 200 | # Test completely at every epoch: calculate accuracy 201 | pred_out, accuracy_out, loss_out = sess.run([pred_Y, accuracy, cost], feed_dict={ 202 | X: X_test, Y: y_test}) 203 | 204 | if config.matplot_lib_enabled: 205 | indep_test_axis.append(i) 206 | test_losses.append(loss_out) 207 | test_accuracies.append(accuracy_out) 208 | 209 | print("traing iter: {},".format(i) + \ 210 | " test accuracy : {},".format(accuracy_out) + \ 211 | " loss : {}".format(loss_out)) 212 | best_accuracy = max(best_accuracy, accuracy_out) 213 | 214 | print("") 215 | print("final test accuracy: {}".format(accuracy_out)) 216 | print("best epoch's test accuracy: {}".format(best_accuracy)) 217 | print("") 218 | 219 | if config.tensor_board_logging_enabled: 220 | print("Run the command line:\n") 221 | print(config.tensorboard_cmd) 222 | print("\nThen open http://0.0.0.0:6006/ into your web browser") 223 | 224 | print(config.model_desc_attched_string) 225 | 226 | 227 | if config.matplot_lib_enabled: 228 | 229 | #for i in range(config.batch_size): 230 | # indep_test_axis.append(i) 231 | #indep_test_axis = [i for i in range(config.batch_size)] 232 | #indep_test_axis = np.array(indep_test_axis) 233 | 234 | #p = PlotUtil("title", indep_test_axis, "x_label", "y_label") 235 | y_bundle = [] 236 | test_losses = np.array(test_losses) 237 | test_accuracies = np.array(test_accuracies) 238 | 239 | y = YaxisBundle(np.array(test_losses), "loss", "b") 240 | y_bundle.append(y) 241 | 242 | y = YaxisBundle(np.array(test_accuracies), "accuracy", "g") 243 | y_bundle.append(y) 244 | 245 | #p.show_plot(y_bundle) 246 | 247 | if config.matplot_lib_for_single_ybundle: 248 | if config.matplot_lib_for_accuracy: 249 | return y_bundle[1] 250 | else : 251 | return y_bundle[0] 252 | return y_bundle 253 | 254 | 255 | 256 | if __name__ == '__main__': 257 | if config.matplot_lib_enabled: 258 | indep_test_axis = [] 259 | for i in range(config.training_epochs): 260 | indep_test_axis.append(i) 261 | 262 | p = PlotUtil("title", np.array(indep_test_axis), "x_label", "y_label") 263 | y_bundle = run_with_config(config) 264 | 265 | p.show_plot(y_bundle) 266 | else: 267 | run_with_config(config) 268 | 269 | 270 | -------------------------------------------------------------------------------- /single_layer_lstm_UCR_dataset.py: -------------------------------------------------------------------------------- 1 | from HAR_data_handler import get_HAR_data 2 | import tensorflow as tf 3 | from sklearn import metrics 4 | from sklearn.utils import shuffle 5 | import numpy as np 6 | from base_config import Config, YaxisBundle, PlotUtil 7 | from UCR_time_series_data_handler import get_dataset_with_series_size 8 | 9 | 10 | 11 | def single_LSTM_cell(input_hidden_tensor, n_outputs): 12 | """ define the basic LSTM layer 13 | argument: 14 | input_hidden_tensor: list a list of tensor, 15 | shape: time_steps*[batch_size,n_inputs] 16 | n_outputs: int num of LSTM layer output 17 | return: 18 | outputs: list a time_steps list of tensor, 19 | shape: time_steps*[batch_size,n_outputs] 20 | """ 21 | with tf.variable_scope("lstm_cell"): 22 | lstm_cell = tf.contrib.rnn.BasicLSTMCell(n_outputs, state_is_tuple=True, forget_bias=0.999) 23 | #outputs, _ = tf.nn.rnn(lstm_cell, input_hidden_tensor, dtype=tf.float32) 24 | outputs, _ = tf.contrib.rnn.static_rnn(lstm_cell, input_hidden_tensor, dtype=tf.float32) 25 | return outputs 26 | 27 | 28 | 29 | 30 | def LSTM_Network(feature_mat, config): 31 | """model a LSTM Network, 32 | it stacks 2 LSTM layers, each layer has n_hidden=32 cells 33 | and 1 output layer, it is a full connet layer 34 | argument: 35 | feature_mat: ndarray fature matrix, shape=[batch_size,time_steps,n_inputs] 36 | config: class containing config of network 37 | return: 38 | : matrix output shape [batch_size,n_classes] 39 | """ 40 | # Exchange dim 1 and dim 0 41 | feature_mat = tf.transpose(feature_mat, [1, 0, 2]) 42 | # New feature_mat's shape: [time_steps, batch_size, n_inputs] 43 | 44 | # Temporarily crush the feature_mat's dimensions 45 | feature_mat = tf.reshape(feature_mat, [-1, config.n_inputs]) 46 | # New feature_mat's shape: [time_steps*batch_size, n_inputs] 47 | 48 | # Split the series because the rnn cell needs time_steps features, each of shape: 49 | hidden = tf.split(axis=0, num_or_size_splits=config.n_steps, value=feature_mat) 50 | print (len(hidden), str(hidden[0].get_shape())) 51 | 52 | outputs = single_LSTM_cell(hidden, config.n_hidden) 53 | 54 | 55 | # Get last time step's output feature for a "many to one" style classifier, 56 | # as in the image describing RNNs at the top of this page 57 | lstm_last_output = outputs[-1] 58 | 59 | # Linear activation 60 | return tf.matmul(lstm_last_output, config.W['output']) + config.biases['output'] 61 | 62 | def one_hot_to_int(y): 63 | """convert label from dense to one hot 64 | argument: 65 | label: ndarray dense label ,shape: [sample_num,1] 66 | return: 67 | one_hot_label: ndarray one hot, shape: [sample_num,n_class] 68 | """ 69 | # e.g.: [[5], [0], [3]] --> [[0, 0, 0, 0, 0, 1], [1, 0, 0, 0, 0, 0], [0, 0, 0, 1, 0, 0]] 70 | 71 | y = y.reshape(len(y)) 72 | n_values = int(np.max(y)) + 1 73 | return np.eye(n_values)[np.array(y, dtype=np.int32)] # Returns FLOATS 74 | 75 | ################################## load data and config ################################## 76 | X_train, y_train, X_test, y_test, series_size = get_dataset_with_series_size(dataset='ElectricDevices') 77 | X_train = X_train.reshape((-1, series_size, 1)) 78 | X_test = X_test.reshape((-1, series_size, 1)) 79 | y_train = y_train.reshape((-1, 1)) 80 | y_test = y_test.reshape((-1, 1)) 81 | y_train = one_hot_to_int(y_train) 82 | y_test = one_hot_to_int(y_test) 83 | 84 | 85 | class SingleLayerConfig(Config): 86 | """ 87 | define a class to store parameters, 88 | the input should be feature mat of training and testing 89 | """ 90 | 91 | def __init__(self): 92 | super(SingleLayerConfig, self).__init__() 93 | # Input data 94 | self.train_count = len(X_train) # 7352 training series 95 | self.test_data_count = len(X_test) # 2947 testing series 96 | self.n_steps = len(X_train[0]) # 128 time_steps per series 97 | 98 | # Trainging 99 | self.learning_rate = 0.005 100 | self.lambda_loss_amount = 0.0015 101 | self.training_epochs = 100 102 | self.batch_size = 1500 103 | 104 | # LSTM structure 105 | self.n_inputs = len(X_train[0][0]) 106 | self.n_hidden = 28 # nb of neurons inside the neural network 107 | self.n_classes = len(y_train[0]) # Final output classes 108 | 109 | 110 | self.model_name = "single_lstm_" + "ElectricDevices" 111 | self.log_folder_suffix = self.attach_log_suffix() 112 | self.logs_path = "/tmp/LSTM_logs/"+self.log_folder_suffix 113 | 114 | self.tensor_board_logging_enabled = True 115 | self.tensorboard_cmd = "tensorboard --logdir="+ self.logs_path 116 | self.model_desc_attched_string = self.attach_mdoel_desc() 117 | self.matplot_lib_enabled = True 118 | self.matplot_lib_for_accuracy =True 119 | self.matplot_lib_for_single_ybundle=False 120 | 121 | 122 | 123 | #config = Config(X_train, X_test) 124 | config = SingleLayerConfig() 125 | 126 | 127 | def run_with_config(config) : #, X_train, y_train, X_test, y_test): 128 | tf.reset_default_graph() # To enable to run multiple things in a loop 129 | config.print_config() 130 | 131 | if config.matplot_lib_enabled: 132 | # To keep track of training's performance 133 | test_losses = [] 134 | test_accuracies = [] 135 | indep_test_axis = [] 136 | 137 | 138 | 139 | config.W = { 140 | 'hidden': tf.Variable(tf.random_normal([config.n_inputs, config.n_hidden])), 141 | 'output': tf.Variable(tf.random_normal([config.n_hidden, config.n_classes])) 142 | } 143 | config.biases = { 144 | 'hidden': tf.Variable(tf.random_normal([config.n_hidden], mean=1.0)), 145 | 'output': tf.Variable(tf.random_normal([config.n_classes])) 146 | } 147 | #----------------------------------- 148 | # Define parameters for model 149 | #----------------------------------- 150 | 151 | 152 | print("Some useful info to get an insight on dataset's shape and normalisation:") 153 | print("features shape, labels shape, each features mean, each features standard deviation") 154 | print(X_test.shape, y_test.shape, 155 | np.mean(X_test), np.std(X_test)) 156 | print("the dataset is therefore properly normalised, as expected.") 157 | 158 | # ------------------------------------------------------ 159 | # step3: Let's get serious and build the neural network 160 | # ------------------------------------------------------ 161 | X = tf.placeholder(tf.float32, [None, config.n_steps, config.n_inputs]) 162 | Y = tf.placeholder(tf.float32, [None, config.n_classes]) 163 | 164 | pred_Y = LSTM_Network(X, config) 165 | 166 | 167 | print "Unregularised variables:" 168 | for unreg in [tf_var.name for tf_var in tf.trainable_variables() if 169 | ("noreg" in tf_var.name or "Bias" in tf_var.name)]: 170 | print unreg 171 | 172 | # Loss,optimizer,evaluation 173 | l2 = config.lambda_loss_amount * \ 174 | sum(tf.nn.l2_loss(tf_var) for tf_var in tf.trainable_variables()) 175 | # Softmax loss and L2 176 | cost = tf.reduce_mean( 177 | tf.nn.softmax_cross_entropy_with_logits(logits=pred_Y, labels=Y)) + l2 178 | optimizer = tf.train.AdamOptimizer( 179 | learning_rate=config.learning_rate).minimize(cost) 180 | 181 | correct_pred = tf.equal(tf.argmax(pred_Y, 1), tf.argmax(Y, 1)) 182 | accuracy = tf.reduce_mean(tf.cast(correct_pred, dtype=tf.float32)) 183 | # ------------------------------------------------------ 184 | # step3.5 : Tensorboard stuff here 185 | # ------------------------------------------------------ 186 | if config.tensor_board_logging_enabled: 187 | tf.summary.scalar("loss", cost) 188 | tf.summary.scalar("accuracy", accuracy) 189 | merged_summary_op = tf.summary.merge_all() 190 | 191 | # -------------------------------------------- 192 | # step4: Hooray, now train the neural network 193 | # -------------------------------------------- 194 | # Note that log_device_placement can be turned ON but will cause console spam. 195 | sess = tf.InteractiveSession(config=tf.ConfigProto(log_device_placement=False)) 196 | tf.global_variables_initializer().run() 197 | 198 | if config.tensor_board_logging_enabled: 199 | # op to write logs to Tensorboard 200 | summary_writer = tf.summary.FileWriter(config.logs_path, graph=tf.get_default_graph()) 201 | 202 | best_accuracy = 0.0 203 | # Start training for each batch and loop epochs 204 | for i in range(config.training_epochs): 205 | for start, end in zip(range(0, config.train_count, config.batch_size), 206 | range(config.batch_size, config.train_count + 1, config.batch_size)): 207 | if config.tensor_board_logging_enabled: 208 | _, summary = sess.run([optimizer, merged_summary_op], 209 | feed_dict={X: X_train[start:end], Y: y_train[start:end]}) 210 | else: 211 | sess.run(optimizer, feed_dict={X: X_train[start:end], Y: y_train[start:end]}) 212 | 213 | if config.tensor_board_logging_enabled: 214 | # Write logs at every iteration 215 | summary_writer.add_summary(summary, i) 216 | 217 | # Test completely at every epoch: calculate accuracy 218 | pred_out, accuracy_out, loss_out = sess.run([pred_Y, accuracy, cost], feed_dict={ 219 | X: X_test, Y: y_test}) 220 | 221 | if config.matplot_lib_enabled: 222 | indep_test_axis.append(i) 223 | test_losses.append(loss_out) 224 | test_accuracies.append(accuracy_out) 225 | 226 | print("traing iter: {},".format(i) + \ 227 | " test accuracy : {},".format(accuracy_out) + \ 228 | " loss : {}".format(loss_out)) 229 | best_accuracy = max(best_accuracy, accuracy_out) 230 | 231 | print("") 232 | print("final test accuracy: {}".format(accuracy_out)) 233 | print("best epoch's test accuracy: {}".format(best_accuracy)) 234 | print("") 235 | 236 | if config.tensor_board_logging_enabled: 237 | print("Run the command line:\n") 238 | print(config.tensorboard_cmd) 239 | print("\nThen open http://0.0.0.0:6006/ into your web browser") 240 | 241 | print(config.model_desc_attched_string) 242 | 243 | 244 | if config.matplot_lib_enabled: 245 | 246 | #for i in range(config.batch_size): 247 | # indep_test_axis.append(i) 248 | #indep_test_axis = [i for i in range(config.batch_size)] 249 | #indep_test_axis = np.array(indep_test_axis) 250 | 251 | #p = PlotUtil("title", indep_test_axis, "x_label", "y_label") 252 | y_bundle = [] 253 | test_losses = np.array(test_losses) 254 | test_accuracies = np.array(test_accuracies) 255 | 256 | y = YaxisBundle(np.array(test_losses), "loss", "b") 257 | y_bundle.append(y) 258 | 259 | y = YaxisBundle(np.array(test_accuracies), "accuracy", "g") 260 | y_bundle.append(y) 261 | 262 | #p.show_plot(y_bundle) 263 | 264 | if config.matplot_lib_for_single_ybundle: 265 | if config.matplot_lib_for_accuracy: 266 | return y_bundle[1] 267 | else : 268 | return y_bundle[0] 269 | return y_bundle 270 | 271 | 272 | 273 | if __name__ == '__main__': 274 | if config.matplot_lib_enabled: 275 | indep_test_axis = [] 276 | for i in range(config.training_epochs): 277 | indep_test_axis.append(i) 278 | 279 | p = PlotUtil("title", np.array(indep_test_axis), "x_label", "y_label") 280 | y_bundle = run_with_config(config) 281 | 282 | p.show_plot(y_bundle) 283 | else: 284 | run_with_config(config) 285 | 286 | 287 | -------------------------------------------------------------------------------- /single_layer_lstm_looper_UCR.py: -------------------------------------------------------------------------------- 1 | from HAR_data_handler import get_HAR_data 2 | import tensorflow as tf 3 | from sklearn import metrics 4 | from sklearn.utils import shuffle 5 | import numpy as np 6 | from base_config import Config, YaxisBundle, PlotUtil 7 | from UCR_time_series_data_handler import get_dataset_with_series_size 8 | 9 | 10 | 11 | def single_LSTM_cell(input_hidden_tensor, n_outputs): 12 | """ define the basic LSTM layer 13 | argument: 14 | input_hidden_tensor: list a list of tensor, 15 | shape: time_steps*[batch_size,n_inputs] 16 | n_outputs: int num of LSTM layer output 17 | return: 18 | outputs: list a time_steps list of tensor, 19 | shape: time_steps*[batch_size,n_outputs] 20 | """ 21 | with tf.variable_scope("lstm_cell"): 22 | lstm_cell = tf.contrib.rnn.BasicLSTMCell(n_outputs, state_is_tuple=True, forget_bias=0.999) 23 | #outputs, _ = tf.nn.rnn(lstm_cell, input_hidden_tensor, dtype=tf.float32) 24 | outputs, _ = tf.contrib.rnn.static_rnn(lstm_cell, input_hidden_tensor, dtype=tf.float32) 25 | return outputs 26 | 27 | 28 | 29 | 30 | def LSTM_Network(feature_mat, config): 31 | """model a LSTM Network, 32 | it stacks 2 LSTM layers, each layer has n_hidden=32 cells 33 | and 1 output layer, it is a full connet layer 34 | argument: 35 | feature_mat: ndarray fature matrix, shape=[batch_size,time_steps,n_inputs] 36 | config: class containing config of network 37 | return: 38 | : matrix output shape [batch_size,n_classes] 39 | """ 40 | # Exchange dim 1 and dim 0 41 | feature_mat = tf.transpose(feature_mat, [1, 0, 2]) 42 | # New feature_mat's shape: [time_steps, batch_size, n_inputs] 43 | 44 | # Temporarily crush the feature_mat's dimensions 45 | feature_mat = tf.reshape(feature_mat, [-1, config.n_inputs]) 46 | # New feature_mat's shape: [time_steps*batch_size, n_inputs] 47 | 48 | # Split the series because the rnn cell needs time_steps features, each of shape: 49 | hidden = tf.split(axis=0, num_or_size_splits=config.n_steps, value=feature_mat) 50 | print (len(hidden), str(hidden[0].get_shape())) 51 | 52 | outputs = single_LSTM_cell(hidden, config.n_hidden) 53 | 54 | 55 | # Get last time step's output feature for a "many to one" style classifier, 56 | # as in the image describing RNNs at the top of this page 57 | lstm_last_output = outputs[-1] 58 | 59 | # Linear activation 60 | return tf.matmul(lstm_last_output, config.W['output']) + config.biases['output'] 61 | 62 | def one_hot_to_int(y): 63 | """convert label from dense to one hot 64 | argument: 65 | label: ndarray dense label ,shape: [sample_num,1] 66 | return: 67 | one_hot_label: ndarray one hot, shape: [sample_num,n_class] 68 | """ 69 | # e.g.: [[5], [0], [3]] --> [[0, 0, 0, 0, 0, 1], [1, 0, 0, 0, 0, 0], [0, 0, 0, 1, 0, 0]] 70 | 71 | y = y.reshape(len(y)) 72 | n_values = int(np.max(y)) + 1 73 | return np.eye(n_values)[np.array(y, dtype=np.int32)] # Returns FLOATS 74 | 75 | ################################## load data and config ################################## 76 | 77 | 78 | 79 | class SingleLayerConfig(Config): 80 | """ 81 | define a class to store parameters, 82 | the input should be feature mat of training and testing 83 | """ 84 | 85 | def __init__(self, UCR_dataset_name ): 86 | super(SingleLayerConfig, self).__init__() 87 | 88 | X_train, y_train, X_test, y_test, series_size = get_dataset_with_series_size(dataset=UCR_dataset_name) 89 | self.X_train = X_train.reshape((-1, series_size, 1)) 90 | self.X_test = X_test.reshape((-1, series_size, 1)) 91 | y_train = y_train.reshape((-1, 1)) 92 | y_test = y_test.reshape((-1, 1)) 93 | self.y_train = one_hot_to_int(y_train) 94 | self.y_test = one_hot_to_int(y_test) 95 | 96 | 97 | 98 | 99 | # Input data 100 | self.train_count = len(self.X_train) # 7352 training series 101 | self.test_data_count = len(self.X_test) # 2947 testing series 102 | self.n_steps = len(self.X_train[0]) # 128 time_steps per series 103 | 104 | # Trainging 105 | self.learning_rate = 0.005 106 | self.lambda_loss_amount = 0.0015 107 | self.training_epochs = 100 108 | self.batch_size = 1500 109 | 110 | # LSTM structure 111 | self.n_inputs = len(self.X_train[0][0]) 112 | self.n_hidden = 28 # nb of neurons inside the neural network 113 | self.n_classes = len(self.y_train[0]) # Final output classes 114 | 115 | 116 | self.model_name = "single_lstm_" + UCR_dataset_name 117 | self.log_folder_suffix = self.attach_log_suffix() 118 | self.logs_path = "/tmp/LSTM_logs/"+self.log_folder_suffix 119 | 120 | self.tensor_board_logging_enabled = True 121 | self.tensorboard_cmd = "tensorboard --logdir="+ self.logs_path 122 | self.model_desc_attched_string = self.attach_mdoel_desc() 123 | self.matplot_lib_enabled = True 124 | self.matplot_lib_for_accuracy =True 125 | self.matplot_lib_for_single_ybundle=False 126 | 127 | 128 | 129 | #config = Config(X_train, X_test) 130 | config = SingleLayerConfig('ElectricDevices') 131 | 132 | 133 | def run_with_config(config) : #, X_train, y_train, X_test, y_test): 134 | tf.reset_default_graph() # To enable to run multiple things in a loop 135 | config.print_config() 136 | 137 | X_train = config.X_train 138 | X_test = config.X_test 139 | 140 | y_train = config.y_train 141 | y_test = config.y_test 142 | 143 | if config.matplot_lib_enabled: 144 | # To keep track of training's performance 145 | test_losses = [] 146 | test_accuracies = [] 147 | indep_test_axis = [] 148 | 149 | 150 | 151 | config.W = { 152 | 'hidden': tf.Variable(tf.random_normal([config.n_inputs, config.n_hidden])), 153 | 'output': tf.Variable(tf.random_normal([config.n_hidden, config.n_classes])) 154 | } 155 | config.biases = { 156 | 'hidden': tf.Variable(tf.random_normal([config.n_hidden], mean=1.0)), 157 | 'output': tf.Variable(tf.random_normal([config.n_classes])) 158 | } 159 | #----------------------------------- 160 | # Define parameters for model 161 | #----------------------------------- 162 | 163 | 164 | print("Some useful info to get an insight on dataset's shape and normalisation:") 165 | print("features shape, labels shape, each features mean, each features standard deviation") 166 | print(X_test.shape, y_test.shape, 167 | np.mean(X_test), np.std(X_test)) 168 | print("the dataset is therefore properly normalised, as expected.") 169 | 170 | # ------------------------------------------------------ 171 | # step3: Let's get serious and build the neural network 172 | # ------------------------------------------------------ 173 | X = tf.placeholder(tf.float32, [None, config.n_steps, config.n_inputs]) 174 | Y = tf.placeholder(tf.float32, [None, config.n_classes]) 175 | 176 | pred_Y = LSTM_Network(X, config) 177 | 178 | 179 | print "Unregularised variables:" 180 | for unreg in [tf_var.name for tf_var in tf.trainable_variables() if 181 | ("noreg" in tf_var.name or "Bias" in tf_var.name)]: 182 | print unreg 183 | 184 | # Loss,optimizer,evaluation 185 | l2 = config.lambda_loss_amount * \ 186 | sum(tf.nn.l2_loss(tf_var) for tf_var in tf.trainable_variables()) 187 | # Softmax loss and L2 188 | cost = tf.reduce_mean( 189 | tf.nn.softmax_cross_entropy_with_logits(logits=pred_Y, labels=Y)) + l2 190 | optimizer = tf.train.AdamOptimizer( 191 | learning_rate=config.learning_rate).minimize(cost) 192 | 193 | correct_pred = tf.equal(tf.argmax(pred_Y, 1), tf.argmax(Y, 1)) 194 | accuracy = tf.reduce_mean(tf.cast(correct_pred, dtype=tf.float32)) 195 | # ------------------------------------------------------ 196 | # step3.5 : Tensorboard stuff here 197 | # ------------------------------------------------------ 198 | if config.tensor_board_logging_enabled: 199 | tf.summary.scalar("loss", cost) 200 | tf.summary.scalar("accuracy", accuracy) 201 | merged_summary_op = tf.summary.merge_all() 202 | 203 | # -------------------------------------------- 204 | # step4: Hooray, now train the neural network 205 | # -------------------------------------------- 206 | # Note that log_device_placement can be turned ON but will cause console spam. 207 | sess = tf.InteractiveSession(config=tf.ConfigProto(log_device_placement=False)) 208 | tf.global_variables_initializer().run() 209 | 210 | if config.tensor_board_logging_enabled: 211 | # op to write logs to Tensorboard 212 | summary_writer = tf.summary.FileWriter(config.logs_path, graph=tf.get_default_graph()) 213 | 214 | best_accuracy = 0.0 215 | # Start training for each batch and loop epochs 216 | for i in range(config.training_epochs): 217 | for start, end in zip(range(0, config.train_count, config.batch_size), 218 | range(config.batch_size, config.train_count + 1, config.batch_size)): 219 | if config.tensor_board_logging_enabled: 220 | _, summary = sess.run([optimizer, merged_summary_op], 221 | feed_dict={X: X_train[start:end], Y: y_train[start:end]}) 222 | else: 223 | sess.run(optimizer, feed_dict={X: X_train[start:end], Y: y_train[start:end]}) 224 | 225 | if config.tensor_board_logging_enabled: 226 | # Write logs at every iteration 227 | summary_writer.add_summary(summary, i) 228 | 229 | # Test completely at every epoch: calculate accuracy 230 | pred_out, accuracy_out, loss_out = sess.run([pred_Y, accuracy, cost], feed_dict={ 231 | X: X_test, Y: y_test}) 232 | 233 | if config.matplot_lib_enabled: 234 | indep_test_axis.append(i) 235 | test_losses.append(loss_out) 236 | test_accuracies.append(accuracy_out) 237 | 238 | print("traing iter: {},".format(i) + \ 239 | " test accuracy : {},".format(accuracy_out) + \ 240 | " loss : {}".format(loss_out)) 241 | best_accuracy = max(best_accuracy, accuracy_out) 242 | 243 | print("") 244 | print("final test accuracy: {}".format(accuracy_out)) 245 | print("best epoch's test accuracy: {}".format(best_accuracy)) 246 | print("") 247 | 248 | if config.tensor_board_logging_enabled: 249 | print("Run the command line:\n") 250 | print(config.tensorboard_cmd) 251 | print("\nThen open http://0.0.0.0:6006/ into your web browser") 252 | 253 | print(config.model_desc_attched_string) 254 | 255 | 256 | if config.matplot_lib_enabled: 257 | 258 | #for i in range(config.batch_size): 259 | # indep_test_axis.append(i) 260 | #indep_test_axis = [i for i in range(config.batch_size)] 261 | #indep_test_axis = np.array(indep_test_axis) 262 | 263 | #p = PlotUtil("title", indep_test_axis, "x_label", "y_label") 264 | y_bundle = [] 265 | test_losses = np.array(test_losses) 266 | test_accuracies = np.array(test_accuracies) 267 | 268 | y = YaxisBundle(np.array(test_losses), "loss", "b") 269 | y_bundle.append(y) 270 | 271 | y = YaxisBundle(np.array(test_accuracies), "accuracy", "g") 272 | y_bundle.append(y) 273 | 274 | #p.show_plot(y_bundle) 275 | 276 | if config.matplot_lib_for_single_ybundle: 277 | if config.matplot_lib_for_accuracy: 278 | return y_bundle[1] 279 | else : 280 | return y_bundle[0] 281 | return y_bundle 282 | 283 | 284 | 285 | if __name__ == '__main__': 286 | if config.matplot_lib_enabled: 287 | indep_test_axis = [] 288 | for i in range(config.training_epochs): 289 | indep_test_axis.append(i) 290 | 291 | p = PlotUtil("title", np.array(indep_test_axis), "x_label", "y_label") 292 | y_bundle = run_with_config(config) 293 | 294 | p.show_plot(y_bundle) 295 | else: 296 | run_with_config(config) 297 | 298 | 299 | -------------------------------------------------------------------------------- /single_two.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/praveendareddy21/Stacked_LSTMS_Highway_Residual_On_TimeSeries_Datasets/4c9e9647b4a134a2b7f63b0289687cc92262d5c8/single_two.png -------------------------------------------------------------------------------- /stacked_HAR.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/praveendareddy21/Stacked_LSTMS_Highway_Residual_On_TimeSeries_Datasets/4c9e9647b4a134a2b7f63b0289687cc92262d5c8/stacked_HAR.png -------------------------------------------------------------------------------- /stacked_two.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/praveendareddy21/Stacked_LSTMS_Highway_Residual_On_TimeSeries_Datasets/4c9e9647b4a134a2b7f63b0289687cc92262d5c8/stacked_two.png -------------------------------------------------------------------------------- /utils/Sine_Wave_Dataset/SlidingWindowGenerator.py: -------------------------------------------------------------------------------- 1 | # 2 | # Author : Praveen 3 | import numpy as np 4 | 5 | class SlidingWindowGenerator(object): 6 | """ 7 | Class for generating Time-series Dataset using sliding window algorithm. 8 | 9 | >>> import numpy as np 10 | >>> X = np.array([0, 1, 7, 3, 4,]) 11 | >>> print SlidingWindowGenerator(X,window_size=3).runSlidingWindow() 12 | [array([0, 1, 7]), array([ 1, 7, 3], array([7, 3, 4]))] 13 | 14 | """ 15 | def __init__(self, data, window_size = 10): 16 | self.window_size = window_size 17 | self.data = data ##TODO assert data is np.array 18 | 19 | def runSlidingWindow(self, max_dataset_size = 0 ): 20 | output = [] 21 | range_len = len(self.data) - self.window_size 22 | if(max_dataset_size != 0 and range_len >= max_dataset_size -1): 23 | range_len = max_dataset_size - 1 24 | for i in range(range_len+1): 25 | output.append(self.data[i: i+self.window_size]) 26 | return output 27 | 28 | if __name__ == '__main__': 29 | X = np.array([0, 1, 7, 3, 4,]) 30 | print (SlidingWindowGenerator(X, window_size=3).runSlidingWindow(max_dataset_size = 2)) 31 | -------------------------------------------------------------------------------- /utils/Sine_Wave_Dataset/time_series_create.py: -------------------------------------------------------------------------------- 1 | ## 2 | ## python util function to create dataset from Serial Observations 3 | ## 4 | import sys 5 | import numpy as np 6 | from SlidingWindowGenerator import SlidingWindowGenerator 7 | 8 | 9 | def loadfile(infile): 10 | data = [] 11 | with open(infile, 'r') as inf: 12 | data = inf.readlines() 13 | return data 14 | 15 | def loadfileWithStrippedNewline(infile): 16 | data = [] 17 | with open(infile, 'r') as inf: 18 | data = inf.readlines() 19 | 20 | stripped_data = [] 21 | for i in data: 22 | stripped_data.append(i.rstrip()) 23 | return stripped_data 24 | 25 | def savefile(outfile, dataset, ids): 26 | with open(outfile,'w') as of: 27 | for id in ids: 28 | of.write(dataset[id]) 29 | 30 | def saveListsToCsvfile(outfile, dataset, ids): 31 | with open(outfile,'w') as of: 32 | for id in ids: 33 | for i, value in enumerate(dataset[id]): 34 | if i == len(dataset[id]) - 1: 35 | of.write(value+"\n") 36 | else: 37 | of.write(value+",") 38 | 39 | 40 | def perform_sliding_winow(infile, ratio, _window_size, _max_dataset_size): 41 | dataset = loadfileWithStrippedNewline(infile) 42 | dataset = SlidingWindowGenerator(dataset, window_size=_window_size).runSlidingWindow(max_dataset_size = _max_dataset_size) 43 | total = len(dataset) 44 | if len(dataset) <= 0: 45 | print('load empty file, quit') 46 | sys.exit(-1) 47 | 48 | if ratio < 0: 49 | print('ratio should be in (0,1), reset it to 0.2') 50 | ratio = 0.2 51 | testcnt = int(total*ratio) 52 | elif ratio > 1: 53 | print('set absolute test number as %d'%ratio) 54 | testcnt = int(ratio) 55 | else: 56 | testcnt = int(total*ratio) 57 | 58 | 59 | id = np.arange(total) 60 | perm = np.random.permutation(id) 61 | 62 | test = perm[:testcnt] 63 | train = perm[testcnt:] 64 | saveListsToCsvfile('train-' + infile, dataset, train) 65 | saveListsToCsvfile('test-' + infile, dataset, test) 66 | 67 | def process_split(infile, ratio): 68 | 69 | dataset = loadfile(infile) 70 | total = len(dataset) 71 | if len(dataset) <= 0: 72 | print('load empty file, quit') 73 | sys.exit(-1) 74 | 75 | if ratio < 0: 76 | print('ratio should be in (0,1), reset it to 0.2') 77 | ratio = 0.2 78 | testcnt = int(total*ratio) 79 | elif ratio > 1: 80 | print('set absolute test number as %d'%ratio) 81 | testcnt = int(ratio) 82 | else: 83 | testcnt = int(total*ratio) 84 | 85 | 86 | id = np.arange(total) 87 | perm = np.random.permutation(id) 88 | 89 | test = perm[:testcnt] 90 | train = perm[testcnt:] 91 | savefile('train-' + infile, dataset, train) 92 | savefile('test-' + infile, dataset, test) 93 | 94 | def process_split_X_Y_to_train_test(infile_X, infile_Y, ratio): 95 | 96 | dataset_X = loadfile(infile_X) 97 | dataset_Y = loadfile(infile_Y) 98 | 99 | total = len(dataset_X) 100 | if len(dataset_X) <= 0: 101 | print('load empty file, quit') 102 | sys.exit(-1) 103 | 104 | if ratio < 0: 105 | print('ratio should be in (0,1), reset it to 0.2') 106 | ratio = 0.2 107 | testcnt = int(total*ratio) 108 | elif ratio > 1: 109 | print('set absolute test number as %d'%ratio) 110 | testcnt = int(ratio) 111 | else: 112 | testcnt = int(total*ratio) 113 | 114 | permutated_indices = np.random.permutation(total) 115 | 116 | savefile('train_X-' + infile, dataset_X, permutated_indices[testcnt:]) 117 | savefile('test_X-' + infile, dataset_X, permutated_indices[:testcnt]) 118 | savefile('train_Y-' + infile, dataset_Y, permutated_indices[testcnt:]) 119 | savefile('test_Y-' + infile, dataset_Y, permutated_indices[:testcnt]) 120 | 121 | if __name__ == '__main__': 122 | """ 123 | python time_series_create.py 0.2 5 0 data.csv 124 | """ 125 | if len(sys.argv) < 5 : 126 | print('usage: time_series_create.py <[seed]>') 127 | print('ratio ex : 0.2 would create test data with 0.2*rows, train data with 0.8*rows ') 128 | sys.exit(-1) 129 | 130 | 131 | ratio = float(sys.argv[1]) 132 | window_size = int(sys.argv[2]) 133 | max_data_size = int(sys.argv[3]) 134 | infile = sys.argv[4] 135 | 136 | if len(sys.argv) == 6: 137 | print('fix random seed to 123') 138 | np.random.seed(seed= 123) 139 | else: 140 | perform_sliding_winow(infile, ratio, window_size, max_data_size) 141 | -------------------------------------------------------------------------------- /utils/subdirectory_reveal.py: -------------------------------------------------------------------------------- 1 | import os 2 | def get_immediate_subdirectories(a_dir): 3 | return [name for name in os.listdir(a_dir) 4 | if os.path.isdir(os.path.join(a_dir, name))] 5 | 6 | 7 | 8 | print(get_immediate_subdirectories(os.getcwd())) 9 | 10 | 11 | print(get_immediate_subdirectories("C:\opt\jenkins\workspace\d_s2\Chutzpah.4.3.0")) 12 | -------------------------------------------------------------------------------- /utils/util.py: -------------------------------------------------------------------------------- 1 | import matplotlib 2 | import numpy as np 3 | from matplotlib import pyplot 4 | import math 5 | 6 | def sigmoid(x): 7 | return 1 / (1 + math.exp(-x)) 8 | 9 | 10 | 11 | 12 | class YaxisBundle: 13 | def __init__(self, y_index_array_input, y_graph_label, y_graph_colour): 14 | self.y_index_array_input = y_index_array_input 15 | self.y_graph_colour = y_graph_colour 16 | self.y_graph_label = y_graph_label 17 | 18 | 19 | 20 | class PlotUtil: 21 | def __init__(self, plot_title, x_index_array_input , x_label, y_label, width = 6, height = 6): 22 | self.x_index_array_input = x_index_array_input 23 | self.plot_title = plot_title 24 | self.x_label = x_label 25 | self.y_label = y_label 26 | self.width = width 27 | self.height = height 28 | def show_plot(self, YaxisBundle_array): 29 | pyplot.figure(figsize=(self.width, self.height)) 30 | 31 | for y in YaxisBundle_array: 32 | pyplot.plot(self.x_index_array_input, np.array(y.y_index_array_input), y.y_graph_colour, label=y.y_graph_label) 33 | 34 | pyplot.title(self.plot_title) 35 | pyplot.legend(loc='upper right', shadow=True) 36 | pyplot.ylabel(self.y_label) 37 | pyplot.xlabel(self.x_label) 38 | pyplot.show() 39 | 40 | 41 | 42 | 43 | 44 | 45 | if __name__ == '__main__': 46 | 47 | test_losses = [] 48 | test_accuracies = [] 49 | indep_test_axis = [] 50 | batch_size = 300 51 | 52 | for i in range(batch_size): 53 | indep_test_axis.append(i) 54 | test_losses.append(3.5 - 1.6 * sigmoid( i/10)) 55 | test_accuracies.append(0.5 + 0.4 * sigmoid(i/10)) 56 | 57 | 58 | p = PlotUtil("title", indep_test_axis, "x_label", "y_label") 59 | y_bundle =[] 60 | 61 | y = YaxisBundle(test_losses,"loss", "b") 62 | y_bundle.append(y) 63 | 64 | y = YaxisBundle(test_accuracies,"accuracy", "g") 65 | y_bundle.append(y) 66 | 67 | p.show_plot(y_bundle) 68 | --------------------------------------------------------------------------------