├── README.md ├── LICENSE └── DBN.py /README.md: -------------------------------------------------------------------------------- 1 | # RBM_DBN 2 | Experimenting with RBMs using scikit-learn on MNISTand simulating a DBN using Keras. 3 | 4 | MNIST data has been used for these experiments. The compressed files can be downloaded [here](http://yann.lecun.com/exdb/mnist/). 5 | 6 | You can use [python-mnist](https://pypi.python.org/pypi/python-mnist/) if you're finding it hard to parse and process the data. 7 | 8 | ## Dependencies 9 | 10 | - numpy==1.12.0 11 | - keras==2.0.6 12 | - scikit_learn==0.19b2 13 | 14 | 15 | ## Files: 16 | 17 | 1. RBM.ipynb - Jupyter Notebook containing experiments and results done on the mnist data, I have included comments wherever necessary 18 | 2. RBM.html - The above file in HTML format for quick viewing 19 | 3. DBN.py - A DBN wrapper simulated using RBMs in scikit-learn followed by Keras. Weights learnt during RBM training are then initialised in a MLP built using Keras Sequential Model. Inline documentation and example code are included. 20 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Kaushik S Kalmady 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /DBN.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from sklearn import linear_model, datasets 3 | from sklearn.metrics import classification_report 4 | from sklearn.neural_network import BernoulliRBM 5 | from keras.models import Sequential 6 | from keras.layers import Dense, Activation 7 | from keras.callbacks import ModelCheckpoint, TensorBoard 8 | import os 9 | import json 10 | import pickle 11 | 12 | 13 | 14 | class DBN(): 15 | 16 | def __init__( 17 | self, 18 | train_data, 19 | targets, 20 | layers, 21 | outputs, 22 | rbm_lr, 23 | rbm_iters, 24 | rbm_dir=None, 25 | test_data = None, 26 | test_targets = None, 27 | epochs = 25, 28 | fine_tune_batch_size = 32, 29 | outdir="tmp/", 30 | logdir="logs/" 31 | 32 | ): 33 | 34 | self.hidden_sizes = layers 35 | self.outputs = outputs 36 | self.targets = targets 37 | self.data = train_data 38 | 39 | if test_data is None: 40 | self.validate = False 41 | else: 42 | self.validate = True 43 | 44 | self.valid_data = test_data 45 | self.valid_labels = test_targets 46 | 47 | self.rbm_learning_rate = rbm_lr 48 | self.rbm_iters = rbm_iters 49 | 50 | self.epochs = epochs 51 | self.nn_batch_size = fine_tune_batch_size 52 | 53 | self.rbm_weights = [] 54 | self.rbm_biases = [] 55 | self.rbm_h_act = [] 56 | 57 | self.model = None 58 | self.history = None 59 | 60 | if not os.path.exists(outdir): 61 | os.makedirs(outdir) 62 | if not os.path.exists(logdir): 63 | os.makedirs(logdir) 64 | 65 | 66 | if outdir[-1]!='/': 67 | outdir = outdir + '/' 68 | 69 | self.outdir = outdir 70 | self.logdir=logdir 71 | 72 | def pretrain(self,save=True): 73 | 74 | visual_layer = self.data 75 | 76 | for i in range(len(self.hidden_sizes)): 77 | print("[DBN] Layer {} Pre-Training".format(i+1)) 78 | 79 | rbm = BernoulliRBM(n_components = self.hidden_sizes[i], n_iter = self.rbm_iters[i], learning_rate = self.rbm_learning_rate[i], verbose = True, batch_size = 32) 80 | rbm.fit(visual_layer) 81 | self.rbm_weights.append(rbm.components_) 82 | self.rbm_biases.append(rbm.intercept_hidden_) 83 | self.rbm_h_act.append(rbm.transform(visual_layer)) 84 | 85 | visual_layer = self.rbm_h_act[-1] 86 | 87 | if save: 88 | with open(self.outdir + "rbm_weights.p", 'wb') as f: 89 | pickle.dump(self.rbm_weights, f) 90 | 91 | with open(self.outdir + "rbm_biases.p", 'wb') as f: 92 | pickle.dump(self.rbm_biases, f) 93 | 94 | with open(self.outdir + "rbm_hidden.p", 'wb') as f: 95 | pickle.dump(self.rbm_h_act, f) 96 | 97 | 98 | 99 | 100 | def finetune(self): 101 | model = Sequential() 102 | for i in range(len(self.hidden_sizes)): 103 | 104 | if i==0: 105 | model.add(Dense(self.hidden_sizes[i], activation='relu', input_dim=self.data.shape[1], name='rbm_{}'.format(i))) 106 | else: 107 | model.add(Dense(self.hidden_sizes[i], activation='relu', name='rbm_{}'.format(i))) 108 | 109 | 110 | model.add(Dense(self.outputs, activation='softmax')) 111 | model.compile(optimizer='Adam', 112 | loss='categorical_crossentropy', 113 | metrics=['accuracy']) 114 | 115 | for i in range(len(self.hidden_sizes)): 116 | layer = model.get_layer('rbm_{}'.format(i)) 117 | layer.set_weights([self.rbm_weights[i].transpose(),self.rbm_biases[i]]) 118 | 119 | checkpointer = ModelCheckpoint(filepath= self.outdir + "dbn_weights.hdf5", verbose=1, save_best_only=True) 120 | tensorboard = TensorBoard(log_dir=self.logdir) 121 | 122 | if self.validate: 123 | self.history = model.fit(trainx, trainy, 124 | epochs = self.epochs, 125 | batch_size = self.nn_batch_size, 126 | validation_data=(self.valid_data, self.valid_labels), 127 | callbacks=[checkpointer, tensorboard]) 128 | else: 129 | self.history = model.fit(trainx, trainy, 130 | epochs = self.epochs, 131 | batch_size = self.nn_batch_size, 132 | callbacks=[checkpointer, tensorboard]) 133 | self.model = model 134 | 135 | def report(self, data, labels): 136 | print(classification_report(np.argmax(labels, axis=1), np.argmax(self.model.predict(data),axis=1))) 137 | 138 | 139 | def save_model(self,filename): 140 | 141 | if self.model is None : 142 | raise ValueError("Run finetune() first") 143 | 144 | with open(self.outdir + filename, mode='w', encoding='utf-8') as outfile: 145 | 146 | data = { 147 | "model_config":self.model.get_config(), 148 | "loss_acc": self.history.history 149 | } 150 | json.dump(data, outfile, indent=2) 151 | 152 | def load_rbm(self): 153 | try: 154 | self.rbm_weights = pickle.load(self.rbm_dir + "rbm_weights.p") 155 | self.rbm_biases = pickle.load(self.rbm_dir + "rbm_biases.p") 156 | self.rbm_h_act = pickle.load(self.rbm_dir + "rbm_hidden.p") 157 | except: 158 | print("No such file or directory.") 159 | 160 | 161 | if __name__ == '__main__': 162 | 163 | trainx = np.load("mnist_train.npy") 164 | trainy= np.load("mnist_trainy.npy") 165 | testx = np.load("mnist_test.npy") 166 | testy = np.load("mnist_testy.npy") 167 | 168 | dbn = DBN(train_data = trainx, targets = trainy, 169 | #test_data = testx, test_targets = testy, 170 | layers = [200], 171 | outputs = 10, 172 | rbm_iters = [40], 173 | rbm_lr = [0.01], 174 | outdir = "mnistrbm/", 175 | logdir = "mnistrbm_logs/" 176 | ) 177 | dbn.pretrain(save=True) 178 | dbn.finetune() 179 | dbn.save_model("mnist_dbn_model.json") 180 | 181 | print("Training Report") 182 | dbn.report(trainx,trainy) 183 | 184 | print("Testing Report") 185 | dbn.report(testx,testy) 186 | 187 | --------------------------------------------------------------------------------