├── .gitignore ├── LICENSE ├── README.md ├── __init__.py ├── deepqlearn.py ├── examples ├── __init__.py ├── autoencoder.py ├── autoencoder_vis.py ├── cifar │ ├── __init__.py │ ├── autoencoder.py │ ├── cifar10.py │ └── dataset.py ├── dark_knowledge.py ├── darkencoder.py ├── dialogue.py ├── faces.py ├── iris.py ├── mnist.py ├── next_letter.py ├── next_word.py ├── next_word_embeddings.py ├── num2img.py ├── plotting ├── sentiment.py ├── similarity.py ├── titanic.py ├── topics.py ├── toy2D.py ├── transforming_autoencoder.py └── udacity_terrain.py ├── layers ├── __init__.py ├── dotproducts.py ├── dropout.py ├── input.py ├── loss.py ├── nonlinearities.py ├── normalization.py ├── operations.py ├── pooling.py └── similarity.py ├── net.py ├── run.py ├── trainers.py ├── util.py ├── vol.py └── vol_util.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.DS_Store 3 | data/ 4 | models/ 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License 2 | 3 | Copyright (c) 2014 Andrej Karpathy 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ConvNetPy 2 | ConvNetPy is a Python port of the wonderful [ConvNetJS](https://github.com/karpathy/convnetjs) I wrote for fun and learning. 3 | 4 | ## Disclaimers 5 | - Given my goals in writing ConvNetPy (fun and learning), some code may be incomplete (almost nothing), may not work (almost nothing), may be badly documented (occasionally), or may be written in bad style (judge for yourself). 6 | - Python is rarely fast enough. ConvNetPy is pure-python so [PyPy](http://pypy.org/) will work so long as you are careful using outside libraries (SciPy, OpenCV, NLTK, etc.). /examples has some examples where I used PyPy to train a model and then did some visualizations with OpenCV in a different Python distribution. PyPy's performance impressed me overall but it is still impractical for large models. 7 | - I have data and models folders in my local copy of ConvNetPy which were unfortunately too large to upload to GitHub (they are used in /examples). I suppose you can email me if you want some piece of data or model I have. 8 | 9 | Thank you very much to the original author (Andrej Karpathy) and to all the contributors to ConvNetJS! 10 | -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/benglard/ConvNetPy/c174bbbe86b4292419fbc83acb7fbffca995e59a/__init__.py -------------------------------------------------------------------------------- /deepqlearn.py: -------------------------------------------------------------------------------- 1 | from util import * 2 | from net import Net 3 | from trainers import Trainer 4 | from vol import Vol 5 | 6 | """ 7 | An agent is in state0 and does action0 8 | environment then assigns reward0 and provides new state, state1 9 | Experience nodes store all this information, which is used in the 10 | Q-learning update step 11 | """ 12 | 13 | class Experience(object): 14 | 15 | def __init__(self, state0, action0, reward0, state1): 16 | self.state0 = state0 17 | self.action0 = action0 18 | self.reward0 = reward0 19 | self.state1 = state1 20 | 21 | """ 22 | A Brain object does all the magic. 23 | over time it receives some inputs and some rewards 24 | and its job is to set the outputs to maximize the expected reward 25 | """ 26 | 27 | class Brain(object): 28 | 29 | def __init__(self, num_states, num_actions, opt={}): 30 | """ 31 | in number of time steps, of temporal memory 32 | the ACTUAL input to the net will be (x,a) temporal_window times, and followed by current x 33 | so to have no information from previous time step going into value function, set to 0. 34 | """ 35 | self.temporal_window = getopt(opt, 'temporal_window', 1) 36 | 37 | """size of experience replay memory""" 38 | self.experience_size = getopt(opt, 'experience_size', 30000) 39 | 40 | """number of examples in experience replay memory before we begin learning""" 41 | self.start_learn_threshold = getopt(opt, 'start_learn_threshold', 42 | int(min(self.experience_size * 0.1, 1000))) 43 | 44 | """gamma is a crucial parameter that controls how much plan-ahead the agent does. In [0,1]""" 45 | self.gamma = getopt(opt, 'gamma', 0.8) 46 | 47 | """number of steps we will learn for""" 48 | self.learning_steps_total = getopt(opt, 'learning_steps_total', 100000) 49 | 50 | """how many steps of the above to perform only random actions (in the beginning)?""" 51 | self.learning_steps_burnin = getopt(opt, 'learning_steps_burnin', 3000) 52 | 53 | """what epsilon value do we bottom out on? 0.0 => purely deterministic policy at end""" 54 | self.epsilon_min = getopt(opt, 'epsilon_min', 0.05) 55 | 56 | """what epsilon to use at test time? (i.e. when learning is disabled)""" 57 | self.epsilon_test_time = getopt(opt, 'epsilon_test_time', 0.01) 58 | 59 | """ 60 | advanced feature. Sometimes a random action should be biased towards some values 61 | for example in flappy bird, we may want to choose to not flap more often 62 | """ 63 | if 'random_action_distribution' in opt: 64 | #this better sum to 1 by the way, and be of length this.num_actions 65 | self.random_action_distribution = opt['random_action_distribution'] 66 | 67 | if len(self.random_action_distribution) != num_actions: 68 | print 'TROUBLE. random_action_distribution should be same length as num_actions.' 69 | 70 | a = self.random_action_distribution 71 | s = sum(a) 72 | if abs(s - 1.0) > 0.0001: 73 | print 'TROUBLE. random_action_distribution should sum to 1!' 74 | else: 75 | self.random_action_distribution = [] 76 | 77 | """ 78 | states that go into neural net to predict optimal action look as 79 | x0,a0,x1,a1,x2,a2,...xt 80 | this variable controls the size of that temporal window. Actions are 81 | encoded as 1-of-k hot vectors 82 | """ 83 | self.net_inputs = num_states * self.temporal_window + num_actions * self.temporal_window + num_states 84 | self.num_states = num_states 85 | self.num_actions = num_actions 86 | self.window_size = max(self.temporal_window, 2) #must be at least 2, but if we want more context even more 87 | self.state_window = zeros(self.window_size) 88 | self.action_window = zeros(self.window_size) 89 | self.reward_window = zeros(self.window_size) 90 | self.net_window = zeros(self.window_size) 91 | 92 | #create [state -> value of all possible actions] modeling net for the value function 93 | layers = [] 94 | if 'layers' in opt: 95 | """ 96 | this is an advanced usage feature, because size of the input to the network, and number of 97 | actions must check out. 98 | """ 99 | layers = opt['layers'] 100 | 101 | if len(layers) < 2: 102 | print 'TROUBLE! must have at least 2 layers' 103 | if layers[0]['type'] != 'input': 104 | print 'TROUBLE! first layer must be input layer!' 105 | if layers[-1]['type'] != 'regression': 106 | print 'TROUBLE! last layer must be input regression!' 107 | if layers[0]['out_depth'] * layers[0]['out_sx'] * layers[0]['out_sy'] != self.net_inputs: 108 | print 'TROUBLE! Number of inputs must be num_states * temporal_window + num_actions * temporal_window + num_states!' 109 | if layers[-1]['num_neurons'] != self.num_actions: 110 | print 'TROUBLE! Number of regression neurons should be num_actions!' 111 | else: 112 | #create a very simple neural net by default 113 | layers.append({'type': 'input', 'out_sx': 1, 'out_sy': 1, 'out_depth': self.net_inputs}) 114 | if 'hidden_layer_sizes' in opt: 115 | #allow user to specify this via the option, for convenience 116 | for size in opt['hidden_layer_sizes']: 117 | layers.append({'type': 'fc', 'num_neurons': size, 'activation': 'relu'}) 118 | layers.append({'type': 'regression', 'num_neurons': self.num_actions}) #value function output 119 | 120 | self.value_net = Net(layers) 121 | 122 | #and finally we need a Temporal Difference Learning trainer! 123 | trainer_ops_default = {'learning_rate': 0.01, 'momentum': 0.0, 'batch_size': 64, 'l2_decay': 0.01} 124 | tdtrainer_options = getopt(opt, 'tdtrainer_options', trainer_ops_default) 125 | self.tdtrainer = Trainer(self.value_net, tdtrainer_options) 126 | 127 | #experience replay 128 | self.experience = [] 129 | 130 | #various housekeeping variables 131 | self.age = 0 #incremented every backward() 132 | self.forward_passes = 0 #incremented every forward() 133 | self.epsilon = 1.0 #controls exploration exploitation tradeoff. Should be annealed over time 134 | self.latest_reward = 0 135 | self.last_input_array = [] 136 | self.average_reward_window = Window(1000, 10) 137 | self.average_loss_window = Window(1000, 10) 138 | self.learning = True 139 | 140 | def random_action(self): 141 | """ 142 | a bit of a helper function. It returns a random action 143 | we are abstracting this away because in future we may want to 144 | do more sophisticated things. For example some actions could be more 145 | or less likely at "rest"/default state. 146 | """ 147 | 148 | if len(random_action_distribution) == 0: 149 | return randi(0, self.num_actions) 150 | else: 151 | #okay, lets do some fancier sampling 152 | p = randf(0, 1.0) 153 | cumprob = 0.0 154 | for k in xrange(self.num_actions): 155 | cumprob += self.random_action_distribution[k] 156 | if p < cumprob: 157 | return k 158 | 159 | def policy(self, s): 160 | """ 161 | compute the value of doing any action in this state 162 | and return the argmax action and its value 163 | """ 164 | 165 | V = Vol(s) 166 | action_values = self.value_net.forward(V) 167 | weights = action_values.w 168 | max_val = max(weights) 169 | max_k = weights.index(maxval) 170 | return { 171 | 'action': max_k, 172 | 'value': max_val 173 | } 174 | 175 | def getNetInput(self, xt): 176 | """ 177 | return s = (x,a,x,a,x,a,xt) state vector 178 | It's a concatenation of last window_size (x,a) pairs and current state x 179 | """ 180 | 181 | w = [] 182 | w.extend(xt) #start with current state 183 | #and now go backwards and append states and actions from history temporal_window times 184 | n = self.window_size 185 | for k in xrange(self.temporal_window): 186 | index = n - 1 - k 187 | w.extend(self.state_window[index]) #state 188 | 189 | #action, encoded as 1-of-k indicator vector. We scale it up a bit because 190 | #we dont want weight regularization to undervalue this information, as it only exists once 191 | action1ofk = zeros(self.num_actions) 192 | action1ofk[index] = 1.0 * self.num_states 193 | w.extend(action1ofk) 194 | 195 | return w 196 | 197 | def forward(self, input_array): 198 | self.forward_passes += 1 199 | self.last_input_array = input_array 200 | 201 | # create network input 202 | action = None 203 | if self.forward_passes > self.temporal_window: 204 | #we have enough to actually do something reasonable 205 | net_input = self.getNetInput(input_array) 206 | if self.learning: 207 | #compute epsilon for the epsilon-greedy policy 208 | self.epsilon = min( 209 | 1.0, 210 | max( 211 | self.epsilon_min, 212 | 1.0 - \ 213 | (self.age - self.learning_steps_burnin) / \ 214 | (self.learning_steps_total - self.learning_steps_burnin) 215 | ) 216 | ) 217 | else: 218 | self.epsilon = self.epsilon_test_time #use test-time value 219 | 220 | rf = randf(0, 1) 221 | if rf < self.epsilon: 222 | #choose a random action with epsilon probability 223 | action = self.random_action() 224 | else: 225 | #otherwise use our policy to make decision 226 | maxact = self.policy(net_input) 227 | action = maxact['action'] 228 | else: 229 | #pathological case that happens first few iterations 230 | #before we accumulate window_size inputs 231 | net_input = [] 232 | action = self.random_action() 233 | 234 | #remember the state and action we took for backward pass 235 | self.net_window.pop(0) 236 | self.net_window.append(net_input) 237 | self.state_window.pop(0) 238 | self.state_window.append(input_array) 239 | self.action_window.pop(0) 240 | self.action_window.append(action) 241 | 242 | def backward(self, reward): 243 | self.latest_reward = reward 244 | self.average_reward_window.add(reward) 245 | self.reward_window.pop(0) 246 | self.reward_window.append(reward) 247 | 248 | if not self.learning: 249 | return 250 | 251 | self.age += 1 252 | 253 | #it is time t+1 and we have to store (s_t, a_t, r_t, s_{t+1}) as new experience 254 | #(given that an appropriate number of state measurements already exist, of course) 255 | if self.forward_passes > self.temporal_window + 1: 256 | n = self.window_size 257 | e = Experience( 258 | self.net_window[n - 2], 259 | self.action_window[n - 2], 260 | self.reward_window[n - 2], 261 | self.net_window[n - 1] 262 | ) 263 | 264 | if len(self.experience) < self.experience_size: 265 | self.experience.append(e) 266 | else: 267 | ri = randi(0, self.experience_size) 268 | self.experience[ri] = e 269 | 270 | #learn based on experience, once we have some samples to go on 271 | #this is where the magic happens... 272 | if len(self.experience) > self.start_learn_threshold: 273 | avcost = 0.0 274 | 275 | for k in xrange(self.tdtrainer.batch_size): 276 | re = randi(0, len(self.experience)) 277 | e = self.experience[re] 278 | x = Vol(1, 1, self.net_inputs) 279 | x.w = e.state0 280 | maxact = self.policy(e.state1) 281 | r = e.reward0 + self.gamma * maxact.value 282 | ystruct = {'dim': e.action0, 'val': r} 283 | stats = self.tdtrainer.train(x, ystruct) 284 | avcost += stats['loss'] 285 | 286 | avcost /= self.tdtrainer.batch_size 287 | self.average_loss_window.add(avcost) -------------------------------------------------------------------------------- /examples/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/benglard/ConvNetPy/c174bbbe86b4292419fbc83acb7fbffca995e59a/examples/__init__.py -------------------------------------------------------------------------------- /examples/autoencoder.py: -------------------------------------------------------------------------------- 1 | from vol import Vol 2 | from vol_util import augment 3 | from net import Net 4 | from trainers import Trainer 5 | from random import sample 6 | 7 | import os, struct 8 | from array import array as pyarray 9 | from subprocess import call 10 | 11 | training_data = None 12 | testing_data = None 13 | n = None 14 | t = None 15 | 16 | # Load mnist data 17 | def load_data(training=True): 18 | """Adapted from http://g.sweyla.com/blog/2012/mnist-numpy/""" 19 | path = "./data" 20 | 21 | if training: 22 | fname_img = os.path.join(path, 'train-images-idx3-ubyte') 23 | fname_lbl = os.path.join(path, 'train-labels-idx1-ubyte') 24 | else: 25 | fname_img = os.path.join(path, 't10k-images-idx3-ubyte') 26 | fname_lbl = os.path.join(path, 't10k-labels-idx1-ubyte') 27 | 28 | # Inputs 29 | fimg = open(fname_img, 'rb') 30 | magic_nr, size, rows, cols = struct.unpack(">IIII", fimg.read(16)) 31 | imgs = pyarray("B", fimg.read()) 32 | fimg.close() 33 | 34 | imgs = [imgs[n:n+784] for n in xrange(0, len(imgs), 784)] 35 | inputs = [] 36 | for img in imgs: 37 | V = Vol(28, 28, 1, 0.0) 38 | V.w = [ (px / 255.0) for px in img ] 39 | inputs.append(V) 40 | 41 | # Outputs 42 | flbl = open(fname_lbl, 'rb') 43 | magic_nr, size = struct.unpack(">II", flbl.read(8)) 44 | labels = pyarray("b", flbl.read()) 45 | flbl.close() 46 | 47 | return zip(inputs, labels) 48 | 49 | def start(): 50 | global training_data, testing_data, n, t 51 | 52 | training_data = load_data() 53 | testing_data = load_data(False) 54 | 55 | print 'Data loaded...' 56 | 57 | layers = [] 58 | layers.append({'type': 'input', 'out_sx': 28, 'out_sy': 28, 'out_depth': 1}) 59 | layers.append({'type': 'fc', 'num_neurons': 100, 'activation': 'sigmoid'}) 60 | layers.append({'type': 'regression', 'num_neurons': 28 * 28}) 61 | print 'Layers made...' 62 | 63 | n = Net(layers) 64 | 65 | print 'Net made...' 66 | print n 67 | 68 | t = Trainer(n, {'method': 'sgd', 'batch_size': 20, 'l2_decay': 0.001}) 69 | print 'Trainer made...' 70 | 71 | def train(): 72 | global training_data, testing_data, n, t 73 | 74 | print 'In training...' 75 | print 'k', 'time\t\t ', 'loss\t ' 76 | print '----------------------------------------------------' 77 | try: 78 | for x, y in training_data: 79 | stats = t.train(x, x.w) 80 | print stats['k'], stats['time'], stats['loss'] 81 | except: #hit control-c or other 82 | return 83 | 84 | def display(python_path, pred, x): 85 | display_cmd = '; '.join([ 86 | 'import cv2, numpy', 87 | 'predw = numpy.array({}, dtype=numpy.float64).reshape(28, 28)'.format(pred), 88 | 'cv2.imshow(\'P\', predw)', 89 | 'xw = numpy.array({}, dtype=numpy.float64).reshape(28, 28)'.format(x), 90 | 'cv2.imshow(\'X\', xw)', 91 | 'cv2.waitKey(0)', 92 | 'cv2.destroyAllWindows()' 93 | ]) 94 | 95 | cmd = '{} -c \"{}\"'.format(os.path.join(python_path, 'python'), display_cmd) 96 | call(cmd, shell=True) 97 | 98 | def test(path=None, test_n=None): 99 | global training_data, testing_data, n, t 100 | 101 | print 'In autoencoder testing' 102 | 103 | path = path or '/opt/local/bin' 104 | test_n = test_n or 5 105 | 106 | for x, y in sample(testing_data, test_n): 107 | print y 108 | pred = n.forward(x).w 109 | display(path, pred, x.w) -------------------------------------------------------------------------------- /examples/autoencoder_vis.py: -------------------------------------------------------------------------------- 1 | from vol import Vol 2 | from vol_util import augment 3 | from net import Net 4 | from trainers import Trainer 5 | from random import sample 6 | from util import maxmin 7 | 8 | import numpy, cv2 # requires numpy and opencv 9 | 10 | import os, struct 11 | from array import array as pyarray 12 | from subprocess import call 13 | 14 | training_data = None 15 | testing_data = None 16 | n = None 17 | t = None 18 | 19 | # Load mnist data 20 | def load_data(training=True): 21 | """Adapted from http://g.sweyla.com/blog/2012/mnist-numpy/""" 22 | path = "./data" 23 | 24 | if training: 25 | fname_img = os.path.join(path, 'train-images-idx3-ubyte') 26 | fname_lbl = os.path.join(path, 'train-labels-idx1-ubyte') 27 | else: 28 | fname_img = os.path.join(path, 't10k-images-idx3-ubyte') 29 | fname_lbl = os.path.join(path, 't10k-labels-idx1-ubyte') 30 | 31 | # Inputs 32 | fimg = open(fname_img, 'rb') 33 | magic_nr, size, rows, cols = struct.unpack(">IIII", fimg.read(16)) 34 | imgs = pyarray("B", fimg.read()) 35 | fimg.close() 36 | 37 | imgs = [imgs[n:n+784] for n in xrange(0, len(imgs), 784)] 38 | inputs = [] 39 | for img in imgs: 40 | V = Vol(28, 28, 1, 0.0) 41 | V.w = [ (px / 255.0) for px in img ] 42 | inputs.append(V) 43 | 44 | # Outputs 45 | flbl = open(fname_lbl, 'rb') 46 | magic_nr, size = struct.unpack(">II", flbl.read(8)) 47 | labels = pyarray("b", flbl.read()) 48 | flbl.close() 49 | 50 | return zip(inputs, labels) 51 | 52 | def start(): 53 | global training_data, testing_data, n, t 54 | 55 | training_data = load_data() 56 | testing_data = load_data(False) 57 | 58 | print 'Data loaded...' 59 | 60 | layers = [] 61 | layers.append({'type': 'input', 'out_sx': 28, 'out_sy': 28, 'out_depth': 1}) 62 | layers.append({'type': 'fc', 'num_neurons': 50, 'activation': 'tanh'}) 63 | layers.append({'type': 'fc', 'num_neurons': 50, 'activation': 'tanh'}) 64 | layers.append({'type': 'fc', 'num_neurons': 2, 'activation': 'tanh'}) 65 | layers.append({'type': 'fc', 'num_neurons': 50, 'activation': 'tanh'}) 66 | layers.append({'type': 'fc', 'num_neurons': 50, 'activation': 'tanh'}) 67 | layers.append({'type': 'regression', 'num_neurons': 28 * 28}) 68 | print 'Layers made...' 69 | 70 | n = Net(layers) 71 | print 'Net made...' 72 | print n 73 | 74 | t = Trainer(n, {'method': 'adadelta', 'learning_rate': 1.0, 'batch_size': 50, 'l2_decay': 0.001, 'l1_decay': 0.001}); 75 | print 'Trainer made...' 76 | 77 | def train(): 78 | global training_data, testing_data, n, t 79 | 80 | print 'In training...' 81 | print 'k', 'time\t\t ', 'loss\t ' 82 | print '----------------------------------------------------' 83 | try: 84 | for x, y in training_data: 85 | stats = t.train(x, x.w) 86 | print stats['k'], stats['time'], stats['loss'] 87 | 88 | if stats['k'] % 1000 == 0: test() 89 | except: 90 | return 91 | 92 | def test(): 93 | global training_data, testing_data, n, t 94 | 95 | print 'In autoencoder testing' 96 | test_n = 100 97 | 98 | xcodes = [] 99 | ycodes = [] 100 | xs = [] 101 | for x, y in sample(testing_data, test_n): 102 | n.forward(x) 103 | xcode, ycode = n.layers[5].out_act.w 104 | xcodes.append(xcode) 105 | ycodes.append(ycode) 106 | nx = numpy.array(x.w).reshape((28,28)) 107 | xs.append(nx) 108 | mmx = maxmin(xcodes) 109 | mmy = maxmin(ycodes) 110 | 111 | dim = 500 112 | img = numpy.zeros((dim, dim)) 113 | img[:] = 255 #white 114 | for xcode, ycode, nx in zip(xcodes, ycodes, xs): 115 | xpos = (dim - 28 * 2) * (xcode - mmx['minv']) / mmx['dv'] + 28 116 | ypos = (dim - 28 * 2) * (ycode - mmy['minv']) / mmy['dv'] + 28 117 | xpos = int(xpos) 118 | ypos = int(ypos) 119 | img[ypos:ypos + 28, xpos:xpos + 28] = nx 120 | cv2.imshow('MNIST', img) 121 | cv2.waitKey(0) 122 | cv2.destroyAllWindows() -------------------------------------------------------------------------------- /examples/cifar/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/benglard/ConvNetPy/c174bbbe86b4292419fbc83acb7fbffca995e59a/examples/cifar/__init__.py -------------------------------------------------------------------------------- /examples/cifar/autoencoder.py: -------------------------------------------------------------------------------- 1 | from vol import Vol 2 | from vol_util import augment 3 | from net import Net 4 | from trainers import Trainer 5 | from random import sample 6 | import numpy 7 | import os 8 | from subprocess import call 9 | 10 | training_data = None 11 | testing_data = None 12 | network = None 13 | t = None 14 | 15 | def load_data(crop, gray, training=True): 16 | filename = './data/cifar10_' 17 | if training: 18 | filename += 'train.npz' 19 | else: 20 | filename += 'test.npz' 21 | 22 | data = numpy.load(filename) 23 | xs = data['x'] 24 | ys = data['y'] 25 | 26 | for i in xrange(len(xs)): 27 | V = Vol(32, 32, 3, 0.0) 28 | for d in xrange(3): 29 | for x in xrange(32): 30 | for y in xrange(32): 31 | px = xs[i][x * 32 + y, d] / 255.0 #- 0.5 32 | V.set(x, y, d, px) 33 | if crop: 34 | V = augment(V, 24, gray) 35 | 36 | y = ys[i] 37 | yield V, y 38 | 39 | def start(conv, crop, gray): 40 | global training_data, testing_data, network, t 41 | 42 | training_data = load_data(crop, gray) 43 | testing_data = load_data(crop, gray, False) 44 | 45 | print 'Data loaded...' 46 | 47 | layers = [] 48 | 49 | dim = 24 if crop else 32 50 | depth = 1 if gray else 3 51 | layers.append({'type': 'input', 'out_sx': dim, 'out_sy': dim, 'out_depth': depth}) 52 | 53 | if conv: 54 | layers.append({'type': 'conv', 'sx': 5, 'filters': 16, 'stride': 1, 'pad': 2, 'activation': 'relu'}) #, 'drop_prob': 0.5}) 55 | layers.append({'type': 'pool', 'sx': 3, 'stride': 2}) #, 'drop_prob': 0.5}) 56 | layers.append({'type': 'conv', 'sx': 5, 'filters': 20, 'stride': 1, 'pad': 2, 'activation': 'relu'}) #, 'drop_prob': 0.5}) 57 | layers.append({'type': 'pool', 'sx': 2, 'stride': 2}) #, 'drop_prob': 0.5}) 58 | #layers.append({'type': 'lrn', 'alpha': 5 * (10 ** -5), 'beta': 0.75, 'k': 1, 'n': 3, 'drop_prob': 0.5}) 59 | else: 60 | layers.append({'type': 'fc', 'num_neurons': 100, 'activation': 'sigmoid'}) 61 | #layers.append({'type': 'fc', 'num_neurons': 100, 'activation': 'sigmoid'}) 62 | #layers.append({'type': 'fc', 'num_neurons': 100, 'activation': 'sigmoid'}) 63 | layers.append({'type': 'regression', 'num_neurons': dim * dim * depth}) 64 | 65 | print 'Layers made...' 66 | 67 | network = Net(layers) 68 | 69 | print 'Net made...' 70 | print network 71 | 72 | t = Trainer(network, {'method': 'sgd', 'batch_size': 4, 'l2_decay': 0.0001}); 73 | 74 | print 'Trainer made...' 75 | print t 76 | 77 | def train(): 78 | global training_data, testing_data, network, t 79 | 80 | print 'In training...' 81 | print 'k', 'time\t\t ', 'loss\t ' 82 | print '----------------------------------------------------' 83 | try: 84 | for x, y in training_data: 85 | stats = t.train(x, x.w) 86 | print stats['k'], stats['time'], stats['loss'] 87 | except: 88 | return 89 | 90 | def display(python_path, pred, x): 91 | display_cmd = '; '.join([ 92 | 'import cv2, numpy', 93 | 'predw = numpy.rot90(numpy.array({}, dtype=numpy.float64).reshape(32, 32, 3))'.format(pred), 94 | 'cv2.imshow(\'P\', predw)', 95 | 'xw = numpy.rot90(numpy.array({}, dtype=numpy.float64).reshape(32, 32, 3))'.format(x), 96 | 'cv2.imshow(\'X\', xw)', 97 | 'cv2.waitKey(0)', 98 | 'cv2.destroyAllWindows()' 99 | ]) 100 | 101 | cmd = '{} -c \"{}\"'.format(os.path.join(python_path, 'python'), display_cmd) 102 | call(cmd, shell=True) 103 | 104 | def test(path=None, test_n=None): 105 | global training_data, testing_data, network, t 106 | 107 | print 'In autoencoder testing' 108 | 109 | path = path or '/opt/local/bin' 110 | test_n = test_n or 5 111 | 112 | #for x, y in sample(testing_data, test_n): 113 | c = 0 114 | while c < test_n: 115 | x, y = next(testing_data) 116 | print y 117 | pred = network.forward(x).w 118 | display(path, pred, x.w) 119 | c += 1 -------------------------------------------------------------------------------- /examples/cifar/cifar10.py: -------------------------------------------------------------------------------- 1 | from vol import Vol 2 | from vol_util import augment 3 | from net import Net 4 | from trainers import Trainer 5 | from random import sample 6 | import numpy 7 | 8 | training_data = None 9 | testing_data = None 10 | network = None 11 | t = None 12 | 13 | def load_data(crop, gray, training=True): 14 | filename = './data/cifar10_' 15 | if training: 16 | filename += 'train.npz' 17 | else: 18 | filename += 'test.npz' 19 | 20 | data = numpy.load(filename) 21 | xs = data['x'] 22 | ys = data['y'] 23 | 24 | for i in xrange(len(xs)): 25 | V = Vol(32, 32, 3, 0.0) 26 | for d in xrange(3): 27 | for x in xrange(32): 28 | for y in xrange(32): 29 | px = xs[i][x * 32 + y, d] / 255.0 - 0.5 30 | V.set(x, y, d, px) 31 | if crop: 32 | V = augment(V, 24, gray) 33 | 34 | y = ys[i] 35 | yield V, y 36 | 37 | def start(conv, crop, gray): 38 | global training_data, testing_data, network, t 39 | 40 | training_data = load_data(crop, gray) 41 | testing_data = load_data(crop, gray, False) 42 | 43 | print 'Data loaded...' 44 | 45 | layers = [] 46 | 47 | dim = 24 if crop else 32 48 | depth = 1 if gray else 3 49 | layers.append({'type': 'input', 'out_sx': dim, 'out_sy': dim, 'out_depth': depth}) 50 | 51 | if conv: 52 | layers.append({'type': 'conv', 'sx': 5, 'filters': 16, 'stride': 1, 'pad': 2, 'activation': 'relu'}) #, 'drop_prob': 0.5}) 53 | layers.append({'type': 'pool', 'sx': 3, 'stride': 2}) #, 'drop_prob': 0.5}) 54 | layers.append({'type': 'conv', 'sx': 5, 'filters': 20, 'stride': 1, 'pad': 2, 'activation': 'relu'}) #, 'drop_prob': 0.5}) 55 | layers.append({'type': 'pool', 'sx': 2, 'stride': 2}) #, 'drop_prob': 0.5}) 56 | #layers.append({'type': 'lrn', 'alpha': 5 * (10 ** -5), 'beta': 0.75, 'k': 1, 'n': 3, 'drop_prob': 0.5}) 57 | else: 58 | layers.append({'type': 'fc', 'num_neurons': 100, 'activation': 'sigmoid'}) 59 | #layers.append({'type': 'fc', 'num_neurons': 100, 'activation': 'sigmoid'}) 60 | #layers.append({'type': 'fc', 'num_neurons': 100, 'activation': 'sigmoid'}) 61 | layers.append({'type': 'softmax', 'num_classes': 10}) 62 | 63 | print 'Layers made...' 64 | 65 | network = Net(layers) 66 | 67 | print 'Net made...' 68 | print network 69 | 70 | t = Trainer(network, {'method': 'sgd', 'batch_size': 4, 'l2_decay': 0.0001}); 71 | 72 | print 'Trainer made...' 73 | 74 | def train(): 75 | global training_data, testing_data, network, t 76 | 77 | print 'In training...' 78 | print 'k', 'time\t\t ', 'loss\t ', 'training accuracy' 79 | print '----------------------------------------------------' 80 | try: 81 | for x, y in training_data: 82 | stats = t.train(x, y) 83 | print stats['k'], stats['time'], stats['loss'], stats['accuracy'] 84 | except: 85 | return 86 | 87 | def test(): 88 | global training_data, testing_data, network, t 89 | 90 | print 'In testing' 91 | right = 0 92 | count = 0 93 | limit = 100 94 | try: 95 | for x, y in testing_data: 96 | network.forward(x) 97 | right += network.getPrediction() == y 98 | 99 | if count == limit: break 100 | count += 1 101 | print count 102 | except: 103 | pass 104 | finally: 105 | accuracy = float(right) / count * 100 106 | print accuracy -------------------------------------------------------------------------------- /examples/cifar/dataset.py: -------------------------------------------------------------------------------- 1 | # This file transforms the cifar10 python dataset 2 | # into a form readable by pypy. This program 3 | # cannot be run with pypy, needs CPython 4 | 5 | import cPickle, os 6 | import numpy 7 | from sys import argv, exit 8 | 9 | if '-path' not in argv: 10 | raise Exception('Specify a local path to cifar data') 11 | 12 | path = argv[argv.index('-path') + 1] 13 | 14 | def unpickle(file): 15 | fo = open(file, 'rb') 16 | dict = cPickle.load(fo) 17 | fo.close() 18 | return dict 19 | 20 | xs = [] 21 | ys = [] 22 | for i in range(1, 6): 23 | filename = 'data_batch_' + str(i) 24 | d = unpickle(os.path.join(path, filename)) 25 | x = d['data'] 26 | y = d['labels'] 27 | xs.append(x) 28 | ys.append(y) 29 | 30 | x = numpy.concatenate(xs) 31 | x = numpy.dstack((x[:, :1024], x[:, 1024:2048], x[:, 2048:])) 32 | y = numpy.concatenate(ys) 33 | 34 | numpy.savez('./data/cifar10_train.npz', x=x, y=y) 35 | 36 | d = unpickle(os.path.join(path, 'test_batch')) 37 | x = d['data'] 38 | x = numpy.dstack((x[:, :1024], x[:, 1024:2048], x[:, 2048:])) 39 | y = d['labels'] 40 | 41 | numpy.savez('./data/cifar10_test.npz', x=x, y=y) -------------------------------------------------------------------------------- /examples/dark_knowledge.py: -------------------------------------------------------------------------------- 1 | from vol import Vol 2 | from vol_util import augment 3 | from net import Net 4 | from trainers import Trainer 5 | from random import sample 6 | 7 | import os, struct, sys 8 | from array import array as pyarray 9 | 10 | training_data = None 11 | testing_data = None 12 | n = None 13 | t = None 14 | 15 | training_data2 = None 16 | n2 = None 17 | t2 = None 18 | 19 | # Load mnist data 20 | def load_data(training=True): 21 | """Adapted from http://g.sweyla.com/blog/2012/mnist-numpy/""" 22 | path = './data' 23 | 24 | if training: 25 | fname_img = os.path.join(path, 'train-images-idx3-ubyte') 26 | fname_lbl = os.path.join(path, 'train-labels-idx1-ubyte') 27 | else: 28 | fname_img = os.path.join(path, 't10k-images-idx3-ubyte') 29 | fname_lbl = os.path.join(path, 't10k-labels-idx1-ubyte') 30 | 31 | # Inputs 32 | fimg = open(fname_img, 'rb') 33 | magic_nr, size, rows, cols = struct.unpack(">IIII", fimg.read(16)) 34 | imgs = pyarray("B", fimg.read()) 35 | fimg.close() 36 | 37 | imgs = [imgs[n:n+784] for n in xrange(0, len(imgs), 784)] 38 | inputs = [] 39 | V = Vol(28, 28, 1, 0.0) 40 | for img in imgs: 41 | V.w = [ (px / 255.0) for px in img ] 42 | inputs.append(augment(V, 24)) 43 | 44 | # Outputs 45 | flbl = open(fname_lbl, 'rb') 46 | magic_nr, size = struct.unpack(">II", flbl.read(8)) 47 | labels = pyarray("b", flbl.read()) 48 | flbl.close() 49 | 50 | return zip(inputs, labels) 51 | 52 | def run_big_net(): 53 | global training_data, testing_data, n, t, training_data2 54 | 55 | training_data = load_data() 56 | testing_data = load_data(False) 57 | training_data2 = [] 58 | 59 | print 'Data loaded...' 60 | 61 | layers = [] 62 | layers.append({'type': 'input', 'out_sx': 24, 'out_sy': 24, 'out_depth': 1}) 63 | layers.append({'type': 'fc', 'num_neurons': 100, 'activation': 'relu', 'drop_prob': 0.5}) 64 | #layers.append({'type': 'fc', 'num_neurons': 800, 'activation': 'relu', 'drop_prob': 0.5}) 65 | layers.append({'type': 'softmax', 'num_classes': 10}) 66 | print 'Layers made...' 67 | 68 | n = Net(layers) 69 | print 'Net made...' 70 | print n 71 | 72 | t = Trainer(n, {'method': 'sgd', 'momentum': 0.0}) 73 | print 'Trainer made...' 74 | 75 | print 'In training...' 76 | print 'k', 'time\t\t ', 'loss\t ', 'training accuracy' 77 | print '----------------------------------------------------' 78 | try: 79 | for x, y in training_data: 80 | stats = t.train(x, y) 81 | print stats['k'], stats['time'], stats['loss'], stats['accuracy'] 82 | training_data2.append((x, n.getPrediction())) 83 | except: #hit control-c or other 84 | pass 85 | 86 | print 'In testing: 5000 trials' 87 | right = 0 88 | count = 5000 89 | for x, y in sample(testing_data, count): 90 | n.forward(x) 91 | right += n.getPrediction() == y 92 | accuracy = float(right) / count * 100 93 | print accuracy 94 | 95 | def run_small_net(): 96 | global training_data2, n2, t2, testing_data 97 | 98 | layers = [] 99 | layers.append({'type': 'input', 'out_sx': 24, 'out_sy': 24, 'out_depth': 1}) 100 | #layers.append({'type': 'fc', 'num_neurons': 50, 'activation': 'relu'}) 101 | layers.append({'type': 'softmax', 'num_classes': 10}) 102 | print 'Layers made...' 103 | 104 | n2 = Net(layers) 105 | print 'Smaller Net made...' 106 | print n2 107 | 108 | t2 = Trainer(n2, {'method': 'sgd', 'momentum': 0.0}) 109 | print 'Trainer made for smaller net...' 110 | 111 | print 'In training of smaller net...' 112 | print 'k', 'time\t\t ', 'loss\t ', 'training accuracy' 113 | print '----------------------------------------------------' 114 | try: 115 | for x, y in training_data2: 116 | stats = t2.train(x, y) 117 | print stats['k'], stats['time'], stats['loss'], stats['accuracy'] 118 | except: #hit control-c or other 119 | pass 120 | 121 | print 'Testing smaller net: 5000 trials' 122 | right = 0 123 | count = 5000 124 | for x, y in sample(testing_data, count): 125 | n2.forward(x) 126 | right += n2.getPrediction() == y 127 | accuracy = float(right) / count * 100 128 | print accuracy -------------------------------------------------------------------------------- /examples/darkencoder.py: -------------------------------------------------------------------------------- 1 | from vol import Vol 2 | from vol_util import augment 3 | from net import Net 4 | from trainers import Trainer 5 | from random import sample 6 | 7 | import os, struct 8 | from array import array as pyarray 9 | from subprocess import call 10 | 11 | training_data = None 12 | training_data2 = None 13 | testing_data = None 14 | n = None 15 | n2 = None 16 | t = None 17 | t2 = None 18 | 19 | # Load mnist data 20 | def load_data(training=True): 21 | """Adapted from http://g.sweyla.com/blog/2012/mnist-numpy/""" 22 | path = "./data" 23 | 24 | if training: 25 | fname_img = os.path.join(path, 'train-images-idx3-ubyte') 26 | fname_lbl = os.path.join(path, 'train-labels-idx1-ubyte') 27 | else: 28 | fname_img = os.path.join(path, 't10k-images-idx3-ubyte') 29 | fname_lbl = os.path.join(path, 't10k-labels-idx1-ubyte') 30 | 31 | # Inputs 32 | fimg = open(fname_img, 'rb') 33 | magic_nr, size, rows, cols = struct.unpack(">IIII", fimg.read(16)) 34 | imgs = pyarray("B", fimg.read()) 35 | fimg.close() 36 | 37 | imgs = [imgs[n:n+784] for n in xrange(0, len(imgs), 784)] 38 | inputs = [] 39 | for img in imgs: 40 | V = Vol(28, 28, 1, 0.0) 41 | V.w = [ (px / 255.0) for px in img ] 42 | inputs.append(V) 43 | 44 | # Outputs 45 | flbl = open(fname_lbl, 'rb') 46 | magic_nr, size = struct.unpack(">II", flbl.read(8)) 47 | labels = pyarray("b", flbl.read()) 48 | flbl.close() 49 | 50 | return zip(inputs, labels) 51 | 52 | def start(): 53 | global training_data, testing_data, n, t 54 | 55 | training_data = load_data() 56 | testing_data = load_data(False) 57 | 58 | print 'Data loaded...' 59 | 60 | layers = [] 61 | layers.append({'type': 'input', 'out_sx': 28, 'out_sy': 28, 'out_depth': 1}) 62 | layers.append({'type': 'fc', 'num_neurons': 100, 'activation': 'sigmoid'}) 63 | layers.append({'type': 'regression', 'num_neurons': 28 * 28}) 64 | 65 | print 'Layers made...' 66 | 67 | n = Net(layers) 68 | 69 | print 'Net made...' 70 | print n 71 | 72 | t = Trainer(n, {'method': 'adadelta', 'batch_size': 20, 'l2_decay': 0.001}); 73 | 74 | print 'Trainer made...' 75 | 76 | def train(): 77 | global training_data, n, t, training_data2 78 | 79 | print 'In training...' 80 | print 'k', 'time\t\t ', 'loss\t ' 81 | print '----------------------------------------------------' 82 | training_data2 = [] 83 | try: 84 | for x, y in training_data: 85 | stats = t.train(x, x.w) 86 | print stats['k'], stats['time'], stats['loss'] 87 | training_data2.append((Vol(n.forward(x).w), y)) 88 | except: #hit control-c or other 89 | return 90 | 91 | def train2(): 92 | global training_data2, n2, t2 93 | 94 | layers = [] 95 | layers.append({'type': 'input', 'out_sx': 28, 'out_sy': 28, 'out_depth': 1}) 96 | layers.append({'type': 'fc', 'num_neurons': 100, 'activation': 'sigmoid'}) 97 | layers.append({'type': 'softmax', 'num_classes': 10}) 98 | print 'Layers made...' 99 | 100 | n2 = Net(layers) 101 | print 'Net made...' 102 | print n2 103 | 104 | t2 = Trainer(n2, {'method': 'adadelta', 'batch_size': 20, 'l2_decay': 0.001}); 105 | print 'Trainer made...' 106 | 107 | print 'In training of smaller net...' 108 | print 'k', 'time\t\t ', 'loss\t ', 'training accuracy' 109 | print '----------------------------------------------------' 110 | try: 111 | for x, y in training_data2: 112 | stats = t2.train(x, y) 113 | print stats['k'], stats['time'], stats['loss'], stats['accuracy'] 114 | except: #hit control-c or other 115 | return 116 | 117 | def test(): 118 | global testing_data, n2 119 | 120 | print 'Testing smaller net: 5000 trials' 121 | right = 0 122 | count = 5000 123 | for x, y in sample(testing_data, count): 124 | n2.forward(x) 125 | right += n2.getPrediction() == y 126 | accuracy = float(right) / count * 100 127 | print accuracy -------------------------------------------------------------------------------- /examples/dialogue.py: -------------------------------------------------------------------------------- 1 | # Requires nltk, nltk data 2 | 3 | from vol import Vol 4 | from net import Net 5 | from trainers import Trainer 6 | 7 | from nltk import FreqDist 8 | from nltk.corpus import nps_chat as corpus 9 | from string import punctuation 10 | from random import shuffle 11 | 12 | training_data = None 13 | testing_data = None 14 | network = None 15 | t = None 16 | N = 0 17 | words = None 18 | labels = None 19 | 20 | # This list of English stop words is taken from the "Glasgow Information 21 | # Retrieval Group". The original list can be found at 22 | # http://ir.dcs.gla.ac.uk/resources/linguistic_utils/stop_words 23 | ENGLISH_STOP_WORDS = frozenset([ 24 | "a", "about", "above", "across", "after", "afterwards", "again", "against", 25 | "all", "almost", "alone", "along", "already", "also", "although", "always", 26 | "am", "among", "amongst", "amoungst", "amount", "an", "and", "another", 27 | "any", "anyhow", "anyone", "anything", "anyway", "anywhere", "are", 28 | "around", "as", "at", "back", "be", "became", "because", "become", 29 | "becomes", "becoming", "been", "before", "beforehand", "behind", "being", 30 | "below", "beside", "besides", "between", "beyond", "bill", "both", 31 | "bottom", "but", "by", "call", "can", "cannot", "cant", "co", "con", 32 | "could", "couldnt", "cry", "de", "describe", "detail", "do", "done", 33 | "down", "due", "during", "each", "eg", "eight", "either", "eleven", "else", 34 | "elsewhere", "empty", "enough", "etc", "even", "ever", "every", "everyone", 35 | "everything", "everywhere", "except", "few", "fifteen", "fify", "fill", 36 | "find", "fire", "first", "five", "for", "former", "formerly", "forty", 37 | "found", "four", "from", "front", "full", "further", "get", "give", "go", 38 | "had", "has", "hasnt", "have", "he", "hence", "her", "here", "hereafter", 39 | "hereby", "herein", "hereupon", "hers", "herself", "him", "himself", "his", 40 | "how", "however", "hundred", "i", "ie", "if", "in", "inc", "indeed", 41 | "interest", "into", "is", "it", "its", "itself", "keep", "last", "latter", 42 | "latterly", "least", "less", "ltd", "made", "many", "may", "me", 43 | "meanwhile", "might", "mill", "mine", "more", "moreover", "most", "mostly", 44 | "move", "much", "must", "my", "myself", "name", "namely", "neither", 45 | "never", "nevertheless", "next", "nine", "no", "nobody", "none", "noone", 46 | "nor", "not", "nothing", "now", "nowhere", "of", "off", "often", "on", 47 | "once", "one", "only", "onto", "or", "other", "others", "otherwise", "our", 48 | "ours", "ourselves", "out", "over", "own", "part", "per", "perhaps", 49 | "please", "put", "rather", "re", "same", "see", "seem", "seemed", 50 | "seeming", "seems", "serious", "several", "she", "should", "show", "side", 51 | "since", "sincere", "six", "sixty", "so", "some", "somehow", "someone", 52 | "something", "sometime", "sometimes", "somewhere", "still", "such", 53 | "system", "take", "ten", "than", "that", "the", "their", "them", 54 | "themselves", "then", "thence", "there", "thereafter", "thereby", 55 | "therefore", "therein", "thereupon", "these", "they", "thick", "thin", 56 | "third", "this", "those", "though", "three", "through", "throughout", 57 | "thru", "thus", "to", "together", "too", "top", "toward", "towards", 58 | "twelve", "twenty", "two", "un", "under", "until", "up", "upon", "us", 59 | "very", "via", "was", "we", "well", "were", "what", "whatever", "when", 60 | "whence", "whenever", "where", "whereafter", "whereas", "whereby", 61 | "wherein", "whereupon", "wherever", "whether", "which", "while", "whither", 62 | "who", "whoever", "whole", "whom", "whose", "why", "will", "with", 63 | "within", "without", "would", "yet", "you", "your", "yours", "yourself", 64 | "yourselves" 65 | ]) 66 | 67 | def load_data(): 68 | global N, words, labels 69 | 70 | posts = corpus.xml_posts()[:10000] 71 | freqs = [ FreqDist(post.text) for post in posts ] 72 | words = list(set(word 73 | for dist in freqs 74 | for word in dist.keys() 75 | if word not in ENGLISH_STOP_WORDS and 76 | word not in punctuation)) 77 | 78 | labels = list(set([ post.get('class') for post in posts ])) 79 | 80 | data = [] 81 | N = len(words) 82 | for post, dist in zip(posts, freqs): 83 | V = Vol(1, 1, N, 0.0) 84 | for i, word in enumerate(words): 85 | V.w[i] = dist.freq(word) 86 | data.append((V, labels.index(post.get('class')))) 87 | 88 | return data 89 | 90 | def start(): 91 | global training_data, testing_data, network, t, N, labels 92 | 93 | data = load_data() 94 | shuffle(data) 95 | size = int(len(data) * 0.01) 96 | training_data, testing_data = data[size:], data[:size] 97 | print 'Data loaded...' 98 | 99 | layers = [] 100 | layers.append({'type': 'input', 'out_sx': 1, 'out_sy': 1, 'out_depth': N}) 101 | layers.append({'type': 'fc', 'num_neurons': 10, 'activation': 'sigmoid'}) 102 | layers.append({'type': 'softmax', 'num_classes': len(labels)}) 103 | print 'Layers made...' 104 | 105 | network = Net(layers) 106 | 107 | print 'Net made...' 108 | print network 109 | 110 | t = Trainer(network, {'method': 'adadelta', 'batch_size': 10, 'l2_decay': 0.0001}); 111 | 112 | def train(): 113 | global training_data, network, t 114 | 115 | print 'In training...' 116 | print 'k', 'time\t\t ', 'loss\t ', 'training accuracy' 117 | print '----------------------------------------------------' 118 | #while True: 119 | try: 120 | for x, y in training_data: 121 | stats = t.train(x, y) 122 | print stats['k'], stats['time'], stats['loss'], stats['accuracy'] 123 | except KeyboardInterrupt: 124 | return 125 | 126 | def test(): 127 | global testing_data, network 128 | 129 | print 'In testing...' 130 | right = 0 131 | for x, y in testing_data: 132 | network.forward(x) 133 | right += network.getPrediction() == y 134 | accuracy = float(right) / len(testing_data) 135 | print accuracy -------------------------------------------------------------------------------- /examples/faces.py: -------------------------------------------------------------------------------- 1 | # Requires scikit-learn 2 | 3 | from vol_util import augment 4 | from vol import Vol 5 | from net import Net 6 | from trainers import Trainer 7 | 8 | from sklearn.cross_validation import train_test_split 9 | from sklearn.datasets import fetch_lfw_people 10 | 11 | training_data = None 12 | testing_data = None 13 | network = None 14 | t = None 15 | 16 | def load_data(): 17 | global training_data, testing_data 18 | 19 | lfw_people = fetch_lfw_people(min_faces_per_person=70, resize=0.4) 20 | 21 | xs = lfw_people.data 22 | ys = lfw_people.target 23 | 24 | inputs = [] 25 | labels = list(ys) 26 | 27 | for face in xs: 28 | V = Vol(50, 37, 1, 0.0) 29 | V.w = list(face) 30 | inputs.append(augment(V, 30)) 31 | 32 | x_tr, x_te, y_tr, y_te = train_test_split(inputs, labels, test_size=0.25) 33 | 34 | training_data = zip(x_tr, y_tr) 35 | testing_data = zip(x_te, y_te) 36 | 37 | print 'Dataset made...' 38 | 39 | def start(): 40 | global network, t 41 | 42 | layers = [] 43 | layers.append({'type': 'input', 'out_sx': 30, 'out_sy': 30, 'out_depth': 1}) 44 | layers.append({'type': 'fc', 'num_neurons': 100, 'activation': 'sigmoid'}) 45 | layers.append({'type': 'softmax', 'num_classes': 7}) 46 | print 'Layers made...' 47 | 48 | network = Net(layers) 49 | print 'Net made...' 50 | print network 51 | 52 | t = Trainer(network, {'method': 'adadelta', 'batch_size': 20, 'l2_decay': 0.001}) 53 | print 'Trainer made...' 54 | print t 55 | 56 | def train(): 57 | global training_data, network, t 58 | 59 | print 'In training...' 60 | print 'k', 'time\t\t ', 'loss\t ', 'training accuracy' 61 | print '----------------------------------------------------' 62 | try: 63 | for x, y in training_data: 64 | stats = t.train(x, y) 65 | print stats['k'], stats['time'], stats['loss'], stats['accuracy'] 66 | except: #hit control-c or other 67 | return 68 | 69 | def test(): 70 | global training_data, testing_data, network, t 71 | 72 | print 'In testing' 73 | right = 0 74 | count = 0 75 | try: 76 | for x, y in testing_data: 77 | network.forward(x) 78 | right += network.getPrediction() == y 79 | print count 80 | count += 1 81 | except: 82 | pass 83 | finally: 84 | accuracy = float(right) / count * 100 85 | print accuracy -------------------------------------------------------------------------------- /examples/iris.py: -------------------------------------------------------------------------------- 1 | # Requires scikit-learn 2 | 3 | from vol import Vol 4 | from net import Net 5 | from trainers import Trainer 6 | 7 | from sklearn.datasets import load_iris 8 | 9 | iris_data = None 10 | network = None 11 | sgd = None 12 | N_TRAIN = 120 13 | 14 | def load_data(): 15 | global iris_data 16 | 17 | data = load_iris() 18 | 19 | xs = data.data 20 | ys = data.target 21 | 22 | inputs = [ Vol(list(row)) for row in xs ] 23 | labels = list(ys) 24 | 25 | iris_data = zip(inputs, labels) 26 | print 'Data loaded...' 27 | 28 | def start(): 29 | global network, sgd 30 | 31 | layers = [] 32 | layers.append({'type': 'input', 'out_sx': 1, 'out_sy': 1, 'out_depth': 4}) 33 | layers.append({'type': 'softmax', 'num_classes': 3}) #svm works too 34 | print 'Layers made...' 35 | 36 | network = Net(layers) 37 | print 'Net made...' 38 | print network 39 | 40 | sgd = Trainer(network, {'momentum': 0.1, 'l2_decay': 0.001}) 41 | print 'Trainer made...' 42 | print sgd 43 | 44 | def train(): 45 | global iris_data, sgd 46 | 47 | print 'In training...' 48 | print 'k', 'time\t\t ', 'loss\t ', 'training accuracy' 49 | print '----------------------------------------------------' 50 | for x, y in iris_data[:N_TRAIN]: 51 | stats = sgd.train(x, y) 52 | print stats['k'], stats['time'], stats['loss'], stats['accuracy'] 53 | 54 | def test(): 55 | global iris_data, network 56 | 57 | print 'In testing...' 58 | right = 0 59 | for x, y in iris_data[N_TRAIN:]: 60 | network.forward(x) 61 | right += network.getPrediction() == y 62 | accuracy = float(right) / (150 - N_TRAIN) * 100 63 | print accuracy -------------------------------------------------------------------------------- /examples/mnist.py: -------------------------------------------------------------------------------- 1 | from vol import Vol 2 | from vol_util import augment 3 | from net import Net 4 | from trainers import Trainer 5 | from random import sample 6 | 7 | import os, struct, sys 8 | from array import array as pyarray 9 | 10 | training_data = None 11 | testing_data = None 12 | n = None 13 | t = None 14 | 15 | # Load mnist data 16 | def load_data(training=True): 17 | """Adapted from http://g.sweyla.com/blog/2012/mnist-numpy/""" 18 | path = './data' 19 | 20 | if training: 21 | fname_img = os.path.join(path, 'train-images-idx3-ubyte') 22 | fname_lbl = os.path.join(path, 'train-labels-idx1-ubyte') 23 | else: 24 | fname_img = os.path.join(path, 't10k-images-idx3-ubyte') 25 | fname_lbl = os.path.join(path, 't10k-labels-idx1-ubyte') 26 | 27 | # Inputs 28 | fimg = open(fname_img, 'rb') 29 | magic_nr, size, rows, cols = struct.unpack(">IIII", fimg.read(16)) 30 | imgs = pyarray("B", fimg.read()) 31 | fimg.close() 32 | 33 | imgs = [imgs[n:n+784] for n in xrange(0, len(imgs), 784)] 34 | inputs = [] 35 | V = Vol(28, 28, 1, 0.0) 36 | for img in imgs: 37 | V.w = [ (px / 255.0) for px in img ] 38 | inputs.append(augment(V, 24)) 39 | 40 | # Outputs 41 | flbl = open(fname_lbl, 'rb') 42 | magic_nr, size = struct.unpack(">II", flbl.read(8)) 43 | labels = pyarray("b", flbl.read()) 44 | flbl.close() 45 | 46 | return zip(inputs, labels) 47 | 48 | def start(conv): 49 | global training_data, testing_data, n, t 50 | 51 | training_data = load_data() 52 | testing_data = load_data(False) 53 | 54 | print 'Data loaded...' 55 | 56 | layers = [] 57 | layers.append({'type': 'input', 'out_sx': 24, 'out_sy': 24, 'out_depth': 1}) 58 | if conv: 59 | layers.append({'type': 'conv', 'sx': 5, 'filters': 8, 'stride': 1, 'pad': 2, 'activation': 'relu', 'drop_prob': 0.5}) 60 | layers.append({'type': 'pool', 'sx': 2, 'stride': 2, 'drop_prob': 0.5}) 61 | else: 62 | layers.append({'type': 'fc', 'num_neurons': 100, 'activation': 'relu'}) 63 | #layers.append({'type': 'sim', 'num_neurons': 100, 'activation': 'mex'}) 64 | layers.append({'type': 'softmax', 'num_classes': 10}) 65 | 66 | print 'Layers made...' 67 | 68 | n = Net(layers) 69 | 70 | print 'Net made...' 71 | print n 72 | 73 | t = Trainer(n, {'method': 'adadelta', 'batch_size': 20, 'l2_decay': 0.001}) 74 | print 'Trainer made...' 75 | 76 | def train(): 77 | global training_data, testing_data, n, t 78 | 79 | print 'In training...' 80 | print 'k', 'time\t\t ', 'loss\t ', 'training accuracy' 81 | print '----------------------------------------------------' 82 | try: 83 | for x, y in training_data: 84 | stats = t.train(x, y) 85 | print stats['k'], stats['time'], stats['loss'], stats['accuracy'] 86 | except: #hit control-c or other 87 | return 88 | 89 | def test(n_test): 90 | global training_data, testing_data, n, t 91 | 92 | print 'In testing: {} trials'.format(n_test) 93 | right = 0 94 | count = n_test 95 | for x, y in sample(testing_data, count): 96 | n.forward(x) 97 | right += n.getPrediction() == y 98 | accuracy = float(right) / count * 100 99 | print accuracy -------------------------------------------------------------------------------- /examples/next_letter.py: -------------------------------------------------------------------------------- 1 | from util import * 2 | from collections import defaultdict 3 | 4 | from vol import Vol 5 | from net import Net 6 | from trainers import Trainer 7 | from random import randint, choice, sample 8 | 9 | N = 100000 10 | training_data = None 11 | n = None 12 | t = None 13 | frequencies = defaultdict(lambda: 0) 14 | 15 | def normalize(): 16 | global frequencies 17 | 18 | s = 1.0 * sum(frequencies[letter] for letter in frequencies) 19 | for letter in frequencies: 20 | frequencies[letter] /= s 21 | 22 | def load_data(train=True): 23 | global N, frequencies 24 | 25 | with open('./data/big.txt', 'r') as infile: 26 | text = infile.read() 27 | 28 | skip = 3 29 | size = skip * N 30 | start = randint(0, len(text) - size) 31 | content = text[start:start+size] 32 | data = [] 33 | 34 | for i in range(0, len(content), skip): 35 | x1, x2, y = content[i:i+skip] 36 | 37 | l1 = ord(x1) 38 | l2 = ord(x2) 39 | frequencies[l1] += 1 40 | frequencies[l2] += 1 41 | 42 | V = Vol(1, 1, 255, 0.0) 43 | V.w[l1] = 1.0 44 | V.w[l2] = 1.0 45 | label = ord(y) 46 | data.append((V, label)) 47 | 48 | normalize() 49 | 50 | return data 51 | 52 | def start(): 53 | global training_data, n, t 54 | 55 | training_data = load_data() 56 | 57 | print 'Data loaded...' 58 | 59 | layers = [] 60 | layers.append({'type': 'input', 'out_sx': 1, 'out_sy': 1, 'out_depth': 255}) 61 | layers.append({'type': 'fc', 'num_neurons': 100, 'activation': 'sigmoid'}) 62 | layers.append({'type': 'softmax', 'num_classes': 255}) 63 | 64 | print 'Layers made...' 65 | 66 | n = Net(layers) 67 | 68 | print 'Net made...' 69 | print n 70 | 71 | t = Trainer(n, {'method': 'adadelta', 'batch_size': 10, 'l2_decay': 0.0001}); 72 | 73 | print 'Trainer made...' 74 | 75 | def train(): 76 | global training_data, n, t 77 | 78 | print 'In training...' 79 | print 'k', 'time\t\t ', 'loss\t ' 80 | print '----------------------------------------------------' 81 | try: 82 | for x, y in training_data: 83 | stats = t.train(x, y) 84 | print stats['k'], stats['time'], stats['loss'] 85 | except: 86 | return 87 | 88 | def test(): 89 | global n, frequencies 90 | 91 | y = weightedSample(lst=frequencies, count=2) 92 | x = Vol(1, 1, 255, 0.0) 93 | x.w[y[0]] = 1.0 94 | x.w[y[1]] = 1.0 95 | 96 | s = '' 97 | for i in xrange(50): 98 | n.forward(x) 99 | pred = n.getPrediction() 100 | pred2 = weightedSample(lst=frequencies, count=1)[0] 101 | 102 | s += chr(pred) + chr(pred2) 103 | 104 | x.w[pred] = 1.0 105 | 106 | x.w[y[0]] = 0.0 107 | x.w[y[1]] = 0.0 108 | 109 | y = (pred, pred2) 110 | print s -------------------------------------------------------------------------------- /examples/next_word.py: -------------------------------------------------------------------------------- 1 | # Requires nltk, nltk data 2 | 3 | from vol import Vol 4 | from net import Net 5 | from trainers import Trainer 6 | 7 | from nltk import RegexpTokenizer 8 | from nltk.util import ngrams 9 | from nltk.corpus import gutenberg as corpus #can use others like inaugural 10 | 11 | from random import shuffle, sample 12 | from sys import exit 13 | 14 | training_data = None 15 | testing_data = None 16 | network = None 17 | t = None 18 | N = 0 19 | tokens = None 20 | 21 | def load_data(): 22 | global N, words 23 | 24 | raw = list(word 25 | for fileid in corpus.fileids() 26 | for word in corpus.words(fileid)) 27 | words = list(token for token in RegexpTokenizer('\w+').tokenize(' '.join(raw)))[100:1000] 28 | tokens = set(words) 29 | tokens_l = list(tokens) 30 | N = len(tokens) 31 | print 'Corpus size: {} words'.format(N) 32 | 33 | step = 4 34 | data = [] 35 | for gram in ngrams(words, step): 36 | w1, w2, w3, pred = gram 37 | V = Vol(1, 1, N, 0.0) 38 | V.w[tokens_l.index(w1)] = 1 39 | V.w[tokens_l.index(w2)] = 1 40 | V.w[tokens_l.index(w3)] = 1 41 | label = tokens_l.index(pred) 42 | data.append((V, label)) 43 | 44 | return data 45 | 46 | def start(): 47 | global training_data, testing_data, network, t, N 48 | 49 | all_data = load_data() 50 | shuffle(all_data) 51 | size = int(len(all_data) * 0.1) 52 | training_data, testing_data = all_data[size:], all_data[:size] 53 | print 'Data loaded, size: {}...'.format(len(all_data)) 54 | 55 | layers = [] 56 | layers.append({'type': 'input', 'out_sx': 1, 'out_sy': 1, 'out_depth': N}) 57 | layers.append({'type': 'fc', 'num_neurons': 50, 'activation': 'sigmoid'}) 58 | layers.append({'type': 'fc', 'num_neurons': 10, 'activation': 'sigmoid'}) 59 | layers.append({'type': 'fc', 'num_neurons': 50, 'activation': 'sigmoid'}) 60 | layers.append({'type': 'softmax', 'num_classes': N}) 61 | 62 | print 'Layers made...' 63 | 64 | network = Net(layers) 65 | 66 | print 'Net made...' 67 | print network 68 | 69 | t = Trainer(network, {'method': 'adadelta', 'batch_size': 10, 'l2_decay': 0.0001}); 70 | 71 | def train(): 72 | global training_data, network, t 73 | 74 | print 'In training...' 75 | print 'k', 'time\t\t ', 'loss\t ', 'training accuracy' 76 | print '----------------------------------------------------' 77 | try: 78 | for x, y in training_data: 79 | stats = t.train(x, y) 80 | print stats['k'], stats['time'], stats['loss'], stats['accuracy'] 81 | except KeyboardInterrupt: 82 | return 83 | 84 | def test(): 85 | global testing_data, network 86 | 87 | print 'In testing...' 88 | right = 0 89 | for x, y in testing_data: 90 | network.forward(x) 91 | right += network.getPrediction() == y 92 | accuracy = float(right) / len(testing_data) 93 | print accuracy -------------------------------------------------------------------------------- /examples/next_word_embeddings.py: -------------------------------------------------------------------------------- 1 | from vol import Vol 2 | from net import Net 3 | from trainers import Trainer 4 | from util import * 5 | 6 | import os 7 | from random import shuffle, sample, random 8 | from sys import exit 9 | 10 | embeddings = None 11 | training_data = None 12 | testing_data = None 13 | network = None 14 | t = None 15 | N = None 16 | tokens_l = None 17 | 18 | def load_data(): 19 | global embeddings, N, tokens_l 20 | 21 | embeddings = {} 22 | raw = file('./data/word_projections-80.txt').read() 23 | raw = raw[9:] 24 | raw = raw.split('\n') 25 | for elem in raw: 26 | try: 27 | data = elem.split() 28 | word = data[0].lower() 29 | vector = [ float(v) for v in data[1:] ] 30 | embeddings[word] = vector 31 | except: 32 | continue 33 | 34 | path = './data/text/train_tiny' 35 | words = list(token 36 | for fname in os.listdir(path) 37 | for token in file(os.path.join(path, fname)).read().split()) 38 | tokens = set(words) 39 | tokens_l = list(tokens) 40 | N = len(tokens) 41 | print 'Corpus size: {} words'.format(N) 42 | 43 | step = 4 44 | data = [] 45 | for n in xrange(0, len(words) - step): 46 | w1, w2, w3, pred = words[n:n+step] 47 | 48 | if not (w1 in embeddings and w2 in embeddings and w3 in embeddings 49 | and pred in embeddings and pred in tokens): continue 50 | 51 | V = Vol(embeddings[w1] + embeddings[w2] + embeddings[w3]) 52 | label = tokens_l.index(pred) 53 | data.append((V, label)) 54 | 55 | return data 56 | 57 | def start(): 58 | global training_data, testing_data, network, t, N 59 | 60 | all_data = load_data() 61 | shuffle(all_data) 62 | size = int(len(all_data) * 0.1) 63 | training_data, testing_data = all_data[size:], all_data[:size] 64 | print 'Data loaded, size: {}...'.format(len(all_data)) 65 | 66 | layers = [] 67 | layers.append({'type': 'input', 'out_sx': 1, 'out_sy': 1, 'out_depth': 240}) 68 | 69 | layers.append({'type': 'fc', 'num_neurons': 200, 'activation': 'sigmoid'}) 70 | layers.append({'type': 'fc', 'num_neurons': 100, 'activation': 'sigmoid'}) 71 | layers.append({'type': 'fc', 'num_neurons': 50, 'activation': 'sigmoid'}) 72 | layers.append({'type': 'fc', 'num_neurons': 10, 'activation': 'sigmoid'}) 73 | layers.append({'type': 'fc', 'num_neurons': 50, 'activation': 'sigmoid'}) 74 | layers.append({'type': 'fc', 'num_neurons': 100, 'activation': 'sigmoid'}) 75 | 76 | #layers.append({'type': 'conv', 'sx': 1, 'filters': 240, 'pad': 0}) #lookup table like 77 | #layers.append({'type': 'fc', 'num_neurons': 200, 'activation': 'tanh', 'drop_prob': 0.5}) 78 | #layers.append({'type': 'fc', 'num_neurons': 100, 'activation': 'tanh', 'drop_prob': 0.5}) 79 | 80 | layers.append({'type': 'softmax', 'num_classes': N}) 81 | 82 | print 'Layers made...' 83 | 84 | network = Net(layers) 85 | 86 | print 'Net made...' 87 | print network 88 | 89 | t = Trainer(network, {'method': 'adadelta', 'batch_size': 10, 'l2_decay': 0.0001}); 90 | 91 | def train(): 92 | global training_data, network, t 93 | 94 | print 'In training...' 95 | print 'k', 'time\t\t ', 'loss\t ', 'training accuracy' 96 | print '----------------------------------------------------' 97 | try: 98 | for x, y in training_data: 99 | stats = t.train(x, y) 100 | print stats['k'], stats['time'], stats['loss'], stats['accuracy'] 101 | except KeyboardInterrupt: 102 | pass 103 | finally: 104 | saveJSON('./models/next_word_embeddings/network.json', network.toJSON()) 105 | 106 | def test_text(text, ngenerate=10, delete=True): 107 | out = '' 108 | for n in xrange(ngenerate): 109 | x = [] 110 | words = text.split() 111 | for word in words: 112 | if word not in embeddings: 113 | return 'word: {} not in corpus'.format(word) 114 | else: 115 | x.extend(embeddings[word]) 116 | output = network.forward(Vol(x)).w 117 | pred = network.getPrediction() 118 | new = tokens_l[pred] if random() < 0.5 else \ 119 | weightedSample(embeddings.keys(), output) 120 | 121 | out += ' ' + new 122 | text = ' '.join(words[1:] + [new]) 123 | return out 124 | 125 | def test(): 126 | global testing_data, network 127 | 128 | try: 129 | print 'In testing...' 130 | right = 0 131 | for x, y in testing_data: 132 | network.forward(x) 133 | right += network.getPrediction() == y 134 | accuracy = float(right) / len(testing_data) 135 | print accuracy 136 | except KeyboardInterrupt: 137 | pass 138 | finally: 139 | print test_text('the answer is') 140 | print test_text('i did this') -------------------------------------------------------------------------------- /examples/num2img.py: -------------------------------------------------------------------------------- 1 | from vol import Vol 2 | from vol_util import augment 3 | from net import Net 4 | from trainers import Trainer 5 | from random import sample 6 | 7 | import os, struct, sys 8 | from array import array as pyarray 9 | from subprocess import call 10 | 11 | training_data = None 12 | testing_data = None 13 | n = None 14 | t = None 15 | 16 | # Load mnist data 17 | def load_data(training=True): 18 | """Adapted from http://g.sweyla.com/blog/2012/mnist-numpy/""" 19 | path = './data' 20 | 21 | if training: 22 | fname_img = os.path.join(path, 'train-images-idx3-ubyte') 23 | fname_lbl = os.path.join(path, 'train-labels-idx1-ubyte') 24 | else: 25 | fname_img = os.path.join(path, 't10k-images-idx3-ubyte') 26 | fname_lbl = os.path.join(path, 't10k-labels-idx1-ubyte') 27 | 28 | # Inputs 29 | fimg = open(fname_img, 'rb') 30 | magic_nr, size, rows, cols = struct.unpack(">IIII", fimg.read(16)) 31 | imgs = pyarray("B", fimg.read()) 32 | fimg.close() 33 | 34 | imgs = [imgs[n:n+784] for n in xrange(0, len(imgs), 784)] 35 | inputs = [] 36 | for img in imgs: 37 | V = Vol(28, 28, 1, 0.0) 38 | V.w = [ (px / 255.0) for px in img ] 39 | inputs.append(V) 40 | 41 | # Outputs 42 | flbl = open(fname_lbl, 'rb') 43 | magic_nr, size = struct.unpack(">II", flbl.read(8)) 44 | labels = pyarray("b", flbl.read()) 45 | outputs = [] 46 | for lbl in labels: 47 | V2 = Vol(1, 1, 10, 0.0) 48 | V2.w[lbl] = 1 49 | outputs.append(V2) 50 | flbl.close() 51 | 52 | return zip(inputs, outputs) 53 | 54 | def start(): 55 | global training_data, testing_data, n, t 56 | 57 | training_data = load_data() 58 | testing_data = load_data(False) 59 | 60 | print 'Data loaded...' 61 | 62 | layers = [] 63 | layers.append({'type': 'input', 'out_sx': 1, 'out_sy': 1, 'out_depth': 10}) 64 | layers.append({'type': 'fc', 'num_neurons': 100, 'activation': 'sigmoid'}) 65 | layers.append({'type': 'regression', 'num_neurons': 28 * 28}) 66 | print 'Layers made...' 67 | 68 | n = Net(layers) 69 | print 'Net made...' 70 | print n 71 | 72 | t = Trainer(n, {'method': 'sgd', 'batch_size': 20, 'l2_decay': 0.001}); 73 | print 'Trainer made...' 74 | print t 75 | 76 | def train(): 77 | global training_data, testing_data, n, t 78 | 79 | print 'In training...' 80 | print 'k', 'time\t\t ', 'loss\t ' 81 | print '----------------------------------------------------' 82 | try: 83 | for x, y in training_data: 84 | stats = t.train(y, x.w) 85 | print stats['k'], stats['time'], stats['loss'] 86 | except: #hit control-c or other 87 | return 88 | 89 | def display(python_path, pred, x): 90 | display_cmd = '; '.join([ 91 | 'import cv2, numpy', 92 | 'predw = numpy.array({}, dtype=numpy.float64).reshape(28, 28)'.format(pred), 93 | 'cv2.imshow(\'P\', predw)', 94 | 'xw = numpy.array({}, dtype=numpy.float64).reshape(28, 28)'.format(x), 95 | 'cv2.imshow(\'X\', xw)', 96 | 'cv2.waitKey(0)', 97 | 'cv2.destroyAllWindows()' 98 | ]) 99 | 100 | cmd = '{} -c \"{}\"'.format(os.path.join(python_path, 'python'), display_cmd) 101 | call(cmd, shell=True) 102 | 103 | def test(path=None, test_n=None): 104 | global training_data, testing_data, n, t 105 | 106 | print 'In autoencoder testing' 107 | 108 | path = path or '/opt/local/bin' 109 | test_n = test_n or 5 110 | 111 | for x, y in sample(testing_data, test_n): 112 | print y.w.index(1) 113 | pred = n.forward(y).w 114 | display(path, pred, x.w) -------------------------------------------------------------------------------- /examples/plotting: -------------------------------------------------------------------------------- 1 | import cv2, numpy 2 | w = numpy.array(aug.w, dtype=numpy.float64).reshape(24, 24) 3 | dw = numpy.array(aug.dw, dtype=numpy.float64).reshape(24, 24) 4 | cv2.imshow('W', w) 5 | cv2.imshow('DW', dw) 6 | cv2.waitKey(0) 7 | cv2.destroyAllWindows() 8 | -------------------------------------------------------------------------------- /examples/sentiment.py: -------------------------------------------------------------------------------- 1 | from vol import Vol 2 | from net import Net 3 | from trainers import Trainer 4 | from util import * 5 | 6 | import os 7 | from random import shuffle, sample, random 8 | from sys import exit 9 | 10 | embeddings = None 11 | training_data = None 12 | testing_data = None 13 | network = None 14 | t = None 15 | 16 | def load_data(): 17 | global embeddings 18 | 19 | embeddings = {} 20 | raw = file('./data/word_projections-80.txt').read() 21 | raw = raw[9:] 22 | raw = raw.split('\n') 23 | for elem in raw: 24 | try: 25 | data = elem.split() 26 | word = data[0].lower() 27 | vector = [ float(v) for v in data[1:] ] 28 | embeddings[word] = vector 29 | except: 30 | continue 31 | 32 | data = [] 33 | raw = file('./data/sentiment-kaggle/train.tsv').read().split('\n')[1:] 34 | for line in raw: 35 | try: 36 | values = line.split('\t') 37 | phrase = values[2] 38 | sentag = int(values[3]) - 1 39 | 40 | x = [] 41 | for word in phrase.split(): 42 | if word in embeddings: 43 | x.append(embeddings[word]) 44 | 45 | avgs = [0.0] * 80 46 | for n in xrange(80): 47 | for vec in x: 48 | avgs[n] += vec[n] 49 | try: 50 | avgs[n] /= float(len(x)) 51 | except: 52 | avgs[n] = 0.0 53 | 54 | V = Vol(avgs) 55 | data.append((V, sentag)) 56 | except: 57 | continue 58 | 59 | return data 60 | 61 | def start(): 62 | global training_data, testing_data, network, t 63 | 64 | all_data = load_data() 65 | shuffle(all_data) 66 | size = int(len(all_data) * 0.1) 67 | training_data, testing_data = all_data[size:], all_data[:size] 68 | print 'Data loaded, size: {}...'.format(len(all_data)) 69 | 70 | layers = [] 71 | layers.append({'type': 'input', 'out_sx': 1, 'out_sy': 1, 'out_depth': 80}) 72 | layers.append({'type': 'fc', 'num_neurons': 200, 'activation': 'sigmoid'}) 73 | layers.append({'type': 'fc', 'num_neurons': 100, 'activation': 'sigmoid'}) 74 | layers.append({'type': 'fc', 'num_neurons': 50, 'activation': 'sigmoid'}) 75 | layers.append({'type': 'fc', 'num_neurons': 10, 'activation': 'sigmoid'}) 76 | layers.append({'type': 'fc', 'num_neurons': 50, 'activation': 'sigmoid'}) 77 | layers.append({'type': 'fc', 'num_neurons': 100, 'activation': 'sigmoid'}) 78 | layers.append({'type': 'softmax', 'num_classes': 4}) 79 | 80 | print 'Layers made...' 81 | 82 | network = Net(layers) 83 | 84 | print 'Net made...' 85 | print network 86 | 87 | t = Trainer(network, {'method': 'adadelta', 'batch_size': 10, 'l2_decay': 0.0001}); 88 | 89 | def train(): 90 | global training_data, network, t 91 | 92 | print 'In training...' 93 | print 'k', 'time\t\t ', 'loss\t ', 'training accuracy' 94 | print '----------------------------------------------------' 95 | try: 96 | for x, y in training_data: 97 | stats = t.train(x, y) 98 | print stats['k'], stats['time'], stats['loss'], stats['accuracy'] 99 | except KeyboardInterrupt: 100 | return 101 | 102 | def test(): 103 | global testing_data, network 104 | 105 | try: 106 | print 'In testing...' 107 | right = 0 108 | for x, y in testing_data: 109 | network.forward(x) 110 | right += network.getPrediction() == y 111 | accuracy = float(right) / len(testing_data) 112 | print accuracy 113 | except KeyboardInterrupt: 114 | return 115 | 116 | def fill(): 117 | global embeddings 118 | 119 | output = 'PhraseId,Sentiment\n' 120 | raw = file('./data/sentiment-kaggle/test.tsv').read().split('\n')[1:] 121 | for idx, line in enumerate(raw): 122 | try: 123 | values = line.split('\t') 124 | phrase_id = values[0] 125 | phrase = values[2] 126 | 127 | x = [] 128 | for word in phrase.split(): 129 | if word in embeddings: 130 | x.append(embeddings[word]) 131 | 132 | avgs = [0.0] * 80 133 | for n in xrange(80): 134 | for vec in x: 135 | avgs[n] += vec[n] 136 | try: 137 | avgs[n] /= float(len(x)) 138 | except: 139 | avgs[n] = 0.0 140 | 141 | network.forward(Vol(avgs)) 142 | output += '{},{}\n'.format(phrase_id, network.getPrediction() + 1) 143 | 144 | print idx 145 | except: 146 | continue 147 | with open('./data/sentiment-kaggle/out1.csv', 'w') as outfile: 148 | outfile.write(output) 149 | 150 | print 'Done' -------------------------------------------------------------------------------- /examples/similarity.py: -------------------------------------------------------------------------------- 1 | # Requires nltk, nltk data, twython 2 | 3 | from vol import Vol 4 | from net import Net 5 | from trainers import Trainer 6 | 7 | from nltk import FreqDist, RegexpTokenizer 8 | from nltk.corpus import gutenberg as corpus #can use others like inaugural 9 | from string import punctuation 10 | from operator import mul 11 | from math import sqrt 12 | from twython import Twython, TwythonError 13 | 14 | training_data = None 15 | network = None 16 | t = None 17 | N = 0 18 | words = None 19 | 20 | # This list of English stop words is taken from the "Glasgow Information 21 | # Retrieval Group". The original list can be found at 22 | # http://ir.dcs.gla.ac.uk/resources/linguistic_utils/stop_words 23 | ENGLISH_STOP_WORDS = frozenset([ 24 | "a", "about", "above", "across", "after", "afterwards", "again", "against", 25 | "all", "almost", "alone", "along", "already", "also", "although", "always", 26 | "am", "among", "amongst", "amoungst", "amount", "an", "and", "another", 27 | "any", "anyhow", "anyone", "anything", "anyway", "anywhere", "are", 28 | "around", "as", "at", "back", "be", "became", "because", "become", 29 | "becomes", "becoming", "been", "before", "beforehand", "behind", "being", 30 | "below", "beside", "besides", "between", "beyond", "bill", "both", 31 | "bottom", "but", "by", "call", "can", "cannot", "cant", "co", "con", 32 | "could", "couldnt", "cry", "de", "describe", "detail", "do", "done", 33 | "down", "due", "during", "each", "eg", "eight", "either", "eleven", "else", 34 | "elsewhere", "empty", "enough", "etc", "even", "ever", "every", "everyone", 35 | "everything", "everywhere", "except", "few", "fifteen", "fify", "fill", 36 | "find", "fire", "first", "five", "for", "former", "formerly", "forty", 37 | "found", "four", "from", "front", "full", "further", "get", "give", "go", 38 | "had", "has", "hasnt", "have", "he", "hence", "her", "here", "hereafter", 39 | "hereby", "herein", "hereupon", "hers", "herself", "him", "himself", "his", 40 | "how", "however", "hundred", "i", "ie", "if", "in", "inc", "indeed", 41 | "interest", "into", "is", "it", "its", "itself", "keep", "last", "latter", 42 | "latterly", "least", "less", "ltd", "made", "many", "may", "me", 43 | "meanwhile", "might", "mill", "mine", "more", "moreover", "most", "mostly", 44 | "move", "much", "must", "my", "myself", "name", "namely", "neither", 45 | "never", "nevertheless", "next", "nine", "no", "nobody", "none", "noone", 46 | "nor", "not", "nothing", "now", "nowhere", "of", "off", "often", "on", 47 | "once", "one", "only", "onto", "or", "other", "others", "otherwise", "our", 48 | "ours", "ourselves", "out", "over", "own", "part", "per", "perhaps", 49 | "please", "put", "rather", "re", "same", "see", "seem", "seemed", 50 | "seeming", "seems", "serious", "several", "she", "should", "show", "side", 51 | "since", "sincere", "six", "sixty", "so", "some", "somehow", "someone", 52 | "something", "sometime", "sometimes", "somewhere", "still", "such", 53 | "system", "take", "ten", "than", "that", "the", "their", "them", 54 | "themselves", "then", "thence", "there", "thereafter", "thereby", 55 | "therefore", "therein", "thereupon", "these", "they", "thick", "thin", 56 | "third", "this", "those", "though", "three", "through", "throughout", 57 | "thru", "thus", "to", "together", "too", "top", "toward", "towards", 58 | "twelve", "twenty", "two", "un", "under", "until", "up", "upon", "us", 59 | "very", "via", "was", "we", "well", "were", "what", "whatever", "when", 60 | "whence", "whenever", "where", "whereafter", "whereas", "whereby", 61 | "wherein", "whereupon", "wherever", "whether", "which", "while", "whither", 62 | "who", "whoever", "whole", "whom", "whose", "why", "will", "with", 63 | "within", "without", "would", "yet", "you", "your", "yours", "yourself", 64 | "yourselves" 65 | ]) 66 | 67 | def volumize(dist): 68 | global words 69 | 70 | V = Vol(1, 1, N, 0.0) 71 | for i, word in enumerate(words): 72 | V.w[i] = dist.freq(word) 73 | return V 74 | 75 | def load_data(): 76 | global N, words 77 | 78 | freqs = [ FreqDist(corpus.words(fileid)) for fileid in corpus.fileids() ] 79 | words = list(set(word 80 | for dist in freqs 81 | for word in dist.keys() 82 | if word not in ENGLISH_STOP_WORDS and 83 | word not in punctuation)) 84 | 85 | data = [] 86 | N = len(words) 87 | for dist in freqs: 88 | x = volumize(dist) 89 | data.append((x, x.w)) 90 | 91 | return data 92 | 93 | def start(): 94 | global training_data, network, t, N 95 | 96 | training_data = load_data() 97 | print 'Data loaded...' 98 | 99 | layers = [] 100 | layers.append({'type': 'input', 'out_sx': 1, 'out_sy': 1, 'out_depth': N}) 101 | layers.append({'type': 'fc', 'num_neurons': 50, 'activation': 'sigmoid'}) 102 | layers.append({'type': 'fc', 'num_neurons': 10, 'activation': 'sigmoid'}) 103 | layers.append({'type': 'fc', 'num_neurons': 50, 'activation': 'sigmoid'}) 104 | layers.append({'type': 'regression', 'num_neurons': N}) 105 | 106 | print 'Layers made...' 107 | 108 | network = Net(layers) 109 | 110 | print 'Net made...' 111 | print network 112 | 113 | t = Trainer(network, {'method': 'adadelta', 'batch_size': 4, 'l2_decay': 0.0001}); 114 | 115 | def train(): 116 | global training_data, network, t 117 | 118 | print 'In training...' 119 | print 'k', 'time\t\t ', 'loss\t ' 120 | print '----------------------------------------------------' 121 | try: 122 | for x, y in training_data: 123 | stats = t.train(x, y) 124 | print stats['k'], stats['time'], stats['loss'] 125 | except: 126 | return 127 | 128 | class GetTweets(object): 129 | 130 | def __init__(self): 131 | self.APP_KEY = "###" 132 | self.APP_SECRET = "###" 133 | 134 | self.twitter = Twython(self.APP_KEY, self.APP_SECRET, oauth_version=2) 135 | self.ACCESS_TOKEN = self.twitter.obtain_access_token() 136 | self.twitter = Twython(self.APP_KEY, access_token=self.ACCESS_TOKEN) 137 | 138 | def get_user(self, username, count=200): 139 | return [ tweet['text'] 140 | for tweet in self.twitter.get_user_timeline(screen_name=username, count=count) ] 141 | 142 | def get_hashtag(self, hashtag, count=200): 143 | query = '#{}'.format(hashtag) 144 | return [ tweet['text'].replace('#', '') 145 | for tweet in self.twitter.search(q=query, count=count)['statuses'] ] 146 | 147 | def doc_code(v): 148 | global network 149 | 150 | network.forward(v) 151 | return network.layers[4].out_act.w 152 | 153 | def cos(v1, v2): 154 | dot = float(sum(map(mul, v1, v2))) 155 | norm1 = sqrt(sum(e for e in v1)) 156 | norm2 = sqrt(sum(e for e in v2)) 157 | return dot / (norm1 * norm2) 158 | 159 | def test(): 160 | gt = GetTweets() 161 | documents = gt.get_hashtag('ferguson', count=20) 162 | documents += gt.get_hashtag('police', count=21) 163 | print 'Query:', documents[-1] 164 | 165 | tokenizer = RegexpTokenizer('\w+') 166 | vols = [] 167 | for doc in documents: 168 | samples = [] 169 | for token in tokenizer.tokenize(doc): 170 | word = token.lower() 171 | if word not in ENGLISH_STOP_WORDS and word not in punctuation: 172 | samples.append(word) 173 | vols.append(volumize(FreqDist(samples))) 174 | 175 | vectors = [ doc_code(v) for v in vols[:-1] ] 176 | query_vec = doc_code(vols[-1]) 177 | 178 | sims = [ cos(v, query_vec) for v in vectors ] 179 | m = max(sims) 180 | print m, documents[sims.index(m)] -------------------------------------------------------------------------------- /examples/titanic.py: -------------------------------------------------------------------------------- 1 | from vol import Vol 2 | from net import Net 3 | from trainers import Trainer 4 | 5 | training_data = [] 6 | testing_data = [] 7 | network = None 8 | sgd = None 9 | N_TRAIN = 800 10 | 11 | def load_data(): 12 | global training_data, testing_data 13 | 14 | train = [ line.split(',') for line in 15 | file('./data/titanic-kaggle/train.csv').read().split('\n')[1:] ] 16 | for ex in train: 17 | PassengerId,Survived,Pclass,Name,NameRest,Sex,Age,SibSp,Parch,Ticket,Fare,Cabin,Embarked = ex 18 | 19 | # Fixing 20 | sex = 0.0 if Sex == 'male' else 1.0 21 | age = 0 if Age == '' else float(Age) 22 | Embarked = Embarked.replace('\r', '') 23 | if Embarked == 'C': 24 | emb = 0.0 25 | elif Embarked == 'Q': 26 | emb = 1.0 27 | else: 28 | emb = 2.0 29 | 30 | vec = [ float(Pclass), sex, age, float(SibSp), float(Parch), float(Fare), emb ] 31 | v = Vol(vec) 32 | training_data.append((v, int(Survived))) 33 | 34 | test = [ line.split(',') for line in 35 | file('./data/titanic-kaggle/test.csv').read().split('\n')[1:] ] 36 | for ex in test: 37 | PassengerId,Pclass,Name,NameRest,Sex,Age,SibSp,Parch,Ticket,Fare,Cabin,Embarked = ex 38 | 39 | # Fixing 40 | sex = 0.0 if Sex == 'male' else 1.0 41 | age = 0 if Age == '' else float(Age) 42 | Embarked = Embarked.replace('\r', '') 43 | if Embarked == 'C': 44 | emb = 0.0 45 | elif Embarked == 'Q': 46 | emb = 1.0 47 | else: 48 | emb = 2.0 49 | fare = 0 if Fare == '' else float(Fare) 50 | 51 | vec = [ float(Pclass), sex, age, float(SibSp), float(Parch), fare, emb ] 52 | testing_data.append(Vol(vec)) 53 | 54 | print 'Data loaded...' 55 | 56 | def start(): 57 | global network, sgd 58 | 59 | layers = [] 60 | layers.append({'type': 'input', 'out_sx': 1, 'out_sy': 1, 'out_depth': 7}) 61 | #layers.append({'type': 'fc', 'num_neurons': 30, 'activation': 'relu'}) 62 | #layers.append({'type': 'fc', 'num_neurons': 30, 'activation': 'relu'}) 63 | layers.append({'type': 'softmax', 'num_classes': 2}) #svm works too 64 | print 'Layers made...' 65 | 66 | network = Net(layers) 67 | print 'Net made...' 68 | print network 69 | 70 | sgd = Trainer(network, {'momentum': 0.2, 'l2_decay': 0.001}) 71 | print 'Trainer made...' 72 | print sgd 73 | 74 | def train(): 75 | global training_data, sgd 76 | 77 | print 'In training...' 78 | print 'k', 'time\t\t ', 'loss\t ', 'training accuracy' 79 | print '----------------------------------------------------' 80 | for x, y in training_data[:N_TRAIN]: 81 | stats = sgd.train(x, y) 82 | print stats['k'], stats['time'], stats['loss'], stats['accuracy'] 83 | 84 | def test(): 85 | global training_data, network 86 | 87 | print 'In testing...' 88 | right = 0 89 | for x, y in training_data[N_TRAIN:]: 90 | network.forward(x) 91 | right += network.getPrediction() == y 92 | accuracy = float(right) / (len(training_data) - N_TRAIN) * 100 93 | print accuracy -------------------------------------------------------------------------------- /examples/topics.py: -------------------------------------------------------------------------------- 1 | # Requires nltk, nltk data 2 | 3 | from vol import Vol 4 | from net import Net 5 | from trainers import Trainer 6 | 7 | from nltk import FreqDist, RegexpTokenizer 8 | from nltk.corpus import gutenberg as corpus #can use others like inaugural 9 | from string import punctuation 10 | 11 | training_data = None 12 | network = None 13 | t = None 14 | N = 0 15 | words = None 16 | 17 | # This list of English stop words is taken from the "Glasgow Information 18 | # Retrieval Group". The original list can be found at 19 | # http://ir.dcs.gla.ac.uk/resources/linguistic_utils/stop_words 20 | ENGLISH_STOP_WORDS = frozenset([ 21 | "a", "about", "above", "across", "after", "afterwards", "again", "against", 22 | "all", "almost", "alone", "along", "already", "also", "although", "always", 23 | "am", "among", "amongst", "amoungst", "amount", "an", "and", "another", 24 | "any", "anyhow", "anyone", "anything", "anyway", "anywhere", "are", 25 | "around", "as", "at", "back", "be", "became", "because", "become", 26 | "becomes", "becoming", "been", "before", "beforehand", "behind", "being", 27 | "below", "beside", "besides", "between", "beyond", "bill", "both", 28 | "bottom", "but", "by", "call", "can", "cannot", "cant", "co", "con", 29 | "could", "couldnt", "cry", "de", "describe", "detail", "do", "done", 30 | "down", "due", "during", "each", "eg", "eight", "either", "eleven", "else", 31 | "elsewhere", "empty", "enough", "etc", "even", "ever", "every", "everyone", 32 | "everything", "everywhere", "except", "few", "fifteen", "fify", "fill", 33 | "find", "fire", "first", "five", "for", "former", "formerly", "forty", 34 | "found", "four", "from", "front", "full", "further", "get", "give", "go", 35 | "had", "has", "hasnt", "have", "he", "hence", "her", "here", "hereafter", 36 | "hereby", "herein", "hereupon", "hers", "herself", "him", "himself", "his", 37 | "how", "however", "hundred", "i", "ie", "if", "in", "inc", "indeed", 38 | "interest", "into", "is", "it", "its", "itself", "keep", "last", "latter", 39 | "latterly", "least", "less", "ltd", "made", "many", "may", "me", 40 | "meanwhile", "might", "mill", "mine", "more", "moreover", "most", "mostly", 41 | "move", "much", "must", "my", "myself", "name", "namely", "neither", 42 | "never", "nevertheless", "next", "nine", "no", "nobody", "none", "noone", 43 | "nor", "not", "nothing", "now", "nowhere", "of", "off", "often", "on", 44 | "once", "one", "only", "onto", "or", "other", "others", "otherwise", "our", 45 | "ours", "ourselves", "out", "over", "own", "part", "per", "perhaps", 46 | "please", "put", "rather", "re", "same", "see", "seem", "seemed", 47 | "seeming", "seems", "serious", "several", "she", "should", "show", "side", 48 | "since", "sincere", "six", "sixty", "so", "some", "somehow", "someone", 49 | "something", "sometime", "sometimes", "somewhere", "still", "such", 50 | "system", "take", "ten", "than", "that", "the", "their", "them", 51 | "themselves", "then", "thence", "there", "thereafter", "thereby", 52 | "therefore", "therein", "thereupon", "these", "they", "thick", "thin", 53 | "third", "this", "those", "though", "three", "through", "throughout", 54 | "thru", "thus", "to", "together", "too", "top", "toward", "towards", 55 | "twelve", "twenty", "two", "un", "under", "until", "up", "upon", "us", 56 | "very", "via", "was", "we", "well", "were", "what", "whatever", "when", 57 | "whence", "whenever", "where", "whereafter", "whereas", "whereby", 58 | "wherein", "whereupon", "wherever", "whether", "which", "while", "whither", 59 | "who", "whoever", "whole", "whom", "whose", "why", "will", "with", 60 | "within", "without", "would", "yet", "you", "your", "yours", "yourself", 61 | "yourselves" 62 | ]) 63 | 64 | def load_data(): 65 | global N, words 66 | 67 | freqs = [ FreqDist(corpus.words(fileid)) for fileid in corpus.fileids() ] 68 | words = list(set(word 69 | for dist in freqs 70 | for word in dist.keys() 71 | if word not in ENGLISH_STOP_WORDS and 72 | word not in punctuation)) 73 | 74 | data = [] 75 | N = len(words) 76 | for dist in freqs: 77 | V = Vol(1, 1, N, 0.0) 78 | for i, word in enumerate(words): 79 | V.w[i] = dist.freq(word) 80 | data.append((V, V.w)) 81 | 82 | return data 83 | 84 | def start(): 85 | global training_data, network, t, N 86 | 87 | training_data = load_data() 88 | print 'Data loaded...' 89 | 90 | layers = [] 91 | layers.append({'type': 'input', 'out_sx': 1, 'out_sy': 1, 'out_depth': N}) 92 | layers.append({'type': 'fc', 'num_neurons': 50, 'activation': 'sigmoid'}) 93 | layers.append({'type': 'fc', 'num_neurons': 10, 'activation': 'sigmoid'}) 94 | layers.append({'type': 'fc', 'num_neurons': 50, 'activation': 'sigmoid'}) 95 | layers.append({'type': 'regression', 'num_neurons': N}) 96 | 97 | print 'Layers made...' 98 | 99 | network = Net(layers) 100 | 101 | print 'Net made...' 102 | print network 103 | 104 | t = Trainer(network, {'method': 'adadelta', 'batch_size': 4, 'l2_decay': 0.0001}); 105 | 106 | def train(): 107 | global training_data, network, t 108 | 109 | print 'In training...' 110 | print 'k', 'time\t\t ', 'loss\t ' 111 | print '----------------------------------------------------' 112 | try: 113 | for x, y in training_data: 114 | stats = t.train(x, y) 115 | print stats['k'], stats['time'], stats['loss'] 116 | except: 117 | return 118 | 119 | def test(): 120 | global N, words, network 121 | 122 | print 'In testing.' 123 | 124 | gettysburg = """Four score and seven years ago our fathers brought forth on this continent, a new nation, conceived in Liberty, and dedicated to the proposition that all men are created equal. Now we are engaged in a great civil war, testing whether that nation, or any nation so conceived and so dedicated, can long endure. We are met on a great battle-field of that war. We have come to dedicate a portion of that field, as a final resting place for those who here gave their lives that that nation might live. It is altogether fitting and proper that we should do this. But, in a larger sense, we can not dedicate -- we can not consecrate -- we can not hallow -- this ground. The brave men, living and dead, who struggled here, have consecrated it, far above our poor power to add or detract. The world will little note, nor long remember what we say here, but it can never forget what they did here. It is for us the living, rather, to be dedicated here to the unfinished work which they who fought here have thus far so nobly advanced. It is rather for us to be here dedicated to the great task remaining before us -- that from these honored dead we take increased devotion to that cause for which they gave the last full measure of devotion -- that we here highly resolve that these dead shall not have died in vain -- that this nation, under God, shall have a new birth of freedom -- and that government of the people, by the people, for the people, shall not perish from the earth.""" 125 | tokenizer = RegexpTokenizer('\w+') 126 | gettysburg_tokens = tokenizer.tokenize(gettysburg) 127 | 128 | samples = [] 129 | for token in gettysburg_tokens: 130 | word = token.lower() 131 | if word not in ENGLISH_STOP_WORDS and word not in punctuation: 132 | samples.append(word) 133 | 134 | dist = FreqDist(samples) 135 | V = Vol(1, 1, N, 0.0) 136 | for i, word in enumerate(words): 137 | V.w[i] = dist.freq(word) 138 | 139 | pred = network.forward(V).w 140 | topics = [] 141 | while len(topics) != 5: 142 | max_act = max(pred) 143 | topic_idx = pred.index(max_act) 144 | topic = words[topic_idx] 145 | 146 | if topic in gettysburg_tokens: 147 | topics.append(topic) 148 | 149 | del pred[topic_idx] 150 | 151 | print 'Topics of the Gettysburg Address:' 152 | print topics -------------------------------------------------------------------------------- /examples/toy2D.py: -------------------------------------------------------------------------------- 1 | from vol import Vol 2 | from net import Net 3 | from trainers import Trainer 4 | from random import randint, shuffle 5 | 6 | def generate(n, train=True): 7 | data = [] 8 | if train: 9 | for i in range(n/2): data.append(((2 + randint(-1, 1), 7 + randint(-1, 1)), 1)) # top right 10 | for i in range(n/2): data.append(((7.5 + randint(-2, 2), 2 + randint(-1, 1)), 0)) # bottom left 11 | else: 12 | for i in range(n/2): data.append(((2 + randint(-1, 1), 7 + randint(-1, 1)), 1)) # top right 13 | for i in range(n/2): data.append(((7 + randint(-1, 1), 2 + randint(-1, 1)), 0)) # bottom left 14 | return [ (Vol(x), label) for x, label in data ] 15 | 16 | N_train = 1000 17 | N_test = 30 18 | 19 | training_data = generate(N_train) 20 | shuffle(training_data) 21 | 22 | testing_data = generate(N_test, False) 23 | shuffle(testing_data) 24 | 25 | print 'Data loaded...' 26 | 27 | layers = [] 28 | layers.append({'type': 'input', 'out_sx': 1, 'out_sy': 1, 'out_depth': 2}) 29 | layers.append({'type':'softmax', 'num_classes': 2}) 30 | 31 | n = Net(layers) 32 | 33 | print 'Net made...' 34 | print n 35 | 36 | t = Trainer(n, {'momentum': 0.1, 'l2_decay': 0.001}) 37 | 38 | print 'Trainer made...' 39 | 40 | def train(): 41 | print 'In training...' 42 | print 'k', 'time\t\t ', 'loss\t ', 'training accuracy' 43 | print '----------------------------------------------------' 44 | for x, y in training_data: 45 | stats = t.train(x, y) 46 | print stats['k'], stats['time'], stats['loss'], stats['accuracy'] 47 | 48 | def test(): 49 | print 'In testing...' 50 | right = 0 51 | for x, y in testing_data: 52 | n.forward(x) 53 | right += n.getPrediction() == y 54 | accuracy = float(right) / N_test * 100 55 | print accuracy -------------------------------------------------------------------------------- /examples/transforming_autoencoder.py: -------------------------------------------------------------------------------- 1 | from vol import Vol 2 | from vol_util import augment 3 | from net import Net 4 | from trainers import Trainer 5 | from random import sample 6 | 7 | import os, struct 8 | from array import array as pyarray 9 | from subprocess import call 10 | 11 | training_data = None 12 | testing_data = None 13 | n = None 14 | t = None 15 | 16 | # Load mnist data 17 | def load_data(training=True): 18 | """Adapted from http://g.sweyla.com/blog/2012/mnist-numpy/""" 19 | path = "./data" 20 | 21 | if training: 22 | fname_img = os.path.join(path, 'train-images-idx3-ubyte') 23 | fname_lbl = os.path.join(path, 'train-labels-idx1-ubyte') 24 | else: 25 | fname_img = os.path.join(path, 't10k-images-idx3-ubyte') 26 | fname_lbl = os.path.join(path, 't10k-labels-idx1-ubyte') 27 | 28 | # Inputs 29 | fimg = open(fname_img, 'rb') 30 | magic_nr, size, rows, cols = struct.unpack(">IIII", fimg.read(16)) 31 | imgs = pyarray("B", fimg.read()) 32 | fimg.close() 33 | 34 | imgs = [imgs[n:n+784] for n in xrange(0, len(imgs), 784)] 35 | inputs = [] 36 | for img in imgs: 37 | V = Vol(28, 28, 1, 0.0) 38 | pxs = [0.0] * 784 39 | for n in xrange(784): 40 | if n % 28 == 0: 41 | pxs[n] = 0 42 | else: 43 | pxs[n] = img[n - 1] 44 | V.w = pxs 45 | inputs.append(V) 46 | 47 | # Outputs 48 | flbl = open(fname_lbl, 'rb') 49 | magic_nr, size = struct.unpack(">II", flbl.read(8)) 50 | labels = pyarray("b", flbl.read()) 51 | flbl.close() 52 | 53 | return zip(inputs, labels) 54 | 55 | def start(): 56 | global training_data, testing_data, n, t 57 | 58 | training_data = load_data() 59 | testing_data = load_data(False) 60 | 61 | print 'Data loaded...' 62 | 63 | layers = [] 64 | layers.append({'type': 'input', 'out_sx': 28, 'out_sy': 28, 'out_depth': 1}) 65 | layers.append({ 66 | 'type': 'capsule', 'num_neurons': 30, 67 | 'num_recog': 3, 'num_gen': 4, 'num_pose': 2, 68 | 'dx': 1, 'dy': 0 69 | }) 70 | layers.append({'type': 'regression', 'num_neurons': 28 * 28}) 71 | print 'Layers made...' 72 | 73 | n = Net(layers) 74 | 75 | print 'Net made...' 76 | print n 77 | 78 | t = Trainer(n, {'method': 'sgd', 'batch_size': 20, 'l2_decay': 0.001}) 79 | print 'Trainer made...' 80 | 81 | def train(): 82 | global training_data, testing_data, n, t 83 | 84 | print 'In training...' 85 | print 'k', 'time\t\t ', 'loss\t ' 86 | print '----------------------------------------------------' 87 | for x, y in training_data: 88 | stats = t.train(x, x.w) 89 | print stats['k'], stats['time'], stats['loss'] 90 | 91 | def display(python_path, pred, x): 92 | display_cmd = '; '.join([ 93 | 'import cv2, numpy', 94 | 'predw = numpy.array({}, dtype=numpy.float64).reshape(28, 28)'.format(pred), 95 | 'cv2.imshow(\'P\', predw)', 96 | 'xw = numpy.array({}, dtype=numpy.float64).reshape(28, 28)'.format(x), 97 | 'cv2.imshow(\'X\', xw)', 98 | 'cv2.waitKey(0)', 99 | 'cv2.destroyAllWindows()' 100 | ]) 101 | 102 | cmd = '{} -c \"{}\"'.format(os.path.join(python_path, 'python'), display_cmd) 103 | call(cmd, shell=True) 104 | 105 | def test(path=None, test_n=None): 106 | global training_data, testing_data, n, t 107 | 108 | print 'In autoencoder testing' 109 | 110 | path = path or '/opt/local/bin' 111 | test_n = test_n or 5 112 | 113 | for x, y in sample(testing_data, test_n): 114 | print y 115 | pred = n.forward(x).w 116 | display(path, pred, x.w) -------------------------------------------------------------------------------- /examples/udacity_terrain.py: -------------------------------------------------------------------------------- 1 | from vol import Vol 2 | from net import Net 3 | from trainers import Trainer 4 | 5 | import random 6 | 7 | training_data = [] 8 | testing_data = [] 9 | 10 | def makeTerrainData(n_points=1000): 11 | global training_data, testing_data 12 | 13 | ############################################################################### 14 | ### from: https://github.com/udacity/ud120-projects/blob/master/choose_your_own/prep_terrain_data.py 15 | ### make the toy dataset 16 | random.seed(42) 17 | grade = [random.random() for ii in range(0,n_points)] 18 | bumpy = [random.random() for ii in range(0,n_points)] 19 | error = [random.random() for ii in range(0,n_points)] 20 | y = [round(grade[ii]*bumpy[ii]+0.3+0.1*error[ii]) for ii in range(0,n_points)] 21 | for ii in range(0, len(y)): 22 | if grade[ii]>0.8 or bumpy[ii]>0.8: 23 | y[ii] = 1.0 24 | 25 | ### split into train/test sets 26 | X = [[gg, ss] for gg, ss in zip(grade, bumpy)] 27 | split = int(0.75*n_points) 28 | X_train = X[0:split] 29 | X_test = X[split:] 30 | y_train = y[0:split] 31 | y_test = y[split:] 32 | 33 | for x, y in zip(X_train, y_train): 34 | training_data.append(( Vol(x), int(y) )) 35 | for x, y in zip(X_test, y_test): 36 | testing_data.append(( Vol(x), int(y) )) 37 | 38 | makeTerrainData(10000) 39 | 40 | layers = [] 41 | layers.append({'type': 'input', 'out_sx': 1, 'out_sy': 1, 'out_depth': 2}) 42 | layers.append({'type': 'fc', 'num_neurons': 45, 'activation': 'relu'}) 43 | layers.append({'type':'softmax', 'num_classes': 2}) 44 | 45 | n = Net(layers) 46 | 47 | print 'Net made...' 48 | print n 49 | 50 | t = Trainer(n, {'momentum': 0.14}) 51 | 52 | print 'Trainer made...' 53 | 54 | def train(): 55 | print 'In training...' 56 | print 'k', 'time\t\t ', 'loss\t ', 'training accuracy' 57 | print '----------------------------------------------------' 58 | for x, y in training_data: 59 | stats = t.train(x, y) 60 | print stats['k'], stats['time'], stats['loss'], stats['accuracy'] 61 | 62 | def test(): 63 | print 'In testing...' 64 | right = 0 65 | for x, y in testing_data: 66 | n.forward(x) 67 | right += n.getPrediction() == y 68 | accuracy = float(right) / len(testing_data) * 100 69 | print accuracy -------------------------------------------------------------------------------- /layers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/benglard/ConvNetPy/c174bbbe86b4292419fbc83acb7fbffca995e59a/layers/__init__.py -------------------------------------------------------------------------------- /layers/dotproducts.py: -------------------------------------------------------------------------------- 1 | from util import getopt, zeros 2 | from vol import Vol 3 | from math import floor 4 | 5 | """This file contains all layers that do dot products with input""" 6 | 7 | class ConvLayer(object): 8 | 9 | """ 10 | Performs convolutions: spatial weight sharing 11 | http://deeplearning.net/tutorial/lenet.html 12 | """ 13 | 14 | def __init__(self, opt={}): 15 | self.out_depth = opt['filters'] 16 | self.sx = opt['sx'] # filter size: should be odd if possible 17 | self.in_depth = opt['in_depth'] 18 | self.in_sx = opt['in_sx'] 19 | self.in_sy = opt['in_sy'] 20 | 21 | # optional 22 | self.sy = getopt(opt, 'sy', self.sx) 23 | self.stride = getopt(opt, 'stride', 1) # stride at which we apply filters to input volume 24 | self.pad = getopt(opt, 'pad', 0) # padding to borders of input volume 25 | self.l1_decay_mul = getopt(opt, 'l1_decay_mul', 0.0) 26 | self.l2_decay_mul = getopt(opt, 'l2_decay_mul', 1.0) 27 | 28 | """ 29 | Note we are doing floor, so if the strided convolution of the filter doesnt fit into the input 30 | volume exactly, the output volume will be trimmed and not contain the (incomplete) computed 31 | final application. 32 | """ 33 | self.out_sx = int(floor((self.in_sx - self.sx + 2 * self.pad) / self.stride + 1)) 34 | self.out_sy = int(floor((self.in_sy - self.sy + 2 * self.pad) / self.stride + 1)) 35 | self.layer_type = 'conv' 36 | 37 | bias = getopt(opt, 'bias_pref', 0.0) 38 | self.filters = [ Vol(self.sx, self.sy, self.in_depth) for i in xrange(self.out_depth) ] 39 | self.biases = Vol(1, 1, self.out_depth, bias) 40 | 41 | def forward(self, V, is_training): 42 | self.in_act = V 43 | A = Vol(self.out_sx, self.out_sy, self.out_depth, 0.0) 44 | 45 | v_sx = V.sx 46 | v_sy = V.sy 47 | xy_stride = self.stride 48 | 49 | for d in xrange(self.out_depth): 50 | f = self.filters[d] 51 | x = -self.pad 52 | y = -self.pad 53 | 54 | for ay in xrange(self.out_sy): 55 | x = -self.pad 56 | for ax in xrange(self.out_sx): 57 | # convolve centered at this particular location 58 | sum_a = 0.0 59 | for fy in xrange(f.sy): 60 | off_y = y + fy 61 | for fx in xrange(f.sx): 62 | # coordinates in the original input array coordinates 63 | off_x = x + fx 64 | if off_y >= 0 and off_y < V.sy and off_x >= 0 and off_x < V.sx: 65 | for fd in xrange(f.depth): 66 | sum_a += f.w[((f.sx * fy) + fx) * f.depth + fd] \ 67 | * V.w[((V.sx * off_y) + off_x) * V.depth + fd] 68 | 69 | sum_a += self.biases.w[d] 70 | A.set(ax, ay, d, sum_a) 71 | 72 | x += xy_stride 73 | y += xy_stride 74 | 75 | self.out_act = A 76 | return self.out_act 77 | 78 | def backward(self): 79 | # compute gradient wrt weights, biases and input data 80 | V = self.in_act 81 | V.dw = zeros(len(V.w)) # zero out gradient 82 | 83 | V_sx = V.sx 84 | V_sy = V.sy 85 | xy_stride = self.stride 86 | 87 | for d in xrange(self.out_depth): 88 | f = self.filters[d] 89 | x = -self.pad 90 | y = -self.pad 91 | for ay in xrange(self.out_sy): 92 | x = -self.pad 93 | for ax in xrange(self.out_sx): 94 | # convolve and add up the gradients 95 | chain_grad = self.out_act.get_grad(ax, ay, d) # gradient from above, from chain rule 96 | for fy in xrange(f.sy): 97 | off_y = y + fy 98 | for fx in xrange(f.sx): 99 | off_x = x + fx 100 | if off_y >= 0 and off_y < V_sy and off_x >= 0 and off_x < V_sx: 101 | # forward prop calculated: a += f.get(fx, fy, fd) * V.get(ox, oy, fd) 102 | #f.add_grad(fx, fy, fd, V.get(off_x, off_y, fd) * chain_grad) 103 | #V.add_grad(off_x, off_y, fd, f.get(fx, fy, fd) * chain_grad) 104 | for fd in xrange(f.depth): 105 | ix1 = ((V.sx * off_y) + off_x) * V.depth + fd 106 | ix2 = ((f.sx * fy) + fx) * f.depth + fd 107 | f.dw[ix2] += V.w[ix1] * chain_grad 108 | V.dw[ix1] += f.w[ix2] * chain_grad 109 | 110 | self.biases.dw[d] += chain_grad 111 | x += xy_stride 112 | y += xy_stride 113 | 114 | def getParamsAndGrads(self): 115 | response = [] 116 | for d in xrange(self.out_depth): 117 | response.append({ 118 | 'params': self.filters[d].w, 119 | 'grads': self.filters[d].dw, 120 | 'l2_decay_mul': self.l2_decay_mul, 121 | 'l1_decay_mul': self.l1_decay_mul 122 | }) 123 | response.append({ 124 | 'params': self.biases.w, 125 | 'grads': self.biases.dw, 126 | 'l2_decay_mul': 0.0, 127 | 'l1_decay_mul': 0.0 128 | }) 129 | return response 130 | 131 | def toJSON(self): 132 | return { 133 | 'sx' : self.sx, 134 | 'sy' : self.sy, 135 | 'stride' : self.stride, 136 | 'in_depth' : self.in_depth, 137 | 'out_depth' : self.out_depth, 138 | 'out_sx' : self.out_sx, 139 | 'out_sy' : self.out_sy, 140 | 'layer_type' : self.layer_type, 141 | 'l1_decay_mul': self.l1_decay_mul, 142 | 'l2_decay_mul': self.l2_decay_mul, 143 | 'pad' : self.pad, 144 | 'filters' : [ f.toJSON() for f in self.filters ], 145 | 'biases' : self.biases.toJSON() 146 | } 147 | 148 | def fromJSON(self, json): 149 | self.sx = json['sx'] 150 | self.sy = json['sy'] 151 | self.stride = json['stride'] 152 | self.in_depth = json['in_depth'] 153 | self.out_depth = json['out_depth'] 154 | self.out_sx = json['out_sx'] 155 | self.out_sy = json['out_sy'] 156 | self.layer_type = json['layer_type'] 157 | self.l1_decay_mul = json['l1_decay_mul'] 158 | self.l2_decay_mul = json['l2_decay_mul'] 159 | self.pad = json['pad'] 160 | self.filters = [ Vol(0, 0, 0, 0).fromJSON(f) for f in json['filters'] ] 161 | self.biases = Vol(0, 0, 0, 0).fromJSON(json['biases']) 162 | 163 | class FullyConnectedLayer(object): 164 | 165 | """ 166 | Fully connected dot products, ie. multi-layer perceptron net 167 | Building block for most networks 168 | http://www.deeplearning.net/tutorial/mlp.html 169 | """ 170 | 171 | def __init__(self, opt={}): 172 | self.out_depth = opt['num_neurons'] 173 | self.l1_decay_mul = getopt(opt, 'l1_decay_mul', 0.0) 174 | self.l2_decay_mul = getopt(opt, 'l2_decay_mul', 1.0) 175 | 176 | self.num_inputs = opt['in_sx'] * opt['in_sy'] * opt['in_depth'] 177 | self.out_sx = 1 178 | self.out_sy = 1 179 | self.layer_type = 'fc' 180 | 181 | bias = getopt(opt, 'bias_pref', 0.0) 182 | self.filters = [ Vol(1, 1, self.num_inputs) for i in xrange(self.out_depth) ] 183 | self.biases = Vol(1, 1, self.out_depth, bias) 184 | 185 | def forward(self, V, in_training): 186 | self.in_act = V 187 | A = Vol(1, 1, self.out_depth, 0.0) 188 | Vw = V.w 189 | 190 | # dot(W, x) + b 191 | for i in xrange(self.out_depth): 192 | sum_a = 0.0 193 | fiw = self.filters[i].w 194 | for d in xrange(self.num_inputs): 195 | sum_a += Vw[d] * fiw[d] 196 | sum_a += self.biases.w[i] 197 | A.w[i] = sum_a 198 | 199 | self.out_act = A 200 | return self.out_act 201 | 202 | def backward(self): 203 | V = self.in_act 204 | V.dw = zeros(len(V.w)) # zero out gradient 205 | 206 | # compute gradient wrt weights and data 207 | for i in xrange(self.out_depth): 208 | fi = self.filters[i] 209 | chain_grad = self.out_act.dw[i] 210 | 211 | for d in xrange(self.num_inputs): 212 | V.dw[d] += fi.w[d] * chain_grad #grad wrt input data 213 | fi.dw[d] += V.w[d] * chain_grad #grad wrt params 214 | 215 | self.biases.dw[i] += chain_grad 216 | 217 | def getParamsAndGrads(self): 218 | response = [] 219 | for d in xrange(self.out_depth): 220 | response.append({ 221 | 'params': self.filters[d].w, 222 | 'grads': self.filters[d].dw, 223 | 'l2_decay_mul': self.l2_decay_mul, 224 | 'l1_decay_mul': self.l1_decay_mul 225 | }) 226 | response.append({ 227 | 'params': self.biases.w, 228 | 'grads': self.biases.dw, 229 | 'l2_decay_mul': 0.0, 230 | 'l1_decay_mul': 0.0 231 | }) 232 | return response 233 | 234 | def toJSON(self): 235 | return { 236 | 'out_depth' : self.out_depth, 237 | 'out_sx' : self.out_sx, 238 | 'out_sy' : self.out_sy, 239 | 'layer_type' : self.layer_type, 240 | 'num_inputs' : self.num_inputs, 241 | 'l1_decay_mul': self.l1_decay_mul, 242 | 'l2_decay_mul': self.l2_decay_mul, 243 | 'filters' : [ f.toJSON() for f in self.filters ], 244 | 'biases' : self.biases.toJSON() 245 | } 246 | 247 | def fromJSON(self, json): 248 | self.out_depth = json['out_depth'] 249 | self.out_sx = json['out_sx'] 250 | self.out_sy = json['out_sy'] 251 | self.layer_type = json['layer_type'] 252 | self.num_inputs = json['num_inputs'] 253 | self.l1_decay_mul = json['l1_decay_mul'] 254 | self.l2_decay_mul = json['l2_decay_mul'] 255 | self.filters = [ Vol(0, 0, 0, 0).fromJSON(f) for f in json['filters'] ] 256 | self.biases = Vol(0, 0, 0, 0).fromJSON(json['biases']) -------------------------------------------------------------------------------- /layers/dropout.py: -------------------------------------------------------------------------------- 1 | from util import getopt, zeros 2 | from random import random 3 | 4 | class DropoutLayer(object): 5 | 6 | """ 7 | Randomly omits half the feature detectors on each training case. 8 | Each neuron tends to learn something 'useful'. 9 | http://arxiv.org/pdf/1207.0580.pdf 10 | """ 11 | 12 | def __init__(self, opt={}): 13 | self.out_sx = opt['in_sx'] 14 | self.out_sy = opt['in_sy'] 15 | self.out_depth = opt['in_depth'] 16 | self.layer_type = 'dropout' 17 | self.drop_prob = getopt(opt, 'drop_prob', 0.5) 18 | self.dropped = zeros(self.out_sx * self.out_sy * self.out_depth) 19 | 20 | def forward(self, V, is_training=False): 21 | self.in_act = V 22 | V2 = V.clone() 23 | N = len(V.w) 24 | 25 | if is_training: 26 | # do dropout 27 | for i in xrange(N): 28 | if random() < self.drop_prob: # drop 29 | V2.w[i] = 0.0 30 | self.dropped[i] = True 31 | else: 32 | self.dropped[i] = False 33 | else: 34 | # scale the activations during prediction 35 | for i in xrange(N): 36 | V2.w[i] *= self.drop_prob 37 | 38 | self.out_act = V2 39 | return self.out_act 40 | 41 | def backward(self): 42 | V = self.in_act 43 | chain_grad = self.out_act 44 | N = len(V.w) 45 | V.dw = zeros(N) # zero out gradient wrt data 46 | for i in xrange(N): 47 | if not self.dropped[i]: 48 | V.dw[i] = chain_grad.dw[i] # copy over the gradient 49 | 50 | def getParamsAndGrads(self): 51 | return [] 52 | 53 | def toJSON(self): 54 | return { 55 | 'out_depth' : self.out_depth, 56 | 'out_sx' : self.out_sx, 57 | 'out_sy' : self.out_sy, 58 | 'layer_type': self.layer_type, 59 | 'drop_prob' : self.drop_prob 60 | } 61 | 62 | def fromJSON(self, json): 63 | self.out_depth = json['out_depth'] 64 | self.out_sx = json['out_sx'] 65 | self.out_sy = json['out_sy'] 66 | self.layer_type = json['layer_type'] 67 | self.drop_prob = json['drop_prob'] -------------------------------------------------------------------------------- /layers/input.py: -------------------------------------------------------------------------------- 1 | from util import getopt 2 | 3 | class InputLayer(object): 4 | 5 | def __init__(self, opt={}): 6 | self.out_sx = opt['out_sx'] 7 | self.out_sy = opt['out_sy'] 8 | self.out_depth = opt['out_depth'] 9 | self.layer_type = 'input' 10 | 11 | def forward(self, V, is_training): 12 | self.in_act = V 13 | self.out_act = V 14 | return self.out_act 15 | 16 | def backward(self): 17 | pass 18 | 19 | def getParamsAndGrads(self): 20 | return [] 21 | 22 | def toJSON(self): 23 | return { 24 | 'out_depth' : self.out_depth, 25 | 'out_sx' : self.out_sx, 26 | 'out_sy' : self.out_sy, 27 | 'layer_type': self.layer_type 28 | } 29 | 30 | def fromJSON(self, json): 31 | self.out_depth = json['out_depth'] 32 | self.out_sx = json['out_sx'] 33 | self.out_sy = json['out_sy'] 34 | self.layer_type = json['layer_type'] -------------------------------------------------------------------------------- /layers/loss.py: -------------------------------------------------------------------------------- 1 | from util import getopt, zeros 2 | from vol import Vol 3 | from math import exp, log 4 | 5 | """ 6 | Layers that implement a loss. Currently these are the layers that 7 | can initiate a backward() pass. In future we probably want a more 8 | flexible system that can accomodate multiple losses to do multi-task 9 | learning, and stuff like that. But for now, one of the layers in this 10 | file must be the final layer in a Net. 11 | """ 12 | 13 | class SoftmaxLayer(object): 14 | 15 | """ 16 | This is a classifier, with N discrete classes from 0 to N-1 17 | it gets a stream of N incoming numbers and computes the softmax 18 | function (exponentiate and normalize to sum to 1 as probabilities should) 19 | """ 20 | 21 | def __init__(self, opt={}): 22 | self.num_inputs = opt['in_sx'] * opt['in_sy'] * opt['in_depth'] 23 | self.out_depth = self.num_inputs 24 | self.out_sx = 1 25 | self.out_sy = 1 26 | self.layer_type = 'softmax' 27 | self.old = None 28 | 29 | def forward(self, V, is_training): 30 | self.in_act = V 31 | A = Vol(1, 1, self.out_depth, 0.0) 32 | 33 | # max activation 34 | max_act = max(V.w) 35 | 36 | # compute exponentials (carefully to not blow up) 37 | # normalize 38 | exps = [ exp(w - max_act) for w in V.w ] 39 | exps_sum = float(sum(exps)) 40 | exps_norm = [ elem / exps_sum for elem in exps ] 41 | 42 | self.es = exps_norm 43 | A.w = exps_norm 44 | 45 | self.out_act = A 46 | return self.out_act 47 | 48 | def backward(self, y): 49 | # compute and accumulate gradient wrt weights and bias of this layer 50 | x = self.in_act 51 | x.dw = zeros(len(x.w)) 52 | 53 | for i in xrange(self.out_depth): 54 | indicator = float(i == y) 55 | mul = - (indicator - self.es[i]) 56 | x.dw[i] = mul 57 | 58 | # loss is the class negative log likelihood 59 | try: 60 | return -log(self.es[y]) 61 | except ValueError: 62 | return -log(0.001) 63 | 64 | def getParamsAndGrads(self): 65 | return [] 66 | 67 | def toJSON(self): 68 | return { 69 | 'out_depth' : self.out_depth, 70 | 'out_sx' : self.out_sx, 71 | 'out_sy' : self.out_sy, 72 | 'layer_type': self.layer_type, 73 | 'num_inputs': self.num_inputs 74 | } 75 | 76 | def fromJSON(self, json): 77 | self.out_depth = json['out_depth'] 78 | self.out_sx = json['out_sx'] 79 | self.out_sy = json['out_sy'] 80 | self.layer_type = json['layer_type'] 81 | self.num_inputs = json['num_inputs'] 82 | 83 | class RegressionLayer(object): 84 | 85 | """ 86 | Implements an L2 regression cost layer, 87 | so penalizes \sum_i(||x_i - y_i||^2), where x is its input 88 | and y is the user-provided array of "correct" values. 89 | """ 90 | 91 | def __init__(self, opt={}): 92 | self.num_inputs = opt['in_sx'] * opt['in_sy'] * opt['in_depth'] 93 | self.out_depth = self.num_inputs 94 | self.out_sx = 1 95 | self.out_sy = 1 96 | self.layer_type = 'regression' 97 | 98 | def forward(self, V, is_training): 99 | self.in_act = V 100 | self.out_act = V 101 | return V 102 | 103 | def backward(self, y): 104 | # y is a list here of size num_inputs 105 | # compute and accumulate gradient wrt weights and bias of this layer 106 | x = self.in_act 107 | x.dw = zeros(len(x.w)) # zero out the gradient of input Vol 108 | loss = 0.0 109 | 110 | if type(y) == list: 111 | for i in xrange(self.out_depth): 112 | dy = x.w[i] - y[i] 113 | x.dw[i] = dy 114 | loss += 2 * dy * dy 115 | else: 116 | # assume it is a dict with entries dim and val 117 | # and we pass gradient only along dimension dim to be equal to val 118 | i = y['dim'] 119 | y_i = y['val'] 120 | dy = x.w[i] - y_i 121 | x.dw[i] = dy 122 | loss += 2 * dy * dy 123 | 124 | return loss 125 | 126 | def getParamsAndGrads(self): 127 | return [] 128 | 129 | def toJSON(self): 130 | return { 131 | 'out_depth' : self.out_depth, 132 | 'out_sx' : self.out_sx, 133 | 'out_sy' : self.out_sy, 134 | 'layer_type': self.layer_type, 135 | 'num_inputs': self.num_inputs 136 | } 137 | 138 | def fromJSON(self, json): 139 | self.out_depth = json['out_depth'] 140 | self.out_sx = json['out_sx'] 141 | self.out_sy = json['out_sy'] 142 | self.layer_type = json['layer_type'] 143 | self.num_inputs = json['num_inputs'] 144 | 145 | class SVMLayer(object): 146 | 147 | """Linear SVM classifier""" 148 | 149 | def __init__(self, opt={}): 150 | self.num_inputs = opt['in_sx'] * opt['in_sy'] * opt['in_depth'] 151 | self.out_depth = self.num_inputs 152 | self.out_sx = 1 153 | self.out_sy = 1 154 | self.layer_type = 'svm' 155 | 156 | def forward(self, V, is_training): 157 | self.in_act = V 158 | self.out_act = V 159 | return V 160 | 161 | def backward(self, y): 162 | # compute and accumulate gradient wrt weights and bias of this layer 163 | x = self.in_act 164 | x.dw = zeros(len(x.w)) # zero out the gradient of input Vol 165 | 166 | yscore = x.w[y] 167 | margin = 1.0 168 | loss = 0.0 169 | for i in xrange(self.out_depth): 170 | if -yscore + x.w[i] + margin > 0: 171 | # Hinge loss: http://en.wikipedia.org/wiki/Hinge_loss 172 | x.dw[i] += 1 173 | x.dw[y] -= 1 174 | loss += -yscore + x.w[i] + margin 175 | 176 | return loss 177 | 178 | def getParamsAndGrads(self): 179 | return [] 180 | 181 | def toJSON(self): 182 | return { 183 | 'out_depth' : self.out_depth, 184 | 'out_sx' : self.out_sx, 185 | 'out_sy' : self.out_sy, 186 | 'layer_type': self.layer_type, 187 | 'num_inputs': self.num_inputs 188 | } 189 | 190 | def fromJSON(self, json): 191 | self.out_depth = json['out_depth'] 192 | self.out_sx = json['out_sx'] 193 | self.out_sy = json['out_sy'] 194 | self.layer_type = json['layer_type'] 195 | self.num_inputs = json['num_inputs'] -------------------------------------------------------------------------------- /layers/nonlinearities.py: -------------------------------------------------------------------------------- 1 | from util import getopt, zeros 2 | from vol import Vol 3 | from math import exp, tanh, log 4 | 5 | # Helper functions 6 | 7 | def relu(x): 8 | return max(0, x) 9 | 10 | def sigmoid(x): 11 | try: 12 | return 1.0 / (1 + exp(-x)) 13 | except: 14 | return 0 15 | 16 | class ReluLayer(object): 17 | 18 | """ 19 | Implements ReLU nonlinearity elementwise 20 | x -> max(0, x) 21 | the output is in [0, inf) 22 | """ 23 | 24 | def __init__(self, opt={}): 25 | self.out_sx = opt['in_sx'] 26 | self.out_sy = opt['in_sy'] 27 | self.out_depth = opt['in_depth'] 28 | self.layer_type = 'relu' 29 | 30 | def forward(self, V, is_training=False): 31 | self.in_act = V 32 | V2 = V.clone() 33 | V2.w = map(relu, V.w) 34 | self.out_act = V2 35 | return self.out_act 36 | 37 | def backward(self): 38 | V = self.in_act 39 | V2 = self.out_act 40 | N = len(V.w) 41 | V.dw = zeros(N) # zero out gradient wrt data 42 | for i in xrange(N): 43 | if V2.w[i] <= 0: # threshold 44 | V.dw[i] = 0 45 | else: 46 | V.dw[i] = V2.dw[i] 47 | 48 | def getParamsAndGrads(self): 49 | return [] 50 | 51 | def toJSON(self): 52 | return { 53 | 'out_depth' : self.out_depth, 54 | 'out_sx' : self.out_sx, 55 | 'out_sy' : self.out_sy, 56 | 'layer_type': self.layer_type 57 | } 58 | 59 | def fromJSON(self, json): 60 | self.out_depth = json['out_depth'] 61 | self.out_sx = json['out_sx'] 62 | self.out_sy = json['out_sy'] 63 | self.layer_type = json['layer_type'] 64 | 65 | class SigmoidLayer(object): 66 | 67 | """ 68 | Implements Sigmoid nnonlinearity elementwise 69 | x -> 1/(1+e^(-x)) 70 | so the output is between 0 and 1 71 | """ 72 | 73 | def __init__(self, opt={}): 74 | self.out_sx = opt['in_sx'] 75 | self.out_sy = opt['in_sy'] 76 | self.out_depth = opt['in_depth'] 77 | self.layer_type = 'sigmoid' 78 | 79 | def forward(self, V, is_training): 80 | self.in_act = V 81 | V2 = V.cloneAndZero() 82 | V2.w = map(sigmoid, V.w) 83 | self.out_act = V2 84 | return self.out_act 85 | 86 | def backward(self): 87 | V = self.in_act 88 | V2 = self.out_act 89 | N = len(V.w) 90 | V.dw = zeros(N) # zero out gradient wrt data 91 | for i in xrange(N): 92 | v2wi = V2.w[i] 93 | V.dw[i] = v2wi * (1.0 - v2wi) * V2.dw[i] 94 | 95 | def getParamsAndGrads(self): 96 | return [] 97 | 98 | def toJSON(self): 99 | return { 100 | 'out_depth' : self.out_depth, 101 | 'out_sx' : self.out_sx, 102 | 'out_sy' : self.out_sy, 103 | 'layer_type': self.layer_type 104 | } 105 | 106 | def fromJSON(self, json): 107 | self.out_depth = json['out_depth'] 108 | self.out_sx = json['out_sx'] 109 | self.out_sy = json['out_sy'] 110 | self.layer_type = json['layer_type'] 111 | 112 | class MaxoutLayer(object): 113 | 114 | """ 115 | Implements Maxout nnonlinearity that computes: 116 | x -> max(x) 117 | where x is a vector of size group_size. Ideally of course, 118 | the input size should be exactly divisible by group_size. 119 | """ 120 | 121 | def __init__(self, opt={}): 122 | self.group_size = getopt(opt, 'group_size', 2) 123 | self.out_sx = opt['in_sx'] 124 | self.out_sy = opt['in_sy'] 125 | self.out_depth = opt['in_depth'] / self.group_size 126 | self.layer_type = 'maxout' 127 | self.switches = zeros(self.out_sx * self.out_sy * self.out_depth) 128 | 129 | def forward(self, V, is_training): 130 | self.in_act = V 131 | N = self.out_depth 132 | V2 = Vol(self.out_sx, self.out_sy, self.out_depth, 0.0) 133 | 134 | if self.out_sx == 1 and self.out_sy == 1: 135 | for i in xrange(N): 136 | offset = i * self.group_size 137 | m = max(V.w[offset:]) 138 | index = V.w[offset:].index(m) 139 | V2.w[i] = m 140 | self.switches[i] = offset + index 141 | else: 142 | switch_counter = 0 143 | for x in xrange(V.sx): 144 | for y in xrange(V.sy): 145 | for i in xrange(N): 146 | ix = i * self.group_size 147 | elem = V.get(x, y, ix) 148 | elem_i = 0 149 | for j in range(1, self.group_size): 150 | elem2 = V.get(x, y, ix + j) 151 | if elem2 > elem: 152 | elem = elem2 153 | elem_i = j 154 | V2.set(x, y, i, elem) 155 | self.switches[i] = ix + elem_i 156 | switch_counter += 1 157 | 158 | self.out_act = V2 159 | return self.out_act 160 | 161 | def backward(self): 162 | V = self.in_act 163 | V2 = self.out_act 164 | N = self.out_depth 165 | V.dw = zeros(len(V.w)) # zero out gradient wrt data 166 | 167 | # pass the gradient through the appropriate switch 168 | if self.sx == 1 and self.sy == 1: 169 | for i in range(N): 170 | chain_grad = V2.dw[i] 171 | V.dw[self.switches[i]] = chain_grad 172 | else: 173 | switch_counter = 0 174 | for x in xrange(V2.sx): 175 | for y in xrange(V2.sy): 176 | for i in xrange(N): 177 | chain_grad = V2.get_grad(x,y,i) 178 | V.set_grad(x, y, self.switches[n], chain_grad) 179 | switch_counter += 1 180 | 181 | def getParamsAndGrads(self): 182 | return [] 183 | 184 | def toJSON(self): 185 | return { 186 | 'out_depth' : self.out_depth, 187 | 'out_sx' : self.out_sx, 188 | 'out_sy' : self.out_sy, 189 | 'layer_type': self.layer_type, 190 | 'group_size': self.group_size 191 | } 192 | 193 | def fromJSON(self, json): 194 | self.out_depth = json['out_depth'] 195 | self.out_sx = json['out_sx'] 196 | self.out_sy = json['out_sy'] 197 | self.layer_type = json['layer_type'] 198 | self.group_size = json['group_size'] 199 | self.switches = zeros(self.group_size) 200 | 201 | class TanhLayer(object): 202 | 203 | """ 204 | Implements Tanh nnonlinearity elementwise 205 | x -> tanh(x) 206 | so the output is between -1 and 1. 207 | """ 208 | 209 | def __init__(self, opt={}): 210 | self.out_sx = opt['in_sx'] 211 | self.out_sy = opt['in_sy'] 212 | self.out_depth = opt['in_depth'] 213 | self.layer_type = 'tanh' 214 | 215 | def forward(self, V, is_training): 216 | self.in_act = V 217 | V2 = V.cloneAndZero() 218 | V2.w = map(tanh, V.w) 219 | self.out_act = V2 220 | return self.out_act 221 | 222 | def backward(self): 223 | V = self.in_act 224 | V2 = self.out_act 225 | N = len(V.w) 226 | V.dw = zeros(N) # zero out gradient wrt data 227 | for i in xrange(N): 228 | v2wi = V2.w[i] 229 | V.dw[i] = (1.0 - v2wi * v2wi) * V2.dw[i] 230 | 231 | def getParamsAndGrads(self): 232 | return [] 233 | 234 | def toJSON(self): 235 | return { 236 | 'out_depth' : self.out_depth, 237 | 'out_sx' : self.out_sx, 238 | 'out_sy' : self.out_sy, 239 | 'layer_type': self.layer_type 240 | } 241 | 242 | def fromJSON(self, json): 243 | self.out_depth = json['out_depth'] 244 | self.out_sx = json['out_sx'] 245 | self.out_sy = json['out_sy'] 246 | self.layer_type = json['layer_type'] -------------------------------------------------------------------------------- /layers/normalization.py: -------------------------------------------------------------------------------- 1 | from util import zeros 2 | 3 | class LocalResponseNormalizationLayer(object): 4 | 5 | """ 6 | Local Response Normalization in window, along depths of volumes 7 | https://code.google.com/p/cuda-convnet/wiki/LayerParams#Local_response_normalization_layer_(same_map) 8 | but 1 is replaced with k. This creates local competition among neurons 9 | along depth, independently at every particular location in the input volume. 10 | """ 11 | 12 | def __init__(self, opt={}): 13 | self.k = opt['k'] 14 | self.n = opt['n'] 15 | self.alpha = opt['alpha'] 16 | self.beta = opt['beta'] 17 | 18 | self.out_sx = opt['in_sx'] 19 | self.out_sy = opt['in_sy'] 20 | self.out_depth = opt['in_depth'] 21 | self.layer_type = 'lrn' 22 | 23 | if self.n % 2 == 0: 24 | print 'Warning: n should be odd for LRN layer.' 25 | 26 | def forward(self, V, in_training): 27 | self.in_act = V 28 | A = V.cloneAndZero() 29 | self.S_cache = V.cloneAndZero() 30 | n2 = self.n / 2 31 | 32 | for x in xrange(V.sx): 33 | for y in xrange(V.sy): 34 | for i in xrange(V.depth): 35 | a_i = V.get(x, y, i) 36 | 37 | # Normalize in a window of size n 38 | den = 0.0 39 | for j in xrange(max(0, i - n2), min(i + n2, V.depth - 1) + 1): 40 | u_f = V.get(x, y, j) 41 | den += u_f * u_f 42 | den *= self.alpha / (float(self.n) ** 2) 43 | den += self.k 44 | self.S_cache.set(x, y, i, den) # will be useful for backprop 45 | den = den ** self.beta 46 | 47 | A.set(x, y, i, a_i / den) 48 | 49 | self.out_act = A 50 | return self.out_act 51 | 52 | def backward(self): 53 | # evaluate gradient wrt data 54 | V = self.in_act 55 | V.dw = zeros(len(V.w)) # zero out gradient wrt data 56 | A = self.out_act 57 | n2 = self.n / 2 58 | 59 | for x in xrange(V.sx): 60 | for y in xrange(V.sy): 61 | for i in xrange(V.depth): 62 | chain_grad = self.out_act.get_grad(x, y, i) 63 | S = self.S_cache.get(x, y, i) 64 | S_b = S ** self.beta 65 | S_b2 = S_b * S_b 66 | 67 | # Normalize in a window of size n 68 | for j in xrange(max(0, i - n2), min(i + n2, V.depth - 1) + 1): 69 | a_j = V.get(x, y, j) 70 | grad = -(a_j ** 2) * self.beta * (S ** (self.beta - 1)) * self.alpha / self.n * 2.0 71 | if j == i: 72 | grad += S_b 73 | grad /= S_b2 74 | grad *= chain_grad 75 | V.add_grad(x, y, j, grad) 76 | 77 | def getParamsAndGrads(self): 78 | return [] 79 | 80 | def toJSON(self): 81 | return { 82 | 'k' : self.k, 83 | 'n' : self.n, 84 | 'alpha' : self.alpha, 85 | 'beta' : self.beta, 86 | 'out_sx' : self.out_sx, 87 | 'out_sy' : self.out_sy, 88 | 'out_depth' : self.out_depth, 89 | 'layer_type': self.layer_type 90 | } 91 | 92 | def fromJSON(self, json): 93 | self.k = json['k'] 94 | self.n = json['n'] 95 | self.alpha = json['alpha'] 96 | self.beta = json['beta'] 97 | self.out_sx = json['out_sx'] 98 | self.out_sy = json['out_sy'] 99 | self.out_depth = json['out_depth'] 100 | self.layer_type = json['layer_type'] -------------------------------------------------------------------------------- /layers/operations.py: -------------------------------------------------------------------------------- 1 | from util import getopt 2 | from vol import Vol 3 | 4 | """ 5 | Layers that perform an operation of inputs. 6 | """ 7 | 8 | class AddLayer(object): 9 | 10 | def __init__(self, opt={}): 11 | self.out_depth = opt['in_depth'] 12 | self.out_sx = opt['in_sx'] 13 | self.out_sy = opt['in_sy'] 14 | self.num_inputs = opt['in_sx'] * opt['in_sy'] * opt['in_depth'] 15 | self.layer_type = 'add' 16 | 17 | self.skip = getopt(opt, 'skip', 0) # skip n activations in input 18 | self.delta = getopt(opt, 'delta', [0] * self.num_inputs) 19 | self.num_neurons = getopt(opt, 'num_neurons', self.num_inputs) 20 | 21 | def forward(self, V, is_training): 22 | self.in_act = V 23 | 24 | A = Vol(1, 1, self.num_inputs, 0.0) 25 | applied = 0 26 | for n in xrange(self.num_inputs): 27 | if n < self.skip: 28 | A.w[n] = V.w[n] 29 | else: 30 | A.w[n] = V.w[n] + self.delta[n - self.skip] 31 | applied += 1 32 | if applied == self.num_neurons: 33 | break 34 | 35 | self.out_act = A 36 | return self.out_act 37 | 38 | def backward(self): 39 | pass 40 | 41 | def getParamsAndGrads(self): 42 | return [] 43 | 44 | def toJSON(self): 45 | return { 46 | 'out_depth' : self.out_depth, 47 | 'out_sx' : self.out_sx, 48 | 'out_sy' : self.out_sy, 49 | 'layer_type': self.layer_type 50 | } 51 | 52 | def fromJSON(self, json): 53 | self.out_depth = json['out_depth'] 54 | self.out_sx = json['out_sx'] 55 | self.out_sy = json['out_sy'] 56 | self.layer_type = json['layer_type'] -------------------------------------------------------------------------------- /layers/pooling.py: -------------------------------------------------------------------------------- 1 | from util import getopt, zeros 2 | from vol import Vol 3 | from math import floor 4 | 5 | class PoolLayer(object): 6 | 7 | """ 8 | Max pooling layer: finds areas of max activation 9 | http://deeplearning.net/tutorial/lenet.html#maxpooling 10 | """ 11 | 12 | def __init__(self, opt={}): 13 | self.sx = opt['sx'] # filter size 14 | self.in_depth = opt['in_depth'] 15 | self.in_sx = opt['in_sx'] 16 | self.in_sy = opt['in_sy'] 17 | 18 | # optional 19 | self.sy = getopt(opt, 'sy', self.sx) 20 | self.stride = getopt(opt, 'stride', 2) 21 | self.pad = getopt(opt, 'pad', 0) # padding to borders of input volume 22 | 23 | self.out_depth = self.in_depth 24 | self.out_sx = int(floor((self.in_sx - self.sx + 2 * self.pad) / self.stride + 1)) 25 | self.out_sy = int(floor((self.in_sy - self.sy + 2 * self.pad) / self.stride + 1)) 26 | self.layer_type = 'pool' 27 | 28 | # Store switches for x,y coordinates for where the max comes from, for each output neuron 29 | switch_size = self.out_sx * self.out_sy * self.out_depth 30 | self.switch_x = zeros(switch_size) 31 | self.switch_y = zeros(switch_size) 32 | 33 | def forward(self, V, is_training): 34 | self.in_act = V 35 | A = Vol(self.out_sx, self.out_sy, self.out_depth, 0.0) 36 | switch_counter = 0 37 | 38 | for d in xrange(self.out_depth): 39 | x = -self.pad 40 | y = -self.pad 41 | for ax in xrange(self.out_sx): 42 | y = -self.pad 43 | for ay in xrange(self.out_sy): 44 | # convolve centered at this particular location 45 | max_a = -99999 46 | win_x, win_y = -1, -1 47 | for fx in xrange(self.sx): 48 | for fy in xrange(self.sy): 49 | off_x = x + fx 50 | off_y = y + fy 51 | if off_y >= 0 and off_y < V.sy \ 52 | and off_x >= 0 and off_x < V.sx: 53 | v = V.get(off_x, off_y, d) 54 | # max pool 55 | if v > max_a: 56 | max_a = v 57 | win_x = off_x 58 | win_y = off_y 59 | 60 | self.switch_x[switch_counter] = win_x 61 | self.switch_y[switch_counter] = win_y 62 | switch_counter += 1 63 | A.set(ax, ay, d, max_a) 64 | 65 | y += self.stride 66 | x += self.stride 67 | 68 | self.out_act = A 69 | return self.out_act 70 | 71 | def backward(self): 72 | # pooling layers have no parameters, so simply compute 73 | # gradient wrt data here 74 | V = self.in_act 75 | V.dw = zeros(len(V.w)) # zero out gradient wrt data 76 | A = self.out_act # computed in forward pass 77 | 78 | n = 0 79 | for d in xrange(self.out_depth): 80 | x = -self.pad 81 | y = -self.pad 82 | for ax in xrange(self.out_sx): 83 | y = -self.pad 84 | for ay in xrange(self.out_sy): 85 | chain_grad = self.out_act.get_grad(ax, ay, d) 86 | V.add_grad(self.switch_x[n], self.switch_y[n], d, chain_grad) 87 | n += 1 88 | y += self.stride 89 | x += self.stride 90 | 91 | def getParamsAndGrads(self): 92 | return [] 93 | 94 | def toJSON(self): 95 | return { 96 | 'sx' : self.sx, 97 | 'sy' : self.sy, 98 | 'stride' : self.stride, 99 | 'in_depth' : self.in_depth, 100 | 'out_depth' : self.out_depth, 101 | 'out_sx' : self.out_sx, 102 | 'out_sy' : self.out_sy, 103 | 'pad' : self.pad, 104 | 'layer_type': self.layer_type 105 | } 106 | 107 | def fromJSON(self, json): 108 | self.out_depth = json['out_depth'] 109 | self.out_sx = json['out_sx'] 110 | self.out_sy = json['out_sy'] 111 | self.layer_type = json['layer_type'] 112 | self.sx = json['sx'] 113 | self.sy = json['sy'] 114 | self.stride = json['stride'] 115 | self.in_depth = json['depth'] 116 | self.pad = json['pad'] 117 | 118 | switch_size = self.out_sx * self.out_sy * self.out_depth 119 | self.switch_x = zeros(switch_size) 120 | self.switch_y = zeros(switch_size) -------------------------------------------------------------------------------- /layers/similarity.py: -------------------------------------------------------------------------------- 1 | from util import getopt, zeros 2 | from vol import Vol 3 | from math import sqrt, log, exp 4 | 5 | class SimilarityLayer(object): 6 | 7 | """ 8 | Computes similarity measures, generalization of dot products 9 | http://arxiv.org/pdf/1410.0781.pdf 10 | """ 11 | 12 | def __init__(self, opt={}): 13 | self.out_depth = opt['num_neurons'] 14 | self.l1_decay_mul = getopt(opt, 'l1_decay_mul', 0.0) 15 | self.l2_decay_mul = getopt(opt, 'l2_decay_mul', 1.0) 16 | 17 | self.num_inputs = opt['in_sx'] * opt['in_sy'] * opt['in_depth'] 18 | self.out_sx = 1 19 | self.out_sy = 1 20 | self.layer_type = 'sim' 21 | 22 | bias = getopt(opt, 'bias_pref', 0.0) 23 | self.filters = [ Vol(1, 1, self.num_inputs) for i in xrange(self.out_depth) ] 24 | self.biases = Vol(1, 1, self.out_depth, bias) 25 | 26 | def forward(self, V, in_training): 27 | self.in_act = V 28 | A = Vol(1, 1, self.out_depth, 0.0) 29 | Vw = V.w 30 | 31 | def norm(vec): 32 | return sqrt(sum(c * c for c in vec)) 33 | 34 | normv = norm(Vw) 35 | 36 | # compute cos sim between V and filters 37 | for i in xrange(self.out_depth): 38 | sum_a = 0.0 39 | fiw = self.filters[i].w 40 | for d in xrange(self.num_inputs): 41 | sum_a += Vw[d] * fiw[d] 42 | sum_a += self.biases.w[i] # dot(W, v) + b 43 | 44 | normf = norm(fiw) 45 | try: 46 | A.w[i] = sum_a / (normv * normf) 47 | except: 48 | A.w[i] = 0 49 | 50 | self.out_act = A 51 | return self.out_act 52 | 53 | def backward(self): 54 | V = self.in_act 55 | V.dw = zeros(len(V.w)) # zero out gradient 56 | 57 | # compute gradient wrt weights and data 58 | for i in xrange(self.out_depth): 59 | fi = self.filters[i] 60 | chain_grad = self.out_act.dw[i] 61 | 62 | for d in xrange(self.num_inputs): 63 | V.dw[d] += fi.w[d] * chain_grad #grad wrt input data 64 | fi.dw[d] += V.w[d] * chain_grad #grad wrt params 65 | 66 | self.biases.dw[i] += chain_grad 67 | 68 | def getParamsAndGrads(self): 69 | response = [] 70 | for d in xrange(self.out_depth): 71 | response.append({ 72 | 'params': self.filters[d].w, 73 | 'grads': self.filters[d].dw, 74 | 'l2_decay_mul': self.l2_decay_mul, 75 | 'l1_decay_mul': self.l1_decay_mul 76 | }) 77 | response.append({ 78 | 'params': self.biases.w, 79 | 'grads': self.biases.dw, 80 | 'l2_decay_mul': 0.0, 81 | 'l1_decay_mul': 0.0 82 | }) 83 | return response 84 | 85 | def toJSON(self): 86 | return { 87 | 'out_depth' : self.out_depth, 88 | 'out_sx' : self.out_sx, 89 | 'out_sy' : self.out_sy, 90 | 'layer_type' : self.layer_type, 91 | 'num_inputs' : self.num_inputs, 92 | 'l1_decay_mul': self.l1_decay_mul, 93 | 'l2_decay_mul': self.l2_decay_mul, 94 | 'filters' : [ f.toJSON() for f in self.filters ], 95 | 'biases' : self.biases.toJSON() 96 | } 97 | 98 | def fromJSON(self, json): 99 | self.out_depth = json['out_depth'] 100 | self.out_sx = json['out_sx'] 101 | self.out_sy = json['out_sy'] 102 | self.layer_type = json['layer_type'] 103 | self.num_inputs = json['num_inputs'] 104 | self.l1_decay_mul = json['l1_decay_mul'] 105 | self.l2_decay_mul = json['l2_decay_mul'] 106 | self.filters = [ Vol(0, 0, 0, 0).fromJSON(f) for f in json['filters'] ] 107 | self.biases = Vol(0, 0, 0, 0).fromJSON(json['biases']) 108 | 109 | class MexLayer(object): 110 | 111 | """ 112 | Computes log(sum(exp())) to fill "the role of activation 113 | functions, max or average pooling, and weights necessary 114 | for classification." 115 | http://arxiv.org/pdf/1410.0781.pdf 116 | """ 117 | 118 | def __init__(self, opt={}): 119 | self.out_sx = opt['in_sx'] 120 | self.out_sy = opt['in_sy'] 121 | self.out_depth = opt['in_depth'] 122 | 123 | self.zeta = getopt(opt, 'zeta', 1.0) 124 | if self.zeta == 0.0: 125 | print 'WARNING: zeta cannot equal 0' 126 | 127 | self.layer_type = 'mex' 128 | 129 | def forward(self, V, is_training): 130 | self.in_act = V 131 | V2 = Vol(self.out_sx, self.out_sy, self.out_depth, 0.0) 132 | 133 | sexp = 0.0 134 | for i in xrange(len(V2.w)): 135 | sexp += exp(V.w[i]) 136 | V2.w[i] = log((sexp / (i + 1))) / self.zeta 137 | 138 | self.out_act = V2 139 | return self.out_act 140 | 141 | def backward(self): 142 | # same as relu 143 | 144 | V = self.in_act 145 | V2 = self.out_act 146 | N = len(V.w) 147 | V.dw = zeros(N) # zero out gradient wrt data 148 | for i in xrange(N): 149 | if V2.w[i] <= 0: # threshold 150 | V.dw[i] = 0 151 | else: 152 | V.dw[i] = V2.dw[i] 153 | 154 | def getParamsAndGrads(self): 155 | return [] 156 | 157 | def toJSON(self): 158 | return { 159 | 'out_depth' : self.out_depth, 160 | 'out_sx' : self.out_sx, 161 | 'out_sy' : self.out_sy, 162 | 'layer_type': self.layer_type 163 | } 164 | 165 | def fromJSON(self, json): 166 | self.out_depth = json['out_depth'] 167 | self.out_sx = json['out_sx'] 168 | self.out_sy = json['out_sy'] 169 | self.layer_type = json['layer_type'] -------------------------------------------------------------------------------- /net.py: -------------------------------------------------------------------------------- 1 | from layers.input import InputLayer 2 | from layers.dropout import DropoutLayer 3 | from layers.nonlinearities import ReluLayer, SigmoidLayer, MaxoutLayer, TanhLayer 4 | from layers.loss import SoftmaxLayer, RegressionLayer, SVMLayer 5 | from layers.normalization import LocalResponseNormalizationLayer 6 | from layers.pooling import PoolLayer 7 | from layers.dotproducts import ConvLayer, FullyConnectedLayer 8 | from layers.similarity import SimilarityLayer, MexLayer 9 | from layers.operations import AddLayer 10 | 11 | class Net(object): 12 | 13 | """ 14 | Net manages a set of layers. 15 | For now: Simple linear order of layers, first layer input and last layer a cost layer 16 | """ 17 | 18 | def __init__(self, layers=None): 19 | self.layers = [] 20 | if layers and type(layers) == list: 21 | self.makeLayers(layers) 22 | 23 | def __str__(self): 24 | return '\n'.join( 25 | '{} {} {} {}'.format( 26 | layer.layer_type, 27 | layer.out_sx, 28 | layer.out_sy, 29 | layer.out_depth 30 | ) for layer in self.layers 31 | ) 32 | 33 | def makeLayers(self, layers): 34 | # Takes a list of layer definitions and creates the network layer objects 35 | 36 | # Checks 37 | if len(layers) < 2: 38 | print 'Error: Net must have at least one input and one softmax layer.' 39 | if layers[0]['type'] != 'input': 40 | print 'Error: First layer should be input.' 41 | 42 | # Add activations and dropouts 43 | def addExtraLayers(): 44 | newLayers = [] 45 | for layer in layers: 46 | layerType = layer['type'] 47 | layerKeys = layer.keys() 48 | 49 | if layerType == 'softmax' or layerType == 'svm': 50 | # add an fc layer 51 | newLayers.append({ 52 | 'type': 'fc', 53 | 'num_neurons': layer['num_classes'] 54 | }) 55 | 56 | if layerType == 'regression': 57 | # add an fc layer 58 | newLayers.append({ 59 | 'type': 'fc', 60 | 'num_neurons': layer['num_neurons'] 61 | }) 62 | 63 | if ((layerType == 'fc' or layerType == 'conv') 64 | and ('bias_pref' not in layerKeys)): 65 | layer['bias_pref'] = 0.0 66 | if 'activation' in layerKeys and layer['activation'] == 'relu': 67 | layer['bias_pref'] = 0.1 # prevent dead relu by chance 68 | 69 | if layerType != 'capsule': 70 | newLayers.append(layer) 71 | 72 | if 'activation' in layerKeys: 73 | layerAct = layer['activation'] 74 | if layerAct in ['relu', 'sigmoid', 'tanh', 'mex']: 75 | newLayers.append({'type': layerAct}) 76 | elif layerAct == 'maxout': 77 | newLayers.append({ 78 | 'type': 'maxout', 79 | 'group_size': layer['group_size'] if group_size in layerKeys else 2 80 | }) 81 | else: 82 | print 'Error: Unsupported activation' 83 | 84 | if 'drop_prob' in layerKeys and layerType != 'dropout': 85 | newLayers.append({ 86 | 'type': 'dropout', 87 | 'drop_prob': layer['drop_prob'] 88 | }) 89 | 90 | if layerType == 'capsule': 91 | fc_recog = {'type': 'fc', 'num_neurons': layer['num_recog']} 92 | pose = {'type': 'add', 'delta': [layer['dx'], layer['dy']], 93 | 'skip': 1, 'num_neurons': layer['num_pose']} 94 | fc_gen = {'type': 'fc', 'num_neurons': layer['num_gen']} 95 | 96 | newLayers.append(fc_recog) 97 | newLayers.append(pose) 98 | newLayers.append(fc_gen) 99 | 100 | return newLayers 101 | 102 | all_layers = addExtraLayers() 103 | 104 | # Create the layers 105 | for i in xrange(len(all_layers)): 106 | layer = all_layers[i] 107 | if i > 0: 108 | prev = self.layers[i - 1] 109 | layer['in_sx'] = prev.out_sx 110 | layer['in_sy'] = prev.out_sy 111 | layer['in_depth'] = prev.out_depth 112 | 113 | layerType = layer['type'] 114 | obj = None 115 | 116 | if layerType == 'fc': obj = FullyConnectedLayer(layer) 117 | elif layerType == 'lrn': obj = LocalResponseNormalizationLayer(layer) 118 | elif layerType == 'dropout': obj = DropoutLayer(layer) 119 | elif layerType == 'input': obj = InputLayer(layer) 120 | elif layerType == 'softmax': obj = SoftmaxLayer(layer) 121 | elif layerType == 'regression': obj = RegressionLayer(layer) 122 | elif layerType == 'conv': obj = ConvLayer(layer) 123 | elif layerType == 'pool': obj = PoolLayer(layer) 124 | elif layerType == 'relu': obj = ReluLayer(layer) 125 | elif layerType == 'sigmoid': obj = SigmoidLayer(layer) 126 | elif layerType == 'tanh': obj = TanhLayer(layer) 127 | elif layerType == 'maxout': obj = MaxoutLayer(layer) 128 | elif layerType == 'svm': obj = SVMLayer(layer) 129 | elif layerType == 'sim': obj = SimilarityLayer(layer) 130 | elif layerType == 'mex': obj = MexLayer(layer) 131 | elif layerType == 'add': obj = AddLayer(layer) 132 | elif layerType == 'capsule': pass 133 | else: print 'Unrecognized layer type' 134 | 135 | if obj: self.layers.append(obj) 136 | 137 | def forward(self, V, is_training=False): 138 | # Forward propogate through the network. 139 | # Trainer will pass is_training=True 140 | activation = self.layers[0].forward(V, is_training) 141 | for i in xrange(1, len(self.layers)): 142 | activation = self.layers[i].forward(activation, is_training) 143 | return activation 144 | 145 | def getCostLoss(self, V, y): 146 | self.forward(V, False) 147 | loss = self.layers[-1].backward(y) 148 | return loss 149 | 150 | def backward(self, y): 151 | # Backprop: compute gradients wrt all parameters 152 | loss = self.layers[-1].backward(y) #last layer assumed loss layer 153 | for i in xrange(len(self.layers) - 2, 0, -1): # first layer assumed input 154 | self.layers[i].backward() 155 | return loss 156 | 157 | def getParamsAndGrads(self): 158 | # Accumulate parameters and gradients for the entire network 159 | return [ pg 160 | for layer in self.layers 161 | for pg in layer.getParamsAndGrads() ] 162 | 163 | def getPrediction(self): 164 | softmax = self.layers[-1] 165 | p = softmax.out_act.w 166 | return p.index(max(p)) 167 | 168 | def toJSON(self): 169 | return { 'layers': [ layer.toJSON() for layer in self.layers ] } 170 | 171 | def fromJSON(self, json): 172 | self.layers = [] 173 | for layer in json['layers']: 174 | layerType = layer['type'] 175 | obj = None 176 | 177 | if layerType == 'fc': obj = FullyConnectedLayer(layer) 178 | elif layerType == 'lrn': obj = LocalResponseNormalizationLayer(layer) 179 | elif layerType == 'dropout': obj = DropoutLayer(layer) 180 | elif layerType == 'input': obj = InputLayer(layer) 181 | elif layerType == 'softmax': obj = SoftmaxLayer(layer) 182 | elif layerType == 'regression': obj = RegressionLayer(layer) 183 | elif layerType == 'conv': obj = ConvLayer(layer) 184 | elif layerType == 'pool': obj = PoolLayer(layer) 185 | elif layerType == 'relu': obj = ReluLayer(layer) 186 | elif layerType == 'sigmoid': obj = SigmoidLayer(layer) 187 | elif layerType == 'tanh': obj = TanhLayer(layer) 188 | elif layerType == 'maxout': obj = MaxoutLayer(layer) 189 | elif layerType == 'svm': obj = SVMLayer(layer) 190 | elif layerType == 'sim': obj = SimilarityLayer(layer) 191 | elif layerType == 'mex': obj = MexLayer(layer) 192 | 193 | obj.fromJSON(layer) 194 | self.layers.append(obj) -------------------------------------------------------------------------------- /run.py: -------------------------------------------------------------------------------- 1 | from sys import argv 2 | 3 | # MNIST test #classic 4 | if '--mnist' in argv: 5 | from examples.mnist import * 6 | start('--conv' in argv) 7 | train() 8 | try: 9 | index = argv.index('-testn') 10 | n = int(argv[index + 1]) 11 | test(n) 12 | except: 13 | test(5000) 14 | 15 | # random 2D data classification test 16 | if '--toy2D' in argv: 17 | from examples.toy2D import * 18 | train() 19 | test() 20 | 21 | # Cifar10 test 22 | if '--cifar10' in argv: 23 | from examples.cifar.cifar10 import * 24 | start('-conv' in argv, '-crop' in argv, '-gray' in argv) 25 | train() 26 | test() 27 | 28 | # Cifar10 autoencoder 29 | if '--cifar10-ae' in argv: 30 | from examples.cifar.autoencoder import * 31 | start('-conv' in argv, '-crop' in argv, '-gray' in argv) 32 | train() 33 | test() 34 | 35 | # Learns to predict the next letter in a sequence (trained on trigrams) 36 | if '--nextletter' in argv: 37 | from examples.next_letter import * 38 | start() 39 | train() 40 | test() 41 | 42 | # Autoencode mnist digits, display them with opencv 43 | if '--autoencoder' in argv: 44 | from examples.autoencoder import * 45 | start() 46 | train() 47 | 48 | path, test_n = None, None 49 | if '-path' in argv: 50 | path = argv[argv.index('-path') + 1] 51 | 52 | if '-testn' in argv: 53 | test_n = int(argv[argv.index('-testn') + 1]) 54 | 55 | test(path=path, test_n=test_n) 56 | 57 | # Uses an autoencoder trained on frequency distributions 58 | # from project gutenberg to do topic modeling 59 | # (using the assumption that words with highest activation == topics) 60 | if '--topics' in argv: 61 | from examples.topics import * 62 | start() 63 | train() 64 | test() 65 | 66 | # Uses an autoencoder trained on frequency distributions 67 | # from project gutenberg to do semantic similarity search 68 | # cos(v, qv) for v in doc, v = wieghts of 10 neuron sigmoid ("compressed code of text") 69 | if '--sim' in argv: 70 | from examples.similarity import * 71 | start() 72 | train() 73 | test() 74 | 75 | # Classify iris dataset, requires scikit-learn 76 | if '--iris' in argv: 77 | from examples.iris import * 78 | load_data() 79 | start() 80 | train() 81 | test() 82 | 83 | # Classify labeled faces in the wild, requires scikit-learn 84 | if '--faces' in argv: 85 | from examples.faces import * 86 | load_data() 87 | start() 88 | train() 89 | test() 90 | 91 | # Predict next word from bigram model 92 | if '--nextword' in argv: 93 | from examples.next_word import * 94 | start() 95 | train() 96 | test() 97 | 98 | # Predict dialogue class from frequency distribution of text 99 | if '--dialogue' in argv: 100 | from examples.dialogue import * 101 | start() 102 | train() 103 | test() 104 | 105 | # Predict next word based on word embeddings 106 | if '--nextworde' in argv: 107 | from examples.next_word_embeddings import * 108 | start() 109 | train() 110 | test() 111 | 112 | # Predict 1 of 4 sentiment tags, kaggle challenge 113 | # https://www.kaggle.com/c/sentiment-analysis-on-movie-reviews 114 | if '--sentiment' in argv: 115 | from examples.sentiment import * 116 | start() 117 | train() 118 | test() 119 | fill() 120 | 121 | # Based on Geoff Hinton's Dark Knowledge talk 122 | # https://www.youtube.com/watch?v=EK61htlw8hY 123 | # Train large net with dropout and jitter, use outputs (soft targets) 124 | # as inputs to smaller neural net -- pretty amazing results 125 | # Larger net has already learned a sort of similarity function btw. inputs 126 | if '--dark-knowledge' in argv: 127 | from examples.dark_knowledge import * 128 | run_big_net() 129 | run_small_net() 130 | 131 | if '--udacity-terrain' in argv: 132 | from examples.udacity_terrain import * 133 | train() 134 | test() 135 | 136 | if '--darkencoder' in argv: 137 | from examples.darkencoder import * 138 | start() 139 | train() 140 | train2() 141 | test() 142 | 143 | # Autoencode mnist digits, display them in 2D with opencv 144 | if '--autoencoder-vis' in argv: 145 | from examples.autoencoder_vis import * 146 | start() 147 | test() 148 | train() 149 | test() 150 | 151 | if '--titanic' in argv: 152 | from examples.titanic import * 153 | load_data() 154 | start() 155 | train() 156 | test() 157 | 158 | if '--mnist-n2i' in argv: 159 | from examples.num2img import * 160 | start() 161 | train() 162 | test() 163 | 164 | if '--tae' in argv: 165 | from examples.transforming_autoencoder import start, train, test 166 | start() 167 | train() 168 | test() -------------------------------------------------------------------------------- /trainers.py: -------------------------------------------------------------------------------- 1 | from util import * 2 | from time import time 3 | from math import sqrt 4 | 5 | class Trainer(object): 6 | 7 | def __init__(self, net, options={}): 8 | 9 | """ 10 | Manages Trainers: 11 | 1. Vanilla SGD 12 | 2. Momentum 13 | 3. Adagrad 14 | 4. Adadelta 15 | 5. Windowgrad 16 | """ 17 | 18 | self.opt = options 19 | self.net = net 20 | self.learning_rate = getopt(options, 'learning_rate', 0.01) 21 | self.l1_decay = getopt(options, 'l1_decay', 0.0) 22 | self.l2_decay = getopt(options, 'l2_decay', 0.0) 23 | self.batch_size = getopt(options, 'batch_size', 1) 24 | self.method = getopt(options, 'method', 'sgd') # sgd/adagrad/adadelta/windowgrad 25 | 26 | self.momentum = getopt(options, 'momentum', 0.9) 27 | self.ro = getopt(options, 'ro', 0.95) # used in adadelta 28 | self.eps = getopt(options, 'eps', (10) ** (-6)) # used in adadelta 29 | 30 | self.k = 0 # iteration counter 31 | self.gsum = [] # last iteration gradients (used for momentum calculations) 32 | self.xsum = [] # used in adadelta 33 | 34 | self.win = Window() 35 | 36 | def __str__(self): return str(self.opt) 37 | __repr__ = __str__ 38 | 39 | def train(self, x, y): 40 | self.k += 1 41 | 42 | start = time() 43 | self.net.forward(x, True) # we are training 44 | end = time() 45 | fwd_time = end - start 46 | 47 | if type(y) is not list: 48 | self.win.add(self.net.getPrediction() == y) 49 | 50 | start = time() 51 | cost_loss = self.net.backward(y) 52 | l2_decay_loss = 0.0 53 | l1_decay_loss = 0.0 54 | end = time() 55 | bwd_time = end - start 56 | 57 | if self.k % self.batch_size == 0: 58 | pglist = self.net.getParamsAndGrads() 59 | 60 | # Initialize lists for accumulators. Will only be done once on first iteration 61 | if (len(self.gsum) == 0 and (self.method != 'sgd' or self.momentum > 0.0)): 62 | """ 63 | Only vanilla sgd doesnt need either lists 64 | momentum needs gsum 65 | adagrad needs gsum 66 | adadelta needs gsum and xsum 67 | """ 68 | for elem in pglist: 69 | self.gsum.append(zeros(len(elem['params']))) 70 | if self.method == 'adadelta': 71 | self.xsum.append(zeros(len(elem['params']))) 72 | else: 73 | self.xsum.append([]) 74 | 75 | # Perform an update for all sets of weights 76 | for i in xrange(len(pglist)): 77 | pg = pglist[i] # param, gradient, other options in future (custom learning rate etc.) 78 | p = pg['params'] 79 | g = pg['grads'] 80 | 81 | # Learning rate for some parameters 82 | l2_decay_mul = getopt(pg, 'l2_decay_mul', 1.0) 83 | l1_decay_mul = getopt(pg, 'l1_decay_mul', 1.0) 84 | l2_decay = self.l2_decay * l2_decay_mul 85 | l1_decay = self.l1_decay * l1_decay_mul 86 | for j in xrange(len(p)): 87 | l2_decay_loss += l2_decay * p[j] * p[j] / 2.0 # accumulate weight decay loss 88 | l1_decay_loss += l1_decay * abs(p[j]) 89 | l1grad = l1_decay * (1 if p[j] > 0 else -1) 90 | l2grad = l2_decay * p[j] 91 | gij = (l2grad + l1grad + g[j]) / float(self.batch_size) # raw batch gradient 92 | 93 | try: 94 | gsumi = self.gsum[i] 95 | xsumi = self.xsum[i] 96 | except: 97 | pass 98 | if self.method == 'adagrad': # adagrad update 99 | gsumi[j] += gij * gij 100 | dx = - self.learning_rate / sqrt(gsumi[j] + self.eps) * gij 101 | p[j] += dx 102 | elif self.method == 'windowgrad': 103 | """ 104 | This is adagrad but with a moving window weighted average 105 | so the gradient is not accumulated over the entire history of the run. 106 | It's also referred to as Idea #1 in Zeiler paper on Adadelta. 107 | """ 108 | gsumi[j] = self.ro * gsumi[j] + (1 - self.ro) * gij * gij 109 | dx = - self.learning_rate / sqrt(gsumi[j] + self.eps) * gij 110 | p[j] += dx 111 | elif self.method == 'adadelta': 112 | gsumi[j] = self.ro * gsumi[j] + (1 - self.ro) * gij * gij 113 | dx = - sqrt((xsumi[j] + self.eps) / (gsumi[j] + self.eps)) * gij 114 | xsumi[j] = self.ro * gsumi[j] + (1 - self.ro) * dx * dx 115 | p[j] += dx 116 | else: # SGD 117 | if self.momentum > 0.0: # Momentum update 118 | dx = self.momentum * gsumi[j] - self.learning_rate * gij # step 119 | gsumi[j] = dx # backup for next iteration 120 | p[j] += dx # apply gradient 121 | else: # Vanilla SGD 122 | p[j] += - self.learning_rate * gij 123 | g[j] = 0.0 # zero out gradient so that we can begin accumulating anew 124 | 125 | return { 126 | 'k': self.k, 127 | 'fwd_time': fwd_time, 128 | 'bwd_time': bwd_time, 129 | 'time': fwd_time + bwd_time, 130 | 'l2_decay_loss': l2_decay_loss, 131 | 'l1_decay_loss': l1_decay_loss, 132 | 'cost_loss': cost_loss, 133 | 'loss': cost_loss + l1_decay_loss + l2_decay_loss, 134 | 'accuracy': self.win.get_average() 135 | } -------------------------------------------------------------------------------- /util.py: -------------------------------------------------------------------------------- 1 | from random import random 2 | from math import sqrt, log, floor 3 | import json 4 | 5 | # Random Number Utilities 6 | 7 | return_v = False 8 | v_val = 0.0 9 | def gaussRandom(): 10 | global return_v, v_val 11 | 12 | if return_v: 13 | return_v = False 14 | return v_val 15 | u = 2 * random() - 1 16 | v = 2 * random() - 1 17 | r = u * u + v * v 18 | if r == 0 or r > 1: 19 | return gaussRandom() 20 | c = sqrt(-2.0 * log(r) / r) 21 | v_val = v * c #cache this 22 | return_v = True 23 | return u * c 24 | 25 | def randf(a, b): 26 | return random() * (b - a) + a 27 | 28 | def randi(a, b): 29 | return int(floor(random() * (b - a) + a)) 30 | 31 | def randn(mu, std): 32 | return mu + gaussRandom() * std 33 | 34 | # Array Utilities 35 | 36 | import numpy 37 | def zeros(n=None): 38 | if not n: 39 | return [] 40 | else: 41 | return [0.0] * int(n) 42 | 43 | def arrContains(arr, elt): 44 | for elem in arr: 45 | if elem == elt: 46 | return True 47 | return False 48 | 49 | def arrUnique(arr): 50 | b = set(arr) 51 | return list(b) 52 | 53 | # Return max and min of a given non-empty list 54 | def maxmin(w): 55 | if len(w) == 0: 56 | return {} 57 | maxv = max(w) 58 | maxi = w.index(maxv) 59 | minv = min(w) 60 | mini = w.index(minv) 61 | return { 62 | 'maxi': maxi, 63 | 'maxv': maxv, 64 | 'mini': mini, 65 | 'minv': minv, 66 | 'dv' : maxv - minv 67 | } 68 | 69 | # Create random permutations of numbers, in range [0 ... n-1] 70 | def randperm(n): 71 | i, j, temp = n - 1, 0, None 72 | array = range(n) 73 | while i: 74 | j = int(floor(random() * (i + 1))) 75 | temp = array[i] 76 | array[i] = array[j] 77 | array[j] = temp 78 | i -= 1 79 | return array 80 | 81 | # Sample for list 'lst' according to probabilities in list probs 82 | # the two lists are of same size, and probs adds up to 1 83 | # or lst is a dictionary with keys and probabilities for values 84 | def weightedSample(lst=None, prob=None): 85 | if not (prob or lst): 86 | return 87 | 88 | if type(lst) != list: 89 | try: 90 | prob = lst.values() 91 | lst = lst.keys() 92 | except: 93 | return 94 | 95 | p = randf(0, 1.0) 96 | cumprob = 0.0 97 | for k in xrange(len(lst)): 98 | cumprob += prob[k] 99 | if p < cumprob: 100 | return lst[k] 101 | 102 | # Syntactic sugar function for getting default parameter values 103 | def getopt(opt, field_name, default_value): 104 | return opt.get(field_name, default_value) 105 | 106 | # Utilities for saving/loading json to/from a file 107 | 108 | def saveJSON(filename, data): 109 | print 'Saving to: {}'.format(filename) 110 | with open(filename, 'w') as outfile: 111 | json.dump(data, outfile) 112 | 113 | def loadJSON(path): 114 | with open(path, 'r') as infile: 115 | return json.load(path) 116 | 117 | class Window(object): 118 | 119 | """ 120 | a window stores _size_ number of values 121 | and returns averages. Useful for keeping running 122 | track of validation or training accuracy during SGD 123 | """ 124 | 125 | def __init__(self, size=100, minsize=20): 126 | self.v = [] 127 | self.size = size 128 | self.minsize = minsize 129 | self.sum = 0 130 | 131 | def add(self, x): 132 | self.v.append(x) 133 | self.sum += x 134 | if len(self.v) > self.size: 135 | xold = self.v.pop(0) 136 | self.sum -= xold 137 | 138 | def get_average(self): 139 | if len(self.v) < self.minsize: 140 | return -1 141 | else: 142 | return 1.0 * self.sum / len(self.v) 143 | 144 | def reset(self): 145 | self.v = [] 146 | self.sum = 0 -------------------------------------------------------------------------------- /vol.py: -------------------------------------------------------------------------------- 1 | from util import * 2 | from math import sqrt 3 | 4 | class Vol(object): 5 | 6 | """ 7 | Vol is the basic building block of all data in a net. 8 | it is essentially just a 3D volume of numbers, with a 9 | width (sx), height (sy), and depth (depth). 10 | It is used to hold data for all filters, all volumes, 11 | all weights, and also stores all gradients w.r.t. 12 | the data. c is optionally a value to initialize the volume 13 | with. If c is missing, fills the Vol with random numbers. 14 | """ 15 | 16 | def __init__(self, sx, sy=None, depth=None, c=None): 17 | 18 | # if sx is a list 19 | if type(sx) in (list, tuple): 20 | # We were given a list in sx, assume 1D volume and fill it up 21 | self.sx = 1 22 | self.sy = 1 23 | self.depth = len(sx) 24 | 25 | self.w = zeros(self.depth) 26 | self.dw = zeros(self.depth) 27 | self.w = [ sx[i] for i in xrange(self.depth) ] 28 | else: 29 | # We were given dimensions of the vol 30 | self.sx = sx 31 | self.sy = sy 32 | self.depth = depth 33 | n = sx * sy * depth 34 | self.w = zeros(n) 35 | self.dw = zeros(n) 36 | 37 | if c == None: 38 | # Weight normalization is done to equalize the output 39 | # variance of every neuron, otherwise neurons with a lot 40 | # of incoming connections have outputs of larger variance 41 | scale = sqrt(1.0 / (self.sx * self.sy * self.depth)) 42 | self.w = [ randn(0.0, scale) for i in xrange(n) ] 43 | else: 44 | self.w = [c] * n 45 | 46 | def __str__(self): 47 | return '\n{}{}\n{}{}\n'.format( 48 | 'W:', self.w, 49 | 'DW:', self.dw 50 | ) 51 | 52 | __repr__ = __str__ 53 | 54 | @property 55 | def size(self): 56 | return (self.sx, self.sy, self.depth) 57 | 58 | def get(self, x, y, d): 59 | ix = ((self.sx * y) + x) * self.depth + d 60 | return self.w[ix] 61 | 62 | def set(self, x, y, d, v): 63 | ix = ((self.sx * y) + x) * self.depth + d 64 | self.w[ix] = v 65 | 66 | def add(self, x, y, d, v): 67 | ix = ((self.sx * y) + x) * self.depth + d 68 | self.w[ix] += v 69 | 70 | def get_grad(self, x, y, d): 71 | ix = ((self.sx * y) + x) * self.depth + d 72 | return self.dw[ix] 73 | 74 | def set_grad(self, x, y, d, v): 75 | ix = ((self.sx * y) + x) * self.depth + d 76 | self.dw[ix] = v 77 | 78 | def add_grad(self, x, y, d, v): 79 | ix = ((self.sx * y) + x) * self.depth + d 80 | self.dw[ix] += v 81 | 82 | def cloneAndZero(self): 83 | return Vol(self.sx, self.sy, self.depth, 0.0) 84 | 85 | def clone(self): 86 | V = Vol(self.sx, self.sy, self.depth, 0.0) 87 | n = len(self.w) 88 | for i in range(n): 89 | V.w[i] = self.w[i] 90 | return V 91 | 92 | def addFrom(self, V): 93 | for i in xrange(len(self.w)): 94 | self.w[i] += V.w[i] 95 | 96 | def addFromScaled(self, V, a): 97 | for i in xrange(len(self.w)): 98 | self.w[i] += a * V.w[i] 99 | 100 | def setConst(self, a): 101 | self.w = [a] * len(self.w) 102 | 103 | def toJSON(self): 104 | return { 105 | 'sx' : self.sx, 106 | 'sy' : self.sy, 107 | 'depth' : self.depth, 108 | 'w' : self.w 109 | } 110 | 111 | def fromJSON(self, json): 112 | self.sx = json['sx'] 113 | self.sy = json['sy'] 114 | self.depth = json['depth'] 115 | 116 | n = self.sx * self.sy * self.depth 117 | self.w = zeros(n) 118 | self.dw = zeros(n) 119 | self.addFrom(json['w']) 120 | 121 | return self -------------------------------------------------------------------------------- /vol_util.py: -------------------------------------------------------------------------------- 1 | from vol import Vol 2 | from util import randi 3 | 4 | def augment(V, crop, grayscale=False): 5 | # note assumes square outputs of size crop x crop 6 | # randomly sample a crop in the input volume 7 | if crop == V.sx: return V 8 | 9 | dx = randi(0, V.sx - crop) 10 | dy = randi(0, V.sy - crop) 11 | 12 | W = Vol(crop, crop, V.depth) 13 | for x in xrange(crop): 14 | for y in xrange(crop): 15 | if x + dx < 0 or x + dx >= V.sx or \ 16 | y + dy < 0 or y + dy >= V.sy: continue 17 | for d in xrange(V.depth): 18 | W.set(x, y, d, V.get(x + dx, y + dy, d)) 19 | 20 | if grayscale: 21 | #flatten into depth=1 array 22 | G = Vol(crop, crop, 1, 0.0) 23 | for i in xrange(crop): 24 | for j in xrange(crop): 25 | G.set(i, j, 0, W.get(i, j, 0)) 26 | W = G 27 | 28 | return W --------------------------------------------------------------------------------