├── .gitignore ├── README.md ├── __init__.py ├── ae.py ├── ca.py ├── da.py ├── dataset.py ├── layer.py ├── out ├── clean_binary_files.sh ├── plot_ae_feats.py ├── plot_cae_feats.py ├── plot_dae_feats.py ├── plot_sa_feats.py ├── plot_samples_fn.py └── utils.py ├── sa.py └── tests ├── __init__.py ├── cae_stack_sample.py ├── dae_stack_sample.py ├── dae_stack_sample_ctd.py ├── plot_cae_feats.py ├── plot_dae_feats.py ├── plot_mlp_feats.py ├── plot_samples_fn.py ├── plot_smlp_feats.py ├── run_cae_exp.py ├── run_cae_exp_simple_sample.py ├── test_ae.py ├── test_cae.py ├── test_dae.py ├── test_dae_pento.py ├── test_pae.py ├── test_pae_tfd.py ├── test_sa.py └── utils.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | 3 | # C extensions 4 | *.so 5 | 6 | # Packages 7 | *.egg 8 | *.egg-info 9 | dist 10 | build 11 | eggs 12 | parts 13 | bin 14 | var 15 | sdist 16 | develop-eggs 17 | .installed.cfg 18 | lib 19 | lib64 20 | 21 | # Installer logs 22 | pip-log.txt 23 | 24 | # Unit test / coverage reports 25 | .coverage 26 | .tox 27 | nosetests.xml 28 | 29 | #Translations 30 | *.mo 31 | 32 | #Mr Developer 33 | .mr.developer.cfg 34 | 35 | #Vim thrashes 36 | *.swp 37 | 38 | #binary data 39 | *.pkl 40 | *.npy 41 | *.gz 42 | *.bz2 43 | *.pyc 44 | 45 | #images 46 | *.png 47 | *.jpg 48 | 49 | #Jobman logs 50 | *.err 51 | *.out 52 | *.condor 53 | gulcehre_db/ 54 | LOGS/ 55 | *.txt 56 | *.conf 57 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | autoencoders 2 | ============ 3 | 4 | Implementation of several different types of autoencoders in Theano. 5 | 6 | * **da.py**: Denoising Autoencoder implementation. 7 | * **ca.py**: Contractive Autoencoder implementation. 8 | * **sa.py**: Sparse Autoencoder implementation. 9 | * **autoencoder.py**: Base class for other autoencoder types and basic implementation of a linear autoencoder. 10 | -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caglar/autoencoders/a002c85205d98c073f9776e2196f70d726a3da06/__init__.py -------------------------------------------------------------------------------- /ae.py: -------------------------------------------------------------------------------- 1 | import theano 2 | import theano.tensor as T 3 | from theano.tensor.shared_randomstreams import RandomStreams 4 | 5 | from layer import AEHiddenLayer 6 | import numpy 7 | 8 | from collections import OrderedDict 9 | 10 | theano.config.warn.subtensor_merge_bug = False 11 | 12 | class Nonlinearity: 13 | RELU = "rectifier" 14 | TANH = "tanh" 15 | SIGMOID = "sigmoid" 16 | 17 | class CostType: 18 | MeanSquared = "MeanSquaredCost" 19 | CrossEntropy = "CrossEntropy" 20 | 21 | class Autoencoder(object): 22 | 23 | def __init__(self, 24 | input, 25 | nvis, 26 | nhid=None, 27 | nvis_dec=None, 28 | nhid_dec=None, 29 | rnd=None, 30 | bhid=None, 31 | cost_type=CostType.MeanSquared, 32 | momentum=1, 33 | num_pieces=1, 34 | L2_reg=-1, 35 | L1_reg=-1, 36 | sparse_initialize=False, 37 | nonlinearity=Nonlinearity.TANH, 38 | bvis=None, 39 | tied_weights=True): 40 | 41 | self.input = input 42 | self.nvis = nvis 43 | self.nhid = nhid 44 | self.bhid = bhid 45 | self.bvis = bvis 46 | self.momentum = momentum 47 | self.nonlinearity = nonlinearity 48 | self.tied_weights = tied_weights 49 | self.gparams = None 50 | 51 | if cost_type == CostType.MeanSquared: 52 | self.cost_type = CostType.MeanSquared 53 | elif cost_type == CostType.CrossEntropy: 54 | self.cost_type = CostType.CrossEntropy 55 | 56 | if self.input is None: 57 | self.input = T.matrix('x') 58 | 59 | if rnd is None: 60 | self.rnd = numpy.random.RandomState(1231) 61 | else: 62 | self.rnd = rnd 63 | 64 | self.srng = RandomStreams(seed=1231) 65 | 66 | self.hidden = AEHiddenLayer(input, 67 | nvis, 68 | nhid, 69 | num_pieces=num_pieces, 70 | n_in_dec=nvis_dec, 71 | n_out_dec=nhid_dec, 72 | activation=None, 73 | tied_weights=tied_weights, 74 | sparse_initialize=sparse_initialize, 75 | rng=rnd) 76 | 77 | self.params = self.hidden.params 78 | 79 | self.L1_reg = L1_reg 80 | self.L2_reg = L2_reg 81 | 82 | self.sparse_initialize = sparse_initialize 83 | 84 | self.L1 = 0 85 | self.L2 = 0 86 | 87 | if L1_reg != -1: 88 | self.L1 += abs(self.hidden.W).sum() 89 | if not tied_weights and 0: 90 | self.L1 += abs(self.hidden.W_prime).sum() 91 | 92 | if L2_reg != -1: 93 | self.L2 += (self.hidden.W**2).sum() 94 | if not tied_weights and 0: 95 | self.L2 += (self.hidden.W_prime**2).sum() 96 | 97 | if input is not None: 98 | self.x = input 99 | else: 100 | self.x = T.matrix('x_input', dtype=theano.config.floatX) 101 | 102 | def nonlinearity_fn(self, d_in=None, recons=False): 103 | if self.nonlinearity == Nonlinearity.SIGMOID: 104 | return T.nnet.sigmoid(d_in) 105 | elif self.nonlinearity == Nonlinearity.RELU and not recons: 106 | return T.maximum(d_in, 0) 107 | elif self.nonlinearity == Nonlinearity.RELU and recons: 108 | return T.nnet.softplus(d_in) 109 | elif self.nonlinearity == Nonlinearity.TANH: 110 | return T.tanh(d_in) 111 | 112 | def encode(self, x_in=None, center=True): 113 | if x_in is None: 114 | x_in = self.x 115 | act = self.nonlinearity_fn(T.dot(x_in, self.hidden.W) + self.hidden.b) 116 | if center: 117 | act = act - act.mean(0) 118 | return act 119 | 120 | def encode_linear(self, x_in=None): 121 | if x_in is None: 122 | x_in = self.x 123 | lin_output = T.dot(x_in, self.hidden.W) + self.hidden.b 124 | return self.nonlinearity_fn(lin_output), lin_output 125 | 126 | def decode(self, h): 127 | return self.nonlinearity_fn(T.dot(h, self.hidden.W_prime) + self.hidden.b_prime) 128 | 129 | def get_rec_cost(self, x_rec): 130 | """ 131 | Returns the reconstruction cost. 132 | """ 133 | if self.cost_type == CostType.MeanSquared: 134 | return T.mean(((self.x - x_rec)**2).sum(axis=1)) 135 | elif self.cost_type == CostType.CrossEntropy: 136 | return T.mean((T.nnet.binary_crossentropy(x_rec, self.x)).mean(axis=1)) 137 | 138 | def kl_divergence(self, p, p_hat): 139 | return p * T.log(p) - T.log(p_hat) + (1 - p) * T.log(1 - p) - (1 - p) * T.log(1 - p_hat) 140 | 141 | def sparsity_penalty(self, h, sparsity_level=0.05, sparse_reg=1e-3, batch_size=-1): 142 | if batch_size == -1 or batch_size == 0: 143 | raise Exception("Invalid batch_size!") 144 | sparsity_level = T.extra_ops.repeat(sparsity_level, self.nhid) 145 | sparsity_penalty = 0 146 | avg_act = h.mean(axis=0) 147 | kl_div = self.kl_divergence(sparsity_level, avg_act) 148 | sparsity_penalty = sparse_reg * kl_div.sum() 149 | # Implement KL divergence here. 150 | return sparsity_penalty 151 | 152 | def act_grads(self, inputs): 153 | h, acts = self.encode_linear(inputs) 154 | h_grad = T.grad(h.sum(), acts) 155 | return (h, h_grad) 156 | 157 | def jacobian_h_x(self, inputs): 158 | h, act_grad = self.act_grads(inputs) 159 | jacobian = self.hidden.W * act_grad.dimshuffle(0, 'x', 1) 160 | return (h, T.reshape(jacobian, newshape=(self.nhid, self.nvis))) 161 | 162 | def compute_jacobian_h_x(self, inputs): 163 | 164 | inputs = theano.shared(inputs.flatten()) 165 | h = self.encode(inputs) 166 | #h = h.flatten() 167 | #inputs = inputs.flatten() 168 | #inputs = T.reshape(inputs, newshape=(self.nvis)) 169 | J = theano.gradient.jacobian(h, inputs) 170 | return h, J 171 | 172 | def sample_one_step(self, x, sigma): 173 | #h, J_t = self.jacobian_h_x(x) 174 | h, J_t = self.compute_jacobian_h_x(x) 175 | eps = self.srng.normal(avg=0, size=(self.nhid, 1), std=sigma) 176 | jacob_w_eps = T.dot(J_t.T, eps) 177 | delta_h = T.dot(J_t, jacob_w_eps) 178 | perturbed_h = h + delta_h.T 179 | x = self.decode(perturbed_h) 180 | return x 181 | 182 | def sample_scan(self, x, sigma, n_steps, samples): 183 | # enable on-the-fly graph computations 184 | # theano.config.compute_test_value = 'raise' 185 | in_val = T.fmatrix("input_values") 186 | #in_val.tag.test_value = numpy.asarray(numpy.random.rand(1, 784), dtype=theano.config.floatX) 187 | s_sigma = T.fscalar("sigma_values") 188 | #s_sigma = numpy.asarray(numpy.random.rand(1), dtype=theano.config.floatX) 189 | mode = "FAST_RUN" 190 | values, updates = theano.scan(fn=self.sample_one_step, 191 | outputs_info=in_val, 192 | non_sequences=s_sigma, 193 | n_steps=n_steps, 194 | mode=mode) 195 | ae_sampler = theano.function(inputs=[in_val, s_sigma], outputs=values[-1], updates=updates) 196 | samples = ae_sampler(x, sigma) 197 | return samples 198 | 199 | def sample_old(self, x, sigma, n_steps): 200 | # enable on-the-fly graph computations 201 | # theano.config.compute_test_value = 'raise' 202 | #in_val = T.fmatrix("input_values") 203 | #in_val.tag.test_value = numpy.asarray(numpy.random.rand(1, 784), dtype=theano.config.floatX) 204 | #s_sigma = T.fscalar("sigma_values") 205 | #s_sigma = numpy.asarray(numpy.random.rand(1), dtype=theano.config.floatX) 206 | #mode = "FAST_RUN" 207 | samples = [] 208 | sample = x 209 | samples.append(x) 210 | for i in xrange(n_steps): 211 | print "Sample %d..." % i 212 | sampler = self.sample_one_step(sample, sigma) 213 | sample = sampler.eval() 214 | samples.append(sample) 215 | return samples 216 | 217 | def get_sgd_updates(self, learning_rate, lr_scaler=1.0, batch_size=-1, sparsity_level=-1, sparse_reg=-1, x_in=None): 218 | h = self.encode(x_in) 219 | x_rec = self.decode(h) 220 | cost = self.get_rec_cost(x_rec) 221 | 222 | if self.L1_reg != -1 and self.L1_reg is not None: 223 | cost += self.L1_reg * self.L1 224 | 225 | if self.L2_reg != -1 and self.L2_reg is not None: 226 | cost += self.L2_reg * self.L2 227 | 228 | if sparsity_level != -1 and sparse_reg != -1: 229 | sparsity_penal = self.sparsity_penalty(h, sparsity_level, sparse_reg, batch_size) 230 | cost += sparsity_penal 231 | 232 | self.gparams = T.grad(cost, self.params) 233 | updates = OrderedDict({}) 234 | for param, gparam in zip(self.params, self.gparams): 235 | updates[param] = self.momentum * param - lr_scaler * learning_rate * gparam 236 | return (cost, updates) 237 | 238 | def fit(self, 239 | data=None, 240 | learning_rate=0.1, 241 | batch_size=100, 242 | n_epochs=20, 243 | lr_scaler=0.998, 244 | weights_file="out/ae_weights_mnist.npy"): 245 | """ 246 | Fit the data to the autoencoder model. Basically this performs 247 | the learning. 248 | """ 249 | if data is None: 250 | raise Exception("Data can't be empty.") 251 | 252 | index = T.lscalar('index') 253 | data_shared = theano.shared(numpy.asarray(data.tolist(), dtype=theano.config.floatX)) 254 | n_batches = data.shape[0] / batch_size 255 | (cost, updates) = self.get_sgd_updates(learning_rate, lr_scaler, batch_size) 256 | train_ae = theano.function([index], 257 | cost, 258 | updates=updates, 259 | givens={ 260 | self.x: data_shared[index * batch_size: (index + 1) * batch_size] 261 | } 262 | ) 263 | 264 | print "Started the training." 265 | ae_costs = [] 266 | 267 | for epoch in xrange(n_epochs): 268 | print "Training at epoch %d" % epoch 269 | for batch_index in xrange(n_batches): 270 | ae_costs.append(train_ae(batch_index)) 271 | print "Training at epoch %d, %f" % (epoch, numpy.mean(ae_costs)) 272 | 273 | print "Saving files..." 274 | numpy.save(weights_file, self.params[0].get_value()) 275 | return ae_costs 276 | -------------------------------------------------------------------------------- /ca.py: -------------------------------------------------------------------------------- 1 | import theano 2 | import theano.tensor as T 3 | from theano.tensor.shared_randomstreams import RandomStreams 4 | import numpy 5 | 6 | from ae import Autoencoder, CostType, Nonlinearity 7 | from collections import OrderedDict 8 | 9 | #Contractive Autoencoder implementation. 10 | class ContractiveAutoencoder(Autoencoder): 11 | 12 | def __init__(self, 13 | input, 14 | nvis, 15 | nhid, 16 | rnd=None, 17 | theano_rng=None, 18 | bhid=None, 19 | sigma=0.06, 20 | nonlinearity=Nonlinearity.SIGMOID, 21 | cost_type=CostType.MeanSquared, 22 | bvis=None): 23 | self.sigma = sigma 24 | # create a Theano random generator that gives symbolic random values 25 | super(ContractiveAutoencoder, self).__init__(input, nvis, nhid, rnd, bhid, cost_type, 26 | nonlinearity=nonlinearity, sparse_initialize=True, bvis=bvis) 27 | if not theano_rng : 28 | theano_rng = RandomStreams(rnd.randint(2 ** 30)) 29 | self.theano_rng = theano_rng 30 | 31 | def get_linear_hidden_outs(self, x_in=None): 32 | if x_in is None: 33 | x_in = self.x 34 | return T.dot(x_in, self.hidden.W) + self.hidden.b 35 | 36 | def contraction_penalty(self, h, linear_hid, contraction_level=0.0, batch_size=-1): 37 | """ 38 | Compute the contraction penalty in the way that Ian describes in his e-mail: 39 | https://groups.google.com/d/topic/pylearn-dev/iY7swxgn-xI/discussion 40 | """ 41 | if batch_size == -1 or batch_size == 0: 42 | raise Exception("Invalid batch_size!") 43 | 44 | grad = T.grad(h.sum(), linear_hid) 45 | jacob = T.dot(T.sqr(grad), T.sqr(self.hidden.W.sum(axis=0))) 46 | frob_norm_jacob = T.sum(jacob) / batch_size 47 | contract_pen = contraction_level * frob_norm_jacob 48 | return contract_pen 49 | 50 | def get_ca_sgd_updates(self, learning_rate, contraction_level, batch_size, x_in=None): 51 | h, linear_hid = self.encode_linear(x_in) 52 | x_rec = self.decode(h) 53 | cost = self.get_rec_cost(x_rec) 54 | contract_penal = self.contraction_penalty(h, linear_hid, contraction_level, batch_size) 55 | cost = cost + contract_penal 56 | gparams = T.grad(cost, self.params) 57 | updates = OrderedDict({}) 58 | for param, gparam in zip(self.params, gparams): 59 | updates[param] = param - learning_rate * gparam 60 | return (cost, updates) 61 | 62 | def sample(self, x, K): 63 | 64 | if x.ndim == 1: 65 | x = x.reshape(1, x.shape[0]) 66 | hn = self.encode(x) 67 | W = self.params[0] 68 | ww = T.dot(W.T, W) 69 | samples = [] 70 | for _ in range(K): 71 | s = hn * (1. - hn) 72 | jj = ww * s.dimshuffle(0, 'x', 1) * s.dimshuffle(0, 1, 'x') 73 | alpha = self.srng.normal(size=hn.shape, 74 | avg=0., 75 | std=self.sigma, 76 | dtype=theano.config.floatX) 77 | 78 | delta = (alpha.dimshuffle(0, 1, 'x') * jj).sum(1) 79 | 80 | zn = self.decode(hn + delta) 81 | hn = self.encode(zn) 82 | #zn2 = self.decode(hn) 83 | samples.append(zn.eval()) 84 | return samples 85 | 86 | def fit(self, 87 | data=None, 88 | learning_rate=0.1, 89 | batch_size=100, 90 | n_epochs=22, 91 | contraction_level=0.1, 92 | shuffle_data=True, 93 | weights_file="out/cae_weights_mnist.npy"): 94 | 95 | if data is None: 96 | raise Exception("Data can't be empty.") 97 | 98 | index = T.iscalar('index') 99 | data = numpy.asarray(data.tolist(), dtype="float32") 100 | data_shared = theano.shared(data) 101 | n_batches = data.shape[0] / batch_size 102 | (cost, updates) = self.get_ca_sgd_updates(learning_rate, contraction_level, batch_size) 103 | 104 | train_ae = theano.function([index], 105 | cost, 106 | updates=updates, 107 | givens={self.x: data_shared[index * batch_size: (index + 1) * batch_size]}) 108 | 109 | print "Started the training." 110 | ae_costs = [] 111 | for epoch in xrange(n_epochs): 112 | if shuffle_data: 113 | print "shuffling the dataset" 114 | numpy.random.shuffle(data) 115 | data_shared.set_value(data) 116 | print "Training at epoch %d" % epoch 117 | for batch_index in xrange(n_batches): 118 | ae_costs.append(train_ae(batch_index)) 119 | print "Training at epoch %d, %f" % (epoch, numpy.mean(ae_costs)) 120 | 121 | print "Saving files..." 122 | numpy.save(weights_file, self.params[0].get_value()) 123 | return ae_costs 124 | -------------------------------------------------------------------------------- /da.py: -------------------------------------------------------------------------------- 1 | import theano 2 | import theano.tensor as T 3 | from theano.tensor.shared_randomstreams import RandomStreams 4 | 5 | import numpy 6 | 7 | from ae import Autoencoder, CostType, Nonlinearity 8 | 9 | class DenoisingAutoencoder(Autoencoder): 10 | 11 | def __init__(self, 12 | input, 13 | nvis, 14 | nhid, 15 | rnd=None, 16 | theano_rng=None, 17 | bhid=None, 18 | cost_type=CostType.MeanSquared, 19 | momentum=1, 20 | L1_reg=-1, 21 | L2_reg=-1, 22 | sparse_initialize=False, 23 | nonlinearity=Nonlinearity.TANH, 24 | bvis=None, 25 | tied_weights=True): 26 | 27 | # create a Theano random generator that gives symbolic random values 28 | super(DenoisingAutoencoder, self).__init__(input, 29 | nvis, 30 | nhid, 31 | rnd, 32 | bhid, 33 | cost_type, 34 | momentum, 35 | L1_reg=L1_reg, 36 | L2_reg=L2_reg, 37 | sparse_initialize=sparse_initialize, 38 | nonlinearity=nonlinearity, 39 | bvis=bvis, 40 | tied_weights=tied_weights) 41 | 42 | if not theano_rng : 43 | theano_rng = RandomStreams(rnd.randint(2 ** 30)) 44 | self.theano_rng = theano_rng 45 | 46 | def corrupt_input(self, in_data, corruption_level): 47 | return self.theano_rng.binomial(self.x.shape, n=1, p=1-corruption_level, 48 | dtype=theano.config.floatX) * self.x 49 | 50 | def get_reconstructed_images(self, data): 51 | h = self.encode(x_in=data) 52 | x_rec = self.decode(h) 53 | return x_rec 54 | 55 | def debug_grads(self, data): 56 | gfn = theano.function([self.x], self.gparams[0]) 57 | print "gradients:" 58 | print gfn(data) 59 | print "params:" 60 | print self.hidden.W.get_value() 61 | 62 | def fit(self, 63 | data=None, 64 | learning_rate=0.1, 65 | learning_rate_decay=None, 66 | batch_size=100, 67 | n_epochs=60, 68 | corruption_level=0.5, 69 | weights_file=None, 70 | sparsity_level=-1, 71 | sparse_reg=-1, 72 | shuffle_data=True, 73 | lr_scaler=1.0, 74 | recons_img_file="out/dae_reconstructed_pento.npy"): 75 | 76 | if data is None: 77 | raise Exception("Data can't be empty.") 78 | 79 | index = T.iscalar('index') 80 | data_shared = theano.shared(numpy.asarray(data.tolist(), dtype=theano.config.floatX)) 81 | n_batches = data.shape[0] / batch_size 82 | 83 | corrupted_input = self.corrupt_input(data_shared, corruption_level) 84 | 85 | (cost, updates) = self.get_sgd_updates(learning_rate, lr_scaler=lr_scaler, batch_size=batch_size, 86 | sparsity_level=sparsity_level, 87 | sparse_reg=sparse_reg, x_in=corrupted_input) 88 | 89 | train_ae = theano.function([index], 90 | cost, 91 | updates=updates, 92 | givens={self.x: data_shared[index * batch_size: (index + 1) * batch_size]}) 93 | 94 | print "Started the training." 95 | ae_costs = [] 96 | batch_index = 0 97 | for epoch in xrange(n_epochs): 98 | idxs = numpy.arange(n_batches) 99 | numpy.random.shuffle(idxs) 100 | print "Training at epoch %d" % epoch 101 | for batch_index in idxs: 102 | ae_costs.append(train_ae(batch_index)) 103 | if False: 104 | print "Cost: ", ae_costs[-1] 105 | self.debug_grads(data_shared.get_value()[batch_index * batch_size: (batch_index + 1) * batch_size]) 106 | print "Training at epoch %d, %f" % (epoch, numpy.mean(ae_costs)) 107 | 108 | if weights_file is not None: 109 | print "Saving weights..." 110 | numpy.save(weights_file, self.params[0].get_value()) 111 | if recons_img_file is not None: 112 | print "Saving reconstructed images..." 113 | x_rec = self.get_reconstructed_images(data_shared) 114 | numpy.save(recons_img_file, x_rec) 115 | return ae_costs 116 | -------------------------------------------------------------------------------- /dataset.py: -------------------------------------------------------------------------------- 1 | from __future__ import division 2 | 3 | import pickle as pkl 4 | import math 5 | import numpy as np 6 | 7 | 8 | class Dataset(object): 9 | 10 | def __init__(self, is_binary=False): 11 | self.is_binary = is_binary 12 | 13 | #Examples 14 | self.Xtrain = None 15 | self.Xtest = None 16 | 17 | #Labels 18 | self.Ytrain = None 19 | self.Ytest = None 20 | 21 | self.Xtrain_pres = None 22 | self.Xtest_pres = None 23 | 24 | self.sparsity = 0.0 25 | self.n_examples = 0 26 | 27 | def _get_data(self, data_path): 28 | if data_path.endswith("pkl") or data_path.endswith("pickle"): 29 | data = pkl.load(open(data_path, "rb")) 30 | else: 31 | data = np.load(data_path) 32 | return data 33 | 34 | def binarize_labels(self, labels=None): 35 | #Largest label is for the images without different object. 36 | last_lbl = np.max(labels) 37 | binarized_lbls = [] 38 | if self.is_binary: 39 | for label in labels: 40 | if label == last_lbl: 41 | binarized_lbls.append(0) 42 | else: 43 | binarized_lbls.append(1) 44 | return binarized_lbls 45 | 46 | def setup_dataset(self, data_path=None, train_split_scale = 0.8): 47 | 48 | data = self._get_data(data_path) 49 | self.n_examples = data[0].shape[0] 50 | ntrain = math.floor(self.n_examples * train_split_scale) 51 | 52 | self.Xtrain = data[0][:ntrain] 53 | self.Xtrain_pres = data[2][:ntrain] 54 | self.Xtest = data[0][ntrain:] 55 | self.Xtest_pre = data[2][ntrain:] 56 | 57 | if train_split_scale != 0.0: 58 | self.Ytrain = np.array(self.binarize_labels(data[1][:ntrain].flatten()) \ 59 | if self.is_binary else data[1][:ntrain].flatten()) 60 | 61 | if train_split_scale != 1.0: 62 | self.Ytest = np.array(self.binarize_labels(data[1][ntrain:].flatten()) \ 63 | if self.is_binary else data[1][ntrain:].flatten()) 64 | 65 | def comp_sparsity(self): 66 | num_sparse_els = 0 67 | for el in self.Xtrain.flatten(): 68 | if el == 0: 69 | num_sparse_els+=1 70 | for el in self.Xtest.flatten(): 71 | if el == 0: 72 | num_sparse_els+=1 73 | self.sparsity = (num_sparse_els/self.n_examples) 74 | return self.sparsity 75 | -------------------------------------------------------------------------------- /layer.py: -------------------------------------------------------------------------------- 1 | from __future__ import division 2 | import numpy 3 | import theano 4 | from theano import tensor as T 5 | 6 | class Layer(object): 7 | """ 8 | A general base layer class for neural networks. 9 | """ 10 | def __init__(self, 11 | input, 12 | n_in, 13 | n_out, 14 | activation=T.nnet.sigmoid, 15 | sparse_initialize=False, 16 | num_pieces=1, 17 | non_zero_units=25, 18 | rng=None): 19 | 20 | self.num_pieces = num_pieces 21 | self.input = input 22 | self.n_in = n_in 23 | self.n_out = n_out 24 | self.rng = rng 25 | self.sparse_initialize = sparse_initialize 26 | self.non_zero_units = non_zero_units 27 | self.W = None 28 | self.b = None 29 | self.sparse_initialize = sparse_initialize 30 | self.activation = activation 31 | 32 | def reset_layer(self): 33 | if self.W is None: 34 | if self.sparse_initialize: 35 | W_values = self.sparse_initialize_weights() 36 | else: 37 | W_values = numpy.asarray(self.rng.uniform( 38 | low=-numpy.sqrt(6. / (self.n_in + self.n_out)), 39 | high=numpy.sqrt(6. / (self.n_in + self.n_out)), 40 | size=(self.n_in, self.n_out)), 41 | dtype=theano.config.floatX) 42 | 43 | if self.activation == theano.tensor.nnet.sigmoid: 44 | W_values *= 4 45 | 46 | self.W = theano.shared(value=W_values, name='W', borrow=True) 47 | 48 | if self.b is None: 49 | b_values = numpy.zeros((self.n_out/self.num_pieces), dtype=theano.config.floatX) 50 | self.b = theano.shared(value=b_values, name='b', borrow=True) 51 | # parameters of the model 52 | self.params = [self.W, self.b] 53 | 54 | def sparse_initialize_weights(self): 55 | #Implement the sparse initialization technique as decribed in 2010 Martens. 56 | W = [] 57 | mu, sigma = 0, 1/self.non_zero_units 58 | 59 | for i in xrange(self.n_in): 60 | row = numpy.zeros(self.n_out) 61 | non_zeros = self.rng.normal(mu, sigma, self.non_zero_units) 62 | #non_zeros /= non_zeros.sum() 63 | non_zero_idxs = self.rng.permutation(self.n_out)[0:self.non_zero_units] 64 | for j in xrange(self.non_zero_units): 65 | row[non_zero_idxs[j]] = non_zeros[j] 66 | W.append(row) 67 | W = numpy.asarray(W, dtype=theano.config.floatX) 68 | return W 69 | 70 | class AEHiddenLayer(Layer): 71 | 72 | def __init__(self, 73 | input, 74 | n_in, 75 | n_out, 76 | n_in_dec=None, 77 | n_out_dec=None, 78 | W=None, 79 | b=None, 80 | num_pieces=1, 81 | bhid=None, 82 | activation=T.nnet.sigmoid, 83 | sparse_initialize=False, 84 | tied_weights=True, 85 | rng=None): 86 | """ 87 | Typical hidden layer of a MLP: units are fully-connected and have 88 | sigmoidal activation function. Weight matrix W is of shape (n_in,n_out) 89 | and the bias vector b is of shape (n_out,). 90 | 91 | NOTE : The nonlinearity used here is tanh 92 | 93 | Hidden unit activation is given by: tanh(dot(input,W) + b) 94 | 95 | :type rng: numpy.random.RandomState 96 | :param rng: a random number generator used to initialize weights 97 | 98 | :type input: theano.tensor.dmatrix 99 | :param input: a symbolic tensor of shape (n_examples, n_in) 100 | 101 | :type n_in: int 102 | 103 | :param n_in: dimensionality of input 104 | :type n_out: int 105 | :param n_out: number of hidden units 106 | 107 | :type activation: theano.Op or function 108 | :param activation: Non linearity to be applied in the hidden layer. 109 | """ 110 | if rng is None: 111 | rng = numpy.random.RandomState() 112 | 113 | super(AEHiddenLayer, self).__init__(input, 114 | n_in, 115 | n_out, 116 | num_pieces=num_pieces, 117 | activation=activation, 118 | sparse_initialize=sparse_initialize, 119 | rng=rng) 120 | 121 | self.reset_layer() 122 | 123 | if W is not None: 124 | self.W = W 125 | if b is not None: 126 | self.b = b 127 | 128 | if bhid is not None: 129 | self.b_prime = bhid 130 | else: 131 | if n_in_dec is not None: 132 | b_values = numpy.zeros((n_out_dec), dtype=theano.config.floatX) 133 | else: 134 | b_values = numpy.zeros((self.n_out/num_pieces), dtype=theano.config.floatX) 135 | 136 | self.b_prime = theano.shared(value=b_values, name='b_prime') 137 | 138 | if tied_weights: 139 | self.W_prime = self.W.T 140 | else: 141 | if n_in_dec is not None and n_out_dec is not None: 142 | W_values = numpy.asarray(self.rng.normal( 143 | loc=0., 144 | scale=0.005, 145 | size=(n_out_dec, n_in_dec)), 146 | dtype=theano.config.floatX) 147 | else: 148 | W_values = numpy.asarray(self.rng.uniform( 149 | low=-numpy.sqrt(6. / (self.n_in + self.n_out)), 150 | high=numpy.sqrt(6. / (self.n_in + self.n_out)), 151 | size=(self.n_out, self.n_in)), 152 | dtype=theano.config.floatX) 153 | 154 | if self.activation == T.nnet.sigmoid: 155 | W_values *= 4 156 | 157 | self.W_prime = theano.shared(value=W_values, name='W', borrow=True) 158 | self.params += [self.W_prime] 159 | 160 | self.params += [self.b_prime] 161 | self.setup_outputs(input) 162 | 163 | def setup_outputs(self, input): 164 | lin_output = T.dot(input, self.W) + self.b 165 | self.output = (lin_output if self.activation is None 166 | else self.activation(lin_output)) 167 | 168 | def get_outputs(self, input): 169 | self.setup_outputs(input) 170 | return self.output 171 | 172 | class HiddenLayer(Layer): 173 | 174 | def __init__(self, input, n_in, n_out, W=None, b=None, activation=T.tanh, rng=None): 175 | """ 176 | Typical hidden layer of a MLP: units are fully-connected and have 177 | sigmoidal activation function. Weight matrix W is of shape (n_in,n_out) 178 | and the bias vector b is of shape (n_out,). 179 | 180 | NOTE : The nonlinearity used here is tanh 181 | 182 | Hidden unit activation is given by: tanh(dot(input,W) + b) 183 | 184 | :type rng: numpy.random.RandomState 185 | :param rng: a random number generator used to initialize weights 186 | 187 | :type input: theano.tensor.dmatrix 188 | :param input: a symbolic tensor of shape (n_examples, n_in) 189 | 190 | :type n_in: int 191 | 192 | :param n_in: dimensionality of input 193 | :type n_out: int 194 | :param n_out: number of hidden units 195 | 196 | :type activation: theano.Op or function 197 | :param activation: Non linearity to be applied in the hidden layer. 198 | """ 199 | if rng is None: 200 | rng = numpy.random.RandomState() 201 | 202 | super(HiddenLayer, self).__init__(input, n_in, n_out, activation=activation, rng=rng) 203 | self.reset_layer() 204 | 205 | if W is not None: 206 | self.W = W 207 | 208 | if b is not None: 209 | self.b = b 210 | 211 | self.setup_outputs(input) 212 | 213 | def setup_outputs(self, input): 214 | lin_output = T.dot(input, self.W) + self.b 215 | self.output = (lin_output if self.activation is None 216 | else self.activation(lin_output)) 217 | 218 | def get_outputs(self, input): 219 | self.setup_outputs(input) 220 | return self.output 221 | 222 | class LogisticRegressionLayer(Layer): 223 | """ 224 | Multi-class Logistic Regression Class. 225 | The logistic regression is fully described by a weight matrix :math:`W` 226 | and bias vector :math:`b`. Classification is done by projecting data 227 | points onto a set of hyperplanes, the distance to which is used to 228 | determine a class membership probability. 229 | """ 230 | def __init__(self, input, n_in, n_out, is_binary=False, threshold=0.4, rng=None): 231 | """ Initialize the parameters of the logistic regression 232 | :type input: theano.tensor.TensorType 233 | :param input: symbolic variable that describes the input of the architecture 234 | (one minibatch) 235 | :type n_in: int 236 | :param n_in: number of input units, the dimension of the space in which 237 | the datapoints lie 238 | :type n_out: int 239 | :param n_out: number of output units, the dimension of the space in 240 | which the labels lie 241 | """ 242 | self.activation = T.nnet.sigmoid 243 | super(LogisticRegressionLayer, self).__init__(input, 244 | n_in, n_out, self.activation, rng) 245 | 246 | self.reset_layer() 247 | 248 | self.is_binary = is_binary 249 | if n_out == 1: 250 | self.is_binary = True 251 | # The number of classes seen 252 | self.n_classes_seen = numpy.zeros(n_out) 253 | # The number of wrong classification made for class i 254 | self.n_wrong_clasif_made = numpy.zeros(n_out) 255 | 256 | self.reset_conf_mat() 257 | # 258 | # compute vector of class-membership probabilities in symbolic form 259 | # self.p_y_given_x = T.nnet.softmax(T.dot(input, self.W) + self.b) 260 | self.p_y_given_x = self.get_class_memberships(self.input) 261 | 262 | if not self.is_binary: 263 | # compute prediction as class whose probability is maximal in 264 | # symbolic form 265 | self.y_decision = T.argmax(self.p_y_given_x, axis=1) 266 | else: 267 | #If the probability is greater than 0.5 assign to the class 1 268 | # otherwise it is 0. Which can also be interpreted as check if 269 | # p(y=1|x)>threshold. 270 | self.y_decision = T.gt(T.flatten(self.p_y_given_x), threshold) 271 | 272 | # parameters of the model 273 | self.params = [self.W, self.b] 274 | 275 | def reset_conf_mat(self): 276 | """ 277 | Reset the confusion matrix. 278 | """ 279 | self.conf_mat = numpy.zeros(shape=(self.n_out, self.n_out), dtype=numpy.dtype(int)) 280 | 281 | def negative_log_likelihood(self, y): 282 | """ Return the mean of the negative log-likelihood of the prediction 283 | of this model under a given target distribution. 284 | 285 | .. math:: 286 | \frac{1}{|\mathcal{D}|} \mathcal{L} (\theta=\{W,b\}, \mathcal{D}) = 287 | \frac{1}{|\mathcal{D}|} \sum_{i=0}^{|\mathcal{D}|} \log(P(Y=y^{(i)}|x^{(i)}, W,b)) \\ 288 | \ell (\theta=\{W,b\}, \mathcal{D}) 289 | 290 | :type y: theano.tensor.TensorType 291 | :param y: corresponds to a vector that gives for each example the 292 | correct label 293 | Note: we use the mean instead of the sum so that 294 | the learning rate is less dependent on the batch size 295 | """ 296 | 297 | # y.shape[0] is (symbolically) the number of rows in y, i.e., 298 | # number of examples (call it n) in the minibatch 299 | # T.arange(y.shape[0]) is a symbolic vector which will contain 300 | # [0,1,2,... n-1] T.log(self.p_y_given_x) is a matrix of 301 | # Log-Probabilities (call it LP) with one row per example and 302 | # one column per class LP[T.arange(y.shape[0]),y] is a vector 303 | # v containing [LP[0,y[0]], LP[1,y[1]], LP[2,y[2]], ..., 304 | # LP[n-1,y[n-1]]] and T.mean(LP[T.arange(y.shape[0]),y]) is 305 | # the mean (across minibatch examples) of the elements in v, 306 | # i.e., the mean log-likelihood across the minibatch. 307 | if self.is_binary: 308 | -T.mean(T.log(self.p_y_given_x)) 309 | return -T.mean(T.log(self.p_y_given_x)[T.arange(y.shape[0]), y]) 310 | 311 | def crossentropy_categorical(self, y): 312 | """ 313 | Find the categorical crossentropy. 314 | """ 315 | return T.mean(T.nnet.categorical_crossentropy(self.p_y_given_x, y)) 316 | 317 | def crossentropy(self, y): 318 | """ 319 | use the theano nnet cross entropy function. Return the mean. 320 | Note: self.p_y_given_x is (batch_size, 1) but y is is (batch_size,) 321 | in order to establish the compliance, we should flatten the p_y_given_x. 322 | """ 323 | return T.mean(T.nnet.binary_crossentropy(T.flatten(self.p_y_given_x), y)) 324 | 325 | def get_class_memberships(self, x): 326 | lin_activation = T.dot(x, self.W) + self.b 327 | if self.is_binary: 328 | """If it is binary return the sigmoid.""" 329 | return T.nnet.sigmoid(lin_activation) 330 | """ 331 | Else return the softmax class memberships. 332 | """ 333 | return T.nnet.softmax(lin_activation) 334 | 335 | def update_conf_mat(self, y, p_y_given_x): 336 | """ 337 | Update the confusion matrix with the given true labels and estimated 338 | labels. 339 | """ 340 | if self.n_out == 1: 341 | y_decision = (p_y_given_x > 0.5) 342 | y_decision = y_decision.astype(int) 343 | else: 344 | y_decision = numpy.argmax(p_y_given_x, axis=1) 345 | for i in xrange(y.shape[0]): 346 | self.conf_mat[y[i]][y_decision[i]] +=1 347 | 348 | def errors(self, y): 349 | """Return a float representing the number of errors in the minibatch 350 | over the total number of examples of the minibatch ; zero one 351 | loss over the size of the minibatch 352 | 353 | :type y: theano.tensor.TensorType 354 | :param y: corresponds to a vector that gives for each example the 355 | correct label 356 | """ 357 | # check if y has same dimension of y_decision 358 | if y.ndim != self.y_decision.ndim: 359 | raise TypeError('y should have the same shape as self.y_decision', 360 | ('y', y.type, 'y_decision', self.y_decision.type)) 361 | # check if y is of the correct datatype 362 | if y.dtype.startswith('int') or y.dtype.startswith('uint'): 363 | # the T.neq operator returns a vector of 0s and 1s, where 1 364 | # represents a mistake in prediction 365 | return T.mean(T.neq(self.y_decision, y)) 366 | else: 367 | raise NotImplementedError() 368 | 369 | def raw_prediction_errors(self, y): 370 | """Return a float representing the number of errors in the minibatch 371 | over the total number of examples of the minibatch ; zero one 372 | loss over the size of the minibatch 373 | 374 | :type y: theano.tensor.TensorType 375 | :param y: corresponds to a vector that gives for each example the 376 | correct label 377 | """ 378 | # check if y has same dimension of y_decision 379 | if y.ndim != self.y_decision.ndim: 380 | raise TypeError('y should have the same shape as self.y_decision', 381 | ('y', y.type, 'y_decision', self.y_decision.type)) 382 | # check if y is of the correct datatype 383 | if y.dtype.startswith('int') or y.dtype.startswith('uint'): 384 | # the T.neq operator returns a vector of 0s and 1s, where 1 385 | # represents a mistake in prediction 386 | return T.neq(self.y_decision, y) 387 | else: 388 | raise NotImplementedError() 389 | 390 | def error_per_classes(self, y): 391 | """Return a float representing the number of errors in the minibatch 392 | over the total number of examples of the minibatch ; zero one 393 | loss over the size of the minibatch 394 | 395 | :type y: theano.tensor.TensorType 396 | :param y: corresponds to a vector that gives for each example the 397 | correct label 398 | """ 399 | # check if y has same dimension of y_decision 400 | if y.ndim != self.y_decision.ndim: 401 | raise TypeError('y should have the same shape as self.y_decision', 402 | ('y', y.type, 'y_decision', self.y_decision.type)) 403 | # check if y is of the correct datatype 404 | if y.dtype.startswith('int') or y.dtype.startswith('uint'): 405 | # the T.neq operator returns a vector of 0s and 1s, where 1 406 | # represents a mistake in prediction 407 | y_decision_res = T.neq(self.y_decision, y) 408 | for (i, y_decision_r) in enumerate(y_decision_res): 409 | self.n_classes_seen[y[i]] += 1 410 | if y_decision_r: 411 | self.n_wrong_clasif_made[y[i]] += 1 412 | pred_per_class = self.n_wrong_clasif_made / self.n_classes_seen 413 | return T.mean(y_decision_res), pred_per_class 414 | else: 415 | raise NotImplementedError() 416 | -------------------------------------------------------------------------------- /out/clean_binary_files.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | rm *.png 3 | rm *.npy 4 | rm dae/*.png 5 | rm dae/*.npy 6 | -------------------------------------------------------------------------------- /out/plot_ae_feats.py: -------------------------------------------------------------------------------- 1 | import PIL.Image 2 | import numpy 3 | from utils import tile_raster_images 4 | import pickle as pkl 5 | 6 | data = numpy.load("pae_mnist_enc_weights.npy") 7 | 8 | image = PIL.Image.fromarray(tile_raster_images( 9 | X=data.T, 10 | img_shape=(48, 48), tile_shape=(4, 600), 11 | tile_spacing=(1, 1))) 12 | 13 | image.save('filters_ae.png') 14 | -------------------------------------------------------------------------------- /out/plot_cae_feats.py: -------------------------------------------------------------------------------- 1 | import PIL.Image 2 | import numpy 3 | from utils import tile_raster_images 4 | import pickle as pkl 5 | 6 | data = numpy.load("cae_mnist_weights.npy") 7 | 8 | image = PIL.Image.fromarray(tile_raster_images( 9 | X=data.T, 10 | img_shape=(28, 28), 11 | tile_shape=(12, 12), 12 | tile_spacing=(1, 1))) 13 | 14 | image.save('sparse_filters_cae.png') 15 | -------------------------------------------------------------------------------- /out/plot_dae_feats.py: -------------------------------------------------------------------------------- 1 | import PIL.Image 2 | import numpy 3 | from utils import tile_raster_images 4 | import pickle as pkl 5 | 6 | data = numpy.load("dae_mnist_weights.npy") 7 | 8 | image = PIL.Image.fromarray(tile_raster_images( 9 | X=data.T, 10 | img_shape=(28, 28), tile_shape=(12, 12), 11 | tile_spacing=(1, 1))) 12 | image.save('mnist_filters_dae.png') 13 | -------------------------------------------------------------------------------- /out/plot_sa_feats.py: -------------------------------------------------------------------------------- 1 | import PIL.Image 2 | import numpy 3 | from utils import tile_raster_images 4 | import pickle as pkl 5 | 6 | data = numpy.load("sa_weights_mnist.npy") 7 | 8 | image = PIL.Image.fromarray(tile_raster_images( 9 | X=data.T, 10 | img_shape=(28, 28), tile_shape=(12, 12), 11 | tile_spacing=(1, 1))) 12 | 13 | image.save('filters_sae.png') 14 | -------------------------------------------------------------------------------- /out/plot_samples_fn.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | import pylab 3 | 4 | def plot_samples(samples, no_of_rows, no_of_cols, pno=1, n_samples=30, img_shp=(28, 28)): 5 | if pno >= n_samples: 6 | return 0 7 | else: 8 | pylab.axis("off") 9 | pylab.subplot(no_of_rows, no_of_cols, pno) 10 | pylab.imshow(samples[pno].reshape(img_shp)) 11 | print pno 12 | plot_samples(samples, no_of_rows, no_of_cols, pno+1, n_samples, img_shp) 13 | 14 | if __name__=="__main__": 15 | data = numpy.load("data_samples.npy") 16 | pylab.gray() 17 | plot_samples(data, 3, 10, pno=0) 18 | pylab.show() 19 | -------------------------------------------------------------------------------- /out/utils.py: -------------------------------------------------------------------------------- 1 | """ This file contains different utility functions that are not connected 2 | in anyway to the networks presented in the tutorials, but rather help in 3 | processing the outputs into a more understandable way. 4 | 5 | For example ``tile_raster_images`` helps in generating a easy to grasp 6 | image from a set of samples or weights. 7 | """ 8 | 9 | 10 | import numpy 11 | 12 | 13 | def scale_to_unit_interval(ndar, eps=1e-8): 14 | """ Scales all values in the ndarray ndar to be between 0 and 1 """ 15 | ndar = ndar.copy() 16 | ndar -= ndar.min() 17 | ndar *= 1.0 / (ndar.max() + eps) 18 | return ndar 19 | 20 | 21 | def tile_raster_images(X, img_shape, tile_shape, tile_spacing=(0, 0), 22 | scale_rows_to_unit_interval=True, 23 | output_pixel_vals=True): 24 | """ 25 | Transform an array with one flattened image per row, into an array in 26 | which images are reshaped and layed out like tiles on a floor. 27 | 28 | This function is useful for visualizing datasets whose rows are images, 29 | and also columns of matrices for transforming those rows 30 | (such as the first layer of a neural net). 31 | 32 | :type X: a 2-D ndarray or a tuple of 4 channels, elements of which can 33 | be 2-D ndarrays or None; 34 | :param X: a 2-D array in which every row is a flattened image. 35 | 36 | :type img_shape: tuple; (height, width) 37 | :param img_shape: the original shape of each image 38 | 39 | :type tile_shape: tuple; (rows, cols) 40 | :param tile_shape: the number of images to tile (rows, cols) 41 | 42 | :param output_pixel_vals: if output should be pixel values (i.e. int8 43 | values) or floats 44 | 45 | :param scale_rows_to_unit_interval: if the values need to be scaled before 46 | being plotted to [0,1] or not 47 | 48 | 49 | :returns: array suitable for viewing as an image. 50 | (See:`PIL.Image.fromarray`.) 51 | :rtype: a 2-d array with same dtype as X. 52 | 53 | """ 54 | 55 | assert len(img_shape) == 2 56 | assert len(tile_shape) == 2 57 | assert len(tile_spacing) == 2 58 | 59 | # The expression below can be re-written in a more C style as 60 | # follows : 61 | # 62 | # out_shape = [0,0] 63 | # out_shape[0] = (img_shape[0]+tile_spacing[0])*tile_shape[0] - 64 | # tile_spacing[0] 65 | # out_shape[1] = (img_shape[1]+tile_spacing[1])*tile_shape[1] - 66 | # tile_spacing[1] 67 | out_shape = [(ishp + tsp) * tshp - tsp for ishp, tshp, tsp 68 | in zip(img_shape, tile_shape, tile_spacing)] 69 | 70 | if isinstance(X, tuple): 71 | assert len(X) == 4 72 | # Create an output numpy ndarray to store the image 73 | if output_pixel_vals: 74 | out_array = numpy.zeros((out_shape[0], out_shape[1], 4), 75 | dtype='uint8') 76 | else: 77 | out_array = numpy.zeros((out_shape[0], out_shape[1], 4), 78 | dtype=X.dtype) 79 | 80 | #colors default to 0, alpha defaults to 1 (opaque) 81 | if output_pixel_vals: 82 | channel_defaults = [0, 0, 0, 255] 83 | else: 84 | channel_defaults = [0., 0., 0., 1.] 85 | 86 | for i in xrange(4): 87 | if X[i] is None: 88 | # if channel is None, fill it with zeros of the correct 89 | # dtype 90 | dt = out_array.dtype 91 | if output_pixel_vals: 92 | dt = 'uint8' 93 | out_array[:, :, i] = numpy.zeros(out_shape, 94 | dtype=dt) + channel_defaults[i] 95 | else: 96 | # use a recurrent call to compute the channel and store it 97 | # in the output 98 | out_array[:, :, i] = tile_raster_images( 99 | X[i], img_shape, tile_shape, tile_spacing, 100 | scale_rows_to_unit_interval, output_pixel_vals) 101 | return out_array 102 | 103 | else: 104 | # if we are dealing with only one channel 105 | H, W = img_shape 106 | Hs, Ws = tile_spacing 107 | 108 | # generate a matrix to store the output 109 | dt = X.dtype 110 | if output_pixel_vals: 111 | dt = 'uint8' 112 | out_array = numpy.zeros(out_shape, dtype=dt) 113 | 114 | for tile_row in xrange(tile_shape[0]): 115 | for tile_col in xrange(tile_shape[1]): 116 | if tile_row * tile_shape[1] + tile_col < X.shape[0]: 117 | this_x = X[tile_row * tile_shape[1] + tile_col] 118 | if scale_rows_to_unit_interval: 119 | # if we should scale values to be between 0 and 1 120 | # do this by calling the `scale_to_unit_interval` 121 | # function 122 | this_img = scale_to_unit_interval( 123 | this_x.reshape(img_shape)) 124 | else: 125 | this_img = this_x.reshape(img_shape) 126 | # add the slice to the corresponding position in the 127 | # output array 128 | c = 1 129 | if output_pixel_vals: 130 | c = 255 131 | out_array[ 132 | tile_row * (H + Hs): tile_row * (H + Hs) + H, 133 | tile_col * (W + Ws): tile_col * (W + Ws) + W 134 | ] = this_img * c 135 | return out_array 136 | 137 | -------------------------------------------------------------------------------- /sa.py: -------------------------------------------------------------------------------- 1 | import theano 2 | import theano.tensor as T 3 | from theano.tensor.shared_randomstreams import RandomStreams 4 | 5 | import numpy 6 | 7 | from ae import Autoencoder, CostType 8 | 9 | #Contractive Autoencoder implementation. 10 | class SparseAutoencoder(Autoencoder): 11 | 12 | def __init__(self, 13 | input, 14 | nvis, 15 | nhid, 16 | rnd=None, 17 | theano_rng=None, 18 | bhid=None, 19 | cost_type=CostType.CrossEntropy, 20 | bvis=None): 21 | 22 | # create a Theano random generator that gives symbolic random values 23 | super(SparseAutoencoder, self).__init__(input, nvis, nhid, rnd, bhid, cost_type, bvis) 24 | if not theano_rng : 25 | theano_rng = RandomStreams(rnd.randint(2 ** 30)) 26 | self.theano_rng = theano_rng 27 | 28 | def get_linear_hidden_outs(self, x_in=None): 29 | if x_in is None: 30 | x_in = self.x 31 | return T.dot(x_in, self.hidden.W) + self.hidden.b 32 | 33 | def kl_divergence(self, p, p_hat): 34 | term1 = p * T.log(p) 35 | term2 = p * T.log(p_hat) 36 | term3 = (1-p) * T.log(1 - p) 37 | term4 = (1-p) * T.log(1 - p_hat) 38 | return term1 - term2 + term3 - term4 39 | 40 | def sparsity_penalty(self, h, sparsity_level=0.05, sparse_reg=1e-3, batch_size=-1): 41 | if batch_size == -1 or batch_size == 0: 42 | raise Exception("Invalid batch_size!") 43 | sparsity_level = T.extra_ops.repeat(sparsity_level, self.nhid) 44 | sparsity_penalty = 0 45 | avg_act = h.mean(axis=0) 46 | kl_div = self.kl_divergence(sparsity_level, avg_act) 47 | sparsity_penalty = sparse_reg * kl_div.sum() 48 | # Implement KL divergence here. 49 | return sparsity_penalty 50 | 51 | def get_sa_sgd_updates(self, learning_rate, sparsity_level, sparse_reg, batch_size, x_in=None): 52 | h = self.encode(x_in) 53 | x_rec = self.decode(h) 54 | cost = self.get_rec_cost(x_rec) 55 | sparsity_penal = self.sparsity_penalty(h, sparsity_level, sparse_reg, batch_size) 56 | cost = cost + sparsity_penal 57 | 58 | gparams = T.grad(cost, self.params) 59 | updates = {} 60 | for param, gparam in zip(self.params, gparams): 61 | updates[param] = param - learning_rate * gparam 62 | return (cost, updates) 63 | 64 | def fit(self, 65 | data=None, 66 | learning_rate=0.08, 67 | batch_size=100, 68 | n_epochs=22, 69 | sparsity_penalty=0.001, 70 | sparsity_level=0.05, 71 | weights_file="out/sa_weights_mnist.npy"): 72 | 73 | if data is None: 74 | raise Exception("Data can't be empty.") 75 | 76 | index = T.lscalar('index') 77 | data_shared = theano.shared(numpy.asarray(data.tolist(), dtype=theano.config.floatX)) 78 | n_batches = data.shape[0] / batch_size 79 | (cost, updates) = self.get_sa_sgd_updates(learning_rate, sparsity_level, sparsity_penalty, batch_size) 80 | 81 | train_ae = theano.function([index], 82 | cost, 83 | updates=updates, 84 | givens={self.x: data_shared[index * batch_size: (index + 1) * batch_size]}) 85 | 86 | print "Started the training." 87 | ae_costs = [] 88 | for epoch in xrange(n_epochs): 89 | print "Training at epoch %d" % epoch 90 | for batch_index in xrange(n_batches): 91 | ae_costs.append(train_ae(batch_index)) 92 | print "Training at epoch %d, %f" % (epoch, numpy.mean(ae_costs)) 93 | 94 | print "Saving files..." 95 | numpy.save(weights_file, self.params[0].get_value()) 96 | return ae_costs 97 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caglar/autoencoders/a002c85205d98c073f9776e2196f70d726a3da06/tests/__init__.py -------------------------------------------------------------------------------- /tests/cae_stack_sample.py: -------------------------------------------------------------------------------- 1 | from ca import ContractiveAutoencoder 2 | from dataset import Dataset 3 | import theano.tensor as T 4 | import theano 5 | import numpy 6 | import cPickle as pkl 7 | 8 | def standardize(data): 9 | """ 10 | Normalize the data with respect to finding the mean and standard deviation of it 11 | and dividing by mean and standard deviation. 12 | """ 13 | mu = numpy.mean(data, axis=1) 14 | sigma = numpy.std(data, axis=1) 15 | 16 | if sigma.nonzero()[0].shape[0] == 0: 17 | raise Exception("Std dev should not be zero") 18 | 19 | norm_data = (data - mu) / sigma 20 | return norm_data 21 | 22 | if __name__=="__main__": 23 | fname = "/data/lisa/data/mnist/mnist.pkl" 24 | input = T.fmatrix("x_input") 25 | 26 | ds = Dataset() 27 | data = ds._get_data(fname) 28 | x_data = numpy.asarray(data[0][0][0:48000], dtype=theano.config.floatX) 29 | 30 | weights_file_l1 = "cae_mnist_weights_l1.npy" 31 | weights_file_l2 = "cae_mnist_weights_l2.npy" 32 | 33 | rnd = numpy.random.RandomState(1231) 34 | nhid_l1 = 900 35 | nhid_l2 = 900 36 | 37 | dae_l1 = ContractiveAutoencoder(input, 38 | nvis=28*28, 39 | nhid=nhid_l1, 40 | cost_type="CrossEntropy", 41 | rnd=rnd) 42 | 43 | #std_data = standardize(x_data) 44 | std_data = numpy.asarray(x_data, dtype=theano.config.floatX) 45 | 46 | dae_l1.fit(learning_rate=3*0.96*1e-2, 47 | data=std_data, 48 | weights_file=weights_file_l1, 49 | contraction_level=0.2, 50 | batch_size=128, 51 | n_epochs=60) 52 | 53 | dae_l1_obj_out = open("cae_l1_obj.pkl", "wb") 54 | pkl.dump(dae_l1, dae_l1_obj_out, protocol=pkl.HIGHEST_PROTOCOL) 55 | 56 | 57 | dae_l1_out = dae_l1.encode(input) 58 | dae_l1_h = dae_l1.encode(std_data) 59 | dae_l1_h_fn = theano.function([], dae_l1_h) 60 | dae_l2_in = dae_l1_h_fn() 61 | dae_l2_in = numpy.asarray(dae_l2_in, dtype=theano.config.floatX) 62 | 63 | dae_l2 = ContractiveAutoencoder(dae_l1_out, 64 | nvis=nhid_l1, 65 | nhid=nhid_l2, 66 | cost_type="CrossEntropy", 67 | rnd=rnd) 68 | 69 | dae_l2.fit(learning_rate=0.95*1e-1, 70 | data=dae_l2_in, 71 | weights_file=weights_file_l2, 72 | contraction_level=0.25, 73 | batch_size=128, 74 | n_epochs=80) 75 | 76 | dae_l2_obj_out = open("cae_l2_obj.pkl", "wb") 77 | pkl.dump(dae_l2, dae_l2_obj_out, protocol=pkl.HIGHEST_PROTOCOL) 78 | 79 | 80 | samples = dae_l2.sample(dae_l2_in[10], 40) 81 | numpy.save(open("data_samples_l2_dae.npy", "wb"), samples) 82 | -------------------------------------------------------------------------------- /tests/dae_stack_sample.py: -------------------------------------------------------------------------------- 1 | from da import DenoisingAutoencoder 2 | from dataset import Dataset 3 | import theano.tensor as T 4 | import theano 5 | import numpy 6 | import cPickle as pkl 7 | 8 | def standardize(data): 9 | """ 10 | Normalize the data with respect to finding the mean and standard deviation of it 11 | and dividing by mean and standard deviation. 12 | """ 13 | mu = numpy.mean(data, axis=0) 14 | sigma = numpy.std(data, axis=0) 15 | 16 | if sigma.nonzero()[0].shape[0] == 0: 17 | raise Exception("Std dev should not be zero") 18 | 19 | norm_data = (data - mu) / sigma 20 | return norm_data 21 | 22 | if __name__=="__main__": 23 | fname = "/data/lisa/data/mnist/mnist_all.pickle" 24 | input = T.fmatrix("x_input") 25 | 26 | ds = Dataset() 27 | data = ds._get_data(fname) 28 | x_data = numpy.asarray(data[0][0:42000], dtype=theano.config.floatX) 29 | 30 | weights_file_l1 = "dae_mnist_weights_l1.npy" 31 | weights_file_l2 = "dae_mnist_weights_l2.npy" 32 | 33 | rnd = numpy.random.RandomState(1231) 34 | nhid_l1 = 800 35 | nhid_l2 = 800 36 | 37 | dae_l1 = DenoisingAutoencoder(input, 38 | nvis=28*28, 39 | nhid=nhid_l1, 40 | L1_reg=9*1e-5, 41 | L2_reg=7*1e-4, 42 | rnd=rnd) 43 | 44 | #std_data = standardize(x_data) 45 | std_data = numpy.asarray(x_data, dtype=theano.config.floatX) 46 | 47 | dae_l1.fit(learning_rate=9.96*1e-3, 48 | shuffle_data=True, 49 | data=std_data, 50 | weights_file=weights_file_l1, 51 | recons_img_file=None, 52 | corruption_level=0.095, 53 | batch_size=40, 54 | n_epochs=1400) 55 | 56 | dae_l1_obj_out = open("dae_l1_obj.pkl", "wb") 57 | pkl.dump(dae_l1, dae_l1_obj_out, protocol=pkl.HIGHEST_PROTOCOL) 58 | 59 | 60 | dae_l1_out = dae_l1.encode(input) 61 | dae_l1_h = dae_l1.encode(std_data) 62 | dae_l1_h_fn = theano.function([], dae_l1_h) 63 | dae_l2_in = dae_l1_h_fn() 64 | dae_l2_in = numpy.asarray(dae_l2_in, dtype=theano.config.floatX) 65 | 66 | dae_l2 = DenoisingAutoencoder(dae_l1_out, 67 | L1_reg=1e-4, 68 | L2_reg=6*1e-4, 69 | nvis=nhid_l1, 70 | nhid=nhid_l2, 71 | rnd=rnd) 72 | 73 | dae_l2.fit(learning_rate=0.95*1e-2, 74 | data=dae_l2_in, 75 | shuffle_data=True, 76 | recons_img_file=None, 77 | weights_file=weights_file_l2, 78 | corruption_level=0.1, 79 | batch_size=25, 80 | n_epochs=1400) 81 | 82 | dae_l2_obj_out = open("dae_l2_obj.pkl", "wb") 83 | pkl.dump(dae_l2, dae_l2_obj_out, protocol=pkl.HIGHEST_PROTOCOL) 84 | 85 | 86 | samples = dae_l2.sample(dae_l2_in[10], 0.1, 1000) 87 | numpy.save(open("data_samples_l2_dae.npy", "wb"), samples) 88 | -------------------------------------------------------------------------------- /tests/dae_stack_sample_ctd.py: -------------------------------------------------------------------------------- 1 | from da import DenoisingAutoencoder 2 | from dataset import Dataset 3 | import theano.tensor as T 4 | import theano 5 | import numpy 6 | import cPickle as pkl 7 | 8 | def standardize(data): 9 | """ 10 | Normalize the data with respect to finding the mean and standard deviation of it 11 | and dividing by mean and standard deviation. 12 | """ 13 | mu = numpy.mean(data, axis=0) 14 | sigma = numpy.std(data, axis=0) 15 | 16 | if sigma.nonzero()[0].shape[0] == 0: 17 | raise Exception("Std dev should not be zero") 18 | 19 | norm_data = (data - mu) / sigma 20 | return norm_data 21 | 22 | if __name__=="__main__": 23 | fname = "/data/lisa/data/mnist/mnist_all.pickle" 24 | input = T.fmatrix("x_input") 25 | 26 | ds = Dataset() 27 | data = ds._get_data(fname) 28 | x_data = numpy.asarray(data[0][0:42000], dtype=theano.config.floatX) 29 | 30 | weights_file_l1 = "dae_mnist_weights_l1.npy" 31 | weights_file_l2 = "dae_mnist_weights_l2.npy" 32 | 33 | rnd = numpy.random.RandomState(1231) 34 | nhid_l1 = 800 35 | nhid_l2 = 800 36 | 37 | dae_l1 = DenoisingAutoencoder(input, 38 | nvis=28*28, 39 | nhid=nhid_l1, 40 | L1_reg=9.3*1e-5, 41 | L2_reg=8*1e-4, 42 | rnd=rnd) 43 | 44 | #std_data = standardize(x_data) 45 | std_data = numpy.asarray(x_data, dtype=theano.config.floatX) 46 | 47 | dae_l1.fit(learning_rate=9.96*1e-3, 48 | shuffle_data=True, 49 | data=std_data, 50 | weights_file=weights_file_l1, 51 | recons_img_file=None, 52 | corruption_level=0.096, 53 | batch_size=30, 54 | n_epochs=1400) 55 | 56 | dae_l1_obj_out = file("dae_l1_obj.pkl", "wb") 57 | pkl.dump(dae_l1, dae_l1_obj_out, protocol=pkl.HIGHEST_PROTOCOL) 58 | 59 | dae_l1_out = dae_l1.encode(input) 60 | dae_l1_h = dae_l1.encode(std_data) 61 | dae_l1_h_fn = theano.function([], dae_l1_h) 62 | dae_l2_in = dae_l1_h_fn() 63 | dae_l2_in = numpy.asarray(dae_l2_in, dtype=theano.config.floatX) 64 | 65 | dae_l2 = DenoisingAutoencoder(dae_l1_out, 66 | L1_reg=1e-4, 67 | L2_reg=6*1e-4, 68 | nvis=nhid_l1, 69 | nhid=nhid_l2, 70 | rnd=rnd) 71 | 72 | dae_l2.fit(learning_rate=0.95*1e-2, 73 | data=dae_l2_in, 74 | shuffle_data=True, 75 | recons_img_file=None, 76 | weights_file=weights_file_l2, 77 | corruption_level=0.1, 78 | batch_size=25, 79 | n_epochs=1400) 80 | 81 | dae_l2_obj_out = file("dae_l2_obj.pkl", "wb") 82 | pkl.dump(dae_l2, dae_l2_obj_out, protocol=pkl.HIGHEST_PROTOCOL) 83 | 84 | 85 | samples = dae_l2.sample(dae_l2_in[10], 0.1, 1000) 86 | numpy.save(open("data_samples_l2_dae.npy", "wb"), samples) 87 | 88 | -------------------------------------------------------------------------------- /tests/plot_cae_feats.py: -------------------------------------------------------------------------------- 1 | import PIL.Image 2 | import numpy 3 | from utils import tile_raster_images 4 | import pickle as pkl 5 | 6 | data = numpy.load("cae_mnist_weights_l1.npy") 7 | 8 | image = PIL.Image.fromarray(tile_raster_images( 9 | X=data.T, 10 | img_shape=(28, 28), tile_shape=(30, 30), 11 | tile_spacing=(1, 1))) 12 | image.save('mnist_filters_cae.png') 13 | -------------------------------------------------------------------------------- /tests/plot_dae_feats.py: -------------------------------------------------------------------------------- 1 | import PIL.Image 2 | import numpy 3 | from utils import tile_raster_images 4 | import pickle as pkl 5 | 6 | data = numpy.load("dae_mnist_weights_l1.npy") 7 | 8 | image = PIL.Image.fromarray(tile_raster_images( 9 | X=data.T, 10 | img_shape=(28, 28), tile_shape=(40, 20), 11 | tile_spacing=(1, 1))) 12 | image.save('mnist_filters_dae.png') 13 | -------------------------------------------------------------------------------- /tests/plot_mlp_feats.py: -------------------------------------------------------------------------------- 1 | import PIL.Image 2 | import numpy 3 | from utils import tile_raster_images 4 | import pickle as pkl 5 | 6 | data = numpy.load("/u/gulcehrc/Documents/Papers/lisa_lab/articles/2012/intermediate_targets/prmlp_1stlayer.npy") 7 | 8 | image = PIL.Image.fromarray(tile_raster_images( 9 | X=data.T, 10 | img_shape=(8, 8), tile_shape=(32, 64), 11 | tile_spacing=(1, 1))) 12 | image.save('pento_filters_ikgnn.png') 13 | -------------------------------------------------------------------------------- /tests/plot_samples_fn.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | import pylab 3 | import cPickle as pkl 4 | 5 | def colored_axis(axes, color="red"): 6 | axes.spines['bottom'].set_color(color) 7 | axes.spines['top'].set_color(color) 8 | axes.spines['right'].set_color(color) 9 | axes.spines['left'].set_color(color) 10 | axes.get_yaxis().set_ticks([]) 11 | axes.get_xaxis().set_ticks([]) 12 | 13 | def plot_samples(samples, no_of_rows, no_of_cols, pno=1, n_samples=100, plot_every=1, img_shp=(28, 14 | 28), axes=None): 15 | if pno == 0: 16 | axes = pylab.subplot(no_of_rows, no_of_cols, pno + 1, aspect='equal') 17 | colored_axis(axes, "red") 18 | pylab.imshow(samples[pno].reshape(img_shp)) 19 | plot_samples(samples, 20 | no_of_rows, 21 | no_of_cols, 22 | pno=pno + plot_every, 23 | n_samples=n_samples, 24 | plot_every=plot_every, 25 | img_shp=img_shp, 26 | axes=axes) 27 | 28 | if pno >= n_samples: 29 | colored_axis(axes, "black") 30 | return 0 31 | else: 32 | plot_no = pno / plot_every 33 | axes = pylab.subplot(no_of_rows, no_of_cols, plot_no + 1, aspect='equal') 34 | colored_axis(axes, "black") 35 | pylab.imshow(samples[pno].reshape(img_shp)) 36 | plot_samples(samples, 37 | no_of_rows, 38 | no_of_cols, 39 | pno=pno + plot_every, 40 | n_samples=n_samples, 41 | plot_every=plot_every, 42 | img_shp=img_shp, 43 | axes=axes) 44 | 45 | if __name__=="__main__": 46 | data = numpy.load("data_samples_l2_dae.npy") 47 | #data = pkl.load(open("samples_file.pkl")) 48 | import ipdb; ipdb.set_trace() 49 | pylab.gray() 50 | plot_samples(data, 5, 8, pno=0, n_samples=40, plot_every=1, img_shp=(30, 30)) 51 | pylab.show() 52 | -------------------------------------------------------------------------------- /tests/plot_smlp_feats.py: -------------------------------------------------------------------------------- 1 | import PIL.Image 2 | import numpy 3 | from utils import tile_raster_images 4 | import pickle as pkl 5 | 6 | #data = pkl.load(open("./online_smlp_lbfgs_std_adadelta.pkl")) 7 | 8 | data = numpy.load("weights.npy") 9 | 10 | import ipdb; ipdb.set_trace() 11 | 12 | image = PIL.Image.fromarray(tile_raster_images( 13 | X=data.T, 14 | img_shape=(8, 8), tile_shape=(32, 64), 15 | tile_spacing=(1, 1))) 16 | 17 | image.save('pento_filters_smlp.png') 18 | -------------------------------------------------------------------------------- /tests/run_cae_exp.py: -------------------------------------------------------------------------------- 1 | from ca import ContractiveAutoencoder 2 | #from dataset import Dataset 3 | 4 | import theano 5 | #import theano.tensor as T 6 | import numpy 7 | import cPickle as pkl 8 | 9 | cae_fhdlr = open("cae_data.pkl", 'rb') 10 | mnist_fhdlr = open("/data/lisa/data/mnist/mnist_all.pickle", 'rb') 11 | 12 | cae = pkl.load(cae_fhdlr) 13 | mnist = pkl.load(mnist_fhdlr) 14 | 15 | data = numpy.asarray(mnist[0][20].reshape(1, 784), dtype=theano.config.floatX) 16 | #data = numpy.asarray(numpy.random.rand(1, 784), dtype=theano.config.floatX) 17 | #samples = [] 18 | 19 | #samples.append(data) 20 | #last_sample = data 21 | #for i in xrange(99): 22 | #last_sample = cae.sample(last_sample, 0.009, 1000) 23 | # samples.append(last_sample) 24 | samples = cae.sample(data, 0.08, 10000) 25 | #samples.insert(0, data) 26 | numpy.save(open("data_samples.npy", 'wb'), samples) 27 | #samples = cae.sample(numpy.asarray(data, dtype=theano.config.floatX), sigma=0.3, n_steps=100) 28 | -------------------------------------------------------------------------------- /tests/run_cae_exp_simple_sample.py: -------------------------------------------------------------------------------- 1 | from ca import ContractiveAutoencoder 2 | #from dataset import Dataset 3 | 4 | import theano 5 | #import theano.tensor as T 6 | import numpy 7 | import cPickle as pkl 8 | 9 | cae_fhdlr = open("cae_data.pkl", 'rb') 10 | mnist_fhdlr = open("/data/lisa/data/mnist/mnist_all.pickle", 'rb') 11 | 12 | cae = pkl.load(cae_fhdlr) 13 | mnist = pkl.load(mnist_fhdlr) 14 | 15 | data = numpy.asarray(mnist[0][1200].reshape(1, 784), dtype=theano.config.floatX) 16 | #data = numpy.asarray(numpy.random.rand(1, 784), dtype=theano.config.floatX) 17 | #samples = [] 18 | 19 | #samples.append(data) 20 | #last_sample = data 21 | #for i in xrange(99): 22 | #last_sample = cae.sample(last_sample, 0.009, 1000) 23 | 24 | n_steps = 200 25 | samples = [] 26 | sample = data 27 | samples.append(data) 28 | 29 | for i in xrange(n_steps): 30 | encoded = cae.encode(sample) 31 | samplefn = cae.decode(encoded) 32 | sample = samplefn.eval() 33 | samples.append(sample) 34 | 35 | numpy.save(open("data_simple_samples.npy", 'wb'), samples) 36 | 37 | #samples = cae.sample(numpy.asarray(data, dtype=theano.config.floatX), sigma=0.3, n_steps=100) 38 | -------------------------------------------------------------------------------- /tests/test_ae.py: -------------------------------------------------------------------------------- 1 | from ae import Autoencoder 2 | from dataset import Dataset 3 | import theano.tensor as T 4 | import numpy 5 | 6 | def test_ae(): 7 | pass 8 | 9 | if __name__=="__main__": 10 | fname = "/data/lisa/data/mnist/mnist_all.pickle" 11 | ds = Dataset() 12 | ds.setup_dataset(data_path=fname, train_split_scale=0.8) 13 | x_data = ds.Xtrain 14 | input = T.dmatrix("x_input") 15 | rnd = numpy.random.RandomState(1231) 16 | ae = Autoencoder(input, nvis=28*28, nhid=500, rnd=rnd) 17 | ae.fit(data=x_data) 18 | -------------------------------------------------------------------------------- /tests/test_cae.py: -------------------------------------------------------------------------------- 1 | from ca import ContractiveAutoencoder 2 | from dataset import Dataset 3 | import theano 4 | import theano.tensor as T 5 | import numpy 6 | 7 | import cPickle as pkl 8 | theano.subtensor_merge_bug=False 9 | 10 | if __name__=="__main__": 11 | fname = "/data/lisa/data/mnist/mnist_all.pickle" 12 | print "Started training CA on data %s " % (fname) 13 | ds = Dataset() 14 | data = ds._get_data(fname) 15 | x_data = numpy.asarray(data[0][0:42000], dtype=theano.config.floatX) 16 | input = T.fmatrix("x_input") 17 | weights_file = "../out/cae_mnist_weights.npy" 18 | 19 | rnd = numpy.random.RandomState(1231) 20 | powerup = PowerupAutoencoder(input, nvis=28*28, nhid=512, num_pieces=10, rnd=rnd) 21 | 22 | powerup.fit(data=x_data, 23 | weights_file=weights_file, 24 | shuffle_data=True, 25 | learning_rate=0.08, 26 | contraction_level=1, 27 | n_epochs=140) 28 | 29 | pae_obj_out = open("pae_data.pkl", "wb") 30 | pkl.dump(cae, cae_obj_out) 31 | 32 | single_data = numpy.array(x_data[0]).reshape((1, 28*28)) 33 | 34 | # samples = [] 35 | # for i in xrange(100): 36 | # samples.append(cae.sample(single_data, 0.1, 120)) 37 | # numpy.save("data_samples.npy", samples) 38 | -------------------------------------------------------------------------------- /tests/test_dae.py: -------------------------------------------------------------------------------- 1 | from da import DenoisingAutoencoder 2 | from dataset import Dataset 3 | import theano.tensor as T 4 | import numpy 5 | 6 | if __name__=="__main__": 7 | fname = "/data/lisa/data/mnist/mnist_all.pickle" 8 | #fname = "/data/lisa/data/pentomino/" 9 | ds = Dataset() 10 | ds.setup_dataset(data_path=fname, train_split_scale=0.8) 11 | x_data = ds.Xtrain 12 | input = T.dmatrix("x_input") 13 | 14 | weights_file = "../out/dae_mnist_weights.npy" 15 | recons_file = "../out/dae_mnist_recons.npy" 16 | rnd = numpy.random.RandomState(1231) 17 | dae = DenoisingAutoencoder(input, nvis=28*28, nhid=600, rnd=rnd) 18 | dae.fit(learning_rate=0.1, data=x_data, weights_file=weights_file, n_epochs=100, recons_img_file=recons_file) 19 | -------------------------------------------------------------------------------- /tests/test_dae_pento.py: -------------------------------------------------------------------------------- 1 | from da import DenoisingAutoencoder 2 | from dataset import Dataset 3 | import theano.tensor as T 4 | import numpy 5 | 6 | if __name__=="__main__": 7 | fname = "/data/lisa/data/pentomino/pento64x64_40k_seed_5365102867_64patches.npy" 8 | ds = Dataset() 9 | ds.setup_dataset(data_path=fname, train_split_scale=0.4) 10 | x_data = ds.Xtrain 11 | input = T.dmatrix("x_input") 12 | rnd = numpy.random.RandomState(1231) 13 | dae = DenoisingAutoencoder(input, 14 | nvis=64*64, 15 | nhid=1500, 16 | rnd=rnd) 17 | 18 | dae.fit(data=x_data, 19 | learning_rate=0.04, 20 | n_epochs=32, 21 | weights_file="out/dae_weights_pento.npy") 22 | -------------------------------------------------------------------------------- /tests/test_pae.py: -------------------------------------------------------------------------------- 1 | from pae import PowerupAutoencoder 2 | from dataset import Dataset 3 | import theano 4 | import theano.tensor as T 5 | import numpy 6 | 7 | import cPickle as pkl 8 | theano.subtensor_merge_bug=False 9 | 10 | if __name__ == "__main__": 11 | 12 | fname = "/data/lisa/data/mnist/mnist.pkl" 13 | print "Started training PAE on data %s " % (fname) 14 | ds = Dataset() 15 | data = ds._get_data(fname) 16 | x_data = numpy.asarray(data[0][0], dtype=theano.config.floatX) 17 | input = T.fmatrix("x_input") 18 | weights_file = "../out/pae_mnist_weights.npy" 19 | 20 | rnd = numpy.random.RandomState(1231) 21 | 22 | powerup = PowerupAutoencoder(input, 23 | nvis=28*28, 24 | nhid=300, 25 | momentum=0.54, 26 | rho=0.94, 27 | num_pieces=10, 28 | #max_col_norm=1.9368, 29 | cost_type="MeanSquaredCost", 30 | L2_reg=1.2*1e-5, 31 | L1_reg=6.62*1e-6, 32 | L1_act_reg=1.8*1e-4, 33 | #p_decay=5.4*1e-4, 34 | tied_weights=False, 35 | rnd=rnd) 36 | lr_scalers = {} 37 | lr_scalers["W"] = 0.09 38 | lr_scalers["b"] = 0.09 39 | lr_scalers["power"] = 0.01 40 | bs=120 41 | 42 | powerup.fit(data=x_data, 43 | weights_file=weights_file, 44 | batch_size=bs, 45 | lr_scalers=lr_scalers, 46 | corruption_level=0.01, 47 | shuffle_data=True, 48 | learning_rate=0.54*1e-3, 49 | n_epochs=320) 50 | 51 | pae_obj_out = open("pae_data.pkl", "wb") 52 | pkl.dump(powerup, pae_obj_out) 53 | encoded = powerup.encode(input, bs) 54 | decoded = powerup.decode(encoded) 55 | rec_fn = theano.function([input], decoded) 56 | reconstructed = rec_fn(data[0][0][0:bs]) 57 | numpy.save("reconstructions.npy", reconstructed) 58 | -------------------------------------------------------------------------------- /tests/test_pae_tfd.py: -------------------------------------------------------------------------------- 1 | from pae import PowerupAutoencoder 2 | from dataset import Dataset 3 | import theano 4 | import theano.tensor as T 5 | import numpy 6 | 7 | from pylearn2.datasets.preprocessing import Standardize, LeCunLCN, GlobalContrastNormalization 8 | from pylearn2.datasets.tfd import TFD 9 | 10 | import cPickle as pkl 11 | theano.subtensor_merge_bug=False 12 | 13 | if __name__ == "__main__": 14 | weights_file = "../out/pae_mnist_enc_weights.npy" 15 | input = T.matrix("X", dtype=theano.config.floatX) 16 | tfd_ds = TFD("unlabeled") 17 | 18 | print "TFD shape: ", tfd_ds.X.shape 19 | gcn = GlobalContrastNormalization() 20 | standardizer = Standardize() 21 | lcn = LeCunLCN(img_shape=(48, 48), channels=[0]) 22 | gcn.apply(tfd_ds, can_fit=True) 23 | standardizer.apply(tfd_ds, can_fit=True) 24 | lcn.apply(tfd_ds) 25 | 26 | rnd = numpy.random.RandomState(1231) 27 | 28 | powerup = PowerupAutoencoder(input, 29 | nvis=48*48, 30 | nhid=500, 31 | momentum=0.66, 32 | rho=0.92, 33 | num_pieces=4, 34 | cost_type="MeanSquaredCost", 35 | L2_reg=8.2*1e-5, 36 | L1_reg=1.2 * 1e-5, 37 | L1_act_reg=8.8*1e-4, 38 | tied_weights=False, 39 | rnd=rnd) 40 | 41 | lr_scalers = {} 42 | lr_scalers["W"] = 0.035 43 | lr_scalers["b"] = 0.05 44 | lr_scalers["power"] = 0.01 45 | bs=60 46 | 47 | powerup.fit(data=tfd_ds.X, 48 | weights_file=weights_file, 49 | batch_size=bs, 50 | lr_scalers=lr_scalers, 51 | corruption_level=0.02, 52 | shuffle_data=True, 53 | learning_rate=0.34*1e-3, 54 | n_epochs=120) 55 | 56 | pae_obj_out = open("pae_data.pkl", "wb") 57 | pkl.dump(powerup, pae_obj_out) 58 | encoded = powerup.encode(input, bs) 59 | decoded = powerup.decode(encoded) 60 | rec_fn = theano.function([input], decoded) 61 | reconstructed = rec_fn(tfd_ds.X[0:bs]) 62 | 63 | numpy.save("reconstructions.npy", reconstructed) 64 | 65 | -------------------------------------------------------------------------------- /tests/test_sa.py: -------------------------------------------------------------------------------- 1 | from sa import SparseAutoencoder 2 | from dataset import Dataset 3 | import theano.tensor as T 4 | import numpy 5 | 6 | if __name__=="__main__": 7 | fname = "/data/lisa/data/mnist/mnist_all.pickle" 8 | print "Started training SA on data %s " % (fname) 9 | ds = Dataset() 10 | ds.setup_dataset(data_path=fname, train_split_scale=0.8) 11 | x_data = ds.Xtrain 12 | input = T.dmatrix("x_input") 13 | rnd = numpy.random.RandomState(1231) 14 | dae = SparseAutoencoder(input, nvis=28*28, nhid=2000, rnd=rnd) 15 | dae.fit(data=x_data) 16 | -------------------------------------------------------------------------------- /tests/utils.py: -------------------------------------------------------------------------------- 1 | """ This file contains different utility functions that are not connected 2 | in anyway to the networks presented in the tutorials, but rather help in 3 | processing the outputs into a more understandable way. 4 | 5 | For example ``tile_raster_images`` helps in generating a easy to grasp 6 | image from a set of samples or weights. 7 | """ 8 | 9 | 10 | import numpy 11 | 12 | 13 | def scale_to_unit_interval(ndar, eps=1e-8): 14 | """ Scales all values in the ndarray ndar to be between 0 and 1 """ 15 | ndar = ndar.copy() 16 | ndar -= ndar.min() 17 | ndar *= 1.0 / (ndar.max() + eps) 18 | return ndar 19 | 20 | 21 | def tile_raster_images(X, img_shape, tile_shape, tile_spacing=(0, 0), 22 | scale_rows_to_unit_interval=True, 23 | output_pixel_vals=True): 24 | """ 25 | Transform an array with one flattened image per row, into an array in 26 | which images are reshaped and layed out like tiles on a floor. 27 | 28 | This function is useful for visualizing datasets whose rows are images, 29 | and also columns of matrices for transforming those rows 30 | (such as the first layer of a neural net). 31 | 32 | :type X: a 2-D ndarray or a tuple of 4 channels, elements of which can 33 | be 2-D ndarrays or None; 34 | :param X: a 2-D array in which every row is a flattened image. 35 | 36 | :type img_shape: tuple; (height, width) 37 | :param img_shape: the original shape of each image 38 | 39 | :type tile_shape: tuple; (rows, cols) 40 | :param tile_shape: the number of images to tile (rows, cols) 41 | 42 | :param output_pixel_vals: if output should be pixel values (i.e. int8 43 | values) or floats 44 | 45 | :param scale_rows_to_unit_interval: if the values need to be scaled before 46 | being plotted to [0,1] or not 47 | 48 | 49 | :returns: array suitable for viewing as an image. 50 | (See:`PIL.Image.fromarray`.) 51 | :rtype: a 2-d array with same dtype as X. 52 | 53 | """ 54 | 55 | assert len(img_shape) == 2 56 | assert len(tile_shape) == 2 57 | assert len(tile_spacing) == 2 58 | 59 | # The expression below can be re-written in a more C style as 60 | # follows : 61 | # 62 | # out_shape = [0,0] 63 | # out_shape[0] = (img_shape[0]+tile_spacing[0])*tile_shape[0] - 64 | # tile_spacing[0] 65 | # out_shape[1] = (img_shape[1]+tile_spacing[1])*tile_shape[1] - 66 | # tile_spacing[1] 67 | out_shape = [(ishp + tsp) * tshp - tsp for ishp, tshp, tsp 68 | in zip(img_shape, tile_shape, tile_spacing)] 69 | 70 | if isinstance(X, tuple): 71 | assert len(X) == 4 72 | # Create an output numpy ndarray to store the image 73 | if output_pixel_vals: 74 | out_array = numpy.zeros((out_shape[0], out_shape[1], 4), 75 | dtype='uint8') 76 | else: 77 | out_array = numpy.zeros((out_shape[0], out_shape[1], 4), 78 | dtype=X.dtype) 79 | 80 | #colors default to 0, alpha defaults to 1 (opaque) 81 | if output_pixel_vals: 82 | channel_defaults = [0, 0, 0, 255] 83 | else: 84 | channel_defaults = [0., 0., 0., 1.] 85 | 86 | for i in xrange(4): 87 | if X[i] is None: 88 | # if channel is None, fill it with zeros of the correct 89 | # dtype 90 | dt = out_array.dtype 91 | if output_pixel_vals: 92 | dt = 'uint8' 93 | out_array[:, :, i] = numpy.zeros(out_shape, 94 | dtype=dt) + channel_defaults[i] 95 | else: 96 | # use a recurrent call to compute the channel and store it 97 | # in the output 98 | out_array[:, :, i] = tile_raster_images( 99 | X[i], img_shape, tile_shape, tile_spacing, 100 | scale_rows_to_unit_interval, output_pixel_vals) 101 | return out_array 102 | 103 | else: 104 | # if we are dealing with only one channel 105 | H, W = img_shape 106 | Hs, Ws = tile_spacing 107 | 108 | # generate a matrix to store the output 109 | dt = X.dtype 110 | if output_pixel_vals: 111 | dt = 'uint8' 112 | out_array = numpy.zeros(out_shape, dtype=dt) 113 | 114 | for tile_row in xrange(tile_shape[0]): 115 | for tile_col in xrange(tile_shape[1]): 116 | if tile_row * tile_shape[1] + tile_col < X.shape[0]: 117 | this_x = X[tile_row * tile_shape[1] + tile_col] 118 | if scale_rows_to_unit_interval: 119 | # if we should scale values to be between 0 and 1 120 | # do this by calling the `scale_to_unit_interval` 121 | # function 122 | this_img = scale_to_unit_interval( 123 | this_x.reshape(img_shape)) 124 | else: 125 | this_img = this_x.reshape(img_shape) 126 | # add the slice to the corresponding position in the 127 | # output array 128 | c = 1 129 | if output_pixel_vals: 130 | c = 255 131 | out_array[ 132 | tile_row * (H + Hs): tile_row * (H + Hs) + H, 133 | tile_col * (W + Ws): tile_col * (W + Ws) + W 134 | ] = this_img * c 135 | return out_array 136 | 137 | --------------------------------------------------------------------------------