├── .gitignore ├── README.md ├── data └── .gitignore ├── deeplearningnet_utils.py ├── definitions ├── .gitignore ├── __init__.py ├── conv_100_5_2-mpool_2-conv_100_5_2-mpool_2-dense_3000-rg-nd-eu-tanh.py ├── conv_100_5_2-mpool_2-dense_3000-nd-eu-tanh.py ├── conv_100_5_2-mpool_2-dense_3000-nd-tanh.py ├── conv_100_5_2-mpool_2-dense_3000-rg-ec.py ├── conv_100_5_2-mpool_2-dense_3000-rg-nd-eu-tanh-purple.py ├── conv_100_5_2-mpool_2-dense_3000-rg-nd-eu-tanh.py ├── conv_100_5_2-mpool_2-dense_3000-rg-nd-eu.py ├── conv_100_5_2-mpool_2-dense_3000-rg-nd.py ├── conv_100_5_2-mpool_2-dense_3000-rg-wd0.001.py ├── conv_100_5_2-mpool_2-dense_3000-rg-wd0.01.py ├── conv_100_5_2-mpool_2-dense_3000-rg-wd0.1.py ├── conv_100_5_2-mpool_2-dense_3000-rg.py ├── conv_100_5_2-mpool_2-dense_3000_dr0.3-nd-eu-tanh.py ├── conv_100_5_2-mpool_2-dense_3000_dr0.5-nd-eu-tanh.py ├── conv_30_5_2-mpool_2-dense_3000-ec.py ├── conv_50_5_2-mpool_2-conv_50_5_2-mpool_2-dense_1000-rg-nd-eu-tanh.py ├── conv_50_5_2-mpool_2-dense_500-rg-ec.py ├── conv_50_5_2-mpool_2-dense_500-rg.py └── flags ├── load_data.py ├── logs └── .gitignore ├── models └── .gitignore ├── our_layers.py ├── our_utils.py ├── plots ├── .gitignore └── example.png └── train_convae.py /.gitignore: -------------------------------------------------------------------------------- 1 | tmp/ 2 | notes 3 | ae_cifar_simple_example.py 4 | *.pyc 5 | todo 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # neural-colorizer 2 | Convolutional autoencoder to colorize greyscale images 3 | 4 | We will write detailed description soon. 5 | 6 | Here is the result of one layer convolutional autoencoder on Cifar-10 dataset(32x32 images). 7 | (original, greyscale, colorized) 8 | 9 | ![alt tag](https://raw.github.com/YerevaNN/neural-colorizer/master/plots/example.png) -------------------------------------------------------------------------------- /data/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore -------------------------------------------------------------------------------- /deeplearningnet_utils.py: -------------------------------------------------------------------------------- 1 | """ This file contains different utility functions that are not connected 2 | in anyway to the networks presented in the tutorials, but rather help in 3 | processing the outputs into a more understandable way. 4 | 5 | For example ``tile_raster_images`` helps in generating a easy to grasp 6 | image from a set of samples or weights. 7 | """ 8 | 9 | import numpy 10 | 11 | def scale_to_unit_interval(ndar, eps=1e-8): 12 | """ Scales all values in the ndarray ndar to be between 0 and 1 """ 13 | ndar = ndar.copy() 14 | ndar -= ndar.min() 15 | ndar *= 1.0 / (ndar.max() + eps) 16 | return ndar 17 | 18 | def tile_raster_images(X, img_shape, tile_shape, tile_spacing=(0, 0), 19 | scale_rows_to_unit_interval=True, 20 | output_pixel_vals=True): 21 | """ 22 | Transform an array with one flattened image per row, into an array in 23 | which images are reshaped and layed out like tiles on a floor. 24 | 25 | This function is useful for visualizing datasets whose rows are images, 26 | and also columns of matrices for transforming those rows 27 | (such as the first layer of a neural net). 28 | 29 | :type X: a 2-D ndarray or a tuple of 4 channels, elements of which can 30 | be 2-D ndarrays or None; 31 | :param X: a 2-D array in which every row is a flattened image. 32 | 33 | :type img_shape: tuple; (height, width) 34 | :param img_shape: the original shape of each image 35 | 36 | :type tile_shape: tuple; (rows, cols) 37 | :param tile_shape: the number of images to tile (rows, cols) 38 | 39 | :param output_pixel_vals: if output should be pixel values (i.e. int8 40 | values) or floats 41 | 42 | :param scale_rows_to_unit_interval: if the values need to be scaled before 43 | being plotted to [0,1] or not 44 | 45 | 46 | :returns: array suitable for viewing as an image. 47 | (See:`Image.fromarray`.) 48 | :rtype: a 2-d array with same dtype as X. 49 | 50 | """ 51 | 52 | assert len(img_shape) == 2 53 | assert len(tile_shape) == 2 54 | assert len(tile_spacing) == 2 55 | 56 | # The expression below can be re-written in a more C style as 57 | # follows : 58 | # 59 | # out_shape = [0,0] 60 | # out_shape[0] = (img_shape[0]+tile_spacing[0])*tile_shape[0] - 61 | # tile_spacing[0] 62 | # out_shape[1] = (img_shape[1]+tile_spacing[1])*tile_shape[1] - 63 | # tile_spacing[1] 64 | out_shape = [ 65 | (ishp + tsp) * tshp - tsp 66 | for ishp, tshp, tsp in zip(img_shape, tile_shape, tile_spacing) 67 | ] 68 | 69 | if isinstance(X, tuple): 70 | assert len(X) == 4 71 | # Create an output numpy ndarray to store the image 72 | if output_pixel_vals: 73 | out_array = numpy.zeros((out_shape[0], out_shape[1], 4), 74 | dtype='uint8') 75 | else: 76 | out_array = numpy.zeros((out_shape[0], out_shape[1], 4), 77 | dtype=X.dtype) 78 | 79 | #colors default to 0, alpha defaults to 1 (opaque) 80 | if output_pixel_vals: 81 | channel_defaults = [0, 0, 0, 255] 82 | else: 83 | channel_defaults = [0., 0., 0., 1.] 84 | 85 | for i in xrange(4): 86 | if X[i] is None: 87 | # if channel is None, fill it with zeros of the correct 88 | # dtype 89 | dt = out_array.dtype 90 | if output_pixel_vals: 91 | dt = 'uint8' 92 | out_array[:, :, i] = numpy.zeros( 93 | out_shape, 94 | dtype=dt 95 | ) + channel_defaults[i] 96 | else: 97 | # use a recurrent call to compute the channel and store it 98 | # in the output 99 | out_array[:, :, i] = tile_raster_images( 100 | X[i], img_shape, tile_shape, tile_spacing, 101 | scale_rows_to_unit_interval, output_pixel_vals) 102 | return out_array 103 | 104 | else: 105 | # if we are dealing with only one channel 106 | H, W = img_shape 107 | Hs, Ws = tile_spacing 108 | 109 | # generate a matrix to store the output 110 | dt = X.dtype 111 | if output_pixel_vals: 112 | dt = 'uint8' 113 | out_array = numpy.zeros(out_shape, dtype=dt) 114 | 115 | for tile_row in xrange(tile_shape[0]): 116 | for tile_col in xrange(tile_shape[1]): 117 | if tile_row * tile_shape[1] + tile_col < X.shape[0]: 118 | this_x = X[tile_row * tile_shape[1] + tile_col] 119 | if scale_rows_to_unit_interval: 120 | # if we should scale values to be between 0 and 1 121 | # do this by calling the `scale_to_unit_interval` 122 | # function 123 | this_img = scale_to_unit_interval( 124 | this_x.reshape(img_shape)) 125 | else: 126 | this_img = this_x.reshape(img_shape) 127 | # add the slice to the corresponding position in the 128 | # output array 129 | c = 1 130 | if output_pixel_vals: 131 | c = 255 132 | out_array[ 133 | tile_row * (H + Hs): tile_row * (H + Hs) + H, 134 | tile_col * (W + Ws): tile_col * (W + Ws) + W 135 | ] = this_img * c 136 | return out_array -------------------------------------------------------------------------------- /definitions/.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | -------------------------------------------------------------------------------- /definitions/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YerevaNN/neural-colorizer/1e3299b2f2b25f3bfb37bf642d78d9e0fdf5476b/definitions/__init__.py -------------------------------------------------------------------------------- /definitions/conv_100_5_2-mpool_2-conv_100_5_2-mpool_2-dense_3000-rg-nd-eu-tanh.py: -------------------------------------------------------------------------------- 1 | import theano.tensor as T 2 | import lasagne 3 | import lasagne.layers as layers 4 | import our_layers 5 | 6 | 7 | def define_model(input_var, **kwargs): 8 | """ Defines the model and returns (network, validation network output) 9 | 10 | -Return layers.get_output(final_layer_name) if validation network output and 11 | train network output are the same 12 | 13 | -For example, return layers.get_output(final_layer_name, deterministic = true) 14 | if there is a dropout layer 15 | 16 | -Use **kwargs to pass model specific parameters 17 | """ 18 | 19 | conv1_filter_count = 100 20 | conv1_filter_size = 5 21 | pool1_size = 2 22 | 23 | conv2_filter_count = 100 24 | conv2_filter_size = 5 25 | pool2_size = 2 26 | 27 | n_dense_units = 3000 28 | 29 | batch_size = input_var.shape[0] 30 | image_size = 32 31 | # note: ignore_border is True by defualt, so rounding down 32 | after_pool1 = image_size // pool1_size 33 | after_pool2 = after_pool1 // pool2_size 34 | 35 | input = layers.InputLayer( 36 | shape = (None, 3, image_size, image_size), 37 | input_var = input_var 38 | ) 39 | 40 | greyscale_input = our_layers.GreyscaleLayer( 41 | incoming = input, 42 | random_greyscale = True, 43 | ) 44 | 45 | conv1 = layers.Conv2DLayer( 46 | incoming = greyscale_input, 47 | num_filters = conv1_filter_count, 48 | filter_size = conv1_filter_size, 49 | stride = 1, 50 | pad = 'same', 51 | nonlinearity = lasagne.nonlinearities.tanh, 52 | ) 53 | 54 | pool1 = layers.MaxPool2DLayer( 55 | incoming = conv1, 56 | pool_size = pool1_size, 57 | stride = pool1_size, 58 | ) 59 | 60 | conv2 = layers.Conv2DLayer( 61 | incoming = pool1, 62 | num_filters = conv2_filter_count, 63 | filter_size = conv2_filter_size, 64 | stride = 1, 65 | pad = 'same', 66 | nonlinearity = lasagne.nonlinearities.tanh, 67 | ) 68 | 69 | pool2 = layers.MaxPool2DLayer( 70 | incoming = conv2, 71 | pool_size = pool2_size, 72 | stride = pool2_size, 73 | ) 74 | 75 | dense1 = layers.DenseLayer( 76 | incoming = pool2, 77 | num_units = n_dense_units, 78 | nonlinearity = lasagne.nonlinearities.tanh, 79 | ) 80 | 81 | pre_unpool2 = layers.DenseLayer( 82 | incoming = dense1, 83 | num_units = conv2_filter_count * (after_pool2 ** 2), 84 | nonlinearity = lasagne.nonlinearities.tanh, 85 | ) 86 | 87 | pre_unpool2 = layers.ReshapeLayer( 88 | incoming = pre_unpool2, 89 | shape = (batch_size, conv2_filter_count) + (after_pool2, after_pool2), 90 | ) 91 | 92 | unpool2 = our_layers.Unpool2DLayer( 93 | incoming = pre_unpool2, 94 | kernel_size = pool2_size, 95 | ) 96 | 97 | deconv2 = layers.Conv2DLayer( 98 | incoming = unpool2, 99 | num_filters = conv1_filter_count, 100 | filter_size = conv2_filter_size, 101 | stride = 1, 102 | pad = 'same', 103 | nonlinearity = lasagne.nonlinearities.tanh, 104 | ) 105 | 106 | unpool1 = our_layers.Unpool2DLayer( 107 | incoming = deconv2, 108 | kernel_size = pool1_size, 109 | ) 110 | 111 | deconv1 = layers.Conv2DLayer( 112 | incoming = unpool1, 113 | num_filters = 3, 114 | filter_size = conv1_filter_size, 115 | stride = 1, 116 | pad = 'same', 117 | nonlinearity = lasagne.nonlinearities.tanh, 118 | ) 119 | 120 | output = layers.ReshapeLayer( 121 | incoming = deconv1, 122 | shape = input_var.shape 123 | ) 124 | 125 | return (output, layers.get_output(output)) 126 | 127 | 128 | def get_cost_updates(network, input_var, output, learning_rate, **kwargs): 129 | """ Defines and returns cost and updates of the network 130 | 131 | -output can be different from layers.get_output(network), because train 132 | and validation networks may differ from each other 133 | 134 | -Use **kwargs to pass model specific parameters 135 | """ 136 | 137 | params = layers.get_all_params(network, trainable = True) 138 | 139 | batch_size = input_var.shape[0] 140 | flat_input = input_var.reshape((batch_size, 3072)) 141 | flat_output= output.reshape((batch_size, 3072)) 142 | 143 | # scale flat_output to [0, 1] 144 | flat_output = (flat_output + 1) / 2 145 | 146 | # cross entrophy loss 147 | #losses = -T.sum(flat_input * T.log(flat_output) + (1 - flat_input) * T.log(1 - flat_output), axis = 1) 148 | 149 | # euclidean loss 150 | losses = T.sum((flat_input - flat_output) ** 2, axis = 1) 151 | 152 | # add saturation loss 153 | #saturation = -T.sum(T.std(T.reshape(flat_output, (batch_size, 3, 1024)), axis = 1), axis = 1) 154 | #losses = losses + 0.2 * saturation 155 | 156 | cost = T.mean(losses) 157 | gradients = T.grad(cost, params) 158 | 159 | # stochastic gradient descent 160 | #updates = [ 161 | # (param, param - learning_rate * gradient) 162 | # for param, gradient in zip(params, gradients) 163 | #] 164 | 165 | # rmsprop 166 | #updates = lasagne.updates.rmsprop(gradients, params, learning_rate = learning_rate) 167 | 168 | # momentum 169 | updates = lasagne.updates.nesterov_momentum(gradients, params, learning_rate = learning_rate) 170 | 171 | return (cost, updates) 172 | -------------------------------------------------------------------------------- /definitions/conv_100_5_2-mpool_2-dense_3000-nd-eu-tanh.py: -------------------------------------------------------------------------------- 1 | import theano.tensor as T 2 | import lasagne 3 | import lasagne.layers as layers 4 | import our_layers 5 | 6 | 7 | def define_model(input_var, **kwargs): 8 | """ Defines the model and returns (network, validation network output) 9 | 10 | -Return layers.get_output(final_layer_name) if validation network output and 11 | train network output are the same 12 | 13 | -For example, return layers.get_output(final_layer_name, deterministic = true) 14 | if there is a dropout layer 15 | 16 | -Use **kwargs to pass model specific parameters 17 | """ 18 | 19 | conv1_filter_count = 100 20 | conv1_filter_size = 5 21 | pool1_size = 2 22 | 23 | n_dense_units = 3000 24 | 25 | batch_size = input_var.shape[0] 26 | image_size = 32 27 | after_conv1 = image_size 28 | after_pool1 = (after_conv1 + pool1_size - 1) // pool1_size 29 | 30 | input = layers.InputLayer( 31 | shape = (None, 3, image_size, image_size), 32 | input_var = input_var 33 | ) 34 | 35 | greyscale_input = our_layers.GreyscaleLayer( 36 | incoming = input, 37 | random_greyscale = False, 38 | ) 39 | 40 | conv1 = layers.Conv2DLayer( 41 | incoming = greyscale_input, 42 | num_filters = conv1_filter_count, 43 | filter_size = conv1_filter_size, 44 | stride = 1, 45 | pad = 'same', 46 | nonlinearity = lasagne.nonlinearities.tanh, 47 | ) 48 | 49 | pool1 = layers.MaxPool2DLayer( 50 | incoming = conv1, 51 | pool_size = pool1_size, 52 | stride = pool1_size, 53 | ) 54 | 55 | dense1 = layers.DenseLayer( 56 | incoming = pool1, 57 | num_units = n_dense_units, 58 | nonlinearity = lasagne.nonlinearities.tanh, 59 | ) 60 | 61 | pre_unpool1 = layers.DenseLayer( 62 | incoming = dense1, 63 | num_units = conv1_filter_count * (after_pool1 ** 2), 64 | nonlinearity = lasagne.nonlinearities.tanh, 65 | ) 66 | 67 | pre_unpool1 = layers.ReshapeLayer( 68 | incoming = pre_unpool1, 69 | shape = (batch_size, conv1_filter_count) + (after_pool1, after_pool1), 70 | ) 71 | 72 | unpool1 = our_layers.Unpool2DLayer( 73 | incoming = pre_unpool1, 74 | kernel_size = pool1_size, 75 | ) 76 | 77 | deconv1 = layers.Conv2DLayer( 78 | incoming = unpool1, 79 | num_filters = 3, 80 | filter_size = conv1_filter_size, 81 | stride = 1, 82 | pad = 'same', 83 | nonlinearity = lasagne.nonlinearities.tanh, 84 | ) 85 | 86 | output = layers.ReshapeLayer( 87 | incoming = deconv1, 88 | shape = input_var.shape, 89 | ) 90 | 91 | return (output, layers.get_output(output)) 92 | 93 | 94 | def get_cost_updates(network, input_var, output, learning_rate, **kwargs): 95 | """ Defines and returns cost and updates of the network 96 | 97 | -output can be different from layers.get_output(network), because train 98 | and validation networks may differ from each other 99 | 100 | -Use **kwargs to pass model specific parameters 101 | """ 102 | 103 | params = layers.get_all_params(network, trainable = True) 104 | 105 | batch_size = input_var.shape[0] 106 | flat_input = input_var.reshape((batch_size, 3072)) 107 | flat_output= output.reshape((batch_size, 3072)) 108 | 109 | # scale flat_output to [0, 1] 110 | flat_output = (flat_output + 1) / 2 111 | 112 | # cross entrophy loss 113 | #losses = -T.sum(flat_input * T.log(flat_output) + (1 - flat_input) * T.log(1 - flat_output), axis = 1) 114 | 115 | # euclidean loss 116 | losses = T.sum((flat_input - flat_output) ** 2, axis = 1) 117 | 118 | # add saturation loss 119 | #saturation = -T.sum(T.std(T.reshape(flat_output, (batch_size, 3, 1024)), axis = 1), axis = 1) 120 | #losses = losses + 0.2 * saturation 121 | 122 | cost = T.mean(losses) 123 | gradients = T.grad(cost, params) 124 | 125 | # stochastic gradient descent 126 | #updates = [ 127 | # (param, param - learning_rate * gradient) 128 | # for param, gradient in zip(params, gradients) 129 | #] 130 | 131 | # rmsprop 132 | #updates = lasagne.updates.rmsprop(gradients, params, learning_rate = learning_rate) 133 | 134 | # momentum 135 | updates = lasagne.updates.nesterov_momentum(gradients, params, learning_rate = learning_rate) 136 | 137 | return (cost, updates) 138 | -------------------------------------------------------------------------------- /definitions/conv_100_5_2-mpool_2-dense_3000-nd-tanh.py: -------------------------------------------------------------------------------- 1 | import theano.tensor as T 2 | import lasagne 3 | import lasagne.layers as layers 4 | import our_layers 5 | 6 | 7 | def define_model(input_var, **kwargs): 8 | """ Defines the model and returns (network, validation network output) 9 | 10 | -Return layers.get_output(final_layer_name) if validation network output and 11 | train network output are the same 12 | 13 | -For example, return layers.get_output(final_layer_name, deterministic = true) 14 | if there is a dropout layer 15 | 16 | -Use **kwargs to pass model specific parameters 17 | """ 18 | 19 | conv1_filter_count = 100 20 | conv1_filter_size = 5 21 | pool1_size = 2 22 | 23 | n_dense_units = 3000 24 | 25 | batch_size = input_var.shape[0] 26 | image_size = 32 27 | after_conv1 = image_size 28 | after_pool1 = (after_conv1 + pool1_size - 1) // pool1_size 29 | 30 | input = layers.InputLayer( 31 | shape = (None, 3, image_size, image_size), 32 | input_var = input_var 33 | ) 34 | 35 | greyscale_input = our_layers.GreyscaleLayer( 36 | incoming = input, 37 | random_greyscale = False, 38 | ) 39 | 40 | conv1 = layers.Conv2DLayer( 41 | incoming = greyscale_input, 42 | num_filters = conv1_filter_count, 43 | filter_size = conv1_filter_size, 44 | stride = 1, 45 | pad = 'same', 46 | nonlinearity = lasagne.nonlinearities.tanh, 47 | ) 48 | 49 | pool1 = layers.MaxPool2DLayer( 50 | incoming = conv1, 51 | pool_size = pool1_size, 52 | stride = pool1_size, 53 | ) 54 | 55 | dense1 = layers.DenseLayer( 56 | incoming = pool1, 57 | num_units = n_dense_units, 58 | nonlinearity = lasagne.nonlinearities.tanh, 59 | ) 60 | 61 | pre_unpool1 = layers.DenseLayer( 62 | incoming = dense1, 63 | num_units = conv1_filter_count * (after_pool1 ** 2), 64 | nonlinearity = lasagne.nonlinearities.tanh, 65 | ) 66 | 67 | pre_unpool1 = layers.ReshapeLayer( 68 | incoming = pre_unpool1, 69 | shape = (batch_size, conv1_filter_count) + (after_pool1, after_pool1), 70 | ) 71 | 72 | unpool1 = our_layers.Unpool2DLayer( 73 | incoming = pre_unpool1, 74 | kernel_size = pool1_size, 75 | ) 76 | 77 | deconv1 = layers.Conv2DLayer( 78 | incoming = unpool1, 79 | num_filters = 3, 80 | filter_size = conv1_filter_size, 81 | stride = 1, 82 | pad = 'same', 83 | nonlinearity = lasagne.nonlinearities.tanh, 84 | ) 85 | 86 | output = layers.ReshapeLayer( 87 | incoming = deconv1, 88 | shape = input_var.shape 89 | ) 90 | 91 | return (output, layers.get_output(output)) 92 | 93 | 94 | def get_cost_updates(network, input_var, output, learning_rate, **kwargs): 95 | """ Defines and returns cost and updates of the network 96 | 97 | -output can be different from layers.get_output(network), because train 98 | and validation networks may differ from each other 99 | 100 | -Use **kwargs to pass model specific parameters 101 | """ 102 | 103 | params = layers.get_all_params(network, trainable = True) 104 | 105 | batch_size = input_var.shape[0] 106 | flat_input = input_var.reshape((batch_size, 3072)) 107 | flat_output= output.reshape((batch_size, 3072)) 108 | 109 | # scale flat_output to [0, 1] 110 | flat_output = (flat_output + 1) / 2 111 | flat_input = T.clip(flat_input, 0.01, 0.99) 112 | flat_output = T.clip(flat_output, 0.01, 0.99) 113 | 114 | # cross entrophy loss 115 | losses = -T.sum(flat_input * T.log(flat_output) + (1 - flat_input) * T.log(1 - flat_output), axis = 1) 116 | 117 | # euclidean loss 118 | #losses = T.sum((flat_input - flat_output) ** 2, axis = 1) 119 | 120 | # add saturation loss 121 | #saturation = -T.sum(T.std(T.reshape(flat_output, (batch_size, 3, 1024)), axis = 1), axis = 1) 122 | #losses = losses + 0.2 * saturation 123 | 124 | cost = T.mean(losses) 125 | gradients = T.grad(cost, params) 126 | 127 | # stochastic gradient descent 128 | #updates = [ 129 | # (param, param - learning_rate * gradient) 130 | # for param, gradient in zip(params, gradients) 131 | #] 132 | 133 | # rmsprop 134 | #updates = lasagne.updates.rmsprop(gradients, params, learning_rate = learning_rate) 135 | 136 | # momentum 137 | updates = lasagne.updates.nesterov_momentum(gradients, params, learning_rate = learning_rate) 138 | 139 | return (cost, updates) 140 | -------------------------------------------------------------------------------- /definitions/conv_100_5_2-mpool_2-dense_3000-rg-ec.py: -------------------------------------------------------------------------------- 1 | import theano.tensor as T 2 | import lasagne 3 | import lasagne.layers as layers 4 | import our_layers 5 | 6 | 7 | def define_model(input_var, **kwargs): 8 | """ Defines the model and returns (network, validation network output) 9 | 10 | -Return layers.get_output(final_layer_name) if validation network output and 11 | train network output are the same 12 | 13 | -For example, return layers.get_output(final_layer_name, deterministic = true) 14 | if there is a dropout layer 15 | 16 | -Use **kwargs to pass model specific parameters 17 | """ 18 | 19 | image_size = 32 20 | conv_filter_count = 100 21 | conv_filter_size = 5 22 | pool_size = 2 23 | n_dense_units = 3000 24 | 25 | input = layers.InputLayer( 26 | shape = (None, 3, image_size, image_size), 27 | input_var = input_var 28 | ) 29 | 30 | greyscale_input = our_layers.GreyscaleLayer( 31 | incoming = input, 32 | random_greyscale = True, 33 | ) 34 | 35 | conv1 = layers.Conv2DLayer( 36 | incoming = greyscale_input, 37 | num_filters = conv_filter_count, 38 | filter_size = conv_filter_size, 39 | stride = 1, 40 | nonlinearity = lasagne.nonlinearities.sigmoid, 41 | ) 42 | 43 | pool1 = layers.MaxPool2DLayer( 44 | incoming = conv1, 45 | pool_size = pool_size, 46 | stride = pool_size, 47 | ) 48 | 49 | dense1 = layers.DenseLayer( 50 | incoming = pool1, 51 | num_units = n_dense_units, 52 | nonlinearity = lasagne.nonlinearities.rectify, 53 | ) 54 | 55 | pre_unpool1 = layers.DenseLayer( 56 | incoming = dense1, 57 | num_units = conv_filter_count * (image_size + conv_filter_size - 1) ** 2 / (pool_size * pool_size), 58 | nonlinearity = lasagne.nonlinearities.linear, 59 | ) 60 | 61 | pre_unpool1 = layers.ReshapeLayer( 62 | incoming = pre_unpool1, 63 | shape = (input_var.shape[0], conv_filter_count) + ((image_size + conv_filter_size - 1) / 2, (image_size + conv_filter_size - 1) / 2), 64 | ) 65 | 66 | unpool1 = our_layers.Unpool2DLayer( 67 | incoming = pre_unpool1, 68 | kernel_size = 2, 69 | ) 70 | 71 | deconv1 = layers.Conv2DLayer( 72 | incoming = unpool1, 73 | num_filters = 3, 74 | filter_size = conv_filter_size, 75 | stride = 1, 76 | nonlinearity = lasagne.nonlinearities.sigmoid, 77 | ) 78 | 79 | deconv1_flatten = layers.ReshapeLayer( 80 | incoming = deconv1, 81 | shape = (input_var.shape[0], 1, 1, 3 * image_size * image_size), 82 | ) 83 | 84 | grey_with_deconv1_flatten = layers.ConcatLayer( 85 | incomings = [deconv1_flatten, greyscale_input], 86 | axis = 1, 87 | ) 88 | 89 | output = layers.DenseLayer( 90 | incoming = grey_with_deconv1_flatten, 91 | num_units = 3 * image_size * image_size, 92 | nonlinearity = lasagne.nonlinearities.linear, 93 | ) 94 | 95 | output = layers.ReshapeLayer( 96 | incoming = deconv1, 97 | shape = input_var.shape 98 | ) 99 | 100 | return (output, layers.get_output(output)) 101 | 102 | 103 | def get_cost_updates(network, input_var, output, learning_rate, **kwargs): 104 | """ Defines and returns cost and updates of the network 105 | 106 | -output can be different from layers.get_output(network), because train 107 | and validation networks may differ from each other 108 | 109 | -Use **kwargs to pass model specific parameters 110 | """ 111 | 112 | params = layers.get_all_params(network, trainable = True) 113 | 114 | batch_size = input_var.shape[0] 115 | flat_input = input_var.reshape((batch_size, 3072)) 116 | flat_output= output.reshape((batch_size, 3072)) 117 | 118 | # cross entrophy loss 119 | losses = -T.sum(flat_input * T.log(flat_output) + (1 - flat_input) * T.log(1 - flat_output), axis = 1) 120 | 121 | # add saturation loss 122 | #saturation = -T.sum(T.std(T.reshape(flat_output, (batch_size, 3, 1024)), axis = 1), axis = 1) 123 | #losses = losses + 0.2 * saturation 124 | 125 | cost = T.mean(losses) 126 | gradients = T.grad(cost, params) 127 | 128 | # stochastic gradient descent 129 | #updates = [ 130 | # (param, param - learning_rate * gradient) 131 | # for param, gradient in zip(params, gradients) 132 | #] 133 | 134 | # rmsprop 135 | #updates = lasagne.updates.rmsprop(gradients, params, learning_rate = learning_rate) 136 | 137 | # momentum 138 | updates = lasagne.updates.nesterov_momentum(gradients, params, learning_rate = learning_rate) 139 | 140 | return (cost, updates) 141 | -------------------------------------------------------------------------------- /definitions/conv_100_5_2-mpool_2-dense_3000-rg-nd-eu-tanh-purple.py: -------------------------------------------------------------------------------- 1 | import theano.tensor as T 2 | import lasagne 3 | import lasagne.layers as layers 4 | import our_layers 5 | 6 | 7 | def define_model(input_var, **kwargs): 8 | """ Defines the model and returns (network, validation network output) 9 | 10 | -Return layers.get_output(final_layer_name) if validation network output and 11 | train network output are the same 12 | 13 | -For example, return layers.get_output(final_layer_name, deterministic = true) 14 | if there is a dropout layer 15 | 16 | -Use **kwargs to pass model specific parameters 17 | """ 18 | 19 | conv1_filter_count = 100 20 | conv1_filter_size = 5 21 | pool1_size = 2 22 | 23 | n_dense_units = 3000 24 | 25 | batch_size = input_var.shape[0] 26 | image_size = 32 27 | after_conv1 = image_size 28 | after_pool1 = (after_conv1 + pool1_size - 1) // pool1_size 29 | 30 | input = layers.InputLayer( 31 | shape = (None, 3, image_size, image_size), 32 | input_var = input_var 33 | ) 34 | 35 | greyscale_input = our_layers.GreyscaleLayer( 36 | incoming = input, 37 | random_greyscale = True, 38 | ) 39 | 40 | conv1 = layers.Conv2DLayer( 41 | incoming = greyscale_input, 42 | num_filters = conv1_filter_count, 43 | filter_size = conv1_filter_size, 44 | stride = 1, 45 | pad = 'same', 46 | nonlinearity = lasagne.nonlinearities.tanh, 47 | ) 48 | 49 | pool1 = layers.MaxPool2DLayer( 50 | incoming = conv1, 51 | pool_size = pool1_size, 52 | stride = pool1_size, 53 | ) 54 | 55 | dense1 = layers.DenseLayer( 56 | incoming = pool1, 57 | num_units = n_dense_units, 58 | nonlinearity = lasagne.nonlinearities.tanh, 59 | ) 60 | 61 | pre_unpool1 = layers.DenseLayer( 62 | incoming = dense1, 63 | num_units = conv1_filter_count * (after_pool1 ** 2), 64 | nonlinearity = lasagne.nonlinearities.tanh, 65 | ) 66 | 67 | pre_unpool1 = layers.ReshapeLayer( 68 | incoming = pre_unpool1, 69 | shape = (batch_size, conv1_filter_count) + (after_pool1, after_pool1), 70 | ) 71 | 72 | unpool1 = our_layers.Unpool2DLayer( 73 | incoming = pre_unpool1, 74 | kernel_size = pool1_size, 75 | ) 76 | 77 | deconv1 = layers.Conv2DLayer( 78 | incoming = unpool1, 79 | num_filters = 3, 80 | filter_size = conv1_filter_size, 81 | stride = 1, 82 | pad = 'same', 83 | nonlinearity = lasagne.nonlinearities.tanh, 84 | ) 85 | 86 | output = layers.ReshapeLayer( 87 | incoming = deconv1, 88 | shape = input_var.shape 89 | ) 90 | 91 | return (output, layers.get_output(output)) 92 | 93 | 94 | def get_cost_updates(network, input_var, output, learning_rate, **kwargs): 95 | """ Defines and returns cost and updates of the network 96 | 97 | -output can be different from layers.get_output(network), because train 98 | and validation networks may differ from each other 99 | 100 | -Use **kwargs to pass model specific parameters 101 | """ 102 | 103 | params = layers.get_all_params(network, trainable = True) 104 | 105 | batch_size = input_var.shape[0] 106 | flat_input = input_var.reshape((batch_size, 3072)) 107 | flat_output= output.reshape((batch_size, 3072)) 108 | 109 | # scale flat_output to [0, 1] 110 | mini = T.min(flat_output, axis = 1).reshape((batch_size, 1)).repeat(3072, axis = 1) 111 | maxi = T.max(flat_output, axis = 1).reshape((batch_size, 1)).repeat(3072, axis = 1) 112 | flat_output = (flat_output - mini) / (maxi - mini) 113 | 114 | # cross entrophy loss 115 | #losses = -T.sum(flat_input * T.log(flat_output) + (1 - flat_input) * T.log(1 - flat_output), axis = 1) 116 | 117 | # euclidean loss 118 | losses = T.sum((flat_input - flat_output) ** 2, axis = 1) 119 | 120 | # add saturation loss 121 | #saturation = -T.sum(T.std(T.reshape(flat_output, (batch_size, 3, 1024)), axis = 1), axis = 1) 122 | #losses = losses + 0.2 * saturation 123 | 124 | cost = T.mean(losses) 125 | gradients = T.grad(cost, params) 126 | 127 | # stochastic gradient descent 128 | #updates = [ 129 | # (param, param - learning_rate * gradient) 130 | # for param, gradient in zip(params, gradients) 131 | #] 132 | 133 | # rmsprop 134 | #updates = lasagne.updates.rmsprop(gradients, params, learning_rate = learning_rate) 135 | 136 | # momentum 137 | updates = lasagne.updates.nesterov_momentum(gradients, params, learning_rate = learning_rate) 138 | 139 | return (cost, updates) 140 | -------------------------------------------------------------------------------- /definitions/conv_100_5_2-mpool_2-dense_3000-rg-nd-eu-tanh.py: -------------------------------------------------------------------------------- 1 | import theano.tensor as T 2 | import lasagne 3 | import lasagne.layers as layers 4 | import our_layers 5 | 6 | 7 | def define_model(input_var, **kwargs): 8 | """ Defines the model and returns (network, validation network output) 9 | 10 | -Return layers.get_output(final_layer_name) if validation network output and 11 | train network output are the same 12 | 13 | -For example, return layers.get_output(final_layer_name, deterministic = true) 14 | if there is a dropout layer 15 | 16 | -Use **kwargs to pass model specific parameters 17 | """ 18 | 19 | conv1_filter_count = 100 20 | conv1_filter_size = 5 21 | pool1_size = 2 22 | 23 | n_dense_units = 3000 24 | 25 | batch_size = input_var.shape[0] 26 | image_size = 32 27 | after_conv1 = image_size 28 | after_pool1 = (after_conv1 + pool1_size - 1) // pool1_size 29 | 30 | input = layers.InputLayer( 31 | shape = (None, 3, image_size, image_size), 32 | input_var = input_var 33 | ) 34 | 35 | greyscale_input = our_layers.GreyscaleLayer( 36 | incoming = input, 37 | random_greyscale = True, 38 | ) 39 | 40 | conv1 = layers.Conv2DLayer( 41 | incoming = greyscale_input, 42 | num_filters = conv1_filter_count, 43 | filter_size = conv1_filter_size, 44 | stride = 1, 45 | pad = 'same', 46 | nonlinearity = lasagne.nonlinearities.tanh, 47 | ) 48 | 49 | pool1 = layers.MaxPool2DLayer( 50 | incoming = conv1, 51 | pool_size = pool1_size, 52 | stride = pool1_size, 53 | ) 54 | 55 | dense1 = layers.DenseLayer( 56 | incoming = pool1, 57 | num_units = n_dense_units, 58 | nonlinearity = lasagne.nonlinearities.tanh, 59 | ) 60 | 61 | pre_unpool1 = layers.DenseLayer( 62 | incoming = dense1, 63 | num_units = conv1_filter_count * (after_pool1 ** 2), 64 | nonlinearity = lasagne.nonlinearities.tanh, 65 | ) 66 | 67 | pre_unpool1 = layers.ReshapeLayer( 68 | incoming = pre_unpool1, 69 | shape = (batch_size, conv1_filter_count) + (after_pool1, after_pool1), 70 | ) 71 | 72 | unpool1 = our_layers.Unpool2DLayer( 73 | incoming = pre_unpool1, 74 | kernel_size = pool1_size, 75 | ) 76 | 77 | deconv1 = layers.Conv2DLayer( 78 | incoming = unpool1, 79 | num_filters = 3, 80 | filter_size = conv1_filter_size, 81 | stride = 1, 82 | pad = 'same', 83 | nonlinearity = lasagne.nonlinearities.tanh, 84 | ) 85 | 86 | output = layers.ReshapeLayer( 87 | incoming = deconv1, 88 | shape = input_var.shape 89 | ) 90 | 91 | return (output, layers.get_output(output)) 92 | 93 | 94 | def get_cost_updates(network, input_var, output, learning_rate, **kwargs): 95 | """ Defines and returns cost and updates of the network 96 | 97 | -output can be different from layers.get_output(network), because train 98 | and validation networks may differ from each other 99 | 100 | -Use **kwargs to pass model specific parameters 101 | """ 102 | 103 | params = layers.get_all_params(network, trainable = True) 104 | 105 | batch_size = input_var.shape[0] 106 | flat_input = input_var.reshape((batch_size, 3072)) 107 | flat_output= output.reshape((batch_size, 3072)) 108 | 109 | # scale flat_output to [0, 1] 110 | flat_output = (flat_output + 1) / 2 111 | 112 | # cross entrophy loss 113 | #losses = -T.sum(flat_input * T.log(flat_output) + (1 - flat_input) * T.log(1 - flat_output), axis = 1) 114 | 115 | # euclidean loss 116 | losses = T.sum((flat_input - flat_output) ** 2, axis = 1) 117 | 118 | # add saturation loss 119 | #saturation = -T.sum(T.std(T.reshape(flat_output, (batch_size, 3, 1024)), axis = 1), axis = 1) 120 | #losses = losses + 0.2 * saturation 121 | 122 | cost = T.mean(losses) 123 | gradients = T.grad(cost, params) 124 | 125 | # stochastic gradient descent 126 | #updates = [ 127 | # (param, param - learning_rate * gradient) 128 | # for param, gradient in zip(params, gradients) 129 | #] 130 | 131 | # rmsprop 132 | #updates = lasagne.updates.rmsprop(gradients, params, learning_rate = learning_rate) 133 | 134 | # momentum 135 | updates = lasagne.updates.nesterov_momentum(gradients, params, learning_rate = learning_rate) 136 | 137 | return (cost, updates) 138 | -------------------------------------------------------------------------------- /definitions/conv_100_5_2-mpool_2-dense_3000-rg-nd-eu.py: -------------------------------------------------------------------------------- 1 | import theano.tensor as T 2 | import lasagne 3 | import lasagne.layers as layers 4 | import our_layers 5 | 6 | 7 | def define_model(input_var, **kwargs): 8 | """ Defines the model and returns (network, validation network output) 9 | 10 | -Return layers.get_output(final_layer_name) if validation network output and 11 | train network output are the same 12 | 13 | -For example, return layers.get_output(final_layer_name, deterministic = true) 14 | if there is a dropout layer 15 | 16 | -Use **kwargs to pass model specific parameters 17 | """ 18 | 19 | conv1_filter_count = 100 20 | conv1_filter_size = 5 21 | pool1_size = 2 22 | 23 | n_dense_units = 3000 24 | 25 | batch_size = input_var.shape[0] 26 | image_size = 32 27 | after_conv1 = image_size 28 | after_pool1 = (after_conv1 + pool1_size - 1) // pool1_size 29 | 30 | input = layers.InputLayer( 31 | shape = (None, 3, image_size, image_size), 32 | input_var = input_var 33 | ) 34 | 35 | greyscale_input = our_layers.GreyscaleLayer( 36 | incoming = input, 37 | random_greyscale = True, 38 | ) 39 | 40 | conv1 = layers.Conv2DLayer( 41 | incoming = greyscale_input, 42 | num_filters = conv1_filter_count, 43 | filter_size = conv1_filter_size, 44 | stride = 1, 45 | pad = 'same', 46 | nonlinearity = lasagne.nonlinearities.sigmoid, 47 | ) 48 | 49 | pool1 = layers.MaxPool2DLayer( 50 | incoming = conv1, 51 | pool_size = pool1_size, 52 | stride = pool1_size, 53 | ) 54 | 55 | dense1 = layers.DenseLayer( 56 | incoming = pool1, 57 | num_units = n_dense_units, 58 | nonlinearity = lasagne.nonlinearities.rectify, 59 | ) 60 | 61 | pre_unpool1 = layers.DenseLayer( 62 | incoming = dense1, 63 | num_units = conv1_filter_count * (after_pool1 ** 2), 64 | nonlinearity = lasagne.nonlinearities.linear, 65 | ) 66 | 67 | pre_unpool1 = layers.ReshapeLayer( 68 | incoming = pre_unpool1, 69 | shape = (batch_size, conv1_filter_count) + (after_pool1, after_pool1), 70 | ) 71 | 72 | unpool1 = our_layers.Unpool2DLayer( 73 | incoming = pre_unpool1, 74 | kernel_size = pool1_size, 75 | ) 76 | 77 | deconv1 = layers.Conv2DLayer( 78 | incoming = unpool1, 79 | num_filters = 3, 80 | filter_size = conv1_filter_size, 81 | stride = 1, 82 | pad = 'same', 83 | nonlinearity = lasagne.nonlinearities.sigmoid, 84 | ) 85 | 86 | output = layers.ReshapeLayer( 87 | incoming = deconv1, 88 | shape = input_var.shape 89 | ) 90 | 91 | return (output, layers.get_output(output)) 92 | 93 | 94 | def get_cost_updates(network, input_var, output, learning_rate, **kwargs): 95 | """ Defines and returns cost and updates of the network 96 | 97 | -output can be different from layers.get_output(network), because train 98 | and validation networks may differ from each other 99 | 100 | -Use **kwargs to pass model specific parameters 101 | """ 102 | 103 | params = layers.get_all_params(network, trainable = True) 104 | 105 | batch_size = input_var.shape[0] 106 | flat_input = input_var.reshape((batch_size, 3072)) 107 | flat_output= output.reshape((batch_size, 3072)) 108 | 109 | # cross entrophy loss 110 | #losses = -T.sum(flat_input * T.log(flat_output) + (1 - flat_input) * T.log(1 - flat_output), axis = 1) 111 | 112 | # euclidean loss 113 | losses = T.sum((flat_input - flat_output) ** 2, axis = 1) 114 | 115 | # add saturation loss 116 | #saturation = -T.sum(T.std(T.reshape(flat_output, (batch_size, 3, 1024)), axis = 1), axis = 1) 117 | #losses = losses + 0.2 * saturation 118 | 119 | cost = T.mean(losses) 120 | gradients = T.grad(cost, params) 121 | 122 | # stochastic gradient descent 123 | #updates = [ 124 | # (param, param - learning_rate * gradient) 125 | # for param, gradient in zip(params, gradients) 126 | #] 127 | 128 | # rmsprop 129 | #updates = lasagne.updates.rmsprop(gradients, params, learning_rate = learning_rate) 130 | 131 | # momentum 132 | updates = lasagne.updates.nesterov_momentum(gradients, params, learning_rate = learning_rate) 133 | 134 | return (cost, updates) 135 | -------------------------------------------------------------------------------- /definitions/conv_100_5_2-mpool_2-dense_3000-rg-nd.py: -------------------------------------------------------------------------------- 1 | import theano.tensor as T 2 | import lasagne 3 | import lasagne.layers as layers 4 | import our_layers 5 | 6 | 7 | def define_model(input_var, **kwargs): 8 | """ Defines the model and returns (network, validation network output) 9 | 10 | -Return layers.get_output(final_layer_name) if validation network output and 11 | train network output are the same 12 | 13 | -For example, return layers.get_output(final_layer_name, deterministic = true) 14 | if there is a dropout layer 15 | 16 | -Use **kwargs to pass model specific parameters 17 | """ 18 | 19 | conv1_filter_count = 100 20 | conv1_filter_size = 5 21 | pool1_size = 2 22 | 23 | n_dense_units = 3000 24 | 25 | batch_size = input_var.shape[0] 26 | image_size = 32 27 | after_conv1 = image_size 28 | after_pool1 = (after_conv1 + pool1_size - 1) // pool1_size 29 | 30 | input = layers.InputLayer( 31 | shape = (None, 3, image_size, image_size), 32 | input_var = input_var 33 | ) 34 | 35 | greyscale_input = our_layers.GreyscaleLayer( 36 | incoming = input, 37 | random_greyscale = True, 38 | ) 39 | 40 | conv1 = layers.Conv2DLayer( 41 | incoming = greyscale_input, 42 | num_filters = conv1_filter_count, 43 | filter_size = conv1_filter_size, 44 | stride = 1, 45 | pad = 'same', 46 | nonlinearity = lasagne.nonlinearities.sigmoid, 47 | ) 48 | 49 | pool1 = layers.MaxPool2DLayer( 50 | incoming = conv1, 51 | pool_size = pool1_size, 52 | stride = pool1_size, 53 | ) 54 | 55 | dense1 = layers.DenseLayer( 56 | incoming = pool1, 57 | num_units = n_dense_units, 58 | nonlinearity = lasagne.nonlinearities.rectify, 59 | ) 60 | 61 | pre_unpool1 = layers.DenseLayer( 62 | incoming = dense1, 63 | num_units = conv1_filter_count * (after_pool1 ** 2), 64 | nonlinearity = lasagne.nonlinearities.linear, 65 | ) 66 | 67 | pre_unpool1 = layers.ReshapeLayer( 68 | incoming = pre_unpool1, 69 | shape = (batch_size, conv1_filter_count) + (after_pool1, after_pool1), 70 | ) 71 | 72 | unpool1 = our_layers.Unpool2DLayer( 73 | incoming = pre_unpool1, 74 | kernel_size = pool1_size, 75 | ) 76 | 77 | deconv1 = layers.Conv2DLayer( 78 | incoming = unpool1, 79 | num_filters = 3, 80 | filter_size = conv1_filter_size, 81 | stride = 1, 82 | pad = 'same', 83 | nonlinearity = lasagne.nonlinearities.sigmoid, 84 | ) 85 | 86 | output = layers.ReshapeLayer( 87 | incoming = deconv1, 88 | shape = input_var.shape 89 | ) 90 | 91 | return (output, layers.get_output(output)) 92 | 93 | 94 | def get_cost_updates(network, input_var, output, learning_rate, **kwargs): 95 | """ Defines and returns cost and updates of the network 96 | 97 | -output can be different from layers.get_output(network), because train 98 | and validation networks may differ from each other 99 | 100 | -Use **kwargs to pass model specific parameters 101 | """ 102 | 103 | params = layers.get_all_params(network, trainable = True) 104 | 105 | batch_size = input_var.shape[0] 106 | flat_input = input_var.reshape((batch_size, 3072)) 107 | flat_output= output.reshape((batch_size, 3072)) 108 | 109 | # cross entrophy loss 110 | losses = -T.sum(flat_input * T.log(flat_output) + (1 - flat_input) * T.log(1 - flat_output), axis = 1) 111 | 112 | # add saturation loss 113 | #saturation = -T.sum(T.std(T.reshape(flat_output, (batch_size, 3, 1024)), axis = 1), axis = 1) 114 | #losses = losses + 0.2 * saturation 115 | 116 | cost = T.mean(losses) 117 | gradients = T.grad(cost, params) 118 | 119 | # stochastic gradient descent 120 | #updates = [ 121 | # (param, param - learning_rate * gradient) 122 | # for param, gradient in zip(params, gradients) 123 | #] 124 | 125 | # rmsprop 126 | #updates = lasagne.updates.rmsprop(gradients, params, learning_rate = learning_rate) 127 | 128 | # momentum 129 | updates = lasagne.updates.nesterov_momentum(gradients, params, learning_rate = learning_rate) 130 | 131 | return (cost, updates) 132 | -------------------------------------------------------------------------------- /definitions/conv_100_5_2-mpool_2-dense_3000-rg-wd0.001.py: -------------------------------------------------------------------------------- 1 | import theano.tensor as T 2 | import lasagne 3 | import lasagne.layers as layers 4 | import our_layers 5 | 6 | 7 | def define_model(input_var, **kwargs): 8 | """ Defines the model and returns (network, validation network output) 9 | 10 | -Return layers.get_output(final_layer_name) if validation network output and 11 | train network output are the same 12 | 13 | -For example, return layers.get_output(final_layer_name, deterministic = true) 14 | if there is a dropout layer 15 | 16 | -Use **kwargs to pass model specific parameters 17 | """ 18 | 19 | image_size = 32 20 | conv_filter_count = 100 21 | conv_filter_size = 5 22 | pool_size = 2 23 | n_dense_units = 3000 24 | 25 | input = layers.InputLayer( 26 | shape = (None, 3, image_size, image_size), 27 | input_var = input_var 28 | ) 29 | 30 | greyscale_input = our_layers.GreyscaleLayer( 31 | incoming = input, 32 | random_greyscale = True, 33 | ) 34 | 35 | conv1 = layers.Conv2DLayer( 36 | incoming = greyscale_input, 37 | num_filters = conv_filter_count, 38 | filter_size = conv_filter_size, 39 | stride = 1, 40 | nonlinearity = lasagne.nonlinearities.sigmoid, 41 | ) 42 | 43 | pool1 = layers.MaxPool2DLayer( 44 | incoming = conv1, 45 | pool_size = pool_size, 46 | stride = pool_size, 47 | ) 48 | 49 | dense1 = layers.DenseLayer( 50 | incoming =pool1, 51 | num_units = n_dense_units, 52 | nonlinearity = lasagne.nonlinearities.rectify, 53 | ) 54 | 55 | pre_unpool1 = layers.DenseLayer( 56 | incoming = dense1, 57 | num_units = conv_filter_count * (image_size + conv_filter_size - 1) ** 2 / (pool_size * pool_size), 58 | nonlinearity = lasagne.nonlinearities.linear, 59 | ) 60 | 61 | pre_unpool1 = layers.ReshapeLayer( 62 | incoming = pre_unpool1, 63 | shape = (input_var.shape[0], conv_filter_count) + ((image_size + conv_filter_size - 1) / 2, (image_size + conv_filter_size - 1) / 2), 64 | ) 65 | 66 | unpool1 = our_layers.Unpool2DLayer( 67 | incoming = pre_unpool1, 68 | kernel_size = 2, 69 | ) 70 | 71 | deconv1 = layers.Conv2DLayer( 72 | incoming = unpool1, 73 | num_filters = 3, 74 | filter_size = conv_filter_size, 75 | stride = 1, 76 | nonlinearity = lasagne.nonlinearities.sigmoid, 77 | ) 78 | 79 | output = layers.ReshapeLayer( 80 | incoming = deconv1, 81 | shape = input_var.shape 82 | ) 83 | 84 | return (output, layers.get_output(output)) 85 | 86 | 87 | def get_cost_updates(network, input_var, output, learning_rate, **kwargs): 88 | """ Defines and returns cost and updates of the network 89 | 90 | -output can be different from layers.get_output(network), because train 91 | and validation networks may differ from each other 92 | 93 | -Use **kwargs to pass model specific parameters 94 | """ 95 | 96 | params = layers.get_all_params(network, trainable = True) 97 | 98 | batch_size = input_var.shape[0] 99 | flat_input = input_var.reshape((batch_size, 3072)) 100 | flat_output= output.reshape((batch_size, 3072)) 101 | 102 | # cross entrophy loss 103 | losses = -T.sum(flat_input * T.log(flat_output) + (1 - flat_input) * T.log(1 - flat_output), axis = 1) 104 | 105 | # add saturation loss 106 | #saturation = -T.sum(T.std(T.reshape(flat_output, (batch_size, 3, 1024)), axis = 1), axis = 1) 107 | #losses = losses + 0.2 * saturation 108 | 109 | cost = T.mean(losses) 110 | 111 | # add weight decay 112 | cost = cost + 0.001 * lasagne.regularization.regularize_network_params( 113 | layer = network, 114 | penalty = lasagne.regularization.l2, 115 | ) 116 | 117 | gradients = T.grad(cost, params) 118 | 119 | # stochastic gradient descent 120 | #updates = [ 121 | # (param, param - learning_rate * gradient) 122 | # for param, gradient in zip(params, gradients) 123 | #] 124 | 125 | # rmsprop 126 | #updates = lasagne.updates.rmsprop(gradients, params, learning_rate = learning_rate) 127 | 128 | # momentum 129 | updates = lasagne.updates.nesterov_momentum(gradients, params, learning_rate = learning_rate) 130 | 131 | return (cost, updates) 132 | -------------------------------------------------------------------------------- /definitions/conv_100_5_2-mpool_2-dense_3000-rg-wd0.01.py: -------------------------------------------------------------------------------- 1 | import theano.tensor as T 2 | import lasagne 3 | import lasagne.layers as layers 4 | import our_layers 5 | 6 | 7 | def define_model(input_var, **kwargs): 8 | """ Defines the model and returns (network, validation network output) 9 | 10 | -Return layers.get_output(final_layer_name) if validation network output and 11 | train network output are the same 12 | 13 | -For example, return layers.get_output(final_layer_name, deterministic = true) 14 | if there is a dropout layer 15 | 16 | -Use **kwargs to pass model specific parameters 17 | """ 18 | 19 | image_size = 32 20 | conv_filter_count = 100 21 | conv_filter_size = 5 22 | pool_size = 2 23 | n_dense_units = 3000 24 | 25 | input = layers.InputLayer( 26 | shape = (None, 3, image_size, image_size), 27 | input_var = input_var 28 | ) 29 | 30 | greyscale_input = our_layers.GreyscaleLayer( 31 | incoming = input, 32 | random_greyscale = True, 33 | ) 34 | 35 | conv1 = layers.Conv2DLayer( 36 | incoming = greyscale_input, 37 | num_filters = conv_filter_count, 38 | filter_size = conv_filter_size, 39 | stride = 1, 40 | nonlinearity = lasagne.nonlinearities.sigmoid, 41 | ) 42 | 43 | pool1 = layers.MaxPool2DLayer( 44 | incoming = conv1, 45 | pool_size = pool_size, 46 | stride = pool_size, 47 | ) 48 | 49 | dense1 = layers.DenseLayer( 50 | incoming =pool1, 51 | num_units = n_dense_units, 52 | nonlinearity = lasagne.nonlinearities.rectify, 53 | ) 54 | 55 | pre_unpool1 = layers.DenseLayer( 56 | incoming = dense1, 57 | num_units = conv_filter_count * (image_size + conv_filter_size - 1) ** 2 / (pool_size * pool_size), 58 | nonlinearity = lasagne.nonlinearities.linear, 59 | ) 60 | 61 | pre_unpool1 = layers.ReshapeLayer( 62 | incoming = pre_unpool1, 63 | shape = (input_var.shape[0], conv_filter_count) + ((image_size + conv_filter_size - 1) / 2, (image_size + conv_filter_size - 1) / 2), 64 | ) 65 | 66 | unpool1 = our_layers.Unpool2DLayer( 67 | incoming = pre_unpool1, 68 | kernel_size = 2, 69 | ) 70 | 71 | deconv1 = layers.Conv2DLayer( 72 | incoming = unpool1, 73 | num_filters = 3, 74 | filter_size = conv_filter_size, 75 | stride = 1, 76 | nonlinearity = lasagne.nonlinearities.sigmoid, 77 | ) 78 | 79 | output = layers.ReshapeLayer( 80 | incoming = deconv1, 81 | shape = input_var.shape 82 | ) 83 | 84 | return (output, layers.get_output(output)) 85 | 86 | 87 | def get_cost_updates(network, input_var, output, learning_rate, **kwargs): 88 | """ Defines and returns cost and updates of the network 89 | 90 | -output can be different from layers.get_output(network), because train 91 | and validation networks may differ from each other 92 | 93 | -Use **kwargs to pass model specific parameters 94 | """ 95 | 96 | params = layers.get_all_params(network, trainable = True) 97 | 98 | batch_size = input_var.shape[0] 99 | flat_input = input_var.reshape((batch_size, 3072)) 100 | flat_output= output.reshape((batch_size, 3072)) 101 | 102 | # cross entrophy loss 103 | losses = -T.sum(flat_input * T.log(flat_output) + (1 - flat_input) * T.log(1 - flat_output), axis = 1) 104 | 105 | # add saturation loss 106 | #saturation = -T.sum(T.std(T.reshape(flat_output, (batch_size, 3, 1024)), axis = 1), axis = 1) 107 | #losses = losses + 0.2 * saturation 108 | 109 | cost = T.mean(losses) 110 | 111 | # add weight decay 112 | reg = sum(T.sum(p ** 2) for p in params) 113 | cost = cost + 0.01 * reg 114 | 115 | #cost = cost + 0.01 * lasagne.regularization.regularize_network_params( 116 | # layer = network, 117 | # penalty = lasagne.regularization.l2, 118 | #) 119 | 120 | gradients = T.grad(cost, params) 121 | 122 | # stochastic gradient descent 123 | #updates = [ 124 | # (param, param - learning_rate * gradient) 125 | # for param, gradient in zip(params, gradients) 126 | #] 127 | 128 | # rmsprop 129 | #updates = lasagne.updates.rmsprop(gradients, params, learning_rate = learning_rate) 130 | 131 | # momentum 132 | updates = lasagne.updates.nesterov_momentum(gradients, params, learning_rate = learning_rate) 133 | 134 | return (cost, updates) 135 | -------------------------------------------------------------------------------- /definitions/conv_100_5_2-mpool_2-dense_3000-rg-wd0.1.py: -------------------------------------------------------------------------------- 1 | import theano.tensor as T 2 | import lasagne 3 | import lasagne.layers as layers 4 | import our_layers 5 | 6 | 7 | def define_model(input_var, **kwargs): 8 | """ Defines the model and returns (network, validation network output) 9 | 10 | -Return layers.get_output(final_layer_name) if validation network output and 11 | train network output are the same 12 | 13 | -For example, return layers.get_output(final_layer_name, deterministic = true) 14 | if there is a dropout layer 15 | 16 | -Use **kwargs to pass model specific parameters 17 | """ 18 | 19 | image_size = 32 20 | conv_filter_count = 100 21 | conv_filter_size = 5 22 | pool_size = 2 23 | n_dense_units = 3000 24 | 25 | input = layers.InputLayer( 26 | shape = (None, 3, image_size, image_size), 27 | input_var = input_var 28 | ) 29 | 30 | greyscale_input = our_layers.GreyscaleLayer( 31 | incoming = input, 32 | random_greyscale = True, 33 | ) 34 | 35 | conv1 = layers.Conv2DLayer( 36 | incoming = greyscale_input, 37 | num_filters = conv_filter_count, 38 | filter_size = conv_filter_size, 39 | stride = 1, 40 | nonlinearity = lasagne.nonlinearities.sigmoid, 41 | ) 42 | 43 | pool1 = layers.MaxPool2DLayer( 44 | incoming = conv1, 45 | pool_size = pool_size, 46 | stride = pool_size, 47 | ) 48 | 49 | dense1 = layers.DenseLayer( 50 | incoming =pool1, 51 | num_units = n_dense_units, 52 | nonlinearity = lasagne.nonlinearities.rectify, 53 | ) 54 | 55 | pre_unpool1 = layers.DenseLayer( 56 | incoming = dense1, 57 | num_units = conv_filter_count * (image_size + conv_filter_size - 1) ** 2 / (pool_size * pool_size), 58 | nonlinearity = lasagne.nonlinearities.linear, 59 | ) 60 | 61 | pre_unpool1 = layers.ReshapeLayer( 62 | incoming = pre_unpool1, 63 | shape = (input_var.shape[0], conv_filter_count) + ((image_size + conv_filter_size - 1) / 2, (image_size + conv_filter_size - 1) / 2), 64 | ) 65 | 66 | unpool1 = our_layers.Unpool2DLayer( 67 | incoming = pre_unpool1, 68 | kernel_size = 2, 69 | ) 70 | 71 | deconv1 = layers.Conv2DLayer( 72 | incoming = unpool1, 73 | num_filters = 3, 74 | filter_size = conv_filter_size, 75 | stride = 1, 76 | nonlinearity = lasagne.nonlinearities.sigmoid, 77 | ) 78 | 79 | output = layers.ReshapeLayer( 80 | incoming = deconv1, 81 | shape = input_var.shape 82 | ) 83 | 84 | return (output, layers.get_output(output)) 85 | 86 | 87 | def get_cost_updates(network, input_var, output, learning_rate, **kwargs): 88 | """ Defines and returns cost and updates of the network 89 | 90 | -output can be different from layers.get_output(network), because train 91 | and validation networks may differ from each other 92 | 93 | -Use **kwargs to pass model specific parameters 94 | """ 95 | 96 | params = layers.get_all_params(network, trainable = True) 97 | 98 | batch_size = input_var.shape[0] 99 | flat_input = input_var.reshape((batch_size, 3072)) 100 | flat_output= output.reshape((batch_size, 3072)) 101 | 102 | # cross entrophy loss 103 | losses = -T.sum(flat_input * T.log(flat_output) + (1 - flat_input) * T.log(1 - flat_output), axis = 1) 104 | 105 | # add saturation loss 106 | #saturation = -T.sum(T.std(T.reshape(flat_output, (batch_size, 3, 1024)), axis = 1), axis = 1) 107 | #losses = losses + 0.2 * saturation 108 | 109 | cost = T.mean(losses) 110 | 111 | # add weight decay 112 | cost = cost + 0.1 * lasagne.regularization.regularize_network_params( 113 | layer = network, 114 | penalty = lasagne.regularization.l2, 115 | ) 116 | 117 | gradients = T.grad(cost, params) 118 | 119 | # stochastic gradient descent 120 | #updates = [ 121 | # (param, param - learning_rate * gradient) 122 | # for param, gradient in zip(params, gradients) 123 | #] 124 | 125 | # rmsprop 126 | #updates = lasagne.updates.rmsprop(gradients, params, learning_rate = learning_rate) 127 | 128 | # momentum 129 | updates = lasagne.updates.nesterov_momentum(gradients, params, learning_rate = learning_rate) 130 | 131 | return (cost, updates) 132 | -------------------------------------------------------------------------------- /definitions/conv_100_5_2-mpool_2-dense_3000-rg.py: -------------------------------------------------------------------------------- 1 | import theano.tensor as T 2 | import lasagne 3 | import lasagne.layers as layers 4 | import our_layers 5 | 6 | 7 | def define_model(input_var, **kwargs): 8 | """ Defines the model and returns (network, validation network output) 9 | 10 | -Return layers.get_output(final_layer_name) if validation network output and 11 | train network output are the same 12 | 13 | -For example, return layers.get_output(final_layer_name, deterministic = true) 14 | if there is a dropout layer 15 | 16 | -Use **kwargs to pass model specific parameters 17 | """ 18 | 19 | image_size = 32 20 | conv_filter_count = 100 21 | conv_filter_size = 5 22 | pool_size = 2 23 | n_dense_units = 3000 24 | 25 | input = layers.InputLayer( 26 | shape = (None, 3, image_size, image_size), 27 | input_var = input_var 28 | ) 29 | 30 | greyscale_input = our_layers.GreyscaleLayer( 31 | incoming = input, 32 | random_greyscale = True, 33 | ) 34 | 35 | conv1 = layers.Conv2DLayer( 36 | incoming = greyscale_input, 37 | num_filters = conv_filter_count, 38 | filter_size = conv_filter_size, 39 | stride = 1, 40 | nonlinearity = lasagne.nonlinearities.sigmoid, 41 | ) 42 | 43 | pool1 = layers.MaxPool2DLayer( 44 | incoming = conv1, 45 | pool_size = pool_size, 46 | stride = pool_size, 47 | ) 48 | 49 | dense1 = layers.DenseLayer( 50 | incoming =pool1, 51 | num_units = n_dense_units, 52 | nonlinearity = lasagne.nonlinearities.rectify, 53 | ) 54 | 55 | pre_unpool1 = layers.DenseLayer( 56 | incoming = dense1, 57 | num_units = conv_filter_count * (image_size + conv_filter_size - 1) ** 2 / (pool_size * pool_size), 58 | nonlinearity = lasagne.nonlinearities.linear, 59 | ) 60 | 61 | pre_unpool1 = layers.ReshapeLayer( 62 | incoming = pre_unpool1, 63 | shape = (input_var.shape[0], conv_filter_count) + ((image_size + conv_filter_size - 1) / 2, (image_size + conv_filter_size - 1) / 2), 64 | ) 65 | 66 | unpool1 = our_layers.Unpool2DLayer( 67 | incoming = pre_unpool1, 68 | kernel_size = 2, 69 | ) 70 | 71 | deconv1 = layers.Conv2DLayer( 72 | incoming = unpool1, 73 | num_filters = 3, 74 | filter_size = conv_filter_size, 75 | stride = 1, 76 | nonlinearity = lasagne.nonlinearities.sigmoid, 77 | ) 78 | 79 | output = layers.ReshapeLayer( 80 | incoming = deconv1, 81 | shape = input_var.shape 82 | ) 83 | 84 | return (output, layers.get_output(output)) 85 | 86 | 87 | def get_cost_updates(network, input_var, output, learning_rate, **kwargs): 88 | """ Defines and returns cost and updates of the network 89 | 90 | -output can be different from layers.get_output(network), because train 91 | and validation networks may differ from each other 92 | 93 | -Use **kwargs to pass model specific parameters 94 | """ 95 | 96 | params = layers.get_all_params(network, trainable = True) 97 | 98 | batch_size = input_var.shape[0] 99 | flat_input = input_var.reshape((batch_size, 3072)) 100 | flat_output= output.reshape((batch_size, 3072)) 101 | 102 | # cross entrophy loss 103 | losses = -T.sum(flat_input * T.log(flat_output) + (1 - flat_input) * T.log(1 - flat_output), axis = 1) 104 | 105 | # add saturation loss 106 | #saturation = -T.sum(T.std(T.reshape(flat_output, (batch_size, 3, 1024)), axis = 1), axis = 1) 107 | #losses = losses + 0.2 * saturation 108 | 109 | cost = T.mean(losses) 110 | gradients = T.grad(cost, params) 111 | 112 | # stochastic gradient descent 113 | #updates = [ 114 | # (param, param - learning_rate * gradient) 115 | # for param, gradient in zip(params, gradients) 116 | #] 117 | 118 | # rmsprop 119 | #updates = lasagne.updates.rmsprop(gradients, params, learning_rate = learning_rate) 120 | 121 | # momentum 122 | updates = lasagne.updates.nesterov_momentum(gradients, params, learning_rate = learning_rate) 123 | 124 | return (cost, updates) 125 | -------------------------------------------------------------------------------- /definitions/conv_100_5_2-mpool_2-dense_3000_dr0.3-nd-eu-tanh.py: -------------------------------------------------------------------------------- 1 | import theano.tensor as T 2 | import lasagne 3 | import lasagne.layers as layers 4 | import our_layers 5 | 6 | lr_policy = 'geom' 7 | lr_base = 0.0004 8 | lr_step = 20 9 | lr_coef = 0.8 10 | 11 | def define_model(input_var, **kwargs): 12 | """ Defines the model and returns (network, validation network output) 13 | 14 | -Return layers.get_output(final_layer_name) if validation network output and 15 | train network output are the same 16 | 17 | -For example, return layers.get_output(final_layer_name, deterministic = true) 18 | if there is a dropout layer 19 | 20 | -Use **kwargs to pass model specific parameters 21 | """ 22 | 23 | conv1_filter_count = 100 24 | conv1_filter_size = 5 25 | pool1_size = 2 26 | 27 | n_dense_units = 3000 28 | 29 | batch_size = input_var.shape[0] 30 | image_size = 32 31 | after_conv1 = image_size 32 | after_pool1 = (after_conv1 + pool1_size - 1) // pool1_size 33 | 34 | input = layers.InputLayer( 35 | shape = (None, 3, image_size, image_size), 36 | input_var = input_var 37 | ) 38 | 39 | greyscale_input = our_layers.GreyscaleLayer( 40 | incoming = input, 41 | random_greyscale = False, 42 | ) 43 | 44 | conv1 = layers.Conv2DLayer( 45 | incoming = greyscale_input, 46 | num_filters = conv1_filter_count, 47 | filter_size = conv1_filter_size, 48 | stride = 1, 49 | pad = 'same', 50 | nonlinearity = lasagne.nonlinearities.tanh, 51 | ) 52 | 53 | pool1 = layers.MaxPool2DLayer( 54 | incoming = conv1, 55 | pool_size = pool1_size, 56 | stride = pool1_size, 57 | ) 58 | 59 | dense1 = layers.DenseLayer( 60 | incoming = pool1, 61 | num_units = n_dense_units, 62 | nonlinearity = lasagne.nonlinearities.tanh, 63 | ) 64 | 65 | dense1 = layers.DropoutLayer( 66 | incoming = dense1, 67 | p = 0.3, 68 | ) 69 | 70 | pre_unpool1 = layers.DenseLayer( 71 | incoming = dense1, 72 | num_units = conv1_filter_count * (after_pool1 ** 2), 73 | nonlinearity = lasagne.nonlinearities.tanh, 74 | ) 75 | 76 | pre_unpool1 = layers.ReshapeLayer( 77 | incoming = pre_unpool1, 78 | shape = (batch_size, conv1_filter_count) + (after_pool1, after_pool1), 79 | ) 80 | 81 | unpool1 = our_layers.Unpool2DLayer( 82 | incoming = pre_unpool1, 83 | kernel_size = pool1_size, 84 | ) 85 | 86 | deconv1 = layers.Conv2DLayer( 87 | incoming = unpool1, 88 | num_filters = 3, 89 | filter_size = conv1_filter_size, 90 | stride = 1, 91 | pad = 'same', 92 | nonlinearity = lasagne.nonlinearities.tanh, 93 | ) 94 | 95 | output = layers.ReshapeLayer( 96 | incoming = deconv1, 97 | shape = input_var.shape 98 | ) 99 | 100 | return (output, layers.get_output(output, deterministic = True)) 101 | 102 | 103 | def get_cost_updates(network, input_var, output, learning_rate, **kwargs): 104 | """ Defines and returns cost and updates of the network 105 | 106 | -output can be different from layers.get_output(network), because train 107 | and validation networks may differ from each other 108 | 109 | -Use **kwargs to pass model specific parameters 110 | """ 111 | 112 | params = layers.get_all_params(network, trainable = True) 113 | 114 | batch_size = input_var.shape[0] 115 | flat_input = input_var.reshape((batch_size, 3072)) 116 | flat_output= output.reshape((batch_size, 3072)) 117 | 118 | # scale flat_output to [0, 1] 119 | flat_output = (flat_output + 1) / 2 120 | 121 | # cross entrophy loss 122 | #losses = -T.sum(flat_input * T.log(flat_output) + (1 - flat_input) * T.log(1 - flat_output), axis = 1) 123 | 124 | # euclidean loss 125 | losses = T.sum((flat_input - flat_output) ** 2, axis = 1) 126 | 127 | # add saturation loss 128 | #saturation = -T.sum(T.std(T.reshape(flat_output, (batch_size, 3, 1024)), axis = 1), axis = 1) 129 | #losses = losses + 0.2 * saturation 130 | 131 | cost = T.mean(losses) 132 | gradients = T.grad(cost, params) 133 | 134 | # stochastic gradient descent 135 | #updates = [ 136 | # (param, param - learning_rate * gradient) 137 | # for param, gradient in zip(params, gradients) 138 | #] 139 | 140 | # rmsprop 141 | #updates = lasagne.updates.rmsprop(gradients, params, learning_rate = learning_rate) 142 | 143 | # momentum 144 | updates = lasagne.updates.nesterov_momentum(gradients, params, learning_rate = learning_rate) 145 | 146 | return (cost, updates) 147 | -------------------------------------------------------------------------------- /definitions/conv_100_5_2-mpool_2-dense_3000_dr0.5-nd-eu-tanh.py: -------------------------------------------------------------------------------- 1 | import theano.tensor as T 2 | import lasagne 3 | import lasagne.layers as layers 4 | import our_layers 5 | 6 | lr_policy = 'geom' 7 | lr_base = 0.0004 8 | lr_step = 20 9 | lr_coef = 0.8 10 | 11 | def define_model(input_var, **kwargs): 12 | """ Defines the model and returns (network, validation network output) 13 | 14 | -Return layers.get_output(final_layer_name) if validation network output and 15 | train network output are the same 16 | 17 | -For example, return layers.get_output(final_layer_name, deterministic = true) 18 | if there is a dropout layer 19 | 20 | -Use **kwargs to pass model specific parameters 21 | """ 22 | 23 | conv1_filter_count = 100 24 | conv1_filter_size = 5 25 | pool1_size = 2 26 | 27 | n_dense_units = 3000 28 | 29 | batch_size = input_var.shape[0] 30 | image_size = 32 31 | after_conv1 = image_size 32 | after_pool1 = (after_conv1 + pool1_size - 1) // pool1_size 33 | 34 | input = layers.InputLayer( 35 | shape = (None, 3, image_size, image_size), 36 | input_var = input_var 37 | ) 38 | 39 | greyscale_input = our_layers.GreyscaleLayer( 40 | incoming = input, 41 | random_greyscale = False, 42 | ) 43 | 44 | conv1 = layers.Conv2DLayer( 45 | incoming = greyscale_input, 46 | num_filters = conv1_filter_count, 47 | filter_size = conv1_filter_size, 48 | stride = 1, 49 | pad = 'same', 50 | nonlinearity = lasagne.nonlinearities.tanh, 51 | ) 52 | 53 | pool1 = layers.MaxPool2DLayer( 54 | incoming = conv1, 55 | pool_size = pool1_size, 56 | stride = pool1_size, 57 | ) 58 | 59 | dense1 = layers.DenseLayer( 60 | incoming = pool1, 61 | num_units = n_dense_units, 62 | nonlinearity = lasagne.nonlinearities.tanh, 63 | ) 64 | 65 | dense1 = layers.DropoutLayer( 66 | incoming = dense1, 67 | ) 68 | 69 | pre_unpool1 = layers.DenseLayer( 70 | incoming = dense1, 71 | num_units = conv1_filter_count * (after_pool1 ** 2), 72 | nonlinearity = lasagne.nonlinearities.tanh, 73 | ) 74 | 75 | pre_unpool1 = layers.ReshapeLayer( 76 | incoming = pre_unpool1, 77 | shape = (batch_size, conv1_filter_count) + (after_pool1, after_pool1), 78 | ) 79 | 80 | unpool1 = our_layers.Unpool2DLayer( 81 | incoming = pre_unpool1, 82 | kernel_size = pool1_size, 83 | ) 84 | 85 | deconv1 = layers.Conv2DLayer( 86 | incoming = unpool1, 87 | num_filters = 3, 88 | filter_size = conv1_filter_size, 89 | stride = 1, 90 | pad = 'same', 91 | nonlinearity = lasagne.nonlinearities.tanh, 92 | ) 93 | 94 | output = layers.ReshapeLayer( 95 | incoming = deconv1, 96 | shape = input_var.shape 97 | ) 98 | 99 | return (output, layers.get_output(output, deterministic = True)) 100 | 101 | 102 | def get_cost_updates(network, input_var, output, learning_rate, **kwargs): 103 | """ Defines and returns cost and updates of the network 104 | 105 | -output can be different from layers.get_output(network), because train 106 | and validation networks may differ from each other 107 | 108 | -Use **kwargs to pass model specific parameters 109 | """ 110 | 111 | params = layers.get_all_params(network, trainable = True) 112 | 113 | batch_size = input_var.shape[0] 114 | flat_input = input_var.reshape((batch_size, 3072)) 115 | flat_output= output.reshape((batch_size, 3072)) 116 | 117 | # scale flat_output to [0, 1] 118 | flat_output = (flat_output + 1) / 2 119 | 120 | # cross entrophy loss 121 | #losses = -T.sum(flat_input * T.log(flat_output) + (1 - flat_input) * T.log(1 - flat_output), axis = 1) 122 | 123 | # euclidean loss 124 | losses = T.sum((flat_input - flat_output) ** 2, axis = 1) 125 | 126 | # add saturation loss 127 | #saturation = -T.sum(T.std(T.reshape(flat_output, (batch_size, 3, 1024)), axis = 1), axis = 1) 128 | #losses = losses + 0.2 * saturation 129 | 130 | cost = T.mean(losses) 131 | gradients = T.grad(cost, params) 132 | 133 | # stochastic gradient descent 134 | #updates = [ 135 | # (param, param - learning_rate * gradient) 136 | # for param, gradient in zip(params, gradients) 137 | #] 138 | 139 | # rmsprop 140 | #updates = lasagne.updates.rmsprop(gradients, params, learning_rate = learning_rate) 141 | 142 | # momentum 143 | updates = lasagne.updates.nesterov_momentum(gradients, params, learning_rate = learning_rate) 144 | 145 | return (cost, updates) 146 | -------------------------------------------------------------------------------- /definitions/conv_30_5_2-mpool_2-dense_3000-ec.py: -------------------------------------------------------------------------------- 1 | import theano.tensor as T 2 | import lasagne 3 | import lasagne.layers as layers 4 | import our_layers 5 | 6 | 7 | def define_model(input_var, **kwargs): 8 | """ Defines the model and returns (network, validation network output) 9 | 10 | -Return layers.get_output(final_layer_name) if validation network output and 11 | train network output are the same 12 | 13 | -For example, return layers.get_output(final_layer_name, deterministic = true) 14 | if there is a dropout layer 15 | 16 | -Use **kwargs to pass model specific parameters 17 | """ 18 | 19 | image_size = 32 20 | conv_filter_count = 30 21 | conv_filter_size = 5 22 | pool_size = 2 23 | n_dense_units = 3000 24 | 25 | input = layers.InputLayer( 26 | shape = (None, 3, image_size, image_size), 27 | input_var = input_var 28 | ) 29 | 30 | greyscale_input = our_layers.GreyscaleLayer( 31 | incoming = input, 32 | random_greyscale = False, 33 | ) 34 | 35 | conv1 = layers.Conv2DLayer( 36 | incoming = greyscale_input, 37 | num_filters = conv_filter_count, 38 | filter_size = conv_filter_size, 39 | stride = 1, 40 | nonlinearity = lasagne.nonlinearities.sigmoid, 41 | ) 42 | 43 | pool1 = layers.MaxPool2DLayer( 44 | incoming = conv1, 45 | pool_size = pool_size, 46 | stride = pool_size, 47 | ) 48 | 49 | dense1 = layers.DenseLayer( 50 | incoming = pool1, 51 | num_units = n_dense_units, 52 | nonlinearity = lasagne.nonlinearities.rectify, 53 | ) 54 | 55 | pre_unpool1 = layers.DenseLayer( 56 | incoming = dense1, 57 | num_units = conv_filter_count * (image_size + conv_filter_size - 1) ** 2 / (pool_size * pool_size), 58 | nonlinearity = lasagne.nonlinearities.linear, 59 | ) 60 | 61 | pre_unpool1 = layers.ReshapeLayer( 62 | incoming = pre_unpool1, 63 | shape = (input_var.shape[0], conv_filter_count) + ((image_size + conv_filter_size - 1) / 2, (image_size + conv_filter_size - 1) / 2), 64 | ) 65 | 66 | unpool1 = our_layers.Unpool2DLayer( 67 | incoming = pre_unpool1, 68 | kernel_size = 2, 69 | ) 70 | 71 | deconv1 = layers.Conv2DLayer( 72 | incoming = unpool1, 73 | num_filters = 3, 74 | filter_size = conv_filter_size, 75 | stride = 1, 76 | nonlinearity = lasagne.nonlinearities.sigmoid, 77 | ) 78 | 79 | deconv1_flatten = layers.ReshapeLayer( 80 | incoming = deconv1, 81 | shape = (input_var.shape[0], 1, 1, 3 * image_size * image_size), 82 | ) 83 | 84 | grey_with_deconv1_flatten = layers.ConcatLayer( 85 | incomings = [deconv1_flatten, greyscale_input], 86 | axis = 1, 87 | ) 88 | 89 | output = layers.DenseLayer( 90 | incoming = grey_with_deconv1_flatten, 91 | num_units = 3 * image_size * image_size, 92 | nonlinearity = lasagne.nonlinearities.linear, 93 | ) 94 | 95 | output = layers.ReshapeLayer( 96 | incoming = deconv1, 97 | shape = input_var.shape 98 | ) 99 | 100 | return (output, layers.get_output(output)) 101 | 102 | 103 | def get_cost_updates(network, input_var, output, learning_rate, **kwargs): 104 | """ Defines and returns cost and updates of the network 105 | 106 | -output can be different from layers.get_output(network), because train 107 | and validation networks may differ from each other 108 | 109 | -Use **kwargs to pass model specific parameters 110 | """ 111 | 112 | params = layers.get_all_params(network, trainable = True) 113 | 114 | batch_size = input_var.shape[0] 115 | flat_input = input_var.reshape((batch_size, 3072)) 116 | flat_output= output.reshape((batch_size, 3072)) 117 | 118 | # cross entrophy loss 119 | losses = -T.sum(flat_input * T.log(flat_output) + (1 - flat_input) * T.log(1 - flat_output), axis = 1) 120 | 121 | # add saturation loss 122 | #saturation = -T.sum(T.std(T.reshape(flat_output, (batch_size, 3, 1024)), axis = 1), axis = 1) 123 | #losses = losses + 0.2 * saturation 124 | 125 | cost = T.mean(losses) 126 | gradients = T.grad(cost, params) 127 | 128 | # stochastic gradient descent 129 | #updates = [ 130 | # (param, param - learning_rate * gradient) 131 | # for param, gradient in zip(params, gradients) 132 | #] 133 | 134 | # rmsprop 135 | #updates = lasagne.updates.rmsprop(gradients, params, learning_rate = learning_rate) 136 | 137 | # momentum 138 | updates = lasagne.updates.nesterov_momentum(gradients, params, learning_rate = learning_rate) 139 | 140 | return (cost, updates) -------------------------------------------------------------------------------- /definitions/conv_50_5_2-mpool_2-conv_50_5_2-mpool_2-dense_1000-rg-nd-eu-tanh.py: -------------------------------------------------------------------------------- 1 | import theano.tensor as T 2 | import lasagne 3 | import lasagne.layers as layers 4 | import our_layers 5 | 6 | 7 | def define_model(input_var, **kwargs): 8 | """ Defines the model and returns (network, validation network output) 9 | 10 | -Return layers.get_output(final_layer_name) if validation network output and 11 | train network output are the same 12 | 13 | -For example, return layers.get_output(final_layer_name, deterministic = true) 14 | if there is a dropout layer 15 | 16 | -Use **kwargs to pass model specific parameters 17 | """ 18 | 19 | conv1_filter_count = 50 20 | conv1_filter_size = 5 21 | pool1_size = 2 22 | 23 | conv2_filter_count = 50 24 | conv2_filter_size = 5 25 | pool2_size = 2 26 | 27 | n_dense_units = 1000 28 | 29 | batch_size = input_var.shape[0] 30 | image_size = 32 31 | # note: ignore_border is True by defualt, so rounding down 32 | after_pool1 = image_size // pool1_size 33 | after_pool2 = after_pool1 // pool2_size 34 | 35 | input = layers.InputLayer( 36 | shape = (None, 3, image_size, image_size), 37 | input_var = input_var 38 | ) 39 | 40 | greyscale_input = our_layers.GreyscaleLayer( 41 | incoming = input, 42 | random_greyscale = True, 43 | ) 44 | 45 | conv1 = layers.Conv2DLayer( 46 | incoming = greyscale_input, 47 | num_filters = conv1_filter_count, 48 | filter_size = conv1_filter_size, 49 | stride = 1, 50 | pad = 'same', 51 | nonlinearity = lasagne.nonlinearities.tanh, 52 | ) 53 | 54 | pool1 = layers.MaxPool2DLayer( 55 | incoming = conv1, 56 | pool_size = pool1_size, 57 | stride = pool1_size, 58 | ) 59 | 60 | conv2 = layers.Conv2DLayer( 61 | incoming = pool1, 62 | num_filters = conv2_filter_count, 63 | filter_size = conv2_filter_size, 64 | stride = 1, 65 | pad = 'same', 66 | nonlinearity = lasagne.nonlinearities.tanh, 67 | ) 68 | 69 | pool2 = layers.MaxPool2DLayer( 70 | incoming = conv2, 71 | pool_size = pool2_size, 72 | stride = pool2_size, 73 | ) 74 | 75 | dense1 = layers.DenseLayer( 76 | incoming = pool2, 77 | num_units = n_dense_units, 78 | nonlinearity = lasagne.nonlinearities.tanh, 79 | ) 80 | 81 | pre_unpool2 = layers.DenseLayer( 82 | incoming = dense1, 83 | num_units = conv2_filter_count * (after_pool2 ** 2), 84 | nonlinearity = lasagne.nonlinearities.tanh, 85 | ) 86 | 87 | pre_unpool2 = layers.ReshapeLayer( 88 | incoming = pre_unpool2, 89 | shape = (batch_size, conv2_filter_count) + (after_pool2, after_pool2), 90 | ) 91 | 92 | unpool2 = our_layers.Unpool2DLayer( 93 | incoming = pre_unpool2, 94 | kernel_size = pool2_size, 95 | ) 96 | 97 | deconv2 = layers.Conv2DLayer( 98 | incoming = unpool2, 99 | num_filters = conv1_filter_count, 100 | filter_size = conv2_filter_size, 101 | stride = 1, 102 | pad = 'same', 103 | nonlinearity = lasagne.nonlinearities.tanh, 104 | ) 105 | 106 | unpool1 = our_layers.Unpool2DLayer( 107 | incoming = deconv2, 108 | kernel_size = pool1_size, 109 | ) 110 | 111 | deconv1 = layers.Conv2DLayer( 112 | incoming = unpool1, 113 | num_filters = 3, 114 | filter_size = conv1_filter_size, 115 | stride = 1, 116 | pad = 'same', 117 | nonlinearity = lasagne.nonlinearities.tanh, 118 | ) 119 | 120 | output = layers.ReshapeLayer( 121 | incoming = deconv1, 122 | shape = input_var.shape 123 | ) 124 | 125 | return (output, layers.get_output(output)) 126 | 127 | 128 | def get_cost_updates(network, input_var, output, learning_rate, **kwargs): 129 | """ Defines and returns cost and updates of the network 130 | 131 | -output can be different from layers.get_output(network), because train 132 | and validation networks may differ from each other 133 | 134 | -Use **kwargs to pass model specific parameters 135 | """ 136 | 137 | params = layers.get_all_params(network, trainable = True) 138 | 139 | batch_size = input_var.shape[0] 140 | flat_input = input_var.reshape((batch_size, 3072)) 141 | flat_output= output.reshape((batch_size, 3072)) 142 | 143 | # scale flat_output to [0, 1] 144 | flat_output = (flat_output + 1) / 2 145 | 146 | # cross entrophy loss 147 | #losses = -T.sum(flat_input * T.log(flat_output) + (1 - flat_input) * T.log(1 - flat_output), axis = 1) 148 | 149 | # euclidean loss 150 | losses = T.sum((flat_input - flat_output) ** 2, axis = 1) 151 | 152 | # add saturation loss 153 | #saturation = -T.sum(T.std(T.reshape(flat_output, (batch_size, 3, 1024)), axis = 1), axis = 1) 154 | #losses = losses + 0.2 * saturation 155 | 156 | cost = T.mean(losses) 157 | gradients = T.grad(cost, params) 158 | 159 | # stochastic gradient descent 160 | #updates = [ 161 | # (param, param - learning_rate * gradient) 162 | # for param, gradient in zip(params, gradients) 163 | #] 164 | 165 | # rmsprop 166 | #updates = lasagne.updates.rmsprop(gradients, params, learning_rate = learning_rate) 167 | 168 | # momentum 169 | updates = lasagne.updates.nesterov_momentum(gradients, params, learning_rate = learning_rate) 170 | 171 | return (cost, updates) 172 | -------------------------------------------------------------------------------- /definitions/conv_50_5_2-mpool_2-dense_500-rg-ec.py: -------------------------------------------------------------------------------- 1 | import theano.tensor as T 2 | import lasagne 3 | import lasagne.layers as layers 4 | import our_layers 5 | 6 | 7 | def define_model(input_var, **kwargs): 8 | """ Defines the model and returns (network, validation network output) 9 | 10 | -Return layers.get_output(final_layer_name) if validation network output and 11 | train network output are the same 12 | 13 | -For example, return layers.get_output(final_layer_name, deterministic = true) 14 | if there is a dropout layer 15 | 16 | -Use **kwargs to pass model specific parameters 17 | """ 18 | 19 | image_size = 32 20 | conv_filter_count = 20 21 | conv_filter_size = 5 22 | pool_size = 2 23 | n_dense_units = 300 24 | 25 | input = layers.InputLayer( 26 | shape = (None, 3, image_size, image_size), 27 | input_var = input_var 28 | ) 29 | 30 | greyscale_input = our_layers.GreyscaleLayer( 31 | incoming = input, 32 | random_greyscale = True, 33 | ) 34 | 35 | conv1 = layers.Conv2DLayer( 36 | incoming = greyscale_input, 37 | num_filters = conv_filter_count, 38 | filter_size = conv_filter_size, 39 | stride = 1, 40 | nonlinearity = lasagne.nonlinearities.sigmoid, 41 | ) 42 | 43 | pool1 = layers.MaxPool2DLayer( 44 | incoming = conv1, 45 | pool_size = pool_size, 46 | stride = pool_size, 47 | ) 48 | 49 | dense1 = layers.DenseLayer( 50 | incoming = pool1, 51 | num_units = n_dense_units, 52 | nonlinearity = lasagne.nonlinearities.rectify, 53 | ) 54 | 55 | pre_unpool1 = layers.DenseLayer( 56 | incoming = dense1, 57 | num_units = conv_filter_count * (image_size + conv_filter_size - 1) ** 2 / (pool_size * pool_size), 58 | nonlinearity = lasagne.nonlinearities.linear, 59 | ) 60 | 61 | pre_unpool1 = layers.ReshapeLayer( 62 | incoming = pre_unpool1, 63 | shape = (input_var.shape[0], conv_filter_count) + ((image_size + conv_filter_size - 1) / 2, (image_size + conv_filter_size - 1) / 2), 64 | ) 65 | 66 | unpool1 = our_layers.Unpool2DLayer( 67 | incoming = pre_unpool1, 68 | kernel_size = 2, 69 | ) 70 | 71 | deconv1 = layers.Conv2DLayer( 72 | incoming = unpool1, 73 | num_filters = 3, 74 | filter_size = conv_filter_size, 75 | stride = 1, 76 | nonlinearity = lasagne.nonlinearities.sigmoid, 77 | ) 78 | 79 | deconv1_flatten = layers.ReshapeLayer( 80 | incoming = deconv1, 81 | shape = (input_var.shape[0], 1, 1, 3 * image_size * image_size), 82 | ) 83 | 84 | grey_with_deconv1_flatten = layers.ConcatLayer( 85 | incomings = [deconv1_flatten, greyscale_input], 86 | axis = 1, 87 | ) 88 | 89 | output = layers.DenseLayer( 90 | incoming = grey_with_deconv1_flatten, 91 | num_units = 3 * image_size * image_size, 92 | nonlinearity = lasagne.nonlinearities.linear, 93 | ) 94 | 95 | output = layers.ReshapeLayer( 96 | incoming = deconv1, 97 | shape = input_var.shape 98 | ) 99 | 100 | return (output, layers.get_output(output)) 101 | 102 | 103 | def get_cost_updates(network, input_var, output, learning_rate, **kwargs): 104 | """ Defines and returns cost and updates of the network 105 | 106 | -output can be different from layers.get_output(network), because train 107 | and validation networks may differ from each other 108 | 109 | -Use **kwargs to pass model specific parameters 110 | """ 111 | 112 | params = layers.get_all_params(network, trainable = True) 113 | 114 | batch_size = input_var.shape[0] 115 | flat_input = input_var.reshape((batch_size, 3072)) 116 | flat_output= output.reshape((batch_size, 3072)) 117 | 118 | # cross entrophy loss 119 | losses = -T.sum(flat_input * T.log(flat_output) + (1 - flat_input) * T.log(1 - flat_output), axis = 1) 120 | 121 | # add saturation loss 122 | #saturation = -T.sum(T.std(T.reshape(flat_output, (batch_size, 3, 1024)), axis = 1), axis = 1) 123 | #losses = losses + 0.2 * saturation 124 | 125 | cost = T.mean(losses) 126 | gradients = T.grad(cost, params) 127 | 128 | # stochastic gradient descent 129 | #updates = [ 130 | # (param, param - learning_rate * gradient) 131 | # for param, gradient in zip(params, gradients) 132 | #] 133 | 134 | # rmsprop 135 | #updates = lasagne.updates.rmsprop(gradients, params, learning_rate = learning_rate) 136 | 137 | # momentum 138 | updates = lasagne.updates.nesterov_momentum(gradients, params, learning_rate = learning_rate) 139 | 140 | return (cost, updates) 141 | -------------------------------------------------------------------------------- /definitions/conv_50_5_2-mpool_2-dense_500-rg.py: -------------------------------------------------------------------------------- 1 | import theano.tensor as T 2 | import lasagne 3 | import lasagne.layers as layers 4 | import our_layers 5 | 6 | 7 | def define_model(input_var, **kwargs): 8 | """ Defines the model and returns (network, validation network output) 9 | 10 | -Return layers.get_output(final_layer_name) if validation network output and 11 | train network output are the same 12 | 13 | -For example, return layers.get_output(final_layer_name, deterministic = true) 14 | if there is a dropout layer 15 | 16 | -Use **kwargs to pass model specific parameters 17 | """ 18 | 19 | image_size = 32 20 | conv_filter_count = 50 21 | conv_filter_size = 5 22 | pool_size = 2 23 | n_dense_units = 500 24 | 25 | input = layers.InputLayer( 26 | shape = (None, 3, image_size, image_size), 27 | input_var = input_var 28 | ) 29 | 30 | greyscale_input = our_layers.GreyscaleLayer( 31 | incoming = input, 32 | random_greyscale = True, 33 | ) 34 | 35 | conv1 = layers.Conv2DLayer( 36 | incoming = greyscale_input, 37 | num_filters = conv_filter_count, 38 | filter_size = conv_filter_size, 39 | stride = 1, 40 | nonlinearity = lasagne.nonlinearities.sigmoid, 41 | ) 42 | 43 | pool1 = layers.MaxPool2DLayer( 44 | incoming = conv1, 45 | pool_size = pool_size, 46 | stride = pool_size, 47 | ) 48 | 49 | dense1 = layers.DenseLayer( 50 | incoming =pool1, 51 | num_units = n_dense_units, 52 | nonlinearity = lasagne.nonlinearities.rectify, 53 | ) 54 | 55 | pre_unpool1 = layers.DenseLayer( 56 | incoming = dense1, 57 | num_units = conv_filter_count * (image_size + conv_filter_size - 1) ** 2 / (pool_size * pool_size), 58 | nonlinearity = lasagne.nonlinearities.linear, 59 | ) 60 | 61 | pre_unpool1 = layers.ReshapeLayer( 62 | incoming = pre_unpool1, 63 | shape = (input_var.shape[0], conv_filter_count) + ((image_size + conv_filter_size - 1) / 2, (image_size + conv_filter_size - 1) / 2), 64 | ) 65 | 66 | unpool1 = our_layers.Unpool2DLayer( 67 | incoming = pre_unpool1, 68 | kernel_size = 2, 69 | ) 70 | 71 | deconv1 = layers.Conv2DLayer( 72 | incoming = unpool1, 73 | num_filters = 3, 74 | filter_size = conv_filter_size, 75 | stride = 1, 76 | nonlinearity = lasagne.nonlinearities.sigmoid, 77 | ) 78 | 79 | output = layers.ReshapeLayer( 80 | incoming = deconv1, 81 | shape = input_var.shape 82 | ) 83 | 84 | return (output, layers.get_output(output)) 85 | 86 | 87 | def get_cost_updates(network, input_var, output, learning_rate, **kwargs): 88 | """ Defines and returns cost and updates of the network 89 | 90 | -output can be different from layers.get_output(network), because train 91 | and validation networks may differ from each other 92 | 93 | -Use **kwargs to pass model specific parameters 94 | """ 95 | 96 | params = layers.get_all_params(network, trainable = True) 97 | 98 | batch_size = input_var.shape[0] 99 | flat_input = input_var.reshape((batch_size, 3072)) 100 | flat_output= output.reshape((batch_size, 3072)) 101 | 102 | # cross entrophy loss 103 | losses = -T.sum(flat_input * T.log(flat_output) + (1 - flat_input) * T.log(1 - flat_output), axis = 1) 104 | 105 | # add saturation loss 106 | #saturation = -T.sum(T.std(T.reshape(flat_output, (batch_size, 3, 1024)), axis = 1), axis = 1) 107 | #losses = losses + 0.2 * saturation 108 | 109 | cost = T.mean(losses) 110 | gradients = T.grad(cost, params) 111 | 112 | # stochastic gradient descent 113 | #updates = [ 114 | # (param, param - learning_rate * gradient) 115 | # for param, gradient in zip(params, gradients) 116 | #] 117 | 118 | # rmsprop 119 | #updates = lasagne.updates.rmsprop(gradients, params, learning_rate = learning_rate) 120 | 121 | # momentum 122 | updates = lasagne.updates.nesterov_momentum(gradients, params, learning_rate = learning_rate) 123 | 124 | return (cost, updates) 125 | -------------------------------------------------------------------------------- /definitions/flags: -------------------------------------------------------------------------------- 1 | rg - random_greyscale 2 | ec - extra connection from greyscale to output 3 | wd - weight decay 4 | nd - new decoder (conv/deconv layers don't change image size) 5 | eu - euclidean loss 6 | tanh - tanh activation, default is sigmoid 7 | dr - dropout -------------------------------------------------------------------------------- /load_data.py: -------------------------------------------------------------------------------- 1 | import cPickle as pickle 2 | import numpy as np 3 | 4 | def load_cifar_train_data(): 5 | """ 6 | train_images format is: (index, channel, height, width) 7 | train_labels format is: (index, label) 8 | """ 9 | train_images = [] 10 | train_labels = np.array([]) 11 | 12 | for iter in range(1, 6): 13 | batch_file_name = 'data/cifar-10-batches-py/data_batch_' + str(iter) 14 | batch_file = open(batch_file_name, 'r') 15 | dict = pickle.load(batch_file) 16 | batch_file.close() 17 | 18 | batch_images = dict['data'] 19 | batch_labels = dict['labels'] 20 | 21 | train_images.append(batch_images) 22 | train_labels = np.append(train_labels, batch_labels) 23 | 24 | train_images = np.vstack(train_images) / 256.0 25 | train_images = train_images.reshape(train_images.shape[0], 3, 32, 32) 26 | 27 | return (train_images, train_labels) 28 | 29 | def load_cifar_val_data(): 30 | """ 31 | val_images format is: (index, channel, height, width) 32 | val_labels format is: (index, label) 33 | """ 34 | file_name = 'data/cifar-10-batches-py/test_batch' 35 | file = open(file_name, 'r') 36 | dict = pickle.load(file) 37 | file.close() 38 | 39 | val_images = dict['data'] / 256.0 40 | val_images = val_images.reshape(val_images.shape[0], 3, 32, 32) 41 | val_labels = np.array(dict['labels']) 42 | 43 | return (val_images, val_labels) -------------------------------------------------------------------------------- /logs/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore 3 | -------------------------------------------------------------------------------- /models/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore -------------------------------------------------------------------------------- /our_layers.py: -------------------------------------------------------------------------------- 1 | import lasagne 2 | from lasagne import layers 3 | import theano.tensor as T 4 | from our_utils import get_greyscale 5 | 6 | class Unpool2DLayer(layers.Layer): 7 | """ 8 | This layer performs unpooling over the last two dimensions 9 | of a 4D tensor. 10 | """ 11 | def __init__(self, incoming, kernel_size, 12 | nonlinearity = lasagne.nonlinearities.linear, **kwargs): 13 | """ 14 | note: kernel_size = stride and kernel is square for simplicity 15 | kernel_size must be int 16 | """ 17 | super(Unpool2DLayer, self).__init__(incoming, **kwargs) 18 | self.kernel_size = kernel_size 19 | self.nonlinearity = nonlinearity 20 | 21 | 22 | def get_output_shape_for(self, input_shape): 23 | output_shape = list(input_shape) 24 | output_shape[2] = input_shape[2] * self.kernel_size 25 | output_shape[3] = input_shape[3] * self.kernel_size 26 | return tuple(output_shape) 27 | 28 | 29 | def get_output_for(self, input, **kwargs): 30 | input_shape = input.shape 31 | output_shape = self.get_output_shape_for(input_shape) 32 | return self.nonlinearity(input.repeat(self.kernel_size, axis = 2).repeat(self.kernel_size, axis = 3)) 33 | 34 | 35 | class GreyscaleLayer(layers.Layer): 36 | """ 37 | This layer calculates the greyscale of the input image 38 | """ 39 | def __init__(self, incoming, random_greyscale = False, random_seed = 123, **kwargs): 40 | """ 41 | input for this layer is 4D tensor 42 | incoming is considered to have shape : (index, channel, height, width) 43 | """ 44 | 45 | super(GreyscaleLayer, self).__init__(incoming, **kwargs) 46 | 47 | self.rng = T.shared_randomstreams.RandomStreams(random_seed) 48 | self.random_greyscale = random_greyscale 49 | 50 | 51 | def get_output_shape_for(self, input_shape): 52 | output_shape = list(input_shape) 53 | output_shape[1] = 1 54 | return tuple(output_shape) 55 | 56 | 57 | def get_output_for(self, input, deterministic_greyscale = False, **kwargs): 58 | """ 59 | 'deterministic_greyscale' is designed to allow using deterministic 60 | greyscale during validation 61 | """ 62 | if (not deterministic_greyscale): 63 | return get_greyscale(input, self.random_greyscale, self.rng) 64 | return get_greyscale(input, False, self.rng) -------------------------------------------------------------------------------- /our_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import PIL.Image as Image 4 | import theano.tensor as T 5 | import lasagne.layers as layers 6 | import cPickle as pickle 7 | from deeplearningnet_utils import tile_raster_images 8 | 9 | 10 | def save_image_from_array(arr, file_name = 'image.png', directory = 'plots'): 11 | """ 12 | arr must be numpy.ndarray : (channel, height, width) or (1, channel, height, width) 13 | there can be 3 channels or 1 channel 14 | """ 15 | 16 | if (arr.dtype != 'uint8'): 17 | arr = np.array(np.floor(arr * 256), dtype = 'uint8') 18 | 19 | # if (1, C, H, W) 20 | if (len(arr.shape) == 4): 21 | arr = arr.reshape((arr.shape[1], arr.shape[2], arr.shape[3])) 22 | 23 | # if greyscale 24 | if (arr.shape[0] == 1): 25 | arr = np.vstack([arr, arr, arr]) 26 | 27 | arr = np.transpose(arr, (1, 2, 0)) 28 | img = Image.fromarray(arr) 29 | 30 | if (not os.path.exists(directory)): 31 | os.makedirs(directory) 32 | img.save(directory + '/' + file_name) 33 | 34 | 35 | def get_greyscale(images, random_greyscale = False, rng = None, rgb = False): 36 | """ 37 | images is 4D array of images (4D tensor or np.ndarray): (index, channel, height, width) 38 | if use_random_params is True must provide rng 39 | 40 | if rgb=True the output will have 3 channels, otherwise 1 channel 41 | 42 | note: when images is T.tensor4 rng must be theano.tensor.RandomStreams 43 | np.array rng must be numpy.random 44 | """ 45 | 46 | cr = 0.29 47 | cg = 0.6 48 | cb = 0.11 49 | 50 | if (random_greyscale): 51 | cr = rng.uniform(low = 0.1, high = 0.8) 52 | cg = rng.uniform(low = 0.1, high = 0.9 - cr) 53 | cb = 1 - cr - cg 54 | 55 | 56 | cnt = images.shape[0] 57 | h = images.shape[2] 58 | w = images.shape[3] 59 | 60 | res = cr * images[0:cnt, 0:1, 0:h, 0:w] + \ 61 | cg * images[0:cnt, 1:2, 0:h, 0:w] + \ 62 | cb * images[0:cnt, 2:3, 0:h, 0:w] 63 | 64 | if (rgb): 65 | if (isinstance(images, np.ndarray)): 66 | res = np.concatenate((res, res, res), axis = 1) 67 | else: 68 | res = T.concatenate((res, res, res), axis = 1) 69 | 70 | return res 71 | 72 | 73 | def print_samples(images, forward, model_name, epoch, suffix = '', columns = 1, directory = 'plots'): 74 | """ 75 | images is an 4D array of images: (index, channel, height, width) 76 | forward is a theano.function which maps images to outputs 77 | note: images can be tensor4 or np.ndarray 78 | """ 79 | 80 | where = directory + "/samples_" + model_name 81 | if (not os.path.exists(where)): 82 | os.makedirs(where) 83 | 84 | if (not isinstance(images, np.ndarray)): 85 | images = np.array(images.eval()) 86 | 87 | # now images is np.ndarray 88 | n_images = images.shape[0] 89 | print "==> Printing %d images to %s" % (n_images, where) 90 | 91 | # we set random_greyscale flag false when showing greyscale images 92 | grey_images = get_greyscale(images, rgb = True) 93 | out_images = forward(images) 94 | 95 | all_images = np.array([]).reshape((0,) + images.shape[1:]) 96 | for index in xrange(n_images): 97 | all_images = np.concatenate([all_images, images[index:(index + 1)]], axis = 0) 98 | all_images = np.concatenate([all_images, grey_images[index:(index + 1)]], axis = 0) 99 | all_images = np.concatenate([all_images, out_images[index:(index + 1)]], axis = 0) 100 | 101 | R = all_images[..., 0, ..., ...] 102 | G = all_images[..., 1, ..., ...] 103 | B = all_images[..., 2, ..., ...] 104 | 105 | all_images = tile_raster_images( 106 | (R, G, B, None), 107 | (32, 32), 108 | (-(-len(images) // columns), 3 * columns), 109 | tile_spacing = (1, 1) 110 | ) 111 | image = Image.fromarray(all_images) 112 | 113 | image_name = "epoch %d" % epoch 114 | image.save(where + '/' + image_name + suffix + ".png") 115 | 116 | 117 | def plot_filters(filters, model_name, epoch, suffix = '', max_num_filters = 100, columns = 1, repeat = 5, directory = 'plots'): 118 | """ Plots filters of convolution layers 119 | 120 | filters is 4D nd.array (filter_index, channels, width, height), channels = 1 or 3 121 | one filters size = repeat * kernerl size 122 | """ 123 | 124 | where = directory + "/filters_" + model_name 125 | if (not os.path.exists(where)): 126 | os.makedirs(where) 127 | 128 | num_filters = min(filters.shape[0], max_num_filters) 129 | print "==> Printing %d images to %s" % (num_filters, where) 130 | 131 | filters = filters[0:num_filters] 132 | filters = filters.repeat(repeat, axis = 2).repeat(repeat, axis = 3) 133 | 134 | if (filters.shape[1] == 3): 135 | R = filters[..., 0, ..., ...] 136 | G = filters[..., 1, ..., ...] 137 | B = filters[..., 2, ..., ...] 138 | else: 139 | R = filters[..., 0, ..., ...] 140 | G = R 141 | B = R 142 | 143 | filters = tile_raster_images( 144 | (R, G, B, None), 145 | filters.shape[2:], 146 | (-(-filters.shape[0] // columns), columns), 147 | tile_spacing = (2, 2) 148 | ) 149 | image = Image.fromarray(filters) 150 | 151 | image_name = "epoch %d" % epoch 152 | image.save(where + '/' + image_name + suffix + ".png") 153 | 154 | 155 | def save_model(network, epoch, model_name, learning_rate = 0.0, directory = 'models'): 156 | """ Saves networks parameters, epoch, learning_rate 157 | """ 158 | params = layers.get_all_param_values(network) 159 | file_name = model_name + "-ep" + str(epoch) + ".pickle" 160 | file_path = directory + '/' + file_name 161 | print "==> Saving model to %s" % file_path 162 | 163 | if (not os.path.exists(directory)): 164 | os.makedirs(directory) 165 | 166 | with open(file_path, 'w') as save_file: 167 | pickle.dump( 168 | obj = { 169 | 'params' : params, 170 | 'epoch' : epoch, 171 | 'learning_rate' : learning_rate, 172 | }, 173 | file = save_file, 174 | protocol = -1 175 | ) 176 | 177 | 178 | def load_model(network, file_name, directory = 'models'): 179 | """ Loades saved network and returns dictonary with keys epoch, learning_rate 180 | 181 | Loades from root_dir/models/file_name 182 | """ 183 | file_path = directory + '/' + file_name 184 | print "==> Loading model from %s" % file_path 185 | 186 | with open(file_path, 'r') as load_file: 187 | dict = pickle.load(load_file) 188 | layers.set_all_param_values(network, dict['params']) 189 | 190 | return { 191 | 'epoch' : dict['epoch'], 192 | 'learning_rate' : dict['learning_rate'] 193 | } 194 | -------------------------------------------------------------------------------- /plots/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore 3 | !example.png 4 | 5 | -------------------------------------------------------------------------------- /plots/example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YerevaNN/neural-colorizer/1e3299b2f2b25f3bfb37bf642d78d9e0fdf5476b/plots/example.png -------------------------------------------------------------------------------- /train_convae.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import time 4 | import imp 5 | 6 | import numpy as np 7 | import theano 8 | import theano.tensor as T 9 | 10 | import lasagne 11 | import lasagne.layers as layers 12 | 13 | import our_utils 14 | from load_data import load_cifar_train_data, load_cifar_val_data 15 | from deeplearningnet_utils import tile_raster_images 16 | 17 | network_name = sys.argv[1] 18 | definition = imp.load_source('definition', 'definitions/%s.py' % network_name) 19 | 20 | def test_convae(batch_size = 100, 21 | n_epochs = 10000, 22 | save_iter = 50, 23 | info_iter = 20, 24 | val_iter = 10, 25 | log_dir = 'logs', 26 | info_dir = 'plots', 27 | save_dir = 'models', 28 | load_dir = 'models', 29 | load_model_name = '', 30 | ): 31 | 32 | # loading dataset 33 | train_set_x = load_cifar_train_data()[0].astype(theano.config.floatX) 34 | train_set_x = theano.shared(train_set_x, borrow = True) 35 | 36 | val_set_x = load_cifar_val_data()[0].astype(theano.config.floatX) 37 | val_set_x = theano.shared(val_set_x, borrow = True) 38 | print "==> Dataset is loaded" 39 | 40 | 41 | # compute number of minibatches for training, validation and testing 42 | n_train_batches = train_set_x.get_value(borrow = True).shape[0] / batch_size 43 | n_val_batches = val_set_x.get_value(borrow = True).shape[0] / batch_size 44 | print "==> Number of train batches: %d" % n_train_batches 45 | print "==> Number of val batches: %d" % n_val_batches 46 | 47 | 48 | # create input and target variables 49 | # input_var and target_var are 4 dimentional tensors: (index, channel, height, width) 50 | input_var = T.tensor4('X') 51 | target_var = T.tensor4('Y') 52 | index = T.lscalar('index') 53 | 54 | 55 | # build network 56 | # target_var is output of validation network and is defined in definition file 57 | # output for train network is always layers.get_output(network) 58 | # these two are seperated, because train and val networks may differ from each other 59 | network, target_var = definition.define_model(input_var) 60 | 61 | 62 | # learning rate, constant 0.001 if not specified 63 | if (hasattr(definition, 'lr_policy')): 64 | lr_policy = definition.lr_policy 65 | if (lr_policy == 'constant'): 66 | lr_base = definition.lr_base 67 | elif (lr_policy == 'geom'): 68 | lr_base = definition.lr_base 69 | lr_step = definition.lr_step 70 | lr_coef = definition.lr_coef 71 | else: 72 | raise Exception("Unknown lr_policy\nSupported: 'constant', 'geom'\n") 73 | else: 74 | lr_policy = 'constant' 75 | lr_base = 0.001 76 | 77 | learning_rate = theano.shared(np.float32(lr_base)) 78 | 79 | print "==> lr_policy = %s" % lr_policy 80 | if (lr_policy == 'constant'): 81 | print "\tbase_lr = %f" % lr_base 82 | lr_desc = "-lr_c%f" % lr_base 83 | elif (lr_policy == 'geom'): 84 | print "\tlr_base = %f" % lr_base 85 | print "\tlr_step = %d" % lr_step 86 | print "\tlr_coef = %f" % lr_coef 87 | lr_desc = "-lr_g_%f_%d_%f" % (lr_base, lr_step, lr_coef) 88 | 89 | 90 | # functions for train network 91 | train_cost, train_updates = definition.get_cost_updates( 92 | network = network, 93 | input_var = input_var, 94 | output = layers.get_output(network), 95 | learning_rate = learning_rate, 96 | ) 97 | 98 | train = theano.function( 99 | [index], 100 | train_cost, 101 | updates = train_updates, 102 | givens = { 103 | input_var: train_set_x[index * batch_size: (index + 1) * batch_size] 104 | } 105 | ) 106 | 107 | 108 | # functions for validation network 109 | # val_updates is a dummy variable 110 | forward = theano.function( 111 | [input_var], 112 | target_var, 113 | ) 114 | 115 | val_cost, val_updates = definition.get_cost_updates( 116 | network = network, 117 | input_var = input_var, 118 | output = target_var, 119 | learning_rate = learning_rate, 120 | ) 121 | 122 | validation = theano.function( 123 | [index], 124 | val_cost, 125 | givens = { 126 | input_var: val_set_x[index * batch_size: (index + 1) * batch_size] 127 | } 128 | ) 129 | 130 | 131 | # loading model 132 | if load_model_name == '': 133 | start_epoch = 0 134 | else: 135 | res = our_utils.load_model( 136 | network = network, 137 | file_name = load_model_name, 138 | directory = load_dir, 139 | ) 140 | start_epoch = res['epoch'] + 1 141 | # learning_rate = res['learning_rate'] 142 | 143 | 144 | # model_name 145 | model_name = network_name + lr_desc 146 | if (len(sys.argv) >= 3): 147 | # to allow run temporary models not changing log files 148 | model_name = sys.argv[2] 149 | print "==> network_name = %s" % network_name 150 | print "==> model_name = %s" % model_name 151 | 152 | 153 | # create log files 154 | if (not os.path.exists(log_dir)): 155 | os.makedirs(log_dir) 156 | 157 | train_loss_file = open(log_dir + '/log_train_' + model_name + '.csv', 'w') 158 | val_losss_file = open(log_dir + '/log_val_' + model_name + '.csv', 'w') 159 | 160 | 161 | # train 162 | print "==> Training has started" 163 | start_time = time.time() 164 | 165 | try: 166 | for epoch in xrange(start_epoch, n_epochs): 167 | 168 | costs = [] 169 | for batch_index in xrange(n_train_batches): 170 | costs.append(train(batch_index)) 171 | 172 | print >> train_loss_file, "%d, %f" % (epoch, np.mean(costs)) 173 | train_loss_file.flush() 174 | print "Training epoch %d took %.0fs, lr=%f, loss=%f" % (epoch, time.time() - start_time, learning_rate.eval(), np.mean(costs)) 175 | 176 | 177 | # validation 178 | if (epoch % val_iter == 1): 179 | costs = [] 180 | for batch_index in xrange(n_val_batches): 181 | costs.append(validation(batch_index)) 182 | 183 | print >> val_losss_file, "%d %f" % (epoch, np.mean(costs)) 184 | val_losss_file.flush() 185 | print "==> Validation loss = %f" % np.mean(costs) 186 | 187 | 188 | # learning rate policy 189 | if (lr_policy == 'geom'): 190 | if (epoch > start_epoch + 1 and epoch % lr_step == 1): 191 | learning_rate *= lr_coef 192 | 193 | 194 | # save 195 | if (epoch % save_iter == 1 and epoch > start_epoch + 1): 196 | our_utils.save_model( 197 | network = network, 198 | epoch = epoch, 199 | model_name = model_name, 200 | learning_rate = learning_rate.eval(), 201 | directory = save_dir, 202 | ) 203 | 204 | 205 | # info 206 | if (epoch % info_iter == 1): 207 | our_utils.print_samples( 208 | images = T.concatenate([train_set_x[0:75], val_set_x[0:75]], axis = 0), 209 | forward = forward, 210 | model_name = model_name, 211 | epoch = epoch, 212 | columns = 5, 213 | directory = info_dir, 214 | ) 215 | our_utils.plot_filters( 216 | filters = layers.get_all_param_values(network)[0], 217 | model_name = model_name, 218 | epoch = epoch, 219 | repeat = 10, 220 | columns = 8, 221 | directory = info_dir, 222 | ) 223 | 224 | start_time = time.time() 225 | 226 | except KeyboardInterrupt: 227 | answer = raw_input("\nWould you like to save the model ? (y / n) ") 228 | if (answer in ["y", "Y", "yes"]): 229 | our_utils.save_model( 230 | network = network, 231 | epoch = epoch, 232 | model_name = model_name, 233 | learning_rate = learning_rate.eval(), 234 | directory = save_dir, 235 | ) 236 | 237 | exit() 238 | 239 | 240 | if __name__ == '__main__': 241 | if (len(sys.argv) <= 1): 242 | sys.exit("Usage: convae.py []") 243 | 244 | test_convae( 245 | batch_size = 100, 246 | load_model_name = '', 247 | info_iter = 20, 248 | val_iter = 10, 249 | save_iter = 100, 250 | ) --------------------------------------------------------------------------------