├── CNN ├── CNN_Test.m ├── MaxPooling.cpp ├── MaxPooling.mexw64 ├── ReLu.m ├── cnnapplygrads.m ├── cnnbp.m ├── cnnff.m ├── cnnnumgradcheck.m ├── cnnsetup.m ├── cnntest.m ├── cnntrain.m ├── expand.m ├── expansionData.m ├── flipall.m ├── loadMNISTImages.m ├── loadMNISTLabels.m ├── mnist_uint8.mat ├── sigm.m ├── softmax.m ├── t10k-images.idx3-ubyte ├── t10k-labels.idx1-ubyte ├── train-images.idx3-ubyte ├── train-labels.idx1-ubyte └── workspace_40.mat └── README.md /CNN/CNN_Test.m: -------------------------------------------------------------------------------- 1 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 2 | % Tutor : PierreHao (Hao ZHANG) 3 | % Company : NanJing Qingsou 4 | % Note : if you publish this code, do not forget @me 5 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 6 | 7 | clear all 8 | close all 9 | %{ 10 | clc; 11 | clear all 12 | load mnist_uint8; 13 | 14 | train_x = double(reshape(train_x',28,28,60000))/255; 15 | test_x = double(reshape(test_x',28,28,10000))/255; 16 | train_y = double(train_y'); 17 | test_y = double(test_y'); 18 | %} 19 | filenameTrain = 'train-images.idx3-ubyte'; 20 | filenameTrainLabel = 'train-labels.idx1-ubyte'; 21 | filenameTest = 't10k-images.idx3-ubyte'; 22 | filenameTestLabel = 't10k-labels.idx1-ubyte'; 23 | 24 | % load MNIST 25 | train_x = loadMNISTImages(filenameTrain); 26 | test_x = loadMNISTImages(filenameTest); 27 | train_y = loadMNISTLabels(filenameTrainLabel); 28 | test_y = loadMNISTLabels(filenameTestLabel); 29 | 30 | % normalization 0.47 31 | train_x = double(train_x)/255; 32 | test_x = double(test_x)/255; 33 | train_y = double(train_y); 34 | test_y = double(test_y); 35 | % expand to 32*32 36 | train_x = expansionData(train_x); 37 | test_x = expansionData(test_x); 38 | 39 | %% ex1 Train a 6c-2s-12c-2s Convolutional neural network 40 | %will run 1 epoch in about 200 second and get around 11.30% error. 41 | %With 100 epochs you'll get around 1.2% error 42 | 43 | rand('state',0) 44 | 45 | cnn.layers = { 46 | struct('type', 'i') %input layer 47 | struct('type', 'c', 'outputmaps', 6, 'kernelsize', 5) %convolution layer outputmaps: numbers of filters 48 | struct('type', 's', 'scale', 2) %sub sampling layer 49 | struct('type', 'c', 'outputmaps', 12, 'kernelsize', 5) %convolution layer 50 | struct('type', 's', 'scale', 2) %subsampling layer 51 | %struct('type', 'c', 'outputmaps', 120, 'kernelsize', 5) %0.0388 52 | %struct('type', 's', 'scale', 2) %subsampling layer 53 | }; 54 | 55 | 56 | opts.alpha = 1; 57 | opts.batchsize = 50; 58 | opts.numepochs = 1; 59 | opts.stepsize = 10; 60 | cnn.activation = 'Sigmoid'; % now we have Relu and Sigmoid activation functions 61 | cnn.pooling_mode = 'Mean'; %now we have Mean and Max pooling 62 | cnn.output = 'Softmax';% noe we have Softmax and Sigmoid output function 63 | opts.iteration = 1; 64 | cnn = cnnsetup(cnn, train_x, train_y); 65 | for i = 1 : opts.iteration 66 | cnn = cnntrain(cnn, train_x, train_y, opts); 67 | [er, bad] = cnntest(cnn, test_x, test_y); 68 | fprintf('%d iterations and rate of error : %d\n',i,er); 69 | %[er1, bad1] = cnntest(cnn, val_x, val_Label); 70 | %fprintf('%d iterations and rate of error (validation) : %d\n',i,er1); 71 | if mod(i,opts.stepsize) == 0 72 | opts.alpha = opts.alpha/10;%change learning rate 73 | end 74 | end 75 | %[er, bad] = cnntest(cnn, test_x, test_y); 76 | fprintf('Taux of error : %d\n',er(i)); 77 | %plot mean squared error 78 | figure; plot(cnn.rL); 79 | assert(er<0.12, 'Too big error'); 80 | -------------------------------------------------------------------------------- /CNN/MaxPooling.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * MaxPooling.c 3 | * 4 | * Implements the max-pooling transfer function. Takes a 4D tensor shaped as 5 | * (rows, cols, nchannels, nsamples) and a pooling shape as (prows, pcols) and 6 | * returns a set of max-values with the corresponding indices in the input 7 | * matrix. 8 | * 9 | * e.g. 10 | * [m, idx] = MaxPooling(IM, [2 2]) 11 | * 12 | * Created on: July 11, 2011 13 | * Author: Jonathan Masci 14 | * 15 | * This file is available under the terms of the GNU GPLv2. 16 | */ 17 | 18 | #include "mex.h" 19 | 20 | #ifdef OPENMP 21 | #include 22 | #endif 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | 29 | #define IDX2F(i,j,ld) ((((j)-1)*(ld))+((i)-1)) 30 | #define IDX2C(i,j,ld) (((j)*(ld))+(i)) 31 | 32 | int debug = 0; 33 | 34 | /** 35 | * Computes the max-pooling for the given 2D map, and no, the name is not a typo. 36 | * All pointers are passed already offset so to avoid cumbersome indexing. 37 | * 38 | * @param ptr_data pointer set to the begin of this map 39 | * @param DATA_DIMS data dimensions 40 | * @param ptr_pool pooling sizes 41 | * @param ptr_out pointer to the output max-values set to the right position 42 | * @param ptr_idx pointer to the output indices set to the right position 43 | */ 44 | template 45 | inline void compute_map_pooling(T *ptr_data, const mwSize *DATA_DIMS, T *ptr_pool, 46 | T *ptr_out, T *ptr_idx, int tile_start) 47 | { 48 | T m; 49 | int idx; 50 | int count = 0; 51 | 52 | for (int col = 0; col < DATA_DIMS[1]; col += ptr_pool[1]) { 53 | for (int row = 0; row < DATA_DIMS[0]; row += ptr_pool[0]) { 54 | if (debug) 55 | fprintf(stderr, "r = %i, c = %i \n", row, col); 56 | 57 | m = -std::numeric_limits::max(); 58 | idx = -1; 59 | for (int pcol = 0; (pcol < ptr_pool[1] && col + pcol < DATA_DIMS[1]); ++pcol) { 60 | for (int prow = 0; (prow < ptr_pool[0] && row + prow < DATA_DIMS[0]); ++prow) { 61 | if (debug) { 62 | fprintf(stderr, "m = %f, data = %f \n", m, ptr_data[IDX2C(row + prow, col + pcol, DATA_DIMS[0])]); 63 | fprintf(stderr, "rr = %i, cc = %i \n --> idx = %i \n", row + prow, col + pcol, idx); 64 | } 65 | 66 | if (ptr_data[IDX2C(row + prow, col + pcol, DATA_DIMS[0])] > m) { 67 | idx = IDX2C(row + prow, col + pcol, DATA_DIMS[0]); 68 | m = ptr_data[idx]; 69 | } 70 | } 71 | } 72 | 73 | if (debug && idx == -1) { 74 | fprintf(stderr, "dioschifoso\n"); 75 | return; 76 | } 77 | 78 | if (debug) 79 | fprintf(stderr, "count = %i\n",count); 80 | 81 | /* idxs are to be used in Matlab and hence a +1 is needed */ 82 | ptr_idx[count] = idx + 1 + tile_start; 83 | ptr_out[count] = m; 84 | count++; 85 | } 86 | } 87 | } 88 | 89 | /** 90 | * This is the wrapper for the actual computation. 91 | * It is a template so that multiple types can be handled. 92 | */ 93 | template 94 | void mexMaxPooling(int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[], mxClassID classID) 95 | { 96 | 97 | /***************************************************************************/ 98 | /** Variables */ 99 | /***************************************************************************/ 100 | mwSize IDX_DIMS[1]; 101 | mwSize DATA_DIMS[4]; 102 | mwSize M_DIMS[4]; 103 | const mwSize *POOL_DIMS; 104 | int DATA_NUMEL; 105 | int POOL_NUMEL; 106 | 107 | /** 108 | * Pointers to data 109 | */ 110 | T *ptr_data = NULL; 111 | T *ptr_pool = NULL; 112 | T *ptr_out = NULL; 113 | T *ptr_idx = NULL; 114 | 115 | /***************************************************************************/ 116 | /** Setting input pointers *************************************************/ 117 | /***************************************************************************/ 118 | ptr_data = (T *)mxGetData(prhs[0]); 119 | ptr_pool = (T *)mxGetData(prhs[1]); 120 | if (debug) 121 | fprintf(stderr,"Pooling size: h=%f, w=%f\n", ptr_pool[0], ptr_pool[1]); 122 | 123 | /***************************************************************************/ 124 | /** Setting parameters *****************************************************/ 125 | /***************************************************************************/ 126 | /* Data dimensions. As also a 2D tensor can be used I fill empty dimensions 127 | * with 1 */ 128 | const mwSize *tmp = mxGetDimensions(prhs[0]); 129 | DATA_DIMS[0] = tmp[0]; 130 | DATA_DIMS[1] = tmp[1]; 131 | if (mxGetNumberOfDimensions(prhs[0]) == 2) { 132 | DATA_DIMS[2] = 1; 133 | DATA_DIMS[3] = 1; 134 | } else if (mxGetNumberOfDimensions(prhs[0]) == 3) { 135 | DATA_DIMS[2] = tmp[2]; 136 | DATA_DIMS[3] = 1; 137 | } else { 138 | DATA_DIMS[2] = tmp[2]; 139 | DATA_DIMS[3] = tmp[3]; 140 | } 141 | 142 | DATA_NUMEL = DATA_DIMS[0] * DATA_DIMS[1] * DATA_DIMS[2] * DATA_DIMS[3]; 143 | if (debug) 144 | fprintf(stderr,"Data size: h=%d, w=%d, z=%d, n=%d (%d)\n", DATA_DIMS[0], DATA_DIMS[1], DATA_DIMS[2], DATA_DIMS[3], DATA_NUMEL); 145 | 146 | /* Output dimensions: the first output argument is of size equals to the input 147 | * whereas the second is of size equals to the number of pooled values. 148 | * Below there is ceil because also non complete tiles are considered when 149 | * input dims are not multiples of pooling dims. */ 150 | M_DIMS[0] = ceil(float(DATA_DIMS[0]) / float(ptr_pool[0])); 151 | M_DIMS[1] = ceil(float(DATA_DIMS[1]) / float(ptr_pool[1])); 152 | M_DIMS[2] = DATA_DIMS[2]; 153 | M_DIMS[3] = DATA_DIMS[3]; 154 | IDX_DIMS[0] = M_DIMS[0] * M_DIMS[1] * M_DIMS[2] * M_DIMS[3]; 155 | if (debug){ 156 | fprintf(stderr,"Each output image has (%d, %d) pooled values, " 157 | "IDXs size: h=%d \n", M_DIMS[0], M_DIMS[1], IDX_DIMS[0]); 158 | fprintf(stderr, "M size: h=%d, w=%d, z=%d, n=%d\n", M_DIMS[0], M_DIMS[1], M_DIMS[2], M_DIMS[3]); 159 | } 160 | 161 | /***************************************************************************/ 162 | /** Variables allocation ***************************************************/ 163 | /***************************************************************************/ 164 | /* OUTPUTS: max-values and corresponding indices */ 165 | plhs[0] = mxCreateNumericArray(4, M_DIMS, classID, mxREAL); 166 | ptr_out = (T *)mxGetData(plhs[0]); 167 | plhs[1] = mxCreateNumericArray(1, IDX_DIMS, classID, mxREAL); 168 | ptr_idx = (T *)mxGetData(plhs[1]); 169 | 170 | /***************************************************************************/ 171 | /** Compute max-pooling ****************************************************/ 172 | /***************************************************************************/ 173 | int tile_start = 0; 174 | int ptr_offset = 0; 175 | int M_sample_size = M_DIMS[0] * M_DIMS[1] * M_DIMS[2]; 176 | int D_sample_size = DATA_DIMS[0] * DATA_DIMS[1] * DATA_DIMS[2]; 177 | 178 | for (int n = 0; n < DATA_DIMS[3]; ++n) { 179 | #ifdef OPENMP 180 | #pragma omp parallel for 181 | #endif 182 | for (int k = 0; k < DATA_DIMS[2]; ++k) { 183 | tile_start = n * M_sample_size + k * M_DIMS[0] * M_DIMS[1]; 184 | ptr_offset = n * D_sample_size + k * DATA_DIMS[0] * DATA_DIMS[1]; 185 | 186 | compute_map_pooling (&ptr_data[ptr_offset], DATA_DIMS, ptr_pool, &ptr_out[tile_start], &ptr_idx[tile_start], ptr_offset); 187 | 188 | if (debug) 189 | fprintf(stderr, "tile_start: %i, ptr_offset: %i\n", tile_start, ptr_offset); 190 | } 191 | } 192 | } 193 | 194 | void mexFunction(int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[]) 195 | { 196 | /***************************************************************************/ 197 | /** Check input ************************************************************/ 198 | /***************************************************************************/ 199 | if (nrhs !=2) 200 | mexErrMsgTxt("Must have 2 input arguments: x, pooling_shape"); 201 | 202 | if (nlhs !=2) 203 | mexErrMsgTxt("Must have 2 output arguments ([max_value, idxs])"); 204 | 205 | if (mxIsComplex(prhs[0]) || !(mxIsClass(prhs[0],"single") || mxIsClass(prhs[0],"double"))) 206 | mexErrMsgTxt("Input data must be real, single/double type"); 207 | 208 | if (mxIsComplex(prhs[1]) || !(mxIsClass(prhs[1],"single") || mxIsClass(prhs[1],"double"))) 209 | mexErrMsgTxt("Pooling dimensions (rows, cols) must be real, single/double type"); 210 | 211 | if (mxGetNumberOfDimensions(prhs[0]) < 2) 212 | mexErrMsgTxt("Input data must have at least 2-dimensions (rows, cols, nchannels, nsamples) " 213 | "\nThe last two dimensions will be considered to be 1."); 214 | 215 | if (mxGetNumberOfDimensions(prhs[1]) != 2) 216 | mexErrMsgTxt("Pooling data must have 2-dimensions (prows, pcols)"); 217 | 218 | mxClassID classID = mxGetClassID(prhs[0]); 219 | 220 | /** This is mainly to avoid two typenames. Should not be a big usability issue. */ 221 | if (mxGetClassID(prhs[1]) != classID) 222 | mexErrMsgTxt("Input data and pooling need to be of the same type"); 223 | 224 | /***************************************************************************/ 225 | /** Switch for the supported data types */ 226 | /***************************************************************************/ 227 | if (classID == mxSINGLE_CLASS) { 228 | if (debug) 229 | fprintf(stderr, "Executing the single version\n"); 230 | 231 | mexMaxPooling(nlhs, plhs, nrhs, prhs, classID); 232 | } else if (classID == mxDOUBLE_CLASS) { 233 | if (debug) 234 | fprintf(stderr, "Executing the double version\n"); 235 | 236 | mexMaxPooling(nlhs, plhs, nrhs, prhs, classID); 237 | } 238 | } 239 | -------------------------------------------------------------------------------- /CNN/MaxPooling.mexw64: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PierreHao/Deep-Learning-ToolBox-CNN/18b8f0975cb3028678dc38a87e1d9fa909c5c254/CNN/MaxPooling.mexw64 -------------------------------------------------------------------------------- /CNN/ReLu.m: -------------------------------------------------------------------------------- 1 | function result = ReLu(x) 2 | result = max(0,x); 3 | end -------------------------------------------------------------------------------- /CNN/cnnapplygrads.m: -------------------------------------------------------------------------------- 1 | function net = cnnapplygrads(net, opts) 2 | for l = 2 : numel(net.layers) 3 | if strcmp(net.layers{l}.type, 'c') 4 | for j = 1 : numel(net.layers{l}.a) 5 | for ii = 1 : numel(net.layers{l - 1}.a) 6 | net.layers{l}.k{ii}{j} = net.layers{l}.k{ii}{j} - opts.alpha * net.layers{l}.dk{ii}{j}; 7 | end 8 | net.layers{l}.b{j} = net.layers{l}.b{j} - opts.alpha * net.layers{l}.db{j}; 9 | end 10 | end 11 | end 12 | 13 | net.ffW = net.ffW - opts.alpha * net.dffW; 14 | net.ffb = net.ffb - opts.alpha * net.dffb; 15 | end 16 | -------------------------------------------------------------------------------- /CNN/cnnbp.m: -------------------------------------------------------------------------------- 1 | function net = cnnbp(net, y) 2 | n = numel(net.layers); 3 | 4 | % error 5 | net.e = net.o - y; 6 | % loss function 7 | net.L = 1/2* sum(net.e(:) .^ 2) / size(net.e, 2); % mean of batchSize images 8 | %net.L = -sum(sum(y .* log(net.o))) / size(net.e, 2); 9 | % backprop deltas 10 | if strcmp(net.output,'Sigmoid') 11 | net.od = net.e .* (net.o .* (1 - net.o)); 12 | else 13 | net.od = net.e; 14 | end 15 | %if we use ReLU, the last layer is not ReLU 16 | net.fvd = (net.ffW' * net.od); % feature vector delta 17 | if strcmp(net.layers{n}.type, 'c') % only conv layers has sigm function 18 | switch net.activation 19 | case 'Sigmoid' 20 | net.fvd = net.fvd .* (net.fv .* (1 - net.fv)); 21 | case 'Relu' 22 | net.fvd = net.fvd.*(net.fv > 0); 23 | otherwise 24 | net.fvd = net.fvd .* (net.fv .* (1 - net.fv)); 25 | end 26 | end 27 | 28 | % reshape feature vector deltas into output map style 29 | sa = size(net.layers{n}.a{1}); 30 | fvnum = sa(1) * sa(2); 31 | for j = 1 : numel(net.layers{n}.a) 32 | net.layers{n}.d{j} = reshape(net.fvd(((j - 1) * fvnum + 1) : j * fvnum, :), sa(1), sa(2), sa(3)); 33 | end 34 | 35 | for l = (n - 1) : -1 : 1 36 | if strcmp(net.layers{l+1}.type, 's')%&& strcmp(net.layers{l}.type, 'c') % pay attention, we can add strcmp(net.layers{l}.type, 'c') 37 | for j = 1 : numel(net.layers{l}.a) 38 | if strcmp(net.pooling_mode,'Mean') 39 | switch net.activation 40 | case 'Sigmoid' 41 | net.layers{l}.d{j} = net.layers{l}.a{j} .* (1 - net.layers{l}.a{j}) .* (expand(net.layers{l + 1}.d{j}, [net.layers{l + 1}.scale net.layers{l + 1}.scale 1]) / net.layers{l + 1}.scale ^ 2); 42 | case 'Relu' 43 | net.layers{l}.d{j} =(expand(net.layers{l + 1}.d{j}, [net.layers{l + 1}.scale net.layers{l + 1}.scale 1]) / (net.layers{l + 1}.scale ^ 2)).*(net.layers{l}.a{j}>0); 44 | otherwise 45 | net.layers{l}.d{j} = net.layers{l}.a{j} .* (1 - net.layers{l}.a{j}) .* (expand(net.layers{l + 1}.d{j}, [net.layers{l + 1}.scale net.layers{l + 1}.scale 1]) / net.layers{l + 1}.scale ^ 2); 46 | end 47 | 48 | elseif strcmp(net.pooling_mode,'Max') 49 | switch net.activation 50 | case 'Sigmoid' 51 | up = zeros(size(net.layers{l}.a{1})); 52 | up(net.layers{l+1}.id{j}) = net.layers{l+1}.d{j}(:);% up is matrix of delta:d not a 53 | net.layers{l}.d{j} = net.layers{l}.a{j} .* (1 - net.layers{l}.a{j}) .* up; 54 | case 'Relu' 55 | up = zeros(size(net.layers{l}.a{1})); 56 | up(net.layers{l+1}.id{j}) = net.layers{l+1}.d{j}(:); 57 | net.layers{l}.d{j} =up.*(net.layers{l}.a{j}>0); 58 | otherwise 59 | up = zeros(size(net.layers{l}.a{1})); 60 | up(net.layers{l+1}.id{j}) = net.layers{l+1}.d{j}(:); 61 | net.layers{l}.d{j} = net.layers{l}.a{j} .* (1 - net.layers{l}.a{j}) .* up; 62 | end 63 | end 64 | end 65 | elseif strcmp(net.layers{l+1}.type, 'c')%&& strcmp(net.layers{l}.type, 's') 66 | for i = 1 : numel(net.layers{l}.a) 67 | z = zeros(size(net.layers{l}.a{1})); 68 | for j = 1 : numel(net.layers{l + 1}.a) 69 | z = z + convn(net.layers{l + 1}.d{j}, rot180(net.layers{l + 1}.k{i}{j}), 'full'); 70 | end 71 | net.layers{l}.d{i} = z; 72 | %{ 73 | %--------------------------dropout------------------------------- 74 | if net.dropoutFraction>0 75 | net.layers{l}.d{i} = net.layers{l}.d{i} .* net.layers{l}.dropOutMask{i}; 76 | end 77 | %----------------------------------------------------------------- 78 | %} 79 | end 80 | end 81 | end 82 | 83 | %% calc gradients 84 | for l = 2 : n 85 | if strcmp(net.layers{l}.type, 'c') 86 | for j = 1 : numel(net.layers{l}.a) 87 | for i = 1 : numel(net.layers{l - 1}.a) 88 | net.layers{l}.dk{i}{j} = convn(flipall(net.layers{l - 1}.a{i}), net.layers{l}.d{j}, 'valid') / size(net.layers{l}.d{j}, 3); 89 | end 90 | net.layers{l}.db{j} = sum(net.layers{l}.d{j}(:)) / size(net.layers{l}.d{j}, 3); 91 | end 92 | end 93 | end 94 | net.dffW = net.od * (net.fv)' / size(net.od, 2); 95 | net.dffb = mean(net.od, 2); 96 | 97 | function X = rot180(X) 98 | X = flipdim(flipdim(X, 1), 2); 99 | end 100 | end 101 | -------------------------------------------------------------------------------- /CNN/cnnff.m: -------------------------------------------------------------------------------- 1 | function net = cnnff(net, x) 2 | n = numel(net.layers); 3 | net.layers{1}.a{1} = x; % x: batchsize 4 | inputmaps = 1; 5 | 6 | for l = 2 : n % for each layer 7 | if strcmp(net.layers{l}.type, 'c') 8 | % !!below can probably be handled by insane matrix operations 9 | for j = 1 : net.layers{l}.outputmaps % for each output map 10 | % create temp output map 11 | z = zeros(size(net.layers{l - 1}.a{1}) - [net.layers{l}.kernelsize - 1 net.layers{l}.kernelsize - 1 0]);% size after convolve 12 | for i = 1 : inputmaps % for each input map 13 | %{ 14 | %-----------------------------dropout-------------------------------- 15 | if net.dropoutFraction > 0 16 | if net.testing == 1 17 | net.layers{l - 1}.a{i} = net.layers{l - 1}.a{i}.*(1 - net.dropoutFraction); 18 | else 19 | net.layers{l - 1}.dropOutMask{i} = (rand(size(net.layers{l - 1}.a{i}))>net.dropoutFraction); 20 | net.layers{l - 1}.a{i} =net.layers{l - 1}.a{i}.*net.layers{l - 1}.dropOutMask{i}; 21 | end 22 | end 23 | %------------------------------------------------------------------- 24 | %} 25 | % convolve with corresponding kernel and add to temp output map 26 | z = z + convn(net.layers{l - 1}.a{i}, net.layers{l}.k{i}{j}, 'valid');%k{i,j} filter of i iput and j output 27 | end 28 | % add bias, pass through nonlinearity 29 | switch net.activation 30 | case 'Sigmoid' 31 | net.layers{l}.a{j} = sigm(z + net.layers{l}.b{j}); 32 | case 'Relu' 33 | net.layers{l}.a{j} = ReLu(z + net.layers{l}.b{j}); 34 | otherwise 35 | net.layers{l}.a{j} = sigm(z + net.layers{l}.b{j}); 36 | end 37 | end 38 | % set number of input maps to this layers number of outputmaps 39 | inputmaps = net.layers{l}.outputmaps; 40 | elseif strcmp(net.layers{l}.type, 's') 41 | % downsample 42 | for j = 1 : inputmaps 43 | switch net.pooling_mode 44 | case 'Mean' 45 | z = convn(net.layers{l - 1}.a{j}, ones(net.layers{l}.scale) / (net.layers{l}.scale ^ 2), 'valid'); % !! replace with variable 46 | net.layers{l}.a{j} = z(1 : net.layers{l}.scale : end, 1 : net.layers{l}.scale : end, :); 47 | case 'Max' 48 | [net.layers{l}.a{j} net.layers{l}.id{j}]= MaxPooling(net.layers{l - 1}.a{j},[net.layers{l}.scale net.layers{l}.scale]); 49 | otherwise 50 | z = convn(net.layers{l - 1}.a{j}, ones(net.layers{l}.scale) / (net.layers{l}.scale ^ 2), 'valid'); % !! replace with variable 51 | net.layers{l}.a{j} = z(1 : net.layers{l}.scale : end, 1 : net.layers{l}.scale : end, :); 52 | end 53 | end 54 | end 55 | end 56 | 57 | % concatenate all end layer feature maps into vector 58 | net.fv = []; 59 | for j = 1 : numel(net.layers{n}.a) % 12 60 | sa = size(net.layers{n}.a{j}); 61 | net.fv = [net.fv; reshape(net.layers{n}.a{j}, sa(1) * sa(2), sa(3))]; 62 | end 63 | % feedforward into output perceptrons 64 | net.o = net.ffW * net.fv + repmat(net.ffb, 1, size(net.fv, 2)); 65 | if strcmp(net.output,'Sigmoid') 66 | net.o = sigm(net.ffW * net.fv + repmat(net.ffb, 1, size(net.fv, 2))); 67 | else 68 | net.o = softmax(net.o); 69 | end 70 | end 71 | -------------------------------------------------------------------------------- /CNN/cnnnumgradcheck.m: -------------------------------------------------------------------------------- 1 | function cnnnumgradcheck(net, x, y) 2 | epsilon = 1e-4; 3 | er = 1e-8; 4 | n = numel(net.layers); 5 | for j = 1 : numel(net.ffb) 6 | net_m = net; net_p = net; 7 | net_p.ffb(j) = net_m.ffb(j) + epsilon; 8 | net_m.ffb(j) = net_m.ffb(j) - epsilon; 9 | net_m = cnnff(net_m, x); net_m = cnnbp(net_m, y); 10 | net_p = cnnff(net_p, x); net_p = cnnbp(net_p, y); 11 | d = (net_p.L - net_m.L) / (2 * epsilon); 12 | e = abs(d - net.dffb(j)); 13 | if e > er 14 | error('numerical gradient checking failed'); 15 | end 16 | end 17 | 18 | for i = 1 : size(net.ffW, 1) 19 | for u = 1 : size(net.ffW, 2) 20 | net_m = net; net_p = net; 21 | net_p.ffW(i, u) = net_m.ffW(i, u) + epsilon; 22 | net_m.ffW(i, u) = net_m.ffW(i, u) - epsilon; 23 | net_m = cnnff(net_m, x); net_m = cnnbp(net_m, y); 24 | net_p = cnnff(net_p, x); net_p = cnnbp(net_p, y); 25 | d = (net_p.L - net_m.L) / (2 * epsilon); 26 | e = abs(d - net.dffW(i, u)); 27 | if e > er 28 | error('numerical gradient checking failed'); 29 | end 30 | end 31 | end 32 | 33 | for l = n : -1 : 2 34 | if strcmp(net.layers{l}.type, 'c') 35 | for j = 1 : numel(net.layers{l}.a) 36 | net_m = net; net_p = net; 37 | net_p.layers{l}.b{j} = net_m.layers{l}.b{j} + epsilon; 38 | net_m.layers{l}.b{j} = net_m.layers{l}.b{j} - epsilon; 39 | net_m = cnnff(net_m, x); net_m = cnnbp(net_m, y); 40 | net_p = cnnff(net_p, x); net_p = cnnbp(net_p, y); 41 | d = (net_p.L - net_m.L) / (2 * epsilon); 42 | e = abs(d - net.layers{l}.db{j}); 43 | if e > er 44 | error('numerical gradient checking failed'); 45 | end 46 | for i = 1 : numel(net.layers{l - 1}.a) 47 | for u = 1 : size(net.layers{l}.k{i}{j}, 1) 48 | for v = 1 : size(net.layers{l}.k{i}{j}, 2) 49 | net_m = net; net_p = net; 50 | net_p.layers{l}.k{i}{j}(u, v) = net_p.layers{l}.k{i}{j}(u, v) + epsilon; 51 | net_m.layers{l}.k{i}{j}(u, v) = net_m.layers{l}.k{i}{j}(u, v) - epsilon; 52 | net_m = cnnff(net_m, x); net_m = cnnbp(net_m, y); 53 | net_p = cnnff(net_p, x); net_p = cnnbp(net_p, y); 54 | d = (net_p.L - net_m.L) / (2 * epsilon); 55 | e = abs(d - net.layers{l}.dk{i}{j}(u, v)); 56 | if e > er 57 | error('numerical gradient checking failed'); 58 | end 59 | end 60 | end 61 | end 62 | end 63 | elseif strcmp(net.layers{l}.type, 's') 64 | % for j = 1 : numel(net.layers{l}.a) 65 | % net_m = net; net_p = net; 66 | % net_p.layers{l}.b{j} = net_m.layers{l}.b{j} + epsilon; 67 | % net_m.layers{l}.b{j} = net_m.layers{l}.b{j} - epsilon; 68 | % net_m = cnnff(net_m, x); net_m = cnnbp(net_m, y); 69 | % net_p = cnnff(net_p, x); net_p = cnnbp(net_p, y); 70 | % d = (net_p.L - net_m.L) / (2 * epsilon); 71 | % e = abs(d - net.layers{l}.db{j}); 72 | % if e > er 73 | % error('numerical gradient checking failed'); 74 | % end 75 | % end 76 | end 77 | end 78 | % keyboard 79 | end 80 | -------------------------------------------------------------------------------- /CNN/cnnsetup.m: -------------------------------------------------------------------------------- 1 | function net = cnnsetup(net, x, y) 2 | %assert(~isOctave() || compare_versions(OCTAVE_VERSION, '3.8.0', '>='), ['Octave 3.8.0 or greater is required for CNNs as there is a bug in convolution in previous versions. See http://savannah.gnu.org/bugs/?39314. Your version is ' myOctaveVersion]); 3 | inputmaps = 1; 4 | mapsize = size(squeeze(x(:, :, 1))); 5 | 6 | for l = 1 : numel(net.layers) % layer 7 | if strcmp(net.layers{l}.type, 's') 8 | mapsize = mapsize / net.layers{l}.scale; 9 | assert(all(floor(mapsize)==mapsize), ['Layer ' num2str(l) ' size must be integer. Actual: ' num2str(mapsize)]); 10 | for j = 1 : inputmaps 11 | net.layers{l}.b{j} = 0; 12 | end 13 | end 14 | if strcmp(net.layers{l}.type, 'c') 15 | mapsize = mapsize - net.layers{l}.kernelsize + 1; 16 | %fan_out = net.layers{l}.outputmaps * net.layers{l}.kernelsize ^ 2; 17 | for j = 1 : net.layers{l}.outputmaps % output map 18 | %fan_in = inputmaps * net.layers{l}.kernelsize ^ 2; 19 | for i = 1 : inputmaps % input map 20 | %net.layers{l}.k{i}{j} = (rand(net.layers{l}.kernelsize) - 0.5) * 2 * sqrt(6 / (fan_in + fan_out)); 21 | net.layers{l}.k{i}{j} = (rand(net.layers{l}.kernelsize) - 0.5); 22 | %net.layers{l}.k{i}{j} = fspecial('gaussian',net.layers{l}.kernelsize,1)*5; 23 | end 24 | net.layers{l}.b{j} = 0; 25 | end 26 | inputmaps = net.layers{l}.outputmaps; 27 | end 28 | end 29 | % 'onum' is the number of labels, that's why it is calculated using size(y, 1). If you have 20 labels so the output of the network will be 20 neurons. 30 | % 'fvnum' is the number of output neurons at the last layer, the layer just before the output layer. 31 | % 'ffb' is the biases of the output neurons. 32 | % 'ffW' is the weights between the last layer and the output neurons. Note that the last layer is fully connected to the output layer, that's why the size of the weights is (onum * fvnum) 33 | fvnum = prod(mapsize)*inputmaps; 34 | %fvnum = net.layers{l}.outputmaps; 35 | onum = size(y, 1); 36 | 37 | net.ffb = zeros(onum, 1); 38 | %net.ffW = (rand(onum, fvnum) - 0.5) * 2 * sqrt(6 / (onum + fvnum)); 39 | net.ffW = (rand(onum, fvnum) - 0.5); 40 | end 41 | -------------------------------------------------------------------------------- /CNN/cnntest.m: -------------------------------------------------------------------------------- 1 | function [er, bad] = cnntest(net, x, y) 2 | % feedforward 3 | net.testing = 1; 4 | net = cnnff(net, x); 5 | [~, h] = max(net.o); 6 | [~, a] = max(y); 7 | bad = find(h ~= a); 8 | %{ 9 | % virsualisation 10 | for i = 1 : numel(net.layers) 11 | for j = 1 : numel(net.layers{i}.a) 12 | imshow(net.layers{i,1}.a{1,j}(:,:,1)'); 13 | figure; 14 | end 15 | end 16 | %} 17 | er = numel(bad) / size(y, 2); 18 | end 19 | -------------------------------------------------------------------------------- /CNN/cnntrain.m: -------------------------------------------------------------------------------- 1 | function net = cnntrain(net, x, y, opts) 2 | net.testing = 0; 3 | m = size(x, 3); % numbers of training data 4 | numbatches = m / opts.batchsize; 5 | if rem(numbatches, 1) ~= 0 6 | error('numbatches not integer'); 7 | end 8 | net.rL = []; 9 | for i = 1 : opts.numepochs 10 | disp(['epoch ' num2str(i) '/' num2str(opts.numepochs)]); 11 | tic; 12 | kk = randperm(m); 13 | for l = 1 : numbatches 14 | batch_x = x(:, :, kk((l - 1) * opts.batchsize + 1 : l * opts.batchsize)); % every image in batch share the same kernel 15 | batch_y = y(:, kk((l - 1) * opts.batchsize + 1 : l * opts.batchsize)); 16 | 17 | net = cnnff(net, batch_x); 18 | net = cnnbp(net, batch_y); 19 | net = cnnapplygrads(net, opts); 20 | if isempty(net.rL) 21 | net.rL(1) = net.L; 22 | end 23 | net.rL(end + 1) = 0.99 * net.rL(end) + 0.01 * net.L; 24 | end 25 | toc; 26 | end 27 | 28 | end 29 | -------------------------------------------------------------------------------- /CNN/expand.m: -------------------------------------------------------------------------------- 1 | function B = expand(A, S) 2 | %EXPAND Replicate and tile each element of an array, similar to repmat. 3 | % EXPAND(A,SZ), for array A and vector SZ replicates each element of A by 4 | % SZ. The results are tiled into an array in the same order as the 5 | % elements of A, so that the result is size: size(A).*SZ. Therefore the 6 | % number of elements of SZ must equal the number of dimensions of A, or in 7 | % MATLAB syntax: length(size(A))==length(SZ) must be true. 8 | % The result will have the same number of dimensions as does A. 9 | % There is no restriction on the number of dimensions for input A. 10 | % 11 | % Examples: 12 | % 13 | % A = [1 2; 3 4]; % 2x2 14 | % SZ = [6 5]; 15 | % B = expand(A,[6 5]) % Creates a 12x10 array. 16 | % 17 | % The following demonstrates equivalence of EXPAND and expansion acheived 18 | % through indexing the individual elements of the array: 19 | % 20 | % A = 1; B = 2; C = 3; D = 4; % Elements of the array to be expanded. 21 | % Mat = [A B;C D]; % The array to expand. 22 | % SZ = [2 3]; % The expansion vector. 23 | % ONES = ones(SZ); % The index array. 24 | % ExpMat1 = [A(ONES),B(ONES);C(ONES),D(ONES)]; % Element expansion. 25 | % ExpMat2 = expand(Mat,SZ); % Calling EXPAND. 26 | % isequal(ExpMat1,ExpMat2) % Yes 27 | % 28 | % 29 | % See also, repmat, meshgrid, ones, zeros, kron 30 | % 31 | % Author: Matt Fig 32 | % Date: 6/20/2009 33 | % Contact: popkenai@yahoo.com 34 | 35 | if nargin < 2 36 | error('Size vector must be provided. See help.'); 37 | end 38 | 39 | SA = size(A); % Get the size (and number of dimensions) of input. 40 | 41 | if length(SA) ~= length(S) 42 | error('Length of size vector must equal ndims(A). See help.') 43 | elseif any(S ~= floor(S)) 44 | error('The size vector must contain integers only. See help.') 45 | end 46 | 47 | T = cell(length(SA), 1); 48 | for ii = length(SA) : -1 : 1 49 | H = zeros(SA(ii) * S(ii), 1); % One index vector into A for each dim. 50 | H(1 : S(ii) : SA(ii) * S(ii)) = 1; % Put ones in correct places. 51 | T{ii} = cumsum(H); % Cumsumming creates the correct order. 52 | end 53 | 54 | B = A(T{:}); % Feed the indices into A. -------------------------------------------------------------------------------- /CNN/expansionData.m: -------------------------------------------------------------------------------- 1 | % expansion 28*28 to 32*32 with 0 2 | function images = expansionData(x) 3 | images = zeros(size(x,1)+4,size(x,2)+4,size(x,3)); 4 | for i = 1 : size(x,3) 5 | for m = 1 : size(x,1) 6 | for n = 1 : size(x,2) 7 | images(m,n,i) = x(m,n,i); 8 | end 9 | end 10 | end 11 | end -------------------------------------------------------------------------------- /CNN/flipall.m: -------------------------------------------------------------------------------- 1 | function X=flipall(X) 2 | for i=1:ndims(X) 3 | X = flipdim(X,i); 4 | end 5 | end -------------------------------------------------------------------------------- /CNN/loadMNISTImages.m: -------------------------------------------------------------------------------- 1 | function images = loadMNISTImages(filename) 2 | %filename = 't10k-images.idx3-ubyte'; 3 | %filename = 'train-images.idx3-ubyte'; 4 | fp = fopen(filename, 'rb'); 5 | assert(fp ~= -1, ['Could not open ', filename, '']); 6 | 7 | magic = fread(fp, 1, 'int32', 0, 'ieee-be'); 8 | assert(magic == 2051, ['Bad magic number in ', filename, '']); 9 | 10 | numImages = fread(fp, 1, 'int32', 0, 'ieee-be'); 11 | numRows = fread(fp, 1, 'int32', 0, 'ieee-be'); 12 | numCols = fread(fp, 1, 'int32', 0, 'ieee-be'); 13 | 14 | images = fread(fp, inf, 'unsigned char'); 15 | images = reshape(images, numCols, numRows, numImages); 16 | %images = permute(images,[2 1 3]);% images' 17 | 18 | fclose(fp); 19 | 20 | % Reshape to #pixels x #examples 21 | %images = reshape(images, size(images, 1) * size(images, 2), size(images, 3)); 22 | % Convert to double and rescale to [0,1] 23 | %images = double(images) / 255; 24 | end -------------------------------------------------------------------------------- /CNN/loadMNISTLabels.m: -------------------------------------------------------------------------------- 1 | function newLabels = loadMNISTLabels(filename) 2 | %loadMNISTLabels returns a [number of MNIST images]x1 matrix containing 3 | %the labels for the MNIST images 4 | 5 | %filename = 'train-labels.idx1-ubyte'; 6 | fp = fopen(filename, 'rb'); 7 | assert(fp ~= -1, ['Could not open ', filename, '']); 8 | 9 | magic = fread(fp, 1, 'int32', 0, 'ieee-be'); 10 | assert(magic == 2049, ['Bad magic number in ', filename, '']); 11 | 12 | numLabels = fread(fp, 1, 'int32', 0, 'ieee-be'); 13 | 14 | labels = fread(fp, inf, 'unsigned char'); 15 | % dim(newLabels) = [10,:] with one value 1, others 0 16 | newLabels = zeros(10,numLabels); 17 | for i = 1 : numLabels 18 | newLabels(labels(i,1)+1,i) = 1; 19 | end 20 | 21 | assert(size(labels,1) == numLabels, 'Mismatch in label count'); 22 | 23 | fclose(fp); 24 | 25 | end 26 | -------------------------------------------------------------------------------- /CNN/mnist_uint8.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PierreHao/Deep-Learning-ToolBox-CNN/18b8f0975cb3028678dc38a87e1d9fa909c5c254/CNN/mnist_uint8.mat -------------------------------------------------------------------------------- /CNN/sigm.m: -------------------------------------------------------------------------------- 1 | function X = sigm(P) 2 | X = 1./(1+exp(-P)); 3 | %X = max(0,P); 4 | end -------------------------------------------------------------------------------- /CNN/softmax.m: -------------------------------------------------------------------------------- 1 | function mu = softmax(eta) 2 | % Softmax function 3 | % mu(i,c) = exp(eta(i,c))/sum_c' exp(eta(i,c')) 4 | 5 | % This file is from matlabtools.googlecode.com 6 | c = 1; 7 | 8 | tmp = exp(c*eta); 9 | denom = sum(tmp, 1); 10 | mu = bsxfun(@rdivide, tmp, denom); 11 | 12 | end -------------------------------------------------------------------------------- /CNN/t10k-images.idx3-ubyte: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PierreHao/Deep-Learning-ToolBox-CNN/18b8f0975cb3028678dc38a87e1d9fa909c5c254/CNN/t10k-images.idx3-ubyte -------------------------------------------------------------------------------- /CNN/t10k-labels.idx1-ubyte: -------------------------------------------------------------------------------- 1 | '                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             -------------------------------------------------------------------------------- /CNN/train-images.idx3-ubyte: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PierreHao/Deep-Learning-ToolBox-CNN/18b8f0975cb3028678dc38a87e1d9fa909c5c254/CNN/train-images.idx3-ubyte -------------------------------------------------------------------------------- /CNN/train-labels.idx1-ubyte: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PierreHao/Deep-Learning-ToolBox-CNN/18b8f0975cb3028678dc38a87e1d9fa909c5c254/CNN/train-labels.idx1-ubyte -------------------------------------------------------------------------------- /CNN/workspace_40.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PierreHao/Deep-Learning-ToolBox-CNN/18b8f0975cb3028678dc38a87e1d9fa909c5c254/CNN/workspace_40.mat -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Deep-Learning-ToolBox-CNN 2 | 3 | Author: PIERRE_HAO 4 | 5 | Data: 2015/05/01 6 | 7 | An improved version for Deep Learning Toolbox CNN 8 | 9 | 1. Add ReLu function 10 | 11 | 2. Add Max pooling function 12 | 13 | 3. Add function to read MNIST which we can get from Yann LeCun Home Page 14 | 15 | 4. Pad size 28*28 to 32*32 with value 0 16 | 17 | 5. Other minor changes which are not important 18 | 19 | 6. Add softmax 20 | 21 | 7. Add stepsize for changing learning rate 22 | 23 | @MASTERSTHESIS\{IMM2012-06284, 24 | 25 | author = "R. B. Palm", 26 | 27 | title = "Prediction as a candidate for learning deep hierarchical models of data", 28 | 29 | year = "2012", 30 | } 31 | --------------------------------------------------------------------------------