├── .travis.yml ├── CAE ├── caeapplygrads.m ├── caebbp.m ├── caebp.m ├── caedown.m ├── caeexamples.m ├── caenumgradcheck.m ├── caesdlm.m ├── caetrain.m ├── caeup.m ├── max3d.m ├── scaesetup.m └── scaetrain.m ├── CNN ├── MaxPooling.cpp ├── MaxPooling.cpp.bak ├── MaxPooling.m ├── MaxPooling.mexw64 ├── MaxPoolingGPU.cpp ├── MaxPoolingGPU.cpp.bak ├── StochasticPooling.cpp ├── StochasticPooling.mexw64 ├── StochaticTest.m ├── cnnapplygrads.m ├── cnnbp - 副本.m ├── cnnbp.m ├── cnnff.m ├── cnnnumgradcheck.m ├── cnnsetup.m ├── cnntest.m ├── cnntrain.m ├── test_example_CNN.m └── test_example_CNN2.m ├── CONTRIBUTING.md ├── DBN ├── dbnsetup.m ├── dbntrain.m ├── dbnunfoldtonn.m ├── rbmdown.m ├── rbmtrain.m └── rbmup.m ├── LICENSE ├── MaxOut ├── nnapplygrads.m ├── nnbp.m ├── nnchecknumgrad.m ├── nneval.m ├── nnff.m ├── nnpredict.m ├── nnsetup - 副本.m ├── nnsetup.m ├── nntest.m ├── nntrain.m ├── nnupdatefigures.m └── test_example_MaxOut.m ├── NN ├── nnapplygrads.m ├── nnbp.m ├── nnchecknumgrad.m ├── nneval.m ├── nnff.m ├── nnpredict.m ├── nnsetup - 副本.m ├── nnsetup.m ├── nntest.m ├── nntrain.m └── nnupdatefigures.m ├── README.md ├── README_header.md ├── REFS.md ├── SAE ├── saesetup.m └── saetrain.m ├── data └── mnist_uint8.mat ├── test_example_NN.m ├── tests.wf ├── runalltests.m ├── test_cnn_gradients_are_numerically_correct.m ├── test_example_CNN.m ├── test_example_DBN.m ├── test_example_NN.m ├── test_example_SAE.m └── test_nn_gradients_are_numerically_correct.m ├── tests ├── RandomDrawing.m ├── runalltests.m ├── test_example_AE.m ├── test_example_CNN.m ├── test_example_DBN.m ├── test_example_NN.m ├── test_example_NN2.m ├── test_example_SAE.m └── test_nn_gradients_are_numerically_correct.m └── util ├── allcomb.m ├── expand.m ├── flicker.m ├── flipall.m ├── fliplrf.m ├── flipudf.m ├── im2patches.m ├── isOctave.m ├── makeLMfilters.m ├── myOctaveVersion.m ├── normalize.m ├── patches2im.m ├── randcorr.m ├── randp.m ├── rnd.m ├── sigm.m ├── sigmrnd.m ├── softmax.m ├── tanh_opt.m ├── visualize.m ├── whiten.m └── zscore.m /.travis.yml: -------------------------------------------------------------------------------- 1 | before_script: 2 | - sudo apt-add-repository ppa:octave/stable --yes 3 | - sudo apt-get update -y 4 | - sudo apt-get install octave -y 5 | - sudo apt-get install liboctave-dev -y 6 | script: 7 | - sh -c "octave tests/runalltests.m" 8 | 9 | notifications: 10 | email: false 11 | -------------------------------------------------------------------------------- /CAE/caeapplygrads.m: -------------------------------------------------------------------------------- 1 | function cae = caeapplygrads(cae) 2 | cae.sv = 0; 3 | for j = 1 : numel(cae.a) 4 | for i = 1 : numel(cae.i) 5 | % cae.vik{i}{j} = cae.momentum * cae.vik{i}{j} + cae.alpha ./ (cae.sigma + cae.ddik{i}{j}) .* cae.dik{i}{j}; 6 | % cae.vok{i}{j} = cae.momentum * cae.vok{i}{j} + cae.alpha ./ (cae.sigma + cae.ddok{i}{j}) .* cae.dok{i}{j}; 7 | cae.vik{i}{j} = cae.alpha * cae.dik{i}{j}; 8 | cae.vok{i}{j} = cae.alpha * cae.dok{i}{j}; 9 | cae.sv = cae.sv + sum(cae.vik{i}{j}(:) .^ 2); 10 | cae.sv = cae.sv + sum(cae.vok{i}{j}(:) .^ 2); 11 | 12 | cae.ik{i}{j} = cae.ik{i}{j} - cae.vik{i}{j}; 13 | cae.ok{i}{j} = cae.ok{i}{j} - cae.vok{i}{j}; 14 | end 15 | % cae.vb{j} = cae.momentum * cae.vb{j} + cae.alpha / (cae.sigma + cae.ddb{j}) * cae.db{j}; 16 | cae.vb{j} = cae.alpha * cae.db{j}; 17 | cae.sv = cae.sv + sum(cae.vb{j} .^ 2); 18 | 19 | cae.b{j} = cae.b{j} - cae.vb{j}; 20 | end 21 | 22 | for i = 1 : numel(cae.o) 23 | % cae.vc{i} = cae.momentum * cae.vc{i} + cae.alpha / (cae.sigma + cae.ddc{i}) * cae.dc{i}; 24 | cae.vc{i} = cae.alpha * cae.dc{i}; 25 | cae.sv = cae.sv + sum(cae.vc{i} .^ 2); 26 | 27 | cae.c{i} = cae.c{i} - cae.vc{i}; 28 | end 29 | end 30 | -------------------------------------------------------------------------------- /CAE/caebbp.m: -------------------------------------------------------------------------------- 1 | function cae = caebbp(cae) 2 | 3 | %% backprop deltas 4 | for i = 1 : numel(cae.o) 5 | % output delta delta 6 | cae.odd{i} = (cae.o{i} .* (1 - cae.o{i}) .* cae.edgemask) .^ 2; 7 | % delta delta c 8 | cae.ddc{i} = sum(cae.odd{i}(:)) / size(cae.odd{i}, 1); 9 | end 10 | 11 | for j = 1 : numel(cae.a) % calc activation delta deltas 12 | z = 0; 13 | for i = 1 : numel(cae.o) 14 | z = z + convn(cae.odd{i}, flipall(cae.ok{i}{j} .^ 2), 'full'); 15 | end 16 | cae.add{j} = (cae.a{j} .* (1 - cae.a{j})) .^ 2 .* z; 17 | end 18 | 19 | %% calc params delta deltas 20 | ns = size(cae.odd{1}, 1); 21 | for j = 1 : numel(cae.a) 22 | cae.ddb{j} = sum(cae.add{j}(:)) / ns; 23 | for i = 1 : numel(cae.o) 24 | cae.ddok{i}{j} = convn(flipall(cae.a{j} .^ 2), cae.odd{i}, 'valid') / ns; 25 | cae.ddik{i}{j} = convn(cae.add{j}, flipall(cae.i{i} .^ 2), 'valid') / ns; 26 | end 27 | end 28 | 29 | end 30 | -------------------------------------------------------------------------------- /CAE/caebp.m: -------------------------------------------------------------------------------- 1 | function cae = caebp(cae, y) 2 | 3 | %% backprop deltas 4 | cae.L = 0; 5 | for i = 1 : numel(cae.o) 6 | % error 7 | cae.e{i} = (cae.o{i} - y{i}) .* cae.edgemask; 8 | % loss function 9 | cae.L = cae.L + 1/2 * sum(cae.e{i}(:) .^2 ) / size(cae.e{i}, 1); 10 | % output delta 11 | cae.od{i} = cae.e{i} .* (cae.o{i} .* (1 - cae.o{i})); 12 | 13 | cae.dc{i} = sum(cae.od{i}(:)) / size(cae.e{i}, 1); 14 | end 15 | 16 | for j = 1 : numel(cae.a) % calc activation deltas 17 | z = 0; 18 | for i = 1 : numel(cae.o) 19 | z = z + convn(cae.od{i}, flipall(cae.ok{i}{j}), 'full'); 20 | end 21 | cae.ad{j} = cae.a{j} .* (1 - cae.a{j}) .* z; 22 | end 23 | 24 | %% calc gradients 25 | ns = size(cae.e{1}, 1); 26 | for j = 1 : numel(cae.a) 27 | cae.db{j} = sum(cae.ad{j}(:)) / ns; 28 | for i = 1 : numel(cae.o) 29 | cae.dok{i}{j} = convn(flipall(cae.a{j}), cae.od{i}, 'valid') / ns; 30 | cae.dik{i}{j} = convn(cae.ad{j}, flipall(cae.i{i}), 'valid') / ns; 31 | end 32 | end 33 | 34 | end 35 | -------------------------------------------------------------------------------- /CAE/caedown.m: -------------------------------------------------------------------------------- 1 | function cae = caedown(cae) 2 | pa = cae.a; 3 | pok = cae.ok; 4 | 5 | for i = 1 : numel(cae.o) 6 | z = 0; 7 | for j = 1 : numel(cae.a) 8 | z = z + convn(pa{j}, pok{i}{j}, 'valid'); 9 | end 10 | cae.o{i} = sigm(z + cae.c{i}); 11 | 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /CAE/caeexamples.m: -------------------------------------------------------------------------------- 1 | %% mnist data 2 | clear all; close all; clc; 3 | load mnist_uint8; 4 | x = cell(100, 1); 5 | N = 600; 6 | for i = 1 : 100 7 | x{i}{1} = reshape(train_x(((i - 1) * N + 1) : (i) * N, :), N, 28, 28) * 255; 8 | end 9 | %% ex 1 10 | scae = { 11 | struct('outputmaps', 10, 'inputkernel', [1 5 5], 'outputkernel', [1 5 5], 'scale', [1 2 2], 'sigma', 0.1, 'momentum', 0.9, 'noise', 0) 12 | }; 13 | 14 | opts.rounds = 1000; 15 | opts.batchsize = 1; 16 | opts.alpha = 0.01; 17 | opts.ddinterval = 10; 18 | opts.ddhist = 0.5; 19 | scae = scaesetup(scae, x, opts); 20 | scae = scaetrain(scae, x, opts); 21 | cae = scae{1}; 22 | 23 | %Visualize the average reconstruction error 24 | plot(cae.rL); 25 | 26 | %Visualize the output kernels 27 | ff=[]; 28 | for i=1:numel(cae.ok{1}); 29 | mm = cae.ok{1}{i}(1,:,:); 30 | ff(i,:) = mm(:); 31 | end; 32 | figure;visualize(ff') 33 | -------------------------------------------------------------------------------- /CAE/caenumgradcheck.m: -------------------------------------------------------------------------------- 1 | function cae = caenumgradcheck(cae, x, y) 2 | epsilon = 1e-4; 3 | er = 1e-6; 4 | disp('performing numerical gradient checking...') 5 | for i = 1 : numel(cae.o) 6 | p_cae = cae; p_cae.c{i} = p_cae.c{i} + epsilon; 7 | m_cae = cae; m_cae.c{i} = m_cae.c{i} - epsilon; 8 | 9 | [m_cae, p_cae] = caerun(m_cae, p_cae, x, y); 10 | d = (p_cae.L - m_cae.L) / (2 * epsilon); 11 | 12 | e = abs(d - cae.dc{i}); 13 | if e > er 14 | disp('OUTPUT BIAS numerical gradient checking failed'); 15 | disp(e); 16 | disp(d / cae.dc{i}); 17 | keyboard 18 | end 19 | end 20 | 21 | for a = 1 : numel(cae.a) 22 | 23 | p_cae = cae; p_cae.b{a} = p_cae.b{a} + epsilon; 24 | m_cae = cae; m_cae.b{a} = m_cae.b{a} - epsilon; 25 | 26 | [m_cae, p_cae] = caerun(m_cae, p_cae, x, y); 27 | d = (p_cae.L - m_cae.L) / (2 * epsilon); 28 | % cae.dok{i}{a}(u) = d; 29 | e = abs(d - cae.db{a}); 30 | if e > er 31 | disp('BIAS numerical gradient checking failed'); 32 | disp(e); 33 | disp(d / cae.db{a}); 34 | keyboard 35 | end 36 | 37 | for i = 1 : numel(cae.o) 38 | for u = 1 : numel(cae.ok{i}{a}) 39 | p_cae = cae; p_cae.ok{i}{a}(u) = p_cae.ok{i}{a}(u) + epsilon; 40 | m_cae = cae; m_cae.ok{i}{a}(u) = m_cae.ok{i}{a}(u) - epsilon; 41 | 42 | [m_cae, p_cae] = caerun(m_cae, p_cae, x, y); 43 | d = (p_cae.L - m_cae.L) / (2 * epsilon); 44 | % cae.dok{i}{a}(u) = d; 45 | e = abs(d - cae.dok{i}{a}(u)); 46 | if e > er 47 | disp('OUTPUT KERNEL numerical gradient checking failed'); 48 | disp(e); 49 | disp(d / cae.dok{i}{a}(u)); 50 | % keyboard 51 | end 52 | end 53 | end 54 | 55 | for i = 1 : numel(cae.i) 56 | for u = 1 : numel(cae.ik{i}{a}) 57 | p_cae = cae; 58 | m_cae = cae; 59 | p_cae.ik{i}{a}(u) = p_cae.ik{i}{a}(u) + epsilon; 60 | m_cae.ik{i}{a}(u) = m_cae.ik{i}{a}(u) - epsilon; 61 | [m_cae, p_cae] = caerun(m_cae, p_cae, x, y); 62 | d = (p_cae.L - m_cae.L) / (2 * epsilon); 63 | % cae.dik{i}{a}(u) = d; 64 | e = abs(d - cae.dik{i}{a}(u)); 65 | if e > er 66 | disp('INPUT KERNEL numerical gradient checking failed'); 67 | disp(e); 68 | disp(d / cae.dik{i}{a}(u)); 69 | end 70 | end 71 | end 72 | end 73 | 74 | disp('done') 75 | 76 | end 77 | 78 | function [m_cae, p_cae] = caerun(m_cae, p_cae, x, y) 79 | m_cae = caeup(m_cae, x); m_cae = caedown(m_cae); m_cae = caebp(m_cae, y); 80 | p_cae = caeup(p_cae, x); p_cae = caedown(p_cae); p_cae = caebp(p_cae, y); 81 | end 82 | 83 | %function checknumgrad(cae,what,x,y) 84 | % epsilon = 1e-4; 85 | % er = 1e-9; 86 | % 87 | % for i = 1 : numel(eval(what)) 88 | % if iscell(eval(['cae.' what])) 89 | % checknumgrad(cae,[what '{' num2str(i) '}'], x, y) 90 | % else 91 | % p_cae = cae; 92 | % m_cae = cae; 93 | % eval(['p_cae.' what '(' num2str(i) ')']) = eval([what '(' num2str(i) ')']) + epsilon; 94 | % eval(['m_cae.' what '(' num2str(i) ')']) = eval([what '(' num2str(i) ')']) - epsilon; 95 | % 96 | % m_cae = caeff(m_cae, x); m_cae = caedown(m_cae); m_cae = caebp(m_cae, y); 97 | % p_cae = caeff(p_cae, x); p_cae = caedown(p_cae); p_cae = caebp(p_cae, y); 98 | % 99 | % d = (p_cae.L - m_cae.L) / (2 * epsilon); 100 | % e = abs(d - eval(['cae.d' what '(' num2str(i) ')'])); 101 | % if e > er 102 | % error('numerical gradient checking failed'); 103 | % end 104 | % end 105 | % end 106 | % 107 | % end 108 | -------------------------------------------------------------------------------- /CAE/caesdlm.m: -------------------------------------------------------------------------------- 1 | function cae = caesdlm(cae, opts, m) 2 | %stochastic diagonal levenberg-marquardt 3 | 4 | %first round 5 | if isfield(cae,'ddok') == 0 6 | cae = caebbp(cae); 7 | end 8 | 9 | %recalculate double grads every opts.ddinterval 10 | if mod(m, opts.ddinterval) == 0 11 | cae_n = caebbp(cae); 12 | 13 | for ii = 1 : numel(cae.o) 14 | cae.ddc{ii} = opts.ddhist * cae.ddc{ii} + (1 - opts.ddhist) * cae_n.ddc{ii}; 15 | end 16 | 17 | for jj = 1 : numel(cae.a) 18 | cae.ddb{jj} = opts.ddhist * cae.ddb{jj} + (1 - opts.ddhist) * cae_n.ddb{jj}; 19 | for ii = 1 : numel(cae.o) 20 | cae.ddok{ii}{jj} = opts.ddhist * cae.ddok{ii}{jj} + (1 - opts.ddhist) * cae_n.ddok{ii}{jj}; 21 | cae.ddik{ii}{jj} = opts.ddhist * cae.ddik{ii}{jj} + (1 - opts.ddhist) * cae_n.ddik{ii}{jj}; 22 | end 23 | end 24 | 25 | end 26 | end -------------------------------------------------------------------------------- /CAE/caetrain.m: -------------------------------------------------------------------------------- 1 | function cae = caetrain(cae, x, opts) 2 | n = cae.inputkernel(1); 3 | cae.rL = []; 4 | for m = 1 : opts.rounds 5 | tic; 6 | disp([num2str(m) '/' num2str(opts.rounds) ' rounds']); 7 | i1 = randi(numel(x)); 8 | l = randi(size(x{i1}{1},1) - opts.batchsize - n + 1); 9 | x1{1} = double(x{i1}{1}(l : l + opts.batchsize - 1, :, :)) / 255; 10 | 11 | if n == 1 %Auto Encoder 12 | x2{1} = x1{1}; 13 | else %Predictive Encoder 14 | x2{1} = double(x{i1}{1}(l + n : l + n + opts.batchsize - 1, :, :)) / 255; 15 | end 16 | % Add noise to input, for denoising stacked autoenoder 17 | x1{1} = x1{1} .* (rand(size(x1{1})) > cae.noise); 18 | 19 | cae = caeup(cae, x1); 20 | cae = caedown(cae); 21 | cae = caebp(cae, x2); 22 | cae = caesdlm(cae, opts, m); 23 | % caenumgradcheck(cae,x1,x2); 24 | cae = caeapplygrads(cae); 25 | 26 | if m == 1 27 | cae.rL(1) = cae.L; 28 | end 29 | % cae.rL(m + 1) = 0.99 * cae.rL(m) + 0.01 * cae.L; 30 | cae.rL(m + 1) = cae.L; 31 | % if cae.sv < 1e-10 32 | % disp('Converged'); 33 | % break; 34 | % end 35 | toc; 36 | end 37 | 38 | end 39 | -------------------------------------------------------------------------------- /CAE/caeup.m: -------------------------------------------------------------------------------- 1 | function cae = caeup(cae, x) 2 | cae.i = x; 3 | 4 | %init temp vars for parrallel processing 5 | pa = cell(size(cae.a)); 6 | pi = cae.i; 7 | pik = cae.ik; 8 | pb = cae.b; 9 | 10 | for j = 1 : numel(cae.a) 11 | z = 0; 12 | for i = 1 : numel(pi) 13 | z = z + convn(pi{i}, pik{i}{j}, 'full'); 14 | end 15 | pa{j} = sigm(z + pb{j}); 16 | 17 | % Max pool. 18 | if ~isequal(cae.scale, [1 1 1]) 19 | pa{j} = max3d(pa{j}, cae.M); 20 | end 21 | 22 | end 23 | cae.a = pa; 24 | 25 | end 26 | -------------------------------------------------------------------------------- /CAE/max3d.m: -------------------------------------------------------------------------------- 1 | function X = max3d(X, M) 2 | ll = size(X); 3 | B=X(M); 4 | B=B+rand(size(B))*1e-12; 5 | B=(B.*(B==repmat(max(B,[],2),[1 size(B,2) 1]))); 6 | X(M) = B; 7 | reshape(X,ll); 8 | end -------------------------------------------------------------------------------- /CAE/scaesetup.m: -------------------------------------------------------------------------------- 1 | function scae = scaesetup(cae, x, opts) 2 | x = x{1}; 3 | for l = 1 : numel(cae) 4 | cae = cae{l}; 5 | ll= [opts.batchsize size(x{1}, 2) size(x{1}, 3)] + cae.inputkernel - 1; 6 | X = zeros(ll); 7 | cae.M = nbmap(X, cae.scale); 8 | bounds = cae.outputmaps * prod(cae.inputkernel) + numel(x) * prod(cae.outputkernel); 9 | for j = 1 : cae.outputmaps % activation maps 10 | cae.a{j} = zeros(size(x{1}) + cae.inputkernel - 1); 11 | for i = 1 : numel(x) % input map 12 | cae.ik{i}{j} = (rand(cae.inputkernel) - 0.5) * 2 * sqrt(6 / bounds); 13 | cae.ok{i}{j} = (rand(cae.outputkernel) - 0.5) * 2 * sqrt(6 / bounds); 14 | cae.vik{i}{j} = zeros(size(cae.ik{i}{j})); 15 | cae.vok{i}{j} = zeros(size(cae.ok{i}{j})); 16 | end 17 | cae.b{j} = 0; 18 | cae.vb{j} = zeros(size(cae.b{j})); 19 | end 20 | 21 | cae.alpha = opts.alpha; 22 | 23 | cae.i = cell(numel(x), 1); 24 | cae.o = cae.i; 25 | 26 | for i = 1 : numel(cae.o) 27 | cae.c{i} = 0; 28 | cae.vc{i} = zeros(size(cae.c{i})); 29 | end 30 | 31 | ss = cae.outputkernel; 32 | 33 | cae.edgemask = zeros([opts.batchsize size(x{1}, 2) size(x{1}, 3)]); 34 | 35 | cae.edgemask(ss(1) : end - ss(1) + 1, ... 36 | ss(2) : end - ss(2) + 1, ... 37 | ss(3) : end - ss(3) + 1) = 1; 38 | 39 | scae{l} = cae; 40 | end 41 | 42 | function B = nbmap(X,n) 43 | assert(numel(n)==3,'n should have 3 elements (x,y,z) scaling.'); 44 | X = reshape(1:numel(X),size(X,1),size(X,2),size(X,3)); 45 | B = zeros(size(X,1)/n(1),prod(n),size(X,2)*size(X,3)/prod(n(2:3))); 46 | u=1; 47 | p=1; 48 | for m=1:size(X,1) 49 | B(u,(p-1)*prod(n(2:3))+1:p*prod(n(2:3)),:) = im2col(squeeze(X(m,:,:)),n(2:3),'distinct'); 50 | p=p+1; 51 | if(mod(m,n(1))==0) 52 | u=u+1; 53 | p=1; 54 | end 55 | end 56 | 57 | end 58 | end 59 | -------------------------------------------------------------------------------- /CAE/scaetrain.m: -------------------------------------------------------------------------------- 1 | function scae = scaetrain(scae, x, opts) 2 | %TODO: Transform x through scae{1} into new x. Only works for a single PAE. 3 | % for i=1:numel(scae) 4 | % scae{i} = paetrain(scae{i}, x, opts); 5 | % end 6 | scae{1} = caetrain(scae{1}, x, opts); 7 | 8 | end -------------------------------------------------------------------------------- /CNN/MaxPooling.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * MaxPooling.c 3 | * 4 | * Implements the max-pooling transfer function. Takes a 4D tensor shaped as 5 | * (rows, cols, nchannels, nsamples) and a pooling shape as (prows, pcols) and 6 | * returns a set of max-values with the corresponding indices in the input 7 | * matrix. 8 | * 9 | * e.g. 10 | * [m, idx] = MaxPooling(IM, [2 2]) 11 | * 12 | * Created on: July 11, 2011 13 | * Author: Jonathan Masci 14 | * 15 | * This file is available under the terms of the GNU GPLv2. 16 | */ 17 | 18 | #include "mex.h" 19 | 20 | #include 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | #define IDX2F(i,j,ld) ((((j)-1)*(ld))+((i)-1)) 28 | #define IDX2C(i,j,ld) (((j)*(ld))+(i)) 29 | 30 | int debug = 0; 31 | 32 | /** 33 | * Computes the max-pooling for the given 2D map, and no, the name is not a typo. 34 | * All pointers are passed already offset so to avoid cumbersome indexing. 35 | * 36 | * @param ptr_data pointer set to the begin of this map 37 | * @param DATA_DIMS data dimensions 38 | * @param ptr_pool pooling sizes 39 | * @param ptr_out pointer to the output max-values set to the right position 40 | * @param ptr_idx pointer to the output indices set to the right position 41 | */ 42 | template 43 | inline void compute_map_pooling(T *ptr_data, const mwSize *DATA_DIMS, T *ptr_pool, 44 | T *ptr_out, T *ptr_idx, int tile_start) 45 | { 46 | T m; 47 | int idx; 48 | int count = 0; 49 | 50 | for (int col = 0; col < DATA_DIMS[1]; col += ptr_pool[1]) { 51 | for (int row = 0; row < DATA_DIMS[0]; row += ptr_pool[0]) { 52 | if (debug) 53 | fprintf(stderr, "r = %i, c = %i \n", row, col); 54 | 55 | m = -std::numeric_limits::max(); 56 | idx = -1; 57 | for (int pcol = 0; (pcol < ptr_pool[1] && col + pcol < DATA_DIMS[1]); ++pcol) { 58 | for (int prow = 0; (prow < ptr_pool[0] && row + prow < DATA_DIMS[0]); ++prow) { 59 | if (debug) { 60 | fprintf(stderr, "m = %f, data = %f \n", m, ptr_data[IDX2C(row + prow, col + pcol, DATA_DIMS[0])]); 61 | fprintf(stderr, "rr = %i, cc = %i \n --> idx = %i \n", row + prow, col + pcol, idx); 62 | } 63 | 64 | if (ptr_data[IDX2C(row + prow, col + pcol, DATA_DIMS[0])] > m) { 65 | idx = IDX2C(row + prow, col + pcol, DATA_DIMS[0]); 66 | m = ptr_data[idx]; 67 | } 68 | } 69 | } 70 | 71 | if (debug && idx == -1) { 72 | fprintf(stderr, "dioschifoso\n"); 73 | return; 74 | } 75 | 76 | if (debug) 77 | fprintf(stderr, "count = %i\n",count); 78 | 79 | /* idxs are to be used in Matlab and hence a +1 is needed */ 80 | ptr_idx[count] = idx + 1 + tile_start; 81 | ptr_out[count] = m; 82 | count++; 83 | } 84 | } 85 | } 86 | 87 | /** 88 | * This is the wrapper for the actual computation. 89 | * It is a template so that multiple types can be handled. 90 | */ 91 | template 92 | void mexMaxPooling(int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[], mxClassID classID) 93 | { 94 | 95 | /***************************************************************************/ 96 | /** Variables */ 97 | /***************************************************************************/ 98 | mwSize IDX_DIMS[1]; 99 | mwSize DATA_DIMS[4]; 100 | mwSize M_DIMS[4]; 101 | const mwSize *POOL_DIMS; 102 | int DATA_NUMEL; 103 | int POOL_NUMEL; 104 | 105 | /** 106 | * Pointers to data 107 | */ 108 | T *ptr_data = NULL; 109 | T *ptr_pool = NULL; 110 | T *ptr_out = NULL; 111 | T *ptr_idx = NULL; 112 | 113 | /***************************************************************************/ 114 | /** Setting input pointers *************************************************/ 115 | /***************************************************************************/ 116 | ptr_data = (T *)mxGetData(prhs[0]); 117 | ptr_pool = (T *)mxGetData(prhs[1]); 118 | if (debug) 119 | fprintf(stderr,"Pooling size: h=%f, w=%f\n", ptr_pool[0], ptr_pool[1]); 120 | 121 | /***************************************************************************/ 122 | /** Setting parameters *****************************************************/ 123 | /***************************************************************************/ 124 | /* Data dimensions. As also a 2D tensor can be used I fill empty dimensions 125 | * with 1 */ 126 | const mwSize *tmp = mxGetDimensions(prhs[0]); 127 | DATA_DIMS[0] = tmp[0]; 128 | DATA_DIMS[1] = tmp[1]; 129 | if (mxGetNumberOfDimensions(prhs[0]) == 2) { 130 | DATA_DIMS[2] = 1; 131 | DATA_DIMS[3] = 1; 132 | } else if (mxGetNumberOfDimensions(prhs[0]) == 3) { 133 | DATA_DIMS[2] = tmp[2]; 134 | DATA_DIMS[3] = 1; 135 | } else { 136 | DATA_DIMS[2] = tmp[2]; 137 | DATA_DIMS[3] = tmp[3]; 138 | } 139 | 140 | DATA_NUMEL = DATA_DIMS[0] * DATA_DIMS[1] * DATA_DIMS[2] * DATA_DIMS[3]; 141 | if (debug) 142 | fprintf(stderr,"Data size: h=%d, w=%d, z=%d, n=%d (%d)\n", DATA_DIMS[0], DATA_DIMS[1], DATA_DIMS[2], DATA_DIMS[3], DATA_NUMEL); 143 | 144 | /* Output dimensions: the first output argument is of size equals to the input 145 | * whereas the second is of size equals to the number of pooled values. 146 | * Below there is ceil because also non complete tiles are considered when 147 | * input dims are not multiples of pooling dims. */ 148 | M_DIMS[0] = ceil(float(DATA_DIMS[0]) / float(ptr_pool[0])); 149 | M_DIMS[1] = ceil(float(DATA_DIMS[1]) / float(ptr_pool[1])); 150 | M_DIMS[2] = DATA_DIMS[2]; 151 | M_DIMS[3] = DATA_DIMS[3]; 152 | IDX_DIMS[0] = M_DIMS[0] * M_DIMS[1] * M_DIMS[2] * M_DIMS[3]; 153 | if (debug){ 154 | fprintf(stderr,"Each output image has (%d, %d) pooled values, " 155 | "IDXs size: h=%d \n", M_DIMS[0], M_DIMS[1], IDX_DIMS[0]); 156 | fprintf(stderr, "M size: h=%d, w=%d, z=%d, n=%d\n", M_DIMS[0], M_DIMS[1], M_DIMS[2], M_DIMS[3]); 157 | } 158 | 159 | /***************************************************************************/ 160 | /** Variables allocation ***************************************************/ 161 | /***************************************************************************/ 162 | /* OUTPUTS: max-values and corresponding indices */ 163 | plhs[0] = mxCreateNumericArray(4, M_DIMS, classID, mxREAL); 164 | ptr_out = (T *)mxGetData(plhs[0]); 165 | plhs[1] = mxCreateNumericArray(1, IDX_DIMS, classID, mxREAL); 166 | ptr_idx = (T *)mxGetData(plhs[1]); 167 | 168 | /***************************************************************************/ 169 | /** Compute max-pooling ****************************************************/ 170 | /***************************************************************************/ 171 | int tile_start = 0; 172 | int ptr_offset = 0; 173 | int M_sample_size = M_DIMS[0] * M_DIMS[1] * M_DIMS[2]; 174 | int D_sample_size = DATA_DIMS[0] * DATA_DIMS[1] * DATA_DIMS[2]; 175 | 176 | for (int n = 0; n < DATA_DIMS[3]; ++n) { 177 | #pragma omp parallel for 178 | for (int k = 0; k < DATA_DIMS[2]; ++k) { 179 | tile_start = n * M_sample_size + k * M_DIMS[0] * M_DIMS[1]; 180 | ptr_offset = n * D_sample_size + k * DATA_DIMS[0] * DATA_DIMS[1]; 181 | 182 | compute_map_pooling (&ptr_data[ptr_offset], DATA_DIMS, ptr_pool, &ptr_out[tile_start], &ptr_idx[tile_start], ptr_offset); 183 | 184 | if (debug) 185 | fprintf(stderr, "tile_start: %i, ptr_offset: %i\n", tile_start, ptr_offset); 186 | } 187 | } 188 | } 189 | 190 | void mexFunction(int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[]) 191 | { 192 | /***************************************************************************/ 193 | /** Check input ************************************************************/ 194 | /***************************************************************************/ 195 | if (nrhs !=2) 196 | mexErrMsgTxt("Must have 2 input arguments: x, pooling_shape"); 197 | 198 | if (nlhs !=2) 199 | mexErrMsgTxt("Must have 2 output arguments ([max_value, idxs])"); 200 | 201 | if (mxIsComplex(prhs[0]) || !(mxIsClass(prhs[0],"single") || mxIsClass(prhs[0],"double"))) 202 | mexErrMsgTxt("Input data must be real, single/double type"); 203 | 204 | if (mxIsComplex(prhs[1]) || !(mxIsClass(prhs[1],"single") || mxIsClass(prhs[1],"double"))) 205 | mexErrMsgTxt("Pooling dimensions (rows, cols) must be real, single/double type"); 206 | 207 | if (mxGetNumberOfDimensions(prhs[0]) < 2) 208 | mexErrMsgTxt("Input data must have at least 2-dimensions (rows, cols, nchannels, nsamples) " 209 | "\nThe last two dimensions will be considered to be 1."); 210 | 211 | if (mxGetNumberOfDimensions(prhs[1]) != 2) 212 | mexErrMsgTxt("Pooling data must have 2-dimensions (prows, pcols)"); 213 | 214 | mxClassID classID = mxGetClassID(prhs[0]); 215 | 216 | /** This is mainly to avoid two typenames. Should not be a big usability issue. */ 217 | if (mxGetClassID(prhs[1]) != classID) 218 | mexErrMsgTxt("Input data and pooling need to be of the same type"); 219 | 220 | /***************************************************************************/ 221 | /** Switch for the supported data types */ 222 | /***************************************************************************/ 223 | if (classID == mxSINGLE_CLASS) { 224 | if (debug) 225 | fprintf(stderr, "Executing the single version\n"); 226 | 227 | mexMaxPooling(nlhs, plhs, nrhs, prhs, classID); 228 | } else if (classID == mxDOUBLE_CLASS) { 229 | if (debug) 230 | fprintf(stderr, "Executing the double version\n"); 231 | 232 | mexMaxPooling(nlhs, plhs, nrhs, prhs, classID); 233 | } 234 | } 235 | -------------------------------------------------------------------------------- /CNN/MaxPooling.cpp.bak: -------------------------------------------------------------------------------- 1 | /* 2 | * MaxPooling.c 3 | * 4 | * Implements the max-pooling transfer function. Takes a 4D tensor shaped as 5 | * (rows, cols, nchannels, nsamples) and a pooling shape as (prows, pcols) and 6 | * returns a set of max-values with the corresponding indices in the input 7 | * matrix. 8 | * 9 | * e.g. 10 | * [m, idx] = MaxPooling(IM, [2 2]) 11 | * 12 | * Created on: July 11, 2011 13 | * Author: Jonathan Masci 14 | * 15 | * This file is available under the terms of the GNU GPLv2. 16 | */ 17 | 18 | #include "mex.h" 19 | 20 | #ifdef OPENMP 21 | #include 22 | #endif 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | 29 | #define IDX2F(i,j,ld) ((((j)-1)*(ld))+((i)-1)) 30 | #define IDX2C(i,j,ld) (((j)*(ld))+(i)) 31 | 32 | int debug = 0; 33 | 34 | /** 35 | * Computes the max-pooling for the given 2D map, and no, the name is not a typo. 36 | * All pointers are passed already offset so to avoid cumbersome indexing. 37 | * 38 | * @param ptr_data pointer set to the begin of this map 39 | * @param DATA_DIMS data dimensions 40 | * @param ptr_pool pooling sizes 41 | * @param ptr_out pointer to the output max-values set to the right position 42 | * @param ptr_idx pointer to the output indices set to the right position 43 | */ 44 | template 45 | inline void compute_map_pooling(T *ptr_data, const mwSize *DATA_DIMS, T *ptr_pool, 46 | T *ptr_out, T *ptr_idx, int tile_start) 47 | { 48 | T m; 49 | int idx; 50 | int count = 0; 51 | 52 | for (int col = 0; col < DATA_DIMS[1]; col += ptr_pool[1]) { 53 | for (int row = 0; row < DATA_DIMS[0]; row += ptr_pool[0]) { 54 | if (debug) 55 | fprintf(stderr, "r = %i, c = %i \n", row, col); 56 | 57 | m = -std::numeric_limits::max(); 58 | idx = -1; 59 | for (int pcol = 0; (pcol < ptr_pool[1] && col + pcol < DATA_DIMS[1]); ++pcol) { 60 | for (int prow = 0; (prow < ptr_pool[0] && row + prow < DATA_DIMS[0]); ++prow) { 61 | if (debug) { 62 | fprintf(stderr, "m = %f, data = %f \n", m, ptr_data[IDX2C(row + prow, col + pcol, DATA_DIMS[0])]); 63 | fprintf(stderr, "rr = %i, cc = %i \n --> idx = %i \n", row + prow, col + pcol, idx); 64 | } 65 | 66 | if (ptr_data[IDX2C(row + prow, col + pcol, DATA_DIMS[0])] > m) { 67 | idx = IDX2C(row + prow, col + pcol, DATA_DIMS[0]); 68 | m = ptr_data[idx]; 69 | } 70 | } 71 | } 72 | 73 | if (debug && idx == -1) { 74 | fprintf(stderr, "dioschifoso\n"); 75 | return; 76 | } 77 | 78 | if (debug) 79 | fprintf(stderr, "count = %i\n",count); 80 | 81 | /* idxs are to be used in Matlab and hence a +1 is needed */ 82 | ptr_idx[count] = idx + 1 + tile_start; 83 | ptr_out[count] = m; 84 | count++; 85 | } 86 | } 87 | } 88 | 89 | /** 90 | * This is the wrapper for the actual computation. 91 | * It is a template so that multiple types can be handled. 92 | */ 93 | template 94 | void mexMaxPooling(int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[], mxClassID classID) 95 | { 96 | 97 | /***************************************************************************/ 98 | /** Variables */ 99 | /***************************************************************************/ 100 | mwSize IDX_DIMS[1]; 101 | mwSize DATA_DIMS[4]; 102 | mwSize M_DIMS[4]; 103 | const mwSize *POOL_DIMS; 104 | int DATA_NUMEL; 105 | int POOL_NUMEL; 106 | 107 | /** 108 | * Pointers to data 109 | */ 110 | T *ptr_data = NULL; 111 | T *ptr_pool = NULL; 112 | T *ptr_out = NULL; 113 | T *ptr_idx = NULL; 114 | 115 | /***************************************************************************/ 116 | /** Setting input pointers *************************************************/ 117 | /***************************************************************************/ 118 | ptr_data = (T *)mxGetData(prhs[0]); 119 | ptr_pool = (T *)mxGetData(prhs[1]); 120 | if (debug) 121 | fprintf(stderr,"Pooling size: h=%f, w=%f\n", ptr_pool[0], ptr_pool[1]); 122 | 123 | /***************************************************************************/ 124 | /** Setting parameters *****************************************************/ 125 | /***************************************************************************/ 126 | /* Data dimensions. As also a 2D tensor can be used I fill empty dimensions 127 | * with 1 */ 128 | const mwSize *tmp = mxGetDimensions(prhs[0]); 129 | DATA_DIMS[0] = tmp[0]; 130 | DATA_DIMS[1] = tmp[1]; 131 | if (mxGetNumberOfDimensions(prhs[0]) == 2) { 132 | DATA_DIMS[2] = 1; 133 | DATA_DIMS[3] = 1; 134 | } else if (mxGetNumberOfDimensions(prhs[0]) == 3) { 135 | DATA_DIMS[2] = tmp[2]; 136 | DATA_DIMS[3] = 1; 137 | } else { 138 | DATA_DIMS[2] = tmp[2]; 139 | DATA_DIMS[3] = tmp[3]; 140 | } 141 | 142 | DATA_NUMEL = DATA_DIMS[0] * DATA_DIMS[1] * DATA_DIMS[2] * DATA_DIMS[3]; 143 | if (debug) 144 | fprintf(stderr,"Data size: h=%d, w=%d, z=%d, n=%d (%d)\n", DATA_DIMS[0], DATA_DIMS[1], DATA_DIMS[2], DATA_DIMS[3], DATA_NUMEL); 145 | 146 | /* Output dimensions: the first output argument is of size equals to the input 147 | * whereas the second is of size equals to the number of pooled values. 148 | * Below there is ceil because also non complete tiles are considered when 149 | * input dims are not multiples of pooling dims. */ 150 | M_DIMS[0] = ceil(float(DATA_DIMS[0]) / float(ptr_pool[0])); 151 | M_DIMS[1] = ceil(float(DATA_DIMS[1]) / float(ptr_pool[1])); 152 | M_DIMS[2] = DATA_DIMS[2]; 153 | M_DIMS[3] = DATA_DIMS[3]; 154 | IDX_DIMS[0] = M_DIMS[0] * M_DIMS[1] * M_DIMS[2] * M_DIMS[3]; 155 | if (debug){ 156 | fprintf(stderr,"Each output image has (%d, %d) pooled values, " 157 | "IDXs size: h=%d \n", M_DIMS[0], M_DIMS[1], IDX_DIMS[0]); 158 | fprintf(stderr, "M size: h=%d, w=%d, z=%d, n=%d\n", M_DIMS[0], M_DIMS[1], M_DIMS[2], M_DIMS[3]); 159 | } 160 | 161 | /***************************************************************************/ 162 | /** Variables allocation ***************************************************/ 163 | /***************************************************************************/ 164 | /* OUTPUTS: max-values and corresponding indices */ 165 | plhs[0] = mxCreateNumericArray(4, M_DIMS, classID, mxREAL); 166 | ptr_out = (T *)mxGetData(plhs[0]); 167 | plhs[1] = mxCreateNumericArray(1, IDX_DIMS, classID, mxREAL); 168 | ptr_idx = (T *)mxGetData(plhs[1]); 169 | 170 | /***************************************************************************/ 171 | /** Compute max-pooling ****************************************************/ 172 | /***************************************************************************/ 173 | int tile_start = 0; 174 | int ptr_offset = 0; 175 | int M_sample_size = M_DIMS[0] * M_DIMS[1] * M_DIMS[2]; 176 | int D_sample_size = DATA_DIMS[0] * DATA_DIMS[1] * DATA_DIMS[2]; 177 | 178 | for (int n = 0; n < DATA_DIMS[3]; ++n) { 179 | #ifdef OPENMP 180 | #pragma omp parallel for 181 | #endif 182 | for (int k = 0; k < DATA_DIMS[2]; ++k) { 183 | tile_start = n * M_sample_size + k * M_DIMS[0] * M_DIMS[1]; 184 | ptr_offset = n * D_sample_size + k * DATA_DIMS[0] * DATA_DIMS[1]; 185 | 186 | compute_map_pooling (&ptr_data[ptr_offset], DATA_DIMS, ptr_pool, &ptr_out[tile_start], &ptr_idx[tile_start], ptr_offset); 187 | 188 | if (debug) 189 | fprintf(stderr, "tile_start: %i, ptr_offset: %i\n", tile_start, ptr_offset); 190 | } 191 | } 192 | } 193 | 194 | void mexFunction(int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[]) 195 | { 196 | /***************************************************************************/ 197 | /** Check input ************************************************************/ 198 | /***************************************************************************/ 199 | if (nrhs !=2) 200 | mexErrMsgTxt("Must have 2 input arguments: x, pooling_shape"); 201 | 202 | if (nlhs !=2) 203 | mexErrMsgTxt("Must have 2 output arguments ([max_value, idxs])"); 204 | 205 | if (mxIsComplex(prhs[0]) || !(mxIsClass(prhs[0],"single") || mxIsClass(prhs[0],"double"))) 206 | mexErrMsgTxt("Input data must be real, single/double type"); 207 | 208 | if (mxIsComplex(prhs[1]) || !(mxIsClass(prhs[1],"single") || mxIsClass(prhs[1],"double"))) 209 | mexErrMsgTxt("Pooling dimensions (rows, cols) must be real, single/double type"); 210 | 211 | if (mxGetNumberOfDimensions(prhs[0]) < 2) 212 | mexErrMsgTxt("Input data must have at least 2-dimensions (rows, cols, nchannels, nsamples) " 213 | "\nThe last two dimensions will be considered to be 1."); 214 | 215 | if (mxGetNumberOfDimensions(prhs[1]) != 2) 216 | mexErrMsgTxt("Pooling data must have 2-dimensions (prows, pcols)"); 217 | 218 | mxClassID classID = mxGetClassID(prhs[0]); 219 | 220 | /** This is mainly to avoid two typenames. Should not be a big usability issue. */ 221 | if (mxGetClassID(prhs[1]) != classID) 222 | mexErrMsgTxt("Input data and pooling need to be of the same type"); 223 | 224 | /***************************************************************************/ 225 | /** Switch for the supported data types */ 226 | /***************************************************************************/ 227 | if (classID == mxSINGLE_CLASS) { 228 | if (debug) 229 | fprintf(stderr, "Executing the single version\n"); 230 | 231 | mexMaxPooling(nlhs, plhs, nrhs, prhs, classID); 232 | } else if (classID == mxDOUBLE_CLASS) { 233 | if (debug) 234 | fprintf(stderr, "Executing the double version\n"); 235 | 236 | mexMaxPooling(nlhs, plhs, nrhs, prhs, classID); 237 | } 238 | } 239 | -------------------------------------------------------------------------------- /CNN/MaxPooling.m: -------------------------------------------------------------------------------- 1 | %MAXPOOLING implements a max-pooling operation for 4D tensors shaped as: 2 | % (rows, cols, channels, samples) and returns the pooled data with the 3 | % corresponding indices. 4 | % 5 | % [m, idx] = MaxPooling(IM, [2 2]) 6 | % 7 | % IM can also be a 2D tensor, the missing dims are set to 1. 8 | 9 | % AUTORIGHTS 10 | % Copyright (C) 2011 Jonathan Masci 11 | % 12 | % This file is available under the terms of the 13 | % GNU GPLv2. 14 | -------------------------------------------------------------------------------- /CNN/MaxPooling.mexw64: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/happynear/DeepLearnToolbox/2539a8cc2dcd0faab7215949b9714d7705c18738/CNN/MaxPooling.mexw64 -------------------------------------------------------------------------------- /CNN/MaxPoolingGPU.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * MaxPooling.c 3 | * 4 | * Implements the max-pooling transfer function. Takes a 4D tensor shaped as 5 | * (rows, cols, nchannels, nsamples) and a pooling shape as (prows, pcols) and 6 | * returns a set of max-values with the corresponding indices in the input 7 | * matrix. 8 | * 9 | * e.g. 10 | * [m, idx] = MaxPooling(IM, [2 2]) 11 | * 12 | * Created on: July 11, 2011 13 | * Author: Jonathan Masci 14 | * 15 | * This file is available under the terms of the GNU GPLv2. 16 | */ 17 | 18 | #include "mex.h" 19 | #include "gpu/mxGPUArray.h" 20 | 21 | #ifdef OPENMP 22 | #include 23 | #endif 24 | 25 | #include 26 | #include 27 | #include 28 | #include 29 | 30 | #define IDX2F(i,j,ld) ((((j)-1)*(ld))+((i)-1)) 31 | #define IDX2C(i,j,ld) (((j)*(ld))+(i)) 32 | 33 | int debug = 0; 34 | 35 | /** 36 | * Computes the max-pooling for the given 2D map, and no, the name is not a typo. 37 | * All pointers are passed already offset so to avoid cumbersome indexing. 38 | * 39 | * @param ptr_data pointer set to the begin of this map 40 | * @param DATA_DIMS data dimensions 41 | * @param ptr_pool pooling sizes 42 | * @param ptr_out pointer to the output max-values set to the right position 43 | * @param ptr_idx pointer to the output indices set to the right position 44 | */ 45 | template 46 | inline void compute_map_pooling(T *ptr_data, const mwSize *DATA_DIMS, T *ptr_pool, 47 | T *ptr_out, T *ptr_idx, int tile_start) 48 | { 49 | T m; 50 | int idx; 51 | int count = 0; 52 | 53 | for (int col = 0; col < DATA_DIMS[1]; col += ptr_pool[1]) { 54 | for (int row = 0; row < DATA_DIMS[0]; row += ptr_pool[0]) { 55 | if (debug) 56 | fprintf(stderr, "r = %i, c = %i \n", row, col); 57 | 58 | m = -std::numeric_limits::max(); 59 | idx = -1; 60 | for (int pcol = 0; (pcol < ptr_pool[1] && col + pcol < DATA_DIMS[1]); ++pcol) { 61 | for (int prow = 0; (prow < ptr_pool[0] && row + prow < DATA_DIMS[0]); ++prow) { 62 | if (debug) { 63 | fprintf(stderr, "m = %f, data = %f \n", m, ptr_data[IDX2C(row + prow, col + pcol, DATA_DIMS[0])]); 64 | fprintf(stderr, "rr = %i, cc = %i \n --> idx = %i \n", row + prow, col + pcol, idx); 65 | } 66 | 67 | if (ptr_data[IDX2C(row + prow, col + pcol, DATA_DIMS[0])] > m) { 68 | idx = IDX2C(row + prow, col + pcol, DATA_DIMS[0]); 69 | m = ptr_data[idx]; 70 | } 71 | } 72 | } 73 | 74 | if (debug && idx == -1) { 75 | fprintf(stderr, "dioschifoso\n"); 76 | return; 77 | } 78 | 79 | if (debug) 80 | fprintf(stderr, "count = %i\n",count); 81 | 82 | /* idxs are to be used in Matlab and hence a +1 is needed */ 83 | ptr_idx[count] = idx + 1 + tile_start; 84 | ptr_out[count] = m; 85 | count++; 86 | } 87 | } 88 | } 89 | 90 | /** 91 | * This is the wrapper for the actual computation. 92 | * It is a template so that multiple types can be handled. 93 | */ 94 | template 95 | void mexMaxPoolingGPU(int nlhs, mxGPUArray *plhs[], int nrhs, const mxArray *prhs[], mxClassID classID) 96 | { 97 | 98 | /***************************************************************************/ 99 | /** Variables */ 100 | /***************************************************************************/ 101 | mwSize IDX_DIMS[1]; 102 | mwSize DATA_DIMS[4]; 103 | mwSize M_DIMS[4]; 104 | const mwSize *POOL_DIMS; 105 | int DATA_NUMEL; 106 | int POOL_NUMEL; 107 | 108 | /** 109 | * Pointers to data 110 | */ 111 | T *ptr_data = NULL; 112 | T *ptr_pool = NULL; 113 | T *ptr_out = NULL; 114 | T *ptr_idx = NULL; 115 | 116 | /***************************************************************************/ 117 | /** Setting input pointers *************************************************/ 118 | /***************************************************************************/ 119 | ptr_data = (T *)mxGetData(prhs[0]); 120 | ptr_pool = (T *)mxGetData(prhs[1]); 121 | if (debug) 122 | fprintf(stderr,"Pooling size: h=%f, w=%f\n", ptr_pool[0], ptr_pool[1]); 123 | 124 | /***************************************************************************/ 125 | /** Setting parameters *****************************************************/ 126 | /***************************************************************************/ 127 | /* Data dimensions. As also a 2D tensor can be used I fill empty dimensions 128 | * with 1 */ 129 | const mwSize *tmp = mxGetDimensions(prhs[0]); 130 | DATA_DIMS[0] = tmp[0]; 131 | DATA_DIMS[1] = tmp[1]; 132 | if (mxGetNumberOfDimensions(prhs[0]) == 2) { 133 | DATA_DIMS[2] = 1; 134 | DATA_DIMS[3] = 1; 135 | } else if (mxGetNumberOfDimensions(prhs[0]) == 3) { 136 | DATA_DIMS[2] = tmp[2]; 137 | DATA_DIMS[3] = 1; 138 | } else { 139 | DATA_DIMS[2] = tmp[2]; 140 | DATA_DIMS[3] = tmp[3]; 141 | } 142 | 143 | DATA_NUMEL = DATA_DIMS[0] * DATA_DIMS[1] * DATA_DIMS[2] * DATA_DIMS[3]; 144 | if (debug) 145 | fprintf(stderr,"Data size: h=%d, w=%d, z=%d, n=%d (%d)\n", DATA_DIMS[0], DATA_DIMS[1], DATA_DIMS[2], DATA_DIMS[3], DATA_NUMEL); 146 | 147 | /* Output dimensions: the first output argument is of size equals to the input 148 | * whereas the second is of size equals to the number of pooled values. 149 | * Below there is ceil because also non complete tiles are considered when 150 | * input dims are not multiples of pooling dims. */ 151 | M_DIMS[0] = ceil(float(DATA_DIMS[0]) / float(ptr_pool[0])); 152 | M_DIMS[1] = ceil(float(DATA_DIMS[1]) / float(ptr_pool[1])); 153 | M_DIMS[2] = DATA_DIMS[2]; 154 | M_DIMS[3] = DATA_DIMS[3]; 155 | IDX_DIMS[0] = M_DIMS[0] * M_DIMS[1] * M_DIMS[2] * M_DIMS[3]; 156 | if (debug){ 157 | fprintf(stderr,"Each output image has (%d, %d) pooled values, " 158 | "IDXs size: h=%d \n", M_DIMS[0], M_DIMS[1], IDX_DIMS[0]); 159 | fprintf(stderr, "M size: h=%d, w=%d, z=%d, n=%d\n", M_DIMS[0], M_DIMS[1], M_DIMS[2], M_DIMS[3]); 160 | } 161 | 162 | /***************************************************************************/ 163 | /** Variables allocation ***************************************************/ 164 | /***************************************************************************/ 165 | /* OUTPUTS: max-values and corresponding indices */ 166 | plhs[0] = mxCreateNumericArray(4, M_DIMS, classID, mxREAL); 167 | ptr_out = (T *)mxGetData(plhs[0]); 168 | plhs[1] = mxCreateNumericArray(1, IDX_DIMS, classID, mxREAL); 169 | ptr_idx = (T *)mxGetData(plhs[1]); 170 | 171 | /***************************************************************************/ 172 | /** Compute max-pooling ****************************************************/ 173 | /***************************************************************************/ 174 | int tile_start = 0; 175 | int ptr_offset = 0; 176 | int M_sample_size = M_DIMS[0] * M_DIMS[1] * M_DIMS[2]; 177 | int D_sample_size = DATA_DIMS[0] * DATA_DIMS[1] * DATA_DIMS[2]; 178 | 179 | for (int n = 0; n < DATA_DIMS[3]; ++n) { 180 | #ifdef OPENMP 181 | #pragma omp parallel for 182 | #endif 183 | for (int k = 0; k < DATA_DIMS[2]; ++k) { 184 | tile_start = n * M_sample_size + k * M_DIMS[0] * M_DIMS[1]; 185 | ptr_offset = n * D_sample_size + k * DATA_DIMS[0] * DATA_DIMS[1]; 186 | 187 | compute_map_pooling (&ptr_data[ptr_offset], DATA_DIMS, ptr_pool, &ptr_out[tile_start], &ptr_idx[tile_start], ptr_offset); 188 | 189 | if (debug) 190 | fprintf(stderr, "tile_start: %i, ptr_offset: %i\n", tile_start, ptr_offset); 191 | } 192 | } 193 | } 194 | 195 | void mexFunction(int nlhs, mxGPUArray *plhs[], int nrhs, const mxGPUArray *prhs[]) 196 | { 197 | /***************************************************************************/ 198 | /** Check input ************************************************************/ 199 | /***************************************************************************/ 200 | mxInitGPU(); 201 | if (nrhs !=2) 202 | mexErrMsgTxt("Must have 2 input arguments: x, pooling_shape"); 203 | 204 | if (nlhs !=2) 205 | mexErrMsgTxt("Must have 2 output arguments ([max_value, idxs])"); 206 | 207 | if (mxIsComplex(prhs[0]) || !(mxIsClass(prhs[0],"single") || mxIsClass(prhs[0],"double"))) 208 | mexErrMsgTxt("Input data must be real, single/double type"); 209 | 210 | if (mxIsComplex(prhs[1]) || !(mxIsClass(prhs[1],"single") || mxIsClass(prhs[1],"double"))) 211 | mexErrMsgTxt("Pooling dimensions (rows, cols) must be real, single/double type"); 212 | 213 | if (mxGetNumberOfDimensions(prhs[0]) < 2) 214 | mexErrMsgTxt("Input data must have at least 2-dimensions (rows, cols, nchannels, nsamples) " 215 | "\nThe last two dimensions will be considered to be 1."); 216 | 217 | if (mxGetNumberOfDimensions(prhs[1]) != 2) 218 | mexErrMsgTxt("Pooling data must have 2-dimensions (prows, pcols)"); 219 | 220 | mxClassID classID = mxGetClassID(prhs[0]); 221 | 222 | /** This is mainly to avoid two typenames. Should not be a big usability issue. */ 223 | if (mxGetClassID(prhs[1]) != classID) 224 | mexErrMsgTxt("Input data and pooling need to be of the same type"); 225 | 226 | /***************************************************************************/ 227 | /** Switch for the supported data types */ 228 | /***************************************************************************/ 229 | if (classID == mxSINGLE_CLASS) { 230 | if (debug) 231 | fprintf(stderr, "Executing the single version\n"); 232 | 233 | mexMaxPoolingGPU(nlhs, plhs, nrhs, prhs, classID); 234 | } else if (classID == mxDOUBLE_CLASS) { 235 | if (debug) 236 | fprintf(stderr, "Executing the double version\n"); 237 | 238 | mexMaxPoolingGPU(nlhs, plhs, nrhs, prhs, classID); 239 | } 240 | else{ 241 | mexMaxPoolingGPU(nlhs, plhs, nrhs, prhs, classID); 242 | } 243 | } 244 | -------------------------------------------------------------------------------- /CNN/MaxPoolingGPU.cpp.bak: -------------------------------------------------------------------------------- 1 | /* 2 | * MaxPooling.c 3 | * 4 | * Implements the max-pooling transfer function. Takes a 4D tensor shaped as 5 | * (rows, cols, nchannels, nsamples) and a pooling shape as (prows, pcols) and 6 | * returns a set of max-values with the corresponding indices in the input 7 | * matrix. 8 | * 9 | * e.g. 10 | * [m, idx] = MaxPooling(IM, [2 2]) 11 | * 12 | * Created on: July 11, 2011 13 | * Author: Jonathan Masci 14 | * 15 | * This file is available under the terms of the GNU GPLv2. 16 | */ 17 | 18 | #include "mex.h" 19 | 20 | #ifdef OPENMP 21 | #include 22 | #endif 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | 29 | #define IDX2F(i,j,ld) ((((j)-1)*(ld))+((i)-1)) 30 | #define IDX2C(i,j,ld) (((j)*(ld))+(i)) 31 | 32 | int debug = 0; 33 | 34 | /** 35 | * Computes the max-pooling for the given 2D map, and no, the name is not a typo. 36 | * All pointers are passed already offset so to avoid cumbersome indexing. 37 | * 38 | * @param ptr_data pointer set to the begin of this map 39 | * @param DATA_DIMS data dimensions 40 | * @param ptr_pool pooling sizes 41 | * @param ptr_out pointer to the output max-values set to the right position 42 | * @param ptr_idx pointer to the output indices set to the right position 43 | */ 44 | template 45 | inline void compute_map_pooling(T *ptr_data, const mwSize *DATA_DIMS, T *ptr_pool, 46 | T *ptr_out, T *ptr_idx, int tile_start) 47 | { 48 | T m; 49 | int idx; 50 | int count = 0; 51 | 52 | for (int col = 0; col < DATA_DIMS[1]; col += ptr_pool[1]) { 53 | for (int row = 0; row < DATA_DIMS[0]; row += ptr_pool[0]) { 54 | if (debug) 55 | fprintf(stderr, "r = %i, c = %i \n", row, col); 56 | 57 | m = -std::numeric_limits::max(); 58 | idx = -1; 59 | for (int pcol = 0; (pcol < ptr_pool[1] && col + pcol < DATA_DIMS[1]); ++pcol) { 60 | for (int prow = 0; (prow < ptr_pool[0] && row + prow < DATA_DIMS[0]); ++prow) { 61 | if (debug) { 62 | fprintf(stderr, "m = %f, data = %f \n", m, ptr_data[IDX2C(row + prow, col + pcol, DATA_DIMS[0])]); 63 | fprintf(stderr, "rr = %i, cc = %i \n --> idx = %i \n", row + prow, col + pcol, idx); 64 | } 65 | 66 | if (ptr_data[IDX2C(row + prow, col + pcol, DATA_DIMS[0])] > m) { 67 | idx = IDX2C(row + prow, col + pcol, DATA_DIMS[0]); 68 | m = ptr_data[idx]; 69 | } 70 | } 71 | } 72 | 73 | if (debug && idx == -1) { 74 | fprintf(stderr, "dioschifoso\n"); 75 | return; 76 | } 77 | 78 | if (debug) 79 | fprintf(stderr, "count = %i\n",count); 80 | 81 | /* idxs are to be used in Matlab and hence a +1 is needed */ 82 | ptr_idx[count] = idx + 1 + tile_start; 83 | ptr_out[count] = m; 84 | count++; 85 | } 86 | } 87 | } 88 | 89 | /** 90 | * This is the wrapper for the actual computation. 91 | * It is a template so that multiple types can be handled. 92 | */ 93 | template 94 | void mexMaxPoolingGPU(int nlhs, mxGPUArray *plhs[], int nrhs, const mxArray *prhs[], mxClassID classID) 95 | { 96 | 97 | /***************************************************************************/ 98 | /** Variables */ 99 | /***************************************************************************/ 100 | mwSize IDX_DIMS[1]; 101 | mwSize DATA_DIMS[4]; 102 | mwSize M_DIMS[4]; 103 | const mwSize *POOL_DIMS; 104 | int DATA_NUMEL; 105 | int POOL_NUMEL; 106 | 107 | /** 108 | * Pointers to data 109 | */ 110 | T *ptr_data = NULL; 111 | T *ptr_pool = NULL; 112 | T *ptr_out = NULL; 113 | T *ptr_idx = NULL; 114 | 115 | /***************************************************************************/ 116 | /** Setting input pointers *************************************************/ 117 | /***************************************************************************/ 118 | ptr_data = (T *)mxGetData(prhs[0]); 119 | ptr_pool = (T *)mxGetData(prhs[1]); 120 | if (debug) 121 | fprintf(stderr,"Pooling size: h=%f, w=%f\n", ptr_pool[0], ptr_pool[1]); 122 | 123 | /***************************************************************************/ 124 | /** Setting parameters *****************************************************/ 125 | /***************************************************************************/ 126 | /* Data dimensions. As also a 2D tensor can be used I fill empty dimensions 127 | * with 1 */ 128 | const mwSize *tmp = mxGetDimensions(prhs[0]); 129 | DATA_DIMS[0] = tmp[0]; 130 | DATA_DIMS[1] = tmp[1]; 131 | if (mxGetNumberOfDimensions(prhs[0]) == 2) { 132 | DATA_DIMS[2] = 1; 133 | DATA_DIMS[3] = 1; 134 | } else if (mxGetNumberOfDimensions(prhs[0]) == 3) { 135 | DATA_DIMS[2] = tmp[2]; 136 | DATA_DIMS[3] = 1; 137 | } else { 138 | DATA_DIMS[2] = tmp[2]; 139 | DATA_DIMS[3] = tmp[3]; 140 | } 141 | 142 | DATA_NUMEL = DATA_DIMS[0] * DATA_DIMS[1] * DATA_DIMS[2] * DATA_DIMS[3]; 143 | if (debug) 144 | fprintf(stderr,"Data size: h=%d, w=%d, z=%d, n=%d (%d)\n", DATA_DIMS[0], DATA_DIMS[1], DATA_DIMS[2], DATA_DIMS[3], DATA_NUMEL); 145 | 146 | /* Output dimensions: the first output argument is of size equals to the input 147 | * whereas the second is of size equals to the number of pooled values. 148 | * Below there is ceil because also non complete tiles are considered when 149 | * input dims are not multiples of pooling dims. */ 150 | M_DIMS[0] = ceil(float(DATA_DIMS[0]) / float(ptr_pool[0])); 151 | M_DIMS[1] = ceil(float(DATA_DIMS[1]) / float(ptr_pool[1])); 152 | M_DIMS[2] = DATA_DIMS[2]; 153 | M_DIMS[3] = DATA_DIMS[3]; 154 | IDX_DIMS[0] = M_DIMS[0] * M_DIMS[1] * M_DIMS[2] * M_DIMS[3]; 155 | if (debug){ 156 | fprintf(stderr,"Each output image has (%d, %d) pooled values, " 157 | "IDXs size: h=%d \n", M_DIMS[0], M_DIMS[1], IDX_DIMS[0]); 158 | fprintf(stderr, "M size: h=%d, w=%d, z=%d, n=%d\n", M_DIMS[0], M_DIMS[1], M_DIMS[2], M_DIMS[3]); 159 | } 160 | 161 | /***************************************************************************/ 162 | /** Variables allocation ***************************************************/ 163 | /***************************************************************************/ 164 | /* OUTPUTS: max-values and corresponding indices */ 165 | plhs[0] = mxCreateNumericArray(4, M_DIMS, classID, mxREAL); 166 | ptr_out = (T *)mxGetData(plhs[0]); 167 | plhs[1] = mxCreateNumericArray(1, IDX_DIMS, classID, mxREAL); 168 | ptr_idx = (T *)mxGetData(plhs[1]); 169 | 170 | /***************************************************************************/ 171 | /** Compute max-pooling ****************************************************/ 172 | /***************************************************************************/ 173 | int tile_start = 0; 174 | int ptr_offset = 0; 175 | int M_sample_size = M_DIMS[0] * M_DIMS[1] * M_DIMS[2]; 176 | int D_sample_size = DATA_DIMS[0] * DATA_DIMS[1] * DATA_DIMS[2]; 177 | 178 | for (int n = 0; n < DATA_DIMS[3]; ++n) { 179 | #ifdef OPENMP 180 | #pragma omp parallel for 181 | #endif 182 | for (int k = 0; k < DATA_DIMS[2]; ++k) { 183 | tile_start = n * M_sample_size + k * M_DIMS[0] * M_DIMS[1]; 184 | ptr_offset = n * D_sample_size + k * DATA_DIMS[0] * DATA_DIMS[1]; 185 | 186 | compute_map_pooling (&ptr_data[ptr_offset], DATA_DIMS, ptr_pool, &ptr_out[tile_start], &ptr_idx[tile_start], ptr_offset); 187 | 188 | if (debug) 189 | fprintf(stderr, "tile_start: %i, ptr_offset: %i\n", tile_start, ptr_offset); 190 | } 191 | } 192 | } 193 | 194 | void mexFunction(int nlhs, mxGPUArray *plhs[], int nrhs, const mxGPUArray *prhs[]) 195 | { 196 | /***************************************************************************/ 197 | /** Check input ************************************************************/ 198 | /***************************************************************************/ 199 | mxInitGPU(); 200 | if (nrhs !=2) 201 | mexErrMsgTxt("Must have 2 input arguments: x, pooling_shape"); 202 | 203 | if (nlhs !=2) 204 | mexErrMsgTxt("Must have 2 output arguments ([max_value, idxs])"); 205 | 206 | if (mxIsComplex(prhs[0]) || !(mxIsClass(prhs[0],"single") || mxIsClass(prhs[0],"double"))) 207 | mexErrMsgTxt("Input data must be real, single/double type"); 208 | 209 | if (mxIsComplex(prhs[1]) || !(mxIsClass(prhs[1],"single") || mxIsClass(prhs[1],"double"))) 210 | mexErrMsgTxt("Pooling dimensions (rows, cols) must be real, single/double type"); 211 | 212 | if (mxGetNumberOfDimensions(prhs[0]) < 2) 213 | mexErrMsgTxt("Input data must have at least 2-dimensions (rows, cols, nchannels, nsamples) " 214 | "\nThe last two dimensions will be considered to be 1."); 215 | 216 | if (mxGetNumberOfDimensions(prhs[1]) != 2) 217 | mexErrMsgTxt("Pooling data must have 2-dimensions (prows, pcols)"); 218 | 219 | mxClassID classID = mxGetClassID(prhs[0]); 220 | 221 | /** This is mainly to avoid two typenames. Should not be a big usability issue. */ 222 | if (mxGetClassID(prhs[1]) != classID) 223 | mexErrMsgTxt("Input data and pooling need to be of the same type"); 224 | 225 | /***************************************************************************/ 226 | /** Switch for the supported data types */ 227 | /***************************************************************************/ 228 | if (classID == mxSINGLE_CLASS) { 229 | if (debug) 230 | fprintf(stderr, "Executing the single version\n"); 231 | 232 | mexMaxPooling(nlhs, plhs, nrhs, prhs, classID); 233 | } else if (classID == mxDOUBLE_CLASS) { 234 | if (debug) 235 | fprintf(stderr, "Executing the double version\n"); 236 | 237 | mexMaxPoolingGPU(nlhs, plhs, nrhs, prhs, classID); 238 | } 239 | } 240 | -------------------------------------------------------------------------------- /CNN/StochasticPooling.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * StochasticPooling.c 3 | * 4 | * Implements the stochastic-pooling transfer function. Takes a 4D tensor shaped as 5 | * (rows, cols, nchannels, nsamples) and a pooling shape as (prows, pcols) and 6 | * returns a set of stochastic sampled values with the corresponding indices in the input 7 | * matrix. 8 | * 9 | * e.g. 10 | * [m, idx] = StochasticPooling(IM, [2 2]) 11 | * 12 | * Created on: March 28, 2014 13 | * Author:Feng Wang 14 | * Based on the max-pooling code provided by: 15 | * Jonathan Masci 16 | * Paper: 17 | * Stochastic Pooling for Regularization of Deep Convolutional Neural Networks, Matthew D. Zeiler 18 | * 19 | * This file is available under the terms of the GNU GPLv2. 20 | */ 21 | 22 | #include "mex.h" 23 | 24 | #include 25 | 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | 32 | #define IDX2F(i,j,ld) ((((j)-1)*(ld))+((i)-1)) 33 | #define IDX2C(i,j,ld) (((j)*(ld))+(i)) 34 | 35 | int debug = 0; 36 | 37 | /** 38 | * Computes the stochastic-pooling for the given 2D map, and no, the name is not a typo. 39 | * All pointers are passed already offset so to avoid cumbersome indexing. 40 | * 41 | * @param ptr_data pointer set to the begin of this map 42 | * @param DATA_DIMS data dimensions 43 | * @param ptr_pool pooling sizes 44 | * @param ptr_out pointer to the output stochastic-values set to the right position 45 | * @param ptr_idx pointer to the output indices set to the right position 46 | */ 47 | template 48 | inline void compute_stochastic_pooling(T *ptr_data, const mwSize *DATA_DIMS, T *ptr_pool, 49 | T *ptr_out, T *ptr_idx, int tile_start) 50 | { 51 | T m; 52 | T sum,rsum; 53 | int idx; 54 | int count = 0; 55 | 56 | for (int col = 0; col < DATA_DIMS[1]; col += ptr_pool[1]) { 57 | for (int row = 0; row < DATA_DIMS[0]; row += ptr_pool[0]) { 58 | if (debug) 59 | fprintf(stderr, "r = %i, c = %i \n", row, col); 60 | 61 | m = 0; 62 | sum = 0; 63 | rsum = 0; 64 | idx = -1; 65 | for (int pcol = 0; (pcol < ptr_pool[1] && col + pcol < DATA_DIMS[1]); ++pcol) { 66 | for (int prow = 0; (prow < ptr_pool[0] && row + prow < DATA_DIMS[0]); ++prow) { 67 | sum += ptr_data[IDX2C(row + prow, col + pcol, DATA_DIMS[0])]; 68 | } 69 | } 70 | float num = rand()%1000; 71 | num *= sum/1000; 72 | for (int pcol = 0; (pcol < ptr_pool[1] && col + pcol < DATA_DIMS[1]); ++pcol) { 73 | for (int prow = 0; (prow < ptr_pool[0] && row + prow < DATA_DIMS[0]); ++prow) { 74 | if (debug) { 75 | fprintf(stderr, "num = %f, rsum = %f, this = %f \n", num, rsum, ptr_data[IDX2C(row + prow, col + pcol, DATA_DIMS[0])]); 76 | } 77 | rsum += ptr_data[IDX2C(row + prow, col + pcol, DATA_DIMS[0])]; 78 | if(num 116 | void mexStochasticPooling(int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[], mxClassID classID) 117 | { 118 | 119 | /***************************************************************************/ 120 | /** Variables */ 121 | /***************************************************************************/ 122 | mwSize IDX_DIMS[1]; 123 | mwSize DATA_DIMS[4]; 124 | mwSize M_DIMS[4]; 125 | const mwSize *POOL_DIMS; 126 | int DATA_NUMEL; 127 | int POOL_NUMEL; 128 | 129 | /** 130 | * Pointers to data 131 | */ 132 | T *ptr_data = NULL; 133 | T *ptr_pool = NULL; 134 | T *ptr_out = NULL; 135 | T *ptr_idx = NULL; 136 | 137 | /***************************************************************************/ 138 | /** Setting input pointers *************************************************/ 139 | /***************************************************************************/ 140 | ptr_data = (T *)mxGetData(prhs[0]); 141 | ptr_pool = (T *)mxGetData(prhs[1]); 142 | if (debug) 143 | fprintf(stderr,"Pooling size: h=%f, w=%f\n", ptr_pool[0], ptr_pool[1]); 144 | 145 | /***************************************************************************/ 146 | /** Setting parameters *****************************************************/ 147 | /***************************************************************************/ 148 | /* Data dimensions. As also a 2D tensor can be used I fill empty dimensions 149 | * with 1 */ 150 | const mwSize *tmp = mxGetDimensions(prhs[0]); 151 | DATA_DIMS[0] = tmp[0]; 152 | DATA_DIMS[1] = tmp[1]; 153 | if (mxGetNumberOfDimensions(prhs[0]) == 2) { 154 | DATA_DIMS[2] = 1; 155 | DATA_DIMS[3] = 1; 156 | } else if (mxGetNumberOfDimensions(prhs[0]) == 3) { 157 | DATA_DIMS[2] = tmp[2]; 158 | DATA_DIMS[3] = 1; 159 | } else { 160 | DATA_DIMS[2] = tmp[2]; 161 | DATA_DIMS[3] = tmp[3]; 162 | } 163 | 164 | DATA_NUMEL = DATA_DIMS[0] * DATA_DIMS[1] * DATA_DIMS[2] * DATA_DIMS[3]; 165 | if (debug) 166 | fprintf(stderr,"Data size: h=%d, w=%d, z=%d, n=%d (%d)\n", DATA_DIMS[0], DATA_DIMS[1], DATA_DIMS[2], DATA_DIMS[3], DATA_NUMEL); 167 | 168 | /* Output dimensions: the first output argument is of size equals to the input 169 | * whereas the second is of size equals to the number of pooled values. 170 | * Below there is ceil because also non complete tiles are considered when 171 | * input dims are not multiples of pooling dims. */ 172 | M_DIMS[0] = ceil(float(DATA_DIMS[0]) / float(ptr_pool[0])); 173 | M_DIMS[1] = ceil(float(DATA_DIMS[1]) / float(ptr_pool[1])); 174 | M_DIMS[2] = DATA_DIMS[2]; 175 | M_DIMS[3] = DATA_DIMS[3]; 176 | IDX_DIMS[0] = M_DIMS[0] * M_DIMS[1] * M_DIMS[2] * M_DIMS[3]; 177 | if (debug){ 178 | fprintf(stderr,"Each output image has (%d, %d) pooled values, " 179 | "IDXs size: h=%d \n", M_DIMS[0], M_DIMS[1], IDX_DIMS[0]); 180 | fprintf(stderr, "M size: h=%d, w=%d, z=%d, n=%d\n", M_DIMS[0], M_DIMS[1], M_DIMS[2], M_DIMS[3]); 181 | } 182 | 183 | /***************************************************************************/ 184 | /** Variables allocation ***************************************************/ 185 | /***************************************************************************/ 186 | /* OUTPUTS: stochastic-values and corresponding indices */ 187 | plhs[0] = mxCreateNumericArray(4, M_DIMS, classID, mxREAL); 188 | ptr_out = (T *)mxGetData(plhs[0]); 189 | plhs[1] = mxCreateNumericArray(1, IDX_DIMS, classID, mxREAL); 190 | ptr_idx = (T *)mxGetData(plhs[1]); 191 | 192 | /***************************************************************************/ 193 | /** Compute stochastic-pooling ****************************************************/ 194 | /***************************************************************************/ 195 | int tile_start = 0; 196 | int ptr_offset = 0; 197 | int M_sample_size = M_DIMS[0] * M_DIMS[1] * M_DIMS[2]; 198 | int D_sample_size = DATA_DIMS[0] * DATA_DIMS[1] * DATA_DIMS[2]; 199 | srand((unsigned)time(0)); 200 | for (int n = 0; n < DATA_DIMS[3]; ++n) { 201 | #pragma omp parallel for 202 | for (int k = 0; k < DATA_DIMS[2]; ++k) { 203 | tile_start = n * M_sample_size + k * M_DIMS[0] * M_DIMS[1]; 204 | ptr_offset = n * D_sample_size + k * DATA_DIMS[0] * DATA_DIMS[1]; 205 | 206 | compute_stochastic_pooling (&ptr_data[ptr_offset], DATA_DIMS, ptr_pool, &ptr_out[tile_start], &ptr_idx[tile_start], ptr_offset); 207 | 208 | if (debug) 209 | fprintf(stderr, "tile_start: %i, ptr_offset: %i\n", tile_start, ptr_offset); 210 | } 211 | } 212 | } 213 | 214 | void mexFunction(int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[]) 215 | { 216 | /***************************************************************************/ 217 | /** Check input ************************************************************/ 218 | /***************************************************************************/ 219 | if (nrhs !=2) 220 | mexErrMsgTxt("Must have 2 input arguments: x, pooling_shape"); 221 | 222 | if (nlhs !=2) 223 | mexErrMsgTxt("Must have 2 output arguments ([pooled_value, idxs])"); 224 | 225 | if (mxIsComplex(prhs[0]) || !(mxIsClass(prhs[0],"single") || mxIsClass(prhs[0],"double"))) 226 | mexErrMsgTxt("Input data must be real, single/double type"); 227 | 228 | if (mxIsComplex(prhs[1]) || !(mxIsClass(prhs[1],"single") || mxIsClass(prhs[1],"double"))) 229 | mexErrMsgTxt("Pooling dimensions (rows, cols) must be real, single/double type"); 230 | 231 | if (mxGetNumberOfDimensions(prhs[0]) < 2) 232 | mexErrMsgTxt("Input data must have at least 2-dimensions (rows, cols, nchannels, nsamples) " 233 | "\nThe last two dimensions will be considered to be 1."); 234 | 235 | if (mxGetNumberOfDimensions(prhs[1]) != 2) 236 | mexErrMsgTxt("Pooling data must have 2-dimensions (prows, pcols)"); 237 | 238 | mxClassID classID = mxGetClassID(prhs[0]); 239 | 240 | /** This is mainly to avoid two typenames. Should not be a big usability issue. */ 241 | if (mxGetClassID(prhs[1]) != classID) 242 | mexErrMsgTxt("Input data and pooling need to be of the same type"); 243 | 244 | /***************************************************************************/ 245 | /** Switch for the supported data types */ 246 | /***************************************************************************/ 247 | if (classID == mxSINGLE_CLASS) { 248 | if (debug) 249 | fprintf(stderr, "Executing the single version\n"); 250 | 251 | mexStochasticPooling(nlhs, plhs, nrhs, prhs, classID); 252 | } else if (classID == mxDOUBLE_CLASS) { 253 | if (debug) 254 | fprintf(stderr, "Executing the double version\n"); 255 | 256 | mexStochasticPooling(nlhs, plhs, nrhs, prhs, classID); 257 | } 258 | } 259 | -------------------------------------------------------------------------------- /CNN/StochasticPooling.mexw64: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/happynear/DeepLearnToolbox/2539a8cc2dcd0faab7215949b9714d7705c18738/CNN/StochasticPooling.mexw64 -------------------------------------------------------------------------------- /CNN/StochaticTest.m: -------------------------------------------------------------------------------- 1 | function pooledFeatures = StochaticTest(poolDim, convolvedFeatures) 2 | %cnnPool Pools the given convolved features 3 | % 4 | % Parameters: 5 | % poolDim - dimension of pooling region 6 | % convolvedFeatures - convolved features to pool (as given by cnnConvolve) 7 | % convolvedFeatures(imageRow, imageCol, featureNum, imageNum) 8 | % 9 | % Returns: 10 | % pooledFeatures - matrix of pooled features in the form 11 | % pooledFeatures(poolRow, poolCol, featureNum, imageNum) 12 | % 13 | 14 | numImages = size(convolvedFeatures, 4); 15 | numFilters = size(convolvedFeatures, 3); 16 | convolvedDim = size(convolvedFeatures, 1); 17 | 18 | pooledFeatures = zeros(convolvedDim / poolDim, ... 19 | convolvedDim / poolDim, numFilters, numImages); 20 | 21 | maxPosition = zeros(convolvedDim,convolvedDim,numFilters,numImages); 22 | 23 | % Instructions: 24 | % Now pool the convolved features in regions of poolDim x poolDim, 25 | % to obtain the 26 | % (convolvedDim/poolDim) x (convolvedDim/poolDim) x numFeatures x numImages 27 | % matrix pooledFeatures, such that 28 | % pooledFeatures(poolRow, poolCol, featureNum, imageNum) is the 29 | % value of the featureNum feature for the imageNum image pooled over the 30 | % corresponding (poolRow, poolCol) pooling region. 31 | % 32 | % Use mean pooling here. 33 | 34 | %%% YOUR CODE HERE %%% 35 | 36 | sumFilter = ones(poolDim,poolDim); 37 | for i=1:numFilters 38 | for j=1:numImages 39 | sumSquareImage=conv2(squeeze(convolvedFeatures(:,:,i,j).^2),sumFilter,'valid'); 40 | sumImage=conv2(squeeze(convolvedFeatures(:,:,i,j)),sumFilter,'valid'); 41 | pooledFeatures(:,:,i,j)=sumSquareImage(1:poolDim:end,1:poolDim:end)./sumImage(1:poolDim:end,1:poolDim:end); 42 | end; 43 | end; 44 | pooledFeatures(isnan(pooledFeatures))=0; 45 | end 46 | 47 | -------------------------------------------------------------------------------- /CNN/cnnapplygrads.m: -------------------------------------------------------------------------------- 1 | function net = cnnapplygrads(net, opts) 2 | mom = 0.5; 3 | net.iter = net.iter +1; 4 | if net.iter == opts.momIncrease 5 | mom = opts.momentum; 6 | end; 7 | for l = 2 : numel(net.layers) 8 | if strcmp(net.layers{l}.type, 'c') 9 | for j = 1 : numel(net.layers{l}.a) 10 | for ii = 1 : numel(net.layers{l - 1}.a) 11 | net.layers{l}.vk{ii}{j} = mom * net.layers{l}.vk{ii}{j} + opts.alpha * net.layers{l}.dk{ii}{j}; 12 | net.layers{l}.k{ii}{j} = net.layers{l}.k{ii}{j} - net.layers{l}.vk{ii}{j}; 13 | end 14 | net.layers{l}.vb{j} = mom * net.layers{l}.vb{j} + opts.alpha * net.layers{l}.db{j}; 15 | net.layers{l}.b{j} = net.layers{l}.b{j} - net.layers{l}.vb{j}; 16 | end 17 | end 18 | end 19 | net.vffW = mom * net.vffW + opts.alpha * net.dffW; 20 | net.ffW = net.ffW - net.vffW; 21 | net.vffb = mom * net.vffb + opts.alpha * net.dffb; 22 | net.ffb = net.ffb - net.vffb; 23 | end 24 | -------------------------------------------------------------------------------- /CNN/cnnbp - 副本.m: -------------------------------------------------------------------------------- 1 | function net = cnnbp(net, y) 2 | n = numel(net.layers); 3 | 4 | % error 5 | net.e = net.o - y; 6 | % loss function 7 | net.L = 1/2* sum(net.e(:) .^ 2) / size(net.e, 2); 8 | 9 | %% backprop deltas 10 | net.od = net.e .* (net.o .* (1 - net.o)); % output delta 11 | net.fvd = (net.ffW' * net.od); % feature vector delta 12 | if strcmp(net.layers{n}.type, 'c') % only conv layers has sigm function 13 | if strcmp(net.layers{n}.activation, 'sigmoid') 14 | net.fvd = net.fvd .* (net.fv .* (1 - net.fv)); 15 | elseif strcmp(net.layers{n}.activation, 'tanh') 16 | net.fvd = net.fvd .* (net.fv .* (1 - net.fv));% need to be exploited 17 | elseif strcmp(net.layers{n}.activation, 'ReLU') 18 | net.fvd = net.fvd .* (net.fv > 0); 19 | end; 20 | end 21 | 22 | % reshape feature vector deltas into output map style 23 | sa = size(net.layers{n}.a{1}); 24 | fvnum = sa(1) * sa(2); 25 | for j = 1 : numel(net.layers{n}.a) 26 | net.layers{n}.d{j} = reshape(net.fvd(((j - 1) * fvnum + 1) : j * fvnum, :), sa(1), sa(2), sa(3)); 27 | end 28 | 29 | for l = (n - 1) : -1 : 1 30 | if strcmp(net.layers{l}.type, 'c') 31 | for j = 1 : numel(net.layers{l}.a) 32 | net.layers{l}.d{j} = net.layers{l}.a{j} .* (1 - net.layers{l}.a{j}) .* (expand(net.layers{l + 1}.d{j}, [net.layers{l + 1}.scale net.layers{l + 1}.scale 1]) / net.layers{l + 1}.scale ^ 2); 33 | end 34 | elseif strcmp(net.layers{l}.type, 's') 35 | for i = 1 : numel(net.layers{l}.a) 36 | z = zeros(size(net.layers{l}.a{1})); 37 | for j = 1 : numel(net.layers{l + 1}.a) 38 | z = z + convn(net.layers{l + 1}.d{j}, rot180(net.layers{l + 1}.k{i}{j}), 'full'); 39 | end 40 | net.layers{l}.d{i} = z; 41 | end 42 | end 43 | end 44 | 45 | %% calc gradients 46 | for l = 2 : n 47 | if strcmp(net.layers{l}.type, 'c') 48 | for j = 1 : numel(net.layers{l}.a) 49 | for i = 1 : numel(net.layers{l - 1}.a) 50 | net.layers{l}.dk{i}{j} = convn(flipall(net.layers{l - 1}.a{i}), net.layers{l}.d{j}, 'valid') / size(net.layers{l}.d{j}, 3); 51 | end 52 | net.layers{l}.db{j} = sum(net.layers{l}.d{j}(:)) / size(net.layers{l}.d{j}, 3); 53 | end 54 | end 55 | end 56 | net.dffW = net.od * (net.fv)' / size(net.od, 2); 57 | net.dffb = mean(net.od, 2); 58 | 59 | function X = rot180(X) 60 | X = flipdim(flipdim(X, 1), 2); 61 | end 62 | end 63 | -------------------------------------------------------------------------------- /CNN/cnnbp.m: -------------------------------------------------------------------------------- 1 | function net = cnnbp(net, y) 2 | n = numel(net.layers); 3 | if strcmp(net.layers{n}.objective, 'sigm') 4 | % error 5 | net.e = net.o - y; 6 | % loss function 7 | net.L = gather(1/2* sum(net.e(:) .^ 2) / size(net.e, 2)); 8 | 9 | net.od = net.e .* (net.o .* (1 - net.o)); % output delta 10 | elseif strcmp(net.layers{n}.objective, 'softmax') 11 | % error 12 | net.e = -1 * (y - net.o)/size(net.layers{1}.a{1},3); 13 | % loss function 14 | net.L = -1 * mean(sum(y.*log(net.o))); 15 | 16 | %% backprop deltas 17 | net.od = net.e; % output delta 18 | end; 19 | net.fvd = (net.ffW' * net.od) * size(net.layers{1}.a{1},3); % feature vector delta 20 | if strcmp(net.layers{n-1}.type, 'c') % only conv layers has sigm function 21 | if strcmp(net.layers{n-1}.activation, 'sigmoid') 22 | net.fvd = net.fvd .* (net.fv .* (1 - net.fv)); 23 | elseif strcmp(net.layers{n-1}.activation, 'tanh') 24 | net.fvd = net.fvd .* (net.fv .* (1 - net.fv));% need to be exploited 25 | elseif strcmp(net.layers{n-1}.activation, 'ReLU') 26 | net.fvd = net.fvd .* (net.fv > 0); 27 | end; 28 | end 29 | 30 | % reshape feature vector deltas into output map style 31 | sa = size(net.layers{n-1}.a{1}); 32 | fvnum = sa(1) * sa(2); 33 | for j = 1 : numel(net.layers{n-1}.a) 34 | net.layers{n-1}.d{j} = reshape(net.fvd(((j - 1) * fvnum + 1) : j * fvnum, :), sa(1), sa(2), sa(3)); 35 | end 36 | 37 | for l = (n - 2) : -1 : 1 38 | if strcmp(net.layers{l}.type, 'c') 39 | for j = 1 : numel(net.layers{l}.a) 40 | if strcmp(net.layers{l}.activation, 'sigmoid') 41 | da = net.layers{l}.a{j} .* (1 - net.layers{l}.a{j}); 42 | elseif strcmp(net.layers{l}.activation, 'tanh') 43 | da = net.layers{l}.a{j} .* (1 - net.layers{l}.a{j});% need to be exploited 44 | elseif strcmp(net.layers{l}.activation, 'ReLU') 45 | da = ( net.layers{l}.a{j} > 0); 46 | end; 47 | if strcmp(net.layers{l + 1}.method, 'a') 48 | net.layers{l}.d{j} = da .* (expand(net.layers{l + 1}.d{j} .* net.layers{l + 1}.dropoutMask{j}, [net.layers{l + 1}.scale net.layers{l + 1}.scale 1]) / net.layers{l + 1}.scale ^ 2); 49 | elseif strcmp(net.layers{l + 1}.method, 'm') 50 | net.layers{l}.d{j} = da .* (expand(net.layers{l + 1}.d{j} .* net.layers{l + 1}.dropoutMask{j}, [net.layers{l + 1}.scale net.layers{l + 1}.scale 1]) .* net.layers{l + 1}.PosMatrix{j}); 51 | elseif strcmp(net.layers{l + 1}.method, 's') 52 | net.layers{l}.d{j} = da .* (expand(net.layers{l + 1}.d{j} .* net.layers{l + 1}.dropoutMask{j}, [net.layers{l + 1}.scale net.layers{l + 1}.scale 1]) .* net.layers{l + 1}.PosMatrix{j}); 53 | end; 54 | end 55 | elseif strcmp(net.layers{l}.type, 's') 56 | for i = 1 : numel(net.layers{l}.a) 57 | z = zeros(size(net.layers{l}.a{1})); 58 | for j = 1 : numel(net.layers{l + 1}.a) 59 | z = z + convn(net.layers{l + 1}.d{j}, rot180(net.layers{l + 1}.k{i}{j}), 'full'); 60 | end 61 | net.layers{l}.d{i} = z; 62 | end 63 | end 64 | end 65 | 66 | %% calc gradients 67 | for l = 2 : n - 1 68 | if strcmp(net.layers{l}.type, 'c') 69 | for j = 1 : numel(net.layers{l}.a) 70 | for i = 1 : numel(net.layers{l - 1}.a) 71 | net.layers{l}.dk{i}{j} = convn(flipall(net.layers{l - 1}.a{i}), net.layers{l}.d{j}, 'valid') / size(net.layers{l}.d{j}, 3); 72 | end 73 | net.layers{l}.db{j} = sum(net.layers{l}.d{j}(:)) / size(net.layers{l}.d{j}, 3); 74 | end 75 | end 76 | end 77 | if strcmp(net.layers{n}.objective, 'sigm') 78 | net.dffW = net.od * (net.fv)' / size(net.od, 2); 79 | net.dffb = mean(net.od, 2); 80 | elseif strcmp(net.layers{n}.objective, 'softmax') 81 | net.dffW = net.od * (net.fv)'; 82 | net.dffb = sum(net.od, 2); 83 | end; 84 | 85 | 86 | function X = rot180(X) 87 | X = flipdim(flipdim(X, 1), 2); 88 | end 89 | end 90 | -------------------------------------------------------------------------------- /CNN/cnnff.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/happynear/DeepLearnToolbox/2539a8cc2dcd0faab7215949b9714d7705c18738/CNN/cnnff.m -------------------------------------------------------------------------------- /CNN/cnnnumgradcheck.m: -------------------------------------------------------------------------------- 1 | function cnnnumgradcheck(net, x, y) 2 | epsilon = 1e-4; 3 | er = 1e-8; 4 | n = numel(net.layers) - 1; 5 | netr = net; 6 | netr = cnnff(netr, x); 7 | netr = cnnbp(netr, y); 8 | % disp('checking ffb...'); 9 | % for j = 1 : numel(net.ffb) 10 | % disp(j); 11 | % net_m = net; net_p = net; 12 | % net_p.ffb(j) = net_m.ffb(j) + epsilon; 13 | % net_m.ffb(j) = net_m.ffb(j) - epsilon; 14 | % net_m = cnnff(net_m, x); net_m = cnnbp(net_m, y); 15 | % net_p = cnnff(net_p, x); net_p = cnnbp(net_p, y); 16 | % d = (net_p.L - net_m.L) / (2 * epsilon); 17 | % e = abs(d - netr.dffb(j)); 18 | % if e > er 19 | % error('numerical gradient checking failed'); 20 | % end 21 | % end 22 | 23 | % disp('checking ffW...'); 24 | % for i = 1 : size(net.ffW, 1) 25 | % for u = 1 : size(net.ffW, 2) 26 | % disp([i u]); 27 | % net_m = net; net_p = net; 28 | % net_p.ffW(i, u) = net_m.ffW(i, u) + epsilon; 29 | % net_m.ffW(i, u) = net_m.ffW(i, u) - epsilon; 30 | % net_m = cnnff(net_m, x); net_m = cnnbp(net_m, y); 31 | % net_p = cnnff(net_p, x); net_p = cnnbp(net_p, y); 32 | % d = (net_p.L - net_m.L) / (2 * epsilon); 33 | % e = abs(d - netr.dffW(i, u)); 34 | % if e > er 35 | % error('numerical gradient checking failed'); 36 | % end 37 | % end 38 | % end 39 | 40 | for l = n : -1 : 2 41 | if strcmp(netr.layers{l}.type, 'c') 42 | for j = 1 : numel(netr.layers{l}.a) 43 | net_m = net; net_p = net; 44 | net_p.layers{l}.b{j} = net_m.layers{l}.b{j} + epsilon; 45 | net_m.layers{l}.b{j} = net_m.layers{l}.b{j} - epsilon; 46 | net_m = cnnff(net_m, x); net_m = cnnbp(net_m, y); 47 | net_p = cnnff(net_p, x); net_p = cnnbp(net_p, y); 48 | d = (net_p.L - net_m.L) / (2 * epsilon); 49 | e = abs(d - netr.layers{l}.db{j}); 50 | if e > er 51 | error('numerical gradient checking failed'); 52 | end 53 | for i = 1 : numel(netr.layers{l - 1}.a) 54 | for u = 1 : size(netr.layers{l}.k{i}{j}, 1) 55 | for v = 1 : size(netr.layers{l}.k{i}{j}, 2) 56 | net_m = net; net_p = net; 57 | net_p.layers{l}.k{i}{j}(u, v) = net_p.layers{l}.k{i}{j}(u, v) + epsilon; 58 | net_m.layers{l}.k{i}{j}(u, v) = net_m.layers{l}.k{i}{j}(u, v) - epsilon; 59 | net_m = cnnff(net_m, x); net_m = cnnbp(net_m, y); 60 | net_p = cnnff(net_p, x); net_p = cnnbp(net_p, y); 61 | d = (net_p.L - net_m.L) / (2 * epsilon); 62 | e = abs(d - netr.layers{l}.dk{i}{j}(u, v)); 63 | if e > er 64 | error('numerical gradient checking failed'); 65 | end 66 | end 67 | end 68 | end 69 | end 70 | elseif strcmp(net.layers{l}.type, 's') 71 | % for j = 1 : numel(net.layers{l}.a) 72 | % net_m = net; net_p = net; 73 | % net_p.layers{l}.b{j} = net_m.layers{l}.b{j} + epsilon; 74 | % net_m.layers{l}.b{j} = net_m.layers{l}.b{j} - epsilon; 75 | % net_m = cnnff(net_m, x); net_m = cnnbp(net_m, y); 76 | % net_p = cnnff(net_p, x); net_p = cnnbp(net_p, y); 77 | % d = (net_p.L - net_m.L) / (2 * epsilon); 78 | % e = abs(d - net.layers{l}.db{j}); 79 | % if e > er 80 | % error('numerical gradient checking failed'); 81 | % end 82 | % end 83 | end 84 | end 85 | % keyboard 86 | end 87 | -------------------------------------------------------------------------------- /CNN/cnnsetup.m: -------------------------------------------------------------------------------- 1 | function net = cnnsetup(net, x, y) 2 | % assert(~isOctave() || compare_versions(OCTAVE_VERSION, '3.8.0', '>='), ['Octave 3.8.0 or greater is required for CNNs as there is a bug in convolution in previous versions. See http://savannah.gnu.org/bugs/?39314. Your version is ' myOctaveVersion]); 3 | inputmaps = 1; 4 | mapsize = size(squeeze(x(:, :, 1))); 5 | global useGpu; 6 | 7 | for l = 1 : numel(net.layers) % layer 8 | if strcmp(net.layers{l}.type, 's') 9 | mapsize = mapsize / net.layers{l}.scale; 10 | assert(all(floor(mapsize)==mapsize), ['Layer ' num2str(l) ' size must be integer. Actual: ' num2str(mapsize)]); 11 | net.layers{l}.inputmaps = inputmaps; 12 | for j = 1 : inputmaps 13 | net.layers{l}.b{j} = 0; 14 | end 15 | end 16 | if strcmp(net.layers{l}.type, 'c') 17 | mapsize = mapsize - net.layers{l}.kernelsize + 1; 18 | fan_out = net.layers{l}.outputmaps * net.layers{l}.kernelsize ^ 2; 19 | net.layers{l}.inputmaps = inputmaps; 20 | for j = 1 : net.layers{l}.outputmaps % output map 21 | fan_in = inputmaps * net.layers{l}.kernelsize ^ 2; 22 | for i = 1 : inputmaps % input map 23 | if useGpu 24 | net.layers{l}.k{i}{j} = (gpuArray.rand(net.layers{l}.kernelsize) - 0.5) * 2 * sqrt(6 / (fan_in + fan_out)); 25 | net.layers{l}.vk{i}{j} = gpuArray.zeros(net.layers{l}.kernelsize); 26 | else 27 | net.layers{l}.k{i}{j} = (rand(net.layers{l}.kernelsize) - 0.5) * 2 * sqrt(6 / (fan_in + fan_out)); 28 | net.layers{l}.vk{i}{j} = zeros(net.layers{l}.kernelsize); 29 | end; 30 | end 31 | if useGpu 32 | net.layers{l}.b{j} = (gpuArray.rand(1,1) - 0.5) * 2 * sqrt(6 / (fan_in + fan_out)); 33 | net.layers{l}.vb{j} = gpuArray.zeros(1,1); 34 | else 35 | net.layers{l}.b{j} = (rand(1,1) - 0.5) * 2 * sqrt(6 / (fan_in + fan_out)); 36 | net.layers{l}.vb{j} = 0; 37 | end; 38 | end 39 | inputmaps = net.layers{l}.outputmaps; 40 | end 41 | end 42 | % 'onum' is the number of labels, that's why it is calculated using size(y, 1). If you have 20 labels so the output of the network will be 20 neurons. 43 | % 'fvnum' is the number of output neurons at the last layer, the layer just before the output layer. 44 | % 'ffb' is the biases of the output neurons. 45 | % 'ffW' is the weights between the last layer and the output neurons. Note that the last layer is fully connected to the output layer, that's why the size of the weights is (onum * fvnum) 46 | fvnum = prod(mapsize) * inputmaps; 47 | onum = size(y, 1); 48 | if useGpu 49 | net.ffb = gpuArray.zeros(onum, 1); 50 | net.vffb = gpuArray.zeros(onum, 1); 51 | net.ffW = (gpuArray.rand(onum, fvnum) - 0.5) * 2 * sqrt(6 / (onum + fvnum)); 52 | net.vffW = gpuArray.zeros(onum, fvnum); 53 | else 54 | net.ffb = zeros(onum, 1); 55 | net.vffb = zeros(onum, 1); 56 | net.ffW = (rand(onum, fvnum) - 0.5) * 2 * sqrt(6 / (onum + fvnum)); 57 | net.vffW = zeros(onum, fvnum); 58 | end; 59 | end 60 | -------------------------------------------------------------------------------- /CNN/cnntest.m: -------------------------------------------------------------------------------- 1 | function [er, bad] = cnntest(net, x, y) 2 | % feedforward 3 | net.testing =true; 4 | net = cnnff(net, x); 5 | [~, h] = max(net.o); 6 | [~, a] = max(y); 7 | bad = find(h ~= a); 8 | 9 | er = numel(bad) / size(y, 2); 10 | end 11 | -------------------------------------------------------------------------------- /CNN/cnntrain.m: -------------------------------------------------------------------------------- 1 | function net = cnntrain(net, x, y, opts) 2 | global useGpu; 3 | m = size(x, 3); 4 | numbatches = m / opts.batchsize; 5 | if rem(numbatches, 1) ~= 0 6 | error('numbatches not integer'); 7 | end 8 | net.rL = []; 9 | for i = 1 : opts.numepochs 10 | % tic; 11 | kk = randperm(m); 12 | for l = 1 : numbatches 13 | if useGpu 14 | batch_x = gpuArray(x(:, :, kk((l - 1) * opts.batchsize + 1 : l * opts.batchsize))); 15 | batch_y = gpuArray(y(:, kk((l - 1) * opts.batchsize + 1 : l * opts.batchsize))); 16 | else 17 | batch_x = x(:, :, kk((l - 1) * opts.batchsize + 1 : l * opts.batchsize)); 18 | batch_y = y(:, kk((l - 1) * opts.batchsize + 1 : l * opts.batchsize)); 19 | end; 20 | 21 | net = cnnff(net, batch_x); 22 | net = cnnbp(net, batch_y); 23 | net = cnnapplygrads(net, opts); 24 | if isempty(net.rL) 25 | net.rL(1) = gather(net.L); 26 | end 27 | net.rL(end + 1) = 0.99 * net.rL(end) + 0.01 * gather(net.L); 28 | if mod(l,10)==0 29 | disp(['epoch ' num2str(i) '/' num2str(opts.numepochs) 'batch ' num2str(l) '/' num2str(numbatches)]); 30 | end; 31 | end 32 | disp(['epoch ' num2str(i) '/' num2str(opts.numepochs) ' error:' num2str(net.L)]); 33 | % toc; 34 | end 35 | 36 | end 37 | -------------------------------------------------------------------------------- /CNN/test_example_CNN.m: -------------------------------------------------------------------------------- 1 | % function test_example_CNN 2 | load mnist_uint8; 3 | global useGpu; 4 | useGpu = false; 5 | 6 | train_x = double(reshape(train_x',28,28,60000))/255; 7 | test_x = double(reshape(test_x',28,28,10000))/255; 8 | train_y = double(train_y'); 9 | test_y = double(test_y'); 10 | 11 | %% ex1 Train a 6c-2s-12c-2s Convolutional neural network 12 | %will run 1 epoch in about 200 second and get around 11% error. 13 | %With 100 epochs you'll get around 1.2% error 14 | 15 | if exist('MaxPooling')~=3 16 | mex MaxPooling.cpp COMPFLAGS="/openmp $COMPFLAGS" CXXFLAGS="\$CFLAGS -fopenmp" LDFLAGS="\$LDFLAGS -fopenmp" -largeArrayDims 17 | end; 18 | if exist('StochasticPooling')~=3 19 | mex StochasticPooling.cpp COMPFLAGS="/openmp $COMPFLAGS" CXXFLAGS="\$CFLAGS -fopenmp" LDFLAGS="\$LDFLAGS -fopenmp" -largeArrayDims 20 | end; 21 | 22 | rand('state',0) 23 | clear cnn; 24 | cnn.layers = { 25 | struct('type', 'i') %input layer 26 | struct('type', 'c', 'outputmaps', 20, 'kernelsize', 5, 'activation', 'ReLU') %convolution layer 27 | struct('type', 's', 'scale', 2, 'method', 'm') %sub sampling layer 'm':maxpooling; 'a':average pooling; 's':stochastic pooling 28 | struct('type', 'c', 'outputmaps', 20, 'kernelsize', 5, 'activation', 'ReLU') %convolution layer 29 | struct('type', 's', 'scale', 2, 'method', 'm') %sub sampling layer 'm':maxpooling; 'a':average pooling; 's':stochastic pooling 30 | % struct('type', 'c', 'outputmaps', 20, 'kernelsize', 2, 'activation', 'ReLU') %convolution layer 31 | % struct('type', 'c', 'outputmaps', 20, 'kernelsize', 5, 'activation', 'ReLU') %convolution layer 32 | % struct('type', 's', 'scale', 2, 'method', 'm') %subsampling layer 33 | struct('type','o','objective','softmax'); 34 | }; 35 | % cnn.layers = { 36 | % struct('type', 'i') %input layer 37 | % struct('type', 'c', 'outputmaps', 6, 'kernelsize', 5, 'activation', 'ReLU') %convolution layer 38 | % struct('type', 's', 'scale', 2, 'method', 'm') %sub sampling layer 'm':maxpooling; 'a':average pooling; 's':stochastic pooling 39 | % struct('type', 'c', 'outputmaps', 12, 'kernelsize', 5, 'activation', 'ReLU') %convolution layer 40 | % struct('type', 's', 'scale', 2, 'method', 'm') %subsampling layer 41 | % struct('type','o','objective','softmax'); 42 | % }; 43 | debug =false; 44 | if debug 45 | opts.alpha = 1; 46 | opts.batchsize = 10; 47 | opts.numepochs = 1; 48 | cnn.testing = false; 49 | cnn = cnnsetup(cnn, train_x(:,:,1:10), train_y(:,1:10)); 50 | cnn = cnntrain(cnn, train_x(:,:,1:10), train_y(:,1:10), opts); 51 | cnnnumgradcheck(cnn, train_x(:,:,1:10), train_y(:,1:10)); 52 | end; 53 | 54 | opts.alpha = 0.1; 55 | opts.alphascale = 0.5; 56 | opts.batchsize = 50; 57 | opts.numepochs = 3; 58 | opts.momentum = 0.95; 59 | opts.momIncrease = 20; 60 | cnn.iter = 1; 61 | cnn.testing = false; 62 | cnn.dropoutFraction = 0; 63 | 64 | cnn = cnnsetup(cnn, train_x, train_y); 65 | % cnn = cnnff(cnn,train_x(:,:,1:1000)); 66 | cnn = cnntrain(cnn, train_x, train_y, opts); 67 | 68 | [er, bad] = cnntest(cnn, test_x, test_y); 69 | 70 | %plot mean squared error 71 | figure; plot(cnn.rL); 72 | assert(er<0.12, 'Too big error'); 73 | -------------------------------------------------------------------------------- /CNN/test_example_CNN2.m: -------------------------------------------------------------------------------- 1 | % function test_example_CNN 2 | % load mnist_uint8; 3 | global useGpu; 4 | useGpu = false; 5 | 6 | trainNum = 700; 7 | idx = randperm(1400); 8 | images = double(images); 9 | images = bsxfun(@rdivide, images, max(images,[],2)); 10 | train_x = images(idx(1:trainNum),:); 11 | labels(labels==0)=10; 12 | groundTruth = full(sparse(labels(1:1400), 1:1400, 1)); 13 | train_y = groundTruth(:,idx(1:trainNum))'; 14 | test_x = images(idx(trainNum+1:1400),:); 15 | test_y = groundTruth(:,idx(trainNum+1:1400))'; 16 | % train_x = double(train_x) / 255; 17 | % test_x = double(test_x) / 255; 18 | train_y = double(train_y); 19 | test_y = double(test_y); 20 | train_x = double(reshape(train_x',30,24,700))/255; 21 | test_x = double(reshape(test_x',30,24,700))/255; 22 | train_y = double(train_y'); 23 | test_y = double(test_y'); 24 | 25 | %% ex1 Train a 6c-2s-12c-2s Convolutional neural network 26 | %will run 1 epoch in about 200 second and get around 11% error. 27 | %With 100 epochs you'll get around 1.2% error 28 | 29 | if exist('MaxPooling')~=3 30 | mex MaxPooling.cpp COMPFLAGS="/openmp $COMPFLAGS" CXXFLAGS="\$CFLAGS -fopenmp" LDFLAGS="\$LDFLAGS -fopenmp" -largeArrayDims 31 | end; 32 | if exist('StochasticPooling')~=3 33 | mex StochasticPooling.cpp COMPFLAGS="/openmp $COMPFLAGS" CXXFLAGS="\$CFLAGS -fopenmp" LDFLAGS="\$LDFLAGS -fopenmp" -largeArrayDims 34 | end; 35 | 36 | rand('state',0) 37 | clear cnn; 38 | % cnn.layers = { 39 | % struct('type', 'i') %input layer 40 | % struct('type', 'c', 'outputmaps', 20, 'kernelsize', 9, 'activation', 'ReLU') %convolution layer 41 | % struct('type', 's', 'scale', 2, 'method', 's') %sub sampling layer 'm':maxpooling; 'a':average pooling; 's':stochastic pooling 42 | % % struct('type', 'c', 'outputmaps', 12, 'kernelsize', 5, 'activation', 'ReLU') %convolution layer 43 | % % struct('type', 's', 'scale', 2, 'method', 'm') %subsampling layer 44 | % struct('type','o','objective','softmax'); 45 | % }; 46 | cnn.layers = { 47 | struct('type', 'i') %input layer 48 | struct('type', 'c', 'outputmaps', 6, 'kernelsize', 5, 'activation', 'sigmoid') %convolution layer 49 | struct('type', 's', 'scale', 2, 'method', 'm') %sub sampling layer 'm':maxpooling; 'a':average pooling; 's':stochastic pooling 50 | % struct('type', 'c', 'outputmaps', 12, 'kernelsize', 2, 'activation', 'ReLU') %convolution layer 51 | % struct('type', 's', 'scale', 2, 'method', 'm') %subsampling layer 52 | struct('type','o','objective','softmax'); 53 | }; 54 | debug =false; 55 | if debug 56 | opts.alpha = 1; 57 | opts.batchsize = 10; 58 | opts.numepochs = 1; 59 | cnn = cnnsetup(cnn, train_x(:,:,1:10), train_y(:,1:10)); 60 | % cnn = cnntrain(cnn, train_x(:,:,1:10), train_y(:,1:10), opts); 61 | cnnnumgradcheck(cnn, train_x(:,:,1:10), train_y(:,1:10)); 62 | end; 63 | 64 | opts.alpha = 0.1; 65 | opts.alphascale = 0.5; 66 | opts.batchsize = 50; 67 | opts.numepochs = 100; 68 | opts.momentum = 0.95; 69 | opts.momIncrease = 20; 70 | cnn.iter = 1; 71 | cnn.testing = false; 72 | cnn.dropoutFraction = 0.5; 73 | 74 | cnn = cnnsetup(cnn, train_x, train_y); 75 | cnn = cnntrain(cnn, train_x, train_y, opts); 76 | 77 | [er, bad] = cnntest(cnn, test_x, test_y); 78 | 79 | %plot mean squared error 80 | figure; plot(cnn.rL); 81 | assert(er<0.12, 'Too big error'); 82 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Thank you so much for wanting to give back to the toolbox. Here's some info on how to contribute: 2 | 3 | #General: 4 | 5 | Don't bunch up changes, e.g. if you have bug-fixes, new features and style changes, rather make 3 seperate pull requests. 6 | 7 | Ensure that you introduce tests/examples for any new functionality 8 | 9 | # Guide 10 | 1. Fork repository 11 | 2. Create a new branch, e.g. `checkout -b my-stuff` 12 | 3. Commit and push your changes to that branch 13 | 4. Make sure that the test works (!) (see known errors) 14 | 5. Create a pull request 15 | 6. I accept your pull request 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /DBN/dbnsetup.m: -------------------------------------------------------------------------------- 1 | function dbn = dbnsetup(dbn, x, opts) 2 | n = size(x, 2); 3 | dbn.sizes = [n, dbn.sizes]; 4 | 5 | for u = 1 : numel(dbn.sizes) - 1 6 | dbn.rbm{u}.alpha = opts.alpha; 7 | dbn.rbm{u}.momentum = opts.momentum; 8 | 9 | dbn.rbm{u}.W = zeros(dbn.sizes(u + 1), dbn.sizes(u)); 10 | dbn.rbm{u}.vW = zeros(dbn.sizes(u + 1), dbn.sizes(u)); 11 | 12 | dbn.rbm{u}.b = zeros(dbn.sizes(u), 1); 13 | dbn.rbm{u}.vb = zeros(dbn.sizes(u), 1); 14 | 15 | dbn.rbm{u}.c = zeros(dbn.sizes(u + 1), 1); 16 | dbn.rbm{u}.vc = zeros(dbn.sizes(u + 1), 1); 17 | end 18 | 19 | end 20 | -------------------------------------------------------------------------------- /DBN/dbntrain.m: -------------------------------------------------------------------------------- 1 | function dbn = dbntrain(dbn, x, opts) 2 | n = numel(dbn.rbm); 3 | 4 | dbn.rbm{1} = rbmtrain(dbn.rbm{1}, x, opts); 5 | for i = 2 : n 6 | x = rbmup(dbn.rbm{i - 1}, x); 7 | dbn.rbm{i} = rbmtrain(dbn.rbm{i}, x, opts); 8 | end 9 | 10 | end 11 | -------------------------------------------------------------------------------- /DBN/dbnunfoldtonn.m: -------------------------------------------------------------------------------- 1 | function nn = dbnunfoldtonn(dbn, outputsize) 2 | %DBNUNFOLDTONN Unfolds a DBN to a NN 3 | % dbnunfoldtonn(dbn, outputsize ) returns the unfolded dbn with a final 4 | % layer of size outputsize added. 5 | if(exist('outputsize','var')) 6 | size = [dbn.sizes outputsize]; 7 | else 8 | size = [dbn.sizes]; 9 | end 10 | nn = nnsetup(size); 11 | for i = 1 : numel(dbn.rbm) 12 | nn.W{i} = [dbn.rbm{i}.c dbn.rbm{i}.W]; 13 | end 14 | end 15 | 16 | -------------------------------------------------------------------------------- /DBN/rbmdown.m: -------------------------------------------------------------------------------- 1 | function x = rbmdown(rbm, x) 2 | x = sigm(repmat(rbm.b', size(x, 1), 1) + x * rbm.W); 3 | end 4 | -------------------------------------------------------------------------------- /DBN/rbmtrain.m: -------------------------------------------------------------------------------- 1 | function rbm = rbmtrain(rbm, x, opts) 2 | assert(isfloat(x), 'x must be a float'); 3 | m = size(x, 1); 4 | numbatches = m / opts.batchsize; 5 | 6 | assert(rem(numbatches, 1) == 0, 'numbatches not integer'); 7 | 8 | for i = 1 : opts.numepochs 9 | kk = randperm(m); 10 | err = 0; 11 | for l = 1 : numbatches 12 | batch = x(kk((l - 1) * opts.batchsize + 1 : l * opts.batchsize), :); 13 | 14 | v1 = batch; 15 | h1 = sigmrnd(repmat(rbm.c', opts.batchsize, 1) + v1 * rbm.W'); 16 | v2 = sigmrnd(repmat(rbm.b', opts.batchsize, 1) + h1 * rbm.W); 17 | h2 = sigmrnd(repmat(rbm.c', opts.batchsize, 1) + v2 * rbm.W'); 18 | 19 | c1 = h1' * v1; 20 | c2 = h2' * v2; 21 | 22 | rbm.vW = rbm.momentum * rbm.vW + rbm.alpha * (c1 - c2) / opts.batchsize; 23 | rbm.vb = rbm.momentum * rbm.vb + rbm.alpha * sum(v1 - v2)' / opts.batchsize; 24 | rbm.vc = rbm.momentum * rbm.vc + rbm.alpha * sum(h1 - h2)' / opts.batchsize; 25 | 26 | rbm.W = rbm.W + rbm.vW; 27 | rbm.b = rbm.b + rbm.vb; 28 | rbm.c = rbm.c + rbm.vc; 29 | 30 | err = err + sum(sum((v1 - v2) .^ 2)) / opts.batchsize; 31 | end 32 | 33 | disp(['epoch ' num2str(i) '/' num2str(opts.numepochs) '. Average reconstruction error is: ' num2str(err / numbatches)]); 34 | 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /DBN/rbmup.m: -------------------------------------------------------------------------------- 1 | function x = rbmup(rbm, x) 2 | x = sigm(repmat(rbm.c', size(x, 1), 1) + x * rbm.W'); 3 | end 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012, Rasmus Berg Palm (rasmusbergpalm@gmail.com) 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 5 | 6 | Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 7 | Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 8 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /MaxOut/nnapplygrads.m: -------------------------------------------------------------------------------- 1 | function nn = nnapplygrads(nn) 2 | %NNAPPLYGRADS updates weights and biases with calculated gradients 3 | % nn = nnapplygrads(nn) returns an neural network structure with updated 4 | % weights and biases 5 | nn.iter = nn.iter + 1; 6 | if nn.iter == nn.momentumSaturate 7 | nn.momemtum = nn.momemtumIncrease; 8 | end; 9 | for i = 1 : (nn.n - 1) 10 | if(nn.weightPenaltyL2>0) 11 | dW = nn.dW{i} + nn.weightPenaltyL2 * [zeros(size(nn.W{i},1),1) nn.W{i}(:,2:end)]; 12 | else 13 | dW = nn.dW{i}; 14 | end 15 | 16 | dW = nn.learningRate * dW; 17 | 18 | if(nn.momentum>0) 19 | nn.vW{i} = nn.momentum*nn.vW{i} + dW; 20 | dW = nn.vW{i}; 21 | end 22 | 23 | nn.W{i} = nn.W{i} - dW; 24 | nn.W{i} = bsxfun(@rdivide, nn.W{i},sqrt(sum(nn.W{i}.^2))*1.9365); 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /MaxOut/nnbp.m: -------------------------------------------------------------------------------- 1 | function nn = nnbp(nn) 2 | %NNBP performs backpropagation 3 | % nn = nnbp(nn) returns an neural network structure with updated weights 4 | 5 | n = nn.n; 6 | switch nn.output 7 | case 'sigm' 8 | d{n} = - nn.e .* (nn.a{n} .* (1 - nn.a{n})); 9 | case {'softmax','linear'} 10 | d{n} = - nn.e; 11 | end 12 | for i = (n - 1) : -1 : 2 13 | 14 | 15 | % Backpropagate first derivatives 16 | if i+1==n % in this case in d{n} there is not the bias term to be removed 17 | d{i} = d{i + 1} * nn.W{i}; % Bishop (5.56) 18 | else % in this case in d{i} the bias term has to be removed 19 | d{i} = d{i + 1}(:,2:end) * nn.W{i}; 20 | end 21 | 22 | if(nn.dropoutFraction>0) 23 | d{i} = d{i} .* [ones(size(d{i},1),1) nn.dropOutMask{i}]; 24 | end 25 | 26 | de = kron(d{i}(:,2:end),ones(1,nn.channels)); 27 | % de = repmat(d{i}(:,2:end),1,nn.channels); 28 | d{i} = [d{i}(:,1) de .* nn.maxMask{i}]; 29 | 30 | end 31 | 32 | for i = 1 : (n - 1) 33 | if i+1==n 34 | nn.dW{i} = (d{i + 1}' * nn.a{i}) / size(d{i + 1}, 1); 35 | else 36 | nn.dW{i} = (d{i + 1}(:,2:end)' * nn.a{i}) / size(d{i + 1}, 1); 37 | end 38 | end 39 | end 40 | -------------------------------------------------------------------------------- /MaxOut/nnchecknumgrad.m: -------------------------------------------------------------------------------- 1 | function nnchecknumgrad(nn, x, y) 2 | epsilon = 1e-6; 3 | er = 1e-7; 4 | n = nn.n; 5 | rng('default'); 6 | nnr = nn; 7 | nnr = nnff(nnr, x, y); 8 | nnr = nnbp(nnr); 9 | for l = 1 : (n - 1) 10 | for i = 2 : size(nn.W{l}, 1) 11 | for j = 1 : size(nn.W{l}, 2) 12 | nn_m = nn; nn_p = nn; 13 | nn_m.W{l}(i, j) = nn.W{l}(i, j) - epsilon; 14 | nn_p.W{l}(i, j) = nn.W{l}(i, j) + epsilon; 15 | rng('default'); 16 | nn_m = nnff(nn_m, x, y); 17 | rng('default'); 18 | nn_p = nnff(nn_p, x, y); 19 | dW = (nn_p.L - nn_m.L) / (2 * epsilon); 20 | e = abs(dW - nnr.dW{l}(i, j)); 21 | disp([num2str(i) ' ' num2str(j) ' ' num2str(dW)]); 22 | 23 | assert(e < er, 'numerical gradient checking failed'); 24 | end 25 | end 26 | end 27 | end 28 | -------------------------------------------------------------------------------- /MaxOut/nneval.m: -------------------------------------------------------------------------------- 1 | function [loss] = nneval(nn, loss, train_x, train_y, val_x, val_y) 2 | %NNEVAL evaluates performance of neural network 3 | % Returns a updated loss struct 4 | assert(nargin == 4 || nargin == 6, 'Wrong number of arguments'); 5 | 6 | % training performance 7 | nn = nnff(nn, train_x, train_y); 8 | loss.train.e(end + 1) = nn.L; 9 | 10 | % validation performance 11 | if nargin == 6 12 | nn = nnff(nn, val_x, val_y); 13 | loss.val.e(end + 1) = nn.L; 14 | end 15 | 16 | %calc misclassification rate if softmax 17 | if strcmp(nn.output,'softmax') 18 | [er_train, ~] = nntest(nn, train_x, train_y); 19 | loss.train.e_frac(end+1) = er_train; 20 | 21 | if nargin == 6 22 | [er_val, ~] = nntest(nn, val_x, val_y); 23 | loss.val.e_frac(end+1) = er_val; 24 | end 25 | end 26 | 27 | end 28 | -------------------------------------------------------------------------------- /MaxOut/nnff.m: -------------------------------------------------------------------------------- 1 | function nn = nnff(nn, x, y) 2 | %NNFF performs a feedforward pass 3 | % nn = nnff(nn, x, y) returns an neural network structure with updated 4 | % layer activations, error and loss (nn.a, nn.e and nn.L) 5 | 6 | n = nn.n; 7 | m = size(x, 1); 8 | 9 | x = [ones(m,1) x]; 10 | nn.a{1} = x; 11 | 12 | %feedforward pass 13 | for i = 2 : n-1 14 | nn.a{i} = nn.a{i - 1} * nn.W{i - 1}'; 15 | nn.a{i} = reshape(nn.a{i},[m, nn.channels, nn.size(i)]); 16 | [nn.a{i}, maxp] = max(nn.a{i},[],2); 17 | maxp = squeeze(maxp); 18 | maxp = maxp + repmat((0:nn.size(i)-1)*nn.channels,m,1); 19 | [I J V] = find(maxp); 20 | nn.maxMask{i} = sparse(I,V,ones(length(V),1),m,nn.size(i) * nn.channels); 21 | nn.a{i} = squeeze(nn.a{i}); 22 | 23 | %dropout 24 | if(nn.dropoutFraction > 0) 25 | if(nn.testing) 26 | nn.a{i} = nn.a{i}.*(1 - nn.dropoutFraction); 27 | else 28 | nn.dropOutMask{i} = (rand(size(nn.a{i}))>nn.dropoutFraction); 29 | nn.a{i} = nn.a{i}.*nn.dropOutMask{i}; 30 | end 31 | end 32 | 33 | %Add the bias term 34 | nn.a{i} = [ones(m,1) nn.a{i}]; 35 | end 36 | switch nn.output 37 | case 'sigm' 38 | nn.a{n} = sigm(nn.a{n - 1} * nn.W{n - 1}'); 39 | case 'linear' 40 | nn.a{n} = nn.a{n - 1} * nn.W{n - 1}'; 41 | case 'softmax' 42 | nn.a{n} = nn.a{n - 1} * nn.W{n - 1}'; 43 | nn.a{n} = exp(bsxfun(@minus, nn.a{n}, max(nn.a{n},[],2))); 44 | nn.a{n} = bsxfun(@rdivide, nn.a{n}, sum(nn.a{n}, 2)); 45 | end 46 | 47 | %error and loss 48 | % size(y) 49 | % size(nn.a{n}) 50 | nn.e = y - nn.a{n}; 51 | 52 | switch nn.output 53 | case {'sigm', 'linear'} 54 | nn.L = 1/2 * sum(sum(nn.e .^ 2)) / m; 55 | case 'softmax' 56 | nn.L = -sum(sum(y .* log(nn.a{n}))) / m; 57 | end 58 | end 59 | -------------------------------------------------------------------------------- /MaxOut/nnpredict.m: -------------------------------------------------------------------------------- 1 | function labels = nnpredict(nn, x) 2 | nn.testing = 1; 3 | nn = nnff(nn, x, zeros(size(x,1), nn.size(end))); 4 | nn.testing = 0; 5 | 6 | [~, i] = max(nn.a{end},[],2); 7 | labels = i; 8 | end 9 | -------------------------------------------------------------------------------- /MaxOut/nnsetup - 副本.m: -------------------------------------------------------------------------------- 1 | function nn = nnsetup(architecture,numChannels) 2 | %NNSETUP creates a Feedforward Backpropagate Neural Network 3 | % nn = nnsetup(architecture) returns an neural network structure with n=numel(architecture) 4 | % layers, architecture being a n x 1 vector of layer sizes e.g. [784 100 10] 5 | 6 | nn.size = architecture; 7 | nn.n = numel(nn.size); 8 | 9 | nn.activation_function = 'MaxOut'; % Activation functions of hidden layers: 'sigm' (sigmoid) or 'tanh_opt' (optimal tanh). 10 | nn.learningRate = 2; % learning rate Note: typically needs to be lower when using 'sigm' activation function and non-normalized inputs. 11 | nn.momentum = 0.5; % Momentum 12 | nn.scaling_learningRate = 1; % Scaling factor for the learning rate (each epoch) 13 | nn.weightPenaltyL2 = 0; % L2 regularization 14 | nn.nonSparsityPenalty = 0; % Non sparsity penalty 15 | nn.sparsityTarget = 0.05; % Sparsity target 16 | nn.inputZeroMaskedFraction = 0.2; % Used for Denoising AutoEncoders 17 | nn.dropoutFraction = 0.5; % Dropout level (http://www.cs.toronto.edu/~hinton/absps/dropout.pdf) 18 | nn.testing = 0; % Internal variable. nntest sets this to one. 19 | nn.output = 'softmax'; % output unit 'sigm' (=logistic), 'softmax' and 'linear' 20 | if ~exist('numChannels','var') 21 | nn.numChannels = 10; 22 | end; 23 | 24 | for i = 2 : nn.n 25 | % weights and weight momentum 26 | nn.W{i - 1} = (rand(nn.size(i), nn.size(i - 1)+1, nn.numChannels) - 0.5) * 2 * 4 * sqrt(6 / (nn.size(i) + nn.size(i - 1) + nn.numChannels)); 27 | nn.vW{i - 1} = zeros(size(nn.W{i - 1})); 28 | 29 | % average activations (for use with sparsity) 30 | nn.p{i} = zeros(1, nn.size(i)); 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /MaxOut/nnsetup.m: -------------------------------------------------------------------------------- 1 | function nn = nnsetup(architecture,channel) 2 | %NNSETUP creates a Feedforward Backpropagate Neural Network 3 | % nn = nnsetup(architecture) returns an neural network structure with n=numel(architecture) 4 | % layers, architecture being a n x 1 vector of layer sizes e.g. [784 100 10] 5 | 6 | nn.size = architecture; 7 | nn.n = numel(nn.size); 8 | 9 | nn.activation_function = 'sigm'; % Activation functions of hidden layers: 'sigm' (sigmoid) or 'tanh_opt' (optimal tanh). 10 | nn.learningRate = 0.1; % learning rate Note: typically needs to be lower when using 'sigm' activation function and non-normalized inputs. 11 | nn.momentum = 0.5; % Momentum 12 | nn.momentumSaturate = 250; % When increase the momentum 13 | nn.momemtumIncrease = 0.75; 14 | nn.scaling_learningRate = 0.5; % Scaling factor for the learning rate (each epoch) 15 | nn.weightPenaltyL2 = 0; % L2 regularization 16 | nn.nonSparsityPenalty = 0; % Non sparsity penalty 17 | nn.sparsityTarget = 0.05; % Sparsity target 18 | nn.inputZeroMaskedFraction = 0; % Used for Denoising AutoEncoders 19 | nn.dropoutFraction = 0; % Dropout level (http://www.cs.toronto.edu/~hinton/absps/dropout.pdf) 20 | nn.testing = 0; % Internal variable. nntest sets this to one. 21 | nn.output = 'softmax'; % output unit 'sigm' (=logistic), 'softmax' and 'linear' 22 | nn.channels = channel; 23 | 24 | for i = 2 : nn.n - 1 25 | % weights and weight momentum 26 | nn.W{i - 1} = (rand(nn.size(i) * nn.channels, nn.size(i - 1)+1) - 0.5) * sqrt(6 / (nn.size(i) + nn.size(i - 1) + nn.channels)); 27 | nn.vW{i - 1} = zeros(size(nn.W{i - 1})); 28 | 29 | % average activations (for use with sparsity) 30 | nn.p{i} = zeros(1, nn.size(i)); 31 | end 32 | nn.W{nn.n-1} = (rand(nn.size(nn.n), nn.size(nn.n - 1)+1) - 0.5) * 2 * 4 * sqrt(6 / (nn.size(nn.n) + nn.size(nn.n - 1))); 33 | nn.vW{nn.n-1} = zeros(size(nn.W{nn.n-1})); 34 | end 35 | -------------------------------------------------------------------------------- /MaxOut/nntest.m: -------------------------------------------------------------------------------- 1 | function [er, bad] = nntest(nn, x, y) 2 | labels = nnpredict(nn, x); 3 | [~, expected] = max(y,[],2); 4 | bad = find(labels ~= expected); 5 | er = numel(bad) / size(x, 1); 6 | bad=labels; 7 | end 8 | -------------------------------------------------------------------------------- /MaxOut/nntrain.m: -------------------------------------------------------------------------------- 1 | function [nn, L] = nntrain(nn, train_x, train_y, opts, val_x, val_y) 2 | %NNTRAIN trains a neural net 3 | % [nn, L] = nnff(nn, x, y, opts) trains the neural network nn with input x and 4 | % output y for opts.numepochs epochs, with minibatches of size 5 | % opts.batchsize. Returns a neural network nn with updated activations, 6 | % errors, weights and biases, (nn.a, nn.e, nn.W, nn.b) and L, the sum 7 | % squared error for each training minibatch. 8 | 9 | assert(isfloat(train_x), 'train_x must be a float'); 10 | assert(nargin == 4 || nargin == 6,'number ofinput arguments must be 4 or 6') 11 | 12 | loss.train.e = []; 13 | loss.train.e_frac = []; 14 | loss.val.e = []; 15 | loss.val.e_frac = []; 16 | opts.validation = 0; 17 | if nargin == 6 18 | opts.validation = 1; 19 | end 20 | 21 | fhandle = []; 22 | if isfield(opts,'plot') && opts.plot == 1 23 | fhandle = figure(); 24 | end 25 | 26 | m = size(train_x, 1); 27 | 28 | batchsize = opts.batchsize; 29 | numepochs = opts.numepochs; 30 | 31 | numbatches = m / batchsize; 32 | 33 | assert(rem(numbatches, 1) == 0, 'numbatches must be a integer'); 34 | 35 | L = zeros(numepochs*numbatches,1); 36 | n = 1; 37 | nn.iter = 1; 38 | for i = 1 : numepochs 39 | tic; 40 | 41 | kk = randperm(m); 42 | for l = 1 : numbatches 43 | batch_x = train_x(kk((l - 1) * batchsize + 1 : l * batchsize), :); 44 | 45 | %Add noise to input (for use in denoising autoencoder) 46 | if(nn.inputZeroMaskedFraction ~= 0) 47 | batch_x = batch_x.*(rand(size(batch_x))>nn.inputZeroMaskedFraction); 48 | end 49 | 50 | batch_y = train_y(kk((l - 1) * batchsize + 1 : l * batchsize), :); 51 | 52 | nn = nnff(nn, batch_x, batch_y); 53 | nn = nnbp(nn); 54 | nn = nnapplygrads(nn); 55 | 56 | L(n) = nn.L; 57 | 58 | n = n + 1; 59 | if mod(l,10)==0 60 | disp(['epoch ' num2str(i) '/' num2str(opts.numepochs) ' batch ' num2str(l) '/' num2str(numbatches) ' error ' num2str(nn.L)]); 61 | end; 62 | end 63 | 64 | t = toc; 65 | 66 | if opts.validation == 1 67 | loss = nneval(nn, loss, train_x, train_y, val_x, val_y); 68 | str_perf = sprintf('; Full-batch train mse = %f, val mse = %f', loss.train.e(end), loss.val.e(end)); 69 | else 70 | loss = nneval(nn, loss, train_x, train_y); 71 | str_perf = sprintf('; Full-batch train err = %f', loss.train.e(end)); 72 | end 73 | if ishandle(fhandle) 74 | nnupdatefigures(nn, fhandle, loss, opts, i); 75 | end 76 | 77 | disp(['epoch ' num2str(i) '/' num2str(opts.numepochs) '. Took ' num2str(t) ' seconds' '. Mini-batch mean squared error on training set is ' num2str(mean(L((n-numbatches):(n-1)))) str_perf]); 78 | nn.learningRate = nn.learningRate * nn.scaling_learningRate; 79 | end 80 | end 81 | 82 | -------------------------------------------------------------------------------- /MaxOut/nnupdatefigures.m: -------------------------------------------------------------------------------- 1 | function nnupdatefigures(nn,fhandle,L,opts,i) 2 | %NNUPDATEFIGURES updates figures during training 3 | if i > 1 %dont plot first point, its only a point 4 | x_ax = 1:i; 5 | % create legend 6 | if opts.validation == 1 7 | M = {'Training','Validation'}; 8 | else 9 | M = {'Training'}; 10 | end 11 | 12 | %create data for plots 13 | if strcmp(nn.output,'softmax') 14 | plot_x = x_ax'; 15 | plot_ye = L.train.e'; 16 | plot_yfrac = L.train.e_frac'; 17 | 18 | else 19 | plot_x = x_ax'; 20 | plot_ye = L.train.e'; 21 | end 22 | 23 | %add error on validation data if present 24 | if opts.validation == 1 25 | plot_x = [plot_x, x_ax']; 26 | plot_ye = [plot_ye,L.val.e']; 27 | end 28 | 29 | 30 | %add classification error on validation data if present 31 | if opts.validation == 1 && strcmp(nn.output,'softmax') 32 | plot_yfrac = [plot_yfrac, L.val.e_frac']; 33 | end 34 | 35 | % plotting 36 | figure(fhandle); 37 | if strcmp(nn.output,'softmax') %also plot classification error 38 | 39 | p1 = subplot(1,2,1); 40 | plot(plot_x,plot_ye); 41 | xlabel('Number of epochs'); ylabel('Error');title('Error'); 42 | title('Error') 43 | legend(p1, M,'Location','NorthEast'); 44 | set(p1, 'Xlim',[0,opts.numepochs + 1]) 45 | 46 | p2 = subplot(1,2,2); 47 | plot(plot_x,plot_yfrac); 48 | xlabel('Number of epochs'); ylabel('Misclassification rate'); 49 | title('Misclassification rate') 50 | legend(p2, M,'Location','NorthEast'); 51 | set(p2, 'Xlim',[0,opts.numepochs + 1]) 52 | 53 | else 54 | 55 | p = plot(plot_x,plot_ye); 56 | xlabel('Number of epochs'); ylabel('Error');title('Error'); 57 | legend(p, M,'Location','NorthEast'); 58 | set(gca, 'Xlim',[0,opts.numepochs + 1]) 59 | 60 | end 61 | drawnow; 62 | end 63 | end 64 | -------------------------------------------------------------------------------- /MaxOut/test_example_MaxOut.m: -------------------------------------------------------------------------------- 1 | % function test_example_MaxOut 2 | load mnist_uint8; 3 | 4 | train_x = double(train_x) / 255; 5 | test_x = double(test_x) / 255; 6 | train_y = double(train_y); 7 | test_y = double(test_y); 8 | 9 | % normalize 10 | [train_x, mu, sigma] = zscore(train_x); 11 | test_x = normalize(test_x, mu, sigma); 12 | 13 | debug = false; 14 | if debug 15 | rand('state',0) 16 | nn = nnsetup([21 5 10],5); 17 | nn.output = 'softmax'; % use softmax output 18 | % nn.dropoutFraction = 0.5; 19 | opts.numepochs = 3; % Number of full sweeps through data 20 | opts.batchsize = 100; % Take a mean gradient step over this many samples 21 | opts.plot = 0; % enable plotting 22 | nnchecknumgrad(nn,randn(100,21),train_y(1:100,:)); 23 | end; 24 | 25 | 26 | rand('state',0) 27 | nn = nnsetup([784 240 240 10],5); 28 | nn.output = 'softmax'; % use softmax output 29 | nn.dropoutFraction = 0.8; 30 | opts.numepochs = 3; % Number of full sweeps through data 31 | opts.batchsize = 100; % Take a mean gradient step over this many samples 32 | opts.plot = 0; % enable plotting 33 | nn = nntrain(nn, train_x, train_y, opts); % nntrain takes validation set as last two arguments (optionally) 34 | 35 | [er, bad] = nntest(nn, test_x, test_y); 36 | assert(er < 0.1, 'Too big error'); 37 | -------------------------------------------------------------------------------- /NN/nnapplygrads.m: -------------------------------------------------------------------------------- 1 | function nn = nnapplygrads(nn) 2 | %NNAPPLYGRADS updates weights and biases with calculated gradients 3 | % nn = nnapplygrads(nn) returns an neural network structure with updated 4 | % weights and biases 5 | 6 | for i = 1 : (nn.n - 1) 7 | if(nn.weightPenaltyL2>0) 8 | dW = nn.dW{i} + nn.weightPenaltyL2 * [zeros(size(nn.W{i},1),1) nn.W{i}(:,2:end)]; 9 | % dW = nn.dW{i} + nn.weightPenaltyL2 * [zeros(size(nn.W{i},1),1) sign(nn.W{i}(:,2:end))]; 10 | else 11 | dW = nn.dW{i}; 12 | end 13 | 14 | nn.rW{i} = 0.9 * nn.rW{i} + 0.1*dW.^2; 15 | dW = nn.learningRate * dW ./ (sqrt(nn.rW{i})+nn.epsilon); 16 | 17 | if(nn.momentum>0) 18 | nn.vW{i} = nn.momentum*nn.vW{i} + dW; 19 | dW = nn.vW{i}; 20 | end 21 | 22 | nn.W{i} = nn.W{i} - dW; 23 | end 24 | if nn.useBatchNormalization 25 | for i = 1 : (nn.n - 2) 26 | nn.rBN{i} = 0.9 * nn.rBN{i} + 0.1*nn.dBN{i}.^2; 27 | dBN = nn.learningRate * nn.dBN{i} ./ (sqrt(nn.rBN{i})+nn.epsilon); 28 | nn.vBN{i} = nn.momentum*nn.vBN{i} + dBN; 29 | nn.gamma{i} = nn.gamma{i} - nn.vBN{i}(1:length(nn.gamma{i})); 30 | nn.beta{i} = nn.beta{i} - nn.vBN{i}(length(nn.gamma{i})+1:end); 31 | end; 32 | end; 33 | if strcmp(nn.activation_function,'ReLU') 34 | da = nn.learningRate * nn.da; 35 | nn.va = nn.momentum*nn.va + da; 36 | nn.ra = nn.ra - nn.va; 37 | end; 38 | 39 | end 40 | -------------------------------------------------------------------------------- /NN/nnbp.m: -------------------------------------------------------------------------------- 1 | function nn = nnbp(nn) 2 | %NNBP performs backpropagation 3 | % nn = nnbp(nn) returns an neural network structure with updated weights 4 | 5 | n = nn.n; 6 | m = size(nn.a{1},1); 7 | sparsityError = 0; 8 | switch nn.output 9 | case 'sigm' 10 | d{n} = - nn.e .* (nn.a{n} .* (1 - nn.a{n})); 11 | case {'softmax','linear', 'ReLU'} 12 | d{n} = - nn.e; 13 | case 'hinge' 14 | if nn.hinge_norm==1 15 | d{n} = -sign(nn.a{n}) .* nn.e; 16 | else 17 | d{n} = -nn.a{n} .* nn.e; 18 | end; 19 | end 20 | for i = (n - 1) : -1 : 2 21 | if(nn.nonSparsityPenalty>0) 22 | pi = repmat(nn.p{i}, size(nn.a{i}, 1), 1); 23 | % sparsityError = [zeros(size(nn.a{i},1),1) nn.nonSparsityPenalty * (-nn.sparsityTarget ./ pi + (1 - nn.sparsityTarget) ./ (1 - pi))]; 24 | sparsityError = [zeros(size(nn.a{i},1),1) nn.nonSparsityPenalty * ones(size(pi))]; 25 | end 26 | 27 | % Backpropagate first derivatives 28 | if i+1==n % in this case in d{n} there is not the bias term to be removed 29 | d{i} = (d{i + 1} * nn.W{i} + sparsityError); % Bishop (5.56) 30 | else % in this case in d{i} the bias term has to be removed 31 | d{i} = (d{i + 1}(:,2:end) * nn.W{i} + sparsityError); 32 | end 33 | 34 | if(nn.dropoutFraction>0)%&&i>=n-3 35 | d{i} = d{i} .* [ones(size(d{i},1),1) nn.dropOutMask{i}]; 36 | end 37 | 38 | % Derivative of the activation function 39 | switch nn.activation_function 40 | case 'sigm' 41 | d_act = nn.a{i} .* (1 - nn.a{i}); 42 | case 'tanh_opt' 43 | d_act = 1.7159 * 2/3 * (1 - 1/(1.7159)^2 * nn.a{i}.^2); 44 | case 'ReLU' 45 | d_act = (nn.a{i} > 0) + nn.ra(i-1) * (nn.a{i} < 0); 46 | tt = d{i}(nn.a{i} < 0) .* nn.a{i}(nn.a{i}<0) / nn.ra(i-1); 47 | % tt = tt(nn.a{i} < 0); 48 | nn.da(i-1) = sum(tt) / size(nn.a{i},1); 49 | case 'linear' 50 | d_act = ones(size(nn.a{i})); 51 | end 52 | d{i} = d{i} .* d_act;%dl/dy 53 | if nn.useBatchNormalization 54 | d_xhat = bsxfun(@times, d{i}(:,2:end), nn.gamma{i-1}); 55 | x_mu = bsxfun(@minus, nn.a_pre{i}, nn.mu{i-1}); 56 | inv_sqrt_sigma = 1 ./ sqrt(nn.sigma2{i-1} + nn.epsilon); 57 | d_sigma2 = -0.5 * sum(d_xhat .* x_mu) .* inv_sqrt_sigma.^3; 58 | d_mu = bsxfun(@times, d_xhat, inv_sqrt_sigma); 59 | d_mu = -1 * sum(d_mu) -2 .* d_sigma2 .* mean(x_mu); 60 | d_gamma = mean(d{i}(:,2:end) .* nn.a_hat{i}); 61 | d_beta = mean(d{i}(:,2:end)); 62 | di1 = bsxfun(@times,d_xhat,inv_sqrt_sigma); 63 | di2 = 2/m * bsxfun(@times, d_sigma2,x_mu); 64 | d{i}(:,2:end) = di1 + di2 + 1/m * repmat(d_mu,m,1); 65 | nn.dBN{i-1} = [d_gamma d_beta]; 66 | nn.d_sigma{i-1} = d_sigma2; 67 | end; 68 | 69 | end 70 | 71 | for i = 1 : (n - 1) 72 | if i+1==n 73 | nn.dW{i} = (d{i + 1}' * nn.a{i}) / size(d{i + 1}, 1); 74 | else 75 | nn.dW{i} = (d{i + 1}(:,2:end)' * nn.a{i}) / size(d{i + 1}, 1); 76 | end 77 | end 78 | end 79 | -------------------------------------------------------------------------------- /NN/nnchecknumgrad.m: -------------------------------------------------------------------------------- 1 | function nnchecknumgrad(nn, x, y) 2 | epsilon = 1e-4; 3 | er = 1e-7; 4 | n = nn.n; 5 | rng('default'); 6 | nnr = nn; 7 | nnr = nnff(nnr, x, y); 8 | nnr = nnbp(nnr); 9 | % for l = 1 : (n - 2) 10 | % nn_m = nn; nn_p = nn; 11 | % nn_m.ra(l) = nn.ra(l) - epsilon; 12 | % nn_p.ra(l) = nn.ra(l) + epsilon; 13 | % rng('default'); 14 | % nn_m = nnff(nn_m, x, y); 15 | % rng('default'); 16 | % nn_p = nnff(nn_p, x, y); 17 | % dW = (nn_p.L - nn_m.L) / (2 * epsilon); 18 | % rdW = nnr.da(l); 19 | % e = abs(dW - rdW); 20 | % 21 | % assert(e < er, 'numerical gradient checking failed'); 22 | % end; 23 | % for l = 1 : (n - 2) 24 | % for j = 1 : size(nn.W{l}, 2)-1 25 | % nn_m = nn; nn_p = nn; 26 | % nn_m.gamma{l}(j) = nn.gamma{l}(j) - epsilon; 27 | % nn_p.gamma{l}(j) = nn.gamma{l}(j) + epsilon; 28 | % rng('default'); 29 | % nn_m = nnff(nn_m, x, y); 30 | % rng('default'); 31 | % nn_p = nnff(nn_p, x, y); 32 | % dW = (nn_p.L - nn_m.L) / (2 * epsilon); 33 | % rdW = nnr.dBN{l}(j); 34 | % e = abs(dW - rdW); 35 | % 36 | % assert(e < er, 'numerical gradient checking failed'); 37 | % end; 38 | % end; 39 | 40 | for l = 1 : (n - 1) 41 | for i = 1 : size(nn.W{l}, 1) 42 | for j = 1 : size(nn.W{l}, 2) 43 | nn_m = nn; nn_p = nn; 44 | nn_m.W{l}(i, j) = nn.W{l}(i, j) - epsilon; 45 | nn_p.W{l}(i, j) = nn.W{l}(i, j) + epsilon; 46 | rng('default'); 47 | nn_m = nnff(nn_m, x, y); 48 | rng('default'); 49 | nn_p = nnff(nn_p, x, y); 50 | dW = (nn_p.L - nn_m.L) / (2 * epsilon); 51 | rdW = nnr.dW{l}(i, j); 52 | e = abs(dW - rdW); 53 | 54 | % assert(e < er, 'numerical gradient checking failed'); 55 | end 56 | end 57 | end 58 | end 59 | -------------------------------------------------------------------------------- /NN/nneval.m: -------------------------------------------------------------------------------- 1 | function [loss] = nneval(nn, loss, train_x, train_y, val_x, val_y) 2 | %NNEVAL evaluates performance of neural network 3 | % Returns a updated loss struct 4 | assert(nargin == 4 || nargin == 6, 'Wrong number of arguments'); 5 | 6 | % training performance 7 | nn = nnff(nn, train_x, train_y); 8 | loss.train.e(end + 1) = nn.L; 9 | 10 | % validation performance 11 | if nargin == 6 12 | nn.testing = 1; 13 | nn = nnff(nn, val_x, val_y); 14 | nn.testing = 0; 15 | loss.val.e(end + 1) = nn.L; 16 | end 17 | 18 | %calc misclassification rate if softmax 19 | if strcmp(nn.output,'softmax')||strcmp(nn.output,'hinge') 20 | [er_train, ~] = nntest(nn, train_x, train_y); 21 | loss.train.e_frac(end+1) = er_train; 22 | 23 | if nargin == 6 24 | [er_val, ~] = nntest(nn, val_x, val_y); 25 | loss.val.e_frac(end+1) = er_val; 26 | end 27 | end 28 | 29 | end 30 | -------------------------------------------------------------------------------- /NN/nnff.m: -------------------------------------------------------------------------------- 1 | function nn = nnff(nn, x, y) 2 | %NNFF performs a feedforward pass 3 | % nn = nnff(nn, x, y) returns an neural network structure with updated 4 | % layer activations, error and loss (nn.a, nn.e and nn.L) 5 | 6 | n = nn.n; 7 | m = size(x, 1); 8 | 9 | x = [ones(m,1) x]; 10 | nn.a{1} = x; 11 | 12 | %feedforward pass 13 | for i = 2 : n-1 14 | if nn.useBatchNormalization 15 | if nn.testing 16 | nn.a_pre{i} = nn.a{i - 1} * nn.W{i - 1}'; 17 | norm_factor = nn.gamma{i-1}./sqrt(nn.mean_sigma2{i-1}+nn.epsilon); 18 | nn.a_hat{i} = bsxfun(@times, nn.a_pre{i}, norm_factor); 19 | nn.a_hat{i} = bsxfun(@plus, nn.a_hat{i}, nn.beta{i-1} - norm_factor .* nn.mean_mu{i-1}); 20 | else 21 | nn.a_pre{i} = nn.a{i - 1} * nn.W{i - 1}'; 22 | nn.mu{i-1} = mean(nn.a_pre{i}); 23 | x_mu = bsxfun(@minus,nn.a_pre{i},nn.mu{i-1}); 24 | nn.sigma2{i-1} = mean(x_mu.^2); 25 | norm_factor = nn.gamma{i-1}./sqrt(nn.sigma2{i-1}+nn.epsilon); 26 | nn.a_hat{i} = bsxfun(@times, nn.a_pre{i}, norm_factor); 27 | nn.a_hat{i} = bsxfun(@plus, nn.a_hat{i}, nn.beta{i-1} - norm_factor .* nn.mu{i-1}); 28 | end; 29 | else 30 | nn.a_hat{i} = nn.a{i - 1} * nn.W{i - 1}'; 31 | end; 32 | 33 | switch nn.activation_function 34 | case 'sigm' 35 | % Calculate the unit's outputs (including the bias term) 36 | nn.a{i} = sigm(nn.a_hat{i}); 37 | case 'tanh_opt' 38 | nn.a{i} = tanh_opt(nn.a_hat{i}); 39 | case 'ReLU' 40 | nn.a{i} = max(nn.a_hat{i}, 0) + nn.ra(i-1) * min(nn.a_hat{i}, 0); 41 | case 'linear' 42 | nn.a{i} = nn.a_hat{i}; 43 | end 44 | 45 | %dropout 46 | if(nn.dropoutFraction > 0)%&&i>=n-3 47 | if(~nn.testing) 48 | nn.dropOutMask{i} = (rand(size(nn.a{i}))>nn.dropoutFraction); 49 | nn.a{i} = nn.a{i}.*nn.dropOutMask{i} / (1 - nn.dropoutFraction); 50 | end 51 | end 52 | 53 | %calculate running exponential activations for use with sparsity 54 | if(nn.nonSparsityPenalty>0) 55 | nn.p{i} = 0.99 * nn.p{i} + 0.01 * mean(nn.a{i}, 1); 56 | end 57 | 58 | %Add the bias term 59 | nn.a{i} = [ones(m,1) nn.a{i}]; 60 | end 61 | switch nn.output 62 | case 'sigm' 63 | nn.a{n} = sigm(nn.a{n - 1} * nn.W{n - 1}'); 64 | case 'linear' 65 | nn.a{n} = nn.a{n - 1} * nn.W{n - 1}'; 66 | case 'softmax' 67 | nn.a{n} = nn.a{n - 1} * nn.W{n - 1}'; 68 | nn.a{n} = exp(bsxfun(@minus, nn.a{n}, max(nn.a{n},[],2))); 69 | nn.a{n} = bsxfun(@rdivide, nn.a{n}, sum(nn.a{n}, 2)); 70 | case 'ReLU' 71 | nn.a{n} = max(nn.a{n - 1} * nn.W{n - 1}',0); 72 | case 'hinge' 73 | nn.a{n} = nn.a{n - 1} * nn.W{n - 1}'; 74 | if ~nn.testing 75 | nn.e = 2 * (y - 0.5); 76 | nn.a{n} = max(0,1 - nn.a{n} .* nn.e); 77 | end; 78 | end 79 | 80 | %error and loss 81 | % size(y) 82 | % size(nn.a{n}) 83 | 84 | 85 | switch nn.output 86 | case {'sigm', 'linear', 'ReLU'} 87 | nn.e = y - nn.a{n}; 88 | nn.L = 1/2 * sum(sum(nn.e .^ 2)) / m; 89 | case 'softmax' 90 | nn.e = y - nn.a{n}; 91 | nn.L = -sum(sum(y .* log(nn.a{n}+1e-10))) / m; 92 | case 'hinge' 93 | if nn.hinge_norm==1 94 | nn.L = sum(sum(abs(nn.a{n}))) / m; 95 | else 96 | nn.L = 1/2 * sum(sum(nn.a{n}.*nn.a{n})) / m; 97 | end; 98 | % nn.a{n}(y==1) = -1 * nn.a{n}(y==1); 99 | end 100 | end 101 | -------------------------------------------------------------------------------- /NN/nnpredict.m: -------------------------------------------------------------------------------- 1 | function labels = nnpredict(nn, x) 2 | nn.testing = 1; 3 | nn = nnff(nn, x, zeros(size(x,1), nn.size(end))); 4 | nn.testing = 0; 5 | 6 | [~, i] = max(nn.a{end},[],2); 7 | labels = i; 8 | end 9 | -------------------------------------------------------------------------------- /NN/nnsetup - 副本.m: -------------------------------------------------------------------------------- 1 | function nn = nnsetup(architecture,numChannels) 2 | %NNSETUP creates a Feedforward Backpropagate Neural Network 3 | % nn = nnsetup(architecture) returns an neural network structure with n=numel(architecture) 4 | % layers, architecture being a n x 1 vector of layer sizes e.g. [784 100 10] 5 | 6 | nn.size = architecture; 7 | nn.n = numel(nn.size); 8 | 9 | nn.activation_function = 'MaxOut'; % Activation functions of hidden layers: 'sigm' (sigmoid) or 'tanh_opt' (optimal tanh). 10 | nn.learningRate = 2; % learning rate Note: typically needs to be lower when using 'sigm' activation function and non-normalized inputs. 11 | nn.momentum = 0.5; % Momentum 12 | nn.scaling_learningRate = 1; % Scaling factor for the learning rate (each epoch) 13 | nn.weightPenaltyL2 = 0; % L2 regularization 14 | nn.nonSparsityPenalty = 0; % Non sparsity penalty 15 | nn.sparsityTarget = 0.05; % Sparsity target 16 | nn.inputZeroMaskedFraction = 0.2; % Used for Denoising AutoEncoders 17 | nn.dropoutFraction = 0.5; % Dropout level (http://www.cs.toronto.edu/~hinton/absps/dropout.pdf) 18 | nn.testing = 0; % Internal variable. nntest sets this to one. 19 | nn.output = 'softmax'; % output unit 'sigm' (=logistic), 'softmax' and 'linear' 20 | if ~exist('numChannels','var') 21 | nn.numChannels = 10; 22 | end; 23 | 24 | for i = 2 : nn.n 25 | % weights and weight momentum 26 | nn.W{i - 1} = (rand(nn.size(i), nn.size(i - 1)+1, nn.numChannels) - 0.5) * 2 * 4 * sqrt(6 / (nn.size(i) + nn.size(i - 1) + nn.numChannels)); 27 | nn.vW{i - 1} = zeros(size(nn.W{i - 1})); 28 | 29 | % average activations (for use with sparsity) 30 | nn.p{i} = zeros(1, nn.size(i)); 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /NN/nnsetup.m: -------------------------------------------------------------------------------- 1 | function nn = nnsetup(architecture) 2 | %NNSETUP creates a Feedforward Backpropagate Neural Network 3 | % nn = nnsetup(architecture) returns an neural network structure with n=numel(architecture) 4 | % layers, architecture being a n x 1 vector of layer sizes e.g. [784 100 10] 5 | 6 | nn.size = architecture; 7 | nn.n = numel(nn.size); 8 | 9 | nn.activation_function = 'sigm'; % Activation functions of hidden layers: 'sigm' (sigmoid) or 'tanh_opt' (optimal tanh). 10 | nn.learningRate = 0.00004; % learning rate Note: typically needs to be lower when using 'sigm' activation function and non-normalized inputs. 11 | nn.momentum = 0.8; % Momentum 12 | nn.scaling_learningRate = 0.9; % Scaling factor for the learning rate (each epoch) 13 | nn.weightPenaltyL2 = 0; % L2 regularization 14 | nn.nonSparsityPenalty = 0; % Non sparsity penalty 15 | nn.sparsityTarget = 0.05; % Sparsity target 16 | nn.inputZeroMaskedFraction = 0; % Used for Denoising AutoEncoders 17 | nn.dropoutFraction = 0; % Dropout level (http://www.cs.toronto.edu/~hinton/absps/dropout.pdf) 18 | nn.testing = 0; % Internal variable. nntest sets this to one. 19 | nn.output = 'sigm'; % output unit 'sigm' (=logistic), 'softmax' and 'linear' 20 | nn.epsilon = 1e-10; % numeric factor for batch normalization 21 | nn.useBatchNormalization = 1; % 22 | 23 | for i = 2 : nn.n 24 | 25 | nn.ra(i-1) = 0.2;%prelu 26 | nn.va(i-1) = 0; 27 | nn.pow(i-1) = 0.5;%pow layer 28 | nn.vpow(i-1) = 0; 29 | 30 | nn.W{i - 1} = randn(nn.size(i), nn.size(i - 1)+1) * sqrt(2 / (nn.size(i) + nn.size(i - 1)) / (1 + nn.ra(i-1)^2));% weights and weight momentum 31 | nn.vW{i - 1} = zeros(size(nn.W{i - 1})); 32 | nn.rW{i - 1} = zeros(size(nn.W{i - 1})); 33 | 34 | nn.beta{i-1} = zeros(1,nn.size(i));%batch normalization 35 | nn.gamma{i-1} = ones(1,nn.size(i)); 36 | nn.sigma2{i-1} = ones(1,nn.size(i)); 37 | nn.mu{i-1} = zeros(1,nn.size(i)); 38 | nn.vBN{i-1} = zeros(1,nn.size(i) * 2); 39 | nn.rBN{i-1} = zeros(1,nn.size(i) * 2); 40 | nn.mean_sigma2{i-1} = zeros(1,nn.size(i)); 41 | nn.mean_mu{i-1} = zeros(1,nn.size(i)); 42 | 43 | % average activations (for use with sparsity) 44 | nn.p{i} = zeros(1, nn.size(i)); 45 | end 46 | nn.ra = nn.ra(1:end-1); 47 | nn.va = nn.va(1:end-1); 48 | end 49 | -------------------------------------------------------------------------------- /NN/nntest.m: -------------------------------------------------------------------------------- 1 | function [er, bad] = nntest(nn, x, y) 2 | labels = nnpredict(nn, x); 3 | [~, expected] = max(y,[],2); 4 | bad = find(labels ~= expected); 5 | er = numel(bad) / size(x, 1); 6 | bad=labels; 7 | end 8 | -------------------------------------------------------------------------------- /NN/nntrain.m: -------------------------------------------------------------------------------- 1 | function [nn, L] = nntrain(nn, train_x, train_y, opts, val_x, val_y) 2 | %NNTRAIN trains a neural net 3 | % [nn, L] = nnff(nn, x, y, opts) trains the neural network nn with input x and 4 | % output y for opts.numepochs epochs, with minibatches of size 5 | % opts.batchsize. Returns a neural network nn with updated activations, 6 | % errors, weights and biases, (nn.a, nn.e, nn.W, nn.b) and L, the sum 7 | % squared error for each training minibatch. 8 | 9 | assert(isfloat(train_x), 'train_x must be a float'); 10 | assert(nargin == 4 || nargin == 6,'number ofinput arguments must be 4 or 6') 11 | 12 | loss.train.e = []; 13 | loss.train.e_frac = []; 14 | loss.val.e = []; 15 | loss.val.e_frac = []; 16 | opts.validation = 0; 17 | if nargin == 6 18 | opts.validation = 1; 19 | end 20 | 21 | fhandle = []; 22 | if isfield(opts,'plot') && opts.plot == 1 23 | fhandle = figure(); 24 | end 25 | 26 | m = size(train_x, 1); 27 | 28 | batchsize = opts.batchsize; 29 | numepochs = opts.numepochs; 30 | 31 | numbatches = m / batchsize; 32 | 33 | assert(rem(numbatches, 1) == 0, 'numbatches must be a integer'); 34 | 35 | L = zeros(numepochs*numbatches,1); 36 | n = 1; 37 | for i = 1 : numepochs 38 | tic; 39 | for j = 2 : nn.n 40 | nn.mean_sigma2{j-1} = 0; 41 | nn.mean_mu{j-1} = 0; 42 | end; 43 | kk = randperm(m); 44 | for l = 1 : numbatches 45 | batch_x = train_x(kk((l - 1) * batchsize + 1 : l * batchsize), :); 46 | 47 | %Add noise to input (for use in denoising autoencoder) 48 | if(nn.inputZeroMaskedFraction ~= 0) 49 | batch_x = batch_x.*(rand(size(batch_x))>nn.inputZeroMaskedFraction); 50 | end 51 | 52 | batch_y = train_y(kk((l - 1) * batchsize + 1 : l * batchsize), :); 53 | 54 | nn = nnff(nn, batch_x, batch_y); 55 | nn = nnbp(nn); 56 | nn = nnapplygrads(nn); 57 | 58 | L(n) = nn.L; 59 | for j = 2 : nn.n 60 | nn.mean_sigma2{j-1} = nn.mean_sigma2{j-1} + nn.sigma2{j-1}; 61 | nn.mean_mu{j-1} = nn.mean_mu{j-1} + nn.mu{j-1}; 62 | end; 63 | 64 | n = n + 1; 65 | if mod(l,10)==0 66 | fprintf('epoch:%d iteration:%d/%d\n',i,l,numbatches); 67 | gradientNorm = []; 68 | for ll = 1:nn.n-1 69 | gradientNorm = [gradientNorm ' ' num2str(norm(nn.dW{ll}(:,2:end)))]; 70 | end; 71 | disp(gradientNorm); 72 | end; 73 | if mod(l,100)==0 74 | disp([nn.ra mean(nn.gamma{2}) mean(nn.beta{2})]); 75 | end; 76 | end 77 | 78 | for j = 2 : nn.n 79 | nn.mean_sigma2{j-1} = nn.mean_sigma2{j-1} / (numbatches - 1); 80 | nn.mean_mu{j-1} = nn.mean_mu{j-1} / numbatches; 81 | end; 82 | 83 | t = toc; 84 | 85 | 86 | if opts.validation == 1 87 | loss = nneval(nn, loss, train_x, train_y, val_x, val_y); 88 | str_perf = sprintf('; Full-batch train mse = %f, val mse = %f', loss.train.e(end), loss.val.e(end)); 89 | else 90 | loss = nneval(nn, loss, train_x, train_y); 91 | str_perf = sprintf('; Full-batch train err = %f', loss.train.e(end)); 92 | end 93 | if ishandle(fhandle) 94 | nnupdatefigures(nn, fhandle, loss, opts, i); 95 | end 96 | 97 | disp(['epoch ' num2str(i) '/' num2str(opts.numepochs) '. Took ' num2str(t) ' seconds' '. Mini-batch mean squared error on training set is ' num2str(mean(L((n-numbatches):(n-1)))) str_perf]); 98 | disp(nn.ra); 99 | nn.learningRate = nn.learningRate * nn.scaling_learningRate; 100 | if nn.learningRate < 0.00001 101 | nn.learningRate = 0.00001; 102 | end; 103 | % if i==5 104 | % nn.dropoutFraction = 0.2; 105 | % end; 106 | end 107 | end 108 | 109 | -------------------------------------------------------------------------------- /NN/nnupdatefigures.m: -------------------------------------------------------------------------------- 1 | function nnupdatefigures(nn,fhandle,L,opts,i) 2 | %NNUPDATEFIGURES updates figures during training 3 | if i > 1 %dont plot first point, its only a point 4 | x_ax = 1:i; 5 | % create legend 6 | if opts.validation == 1 7 | M = {'Training','Validation'}; 8 | else 9 | M = {'Training'}; 10 | end 11 | 12 | %create data for plots 13 | if strcmp(nn.output,'softmax')||strcmp(nn.output,'hinge') 14 | plot_x = x_ax'; 15 | plot_ye = L.train.e'; 16 | plot_yfrac = L.train.e_frac'; 17 | 18 | else 19 | plot_x = x_ax'; 20 | plot_ye = L.train.e'; 21 | end 22 | 23 | %add error on validation data if present 24 | if opts.validation == 1 25 | plot_x = [plot_x, x_ax']; 26 | plot_ye = [plot_ye,L.val.e']; 27 | end 28 | 29 | 30 | %add classification error on validation data if present 31 | if opts.validation == 1 && (strcmp(nn.output,'softmax')||strcmp(nn.output,'hinge')) 32 | plot_yfrac = [plot_yfrac, L.val.e_frac']; 33 | end 34 | 35 | % plotting 36 | figure(fhandle); 37 | if strcmp(nn.output,'softmax')||strcmp(nn.output,'hinge') %also plot classification error 38 | 39 | p1 = subplot(1,2,1); 40 | plot(plot_x,plot_ye); 41 | xlabel('Number of epochs'); ylabel('Error');title('Error'); 42 | title('Error') 43 | legend(p1, M,'Location','NorthEast'); 44 | set(p1, 'Xlim',[0,opts.numepochs + 1]) 45 | 46 | p2 = subplot(1,2,2); 47 | plot(plot_x,plot_yfrac); 48 | xlabel('Number of epochs'); ylabel('Misclassification rate'); 49 | title('Misclassification rate') 50 | legend(p2, M,'Location','NorthEast'); 51 | set(p2, 'Xlim',[0,opts.numepochs + 1]) 52 | 53 | else 54 | 55 | p = plot(plot_x,plot_ye); 56 | xlabel('Number of epochs'); ylabel('Error');title('Error'); 57 | legend(p, M,'Location','NorthEast'); 58 | set(gca, 'Xlim',[0,opts.numepochs + 1]) 59 | 60 | end 61 | drawnow; 62 | end 63 | end 64 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | DeepLearnToolbox 3 | ================ 4 | 5 | A Matlab toolbox for Deep Learning. 6 | 7 | Deep Learning is a new subfield of machine learning that focuses on learning deep hierarchical models of data. 8 | It is inspired by the human brain's apparent deep (layered, hierarchical) architecture. 9 | A good overview of the theory of Deep Learning theory is 10 | [Learning Deep Architectures for AI](http://www.iro.umontreal.ca/~bengioy/papers/ftml_book.pdf) 11 | 12 | For a more informal introduction, see the following videos by Geoffrey Hinton and Andrew Ng. 13 | 14 | * [The Next Generation of Neural Networks](http://www.youtube.com/watch?v=AyzOUbkUf3M) (Hinton, 2007) 15 | * [Recent Developments in Deep Learning](http://www.youtube.com/watch?v=VdIURAu1-aU) (Hinton, 2010) 16 | * [Unsupervised Feature Learning and Deep Learning](http://www.youtube.com/watch?v=ZmNOAtZIgIk) (Ng, 2011) 17 | 18 | If you use this toolbox in your research please cite [Prediction as a candidate for learning deep hierarchical models of data](http://www2.imm.dtu.dk/pubdb/views/publication_details.php?id=6284) 19 | 20 | ``` 21 | @MASTERSTHESIS\{IMM2012-06284, 22 | author = "R. B. Palm", 23 | title = "Prediction as a candidate for learning deep hierarchical models of data", 24 | year = "2012", 25 | } 26 | ``` 27 | 28 | Contact: rasmusbergpalm at gmail dot com 29 | 30 | Directories included in the toolbox 31 | ----------------------------------- 32 | 33 | `NN/` - A library for Feedforward Backpropagation Neural Networks 34 | 35 | `CNN/` - A library for Convolutional Neural Networks 36 | 37 | `DBN/` - A library for Deep Belief Networks 38 | 39 | `SAE/` - A library for Stacked Auto-Encoders 40 | 41 | `CAE/` - A library for Convolutional Auto-Encoders 42 | 43 | `util/` - Utility functions used by the libraries 44 | 45 | `data/` - Data used by the examples 46 | 47 | `tests/` - unit tests to verify toolbox is working 48 | 49 | For references on each library check REFS.md 50 | 51 | Setup 52 | ----- 53 | 54 | 1. Download. 55 | 2. addpath(genpath('DeepLearnToolbox')); 56 | 57 | Known errors 58 | ------------------------------ 59 | 60 | `test_cnn_gradients_are_numerically_correct` fails on Octave because of a bug in Octave's convn implementation. See http://savannah.gnu.org/bugs/?39314 61 | 62 | `test_example_CNN` fails in Octave for the same reason. 63 | Example: Deep Belief Network 64 | --------------------- 65 | ```matlab 66 | 67 | function test_example_DBN 68 | load mnist_uint8; 69 | 70 | train_x = double(train_x) / 255; 71 | test_x = double(test_x) / 255; 72 | train_y = double(train_y); 73 | test_y = double(test_y); 74 | 75 | %% ex1 train a 100 hidden unit RBM and visualize its weights 76 | rand('state',0) 77 | dbn.sizes = [100]; 78 | opts.numepochs = 1; 79 | opts.batchsize = 100; 80 | opts.momentum = 0; 81 | opts.alpha = 1; 82 | dbn = dbnsetup(dbn, train_x, opts); 83 | dbn = dbntrain(dbn, train_x, opts); 84 | figure; visualize(dbn.rbm{1}.W'); % Visualize the RBM weights 85 | 86 | %% ex2 train a 100-100 hidden unit DBN and use its weights to initialize a NN 87 | rand('state',0) 88 | %train dbn 89 | dbn.sizes = [100 100]; 90 | opts.numepochs = 1; 91 | opts.batchsize = 100; 92 | opts.momentum = 0; 93 | opts.alpha = 1; 94 | dbn = dbnsetup(dbn, train_x, opts); 95 | dbn = dbntrain(dbn, train_x, opts); 96 | 97 | %unfold dbn to nn 98 | nn = dbnunfoldtonn(dbn, 10); 99 | nn.activation_function = 'sigm'; 100 | 101 | %train nn 102 | opts.numepochs = 1; 103 | opts.batchsize = 100; 104 | nn = nntrain(nn, train_x, train_y, opts); 105 | [er, bad] = nntest(nn, test_x, test_y); 106 | 107 | assert(er < 0.10, 'Too big error'); 108 | 109 | ``` 110 | 111 | 112 | Example: Stacked Auto-Encoders 113 | --------------------- 114 | ```matlab 115 | 116 | function test_example_SAE 117 | load mnist_uint8; 118 | 119 | train_x = double(train_x)/255; 120 | test_x = double(test_x)/255; 121 | train_y = double(train_y); 122 | test_y = double(test_y); 123 | 124 | %% ex1 train a 100 hidden unit SDAE and use it to initialize a FFNN 125 | % Setup and train a stacked denoising autoencoder (SDAE) 126 | rand('state',0) 127 | sae = saesetup([784 100]); 128 | sae.ae{1}.activation_function = 'sigm'; 129 | sae.ae{1}.learningRate = 1; 130 | sae.ae{1}.inputZeroMaskedFraction = 0.5; 131 | opts.numepochs = 1; 132 | opts.batchsize = 100; 133 | sae = saetrain(sae, train_x, opts); 134 | visualize(sae.ae{1}.W{1}(:,2:end)') 135 | 136 | % Use the SDAE to initialize a FFNN 137 | nn = nnsetup([784 100 10]); 138 | nn.activation_function = 'sigm'; 139 | nn.learningRate = 1; 140 | nn.W{1} = sae.ae{1}.W{1}; 141 | 142 | % Train the FFNN 143 | opts.numepochs = 1; 144 | opts.batchsize = 100; 145 | nn = nntrain(nn, train_x, train_y, opts); 146 | [er, bad] = nntest(nn, test_x, test_y); 147 | assert(er < 0.16, 'Too big error'); 148 | 149 | ``` 150 | 151 | 152 | Example: Convolutional Neural Nets 153 | --------------------- 154 | ```matlab 155 | 156 | function test_example_CNN 157 | load mnist_uint8; 158 | 159 | train_x = double(reshape(train_x',28,28,60000))/255; 160 | test_x = double(reshape(test_x',28,28,10000))/255; 161 | train_y = double(train_y'); 162 | test_y = double(test_y'); 163 | 164 | %% ex1 Train a 6c-2s-12c-2s Convolutional neural network 165 | %will run 1 epoch in about 200 second and get around 11% error. 166 | %With 100 epochs you'll get around 1.2% error 167 | rand('state',0) 168 | cnn.layers = { 169 | struct('type', 'i') %input layer 170 | struct('type', 'c', 'outputmaps', 6, 'kernelsize', 5) %convolution layer 171 | struct('type', 's', 'scale', 2) %sub sampling layer 172 | struct('type', 'c', 'outputmaps', 12, 'kernelsize', 5) %convolution layer 173 | struct('type', 's', 'scale', 2) %subsampling layer 174 | }; 175 | cnn = cnnsetup(cnn, train_x, train_y); 176 | 177 | opts.alpha = 1; 178 | opts.batchsize = 50; 179 | opts.numepochs = 1; 180 | 181 | cnn = cnntrain(cnn, train_x, train_y, opts); 182 | 183 | [er, bad] = cnntest(cnn, test_x, test_y); 184 | 185 | %plot mean squared error 186 | figure; plot(cnn.rL); 187 | 188 | assert(er<0.12, 'Too big error'); 189 | 190 | ``` 191 | 192 | 193 | Example: Neural Networks 194 | --------------------- 195 | ```matlab 196 | 197 | function test_example_NN 198 | load mnist_uint8; 199 | 200 | train_x = double(train_x) / 255; 201 | test_x = double(test_x) / 255; 202 | train_y = double(train_y); 203 | test_y = double(test_y); 204 | 205 | % normalize 206 | [train_x, mu, sigma] = zscore(train_x); 207 | test_x = normalize(test_x, mu, sigma); 208 | 209 | %% ex1 vanilla neural net 210 | rand('state',0) 211 | nn = nnsetup([784 100 10]); 212 | opts.numepochs = 1; % Number of full sweeps through data 213 | opts.batchsize = 100; % Take a mean gradient step over this many samples 214 | [nn, L] = nntrain(nn, train_x, train_y, opts); 215 | 216 | [er, bad] = nntest(nn, test_x, test_y); 217 | 218 | assert(er < 0.08, 'Too big error'); 219 | 220 | %% ex2 neural net with L2 weight decay 221 | rand('state',0) 222 | nn = nnsetup([784 100 10]); 223 | 224 | nn.weightPenaltyL2 = 1e-4; % L2 weight decay 225 | opts.numepochs = 1; % Number of full sweeps through data 226 | opts.batchsize = 100; % Take a mean gradient step over this many samples 227 | 228 | nn = nntrain(nn, train_x, train_y, opts); 229 | 230 | [er, bad] = nntest(nn, test_x, test_y); 231 | assert(er < 0.1, 'Too big error'); 232 | 233 | 234 | %% ex3 neural net with dropout 235 | rand('state',0) 236 | nn = nnsetup([784 100 10]); 237 | 238 | nn.dropoutFraction = 0.5; % Dropout fraction 239 | opts.numepochs = 1; % Number of full sweeps through data 240 | opts.batchsize = 100; % Take a mean gradient step over this many samples 241 | 242 | nn = nntrain(nn, train_x, train_y, opts); 243 | 244 | [er, bad] = nntest(nn, test_x, test_y); 245 | assert(er < 0.1, 'Too big error'); 246 | 247 | %% ex4 neural net with sigmoid activation function 248 | rand('state',0) 249 | nn = nnsetup([784 100 10]); 250 | 251 | nn.activation_function = 'sigm'; % Sigmoid activation function 252 | nn.learningRate = 1; % Sigm require a lower learning rate 253 | opts.numepochs = 1; % Number of full sweeps through data 254 | opts.batchsize = 100; % Take a mean gradient step over this many samples 255 | 256 | nn = nntrain(nn, train_x, train_y, opts); 257 | 258 | [er, bad] = nntest(nn, test_x, test_y); 259 | assert(er < 0.1, 'Too big error'); 260 | 261 | %% ex5 plotting functionality 262 | rand('state',0) 263 | nn = nnsetup([784 20 10]); 264 | opts.numepochs = 5; % Number of full sweeps through data 265 | nn.output = 'softmax'; % use softmax output 266 | opts.batchsize = 1000; % Take a mean gradient step over this many samples 267 | opts.plot = 1; % enable plotting 268 | 269 | nn = nntrain(nn, train_x, train_y, opts); 270 | 271 | [er, bad] = nntest(nn, test_x, test_y); 272 | assert(er < 0.1, 'Too big error'); 273 | 274 | %% ex6 neural net with sigmoid activation and plotting of validation and training error 275 | % split training data into training and validation data 276 | vx = train_x(1:10000,:); 277 | tx = train_x(10001:end,:); 278 | vy = train_y(1:10000,:); 279 | ty = train_y(10001:end,:); 280 | 281 | rand('state',0) 282 | nn = nnsetup([784 20 10]); 283 | nn.output = 'softmax'; % use softmax output 284 | opts.numepochs = 5; % Number of full sweeps through data 285 | opts.batchsize = 1000; % Take a mean gradient step over this many samples 286 | opts.plot = 1; % enable plotting 287 | nn = nntrain(nn, tx, ty, opts, vx, vy); % nntrain takes validation set as last two arguments (optionally) 288 | 289 | [er, bad] = nntest(nn, test_x, test_y); 290 | assert(er < 0.1, 'Too big error'); 291 | 292 | ``` 293 | 294 | 295 | 296 | 297 | [![Bitdeli Badge](https://d2weczhvl823v0.cloudfront.net/rasmusbergpalm/deeplearntoolbox/trend.png)](https://bitdeli.com/free "Bitdeli Badge") 298 | 299 | -------------------------------------------------------------------------------- /README_header.md: -------------------------------------------------------------------------------- 1 | [![Bitdeli Badge](https://d2weczhvl823v0.cloudfront.net/rasmusbergpalm/deeplearntoolbox/trend.png)](https://bitdeli.com/free "Bitdeli Badge") 2 | 3 | DeepLearnToolbox 4 | ================ 5 | 6 | A Matlab toolbox for Deep Learning. 7 | 8 | Deep Learning is a new subfield of machine learning that focuses on learning deep hierarchical models of data. 9 | It is inspired by the human brain's apparent deep (layered, hierarchical) architecture. 10 | A good overview of the theory of Deep Learning theory is 11 | [Learning Deep Architectures for AI](http://www.iro.umontreal.ca/~bengioy/papers/ftml_book.pdf) 12 | 13 | For a more informal introduction, see the following videos by Geoffrey Hinton and Andrew Ng. 14 | 15 | * [The Next Generation of Neural Networks](http://www.youtube.com/watch?v=AyzOUbkUf3M) (Hinton, 2007) 16 | * [Recent Developments in Deep Learning](http://www.youtube.com/watch?v=VdIURAu1-aU) (Hinton, 2010) 17 | * [Unsupervised Feature Learning and Deep Learning](http://www.youtube.com/watch?v=ZmNOAtZIgIk) (Ng, 2011) 18 | 19 | If you use this toolbox in your research please cite [Prediction as a candidate for learning deep hierarchical models of data](http://www2.imm.dtu.dk/pubdb/views/publication_details.php?id=6284) 20 | 21 | ``` 22 | @MASTERSTHESIS\{IMM2012-06284, 23 | author = "R. B. Palm", 24 | title = "Prediction as a candidate for learning deep hierarchical models of data", 25 | year = "2012", 26 | } 27 | ``` 28 | 29 | Contact: rasmusbergpalm at gmail dot com 30 | 31 | Directories included in the toolbox 32 | ----------------------------------- 33 | 34 | `NN/` - A library for Feedforward Backpropagation Neural Networks 35 | 36 | `CNN/` - A library for Convolutional Neural Networks 37 | 38 | `DBN/` - A library for Deep Belief Networks 39 | 40 | `SAE/` - A library for Stacked Auto-Encoders 41 | 42 | `CAE/` - A library for Convolutional Auto-Encoders 43 | 44 | `util/` - Utility functions used by the libraries 45 | 46 | `data/` - Data used by the examples 47 | 48 | `tests/` - unit tests to verify toolbox is working 49 | 50 | For references on each library check REFS.md 51 | 52 | Setup 53 | ----- 54 | 55 | 1. Download. 56 | 2. addpath(genpath('DeepLearnToolbox')); 57 | 58 | Known errors 59 | ------------------------------ 60 | 61 | `test_cnn_gradients_are_numerically_correct` fails on Octave because of a bug in Octave's convn implementation. See http://savannah.gnu.org/bugs/?39314 62 | 63 | `test_example_CNN` fails in Octave for the same reason. 64 | -------------------------------------------------------------------------------- /REFS.md: -------------------------------------------------------------------------------- 1 | Deep Belief Nets 2 | ---------------- 3 | 4 | * ["A Fast Learning Algorithm for Deep Belief Nets"](http://www.cs.toronto.edu/~hinton/absps/ncfast.pdf) Geoffrey Hinton 2006 - Introduces contrastive divergence and DBNs 5 | * ["A Practical Guide to Training Restricted Boltzmann Machines"](http://www.cs.toronto.edu/~hinton/absps/guideTR.pdf) Geoffrey Hinton 2010 - How to implement DBNs 6 | 7 | Convolutional Neural Nets 8 | ------------------------- 9 | 10 | * ["Handwritten Digit Recognition with a Back-Propagation Network"](http://yann.lecun.com/exdb/publis/pdf/lecun-90c.pdf) Yann LeCun 1990 - Introduces CNNs 11 | * ["Notes on Convolutional Neural Networks"](http://cogprints.org/5869/1/cnn_tutorial.pdf) Jake Bouvrie 2006 - How to implement CNNs 12 | 13 | Auto Encoders 14 | ------------- 15 | 16 | * ["Extracting and Composing Robust Features with Denoising Autoencoders"](http://www.iro.umontreal.ca/~vincentp/Publications/vincent_icml_2008.pdf) Pascal Vincent 2008 - Introduces the Denoising Autoencoder 17 | -------------------------------------------------------------------------------- /SAE/saesetup.m: -------------------------------------------------------------------------------- 1 | function sae = saesetup(size) 2 | for u = 2 : numel(size) 3 | sae.ae{u-1} = nnsetup([size(u-1) size(u) size(u-1)]); 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /SAE/saetrain.m: -------------------------------------------------------------------------------- 1 | function sae = saetrain(sae, x, opts) 2 | for i = 1 : numel(sae.ae); 3 | disp(['Training AE ' num2str(i) '/' num2str(numel(sae.ae))]); 4 | sae.ae{i} = nntrain(sae.ae{i}, x, x, opts); 5 | t = nnff(sae.ae{i}, x, x); 6 | x = t.a{2}; 7 | %remove bias term 8 | x = x(:,2:end); 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /data/mnist_uint8.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/happynear/DeepLearnToolbox/2539a8cc2dcd0faab7215949b9714d7705c18738/data/mnist_uint8.mat -------------------------------------------------------------------------------- /test_example_NN.m: -------------------------------------------------------------------------------- 1 | % function test_example_NN 2 | load mnist_uint8; 3 | 4 | train_x = double(train_x) / 255; 5 | test_x = double(test_x) / 255; 6 | train_y = double(train_y); 7 | test_y = double(test_y); 8 | 9 | % normalize 10 | [train_x, mu, sigma] = zscore(train_x); 11 | test_x = normalize(test_x, mu, sigma); 12 | 13 | %% ex1 vanilla neural net 14 | rand('state',0) 15 | nn = nnsetup([784 100 10]); 16 | opts.numepochs = 10; % Number of full sweeps through data 17 | opts.batchsize = 100; % Take a mean gradient step over this many samples 18 | [nn, L] = nntrain(nn, train_x, train_y, opts); 19 | 20 | [er, bad] = nntest(nn, test_x, test_y); 21 | 22 | assert(er < 0.08, 'Too big error'); 23 | 24 | %% ex2 neural net with L2 weight decay 25 | % rand('state',0) 26 | % nn = nnsetup([784 100 10]); 27 | % 28 | % nn.weightPenaltyL2 = 1e-4; % L2 weight decay 29 | % opts.numepochs = 1; % Number of full sweeps through data 30 | % opts.batchsize = 100; % Take a mean gradient step over this many samples 31 | % 32 | % nn = nntrain(nn, train_x, train_y, opts); 33 | % 34 | % [er, bad] = nntest(nn, test_x, test_y); 35 | % assert(er < 0.1, 'Too big error'); 36 | % 37 | % 38 | % %% ex3 neural net with dropout 39 | % rand('state',0) 40 | % nn = nnsetup([784 100 10]); 41 | % 42 | % nn.dropoutFraction = 0.5; % Dropout fraction 43 | % opts.numepochs = 1; % Number of full sweeps through data 44 | % opts.batchsize = 100; % Take a mean gradient step over this many samples 45 | % 46 | % nn = nntrain(nn, train_x, train_y, opts); 47 | % 48 | % [er, bad] = nntest(nn, test_x, test_y); 49 | % assert(er < 0.1, 'Too big error'); 50 | % 51 | % %% ex4 neural net with sigmoid activation function 52 | % rand('state',0) 53 | % nn = nnsetup([784 100 10]); 54 | % 55 | % nn.activation_function = 'sigm'; % Sigmoid activation function 56 | % nn.learningRate = 1; % Sigm require a lower learning rate 57 | % opts.numepochs = 1; % Number of full sweeps through data 58 | % opts.batchsize = 100; % Take a mean gradient step over this many samples 59 | % 60 | % nn = nntrain(nn, train_x, train_y, opts); 61 | % 62 | % [er, bad] = nntest(nn, test_x, test_y); 63 | % assert(er < 0.1, 'Too big error'); 64 | % 65 | % %% ex5 plotting functionality 66 | % rand('state',0) 67 | % nn = nnsetup([784 20 10]); 68 | % opts.numepochs = 5; % Number of full sweeps through data 69 | % nn.output = 'softmax'; % use softmax output 70 | % opts.batchsize = 1000; % Take a mean gradient step over this many samples 71 | % opts.plot = 1; % enable plotting 72 | % 73 | % nn = nntrain(nn, train_x, train_y, opts); 74 | % 75 | % [er, bad] = nntest(nn, test_x, test_y); 76 | % assert(er < 0.1, 'Too big error'); 77 | % 78 | % %% ex6 neural net with sigmoid activation and plotting of validation and training error 79 | % % split training data into training and validation data 80 | % vx = train_x(1:10000,:); 81 | % tx = train_x(10001:end,:); 82 | % vy = train_y(1:10000,:); 83 | % ty = train_y(10001:end,:); 84 | % 85 | % rand('state',0) 86 | % nn = nnsetup([784 20 10]); 87 | % nn.output = 'softmax'; % use softmax output 88 | % opts.numepochs = 5; % Number of full sweeps through data 89 | % opts.batchsize = 1000; % Take a mean gradient step over this many samples 90 | % opts.plot = 1; % enable plotting 91 | % nn = nntrain(nn, tx, ty, opts, vx, vy); % nntrain takes validation set as last two arguments (optionally) 92 | % 93 | % [er, bad] = nntest(nn, test_x, test_y); 94 | % assert(er < 0.1, 'Too big error'); 95 | -------------------------------------------------------------------------------- /tests.wf/runalltests.m: -------------------------------------------------------------------------------- 1 | clear all; close all; clc; 2 | runtests -------------------------------------------------------------------------------- /tests.wf/test_cnn_gradients_are_numerically_correct.m: -------------------------------------------------------------------------------- 1 | function test_cnn_gradients_are_numerically_correct 2 | batch_x = rand(28,28,5); 3 | batch_y = rand(10,5); 4 | cnn.layers = { 5 | struct('type', 'i') %input layer 6 | struct('type', 'c', 'outputmaps', 2, 'kernelsize', 5) %convolution layer 7 | struct('type', 's', 'scale', 2) %sub sampling layer 8 | struct('type', 'c', 'outputmaps', 2, 'kernelsize', 5) %convolution layer 9 | struct('type', 's', 'scale', 2) %subsampling layer 10 | }; 11 | cnn = cnnsetup(cnn, batch_x, batch_y); 12 | 13 | cnn = cnnff(cnn, batch_x); 14 | cnn = cnnbp(cnn, batch_y); 15 | cnnnumgradcheck(cnn, batch_x, batch_y); -------------------------------------------------------------------------------- /tests.wf/test_example_CNN.m: -------------------------------------------------------------------------------- 1 | % function test_example_CNN 2 | load mnist_uint8; 3 | 4 | train_x = double(reshape(train_x',28,28,60000))/255; 5 | test_x = double(reshape(test_x',28,28,10000))/255; 6 | train_y = double(train_y'); 7 | test_y = double(test_y'); 8 | 9 | %% ex1 Train a 6c-2s-12c-2s Convolutional neural network 10 | %will run 1 epoch in about 200 second and get around 11% error. 11 | %With 100 epochs you'll get around 1.2% error 12 | rng(0) 13 | cnn.layers = { 14 | struct('type', 'i') %input layer 15 | struct('type', 'c', 'outputmaps', 6, 'kernelsize', 5) %convolution layer 16 | struct('type', 's', 'scale', 2) %sub sampling layer 17 | struct('type', 'c', 'outputmaps', 12, 'kernelsize', 5) %convolution layer 18 | struct('type', 's', 'scale', 2) %subsampling layer 19 | }; 20 | cnn = cnnsetup(cnn, train_x, train_y); 21 | 22 | opts.alpha = 1; 23 | opts.batchsize = 50; 24 | opts.numepochs = 1; 25 | 26 | cnn = cnntrain(cnn, train_x, train_y, opts); 27 | 28 | [er, bad] = cnntest(cnn, test_x, test_y); 29 | 30 | %plot mean squared error 31 | figure; plot(cnn.rL); 32 | 33 | assert(er<0.12, 'Too big error'); 34 | -------------------------------------------------------------------------------- /tests.wf/test_example_DBN.m: -------------------------------------------------------------------------------- 1 | % function test_example_DBN 2 | load mnist_uint8; 3 | % S1=cv.FileStorage('Training Biometrika Live.yml'); 4 | % S2=cv.FileStorage('Training Biometrika Spoof.yml'); 5 | % GRAY1=S1.allPatch; 6 | % GRAY2=S2.allPatch; 7 | % S3=cv.FileStorage('Testing Biometrika Live.yml'); 8 | % S4=cv.FileStorage('Testing Biometrika Spoof.yml'); 9 | % GRAY3=S3.allPatch; 10 | % GRAY4=S4.allPatch; 11 | 12 | % train_x = [GRAY1'; GRAY2']; 13 | % train_y = zeros(size(train_x,1),2); 14 | % train_y(1:size(GRAY1,2),1)=1; 15 | % train_y(size(GRAY1,2)+1:end,2) = 1; 16 | % test_x = [GRAY3'; GRAY4']; 17 | % test_y=train_y; 18 | clear dbn; 19 | clear nn; 20 | clear opts; 21 | clear options; 22 | 23 | train_x = [train_real_patch';train_fake_patch']; 24 | train_y = zeros(size(train_x,1),2); 25 | train_y(1:size(train_real_patch,2),1)=1; 26 | train_y(size(train_real_patch,2)+1:end,2)=1; 27 | test_x = [test_real_patch';test_fake_patch']; 28 | test_y=train_y; 29 | 30 | train_x = double(train_x) / 255; 31 | test_x = double(test_x) / 255; 32 | train_y = double(train_y); 33 | test_y = double(test_y); 34 | 35 | %% ex1 train a 100 hidden unit RBM and visualize its weights 36 | rng(0); 37 | dbn.sizes = [8]; 38 | opts.numepochs = 1; 39 | opts.batchsize = 100; 40 | opts.momentum = 0; 41 | opts.alpha = 1; 42 | dbn = dbnsetup(dbn, train_x, opts); 43 | dbn = dbntrain(dbn, train_x, opts); 44 | figure; visualize(dbn.rbm{1}.W'); % Visualize the RBM weights 45 | 46 | %% ex2 train a 100-100 hidden unit DBN and use its weights to initialize a NN 47 | rng(4314); 48 | %train dbn 49 | dbn.sizes = [8 8]; 50 | opts.numepochs = 1; 51 | opts.batchsize = 100; 52 | opts.momentum = 0; 53 | opts.alpha = 1; 54 | dbn = dbnsetup(dbn, train_x, opts); 55 | dbn = dbntrain(dbn, train_x, opts); 56 | 57 | %unfold dbn to nn 58 | nn = dbnunfoldtonn(dbn, 2); 59 | nn.activation_function = 'sigm'; 60 | 61 | %train nn 62 | opts.numepochs = 1; 63 | opts.batchsize = 100; 64 | nn = nntrain(nn, train_x, train_y, opts); 65 | [er, bad] = nntest(nn, test_x, test_y); 66 | 67 | assert(er < 0.10, 'Too big error'); 68 | -------------------------------------------------------------------------------- /tests.wf/test_example_NN.m: -------------------------------------------------------------------------------- 1 | function test_example_NN 2 | load mnist_uint8; 3 | 4 | train_x = double(train_x) / 255; 5 | test_x = double(test_x) / 255; 6 | train_y = double(train_y); 7 | test_y = double(test_y); 8 | 9 | % normalize 10 | [train_x, mu, sigma] = zscore(train_x); 11 | test_x = normalize(test_x, mu, sigma); 12 | 13 | %% ex1 vanilla neural net 14 | rng(0); 15 | nn = nnsetup([784 100 10]); 16 | opts.numepochs = 1; % Number of full sweeps through data 17 | opts.batchsize = 100; % Take a mean gradient step over this many samples 18 | [nn, L] = nntrain(nn, train_x, train_y, opts); 19 | 20 | [er, bad] = nntest(nn, test_x, test_y); 21 | 22 | assert(er < 0.08, 'Too big error'); 23 | 24 | % Make an artificial one and verify that we can predict it 25 | x = zeros(1,28,28); 26 | x(:, 14:15, 6:22) = 1; 27 | x = reshape(x,1,28^2); 28 | figure; visualize(x'); 29 | predicted = nnpredict(nn,x)-1; 30 | 31 | assert(predicted == 1); 32 | %% ex2 neural net with L2 weight decay 33 | rng(0); 34 | nn = nnsetup([784 100 10]); 35 | 36 | nn.weightPenaltyL2 = 1e-4; % L2 weight decay 37 | opts.numepochs = 1; % Number of full sweeps through data 38 | opts.batchsize = 100; % Take a mean gradient step over this many samples 39 | 40 | nn = nntrain(nn, train_x, train_y, opts); 41 | 42 | [er, bad] = nntest(nn, test_x, test_y); 43 | assert(er < 0.1, 'Too big error'); 44 | 45 | 46 | %% ex3 neural net with dropout 47 | rng(0); 48 | nn = nnsetup([784 100 10]); 49 | 50 | nn.dropoutFraction = 0.5; % Dropout fraction 51 | opts.numepochs = 1; % Number of full sweeps through data 52 | opts.batchsize = 100; % Take a mean gradient step over this many samples 53 | 54 | nn = nntrain(nn, train_x, train_y, opts); 55 | 56 | [er, bad] = nntest(nn, test_x, test_y); 57 | assert(er < 0.1, 'Too big error'); 58 | 59 | %% ex4 neural net with sigmoid activation function 60 | rng(0); 61 | nn = nnsetup([784 100 10]); 62 | 63 | nn.activation_function = 'sigm'; % Sigmoid activation function 64 | nn.learningRate = 1; % Sigm require a lower learning rate 65 | opts.numepochs = 1; % Number of full sweeps through data 66 | opts.batchsize = 100; % Take a mean gradient step over this many samples 67 | 68 | nn = nntrain(nn, train_x, train_y, opts); 69 | 70 | [er, bad] = nntest(nn, test_x, test_y); 71 | assert(er < 0.1, 'Too big error'); 72 | 73 | %% ex5 plotting functionality 74 | rng(0); 75 | nn = nnsetup([784 20 10]); 76 | opts.numepochs = 5; % Number of full sweeps through data 77 | nn.output = 'softmax'; % use softmax output 78 | opts.batchsize = 1000; % Take a mean gradient step over this many samples 79 | opts.plot = 1; % enable plotting 80 | 81 | nn = nntrain(nn, train_x, train_y, opts); 82 | 83 | [er, bad] = nntest(nn, test_x, test_y); 84 | assert(er < 0.1, 'Too big error'); 85 | 86 | %% ex6 neural net with sigmoid activation and plotting of validation and training error 87 | % split training data into training and validation data 88 | vx = train_x(1:10000,:); 89 | tx = train_x(10001:end,:); 90 | vy = train_y(1:10000,:); 91 | ty = train_y(10001:end,:); 92 | 93 | rng(0); 94 | nn = nnsetup([784 20 10]); 95 | nn.output = 'softmax'; % use softmax output 96 | opts.numepochs = 5; % Number of full sweeps through data 97 | opts.batchsize = 1000; % Take a mean gradient step over this many samples 98 | opts.plot = 1; % enable plotting 99 | nn = nntrain(nn, tx, ty, opts, vx, vy); % nntrain takes validation set as last two arguments (optionally) 100 | 101 | [er, bad] = nntest(nn, test_x, test_y); 102 | assert(er < 0.1, 'Too big error'); -------------------------------------------------------------------------------- /tests.wf/test_example_SAE.m: -------------------------------------------------------------------------------- 1 | %function test_example_SAE 2 | %load mnist_uint8; 3 | % S1=cv.FileStorage('Training Biometrika Live.yml'); 4 | % S2=cv.FileStorage('Training Biometrika Spoof.yml'); 5 | % GRAY1=S1.allPatch; 6 | % GRAY2=S2.allPatch; 7 | % S3=cv.FileStorage('Testing Biometrika Live.yml'); 8 | % S4=cv.FileStorage('Testing Biometrika Spoof.yml'); 9 | % GRAY3=S3.allPatch; 10 | % GRAY4=S4.allPatch; 11 | 12 | % GRAY1=Trans*double(GRAY1); 13 | % GRAY2=Trans*double(GRAY2); 14 | 15 | % train_x = [GRAY1'; GRAY2']; 16 | % train_y = zeros(size(train_x,1),2); 17 | % train_y(1:size(GRAY1,2),1)=1; 18 | % train_y(size(GRAY1,2)+1:end,2) = 1; 19 | % test_x = [GRAY3'; GRAY4']; 20 | % test_y=train_y; 21 | 22 | train_x=[train_real_patch train_fake_patch]'; 23 | test_x=[test_real_patch test_fake_patch]'; 24 | train_y=zeros(size(train_real_patch,2)+size(train_fake_patch,2),2); 25 | train_y(1:size(train_real_patch,2),1)=1; 26 | train_y(size(train_real_patch,2)+1:size(train_real_patch,2)*2,2)=1; 27 | test_y=train_y; 28 | 29 | train_x = double(train_x)/255; 30 | test_x = double(test_x)/255; 31 | train_y = double(train_y); 32 | test_y = double(test_y); 33 | 34 | layer1=size(train_x,2); 35 | layer2=12; 36 | 37 | %% ex1 train a 100 hidden unit SDAE and use it to initialize a FFNN 38 | % Setup and train a stacked denoising autoencoder (SDAE) 39 | rng(0); 40 | sae = saesetup([layer1 layer2]); 41 | sae.ae{1}.activation_function = 'sigm'; 42 | sae.ae{1}.learningRate = 1; 43 | sae.ae{1}.inputZeroMaskedFraction = 0.16; 44 | opts.numepochs = 1; 45 | opts.batchsize = 50; 46 | sae = saetrain(sae, train_x, opts); 47 | visualize(sae.ae{1}.W{1}(:,2:end)') 48 | 49 | % Use the SDAE to initialize a FFNN 50 | nn = nnsetup([layer1 layer2 2]); 51 | nn.activation_function = 'sigm'; 52 | nn.learningRate = 1; 53 | nn.W{1} = sae.ae{1}.W{1}; 54 | 55 | % Train the FFNN 56 | opts.numepochs = 1; 57 | opts.batchsize = 50; 58 | nn = nntrain(nn, train_x, train_y, opts); 59 | [er, bad] = nntest(nn, test_x, test_y); 60 | er 61 | %assert(er < 0.25, 'Too big error'); 62 | 63 | %% ex2 train a 100-100 hidden unit SDAE and use it to initialize a FFNN 64 | % Setup and train a stacked denoising autoencoder (SDAE) 65 | rng(0); 66 | sae = saesetup([layer1 layer2 layer2]); 67 | sae.ae{1}.activation_function = 'sigm'; 68 | sae.ae{1}.learningRate = 1; 69 | sae.ae{1}.inputZeroMaskedFraction = 0.16; 70 | 71 | sae.ae{2}.activation_function = 'sigm'; 72 | sae.ae{2}.learningRate = 1; 73 | sae.ae{2}.inputZeroMaskedFraction = 0.16; 74 | 75 | opts.numepochs = 1; 76 | opts.batchsize = 50; 77 | sae = saetrain(sae, train_x, opts); 78 | visualize(sae.ae{1}.W{1}(:,2:end)') 79 | 80 | % Use the SDAE to initialize a FFNN 81 | nn = nnsetup([layer1 layer2 layer2 2]); 82 | nn.activation_function = 'sigm'; 83 | nn.learningRate = 1; 84 | 85 | %add pretrained weights 86 | nn.W{1} = sae.ae{1}.W{1}; 87 | nn.W{2} = sae.ae{2}.W{1}; 88 | 89 | % Train the FFNN 90 | opts.numepochs = 1; 91 | opts.batchsize = 50; 92 | nn = nntrain(nn, train_x, train_y, opts); 93 | [er, bad] = nntest(nn, test_x, test_y); 94 | er 95 | assert(er < 0.25, 'Too big error'); 96 | -------------------------------------------------------------------------------- /tests.wf/test_nn_gradients_are_numerically_correct.m: -------------------------------------------------------------------------------- 1 | function test_nn_gradients_are_numerically_correct 2 | batch_x = rand(20, 5); 3 | batch_y = rand(20, 2); 4 | 5 | for output = {'sigm', 'linear', 'softmax'} 6 | y=batch_y; 7 | if(strcmp(output,'softmax')) 8 | % softmax output requires a binary output vector 9 | y=(y==repmat(max(y,[],2),1,size(y,2))); 10 | end 11 | 12 | for activation_function = {'sigm', 'tanh_opt'} 13 | for dropoutFraction = {0 rand()} 14 | nn = nnsetup([5 3 4 2]); 15 | 16 | nn.activation_function = activation_function{1}; 17 | nn.output = output{1}; 18 | nn.dropoutFraction = dropoutFraction{1}; 19 | 20 | rng(0) 21 | nn = nnff(nn, batch_x, y); 22 | nn = nnbp(nn); 23 | nnchecknumgrad(nn, batch_x, y); 24 | end 25 | end 26 | end -------------------------------------------------------------------------------- /tests/RandomDrawing.m: -------------------------------------------------------------------------------- 1 | 2 | layers = randi(30,1,10); 3 | 4 | nn = nnsetup([2 layers 3]); 5 | nn.activation_function = 'ReLU'; 6 | nn.output = 'sigm'; 7 | nn.useBatchNormalization = 1; 8 | 9 | output_h = 600; 10 | output_w = 800; 11 | 12 | [I,J] = ind2sub([output_h,output_w],(1:output_h*output_w)'); 13 | 14 | nn = nnff(nn,[I J],zeros(size(I,1),3)); 15 | 16 | output = nn.a{length(nn.a)}; 17 | output = zscore(output); 18 | output = reshape(output,[output_h,output_w,3]); 19 | 20 | imshow(uint8(output*255)); -------------------------------------------------------------------------------- /tests/runalltests.m: -------------------------------------------------------------------------------- 1 | clear all; close all; clc; 2 | 3 | addpath(genpath('.')); 4 | dirlist = dir('tests/test_*'); 5 | for i = 1:length(dirlist) 6 | name = dirlist(i).name(1:end-2); 7 | feval(name) 8 | end 9 | -------------------------------------------------------------------------------- /tests/test_example_AE.m: -------------------------------------------------------------------------------- 1 | % function test_example_NN 2 | load mnist_uint8; 3 | 4 | train_x = double(train_x) / 255; 5 | test_x = double(test_x) / 255; 6 | train_y = double(train_y); 7 | test_y = double(test_y); 8 | 9 | % normalize 10 | [train_x, mu, sigma] = zscore(train_x); 11 | test_x = normalize(test_x, mu, sigma); 12 | 13 | %% ex1 vanilla neural net 14 | rand('state',0) 15 | nn = nnsetup([784 100 784]); 16 | opts.numepochs = 10; % Number of full sweeps through data 17 | opts.batchsize = 100; % Take a mean gradient step over this many samples 18 | [nn, L] = nntrain(nn, train_x, train_x, opts); 19 | visualize(nn.W{1}(:,2:end)'); 20 | % [er, bad] = nntest(nn, test_x, test_y); 21 | 22 | assert(er < 0.08, 'Too big error'); 23 | 24 | %% ex2 neural net with L2 weight decay 25 | rand('state',0) 26 | nn = nnsetup([784 100 10]); 27 | 28 | nn.weightPenaltyL2 = 1e-4; % L2 weight decay 29 | opts.numepochs = 1; % Number of full sweeps through data 30 | opts.batchsize = 100; % Take a mean gradient step over this many samples 31 | 32 | nn = nntrain(nn, train_x, train_y, opts); 33 | 34 | [er, bad] = nntest(nn, test_x, test_y); 35 | assert(er < 0.1, 'Too big error'); 36 | 37 | 38 | %% ex3 neural net with dropout 39 | rand('state',0) 40 | nn = nnsetup([784 100 10]); 41 | 42 | nn.dropoutFraction = 0.5; % Dropout fraction 43 | opts.numepochs = 1; % Number of full sweeps through data 44 | opts.batchsize = 100; % Take a mean gradient step over this many samples 45 | 46 | nn = nntrain(nn, train_x, train_y, opts); 47 | 48 | [er, bad] = nntest(nn, test_x, test_y); 49 | assert(er < 0.1, 'Too big error'); 50 | 51 | %% ex4 neural net with sigmoid activation function 52 | rand('state',0) 53 | nn = nnsetup([784 100 10]); 54 | 55 | nn.activation_function = 'sigm'; % Sigmoid activation function 56 | nn.learningRate = 1; % Sigm require a lower learning rate 57 | opts.numepochs = 1; % Number of full sweeps through data 58 | opts.batchsize = 100; % Take a mean gradient step over this many samples 59 | 60 | nn = nntrain(nn, train_x, train_y, opts); 61 | 62 | [er, bad] = nntest(nn, test_x, test_y); 63 | assert(er < 0.1, 'Too big error'); 64 | 65 | %% ex5 plotting functionality 66 | rand('state',0) 67 | nn = nnsetup([784 20 10]); 68 | opts.numepochs = 5; % Number of full sweeps through data 69 | nn.output = 'softmax'; % use softmax output 70 | opts.batchsize = 1000; % Take a mean gradient step over this many samples 71 | opts.plot = 1; % enable plotting 72 | 73 | nn = nntrain(nn, train_x, train_y, opts); 74 | 75 | [er, bad] = nntest(nn, test_x, test_y); 76 | assert(er < 0.1, 'Too big error'); 77 | 78 | %% ex6 neural net with sigmoid activation and plotting of validation and training error 79 | % split training data into training and validation data 80 | vx = train_x(1:10000,:); 81 | tx = train_x(10001:end,:); 82 | vy = train_y(1:10000,:); 83 | ty = train_y(10001:end,:); 84 | 85 | rand('state',0) 86 | nn = nnsetup([784 20 10]); 87 | nn.output = 'softmax'; % use softmax output 88 | opts.numepochs = 5; % Number of full sweeps through data 89 | opts.batchsize = 1000; % Take a mean gradient step over this many samples 90 | opts.plot = 1; % enable plotting 91 | nn = nntrain(nn, tx, ty, opts, vx, vy); % nntrain takes validation set as last two arguments (optionally) 92 | 93 | [er, bad] = nntest(nn, test_x, test_y); 94 | assert(er < 0.1, 'Too big error'); 95 | -------------------------------------------------------------------------------- /tests/test_example_CNN.m: -------------------------------------------------------------------------------- 1 | % function test_example_CNN 2 | load mnist_uint8; 3 | global useGpu; 4 | 5 | train_x = double(reshape(train_x',28,28,60000))/255; 6 | test_x = double(reshape(test_x',28,28,10000))/255; 7 | train_y = double(train_y'); 8 | test_y = double(test_y'); 9 | 10 | %% ex1 Train a 6c-2s-12c-2s Convolutional neural network 11 | %will run 1 epoch in about 200 second and get around 11% error. 12 | %With 100 epochs you'll get around 1.2% error 13 | 14 | rand('state',0) 15 | clear cnn; 16 | cnn.layers = { 17 | struct('type', 'i') %input layer 18 | struct('type', 'c', 'outputmaps', 6, 'kernelsize', 6, 'activation', 'ReLU') %convolution layer 19 | struct('type', 's', 'scale', 2, 'method', 'm') %sub sampling layer 'm':maxpooling; 'a':average pooling; 's':stochastic pooling 20 | % struct('type', 'c', 'outputmaps', 12, 'kernelsize', 5, 'activation', 'ReLU') %convolution layer 21 | % struct('type', 's', 'scale', 2, 'method', 'm') %subsampling layer 22 | }; 23 | debug =true; 24 | if debug 25 | opts.alpha = 1; 26 | opts.batchsize = 10; 27 | opts.numepochs = 1; 28 | cnn = cnnsetup(cnn, train_x(:,:,1:10), train_y(:,1:10)); 29 | % cnn = cnntrain(cnn, train_x(:,:,1:10), train_y(:,1:10), opts); 30 | cnnnumgradcheck(cnn, train_x(:,:,1:10), train_y(:,1:10)); 31 | end; 32 | 33 | opts.alpha = 1; 34 | opts.batchsize = 50; 35 | opts.numepochs = 1; 36 | 37 | cnn = cnnsetup(cnn, train_x, train_y); 38 | cnn = cnntrain(cnn, train_x, train_y, opts); 39 | 40 | [er, bad] = cnntest(cnn, test_x, test_y); 41 | 42 | %plot mean squared error 43 | figure; plot(cnn.rL); 44 | assert(er<0.12, 'Too big error'); 45 | -------------------------------------------------------------------------------- /tests/test_example_DBN.m: -------------------------------------------------------------------------------- 1 | % function test_example_DBN 2 | % load mnist_uint8; 3 | 4 | % train_x = double(train_x) / 255; 5 | % test_x = double(test_x) / 255; 6 | % train_y = double(train_y); 7 | % test_y = double(test_y); 8 | % S = cv.FileStorage('E:\LearnOpenCV\nn\data2.yml'); 9 | % test_x = S.images(1:2000,:); 10 | % test_x = reshape(test_x,2000,30,24); 11 | % labels = S.labels; 12 | % labels(labels==0)=10; 13 | % groundTruth = full(sparse(double(labels(1:2000)), 1:2000, 1)); 14 | % test_y = groundTruth(:,1:2000)'; 15 | % 16 | % S = cv.FileStorage('E:\LearnOpenCV\nn\trans1400.yml'); 17 | % train_x = S.images(1:1400,:); 18 | % train_x = reshape(train_x,1400,30,24); 19 | % labels = S.labels; 20 | % labels(labels==0)=10; 21 | % groundTruth = full(sparse(double(labels(1:1400)), 1:1400, 1)); 22 | % train_y = groundTruth(:,1:1400)'; 23 | 24 | %% ex1 train a 100 hidden unit RBM and visualize its weights 25 | rand('state',0) 26 | dbn.sizes = [100]; 27 | opts.numepochs = 1; 28 | opts.batchsize = 100; 29 | opts.momentum = 0; 30 | opts.alpha = 1; 31 | dbn = dbnsetup(dbn, train_x, opts); 32 | dbn = dbntrain(dbn, train_x, opts); 33 | figure; visualize(dbn.rbm{1}.W'); % Visualize the RBM weights 34 | 35 | %% ex2 train a 100-100 hidden unit DBN and use its weights to initialize a NN 36 | rand('state',0) 37 | %train dbn 38 | dbn.sizes = [100 50 10 10]; 39 | opts.numepochs = 1; 40 | opts.batchsize = 100; 41 | opts.momentum = 0; 42 | opts.alpha = 1; 43 | dbn = dbnsetup(dbn, train_x, opts); 44 | dbn = dbntrain(dbn, train_x, opts); 45 | 46 | %unfold dbn to nn 47 | nn = dbnunfoldtonn(dbn, 2); 48 | nn.activation_function = 'sigm'; 49 | nn.output = 'softmax'; % use softmax output 50 | nn.dropoutFraction = 0.5; 51 | 52 | %train nn 53 | opts.numepochs = 30; 54 | opts.batchsize = 100; 55 | opts.plot = 1; 56 | nn = nntrain(nn, train_x, train_y, opts); 57 | [er, bad] = nntest(nn, test_x, test_y); 58 | 59 | assert(er < 0.10, 'Too big error'); 60 | -------------------------------------------------------------------------------- /tests/test_example_NN.m: -------------------------------------------------------------------------------- 1 | % function test_example_NN 2 | load mnist_uint8; 3 | 4 | train_x = double(train_x) / 255; 5 | test_x = double(test_x) / 255; 6 | train_y = double(train_y); 7 | test_y = double(test_y); 8 | 9 | % normalize 10 | [train_x, mu, sigma] = zscore(train_x); 11 | test_x = normalize(test_x, mu, sigma); 12 | 13 | %% ex1 vanilla neural net 14 | rand('state',0) 15 | nn = nnsetup([784 100 10]); 16 | opts.numepochs = 10; % Number of full sweeps through data 17 | opts.batchsize = 100; % Take a mean gradient step over this many samples 18 | [nn, L] = nntrain(nn, train_x, train_y, opts); 19 | 20 | [er, bad] = nntest(nn, test_x, test_y); 21 | 22 | assert(er < 0.08, 'Too big error'); 23 | 24 | %% ex2 neural net with L2 weight decay 25 | % rand('state',0) 26 | % nn = nnsetup([784 100 10]); 27 | % 28 | % nn.weightPenaltyL2 = 1e-4; % L2 weight decay 29 | % opts.numepochs = 1; % Number of full sweeps through data 30 | % opts.batchsize = 100; % Take a mean gradient step over this many samples 31 | % 32 | % nn = nntrain(nn, train_x, train_y, opts); 33 | % 34 | % [er, bad] = nntest(nn, test_x, test_y); 35 | % assert(er < 0.1, 'Too big error'); 36 | % 37 | % 38 | % %% ex3 neural net with dropout 39 | % rand('state',0) 40 | % nn = nnsetup([784 100 10]); 41 | % 42 | % nn.dropoutFraction = 0.5; % Dropout fraction 43 | % opts.numepochs = 1; % Number of full sweeps through data 44 | % opts.batchsize = 100; % Take a mean gradient step over this many samples 45 | % 46 | % nn = nntrain(nn, train_x, train_y, opts); 47 | % 48 | % [er, bad] = nntest(nn, test_x, test_y); 49 | % assert(er < 0.1, 'Too big error'); 50 | % 51 | % %% ex4 neural net with sigmoid activation function 52 | % rand('state',0) 53 | % nn = nnsetup([784 100 10]); 54 | % 55 | % nn.activation_function = 'sigm'; % Sigmoid activation function 56 | % nn.learningRate = 1; % Sigm require a lower learning rate 57 | % opts.numepochs = 1; % Number of full sweeps through data 58 | % opts.batchsize = 100; % Take a mean gradient step over this many samples 59 | % 60 | % nn = nntrain(nn, train_x, train_y, opts); 61 | % 62 | % [er, bad] = nntest(nn, test_x, test_y); 63 | % assert(er < 0.1, 'Too big error'); 64 | % 65 | % %% ex5 plotting functionality 66 | % rand('state',0) 67 | % nn = nnsetup([784 20 10]); 68 | % opts.numepochs = 5; % Number of full sweeps through data 69 | % nn.output = 'softmax'; % use softmax output 70 | % opts.batchsize = 1000; % Take a mean gradient step over this many samples 71 | % opts.plot = 1; % enable plotting 72 | % 73 | % nn = nntrain(nn, train_x, train_y, opts); 74 | % 75 | % [er, bad] = nntest(nn, test_x, test_y); 76 | % assert(er < 0.1, 'Too big error'); 77 | % 78 | % %% ex6 neural net with sigmoid activation and plotting of validation and training error 79 | % % split training data into training and validation data 80 | % vx = train_x(1:10000,:); 81 | % tx = train_x(10001:end,:); 82 | % vy = train_y(1:10000,:); 83 | % ty = train_y(10001:end,:); 84 | % 85 | % rand('state',0) 86 | % nn = nnsetup([784 20 10]); 87 | % nn.output = 'softmax'; % use softmax output 88 | % opts.numepochs = 5; % Number of full sweeps through data 89 | % opts.batchsize = 1000; % Take a mean gradient step over this many samples 90 | % opts.plot = 1; % enable plotting 91 | % nn = nntrain(nn, tx, ty, opts, vx, vy); % nntrain takes validation set as last two arguments (optionally) 92 | % 93 | % [er, bad] = nntest(nn, test_x, test_y); 94 | % assert(er < 0.1, 'Too big error'); 95 | -------------------------------------------------------------------------------- /tests/test_example_NN2.m: -------------------------------------------------------------------------------- 1 | % function test_example_NN 2 | load mnist_uint8; 3 | train_x = double(train_x) / 255; 4 | test_x = double(test_x) / 255; 5 | train_y = double(train_y); 6 | test_y = double(test_y); 7 | % 8 | % % normalize 9 | % [train_x, mu, sigma] = zscore(train_x); 10 | % test_x = normalize(test_x, mu, sigma); 11 | close all; 12 | % trainNum = 1000; 13 | % idx = randperm(2000); 14 | % images = double(images); 15 | % images = bsxfun(@rdivide, images, sum(images,2)); 16 | % train_x = images(idx(1:trainNum),:); 17 | % labels(labels==0)=10; 18 | % groundTruth = full(sparse(labels(1:2000), 1:2000, 1)); 19 | % train_y = groundTruth(:,idx(1:trainNum))'; 20 | % test_x = images(idx(trainNum+1:2000),:); 21 | % test_y = groundTruth(:,idx(trainNum+1:2000))'; 22 | 23 | % S = cv.FileStorage('E:\LearnOpenCV\nn\data2.yml'); 24 | % test_x = S.images(1:2000,:); 25 | % labels = S.labels; 26 | % labels(labels==0)=10; 27 | % groundTruth = full(sparse(double(labels(1:2000)), 1:2000, 1)); 28 | % test_y = groundTruth(:,1:2000)'; 29 | % 30 | % S = cv.FileStorage('E:\LearnOpenCV\nn\data1.yml'); 31 | % num = floor(size(S.images,1)/100)*100; 32 | % train_x = S.images(1:num,:); 33 | % labels = S.labels; 34 | % labels(labels==0)=10; 35 | % groundTruth = full(sparse(double(labels(1:num)), 1:num, 1)); 36 | % train_y = groundTruth(:,1:num)'; 37 | 38 | % train_x = double(train_x) / 255; 39 | % test_x = double(test_x) / 255; 40 | % train_x = ta; 41 | % train_y = [group_train==-1 group_train==1]; 42 | % test_x = sa; 43 | % test_y = [group_sample==-1 group_sample==1]; 44 | % train_x = [train_x;test_x]; 45 | % train_y = [train_y;test_y]; 46 | 47 | % normalize 48 | [train_x, mu, sigma] = zscore(train_x); 49 | test_x = normalize(test_x, mu, sigma); 50 | % all_x = normalize(images, mu, sigma); 51 | % all_y = full(sparse(labels, 1:length(labels), 1))'; 52 | clear nn; 53 | 54 | % % ex1 vanilla neural net 55 | % rand('state',0) 56 | % nn = nnsetup([720 100 10]); 57 | % opts.numepochs = 1; % Number of full sweeps through data 58 | % opts.batchsize = 100; % Take a mean gradient step over this many samples 59 | % [nn, L] = nntrain(nn, train_x, train_y, opts); 60 | % 61 | % [er, bad] = nntest(nn, test_x, test_y); 62 | % 63 | % disp(['rate: ', num2str((1-er)*100) '%']); 64 | % % assert(er < 0.08, 'Too big error'); 65 | % 66 | % %% ex2 neural net with L2 weight decay 67 | % rand('state',0) 68 | % nn = nnsetup([720 100 10]); 69 | % 70 | % nn.weightPenaltyL2 = 1e-4; % L2 weight decay 71 | % opts.numepochs = 1; % Number of full sweeps through data 72 | % opts.batchsize = 100; % Take a mean gradient step over this many samples 73 | % 74 | % nn = nntrain(nn, train_x, train_y, opts); 75 | % 76 | % [er, bad] = nntest(nn, test_x, test_y); 77 | % disp(['rate: ', num2str((1-er)*100) '%']); 78 | % % assert(er < 0.1, 'Too big error'); 79 | % 80 | % 81 | % %% ex3 neural net with dropout 82 | % rand('state',0) 83 | % nn = nnsetup([720 100 10]); 84 | % 85 | % nn.dropoutFraction = 0.5; % Dropout fraction 86 | % opts.numepochs = 1; % Number of full sweeps through data 87 | % opts.batchsize = 100; % Take a mean gradient step over this many samples 88 | % 89 | % nn = nntrain(nn, train_x, train_y, opts); 90 | % 91 | % [er, bad] = nntest(nn, test_x, test_y); 92 | % disp(['rate: ', num2str((1-er)*100) '%']); 93 | % % assert(er < 0.1, 'Too big error'); 94 | % 95 | % %% ex4 neural net with sigmoid activation function 96 | % rand('state',0) 97 | % nn = nnsetup([720 100 10]); 98 | % 99 | % nn.activation_function = 'sigm'; % Sigmoid activation function 100 | % nn.learningRate = 1; % Sigm require a lower learning rate 101 | % opts.numepochs = 1; % Number of full sweeps through data 102 | % opts.batchsize = 100; % Take a mean gradient step over this many samples 103 | % 104 | % nn = nntrain(nn, train_x, train_y, opts); 105 | % 106 | % [er, bad] = nntest(nn, test_x, test_y); 107 | % disp(['rate: ', num2str((1-er)*100) '%']); 108 | % % assert(er < 0.1, 'Too big error'); 109 | % S = struct('W1', nn.W{1}', 'W2', nn.W{2}','mu', mu, 'sigma', sigma); 110 | % cv.FileStorage('weight.yml',S); 111 | 112 | %% ex5 plotting functionality 113 | 114 | % rand('state',0) 115 | nn = nnsetup([784 500 300 10]);%400 400 400 200 200 200 200 200 1000 1000 800 400 200 200 200 200 116 | 117 | % rand('state',0) 118 | % ae = nnsetup([2900 1000 2900]); 119 | % opts.numepochs = 100; % Number of full sweeps through data 120 | % ae.activation_function = 'linear'; 121 | % ae.output = 'linear'; % use softmax output 122 | % opts.batchsize = 100; % Take a mean gradient step over this many samples 123 | % opts.plot = 1; % enable plotting 124 | % nn.nonSparsityPenalty = 0.1; 125 | % ae.dropoutFraction = 0.5; 126 | % nn.inputZeroMaskedFraction = 0.6; 127 | 128 | % ae = nntrain(ae, train_x, train_x, opts); 129 | 130 | % aeW1 = ae.W{1}; 131 | 132 | % rand('state',0) 133 | % ae = nnsetup([8192 600 50 8192]); 134 | % ae.W{1} = aeW1; 135 | % opts.numepochs = 3; % Number of full sweeps through data 136 | % ae.activation_function = 'sigm'; 137 | % ae.output = 'sigm'; % use softmax output 138 | % opts.batchsize = 100; % Take a mean gradient step over this many samples 139 | % opts.plot = 1; % enable plotting 140 | % nn.nonSparsityPenalty = 0.1; 141 | % % ae.dropoutFraction = 0.5; 142 | % % nn.inputZeroMaskedFraction = 0.6; 143 | 144 | % ae = nntrain(ae, train_x, train_x, opts); 145 | 146 | % nn.W{1} = ae.W{1}; 147 | % nn.W{2} = ae.W{2}; 148 | % nn.W{1} = 0; 149 | % nn.W{1}(1)=0.55;nn.W{1}(311)=0.5;nn.W{1}(312)=-0.5; 150 | opts.numepochs = 20; % Number of full sweeps through data 151 | nn.activation_function = 'ReLU'; 152 | nn.output = 'hinge'; % use softmax output 153 | nn.hinge_norm = 1; 154 | opts.batchsize = 200; % Take a mean gradient step over this many samples 155 | opts.plot = 1; % enable plotting 156 | % nn.useBatchNormalization = 0; % 157 | % nn.momentum = 0; 158 | nn.learningRate = 0.0004; 159 | % nn.weightPenaltyL2 = 0.0005; 160 | % nn.nonSparsityPenalty = 0.01; 161 | % nn.dropoutFraction = 0.5; 162 | % nn.inputZeroMaskedFraction = 0.2; 163 | 164 | nn = nntrain(nn, train_x, train_y, opts,test_x,test_y); 165 | 166 | % predict_x = [Chuqinlv{2}-Chuqinlv{3} ChuQinTime{2}-ChuQinTime{3}]; 167 | nn.testing = 1; 168 | nn=nnff(nn,test_x,rand(91,1)); 169 | predict_y = nn.a{end}; 170 | predict_y = (predict_y-0.55)*2; 171 | vv = predict_y - mean(predict_y); 172 | predict_y = rank(:,1)/size(rank,1)*0.9+0.05 - predict_y; 173 | [~,idx] = sort(predict_y); 174 | pry = zeros(size(predict_y,1),1); 175 | for i=1:size(predict_y,1) 176 | pry(idx(i))=i; 177 | end; 178 | rate = 1 - 6*norm(pry-rank(:,2)).^2/size(predict_y,1)/(size(predict_y,1)^2-1); 179 | disp(rate); 180 | 181 | [er, bad] = nntest(nn, test_x, test_y); 182 | disp(['rate: ', num2str((1-er)*100) '%']); 183 | assert(er < 0.1, 'Too big error'); 184 | % S = struct('W1', nn.W{1}', 'W2', nn.W{2}','mu', mu, 'sigma', sigma); 185 | % cv.FileStorage('weight.yml',S); 186 | % [er, bad] = nntest(nn, all_x, all_y); 187 | %% ex6 neural net with sigmoid activation and plotting of validation and training error 188 | % split training data into training and validation data 189 | % vx = train_x(1:10000,:); 190 | % tx = train_x(10001:end,:); 191 | % vy = train_y(1:10000,:); 192 | % ty = train_y(10001:end,:); 193 | % 194 | % rand('state',0) 195 | % nn = nnsetup([784 20 10]); 196 | % nn.output = 'softmax'; % use softmax output 197 | % opts.numepochs = 5; % Number of full sweeps through data 198 | % opts.batchsize = 1000; % Take a mean gradient step over this many samples 199 | % opts.plot = 1; % enable plotting 200 | % nn = nntrain(nn, tx, ty, opts, vx, vy); % nntrain takes validation set as last two arguments (optionally) 201 | % 202 | % [er, bad] = nntest(nn, test_x, test_y); 203 | % assert(er < 0.1, 'Too big error'); 204 | -------------------------------------------------------------------------------- /tests/test_example_SAE.m: -------------------------------------------------------------------------------- 1 | % function test_example_SAE 2 | % load mnist_uint8; 3 | 4 | S = cv.FileStorage('E:\LearnOpenCV\nn\data2.yml'); 5 | num = floor(size(S.images,1)/100)*100; 6 | test_x = S.images(1:num,:); 7 | test_x = bsxfun(@rdivide,test_x,sum(test_x,2)); 8 | % test_x = reshape(test_x,num,30,24); 9 | labels = S.labels; 10 | labels(labels==0)=10; 11 | groundTruth = full(sparse(double(labels(1:num)), 1:num, 1)); 12 | test_y = groundTruth(:,1:num)'; 13 | 14 | S = cv.FileStorage('E:\LearnOpenCV\nn\data1.yml'); 15 | num = floor(size(S.images,1)/100)*100; 16 | train_x = S.images(1:num,:); 17 | train_x = bsxfun(@rdivide,train_x,sum(train_x,2)); 18 | % train_x = reshape(train_x,num,30,24); 19 | labels = S.labels; 20 | labels(labels==0)=10; 21 | groundTruth = full(sparse(double(labels(1:num)), 1:num, 1)); 22 | train_y = groundTruth(:,1:num)'; 23 | 24 | % train_x = double(train_x)/255; 25 | % test_x = double(test_x)/255; 26 | % train_y = double(train_y); 27 | % test_y = double(test_y); 28 | 29 | %% ex1 train a 100 hidden unit SDAE and use it to initialize a FFNN 30 | % Setup and train a stacked denoising autoencoder (SDAE) 31 | rand('state',0) 32 | sae = saesetup([784 100]); 33 | sae.ae{1}.activation_function = 'ReLU'; 34 | sae.ae{1}.output = 'ReLU'; 35 | % sae.ae{1}.inputZeroMaskedFraction = 0.5; 36 | % sae.ae{1}.nonSparsityPenalty = 0.5; 37 | opts.numepochs = 1; 38 | opts.batchsize = 100; 39 | sae = saetrain(sae, train_x, opts); 40 | visualize(sae.ae{1}.W{1}(:,2:end)') 41 | sum(sae.ae{1}.a{1}(:,2:end)>0,2) 42 | 43 | % Use the SDAE to initialize a FFNN 44 | nn = nnsetup([720 100 10]); 45 | nn.activation_function = 'sigm'; 46 | nn.learningRate = 1; 47 | nn.W{1} = sae.ae{1}.W{1}; 48 | 49 | % Train the FFNN 50 | opts.numepochs = 50; 51 | opts.batchsize = 100; 52 | nn = nntrain(nn, train_x, train_y, opts); 53 | [er, bad] = nntest(nn, test_x, test_y); 54 | assert(er < 0.16, 'Too big error'); 55 | -------------------------------------------------------------------------------- /tests/test_nn_gradients_are_numerically_correct.m: -------------------------------------------------------------------------------- 1 | function test_nn_gradients_are_numerically_correct 2 | batch_x = rand(20, 5); 3 | batch_y = rand(20, 2); 4 | 5 | for output = {'sigm', 'linear', 'softmax'} 6 | y=batch_y; 7 | if(strcmp(output,'softmax')) 8 | % softmax output requires a binary output vector 9 | y=(y==repmat(max(y,[],2),1,size(y,2))); 10 | end 11 | 12 | for activation_function = {'sigm', 'tanh_opt'} 13 | for dropoutFraction = {0 rand()} 14 | nn = nnsetup([5 3 4 2]); 15 | 16 | nn.activation_function = activation_function{1}; 17 | nn.output = output{1}; 18 | nn.dropoutFraction = dropoutFraction{1}; 19 | 20 | rand('state',0) 21 | nn = nnff(nn, batch_x, y); 22 | nn = nnbp(nn); 23 | nnchecknumgrad(nn, batch_x, y); 24 | end 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /util/allcomb.m: -------------------------------------------------------------------------------- 1 | function A = allcomb(varargin) 2 | % ALLCOMB - All combinations 3 | % B = ALLCOMB(A1,A2,A3,...,AN) returns all combinations of the elements 4 | % in A1, A2, ..., and AN. B is P-by-N matrix is which P is the product 5 | % of the number of elements of the N inputs. 6 | % Empty inputs yields an empty matrix B of size 0-by-N. Note that 7 | % previous versions (1.x) simply ignored empty inputs. 8 | % 9 | % Example: 10 | % allcomb([1 3 5],[-3 8],[0 1]) ; 11 | % 1 -3 0 12 | % 1 -3 1 13 | % 1 8 0 14 | % ... 15 | % 5 -3 1 16 | % 5 8 0 17 | % 5 8 1 18 | % 19 | % ALLCOMB(A1,..AN,'matlab') causes the first column to change fastest. 20 | % This is more consistent with matlab indexing. Example: 21 | % allcomb(1:2,3:4,5:6,'matlab') %-> 22 | % 1 3 5 23 | % 2 3 5 24 | % 1 4 5 25 | % ... 26 | % 2 4 6 27 | % 28 | % This functionality is also known as the cartesian product. 29 | % 30 | % See also NCHOOSEK, PERMS, 31 | % and COMBN (Matlab Central FEX) 32 | 33 | % for Matlab R13+ 34 | % version 2.1 (feb 2011) 35 | % (c) Jos van der Geest 36 | % email: jos@jasen.nl 37 | 38 | % History 39 | % 1.1 (feb 2006), removed minor bug when entering empty cell arrays; 40 | % added option to let the first input run fastest (suggestion by JD) 41 | % 1.2 (jan 2010), using ii as an index on the left-hand for the multiple 42 | % output by NDGRID. Thanks to Jan Simon, for showing this little trick 43 | % 2.0 (dec 2010). Bruno Luong convinced me that an empty input should 44 | % return an empty output. 45 | % 2.1 (feb 2011). A cell as input argument caused the check on the last 46 | % argument (specifying the order) to crash. 47 | 48 | error(nargchk(1,Inf,nargin)) ; 49 | 50 | % check for empty inputs 51 | q = ~cellfun('isempty',varargin) ; 52 | if any(~q), 53 | warning('ALLCOMB:EmptyInput','Empty inputs result in an empty output.') ; 54 | A = zeros(0,nargin) ; 55 | else 56 | 57 | ni = sum(q) ; 58 | 59 | argn = varargin{end} ; 60 | ischar(argn) 61 | if ischar(argn) && (strcmpi(argn,'matlab') || strcmpi(argn,'john')), 62 | % based on a suggestion by JD on the FEX 63 | ni = ni-1 ; 64 | ii = 1:ni ; 65 | q(end) = 0 ; 66 | else 67 | % enter arguments backwards, so last one (AN) is changing fastest 68 | ii = ni:-1:1 ; 69 | end 70 | 71 | if ni==0, 72 | A = [] ; 73 | else 74 | args = varargin(q) ; 75 | if ~all(cellfun('isclass',args,'double')), 76 | error('All arguments should be arrays of doubles') ; 77 | end 78 | if ni==1, 79 | A = args{1}(:) ; 80 | else 81 | % flip using ii if last column is changing fastest 82 | [A{ii}] = ndgrid(args{ii}) ; 83 | % concatenate 84 | A = reshape(cat(ni+1,A{:}),[],ni) ; 85 | end 86 | end 87 | end 88 | -------------------------------------------------------------------------------- /util/expand.m: -------------------------------------------------------------------------------- 1 | function B = expand(A, S) 2 | %EXPAND Replicate and tile each element of an array, similar to repmat. 3 | % EXPAND(A,SZ), for array A and vector SZ replicates each element of A by 4 | % SZ. The results are tiled into an array in the same order as the 5 | % elements of A, so that the result is size: size(A).*SZ. Therefore the 6 | % number of elements of SZ must equal the number of dimensions of A, or in 7 | % MATLAB syntax: length(size(A))==length(SZ) must be true. 8 | % The result will have the same number of dimensions as does A. 9 | % There is no restriction on the number of dimensions for input A. 10 | % 11 | % Examples: 12 | % 13 | % A = [1 2; 3 4]; % 2x2 14 | % SZ = [6 5]; 15 | % B = expand(A,[6 5]) % Creates a 12x10 array. 16 | % 17 | % The following demonstrates equivalence of EXPAND and expansion acheived 18 | % through indexing the individual elements of the array: 19 | % 20 | % A = 1; B = 2; C = 3; D = 4; % Elements of the array to be expanded. 21 | % Mat = [A B;C D]; % The array to expand. 22 | % SZ = [2 3]; % The expansion vector. 23 | % ONES = ones(SZ); % The index array. 24 | % ExpMat1 = [A(ONES),B(ONES);C(ONES),D(ONES)]; % Element expansion. 25 | % ExpMat2 = expand(Mat,SZ); % Calling EXPAND. 26 | % isequal(ExpMat1,ExpMat2) % Yes 27 | % 28 | % 29 | % See also, repmat, meshgrid, ones, zeros, kron 30 | % 31 | % Author: Matt Fig 32 | % Date: 6/20/2009 33 | % Contact: popkenai@yahoo.com 34 | 35 | if nargin < 2 36 | error('Size vector must be provided. See help.'); 37 | end 38 | 39 | SA = size(A); % Get the size (and number of dimensions) of input. 40 | 41 | if length(SA) ~= length(S) 42 | error('Length of size vector must equal ndims(A). See help.') 43 | elseif any(S ~= floor(S)) 44 | error('The size vector must contain integers only. See help.') 45 | end 46 | 47 | T = cell(length(SA), 1); 48 | for ii = length(SA) : -1 : 1 49 | H = zeros(SA(ii) * S(ii), 1); % One index vector into A for each dim. 50 | H(1 : S(ii) : SA(ii) * S(ii)) = 1; % Put ones in correct places. 51 | T{ii} = cumsum(H); % Cumsumming creates the correct order. 52 | end 53 | 54 | B = A(T{:}); % Feed the indices into A. -------------------------------------------------------------------------------- /util/flicker.m: -------------------------------------------------------------------------------- 1 | function flicker(X,fps) 2 | figure; 3 | colormap gray; 4 | axis image; 5 | while 1 6 | for i=1:size(X,1); 7 | imagesc(squeeze(X(i,:,:))); drawnow; 8 | pause(1/fps); 9 | end 10 | end 11 | end -------------------------------------------------------------------------------- /util/flipall.m: -------------------------------------------------------------------------------- 1 | function X=flipall(X) 2 | for i=1:ndims(X) 3 | X = flipdim(X,i); 4 | end 5 | end -------------------------------------------------------------------------------- /util/fliplrf.m: -------------------------------------------------------------------------------- 1 | function y = fliplrf(x) 2 | %FLIPLR Flip matrix in left/right direction. 3 | % FLIPLR(X) returns X with row preserved and columns flipped 4 | % in the left/right direction. 5 | % 6 | % X = 1 2 3 becomes 3 2 1 7 | % 4 5 6 6 5 4 8 | % 9 | % Class support for input X: 10 | % float: double, single 11 | % 12 | % See also FLIPUD, ROT90, FLIPDIM. 13 | 14 | % Copyright 1984-2010 The MathWorks, Inc. 15 | % $Revision: 5.9.4.4 $ $Date: 2010/02/25 08:08:47 $ 16 | 17 | % if ~ismatrix(x) 18 | % error('MATLAB:fliplr:SizeX', 'X must be a 2-D matrix.'); 19 | % end 20 | y = x(:,end:-1:1); 21 | -------------------------------------------------------------------------------- /util/flipudf.m: -------------------------------------------------------------------------------- 1 | function y = flipudf(x) 2 | %FLIPUD Flip matrix in up/down direction. 3 | % FLIPUD(X) returns X with columns preserved and rows flipped 4 | % in the up/down direction. For example, 5 | % 6 | % X = 1 4 becomes 3 6 7 | % 2 5 2 5 8 | % 3 6 1 4 9 | % 10 | % Class support for input X: 11 | % float: double, single 12 | % 13 | % See also FLIPLR, ROT90, FLIPDIM. 14 | 15 | % Copyright 1984-2010 The MathWorks, Inc. 16 | % $Revision: 5.9.4.4 $ $Date: 2010/02/25 08:08:49 $ 17 | 18 | % if ~ismatrix(x) 19 | % error('MATLAB:flipud:SizeX', 'X must be a 2-D matrix.'); 20 | % end 21 | y = x(end:-1:1,:); 22 | -------------------------------------------------------------------------------- /util/im2patches.m: -------------------------------------------------------------------------------- 1 | function patches = im2patches(im,m,n) 2 | assert(rem(size(im,1),m)==0) 3 | assert(rem(size(im,2),n)==0) 4 | 5 | patches = []; 6 | for i=1:m:size(im,1) 7 | for u=1:n:size(im,2) 8 | patch = im(i:i+n-1,u:u+m-1); 9 | patches = [patches patch(:)]; 10 | end 11 | end 12 | patches = patches'; 13 | end -------------------------------------------------------------------------------- /util/isOctave.m: -------------------------------------------------------------------------------- 1 | %detects if we're running Octave 2 | function result = isOctave() 3 | result = exist('OCTAVE_VERSION') ~= 0; 4 | end -------------------------------------------------------------------------------- /util/makeLMfilters.m: -------------------------------------------------------------------------------- 1 | function F=makeLMfilters 2 | % Returns the LML filter bank of size 49x49x48 in F. To convolve an 3 | % image I with the filter bank you can either use the matlab function 4 | % conv2, i.e. responses(:,:,i)=conv2(I,F(:,:,i),'valid'), or use the 5 | % Fourier transform. 6 | 7 | SUP=49; % Support of the largest filter (must be odd) 8 | SCALEX=sqrt(2).^[1:3]; % Sigma_{x} for the oriented filters 9 | NORIENT=6; % Number of orientations 10 | 11 | NROTINV=12; 12 | NBAR=length(SCALEX)*NORIENT; 13 | NEDGE=length(SCALEX)*NORIENT; 14 | NF=NBAR+NEDGE+NROTINV; 15 | F=zeros(SUP,SUP,NF); 16 | hsup=(SUP-1)/2; 17 | [x,y]=meshgrid([-hsup:hsup],[hsup:-1:-hsup]); 18 | orgpts=[x(:) y(:)]'; 19 | 20 | count=1; 21 | for scale=1:length(SCALEX), 22 | for orient=0:NORIENT-1, 23 | angle=pi*orient/NORIENT; % Not 2pi as filters have symmetry 24 | c=cos(angle);s=sin(angle); 25 | rotpts=[c -s;s c]*orgpts; 26 | F(:,:,count)=makefilter(SCALEX(scale),0,1,rotpts,SUP); 27 | F(:,:,count+NEDGE)=makefilter(SCALEX(scale),0,2,rotpts,SUP); 28 | count=count+1; 29 | end; 30 | end; 31 | 32 | count=NBAR+NEDGE+1; 33 | SCALES=sqrt(2).^[1:4]; 34 | for i=1:length(SCALES), 35 | F(:,:,count)=normalise(fspecial('gaussian',SUP,SCALES(i))); 36 | F(:,:,count+1)=normalise(fspecial('log',SUP,SCALES(i))); 37 | F(:,:,count+2)=normalise(fspecial('log',SUP,3*SCALES(i))); 38 | count=count+3; 39 | end; 40 | return 41 | 42 | function f=makefilter(scale,phasex,phasey,pts,sup) 43 | gx=gauss1d(3*scale,0,pts(1,:),phasex); 44 | gy=gauss1d(scale,0,pts(2,:),phasey); 45 | f=normalise(reshape(gx.*gy,sup,sup)); 46 | return 47 | 48 | function g=gauss1d(sigma,mean,x,ord) 49 | % Function to compute gaussian derivatives of order 0 <= ord < 3 50 | % evaluated at x. 51 | 52 | x=x-mean;num=x.*x; 53 | variance=sigma^2; 54 | denom=2*variance; 55 | g=exp(-num/denom)/(pi*denom)^0.5; 56 | switch ord, 57 | case 1, g=-g.*(x/variance); 58 | case 2, g=g.*((num-variance)/(variance^2)); 59 | end; 60 | return 61 | 62 | function f=normalise(f), f=f-mean(f(:)); f=f/sum(abs(f(:))); return -------------------------------------------------------------------------------- /util/myOctaveVersion.m: -------------------------------------------------------------------------------- 1 | % return OCTAVE_VERSION or 'undefined' as a string 2 | function result = myOctaveVersion() 3 | if isOctave() 4 | result = OCTAVE_VERSION; 5 | else 6 | result = 'undefined'; 7 | end 8 | -------------------------------------------------------------------------------- /util/normalize.m: -------------------------------------------------------------------------------- 1 | function x = normalize(x, mu, sigma) 2 | x=bsxfun(@minus,x,mu); 3 | x=bsxfun(@rdivide,x,sigma); 4 | end 5 | -------------------------------------------------------------------------------- /util/patches2im.m: -------------------------------------------------------------------------------- 1 | function im = patches2im(patches,n,m) 2 | k=1; 3 | im = zeros(n,m); 4 | for i=1:10:800 5 | for u=1:10:1140 6 | patch = patches(:,k); 7 | im(u:u+9,i:i+9) = reshape(patch,10,10); 8 | k = k+1; 9 | end 10 | end 11 | end -------------------------------------------------------------------------------- /util/randcorr.m: -------------------------------------------------------------------------------- 1 | function x=randcorr(n,R) 2 | % RANDCORR Generates corremlated random variables 3 | % Generates n vector valued variates with uniform marginals and correlation 4 | % matrix R. 5 | % Returns an nxk matrix, where k is the order of R. 6 | k=size(R,1); 7 | R=2*sin((pi/6)*R); 8 | x=normcdf(randn(n,k)*chol(R)); -------------------------------------------------------------------------------- /util/randp.m: -------------------------------------------------------------------------------- 1 | function X = randp(P,varargin) 2 | % RANDP - pick random values with relative probability 3 | % 4 | % R = RANDP(PROB,..) returns integers in the range from 1 to 5 | % NUMEL(PROB) with a relative probability, so that the value X is 6 | % present approximately (PROB(X)./sum(PROB)) times in the matrix R. 7 | % 8 | % All values of PROB should be equal to or larger than 0. 9 | % 10 | % RANDP(PROB,N) is an N-by-N matrix, RANDP(PROB,M,N) and 11 | % RANDP(PROB,[M,N]) are M-by-N matrices. RANDP(PROB, M1,M2,M3,...) or 12 | % RANDP(PROB,[M1,M2,M3,...]) generate random arrays. 13 | % RANDP(PROB,SIZE(A)) is the same size as A. 14 | % 15 | % Example: 16 | % R = randp([1 3 2],1,10000) 17 | % % return a row vector with 10000 values with about 16650% 2 18 | % histc(R,1:3) ./ numel(R) 19 | % 20 | % R = randp([1 1 0 0 1],10,1) 21 | % % 10 samples evenly drawn from [1 2 5] 22 | % 23 | % 24 | % Also see RAND, RANDPERM 25 | % RANDPERMBREAK, RANDINTERVAL, RANDSWAP (MatLab File Exchange) 26 | 27 | % Created for Matlab R13+ 28 | % version 2.0 (feb 2009) 29 | % (c) Jos van der Geest 30 | % email: jos@jasen.nl 31 | % 32 | % File history: 33 | % 1.0 (nov 2005) - created 34 | % 1.1 (nov 2005) - modified slightly to check input arguments to RAND first 35 | % 1.2 (aug 2006) - fixed bug when called with scalar argument P 36 | % 2.0 (feb 2009) - use HISTC for creating the integers (faster and simplier than 37 | % previous algorithm) 38 | 39 | error(nargchk(2,Inf,nargin)) ; 40 | 41 | try 42 | X = rand(varargin{:}) ; 43 | catch 44 | E = lasterror ; 45 | E.message = strrep(E.message,'rand','randp') ; 46 | rethrow(E) ; 47 | end 48 | 49 | P = P(:) ; 50 | 51 | if any(P<0), 52 | error('All probabilities should be 0 or larger.') ; 53 | end 54 | 55 | if isempty(P) || sum(P)==0 56 | warning([mfilename ':ZeroProbabilities'],'All zero probabilities') ; 57 | X(:) = 0 ; 58 | else 59 | [junk,X] = histc(X,[0 ; cumsum(P(:))] ./ sum(P)) ; 60 | end 61 | 62 | % Method used before version 2 63 | % X = rand(varargin{:}) ; 64 | % sz = size(X) ; 65 | % P = reshape(P,1,[]) ; % row vector 66 | % P = cumsum(P) ./ sum(P) ; 67 | % X = repmat(X(:),1,numel(P)) < repmat(P,numel(X),1) ; 68 | % X = numel(P) - sum(X,2) + 1 ; 69 | % X = reshape(X,sz) ; 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | -------------------------------------------------------------------------------- /util/rnd.m: -------------------------------------------------------------------------------- 1 | function y = rnd(x) 2 | y = double(x>rand()); 3 | end -------------------------------------------------------------------------------- /util/sigm.m: -------------------------------------------------------------------------------- 1 | function X = sigm(P) 2 | X = 1./(1+exp(-P)); 3 | end -------------------------------------------------------------------------------- /util/sigmrnd.m: -------------------------------------------------------------------------------- 1 | function X = sigmrnd(P) 2 | % X = double(1./(1+exp(-P)))+1*randn(size(P)); 3 | X = double(1./(1+exp(-P)) > rand(size(P))); 4 | end -------------------------------------------------------------------------------- /util/softmax.m: -------------------------------------------------------------------------------- 1 | function mu = softmax(eta) 2 | % Softmax function 3 | % mu(i,c) = exp(eta(i,c))/sum_c' exp(eta(i,c')) 4 | 5 | % This file is from matlabtools.googlecode.com 6 | c = 3; 7 | 8 | tmp = exp(c*eta); 9 | denom = sum(tmp, 2); 10 | mu = bsxfun(@rdivide, tmp, denom); 11 | 12 | end -------------------------------------------------------------------------------- /util/tanh_opt.m: -------------------------------------------------------------------------------- 1 | function f=tanh_opt(A) 2 | f=1.7159*tanh(2/3.*A); 3 | end -------------------------------------------------------------------------------- /util/visualize.m: -------------------------------------------------------------------------------- 1 | function r=visualize(X, mm, s1, s2) 2 | %FROM RBMLIB http://code.google.com/p/matrbm/ 3 | %Visualize weights X. If the function is called as a void method, 4 | %it does the plotting. But if the function is assigned to a variable 5 | %outside of this code, the formed image is returned instead. 6 | if ~exist('mm','var') 7 | mm = [min(X(:)) max(X(:))]; 8 | end 9 | if ~exist('s1','var') 10 | s1 = 0; 11 | end 12 | if ~exist('s2','var') 13 | s2 = 0; 14 | end 15 | 16 | [D,N]= size(X); 17 | s=sqrt(D); 18 | if s==floor(s) || (s1 ~=0 && s2 ~=0) 19 | if (s1 ==0 || s2 ==0) 20 | s1 = s; s2 = s; 21 | end 22 | %its a square, so data is probably an image 23 | num=ceil(sqrt(N)); 24 | a=mm(2)*ones(num*s2+num-1,num*s1+num-1); 25 | x=0; 26 | y=0; 27 | for i=1:N 28 | im = reshape(X(:,i),s1,s2)'; 29 | a(x*s2+1+x : x*s2+s2+x, y*s1+1+y : y*s1+s1+y)=im; 30 | x=x+1; 31 | if(x>=num) 32 | x=0; 33 | y=y+1; 34 | end 35 | end 36 | d=true; 37 | else 38 | %there is not much we can do 39 | a=X; 40 | end 41 | 42 | %return the image, or plot the image 43 | if nargout==1 44 | r=a; 45 | else 46 | 47 | imagesc(a, [mm(1) mm(2)]); 48 | axis equal 49 | colormap gray 50 | 51 | end 52 | -------------------------------------------------------------------------------- /util/whiten.m: -------------------------------------------------------------------------------- 1 | function X = whiten(X, fudgefactor) 2 | C = cov(X); 3 | M = mean(X); 4 | [V,D] = eig(C); 5 | P = V * diag(sqrt(1./(diag(D) + fudgefactor))) * V'; 6 | X = bsxfun(@minus, X, M) * P; 7 | end -------------------------------------------------------------------------------- /util/zscore.m: -------------------------------------------------------------------------------- 1 | function [x, mu, sigma] = zscore(x) 2 | mu=mean(x); 3 | sigma=max(std(x),eps); 4 | x=bsxfun(@minus,x,mu); 5 | x=bsxfun(@rdivide,x,sigma); 6 | end 7 | --------------------------------------------------------------------------------