├── CalcErrorRate.m ├── CalcRmse.m ├── GetDroppedDBN.m ├── GetOnInd.m ├── README.md ├── SetLinearMapping.m ├── code1.m ├── evaMNIST.m ├── h2v.m ├── harvest_moon.cvv ├── iris_code1.m ├── linearMapping.m ├── mnistread.m ├── pretrainDBN.m ├── pretrainRBM.m ├── randDBN.m ├── randRBM.m ├── sigmoid.m ├── testDNN.m ├── testMNIST.m ├── trainDBN.m ├── trainMNIST.m ├── v2h.m └── v2hall.m /CalcErrorRate.m: -------------------------------------------------------------------------------- 1 | % CalcErrorRate: calculate error rate 2 | % 3 | % ErrorRate = CalcErrorRate( dbn, IN, OUT ) 4 | % 5 | % 6 | %Output parameters: 7 | % ErrorRate: error rate 8 | % 9 | % 10 | %Input parameters: 11 | % dbn: network 12 | % IN: input data, where # of row is # of data and # of col is # of input features 13 | % OUT: output data, where # of row is # of data and # of col is # of output labels 14 | % 15 | % 16 | %Version: 20131213 17 | 18 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 19 | % Deep Neural Network: % 20 | % % 21 | % Copyright (C) 2013 Masayuki Tanaka. All rights reserved. % 22 | % mtanaka@ctrl.titech.ac.jp % 23 | % % 24 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 25 | function ErrorRate = CalcErrorRate( dbn, IN, OUT ) 26 | out = v2h( dbn, IN ); 27 | [m ind] = max(out,[],2); 28 | out = zeros(size(out)); 29 | for i=1:size(out,1) 30 | out(i,ind(i))=1; 31 | end 32 | 33 | ErrorRate = abs(OUT-out); 34 | ErrorRate = mean(sum(ErrorRate,2)/2); 35 | 36 | end 37 | 38 | -------------------------------------------------------------------------------- /CalcRmse.m: -------------------------------------------------------------------------------- 1 | % CalcRmse: calculate the rmse between predictions and OUTs 2 | % 3 | % [rmse AveErrNum] = CalcRmse( dbn, IN, OUT ) 4 | % 5 | % 6 | %Output parameters: 7 | % rmse: the rmse between predictions and OUTs 8 | % AveErrNum: average error number after binarization 9 | % 10 | % 11 | %Input parameters: 12 | % dbn: network 13 | % IN: input data, where # of row is # of data and # of col is # of input features 14 | % OUT: output data, where # of row is # of data and # of col is # of output labels 15 | % 16 | % 17 | %Version: 20130727 18 | 19 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 20 | % Deep Neural Network: % 21 | % % 22 | % Copyright (C) 2013 Masayuki Tanaka. All rights reserved. % 23 | % mtanaka@ctrl.titech.ac.jp % 24 | % % 25 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 26 | function [rmse AveErrNum] = CalcRmse( dbn, IN, OUT ) 27 | out = v2h( dbn, IN ); 28 | 29 | err = power( OUT - out, 2 ); 30 | rmse = sqrt( sum(err(:)) / numel(err) ); 31 | 32 | bout = out > 0.5; 33 | BOUT = OUT > 0.5; 34 | 35 | err = abs( BOUT - bout ); 36 | AveErrNum = mean( sum(err,2) ); 37 | end 38 | -------------------------------------------------------------------------------- /GetDroppedDBN.m: -------------------------------------------------------------------------------- 1 | % GetDroppedDBN: get dropped dbn 2 | % 3 | % [DropedDBN OnInd] = GetDroppedDBN(dbn, DropOutRate, strbm) 4 | % 5 | % 6 | %Output parameters: 7 | % DropedDBN: the generated dropped Deep Belief Nets (DBN) model 8 | % OnInd: indexes which are used (not dropped) nodes 9 | % 10 | % 11 | %Input parameters: 12 | % dbn: the Original Deep Belief Nets (DBN) model 13 | % DropOutRate: 0 < DropOutRate < 1 14 | % strbm (optional): started rbm layer to dropout (Default: 1) 15 | % 16 | % 17 | %Reference: 18 | %for details of the dropout 19 | % Hinton et al, Improving neural networks by preventing co-adaptation of feature detectors, 2012. 20 | % 21 | % 22 | %Version: 20130920 23 | 24 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 25 | % Deep Neural Network: % 26 | % % 27 | % Copyright (C) 2013 Masayuki Tanaka. All rights reserved. % 28 | % mtanaka@ctrl.titech.ac.jp % 29 | % % 30 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 31 | function [DropedDBN OnInd] = GetDroppedDBN(dbn, DropOutRate, strbm) 32 | 33 | if( ~exist('strbm', 'var') || isempty(strbm) ) 34 | strbm = 1; 35 | end 36 | 37 | nrbm = numel(dbn.rbm); 38 | 39 | OnInd = GetOnInd(dbn, DropOutRate, strbm); 40 | 41 | DropedDBN.type = dbn.type; 42 | DropedDBN.rbm = cell(nrbm,1); 43 | 44 | for n=1:nrbm-1 45 | DropedDBN.rbm{n}.type = dbn.rbm{n}.type; 46 | DropedDBN.rbm{n}.W = dbn.rbm{n}.W(OnInd{n},OnInd{n+1}); 47 | DropedDBN.rbm{n}.b = dbn.rbm{n}.b(1,OnInd{n+1}); 48 | DropedDBN.rbm{n}.c = dbn.rbm{n}.c(1,OnInd{n}); 49 | if( isequal(dbn.rbm{n}.type(1:2), 'GB') ) 50 | DropedDBN.rbm{n}.sig = dbn.rbm{n}.sig(1,OnInd{n}); 51 | end 52 | end 53 | 54 | n = nrbm; 55 | DropedDBN.rbm{n}.type = dbn.rbm{n}.type; 56 | DropedDBN.rbm{n}.W = dbn.rbm{n}.W(OnInd{n},:); 57 | DropedDBN.rbm{n}.b = dbn.rbm{n}.b; 58 | DropedDBN.rbm{n}.c = dbn.rbm{n}.c(1,OnInd{n}); 59 | if( isequal(dbn.rbm{n}.type(1:2), 'GB') ) 60 | DropedDBN.rbm{n}.sig = dbn.rbm{n}.sig(1,OnInd{n}); 61 | end 62 | -------------------------------------------------------------------------------- /GetOnInd.m: -------------------------------------------------------------------------------- 1 | % GetOnInd: get indexes which are used (not dropped) nodes 2 | % 3 | % OnInd = GetOnInd( dbn, DropOutRate, strbm ) 4 | % 5 | % 6 | %Output parameters: 7 | % OnInd: indexes which are used (not dropped) nodes 8 | % 9 | % 10 | %Input parameters: 11 | % dbn: the Original Deep Belief Nets (DBN) model 12 | % DropOutRate: 0 < DropOutRate < 1 13 | % strbm (optional): started rbm layer to dropout (Default: 1) 14 | % 15 | % 16 | %Reference: 17 | %for details of the dropout 18 | % Hinton et al, Improving neural networks by preventing co-adaptation of feature detectors, 2012. 19 | % 20 | % 21 | %Version: 20130821 22 | 23 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 24 | % Deep Neural Network: % 25 | % % 26 | % Copyright (C) 2013 Masayuki Tanaka. All rights reserved. % 27 | % mtanaka@ctrl.titech.ac.jp % 28 | % % 29 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 30 | function OnInd = GetOnInd( dbn, DropOutRate, strbm ) 31 | 32 | if( ~exist('strbm', 'var') || isempty(strbm) ) 33 | strbm = 1; 34 | end 35 | 36 | OnInd = cell(numel(dbn.rbm),1); 37 | 38 | for n=1:numel(dbn.rbm) 39 | dimV = size(dbn.rbm{n}.W,1); 40 | if( n >= strbm ) 41 | OnNum = round(dimV*DropOutRate(n)); 42 | OnInd{n} = sort(randperm(dimV, OnNum)); 43 | else 44 | OnInd{n} = 1:dimV; 45 | end 46 | end -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # code 2 | کدهای پروژه 3 | done by zahra faezi is a master's student from south Tehran university 4 | 40114140111041 student number 5 | -------------------------------------------------------------------------------- /SetLinearMapping.m: -------------------------------------------------------------------------------- 1 | % SetLinearMapping: set the RBM associated to the linear mapping to the last layer 2 | % 3 | % dbn = SetLinearMapping( dbn, IN, OUT ) 4 | % 5 | % 6 | %Input parameters: 7 | % dbn: the Deep Belief Nets (DBN) model 8 | % IN: visible (input) variables, where # of row is number of data and # of col is # of visible (input) nodes 9 | % OUT: teaching data, where # of row is number of data and # of col is # of hidden (output) nodes 10 | % 11 | % 12 | %Output parameters: 13 | % dbn: the set Deep Belief Nets (DBN) model 14 | % 15 | % 16 | %Example: 17 | % datanum = 1024; 18 | % outputnum = 16; 19 | % hiddennum = 8; 20 | % inputnum = 4; 21 | % 22 | % inputdata = rand(datanum, inputnum); 23 | % outputdata = rand(datanum, outputnum); 24 | % 25 | % dbn = randDBN([inputnum, hiddennum, outputnum]); 26 | % dbn = pretrainDBN( dbn, inputdata ); 27 | % dbn = SetLinearMapping( dbn, inputdata, outputdata ); 28 | % dbn = trainDBN( dbn, inputdata, outputdata ); 29 | % 30 | % estimate = v2h( dbn, inputdata ); 31 | 32 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 33 | % Deep Neural Network: % 34 | % % 35 | % Copyright (C) 2013 Masayuki Tanaka. All rights reserved. % 36 | % mtanaka@ctrl.titech.ac.jp % 37 | % % 38 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 39 | function dbn = SetLinearMapping( dbn, IN, OUT ) 40 | nrbm = numel(dbn.rbm); 41 | if( nrbm > 1 ) 42 | Hall = v2hall( dbn, IN ); 43 | dbn.rbm{nrbm}.W = linearMapping( Hall{nrbm-1}, OUT ); 44 | dbn.rbm{nrbm}.b = -0.5 * ones(size(dbn.rbm{nrbm}.b)); 45 | else 46 | dbn.rbm{nrbm}.W = linearMapping( IN, OUT ); 47 | dbn.rbm{nrbm}.b = -0.5 * ones(size(dbn.rbm{nrbm}.b)); 48 | end 49 | -------------------------------------------------------------------------------- /code1.m: -------------------------------------------------------------------------------- 1 | clear 2 | 3 | addpath('DeepNeuralNetwork') 4 | 5 | 6 | num = 1000; 7 | 8 | IN = rand(num,32); 9 | OUT = rand(num,4); 10 | 11 | nodes =[32 16 4]; 12 | 13 | dnn = randDBN(nodes); 14 | 15 | 16 | opts.MaxIter = 1000; 17 | opts.BatchSize = num/4; 18 | opts.Verbose = true; 19 | opts.StepRatio=0.1; 20 | opts.DropOutRate=.05; 21 | opts.object = 'CrossEntropy'; 22 | 23 | 24 | 25 | dnn =pretrainDBN(dnn,IN,opts); 26 | dnn =SetLinearMapping(dnn,IN,OUT); 27 | 28 | dnn = trainDBN(dnn,IN,OUT,opts); 29 | 30 | rmse = CalcRmse(dnn,IN,OUT) -------------------------------------------------------------------------------- /evaMNIST.m: -------------------------------------------------------------------------------- 1 | clear all; 2 | 3 | clear all; 4 | addpath('..'); 5 | 6 | mnistfiles = 0; 7 | if( exist('t10k-images-idx3-ubyte', 'file') == 2 ) 8 | mnistfiles = mnistfiles + 1; 9 | end 10 | if( exist('t10k-labels-idx1-ubyte', 'file') == 2 ) 11 | mnistfiles = mnistfiles + 1; 12 | end 13 | if( exist('train-images-idx3-ubyte', 'file') == 2 ) 14 | mnistfiles = mnistfiles + 1; 15 | end 16 | if( exist('train-labels-idx1-ubyte', 'file') == 2 ) 17 | mnistfiles = mnistfiles + 1; 18 | end 19 | 20 | if( mnistfiles < 4 ) 21 | warning('Can not find mnist data files. Please download four data files from http://yann.lecun.com/exdb/mnist/ . Unzip and copy them to same folder as testMNIST.m'); 22 | return; 23 | end 24 | 25 | [TrainImages TrainLabels TestImages TestLabels] = mnistread(); 26 | 27 | load mnistbbdbn; 28 | 29 | rmse= CalcRmse(bbdbn, TrainImages, TrainLabels); 30 | ErrorRate= CalcErrorRate(bbdbn, TrainImages, TrainLabels); 31 | fprintf( 'For training data:\n' ); 32 | fprintf( 'rmse: %g\n', rmse ); 33 | fprintf( 'ErrorRate: %g\n', ErrorRate ); 34 | 35 | rmse= CalcRmse(bbdbn, TestImages, TestLabels); 36 | ErrorRate= CalcErrorRate(bbdbn, TestImages, TestLabels); 37 | fprintf( 'For test data:\n' ); 38 | fprintf( 'rmse: %g\n', rmse ); 39 | fprintf( 'ErrorRate: %g\n', ErrorRate ); 40 | 41 | -------------------------------------------------------------------------------- /h2v.m: -------------------------------------------------------------------------------- 1 | % h2v: to transform from hidden (output) variables to visible (input) variables 2 | % 3 | % V = h2v(dnn, H) 4 | % 5 | % 6 | %Output parameters: 7 | % V: visible (input) variables, where # of row is number of data and # of col is # of visible (input) nodes 8 | % 9 | % 10 | %Input parameters: 11 | % dnn: the Deep Neural Network model (dbn, rbm) 12 | % H: hidden (output) variables, where # of row is number of data and # of col is # of hidden (output) nodes 13 | % 14 | % 15 | %Example: 16 | % datanum = 1024; 17 | % outputnum = 16; 18 | % inputnum = 4; 19 | % 20 | % outputdata = rand(datanum, outputnum); 21 | % 22 | % dnn = randRBM( inputnum, outputnum ); 23 | % inputdata = h2v( dnn, outputdata ); 24 | % 25 | % 26 | %Version: 20130830 27 | 28 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 29 | % Deep Neural Network: % 30 | % % 31 | % Copyright (C) 2013 Masayuki Tanaka. All rights reserved. % 32 | % mtanaka@ctrl.titech.ac.jp % 33 | % % 34 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 35 | function V = h2v(dnn, H) 36 | 37 | ind1 = numel(dnn.type); 38 | ind0 = ind1-2; 39 | type = dnn.type(ind0:ind1); 40 | 41 | if( isequal(dnn.type, 'BBRBM') ) 42 | V = sigmoid( bsxfun(@plus, H * dnn.W', dnn.c ) ); 43 | 44 | elseif( isequal(dnn.type, 'GBRBM') ) 45 | h = bsxfun(@times, H * dnn.W', dnn.sig); 46 | V = bsxfun(@plus, h, dnn.c ); 47 | 48 | elseif( isequal(dnn.type, 'BBPRBM') ) 49 | w2 = dnn.W .* dnn.W; 50 | pp = H .* ( 1-H ); 51 | mu = bsxfun(@plus, H * dnn.W', dnn.c ); 52 | s2 = pp * w2'; 53 | V = sigmoid( mu ./ sqrt( 1 + s2 * pi / 8 ) ); 54 | 55 | elseif( isequal(type, 'DBN') ) 56 | nrbm = numel( dnn.rbm ); 57 | V0 = H; 58 | for i=nrbm:-1:1 59 | V1 = h2v( dnn.rbm{i}, V0 ); 60 | V0 = V1; 61 | end 62 | V = V1; 63 | 64 | end 65 | -------------------------------------------------------------------------------- /harvest_moon.cvv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mahdeslami11/code/3e74e3e29c80131b725bd5f174489efdc7033a15/harvest_moon.cvv -------------------------------------------------------------------------------- /iris_code1.m: -------------------------------------------------------------------------------- 1 | clear 2 | 3 | warning off 4 | 5 | [x,t]=iris_dataset; 6 | 7 | x = x'; 8 | t = t'; 9 | 10 | nodes = [4 3]; 11 | 12 | dnn = randDBN(nodes); 13 | 14 | opts.MaxIter = 1000; 15 | opts.BatchSize = 150/4; 16 | opts.Verbose = true; 17 | opts.StepRatio=0.1; 18 | opts.DropOutRate=.05; 19 | 20 | 21 | 22 | dnn =pretrainDBN(dnn,x,opts); 23 | dnn =SetLinearMapping(dnn,x,t); 24 | 25 | opts.Layer = 0; 26 | dnn = trainDBN(dnn,x,t,opts); 27 | 28 | rmse = CalcRmse(dnn,x,t) -------------------------------------------------------------------------------- /linearMapping.m: -------------------------------------------------------------------------------- 1 | % linearMaping: calculate the linear mapping matrix between the input data and the output data 2 | % 3 | % M = linearMapping( IN, OUT ) 4 | % 5 | % 6 | %Output parameters: 7 | % M: The linear mapping matrix 8 | % 9 | % 10 | %Input parameters: 11 | % IN: input data, where # of row is # of data and # of col is # of input features 12 | % OUT: output data, where # of row is # of data and # of col is # of output labels 13 | % 14 | % 15 | %Example: 16 | % datanum = 1024; 17 | % outputnum = 16; 18 | % inputnum = 4; 19 | % 20 | % inputdata = rand(datanum, inputnum); 21 | % outputdata = rand(datanum, outputnum); 22 | % 23 | % M = linearMapping(inputdata, outputdata); 24 | % 25 | % 26 | %Version: 20130727 27 | 28 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 29 | % Deep Neural Network: % 30 | % % 31 | % Copyright (C) 2013 Masayuki Tanaka. All rights reserved. % 32 | % mtanaka@ctrl.titech.ac.jp % 33 | % % 34 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 35 | function M = linearMapping( IN, OUT ) 36 | M = pinv(IN) * OUT; 37 | 38 | %OUT = IN * M; 39 | -------------------------------------------------------------------------------- /mnistread.m: -------------------------------------------------------------------------------- 1 | function [ TrainImages, TrainLabels, TestImages, TestLabels ] = mnistread( mnistfilenames ) 2 | 3 | if( ~exist('mnistfilenames') ) 4 | mnistfilenames = cell(4,1); 5 | mnistfilenames{1} = 'train-images-idx3-ubyte'; 6 | mnistfilenames{2} = 'train-labels-idx1-ubyte'; 7 | mnistfilenames{3} = 't10k-images-idx3-ubyte'; 8 | mnistfilenames{4} = 't10k-labels-idx1-ubyte'; 9 | end 10 | 11 | TrainImages = mnistimageread( mnistfilenames{1} ); 12 | TrainLabels = mnistlabelread( mnistfilenames{2} ); 13 | TestImages = mnistimageread( mnistfilenames{3} ); 14 | TestLabels = mnistlabelread( mnistfilenames{4} ); 15 | 16 | TrainImages = single(TrainImages)/255.0; 17 | TestImages = single(TestImages)/255.0; 18 | 19 | TrainLabels = single(TrainLabels); 20 | TestLabels = single(TestLabels); 21 | 22 | end 23 | 24 | function images = mnistimageread( imagefile ) 25 | fid = fopen( imagefile, 'rb'); 26 | magic = fread(fid, 1, '*int32',0,'b'); 27 | nimgs = fread(fid, 1, '*int32',0,'b'); 28 | nrows = fread(fid, 1, '*int32',0,'b'); 29 | ncols = fread(fid, 1, '*int32',0,'b'); 30 | images = fread(fid, inf, '*uint8',0,'b'); 31 | fclose( fid ); 32 | 33 | if( magic ~= 2051 ) 34 | warning( sprintf( '%s is not MNIST image file.', imagefile ) ); 35 | images = []; 36 | return; 37 | end 38 | 39 | images = reshape( images, [nrows*ncols, nimgs] )'; 40 | for i=1:nimgs 41 | img = reshape( images(i,:), [nrows ncols] )'; 42 | images(i,:) = reshape(img, [1 nrows*ncols]); 43 | end 44 | end 45 | 46 | function labels = mnistlabelread( labelfile ) 47 | fid = fopen( labelfile, 'rb'); 48 | magic = fread(fid, 1, '*int32',0,'b'); 49 | nlabels = fread(fid, 1, '*int32',0,'b'); 50 | ind = fread(fid, inf, '*uint8',0,'b'); 51 | fclose( fid ); 52 | 53 | if( magic ~= 2049 ) 54 | warning( sprintf( '%s is not MNIST label file.', labelfile ) ); 55 | labels = []; 56 | return; 57 | end 58 | 59 | labels = zeros( nlabels, 10 ); 60 | ind = ind + 1; 61 | for i=1:nlabels 62 | labels(i, ind(i)) = 1; 63 | end 64 | end 65 | 66 | 67 | -------------------------------------------------------------------------------- /pretrainDBN.m: -------------------------------------------------------------------------------- 1 | % pretrainDBN: pre-training the Deep Belief Nets (DBN) model by Contrastive Divergence Learning 2 | % 3 | % dbn = pretrainDBN(dbn, V, opts) 4 | % 5 | % 6 | %Output parameters: 7 | % dbn: the trained Deep Belief Nets (DBN) model 8 | % 9 | % 10 | %Input parameters: 11 | % dbn: the initial Deep Belief Nets (DBN) model 12 | % V: visible (input) variables, where # of row is number of data and # of col is # of visible (input) nodes 13 | % opts (optional): options 14 | % 15 | % options (defualt value): 16 | % opts.LayerNum: # of tarining RBMs counted from input layer (all layer) 17 | % opts.MaxIter: Maxium iteration number (100) 18 | % opts.InitialMomentum: Initial momentum until InitialMomentumIter (0.5) 19 | % opts.InitialMomentumIter: Iteration number for initial momentum (5) 20 | % opts.FinalMomentum: Final momentum after InitialMomentumIter (0.9) 21 | % opts.WeightCost: Weight cost (0.0002) 22 | % opts.DropOutRate: List of Dropout rates for each layer (0) 23 | % opts.StepRatio: Learning step size (0.01) 24 | % opts.BatchSize: # of mini-batch data (# of all data) 25 | % opts.Verbose: verbose or not (false) 26 | % opts.SparseQ: q parameter of sparse learning (0) 27 | % opts.SparseLambda: lambda parameter (weight) of sparse learning (0) 28 | % 29 | % 30 | %Example: 31 | % datanum = 1024; 32 | % outputnum = 16; 33 | % hiddennum = 8; 34 | % inputnum = 4; 35 | % 36 | % inputdata = rand(datanum, inputnum); 37 | % outputdata = rand(datanum, outputnum); 38 | % 39 | % dbn = randDBN([inputnum, hiddennum, outputnum]); 40 | % dbn = pretrainDBN( dbn, inputdata ); 41 | % dbn = SetLinearMapping( dbn, inputdata, outputdata ); 42 | % dbn = trainDBN( dbn, inputdata, outputdata ); 43 | % 44 | % estimate = v2h( dbn, inputdata ); 45 | % 46 | % 47 | %Reference: 48 | %for details of the dropout 49 | % Hinton et al, Improving neural networks by preventing co-adaptation of feature detectors, 2012. 50 | %for details of the sparse learning 51 | % Lee et al, Sparse deep belief net model for visual area V2, NIPS 2008. 52 | % 53 | % 54 | %Version: 20130821 55 | 56 | 57 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 58 | % Deep Neural Network: % 59 | % % 60 | % Copyright (C) 2013 Masayuki Tanaka. All rights reserved. % 61 | % mtanaka@ctrl.titech.ac.jp % 62 | % % 63 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 64 | function dbn = pretrainDBN(dbn, V, opts) 65 | 66 | LayerNum = numel( dbn.rbm ); 67 | DropOutRate = zeros(LayerNum,1); 68 | 69 | X = V; 70 | 71 | if( exist('opts' ) ) 72 | if( isfield(opts,'LayerNum') ) 73 | LayerNum = opts.LayerNum; 74 | end 75 | if( isfield(opts,'DropOutRate') ) 76 | DropOutRate = opts.DropOutRate; 77 | if( numel( DropOutRate ) == 1 ) 78 | DropOutRate = ones(LayerNum,1) * DropOutRate; 79 | end 80 | end 81 | 82 | else 83 | opts = []; 84 | end 85 | 86 | for i=1:LayerNum 87 | opts.DropOutRate = DropOutRate(i); 88 | dbn.rbm{i} = pretrainRBM(dbn.rbm{i}, X, opts); 89 | X0 = X; 90 | X = v2h( dbn.rbm{i}, X0 ); 91 | end 92 | -------------------------------------------------------------------------------- /pretrainRBM.m: -------------------------------------------------------------------------------- 1 | % pretrainRBM: pre-training the restricted boltzmann machine (RBM) model by Contrastive Divergence Learning 2 | % 3 | % rbm = pretrainRBM(rbm, V, opts) 4 | % 5 | % 6 | %Output parameters: 7 | % rbm: the restricted boltzmann machine (RBM) model 8 | % 9 | % 10 | %Input parameters: 11 | % rbm: the initial boltzmann machine (RBM) model 12 | % V: visible (input) variables, where # of row is number of data and # of col is # of visible (input) nodes 13 | % opts (optional): options 14 | % 15 | % options (defualt value): 16 | % opts.MaxIter: Maxium iteration number (100) 17 | % opts.InitialMomentum: Initial momentum until InitialMomentumIter (0.5) 18 | % opts.InitialMomentumIter: Iteration number for initial momentum (5) 19 | % opts.FinalMomentum: Final momentum after InitialMomentumIter (0.9) 20 | % opts.WeightCost: Weight cost (0.0002) 21 | % opts.DropOutRate: Dropour rate (0) 22 | % opts.StepRatio: Learning step size (0.01) 23 | % opts.BatchSize: # of mini-batch data (# of all data) 24 | % opts.Verbose: verbose or not (false) 25 | % opts.SparseQ: q parameter of sparse learning (0) 26 | % opts.SparseLambda: lambda parameter (weight) of sparse learning (0) 27 | % 28 | % 29 | %Example: 30 | % datanum = 1024; 31 | % outputnum = 16; 32 | % inputnum = 4; 33 | % 34 | % inputdata = rand(datanum, inputnum); 35 | % outputdata = rand(datanum, outputnum); 36 | % 37 | % rbm = randRBM(inputnum, outputnum); 38 | % rbm = pretrainRBM( rbm, inputdata ); 39 | % 40 | % 41 | %Reference: 42 | %for details of the dropout 43 | % Hinton et al, Improving neural networks by preventing co-adaptation of feature detectors, 2012. 44 | %for details of the sparse learning 45 | % Lee et al, Sparse deep belief net model for visual area V2, NIPS 2008. 46 | %for implimentation of contrastive divergence learning 47 | % http://read.pudn.com/downloads103/sourcecode/math/421402/drtoolbox/techniques/train_rbm.m__.htm 48 | % 49 | % 50 | %Version: 20131022 51 | 52 | 53 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 54 | % Deep Neural Network: % 55 | % % 56 | % Copyright (C) 2013 Masayuki Tanaka. All rights reserved. % 57 | % mtanaka@ctrl.titech.ac.jp % 58 | % % 59 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 60 | function rbm = pretrainRBM(rbm, V, opts ) 61 | 62 | % Important parameters 63 | InitialMomentum = 0.5; % momentum for first five iterations 64 | FinalMomentum = 0.9; % momentum for remaining iterations 65 | WeightCost = 0.0002; % costs of weight update 66 | InitialMomentumIter = 5; 67 | 68 | MaxIter = 100; 69 | DropOutRate = 0; 70 | StepRatio = 0.01; 71 | BatchSize = 0; 72 | Verbose = false; 73 | 74 | SparseQ = 0; 75 | SparseLambda = 0; 76 | 77 | 78 | if( exist('opts' ) ) 79 | if( isfield(opts,'MaxIter') ) 80 | MaxIter = opts.MaxIter; 81 | end 82 | if( isfield(opts,'InitialMomentum') ) 83 | InitialMomentum = opts.InitialMomentum; 84 | end 85 | if( isfield(opts,'InitialMomentumIter') ) 86 | InitialMomentumIter = opts.InitialMomentumIter; 87 | end 88 | if( isfield(opts,'FinalMomentum') ) 89 | FinalMomentum = opts.FinalMomentum; 90 | end 91 | if( isfield(opts,'WeightCost') ) 92 | WeightCost = opts.WeightCost; 93 | end 94 | if( isfield(opts,'DropOutRate') ) 95 | DropOutRate = opts.DropOutRate; 96 | end 97 | if( isfield(opts,'StepRatio') ) 98 | StepRatio = opts.StepRatio; 99 | end 100 | if( isfield(opts,'BatchSize') ) 101 | BatchSize = opts.BatchSize; 102 | end 103 | if( isfield(opts,'Verbose') ) 104 | Verbose = opts.Verbose; 105 | end 106 | if( isfield(opts,'SparseQ') ) 107 | SparseQ = opts.SparseQ; 108 | end 109 | if( isfield(opts,'SparseLambda') ) 110 | SparseLambda = opts.SparseLambda; 111 | end 112 | 113 | else 114 | opts = []; 115 | end 116 | 117 | num = size(V,1); 118 | dimH = size(rbm.b, 2); 119 | dimV = size(rbm.c, 2); 120 | 121 | if( BatchSize <= 0 ) 122 | BatchSize = num; 123 | end 124 | 125 | if( DropOutRate > 0 ) 126 | DropOutNum = round(dimV * DropOutRate); 127 | DropOutRate = DropOutNum / num; 128 | end 129 | 130 | 131 | deltaW = zeros(dimV, dimH); 132 | deltaB = zeros(1, dimH); 133 | deltaC = zeros(1, dimV); 134 | 135 | if( Verbose ) 136 | timer = tic; 137 | end 138 | 139 | for iter=1:MaxIter 140 | 141 | 142 | % Set momentum 143 | if( iter <= InitialMomentumIter ) 144 | momentum = InitialMomentum; 145 | else 146 | momentum = FinalMomentum; 147 | end 148 | 149 | if( SparseLambda > 0 ) 150 | dsW = zeros(dimV, dimH); 151 | dsB = zeros(1, dimH); 152 | 153 | vis0 = V; 154 | hid0 = v2h( rbm, vis0 ); 155 | 156 | dH = hid0 .* ( 1.0 - hid0 ); 157 | sH = sum( hid0, 1 ); 158 | end 159 | 160 | if( SparseLambda > 0 ) 161 | mH = sH / num; 162 | sdH = sum( dH, 1 ); 163 | svdH = dH' * vis0; 164 | 165 | dsW = dsW + SparseLambda * 2.0 * bsxfun(@times, (SparseQ-mH)', svdH)'; 166 | dsB = dsB + SparseLambda * 2.0 * (SparseQ-mH) .* sdH; 167 | end 168 | 169 | 170 | ind = randperm(num); 171 | for batch=1:BatchSize:num 172 | 173 | bind = ind(batch:min([batch + BatchSize - 1, num])); 174 | 175 | if( DropOutRate > 0 ) 176 | cMat = zeros(dimV,1); 177 | p = randperm(dimV, DropOutNum); 178 | cMat(p) = 1; 179 | cMat = diag(cMat); 180 | end 181 | 182 | % Gibbs sampling step 0 183 | vis0 = double(V(bind,:)); % Set values of visible nodes 184 | if( DropOutRate > 0 ) 185 | vis0 = vis0 * cMat; 186 | end 187 | hid0 = v2h( rbm, vis0 ); % Compute hidden nodes 188 | 189 | % Gibbs sampling step 1 190 | if( isequal(rbm.type(3), 'P') ) 191 | bhid0 = hid0; 192 | else 193 | bhid0 = double( rand(size(hid0)) < hid0 ); 194 | end 195 | vis1 = h2v( rbm, bhid0 ); % Compute visible nodes 196 | if( DropOutRate > 0 ) 197 | vis1 = vis1 * cMat; 198 | end 199 | hid1 = v2h( rbm, vis1 ); % Compute hidden nodes 200 | 201 | posprods = hid0' * vis0; 202 | negprods = hid1' * vis1; 203 | % Compute the weights update by contrastive divergence 204 | 205 | dW = (posprods - negprods)'; 206 | dB = (sum(hid0, 1) - sum(hid1, 1)); 207 | dC = (sum(vis0, 1) - sum(vis1, 1)); 208 | 209 | if( strcmpi( 'GBRBM', rbm.type ) ) 210 | dW = bsxfun(@rdivide, dW, rbm.sig'); 211 | dC = bsxfun(@rdivide, dC, rbm.sig .* rbm.sig); 212 | end 213 | 214 | deltaW = momentum * deltaW + (StepRatio / num) * dW; 215 | deltaB = momentum * deltaB + (StepRatio / num) * dB; 216 | deltaC = momentum * deltaC + (StepRatio / num) * dC; 217 | 218 | if( SparseLambda > 0 ) 219 | deltaW = deltaW + numel(bind) / num * dsW; 220 | deltaB = deltaB + numel(bind) / num * dsB; 221 | end 222 | 223 | % Update the network weights 224 | rbm.W = rbm.W + deltaW - WeightCost * rbm.W; 225 | rbm.b = rbm.b + deltaB; 226 | rbm.c = rbm.c + deltaC; 227 | 228 | end 229 | 230 | if( SparseLambda > 0 && strcmpi( 'GBRBM', rbm.type ) ) 231 | dsW = bsxfun(@rdivide, dsW, rbm.sig'); 232 | end 233 | 234 | 235 | if( Verbose ) 236 | H = v2h( rbm, V ); 237 | Vr = h2v( rbm, H ); 238 | err = power( V - Vr, 2 ); 239 | rmse = sqrt( sum(err(:)) / numel(err) ); 240 | 241 | totalti = toc(timer); 242 | aveti = totalti / iter; 243 | estti = (MaxIter-iter) * aveti; 244 | eststr = datestr(datenum(0,0,0,0,0,estti),'DD:HH:MM:SS'); 245 | 246 | fprintf( '%3d : %9.4f %9.4f %9.4f %s\n', iter, rmse, mean(H(:)), aveti, eststr ); 247 | end 248 | end 249 | 250 | -------------------------------------------------------------------------------- /randDBN.m: -------------------------------------------------------------------------------- 1 | % randDBN: get randomized Deep Belief Nets (DBN) model 2 | % 3 | % dbn = randDBN( dims, type ) 4 | % 5 | % 6 | %Output parameters: 7 | % dbn: the randomized Deep Belief Nets (DBN) model 8 | % 9 | % 10 | %Input parameters: 11 | % dims: number of nodes 12 | % type (optional): (default: 'BBDBN' ) 13 | % 'BBDBN': all RBMs are the Bernoulli-Bernoulli RBMs 14 | % 'GBDBN': the input RBM is the Gaussian-Bernoulli RBM, other RBMs are the Bernoulli-Bernoulli RBMs 15 | % 16 | % 17 | %Version: 20130830 18 | 19 | 20 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 21 | % Deep Neural Network: % 22 | % % 23 | % Copyright (C) 2013 Masayuki Tanaka. All rights reserved. % 24 | % mtanaka@ctrl.titech.ac.jp % 25 | % % 26 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 27 | function dbn = randDBN( dims, type ) 28 | 29 | if( ~exist('type', 'var') || isempty(type) ) 30 | type = 'BBDBN'; 31 | end 32 | 33 | if( strcmpi( 'GB', type(1:2) ) ) 34 | dbn.type = 'GBDBN'; 35 | rbmtype = 'GBRBM'; 36 | elseif( strcmpi( 'BBP', type(1:3) ) ) 37 | dbn.type = 'BBPDBN'; 38 | rbmtype = 'BBPRBM'; 39 | else 40 | dbn.type = 'BBDBN'; 41 | rbmtype = 'BBRBM'; 42 | end 43 | 44 | dbn.rbm = cell( numel(dims)-1, 1 ); 45 | 46 | i = 1; 47 | dbn.rbm{i} = randRBM( dims(i), dims(i+1), rbmtype ); 48 | 49 | for i=2:numel(dbn.rbm) - 1 50 | dbn.rbm{i} = randRBM( dims(i), dims(i+1), rbmtype ); 51 | end 52 | 53 | i = numel(dbn.rbm); 54 | if( strcmp( 'P', type(3) ) ) 55 | dbn.rbm{i} = randRBM( dims(i), dims(i+1), 'BBPRBM' ); 56 | else 57 | dbn.rbm{i} = randRBM( dims(i), dims(i+1), 'BBRBM' ); 58 | end 59 | -------------------------------------------------------------------------------- /randRBM.m: -------------------------------------------------------------------------------- 1 | % randRBM: get randomized restricted boltzmann machine (RBM) model 2 | % 3 | % rbm = randRBM( dimV, dimH, type ) 4 | % 5 | % 6 | %Output parameters: 7 | % dbn: the randomized restricted boltzmann machine (RBM) model 8 | % 9 | % 10 | %Input parameters: 11 | % dimV: number of visible (input) nodes 12 | % dimH: number of hidden (output) nodes 13 | % type (optional): (default: 'BBRBM' ) 14 | % 'BBRBM': the Bernoulli-Bernoulli RBM 15 | % 'GBRBM': the Gaussian-Bernoulli RBM 16 | % 17 | % 18 | %Version: 20130830 19 | 20 | 21 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 22 | % Deep Neural Network: % 23 | % % 24 | % Copyright (C) 2013 Masayuki Tanaka. All rights reserved. % 25 | % mtanaka@ctrl.titech.ac.jp % 26 | % % 27 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 28 | function rbm = randRBM( dimV, dimH, type ) 29 | 30 | if( ~exist('type', 'var') || isempty(type) ) 31 | type = 'BBRBM'; 32 | end 33 | 34 | if( strcmpi( 'GB', type(1:2) ) ) 35 | rbm.type = 'GBRBM'; 36 | rbm.W = randn(dimV, dimH) * 0.1; 37 | rbm.b = zeros(1, dimH); 38 | rbm.c = zeros(1, dimV); 39 | rbm.sig = ones(1, dimV); 40 | else 41 | rbm.type = type; 42 | rbm.W = randn(dimV, dimH) * 0.1; 43 | rbm.b = zeros(1, dimH); 44 | rbm.c = zeros(1, dimV); 45 | end 46 | 47 | -------------------------------------------------------------------------------- /sigmoid.m: -------------------------------------------------------------------------------- 1 | % sigmoid: calculate sigmoid function 2 | % 3 | % y = sigmoid(x) 4 | % 5 | % y = 1.0 ./ ( 1.0 + exp(-x) ); 6 | % 7 | % 8 | %Version: 20130727 9 | 10 | 11 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 12 | % Deep Neural Network: % 13 | % % 14 | % Copyright (C) 2013 Masayuki Tanaka. All rights reserved. % 15 | % mtanaka@ctrl.titech.ac.jp % 16 | % % 17 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 18 | function y = sigmoid(x) 19 | 20 | y = 1.0 ./ ( 1.0 + exp(-x) ); 21 | -------------------------------------------------------------------------------- /testDNN.m: -------------------------------------------------------------------------------- 1 | clear all; 2 | 3 | num = 1000; 4 | nodes = [32 16 8 4]; 5 | 6 | IN = rand(num,32); 7 | OUT = rand(num,4); 8 | 9 | dnn = randDBN( nodes ); 10 | %dnn = randDBN( nodes, 'BBPDBN' ); % ICPR 2014 11 | %dnn = randDBN( nodes, 'GBDBN' ); 12 | nrbm = numel(dnn.rbm); 13 | 14 | opts.MaxIter = 20; 15 | opts.BatchSize = num/4; 16 | opts.Verbose = true; 17 | opts.StepRatio = 0.1; 18 | opts.Layer = nrbm-1; 19 | opts.DropOutRate = 0.5; 20 | opts.Object = 'CrossEntropy'; 21 | 22 | dnn = pretrainDBN(dnn, IN, opts); 23 | dnn= SetLinearMapping(dnn, IN, OUT); 24 | 25 | opts.Layer = 0; 26 | dnn = trainDBN(dnn, IN, OUT, opts); 27 | rmse = CalcRmse(dnn, IN, OUT); 28 | rmse 29 | -------------------------------------------------------------------------------- /testMNIST.m: -------------------------------------------------------------------------------- 1 | trainMNIST; 2 | evaMNIST; 3 | -------------------------------------------------------------------------------- /trainDBN.m: -------------------------------------------------------------------------------- 1 | % trainDBN: training the Deep Belief Nets (DBN) model by back projection algorithm 2 | % 3 | % [dbn rmse] = trainDBN( dbn, IN, OUT, opts) 4 | % 5 | % 6 | %Output parameters: 7 | % dbn: the trained Deep Belief Nets (DBN) model 8 | % rmse: the rmse between the teaching data and the estimates 9 | % 10 | % 11 | %Input parameters: 12 | % dbn: the initial Deep Belief Nets (DBN) model 13 | % IN: visible (input) variables, where # of row is number of data and # of col is # of visible (input) nodes 14 | % OUT: teaching hidden (output) variables, where # of row is number of data and # of col is # of hidden (output) nodes 15 | % opts (optional): options 16 | % 17 | % options (defualt value): 18 | % opts.LayerNum: # of tarining RBMs counted from output layer (all layer) 19 | % opts.MaxIter: Maxium iteration number (100) 20 | % opts.InitialMomentum: Initial momentum until InitialMomentumIter (0.5) 21 | % opts.InitialMomentumIter: Iteration number for initial momentum (5) 22 | % opts.FinalMomentum: Final momentum after InitialMomentumIter (0.9) 23 | % opts.WeightCost: Weight cost (0.0002) 24 | % opts.DropOutRate: List of Dropout rates for each layer (0) 25 | % opts.StepRatio: Learning step size (0.01) 26 | % opts.BatchSize: # of mini-batch data (# of all data) 27 | % opts.Object: specify the object function ('Square') 28 | % 'Square' 29 | % 'CrossEntorpy' 30 | % opts.Verbose: verbose or not (false) 31 | % 32 | % 33 | %Example: 34 | % datanum = 1024; 35 | % outputnum = 16; 36 | % hiddennum = 8; 37 | % inputnum = 4; 38 | % 39 | % inputdata = rand(datanum, inputnum); 40 | % outputdata = rand(datanum, outputnum); 41 | % 42 | % dbn = randDBN([inputnum, hiddennum, outputnum]); 43 | % dbn = pretrainDBN( dbn, inputdata ); 44 | % dbn = SetLinearMapping( dbn, inputdata, outputdata ); 45 | % dbn = trainDBN( dbn, inputdata, outputdata ); 46 | % 47 | % estimate = v2h( dbn, inputdata ); 48 | % 49 | % 50 | %Reference: 51 | %for details of the dropout 52 | % Hinton et al, Improving neural networks by preventing co-adaptation of feature detectors, 2012. 53 | % 54 | % 55 | %Version: 20131024 56 | 57 | 58 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 59 | % Deep Neural Network: % 60 | % % 61 | % Copyright (C) 2013 Masayuki Tanaka. All rights reserved. % 62 | % mtanaka@ctrl.titech.ac.jp % 63 | % % 64 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 65 | function [dbn rmse] = trainDBN( dbn, IN, OUT, opts) 66 | 67 | % Important parameters 68 | InitialMomentum = 0.5; % momentum for first five iterations 69 | FinalMomentum = 0.9; % momentum for remaining iterations 70 | WeightCost = 0.0002; % costs of weight update 71 | InitialMomentumIter = 5; 72 | 73 | MaxIter = 100; 74 | StepRatio = 0.01; 75 | BatchSize = 0; 76 | Verbose = false; 77 | 78 | Layer = 0; 79 | strbm = 1; 80 | 81 | nrbm = numel( dbn.rbm ); 82 | DropOutRate = zeros(nrbm,1); 83 | 84 | OBJECTSQUARE = 1; 85 | OBJECTCROSSENTROPY = 2; 86 | Object = OBJECTSQUARE; 87 | 88 | TestIN = []; 89 | TestOUT = []; 90 | fp = []; 91 | 92 | debug = 0; 93 | 94 | if( exist('opts' ) ) 95 | if( isfield(opts,'MaxIter') ) 96 | MaxIter = opts.MaxIter; 97 | end 98 | if( isfield(opts,'InitialMomentum') ) 99 | InitialMomentum = opts.InitialMomentum; 100 | end 101 | if( isfield(opts,'InitialMomentumIter') ) 102 | InitialMomentumIter = opts.InitialMomentumIter; 103 | end 104 | if( isfield(opts,'FinalMomentum') ) 105 | FinalMomentum = opts.FinalMomentum; 106 | end 107 | if( isfield(opts,'WeightCost') ) 108 | WeightCost = opts.WeightCost; 109 | end 110 | if( isfield(opts,'DropOutRate') ) 111 | DropOutRate = opts.DropOutRate; 112 | if( numel(DropOutRate) == 1 ) 113 | DropOutRate = ones(nrbm,1) * DropOutRate; 114 | end 115 | end 116 | if( isfield(opts,'StepRatio') ) 117 | StepRatio = opts.StepRatio; 118 | end 119 | if( isfield(opts,'BatchSize') ) 120 | BatchSize = opts.BatchSize; 121 | end 122 | if( isfield(opts,'Verbose') ) 123 | Verbose = opts.Verbose; 124 | end 125 | if( isfield(opts,'Layer') ) 126 | Layer = opts.Layer; 127 | end 128 | if( isfield(opts,'Object') ) 129 | if( strcmpi( opts.Object, 'Square' ) ) 130 | Object = OBJECTSQUARE; 131 | elseif( strcmpi( opts.Object, 'CrossEntropy' ) ) 132 | Object = OBJECTCROSSENTROPY; 133 | end 134 | end 135 | if( isfield(opts,'TestIN') ) 136 | TestIN = opts.TestIN; 137 | end 138 | if( isfield(opts,'TestOUT') ) 139 | TestOUT = opts.TestOUT; 140 | end 141 | if( isfield(opts,'LogFilename') ) 142 | fp = fopen( opts.LogFilename, 'w' ); 143 | end 144 | if( isfield(opts,'Debug') ) 145 | debug = opts.Debug; 146 | end 147 | end 148 | 149 | num = size(IN,1); 150 | if( BatchSize <= 0 ) 151 | BatchSize = num; 152 | end 153 | 154 | if( Layer > 0 ) 155 | strbm = nrbm - Layer + 1; 156 | end 157 | 158 | deltaDbn = dbn; 159 | for n=strbm:nrbm 160 | deltaDbn.rbm{n}.W = zeros(size(dbn.rbm{n}.W)); 161 | deltaDbn.rbm{n}.b = zeros(size(dbn.rbm{n}.b)); 162 | end 163 | 164 | if( Layer > 0 ) 165 | strbm = nrbm - Layer + 1; 166 | end 167 | 168 | if( sum(DropOutRate > 0) ) 169 | OnInd = GetOnInd( dbn, DropOutRate, strbm ); 170 | for n=max([2,strbm]):nrbm 171 | dbn.rbm{n}.W = dbn.rbm{n}.W / numel(OnInd{n-1}) * size(dbn.rbm{n-1}.W,2); 172 | end 173 | end 174 | 175 | if( Verbose ) 176 | timer = tic; 177 | end 178 | 179 | for iter=1:MaxIter 180 | 181 | % Set momentum 182 | if( iter <= InitialMomentumIter ) 183 | momentum = InitialMomentum; 184 | else 185 | momentum = FinalMomentum; 186 | end 187 | 188 | ind = randperm(num); 189 | for batch=1:BatchSize:num 190 | bind = ind(batch:min([batch + BatchSize - 1, num])); 191 | 192 | if( isequal(dbn.type(3), 'P') ) 193 | 194 | Hall = v2hall( dbn, IN(bind,:) ); 195 | for n=nrbm:-1:strbm 196 | if( n-1 > 0 ) 197 | in = Hall{n-1}; 198 | else 199 | in = IN(bind,:); 200 | end 201 | 202 | [intDerDel intDerTau] = internal( dbn.rbm{n}, in ); 203 | derSgm = Hall{n} .* ( 1 - Hall{n} ); 204 | if( n+1 > nrbm ) 205 | derDel = intDerDel .* ( Hall{nrbm} - OUT(bind,:) ); 206 | derTau = intDerTau .* ( Hall{nrbm} - OUT(bind,:) ); 207 | if( Object == OBJECTSQUARE ) 208 | derDel = derDel .* derSgm; 209 | derTau = derTau .* derSgm; 210 | end 211 | else 212 | al = derDel * dbn.rbm{n+1}.W' + derTau * ( dbn.rbm{n+1}.W .* dbn.rbm{n+1}.W )' .* ( 1 - 2 * Hall{n} ); 213 | 214 | derDel = al .* derSgm .* intDerDel; 215 | derTau = al .* derSgm .* intDerTau; 216 | end 217 | 218 | deltaW = ( in' * derDel + 2 * (in .* (1-in))' * derTau .* dbn.rbm{n}.W ) / numel(bind); 219 | deltab = mean(derDel,1); 220 | 221 | deltaDbn.rbm{n}.W = momentum * deltaDbn.rbm{n}.W - StepRatio * deltaW; 222 | deltaDbn.rbm{n}.b = momentum * deltaDbn.rbm{n}.b - StepRatio * deltab; 223 | 224 | if( debug ) 225 | EP = 1E-8; 226 | dif = zeros(size(dbn.rbm{n}.W)); 227 | for i=1:size(dif,1) 228 | for j=1:size(dif,2) 229 | tDBN = dbn; 230 | tDBN.rbm{n}.W(i,j) = tDBN.rbm{n}.W(i,j) - EP; 231 | er0 = ObjectFunc( tDBN, IN(bind,:), OUT(bind,:), opts ); 232 | tDBN = dbn; 233 | tDBN.rbm{n}.W(i,j) = tDBN.rbm{n}.W(i,j) + EP; 234 | er1 = ObjectFunc( tDBN, IN(bind,:), OUT(bind,:), opts ); 235 | d = (er1-er0)/(2*EP); 236 | dif(i,j) = abs(d - deltaW(i,j) ) / size(OUT,2); 237 | end 238 | end 239 | fprintf( 'max err %d : %g\n', n, max(dif(:)) ); 240 | end 241 | 242 | end 243 | else 244 | trainDBN = dbn; 245 | if( DropOutRate > 0 ) 246 | [trainDBN OnInd] = GetDroppedDBN( trainDBN, DropOutRate, strbm ); 247 | Hall = v2hall( trainDBN, IN(bind,OnInd{1}) ); 248 | else 249 | Hall = v2hall( trainDBN, IN(bind,:) ); 250 | end 251 | 252 | 253 | for n=nrbm:-1:strbm 254 | derSgm = Hall{n} .* ( 1 - Hall{n} ); 255 | if( n+1 > nrbm ) 256 | der = ( Hall{nrbm} - OUT(bind,:) ); 257 | if( Object == OBJECTSQUARE ) 258 | der = derSgm .* der; 259 | end 260 | else 261 | der = derSgm .* ( der * trainDBN.rbm{n+1}.W' ); 262 | end 263 | 264 | if( n-1 > 0 ) 265 | in = Hall{n-1}; 266 | else 267 | if( DropOutRate > 0 ) 268 | in = IN(bind,OnInd{1}); 269 | else 270 | in = IN(bind, :); 271 | end 272 | end 273 | 274 | in = cat(2, ones(numel(bind),1), in); 275 | 276 | deltaWb = in' * der / numel(bind); 277 | deltab = deltaWb(1,:); 278 | deltaW = deltaWb(2:end,:); 279 | 280 | if( strcmpi( dbn.rbm{n}.type, 'GBRBM' ) ) 281 | deltaW = bsxfun( @rdivide, deltaW, trainDBN.rbm{n}.sig' ); 282 | end 283 | 284 | deltaDbn.rbm{n}.W = momentum * deltaDbn.rbm{n}.W; 285 | deltaDbn.rbm{n}.b = momentum * deltaDbn.rbm{n}.b; 286 | 287 | if( DropOutRate > 0 ) 288 | if( n == nrbm ) 289 | deltaDbn.rbm{n}.W(OnInd{n},:) = deltaDbn.rbm{n}.W(OnInd{n},:) - StepRatio * deltaW; 290 | deltaDbn.rbm{n}.b = deltaDbn.rbm{n}.b - StepRatio * deltab; 291 | else 292 | deltaDbn.rbm{n}.W(OnInd{n},OnInd{n+1}) = deltaDbn.rbm{n}.W(OnInd{n},OnInd{n+1}) - StepRatio * deltaW; 293 | deltaDbn.rbm{n}.b(1,OnInd{n+1}) = deltaDbn.rbm{n}.b(1,OnInd{n+1}) - StepRatio * deltab; 294 | end 295 | else 296 | deltaDbn.rbm{n}.W = deltaDbn.rbm{n}.W - StepRatio * deltaW; 297 | deltaDbn.rbm{n}.b = deltaDbn.rbm{n}.b - StepRatio * deltab; 298 | end 299 | 300 | if( debug ) 301 | EP = 1E-8; 302 | dif = zeros(size(trainDBN.rbm{n}.W)); 303 | for i=1:size(dif,1) 304 | for j=1:size(dif,2) 305 | tDBN = trainDBN; 306 | tDBN.rbm{n}.W(i,j) = tDBN.rbm{n}.W(i,j) - EP; 307 | er0 = ObjectFunc( tDBN, IN(bind,:), OUT(bind,:), opts ); 308 | tDBN = trainDBN; 309 | tDBN.rbm{n}.W(i,j) = tDBN.rbm{n}.W(i,j) + EP; 310 | er1 = ObjectFunc( tDBN, IN(bind,:), OUT(bind,:), opts ); 311 | d = (er1-er0)/(2*EP); 312 | dif(i,j) = abs(d - deltaW(i,j) ) / size(OUT,2); 313 | end 314 | end 315 | fprintf( 'max err: %g\n', max(dif(:)) ); 316 | end 317 | 318 | end 319 | end 320 | 321 | for n=strbm:nrbm 322 | dbn.rbm{n}.W = dbn.rbm{n}.W + deltaDbn.rbm{n}.W; 323 | dbn.rbm{n}.b = dbn.rbm{n}.b + deltaDbn.rbm{n}.b; 324 | end 325 | 326 | end 327 | 328 | if( Verbose ) 329 | tdbn = dbn; 330 | if( sum(DropOutRate > 0) ) 331 | OnInd = GetOnInd( tdbn, DropOutRate, strbm ); 332 | for n=max([2,strbm]):nrbm 333 | tdbn.rbm{n}.W = tdbn.rbm{n}.W * numel(OnInd{n-1}) / size(tdbn.rbm{n-1}.W,2); 334 | end 335 | end 336 | out = v2h( tdbn, IN ); 337 | err = power( OUT - out, 2 ); 338 | rmse = sqrt( sum(err(:)) / numel(err) ); 339 | msg = sprintf('%3d : %9.4f', iter, rmse ); 340 | 341 | if( ~isempty( TestIN ) && ~isempty( TestOUT ) ) 342 | out = v2h( tdbn, TestIN ); 343 | err = power( TestOUT - out, 2 ); 344 | rmse = sqrt( sum(err(:)) / numel(err) ); 345 | msg = [msg,' ',sprintf('%9.4f', rmse )]; 346 | end 347 | 348 | totalti = toc(timer); 349 | aveti = totalti / iter; 350 | estti = (MaxIter-iter) * aveti; 351 | eststr = datestr(datenum(0,0,0,0,0,estti),'DD:HH:MM:SS'); 352 | 353 | fprintf( '%s %s\n', msg, eststr ); 354 | if( ~isempty( fp ) ) 355 | fprintf( fp, '%s %s\n', msg, eststr ); 356 | end 357 | end 358 | end 359 | 360 | if( sum(DropOutRate > 0) ) 361 | OnInd = GetOnInd( dbn, DropOutRate, strbm ); 362 | for n=max([2,strbm]):nrbm 363 | dbn.rbm{n}.W = dbn.rbm{n}.W * numel(OnInd{n-1}) / size(dbn.rbm{n-1}.W,2); 364 | end 365 | end 366 | 367 | if( ~isempty( fp ) ) 368 | fclose(fp); 369 | end 370 | 371 | end 372 | 373 | function [del tau] = internal(rbm,IN) 374 | w2 = rbm.W .* rbm.W; 375 | pp = IN .* ( 1-IN ); 376 | mu = bsxfun(@plus, IN * rbm.W, rbm.b ); 377 | s2 = pp * w2; 378 | 379 | tmp = 1 + s2 * (pi / 8); 380 | del = power( tmp, -1/2); 381 | tau = -(pi/16) * mu .* del ./ tmp; 382 | end -------------------------------------------------------------------------------- /trainMNIST.m: -------------------------------------------------------------------------------- 1 | clear all; 2 | addpath('..'); 3 | 4 | mnistfiles = 0; 5 | if( exist('t10k-images-idx3-ubyte', 'file') == 2 ) 6 | mnistfiles = mnistfiles + 1; 7 | end 8 | if( exist('t10k-labels-idx1-ubyte', 'file') == 2 ) 9 | mnistfiles = mnistfiles + 1; 10 | end 11 | if( exist('train-images-idx3-ubyte', 'file') == 2 ) 12 | mnistfiles = mnistfiles + 1; 13 | end 14 | if( exist('train-labels-idx1-ubyte', 'file') == 2 ) 15 | mnistfiles = mnistfiles + 1; 16 | end 17 | 18 | if( mnistfiles < 4 ) 19 | warning('Can not find mnist data files. Please download four data files from http://yann.lecun.com/exdb/mnist/ . Unzip and copy them to same folder as testMNIST.m'); 20 | return; 21 | end 22 | 23 | [TrainImages TrainLabels TestImages TestLabels] = mnistread(); 24 | 25 | % if you want to test with small number of samples, 26 | % please uncomment the following 27 | %{ 28 | TrainNum = 10000; % up to 60000 29 | TestNum = 100; % up to 10000 30 | 31 | TrainImages = TrainImages(1:TrainNum,:); 32 | TrainLabels = TrainLabels(1:TrainNum,:); 33 | 34 | TestImages = TestImages(1:TestNum,:); 35 | TestLabels = TestLabels(1:TestNum,:); 36 | %} 37 | 38 | nodes = [784 800 800 10]; % just example! 39 | bbdbn = randDBN( nodes, 'BBDBN' ); 40 | nrbm = numel(bbdbn.rbm); 41 | 42 | opts.MaxIter = 1000; 43 | opts.BatchSize = 100; 44 | opts.Verbose = true; 45 | opts.StepRatio = 0.1; 46 | opts.object = 'CrossEntropy'; 47 | 48 | opts.Layer = nrbm-1; 49 | bbdbn = pretrainDBN(bbdbn, TrainImages, opts); 50 | bbdbn= SetLinearMapping(bbdbn, TrainImages, TrainLabels); 51 | 52 | opts.Layer = 0; 53 | bbdbn = trainDBN(bbdbn, TrainImages, TrainLabels, opts); 54 | 55 | save('mnistbbdbn.mat', 'bbdbn' ); 56 | 57 | 58 | 59 | 60 | -------------------------------------------------------------------------------- /v2h.m: -------------------------------------------------------------------------------- 1 | % v2h: to transform from visible (input) variables to hidden (output) variables 2 | % 3 | % H = h2v(dnn, V) 4 | % 5 | % 6 | %Output parameters: 7 | % H: hidden (output) variables, where # of row is number of data and # of col is # of hidden (output) nodes 8 | % 9 | % 10 | %Input parameters: 11 | % dnn: the Deep Neural Network model (dbn, rbm) 12 | % V: visible (input) variables, where # of row is number of data and # of col is # of visible (input) nodes 13 | % 14 | % 15 | %Example: 16 | % datanum = 1024; 17 | % outputnum = 16; 18 | % inputnum = 4; 19 | % 20 | % inputdata = rand(datanum, outputnum); 21 | % 22 | % dnn = randRBM( inputnum, outputnum ); 23 | % outputdata = v2h( dnn, input ); 24 | % 25 | % 26 | %Version: 20130830 27 | 28 | 29 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 30 | % Deep Neural Network: % 31 | % % 32 | % Copyright (C) 2013 Masayuki Tanaka. All rights reserved. % 33 | % mtanaka@ctrl.titech.ac.jp % 34 | % % 35 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 36 | function H = v2h(dnn, V) 37 | 38 | ind1 = numel(dnn.type); 39 | ind0 = ind1-2; 40 | type = dnn.type(ind0:ind1); 41 | 42 | if( isequal(dnn.type, 'BBRBM') ) 43 | H = sigmoid( bsxfun(@plus, V * dnn.W, dnn.b ) ); 44 | 45 | elseif( isequal(dnn.type, 'GBRBM') ) 46 | v = bsxfun(@rdivide, V, dnn.sig ); 47 | H = sigmoid( bsxfun(@plus, v * dnn.W, dnn.b ) ); 48 | 49 | elseif( isequal(dnn.type, 'BBPRBM') ) 50 | w2 = dnn.W .* dnn.W; 51 | pp = V .* ( 1-V ); 52 | mu = bsxfun(@plus, V * dnn.W, dnn.b ); 53 | s2 = pp * w2; 54 | H = sigmoid( mu ./ sqrt( 1 + s2 * pi / 8 ) ); 55 | 56 | elseif( isequal(type, 'DBN') ) 57 | nrbm = numel( dnn.rbm ); 58 | H0 = V; 59 | for i=1:nrbm 60 | H1 = v2h( dnn.rbm{i}, H0 ); 61 | H0 = H1; 62 | end 63 | H = H1; 64 | 65 | end 66 | -------------------------------------------------------------------------------- /v2hall.m: -------------------------------------------------------------------------------- 1 | % v2hall: to transform from visible (input) variables to all hidden (output) variables 2 | % 3 | % Hall = h2val(dnn, V) 4 | % 5 | % 6 | %Output parameters: 7 | % Hall: all hidden (output) variables, where # of row is number of data and # of col is # of hidden (output) nodes 8 | % 9 | % 10 | %Input parameters: 11 | % dnn: the Deep Neural Network model (dbn, rbm) 12 | % V: visible (input) variables, where # of row is number of data and # of col is # of visible (input) nodes 13 | % 14 | % 15 | %Version: 20130830 16 | 17 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 18 | % Deep Neural Network: % 19 | % % 20 | % Copyright (C) 2013 Masayuki Tanaka. All rights reserved. % 21 | % mtanaka@ctrl.titech.ac.jp % 22 | % % 23 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 24 | function Hall = v2hall(dnn, V) 25 | 26 | ind1 = numel(dnn.type); 27 | ind0 = ind1-2; 28 | type = dnn.type(ind0:ind1); 29 | 30 | if( isequal(type, 'RBM') ) 31 | Hall = cell(1,1); 32 | Hall{1} = v2h( dnn, V ); 33 | 34 | elseif( isequal(type, 'DBN') ) 35 | nrbm = numel( dnn.rbm ); 36 | Hall = cell(nrbm,1); 37 | H0 = V; 38 | for i=1:nrbm 39 | H1 = v2h( dnn.rbm{i}, H0 ); 40 | H0 = H1; 41 | Hall{i} = H1; 42 | end 43 | end 44 | 45 | --------------------------------------------------------------------------------