├── CG_MNIST.m ├── CG_MNIST_INIT.m ├── Deep-FSpaper.pdf ├── DeepFS.zip ├── DeepFS2.m ├── DeepFS2_small.m ├── MNIST_save.m ├── Mini_DeepFs.m ├── README.md ├── ReadME.txt ├── backprop.m ├── converter.m ├── converter_all.m ├── converter_dexter.m ├── dbm_mf.m ├── dispims.m ├── kullback-Leibler.m ├── kullback_Leibler.m ├── license.txt ├── makebatches.m ├── makebatches_all.m ├── makebatches_dexter.m ├── mf.m ├── mf_class.m ├── minimize.m ├── pvh.m ├── pvh0.m ├── rbm.m ├── rbm_MBF.m ├── rbm_l2.m ├── rbm_test17b2.m └── testerr.m /CG_MNIST.m: -------------------------------------------------------------------------------- 1 | function [f, df] = ECG1(VV,Dim,XX,target,temp_h2); 2 | 3 | numdims = Dim(1); 4 | numhids = Dim(2); 5 | numpens = Dim(3); 6 | N = size(XX,1); 7 | 8 | X=VV; 9 | % Do decomversion. 10 | w1_vishid = reshape(X(1:numdims*numhids),numdims,numhids); 11 | xxx = numdims*numhids; 12 | w1_penhid = reshape(X(xxx+1:xxx+numpens*numhids),numpens,numhids); 13 | xxx = xxx+numpens*numhids; 14 | hidpen = reshape(X(xxx+1:xxx+numhids*numpens),numhids,numpens); 15 | xxx = xxx+numhids*numpens; 16 | w_class = reshape(X(xxx+1:xxx+numpens*10),numpens,10); 17 | xxx = xxx+numpens*10; 18 | hidbiases = reshape(X(xxx+1:xxx+numhids),1,numhids); 19 | xxx = xxx+numhids; 20 | penbiases = reshape(X(xxx+1:xxx+numpens),1,numpens); 21 | xxx = xxx+numpens; 22 | topbiases = reshape(X(xxx+1:xxx+10),1,10); 23 | xxx = xxx+10; 24 | 25 | bias_hid= repmat(hidbiases,N,1); 26 | bias_pen = repmat(penbiases,N,1); 27 | bias_top = repmat(topbiases,N,1); 28 | 29 | w1probs = 1./(1 + exp(-XX*w1_vishid -temp_h2*w1_penhid - bias_hid )); 30 | w2probs = 1./(1 + exp(-w1probs*hidpen - bias_pen)); 31 | targetout = exp(w2probs*w_class + bias_top ); 32 | targetout = targetout./repmat(sum(targetout,2),1,10); 33 | 34 | f = -sum(sum( target(:,1:end).*log(targetout))); 35 | 36 | IO = (targetout-target(:,1:end)); 37 | Ix_class=IO; 38 | dw_class = w2probs'*Ix_class; 39 | dtopbiases = sum(Ix_class); 40 | 41 | Ix2 = (Ix_class*w_class').*w2probs.*(1-w2probs); 42 | dw2_hidpen = w1probs'*Ix2; 43 | dw2_biases = sum(Ix2); 44 | 45 | Ix1 = (Ix2*hidpen').*w1probs.*(1-w1probs); 46 | dw1_penhid = temp_h2'*Ix1; 47 | 48 | dw1_vishid = XX'*Ix1; 49 | dw1_biases = sum(Ix1); 50 | 51 | df = [dw1_vishid(:)' dw1_penhid(:)' dw2_hidpen(:)' dw_class(:)' dw1_biases(:)' dw2_biases(:)' dtopbiases(:)']'; 52 | 53 | 54 | -------------------------------------------------------------------------------- /CG_MNIST_INIT.m: -------------------------------------------------------------------------------- 1 | function [f, df] = ECG1(VV,Dim,XX,target,temp_h2); 2 | 3 | numdims = Dim(1); 4 | numhids = Dim(2); 5 | numpens = Dim(3); 6 | N = size(XX,1); 7 | 8 | X=VV; 9 | % Do decomversion. 10 | w1_vishid = reshape(X(1:numdims*numhids),numdims,numhids); 11 | xxx = numdims*numhids; 12 | w1_penhid = reshape(X(xxx+1:xxx+numpens*numhids),numpens,numhids); 13 | xxx = xxx+numpens*numhids; 14 | hidpen = reshape(X(xxx+1:xxx+numhids*numpens),numhids,numpens); 15 | xxx = xxx+numhids*numpens; 16 | w_class = reshape(X(xxx+1:xxx+numpens*size(target,2)),numpens,size(target,2)); 17 | xxx = xxx+numpens*size(target,2); 18 | hidbiases = reshape(X(xxx+1:xxx+numhids),1,numhids); 19 | xxx = xxx+numhids; 20 | penbiases = reshape(X(xxx+1:xxx+numpens),1,numpens); 21 | xxx = xxx+numpens; 22 | topbiases = reshape(X(xxx+1:xxx+size(target,2)),1,size(target,2)); 23 | 24 | xxx = xxx+size(target,2); 25 | 26 | bias_hid= repmat(hidbiases,N,1); 27 | bias_pen = repmat(penbiases,N,1); 28 | bias_top = repmat(topbiases,N,1); 29 | 30 | w1probs = 1./(1 + exp(-XX*w1_vishid -temp_h2*w1_penhid - bias_hid )); 31 | w2probs = 1./(1 + exp(-w1probs*hidpen - bias_pen)); 32 | targetout = exp(w2probs*w_class + bias_top ); 33 | targetout = targetout./repmat(sum(targetout,2),1,size(target,2)); 34 | 35 | f = -sum(sum( target(:,1:end).*log(targetout))); 36 | 37 | IO = (targetout-target(:,1:end)); 38 | Ix_class=IO; 39 | dw_class = w2probs'*Ix_class; 40 | dtopbiases = sum(Ix_class); 41 | 42 | Ix2 = (Ix_class*w_class').*w2probs.*(1-w2probs); 43 | dw2_hidpen = w1probs'*Ix2; 44 | dw2_biases = sum(Ix2); 45 | 46 | Ix1 = (Ix2*hidpen').*w1probs.*(1-w1probs); 47 | dw1_penhid = temp_h2'*Ix1; 48 | dw1_vishid = XX'*Ix1; 49 | dw1_biases = sum(Ix1); 50 | 51 | dhidpen = 0*dw2_hidpen; 52 | dw1_penhid = 0*dw1_penhid; 53 | dw1_vishid = 0*dw1_vishid; 54 | dw2_biases = 0*dw2_biases; 55 | dw1_biases = 0*dw1_biases; 56 | 57 | df = [dw1_vishid(:)' dw1_penhid(:)' dw2_hidpen(:)' dw_class(:)' dw1_biases(:)' dw2_biases(:)' dtopbiases(:)']'; 58 | 59 | 60 | -------------------------------------------------------------------------------- /Deep-FSpaper.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcosma/Deep-FS/59deb3bd855b98c07e9fad763f4a037dc1eeef25/Deep-FSpaper.pdf -------------------------------------------------------------------------------- /DeepFS.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcosma/Deep-FS/59deb3bd855b98c07e9fad763f4a037dc1eeef25/DeepFS.zip -------------------------------------------------------------------------------- /DeepFS2.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcosma/Deep-FS/59deb3bd855b98c07e9fad763f4a037dc1eeef25/DeepFS2.m -------------------------------------------------------------------------------- /DeepFS2_small.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcosma/Deep-FS/59deb3bd855b98c07e9fad763f4a037dc1eeef25/DeepFS2_small.m -------------------------------------------------------------------------------- /MNIST_save.m: -------------------------------------------------------------------------------- 1 | % first run demo_pixel.m until loading 2 | clc 3 | clear all 4 | close all 5 | 6 | randn('state',100); 7 | rand('state',100); 8 | warning off 9 | 10 | 11 | fprintf(1,'Converting Raw files into Matlab format \n'); 12 | converter; 13 | 14 | fprintf(1,'Pretraining a Deep Boltzmann Machine. \n'); 15 | makebatches; 16 | [numcases numdims numbatches]=size(batchdata); 17 | tic 18 | %%%%%% Training 1st layer %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 19 | % numhid=500; maxepoch=100; 20 | numhid=500; maxepoch=10 21 | 22 | fprintf(1,'Pretraining Layer 1 with RBM: %d-%d \n',numdims,numhid); 23 | restart=1; 24 | %%%%%%%%%%%%%%%%%%%%%%%%%%%% 25 | %training data 26 | trainMNIST =zeros(60000,784); 27 | trainTargets =zeros(size(trainMNIST,1),10); 28 | % trainMNIST = []; 29 | % trainTargets = []; 30 | % trainTargets = []; 31 | for i= 1:size(batchdata,1) 32 | for j=1:size(batchdata,3) 33 | % trainMNIST = [trainMNIST;batchdata(i, :, j)]; 34 | % trainTargets = [trainTargets; batchtargets(i,:,j)]; 35 | trainMNIST((i-1)*600+j,:) =batchdata(i, :, j); 36 | trainTargets((i-1)*600+j,:) =batchtargets (i,:,j); 37 | end 38 | end 39 | % save('trainMNIST.mat', 'trainMNIST') 40 | % save('trainTargets.mat','trainTargets') 41 | testMNIST =zeros(10000,784); 42 | testTargets =zeros(size(testMNIST,1),10); 43 | for i= 1:size(testbatchdata,1) 44 | for j=1:size(testbatchdata,3) 45 | testMNIST((i-1)*100+j,:) =testbatchdata(i, :, j); 46 | testTargets((i-1)*100+j,:) =testbatchtargets (i,:,j); 47 | end 48 | end 49 | % save('testMNIST.mat','testMNIST') 50 | % save('testTargets.mat','testTargets') -------------------------------------------------------------------------------- /Mini_DeepFs.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcosma/Deep-FS/59deb3bd855b98c07e9fad763f4a037dc1eeef25/Mini_DeepFs.m -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Deep-FS 2 | Deep-Feature Selection 3 | ------------------------------------------------------------------------------ 4 | READ ME - Instructions on how to run the code. 5 | ------------------------------------------------------------------------------ 6 | 7 | Download MNIST dataset to the same folder that the MATLAB code are as follows. 8 | 9 | Step 1: Download from http://yann.lecun.com/exdb/mnist the following 4 files: 10 | 11 | o train-images-idx3-ubyte.gz 12 | o train-labels-idx1-ubyte.gz 13 | o t10k-images-idx3-ubyte.gz 14 | o t10k-labels-idx1-ubyte.gz 15 | Step 2: Unzip these 4 files by executing: 16 | o gunzip train-images-idx3-ubyte.gz 17 | o gunzip train-labels-idx1-ubyte.gz 18 | o gunzip t10k-images-idx3-ubyte.gz 19 | o gunzip t10k-labels-idx1-ubyte.gz 20 | 21 | If unzipping with WinZip, make sure the file names have not been changed by WinZip. 22 | 23 | Step 3: Run DeepFS2.m 24 | 25 | Step 4: The program will return two options. 26 | 27 | Enter 1 to run DeepFS to select features, or 28 | Enter 2 to first select features using DeepFS and then train a Deep Boltzmann machine (DBM) on the selected data. 29 | 30 | The principle of DeepFS is described in [1]. The DBM in [2] is used in this code. 31 | 32 | [1] Aboozar Taherkhani, Georgina Cosma, T. M McGinnity, Deep-FS: A feature selection algorithm for Deep Boltzmann Machines, 33 | Neurocomputing, Volume 322, 2018, Pages 22-37, ISSN 0925-2312, https://doi.org/10.1016/j.neucom.2018.09.040. 34 | (http://www.sciencedirect.com/science/article/pii/S0925231218311020) 35 | Keywords: Deep Boltzmann Machine; Deep learning; Deep Neural Networks; Feature selection; Restricted Boltzmann Machine; Generative models; Missing features 36 | 37 | [2] Learning Deep Boltzmann Machines, http://www.cs.toronto.edu/~rsalakhu/DBM.html 38 | 39 | Please e-mail us if you find bugs. 40 | Dr Aboozar Taherkhani: 41 | aboozar.taherkhani@ntu.ac.uk 42 | 43 | Dr Georgina Cosma 44 | georgina.cosma@ntu.ac.uk 45 | -------------------------------------------------------------------------------- /ReadME.txt: -------------------------------------------------------------------------------- 1 | ------------------------------------------------------------------------------ 2 | READ ME - Instructions on how to run the code. 3 | ------------------------------------------------------------------------------ 4 | 5 | Download MNIST dataset to the same folder that the MATLAB code are as follows. 6 | 7 | Step 1: Download from http://yann.lecun.com/exdb/mnist the following 4 files: 8 | 9 | o train-images-idx3-ubyte.gz 10 | o train-labels-idx1-ubyte.gz 11 | o t10k-images-idx3-ubyte.gz 12 | o t10k-labels-idx1-ubyte.gz 13 | Step 2: Unzip these 4 files by executing: 14 | o gunzip train-images-idx3-ubyte.gz 15 | o gunzip train-labels-idx1-ubyte.gz 16 | o gunzip t10k-images-idx3-ubyte.gz 17 | o gunzip t10k-labels-idx1-ubyte.gz 18 | 19 | If unzipping with WinZip, make sure the file names have not been changed by WinZip. 20 | 21 | Step 3: Run DeepFS2.m 22 | 23 | Step 4: The program will return two options. 24 | 25 | Enter 1 to run DeepFS to select features, or 26 | Enter 2 to first select features using DeepFS and then train a Deep Boltzmann machine (DBM) on the selected data. 27 | 28 | The principle of DeepFS is described in [1]. The DBM in [2] is used in this code. 29 | 30 | [1] [1] Aboozar Taherkhani, Georgina Cosma, T. M McGinnity, Deep-FS: A feature selection algorithm for Deep Boltzmann Machines, 31 | Neurocomputing, Volume 322, 2018, Pages 22-37, 32 | ISSN 0925-2312, https://doi.org/10.1016/j.neucom.2018.09.040. 33 | (http://www.sciencedirect.com/science/article/pii/S0925231218311020) 34 | 35 | 36 | [2] Learning Deep Boltzmann Machines, http://www.cs.toronto.edu/~rsalakhu/DBM.html 37 | 38 | 39 | Please e-mail me if us find bugs. 40 | Dr Aboozar Taherkhani: 41 | aboozar.taherkhani@ntu.ac.uk 42 | 43 | Dr Georgina Cosma 44 | georgina.cosma@ntu.ac.uk 45 | -------------------------------------------------------------------------------- /backprop.m: -------------------------------------------------------------------------------- 1 | % Version 1.000 2 | % 3 | % Code provided by Ruslan Salakhutdinov 4 | % 5 | % Permission is granted for anyone to copy, use, modify, or distribute this 6 | % program and accompanying programs and documents for any purpose, provided 7 | % this copyright notice is retained and prominently displayed, along with 8 | % a note saying that the original programs are available from our 9 | % web page. 10 | % The programs and documents are distributed without any warranty, express or 11 | % implied. As the programs were written for research purposes only, they have 12 | % not been tested to the degree that would be advisable in any important 13 | % application. All use of these programs is entirely at the user's own risk. 14 | 15 | 16 | test_err=[]; 17 | test_crerr=[]; 18 | train_err=[]; 19 | train_crerr=[]; 20 | 21 | fprintf(1,'\nTraining discriminative model on MNIST by minimizing cross entropy error. \n'); 22 | fprintf(1,'60 batches of 1000 cases each. \n'); 23 | 24 | [numcases numdims numbatches]=size(batchdata); 25 | N=numcases; 26 | 27 | load fullmnist_dbm 28 | [numdims numhids] = size(vishid); 29 | [numhids numpens] = size(hidpen); 30 | 31 | %%%%%% Preprocess the data %%%%%%%%%%%%%%%%%%%%%% 32 | 33 | [testnumcases testnumdims testnumbatches]=size(testbatchdata); 34 | N=testnumcases; 35 | temp_h2_test = zeros(testnumcases,numpens,testnumbatches); 36 | for batch = 1:testnumbatches 37 | data = [testbatchdata(:,:,batch)]; 38 | [temp_h1, temp_h2] = ... 39 | mf_class(data,vishid,hidbiases,visbiases,hidpen,penbiases); 40 | temp_h2_test(:,:,batch) = temp_h2; 41 | end 42 | 43 | [numcases numdims numbatches]=size(batchdata); 44 | N=numcases; 45 | temp_h2_train = zeros(numcases,numpens,numbatches); 46 | for batch = 1:numbatches 47 | data = [batchdata(:,:,batch)]; 48 | [temp_h1, temp_h2] = ... 49 | mf_class(data,vishid,hidbiases,visbiases,hidpen,penbiases); 50 | temp_h2_train(:,:,batch) = temp_h2; 51 | end 52 | 53 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 54 | 55 | w1_penhid = hidpen'; 56 | w1_vishid = vishid; 57 | w2 = hidpen; 58 | h1_biases = hidbiases; h2_biases = penbiases; 59 | 60 | Odim=size(testbatchtargets,2); 61 | w_class = 0.1*randn(numpens,Odim); 62 | topbiases = 0.1*randn(1,Odim); 63 | 64 | for epoch = 1:maxepoch 65 | 66 | %%%% TEST STATS 67 | %%%% Error rates 68 | [testnumcases testnumdims testnumbatches]=size(testbatchdata); 69 | N=testnumcases; 70 | bias_hid= repmat(h1_biases,N,1); 71 | bias_pen = repmat(h2_biases,N,1); 72 | bias_top = repmat(topbiases,N,1); 73 | 74 | err=0; 75 | err_cr=0; 76 | counter=0; 77 | for batch = 1:testnumbatches 78 | data = [testbatchdata(:,:,batch)]; 79 | temp_h2 = temp_h2_test(:,:,batch); 80 | target = [testbatchtargets(:,:,batch)]; 81 | 82 | w1probs = 1./(1 + exp(-data*w1_vishid -temp_h2*w1_penhid - bias_hid )); 83 | w2probs = 1./(1 + exp(-w1probs*w2 - bias_pen)); 84 | targetout = exp(w2probs*w_class + bias_top ); 85 | targetout = targetout./repmat(sum(targetout,2),1,Odim); 86 | [I J]=max(targetout,[],2); 87 | [I1 J1]=max(target,[],2); 88 | counter=counter+length(find(J~=J1)); 89 | err_cr = err_cr- sum(sum( target(:,1:end).*log(targetout))) ; 90 | end 91 | 92 | test_err(epoch)=counter; 93 | test_crerr(epoch)=err_cr; 94 | fprintf(1,'\nepoch %d test misclassification err %d (out of 10000), test cross entropy error %f \n',epoch,test_err(epoch),test_crerr(epoch)); 95 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 96 | 97 | %%%% TRAINING STATS 98 | %%%% Error rates 99 | [numcases numdims numbatches]=size(batchdata); 100 | N=numcases; 101 | err=0; 102 | err_cr=0; 103 | counter=0; 104 | for batch = 1:numbatches 105 | data = [batchdata(:,:,batch)]; 106 | temp_h2 = temp_h2_train(:,:,batch); 107 | target = [batchtargets(:,:,batch)]; 108 | 109 | w1probs = 1./(1 + exp(-data*w1_vishid -temp_h2*w1_penhid - bias_hid )); 110 | w2probs = 1./(1 + exp(-w1probs*w2 - bias_pen)); 111 | targetout = exp(w2probs*w_class + bias_top ); 112 | targetout = targetout./repmat(sum(targetout,2),1,Odim); 113 | [I J]=max(targetout,[],2); 114 | [I1 J1]=max(target,[],2); 115 | counter=counter+length(find(J~=J1)); 116 | 117 | err_cr = err_cr- sum(sum( target(:,1:end).*log(targetout))) ; 118 | end 119 | 120 | train_err(epoch)=counter; 121 | train_crerr(epoch)=err_cr; 122 | fprintf(1,'epoch %d train misclassification err %d train (out of 60000), train cross entropy error %f \n',epoch, train_err(epoch),train_crerr(epoch)); 123 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 124 | 125 | save backprop_weights w1_vishid w1_penhid w2 w_class h1_biases h2_biases topbiases test_err test_crerr train_err train_crerr 126 | 127 | %%% Do Conjugate Gradient Optimization 128 | 129 | % rr = randperm(600); 130 | % for batch = 1:numbatches/100 131 | % %fprintf(1,'epoch %d batch %d\r',epoch,batch); 132 | % data = zeros(10000,numdims); 133 | % temp_h2 = zeros(10000,numpens); 134 | % targets = zeros(10000,10); 135 | % tt1=(batch-1)*100+1:batch*100; 136 | % for tt=1:100 137 | % data( (tt-1)*100+1:tt*100,:) = batchdata(:,:,rr(tt1(tt))); 138 | % temp_h2( (tt-1)*100+1:tt*100,:) = temp_h2_train(:,:,rr(tt1(tt))); 139 | % targets( (tt-1)*100+1:tt*100,:) = batchtargets(:,:,rr(tt1(tt))); 140 | % end 141 | 142 | rr = randperm(numbatches); 143 | for batch = 1:numbatches/numcases 144 | %fprintf(1,'epoch %d batch %d\r',epoch,batch); 145 | data = zeros(numcases^2,numdims); 146 | temp_h2 = zeros(numcases^2,numpens); 147 | targets = zeros(numcases^2,size(batchtargets,2)); 148 | tt1=(batch-1)*numcases+1:batch*numcases; 149 | for tt=1:numcases 150 | data( (tt-1)*numcases+1:tt*numcases,:) = batchdata(:,:,rr(tt1(tt))); 151 | temp_h2( (tt-1)*numcases+1:tt*numcases,:) = temp_h2_train(:,:,rr(tt1(tt))); 152 | targets( (tt-1)*numcases+1:tt*numcases,:) = batchtargets(:,:,rr(tt1(tt))); 153 | end 154 | 155 | %%%%%%%% DO CG with 3 linesearches 156 | 157 | VV = [w1_vishid(:)' w1_penhid(:)' w2(:)' w_class(:)' h1_biases(:)' h2_biases(:)' topbiases(:)']'; 158 | Dim = [numdims; numhids; numpens; ]; 159 | 160 | % checkgrad('CG_MNIST_INIT',VV,10^-5,Dim,data,targets); 161 | max_iter=3; 162 | if epoch<6 163 | [X, fX, num_iter,ecg_XX] = minimize(VV,'CG_MNIST_INIT',max_iter,Dim,data,targets,temp_h2); 164 | else 165 | [X, fX, num_iter,ecg_XX] = minimize(VV,'CG_MNIST',max_iter,Dim,data,targets,temp_h2); 166 | end 167 | w1_vishid = reshape(X(1:numdims*numhids),numdims,numhids); 168 | xxx = numdims*numhids; 169 | w1_penhid = reshape(X(xxx+1:xxx+numpens*numhids),numpens,numhids); 170 | xxx = xxx+numpens*numhids; 171 | w2 = reshape(X(xxx+1:xxx+numhids*numpens),numhids,numpens); 172 | xxx = xxx+numhids*numpens; 173 | w_class = reshape(X(xxx+1:xxx+numpens*10),numpens,10); 174 | xxx = xxx+numpens*10; 175 | h1_biases = reshape(X(xxx+1:xxx+numhids),1,numhids); 176 | xxx = xxx+numhids; 177 | h2_biases = reshape(X(xxx+1:xxx+numpens),1,numpens); 178 | xxx = xxx+numpens; 179 | topbiases = reshape(X(xxx+1:xxx+10),1,10); 180 | xxx = xxx+10; 181 | 182 | end 183 | 184 | end 185 | err_III=[test_err(epoch), train_err(epoch)]; 186 | 187 | 188 | -------------------------------------------------------------------------------- /converter.m: -------------------------------------------------------------------------------- 1 | % Version 1.000 2 | % 3 | % Code provided by Ruslan Salakhutdinov 4 | % 5 | % Permission is granted for anyone to copy, use, modify, or distribute this 6 | % program and accompanying programs and documents for any purpose, provided 7 | % this copyright notice is retained and prominently displayed, along with 8 | % a note saying that the original programs are available from our 9 | % web page. 10 | % The programs and documents are distributed without any warranty, express or 11 | % implied. As the programs were written for research purposes only, they have 12 | % not been tested to the degree that would be advisable in any important 13 | % application. All use of these programs is entirely at the user's own risk. 14 | 15 | % This program reads raw MNIST files available at 16 | % http://yann.lecun.com/exdb/mnist/ 17 | % and converts them to files in matlab format 18 | % Before using this program you first need to download files: 19 | % train-images-idx3-ubyte.gz train-labels-idx1-ubyte.gz 20 | % t10k-images-idx3-ubyte.gz t10k-labels-idx1-ubyte.gz 21 | % and gunzip them. You need to allocate some space for this. 22 | 23 | % This program was originally written by Yee Whye Teh 24 | 25 | % Work with test files first 26 | fprintf(1,'You first need to download files:\n train-images-idx3-ubyte.gz\n train-labels-idx1-ubyte.gz\n t10k-images-idx3-ubyte.gz\n t10k-labels-idx1-ubyte.gz\n from http://yann.lecun.com/exdb/mnist/\n and gunzip them \n'); 27 | 28 | f = fopen('t10k-images-idx3-ubyte','r'); 29 | [a,count] = fread(f,4,'int32'); 30 | 31 | g = fopen('t10k-labels-idx1-ubyte','r'); 32 | [l,count] = fread(g,2,'int32'); 33 | 34 | fprintf(1,'Starting to convert Test MNIST images (prints 10 dots) \n'); 35 | n = 1000; 36 | 37 | Df = cell(1,10); 38 | for d=0:9, 39 | Df{d+1} = fopen(['test' num2str(d) '.ascii'],'w'); 40 | end; 41 | 42 | for i=1:10, 43 | fprintf('.'); 44 | rawimages = fread(f,28*28*n,'uchar'); 45 | rawlabels = fread(g,n,'uchar'); 46 | rawimages = reshape(rawimages,28*28,n); 47 | 48 | for j=1:n, 49 | fprintf(Df{rawlabels(j)+1},'%3d ',rawimages(:,j)); 50 | fprintf(Df{rawlabels(j)+1},'\n'); 51 | end; 52 | end; 53 | 54 | fprintf(1,'\n'); 55 | for d=0:9, 56 | fclose(Df{d+1}); 57 | D = load(['test' num2str(d) '.ascii'],'-ascii'); 58 | fprintf('%5d Digits of class %d\n',size(D,1),d); 59 | save(['test' num2str(d) '.mat'],'D','-mat'); 60 | end; 61 | 62 | 63 | % Work with trainig files second 64 | f = fopen('train-images-idx3-ubyte','r'); 65 | [a,count] = fread(f,4,'int32'); 66 | 67 | g = fopen('train-labels-idx1-ubyte','r'); 68 | [l,count] = fread(g,2,'int32'); 69 | 70 | fprintf(1,'Starting to convert Training MNIST images (prints 60 dots)\n'); 71 | n = 1000; 72 | 73 | Df = cell(1,10); 74 | for d=0:9, 75 | Df{d+1} = fopen(['digit' num2str(d) '.ascii'],'w'); 76 | end; 77 | 78 | for i=1:60, 79 | fprintf('.'); 80 | rawimages = fread(f,28*28*n,'uchar'); 81 | rawlabels = fread(g,n,'uchar'); 82 | rawimages = reshape(rawimages,28*28,n); 83 | 84 | for j=1:n, 85 | fprintf(Df{rawlabels(j)+1},'%3d ',rawimages(:,j)); 86 | fprintf(Df{rawlabels(j)+1},'\n'); 87 | end; 88 | end; 89 | 90 | fprintf(1,'\n'); 91 | for d=0:9, 92 | fclose(Df{d+1}); 93 | D = load(['digit' num2str(d) '.ascii'],'-ascii'); 94 | fprintf('%5d Digits of class %d\n',size(D,1),d); 95 | save(['digit' num2str(d) '.mat'],'D','-mat'); 96 | end; 97 | 98 | dos('rm *.ascii'); 99 | -------------------------------------------------------------------------------- /converter_all.m: -------------------------------------------------------------------------------- 1 | % Version 1.000 2 | % Permission is granted for anyone to copy, use, modify, or distribute this 3 | % program and accompanying programs and documents for any purpose, provided 4 | % this copyright notice is retained and prominently displayed, along with 5 | % a note saying that the original programs are available from our 6 | % web page. 7 | % The programs and documents are distributed without any warranty, express or 8 | % implied. As the programs were written for research purposes only, they have 9 | % not been tested to the degree that would be advisable in any important 10 | % application. All use of these programs is entirely at the user's own risk. 11 | 12 | %%%%%%%%%%%%%%%load dexter: 1) training dataset 13 | % [classes, digitdata] = load_dexter();%load train dat 14 | %%%%%load other: 15 | % dataPath1 = {'\ARCENE\arcene', '\DEXTER\dexter', '\DOROTHEA\dorothea', '\GISETTE\gisette', '\MADELON\madelon'} 16 | 17 | fprintf('load %s ... \n', selection_method) 18 | dataPath1 =['\' upper(selection_method) '\' lower(selection_method) ]; 19 | filename = [rootPath dataPath1]; 20 | 21 | load(filename) 22 | %addpath('C:\Users\ERD204\OneDrive - Nottingham Trent University\Deep_FS_MATLA_Neurocomputing\MATLAB\DataNcode') 23 | 24 | % data_stats(Data);%ad DataNcode to path the 25 | 26 | X_train=Data.train.X; 27 | Y_train = Data.train.Y; 28 | %a=[X_train, Y_train]; 29 | %merg with validation data 30 | % X_train = [X_train; Data.valid.X]; 31 | % Y_train = [Y_train; Data.valid.Y]; 32 | X_valid = Data.valid.X; 33 | Y_valid = Data.valid.Y; 34 | 35 | % test data 36 | X_test = Data.test.X; 37 | Y_test = Data.test.Y; 38 | %% 39 | % switch lower(selection_method) 40 | % case 'dexter' 41 | % [classes, digitdata] = load_dexter();%load train dat 42 | % case 'arcene' 43 | % 44 | % otherwise 45 | % disp('Unknown dataset,') 46 | % end 47 | %%%%%%%%%%%%Remove fitutre vector wich is zero: 48 | 49 | ind=[]; 50 | mmm=[]; 51 | for j=1: size(X_train,2) 52 | if max(X_train(:,j)) == min (X_train(:,j)) 53 | ind=[ind, j]; 54 | % mmm= [mmm,max(digitdata(:,j))]; 55 | 56 | end 57 | end 58 | % digitdata2 = digitdata; 59 | X_train(:,ind)=[]; 60 | X_valid(:,ind)=[]; 61 | 62 | X_test(:,ind)=[]; 63 | 64 | %%%%%%%%%%%%% 65 | % save('dexter_train.mat', 'Y_train', 'X_train') 66 | 67 | filenamSave=selection_method; 68 | save([filenamSave '_train.mat'], 'Y_train', 'X_train') 69 | clear Y_train X_train; 70 | 71 | %%%%%%%%%%%%%%%2) validation dataset 72 | % [val_classes, val_digitdata] = load_dexter_valid();% load validation data 73 | % val_digitdata(:,ind)=[]; 74 | % 75 | % save('dexter_validation.mat', 'val_classes', 'val_digitdata') 76 | % clear val_classes val_digitdata 77 | 78 | save([filenamSave '_validation.mat'], 'Y_valid', 'X_valid') 79 | clear Y_valid X_valid; 80 | 81 | 82 | % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 83 | 84 | % function [classes, vectors] = load_dexter() 85 | % % fprintf('\nNO PARAMETERS GIVEN! Loading & evaluating DEXTER data set.\n\n'); 86 | % fprintf('\n Loading DEXTER data set.\n\n'); 87 | % fprintf('DEXTER is a text classification problem in a bag-of-word\n'); 88 | % fprintf('representation. This is a two-class classification problem\n'); 89 | % fprintf('with sparse continuous input variables.\n'); 90 | % fprintf('This dataset is one of five datasets of the NIPS 2003 feature\n'); 91 | % fprintf('selection challenge.\n\n'); 92 | % 93 | % fprintf('http://archive.ics.uci.edu/ml/datasets/Dexter\n\n'); 94 | % 95 | % n = 300; 96 | % dim = 20000; 97 | % 98 | % vectors = zeros(n, dim); 99 | % classes = zeros(n, 1); 100 | % 101 | % 102 | % % fid = fopen('example_datasets/dexter_train.data', 'r'); 103 | % % fid = fopen('C:\Users\cmp3tahera\OneDrive - Nottingham Trent University\Deep_FS_MATLA_Neurocomputing\MATLAB\hub-toolbox-matlab-master\DEXTER\DEXTER\dexter_train.data', 'r'); 104 | % fid = fopen('C:\Users\ERD204\OneDrive - Nottingham Trent University\Deep_FS_MATLA_Neurocomputing\MATLAB\hub-toolbox-matlab-master\DEXTER\DEXTER\dexter_train.data', 'r'); 105 | % 106 | % 107 | % for i=1:n 108 | % tline = fgetl(fid); 109 | % d = sscanf(tline, '%d:%d'); 110 | % vectors(i,d(1:2:end)) = d(2:2:end); 111 | % end 112 | % fclose(fid); 113 | % 114 | % % fid = fopen('example_datasets/dexter_train.labels', 'r'); 115 | % % fid = fopen('C:\Users\cmp3tahera\OneDrive - Nottingham Trent University\Deep_FS_MATLA_Neurocomputing\MATLAB\hub-toolbox-matlab-master\DEXTER\DEXTER\dexter_train.labels', 'r'); 116 | % fid = fopen('C:\Users\ERD204\OneDrive - Nottingham Trent University\Deep_FS_MATLA_Neurocomputing\MATLAB\hub-toolbox-matlab-master\DEXTER\DEXTER\dexter_train.labels', 'r'); 117 | % 118 | % for i=1:n 119 | % tline = fgetl(fid); 120 | % d = sscanf(tline, '%d'); 121 | % classes(i) = d(1); 122 | % end 123 | % fclose(fid); 124 | % 125 | % % D = cosine_distance(vectors); 126 | % 127 | % end 128 | % 129 | % 130 | % 131 | % function [classes, vectors] = load_dexter_valid() 132 | % % fprintf('\nNO PARAMETERS GIVEN! Loading & evaluating DEXTER data set.\n\n'); 133 | % fprintf('\n Loading DEXTER data set.\n\n'); 134 | % fprintf('DEXTER is a text classification problem in a bag-of-word\n'); 135 | % fprintf('representation. This is a two-class classification problem\n'); 136 | % fprintf('with sparse continuous input variables.\n'); 137 | % fprintf('This dataset is one of five datasets of the NIPS 2003 feature\n'); 138 | % fprintf('selection challenge.\n\n'); 139 | % 140 | % fprintf('http://archive.ics.uci.edu/ml/datasets/Dexter\n\n'); 141 | % 142 | % n = 300; 143 | % dim = 20000; 144 | % 145 | % vectors = zeros(n, dim); 146 | % classes = zeros(n, 1); 147 | % 148 | % 149 | % % fid = fopen('example_datasets/dexter_train.data', 'r'); 150 | % % fid = fopen('C:\Users\cmp3tahera\OneDrive - Nottingham Trent University\Deep_FS_MATLA_Neurocomputing\MATLAB\hub-toolbox-matlab-master\DEXTER\DEXTER\dexter_train.data', 'r'); 151 | % %vlidation dat: 152 | % fid = fopen('C:\Users\cmp3tahera\OneDrive - Nottingham Trent University\Deep_FS_MATLA_Neurocomputing\MATLAB\hub-toolbox-matlab-master\DEXTER\DEXTER\dexter_valid.data', 'r'); 153 | % 154 | % for i=1:n 155 | % tline = fgetl(fid); 156 | % d = sscanf(tline, '%d:%d'); 157 | % vectors(i,d(1:2:end)) = d(2:2:end); 158 | % end 159 | % fclose(fid); 160 | % 161 | % % fid = fopen('example_datasets/dexter_train.labels', 'r'); 162 | % fid = fopen('C:\Users\cmp3tahera\OneDrive - Nottingham Trent University\Deep_FS_MATLA_Neurocomputing\MATLAB\hub-toolbox-matlab-master\DEXTER\dexter_valid.labels', 'r'); 163 | % 164 | % for i=1:n 165 | % tline = fgetl(fid); 166 | % d = sscanf(tline, '%d'); 167 | % classes(i) = d(1); 168 | % end 169 | % fclose(fid); 170 | % 171 | % % D = cosine_distance(vectors); 172 | % 173 | % end 174 | 175 | 176 | 177 | -------------------------------------------------------------------------------- /converter_dexter.m: -------------------------------------------------------------------------------- 1 | % Version 1.000 2 | % Permission is granted for anyone to copy, use, modify, or distribute this 3 | % program and accompanying programs and documents for any purpose, provided 4 | % this copyright notice is retained and prominently displayed, along with 5 | % a note saying that the original programs are available from our 6 | % web page. 7 | % The programs and documents are distributed without any warranty, express or 8 | % implied. As the programs were written for research purposes only, they have 9 | % not been tested to the degree that would be advisable in any important 10 | % application. All use of these programs is entirely at the user's own risk. 11 | 12 | %%%%%%%%%%%%%%%load dexter: 1) training dataset 13 | [classes, digitdata] = load_dexter();%load train dat 14 | %%%%%%%%%%%%Remove fitutre vector wich is zero: 15 | 16 | ind=[]; 17 | mmm=[]; 18 | for j=1: 20000 19 | if max(digitdata(:,j)) == min (digitdata(:,j)) 20 | ind=[ind, j]; 21 | % mmm= [mmm,max(digitdata(:,j))]; 22 | 23 | end 24 | end 25 | % digitdata2 = digitdata; 26 | digitdata(:,ind)=[]; 27 | 28 | %%%%%%%%%%%%% 29 | save('dexter_train.mat', 'classes', 'digitdata') 30 | clear classes digitdata; 31 | % maxV=max(max(digitdata)); 32 | % digitdata = digitdata/maxV; 33 | % 34 | % targets = tocategorical(classes); 35 | % 36 | % totnum=size(digitdata,1); 37 | % fprintf(1, 'Size of the training dataset= %5d \n', totnum); 38 | % 39 | % rand('state',0); %so we know the permutation of the training data 40 | % batchsize = 100 41 | % [batchtargets, batchdata] = puInBatch(targets, digitdata, batchsize); 42 | % clear digitdata targets; 43 | 44 | %%%%%%%%%%%%%%%2) validation dataset 45 | [val_classes, val_digitdata] = load_dexter_valid();% load validation data 46 | val_digitdata(:,ind)=[]; 47 | 48 | save('dexter_validation.mat', 'val_classes', 'val_digitdata') 49 | clear val_classes val_digitdata 50 | 51 | % maxV=max(max(digitdata)); 52 | % digitdata = digitdata/maxV; 53 | % 54 | % targets = tocategorical(classes); 55 | % 56 | % totnum=size(digitdata,1); 57 | % fprintf(1, 'Size of the validation dataset= %5d \n', totnum); 58 | % [testbatchtargets, testbatchdata] = puInBatch(targets, digitdata, batchsize); 59 | % clear digitdata targets; 60 | % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 61 | % 62 | % %%% Reset random seeds 63 | % rand('state',sum(100*clock)); 64 | % randn('state',sum(100*clock)); 65 | 66 | function [classes, vectors] = load_dexter() 67 | % fprintf('\nNO PARAMETERS GIVEN! Loading & evaluating DEXTER data set.\n\n'); 68 | fprintf('\n Loading DEXTER data set.\n\n'); 69 | fprintf('DEXTER is a text classification problem in a bag-of-word\n'); 70 | fprintf('representation. This is a two-class classification problem\n'); 71 | fprintf('with sparse continuous input variables.\n'); 72 | fprintf('This dataset is one of five datasets of the NIPS 2003 feature\n'); 73 | fprintf('selection challenge.\n\n'); 74 | 75 | fprintf('http://archive.ics.uci.edu/ml/datasets/Dexter\n\n'); 76 | 77 | n = 300; 78 | dim = 20000; 79 | 80 | vectors = zeros(n, dim); 81 | classes = zeros(n, 1); 82 | 83 | 84 | % fid = fopen('example_datasets/dexter_train.data', 'r'); 85 | % fid = fopen('C:\Users\cmp3tahera\OneDrive - Nottingham Trent University\Deep_FS_MATLA_Neurocomputing\MATLAB\hub-toolbox-matlab-master\DEXTER\DEXTER\dexter_train.data', 'r'); 86 | fid = fopen('C:\Users\ERD204\OneDrive - Nottingham Trent University\Deep_FS_MATLA_Neurocomputing\MATLAB\hub-toolbox-matlab-master\DEXTER\DEXTER\dexter_train.data', 'r'); 87 | 88 | 89 | for i=1:n 90 | tline = fgetl(fid); 91 | d = sscanf(tline, '%d:%d'); 92 | vectors(i,d(1:2:end)) = d(2:2:end); 93 | end 94 | fclose(fid); 95 | 96 | % fid = fopen('example_datasets/dexter_train.labels', 'r'); 97 | % fid = fopen('C:\Users\cmp3tahera\OneDrive - Nottingham Trent University\Deep_FS_MATLA_Neurocomputing\MATLAB\hub-toolbox-matlab-master\DEXTER\DEXTER\dexter_train.labels', 'r'); 98 | fid = fopen('C:\Users\ERD204\OneDrive - Nottingham Trent University\Deep_FS_MATLA_Neurocomputing\MATLAB\hub-toolbox-matlab-master\DEXTER\DEXTER\dexter_train.labels', 'r'); 99 | 100 | for i=1:n 101 | tline = fgetl(fid); 102 | d = sscanf(tline, '%d'); 103 | classes(i) = d(1); 104 | end 105 | fclose(fid); 106 | 107 | % D = cosine_distance(vectors); 108 | 109 | end 110 | 111 | % function [targets] = tocategorical(classes) 112 | % targets = []; 113 | % for i=1: length(classes) 114 | % if classes(i) == -1 115 | % targets = [targets;[1 0]]; 116 | % elseif classes(i) == 1 117 | % targets = [targets; [0 1]]; 118 | % end 119 | % end 120 | % end 121 | 122 | % function [batchtargets, batchdata] = puInBatch(targets, digitdata, batchsize) 123 | % %get the tardet, data and batch size and put in in batches with the size of 124 | % %batchsize 125 | % totnum=size(digitdata,1); 126 | % randomorder=randperm(totnum); 127 | % 128 | % % batchsize = 100; 129 | % numbatches=totnum/batchsize; 130 | % numdims = size(digitdata,2); 131 | % batchdata = zeros(batchsize, numdims, numbatches); 132 | % batchtargets = zeros(batchsize,size(targets, 2), numbatches); 133 | % 134 | % for b=1:numbatches 135 | % batchdata(:,:,b) = digitdata(randomorder(1+(b-1)*batchsize:b*batchsize), :); 136 | % batchtargets(:,:,b) = targets(randomorder(1+(b-1)*batchsize:b*batchsize), :); 137 | % end 138 | % % clear digitdata targets; 139 | % end 140 | 141 | function [classes, vectors] = load_dexter_valid() 142 | % fprintf('\nNO PARAMETERS GIVEN! Loading & evaluating DEXTER data set.\n\n'); 143 | fprintf('\n Loading DEXTER data set.\n\n'); 144 | fprintf('DEXTER is a text classification problem in a bag-of-word\n'); 145 | fprintf('representation. This is a two-class classification problem\n'); 146 | fprintf('with sparse continuous input variables.\n'); 147 | fprintf('This dataset is one of five datasets of the NIPS 2003 feature\n'); 148 | fprintf('selection challenge.\n\n'); 149 | 150 | fprintf('http://archive.ics.uci.edu/ml/datasets/Dexter\n\n'); 151 | 152 | n = 300; 153 | dim = 20000; 154 | 155 | vectors = zeros(n, dim); 156 | classes = zeros(n, 1); 157 | 158 | 159 | % fid = fopen('example_datasets/dexter_train.data', 'r'); 160 | % fid = fopen('C:\Users\cmp3tahera\OneDrive - Nottingham Trent University\Deep_FS_MATLA_Neurocomputing\MATLAB\hub-toolbox-matlab-master\DEXTER\DEXTER\dexter_train.data', 'r'); 161 | %vlidation dat: 162 | fid = fopen('C:\Users\cmp3tahera\OneDrive - Nottingham Trent University\Deep_FS_MATLA_Neurocomputing\MATLAB\hub-toolbox-matlab-master\DEXTER\DEXTER\dexter_valid.data', 'r'); 163 | 164 | for i=1:n 165 | tline = fgetl(fid); 166 | d = sscanf(tline, '%d:%d'); 167 | vectors(i,d(1:2:end)) = d(2:2:end); 168 | end 169 | fclose(fid); 170 | 171 | % fid = fopen('example_datasets/dexter_train.labels', 'r'); 172 | fid = fopen('C:\Users\cmp3tahera\OneDrive - Nottingham Trent University\Deep_FS_MATLA_Neurocomputing\MATLAB\hub-toolbox-matlab-master\DEXTER\dexter_valid.labels', 'r'); 173 | 174 | for i=1:n 175 | tline = fgetl(fid); 176 | d = sscanf(tline, '%d'); 177 | classes(i) = d(1); 178 | end 179 | fclose(fid); 180 | 181 | % D = cosine_distance(vectors); 182 | 183 | end 184 | 185 | 186 | 187 | -------------------------------------------------------------------------------- /dbm_mf.m: -------------------------------------------------------------------------------- 1 | % Version 1.000 2 | % 3 | % Code provided by Ruslan Salakhutdinov 4 | % 5 | % Permission is granted for anyone to copy, use, modify, or distribute this 6 | % program and accompanying programs and documents for any purpose, provided 7 | % this copyright notice is retained and prominently displayed, along with 8 | % a note saying that the original programs are available from our 9 | % web page. 10 | % The programs and documents are distributed without any warranty, express or 11 | % implied. As the programs were written for research purposes only, they have 12 | % not been tested to the degree that would be advisable in any important 13 | % application. All use of these programs is entirely at the user's own risk. 14 | 15 | close all 16 | if restart ==1, 17 | epsilonw = 0.001; % Learning rate for weights 18 | epsilonvb = 0.001; % Learning rate for biases of visible units 19 | epsilonhb = 0.001; % Learning rate for biases of hidden units 20 | weightcost = 0.0002; 21 | initialmomentum = 0.5; 22 | finalmomentum = 0.9; 23 | 24 | [numcases numdims numbatches]=size(batchdata); 25 | 26 | % numlab=10; 27 | if exist('targets')==1 28 | numlab = size(targets,2);%other 29 | else 30 | numlab = size(batchtargets,2);%%%%MNIST 31 | end 32 | numdim=numdims; 33 | 34 | restart=0; 35 | epoch=1; 36 | % Initializing symmetric weights and biases. 37 | 38 | vishid = 0.001*randn(numdim, numhid); 39 | hidpen = 0.001*randn(numhid,numpen); 40 | 41 | labpen = 0.001*randn(numlab,numpen); 42 | 43 | hidbiases = zeros(1,numhid); 44 | visbiases = zeros(1,numdim); 45 | penbiases = zeros(1,numpen); 46 | labbiases = zeros(1,numlab); 47 | 48 | poshidprobs = zeros(numcases,numhid); 49 | neghidprobs = zeros(numcases,numhid); 50 | posprods = zeros(numdim,numhid); 51 | negprods = zeros(numdim,numhid); 52 | 53 | 54 | vishidinc = zeros(numdim,numhid); 55 | hidpeninc = zeros(numhid,numpen); 56 | labpeninc = zeros(numlab,numpen); 57 | 58 | 59 | hidbiasinc = zeros(1,numhid); 60 | visbiasinc = zeros(1,numdim); 61 | penbiasinc = zeros(1,numpen); 62 | labbiasinc = zeros(1,numlab); 63 | 64 | %%%% This code also adds sparcity penalty 65 | sparsetarget = .2; 66 | sparsetarget2 = .1; 67 | sparsecost = .001; 68 | sparsedamping = .9; 69 | 70 | hidbiases = 0*log(sparsetarget/(1-sparsetarget))*ones(1,numhid); 71 | hidmeans = sparsetarget*ones(1,numhid); 72 | penbiases = 0*log(sparsetarget2/(1-sparsetarget2))*ones(1,numpen); 73 | penmeans = sparsetarget2*ones(1,numpen); 74 | 75 | load fullmnistpo.mat 76 | 77 | hidpen = vishid; 78 | penbiases = hidbiases; 79 | visbiases_l2 = visbiases; 80 | labpen = labhid; 81 | clear labhid; 82 | 83 | load fullmnistvh.mat 84 | hidrecbiases = hidbiases; 85 | hidbiases = (hidbiases + visbiases_l2); 86 | epoch=1; 87 | 88 | neghidprobs = (rand(numcases,numhid)); 89 | neglabstates = 1/10*(ones(numcases,numlab)); 90 | data = round(rand(numcases,numdims)); 91 | neghidprobs = 1./(1 + exp(-data*(2*vishid) - repmat(hidbiases,numcases,1))); 92 | 93 | epsilonw = epsilonw/(1.000015^((epoch-1)*600)); 94 | epsilonvb = epsilonvb/(1.000015^((epoch-1)*600)); 95 | epsilonhb = epsilonhb/(1.000015^((epoch-1)*600)); 96 | 97 | tot = 0; 98 | end 99 | 100 | for epoch = epoch:maxepoch 101 | [numcases numdims numbatches]=size(batchdata); 102 | 103 | % fprintf(1,'epoch %d \t eps %f\r',epoch,epsilonw); 104 | errsum=0; 105 | 106 | [numcases numdims numbatches]=size(batchdata); 107 | 108 | counter=0; 109 | rr = randperm(numbatches); 110 | batch=0; 111 | for batch_rr = rr; %1:numbatches, 112 | batch=batch+1; 113 | %fprintf(1,'epoch %d batch %d\r',epoch,batch); 114 | tot=tot+1; 115 | epsilonw = max(epsilonw/1.000015,0.00010); 116 | epsilonvb = max(epsilonvb/1.000015,0.00010); 117 | epsilonhb = max(epsilonhb/1.000015,0.00010); 118 | 119 | 120 | %%%%%%%%% START POSITIVE PHASE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 121 | data = batchdata(:,:,batch); 122 | targets = batchtargets(:,:,batch); 123 | data = double(data > rand(numcases,numdim)); 124 | 125 | %%%%% First fo MF %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 126 | [poshidprobs, pospenprobs] = ... 127 | mf(data,targets,vishid,hidbiases,visbiases,hidpen,penbiases,labpen,hidrecbiases); 128 | 129 | 130 | bias_hid= repmat(hidbiases,numcases,1); 131 | bias_pen = repmat(penbiases,numcases,1); 132 | bias_vis = repmat(visbiases,numcases,1); 133 | bias_lab = repmat(labbiases,numcases,1); 134 | 135 | posprods = data' * poshidprobs; 136 | posprodspen = poshidprobs'*pospenprobs; 137 | posprodslabpen = targets'*pospenprobs; 138 | 139 | poshidact = sum(poshidprobs); 140 | pospenact = sum(pospenprobs); 141 | poslabact = sum(targets); 142 | posvisact = sum(data); 143 | 144 | 145 | %%%%%%%%% END OF POSITIVE PHASE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 146 | 147 | negdata_CD1 = 1./(1 + exp(-poshidprobs*vishid' - bias_vis)); 148 | totin = bias_lab + pospenprobs*labpen'; 149 | poslabprobs1 = exp(totin); 150 | targetout = poslabprobs1./(sum(poslabprobs1,2)*ones(1,numlab)); 151 | [I J]=max(targetout,[],2); 152 | [I1 J1]=max(targets,[],2); 153 | counter=counter+length(find(J==J1)); 154 | 155 | 156 | 157 | %%%%% START NEGATIVE PHASE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 158 | for iter=1:5 159 | neghidstates = neghidprobs > rand(numcases,numhid); 160 | 161 | negpenprobs = 1./(1 + exp(-neghidstates*hidpen - neglabstates*labpen - bias_pen)); 162 | negpenstates = negpenprobs > rand(numcases,numpen); 163 | 164 | negdataprobs = 1./(1 + exp(-neghidstates*vishid' - bias_vis)); 165 | negdata = negdataprobs > rand(numcases,numdim); 166 | 167 | totin = negpenstates*labpen' + bias_lab; 168 | neglabprobs = exp(totin); 169 | neglabprobs = neglabprobs./(sum(neglabprobs,2)*ones(1,numlab)); 170 | 171 | xx = cumsum(neglabprobs,2); 172 | xx1 = rand(numcases,1); 173 | neglabstates = neglabstates*0; 174 | for jj=1:numcases 175 | index = min(find(xx1(jj) <= xx(jj,:))); 176 | neglabstates(jj,index) = 1; 177 | end 178 | xxx = sum(sum(neglabstates)) ; 179 | 180 | totin = negdata*vishid + bias_hid + negpenstates*hidpen'; 181 | neghidprobs = 1./(1 + exp(-totin)); 182 | 183 | end 184 | negpenprobs = 1./(1 + exp(-neghidprobs*hidpen - neglabprobs*labpen - bias_pen)); 185 | 186 | negprods = negdata'*neghidprobs; 187 | negprodspen = neghidprobs'*negpenprobs; 188 | neghidact = sum(neghidprobs); 189 | negpenact = sum(negpenprobs); 190 | negvisact = sum(negdata); 191 | neglabact = sum(neglabstates); 192 | negprodslabpen = neglabstates'*negpenprobs; 193 | 194 | 195 | %%%%%%%%% END OF NEGATIVE PHASE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 196 | err= sum(sum( (data-negdata_CD1).^2 )); 197 | errsum = err + errsum; 198 | 199 | if epoch>5, 200 | momentum=finalmomentum; 201 | else 202 | momentum=initialmomentum; 203 | end; 204 | 205 | %%%%%%%%% UPDATE WEIGHTS AND BIASES %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 206 | 207 | visbiasinc = momentum*visbiasinc + (epsilonvb/numcases)*(posvisact-negvisact); 208 | labbiasinc = momentum*labbiasinc + (epsilonvb/numcases)*(poslabact-neglabact); 209 | 210 | hidmeans = sparsedamping*hidmeans + (1-sparsedamping)*poshidact/numcases; 211 | sparsegrads = sparsecost*(repmat(hidmeans,numcases,1)-sparsetarget); 212 | 213 | penmeans = sparsedamping*penmeans + (1-sparsedamping)*pospenact/numcases; 214 | sparsegrads2 = sparsecost*(repmat(penmeans,numcases,1)-sparsetarget2); 215 | 216 | labpeninc = momentum*labpeninc + ... 217 | epsilonw*( (posprodslabpen-negprodslabpen)/numcases - weightcost*labpen); 218 | 219 | vishidinc = momentum*vishidinc + ... 220 | epsilonw*( (posprods-negprods)/numcases - weightcost*vishid - ... 221 | data'*sparsegrads/numcases ); 222 | hidbiasinc = momentum*hidbiasinc + epsilonhb/numcases*(poshidact-neghidact) ... 223 | -epsilonhb/numcases*sum(sparsegrads); 224 | 225 | hidpeninc = momentum*hidpeninc + ... 226 | epsilonw*( (posprodspen-negprodspen)/numcases - weightcost*hidpen - ... 227 | poshidprobs'*sparsegrads2/numcases - (pospenprobs'*sparsegrads)'/numcases ); 228 | penbiasinc = momentum*penbiasinc + epsilonhb/numcases*(pospenact-negpenact) ... 229 | -epsilonhb/numcases*sum(sparsegrads2); 230 | 231 | vishid = vishid + vishidinc; 232 | hidpen = hidpen + hidpeninc; 233 | labpen = labpen + labpeninc; 234 | visbiases = visbiases + visbiasinc; 235 | hidbiases = hidbiases + hidbiasinc; 236 | penbiases = penbiases + penbiasinc; 237 | labbiases = labbiases + labbiasinc; 238 | %%%%%%%%%%%%%%% END OF UPDATES %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 239 | % if rem(batch,50)==0 240 | % figure(1); 241 | % dispims(negdata',28,28); 242 | % end 243 | 244 | end 245 | % fprintf(1, 'epoch %4i reconstruction error %6.1f \n Number of misclassified training cases %d (out of 60000) \n', epoch, errsum,60000-counter); 246 | 247 | save fullmnist_dbm labpen labbiases hidpen penbiases vishid hidbiases visbiases epoch; 248 | 249 | end; 250 | err_II=60000-counter; 251 | 252 | 253 | 254 | 255 | 256 | -------------------------------------------------------------------------------- /dispims.m: -------------------------------------------------------------------------------- 1 | function [imdisp] = dispims(imstack,drows,dcols,flip,border,n2,fud) 2 | % [imdisp] = dispims(imstack,drows,dcols,flip,border,frame_rows) 3 | % 4 | % display a stack of images 5 | % Originally written by Sam Roweis 6 | 7 | 8 | [pp,N] = size(imstack); 9 | if(nargin<7) fud=0; end 10 | if(nargin<6) n2=ceil(sqrt(N)); end 11 | 12 | if(nargin<3) dcols=drows; end 13 | if(nargin<4) flip=0; end 14 | if(nargin<5) border=2; end 15 | 16 | drb=drows+border; 17 | dcb=dcols+border; 18 | 19 | imdisp=min(imstack(:))+zeros(n2*drb,ceil(N/n2)*dcb); 20 | 21 | for nn=1:N 22 | 23 | ii=rem(nn,n2); if(ii==0) ii=n2; end 24 | jj=ceil(nn/n2); 25 | 26 | if(flip) 27 | daimg = reshape(imstack(:,nn),dcols,drows)'; 28 | else 29 | daimg = reshape(imstack(:,nn),drows,dcols); 30 | end 31 | 32 | imdisp(((ii-1)*drb+1):(ii*drb-border),((jj-1)*dcb+1):(jj*dcb-border))=daimg'; 33 | 34 | end 35 | 36 | if(fud) 37 | imdisp=flipud(imdisp); 38 | end 39 | 40 | imagesc(imdisp); colormap gray; axis equal; axis off; 41 | drawnow; 42 | 43 | -------------------------------------------------------------------------------- /kullback-Leibler.m: -------------------------------------------------------------------------------- 1 | Apdf = normpdf([-10:0.1:10], -1, 1); 2 | Bpdf = normpdf([-10:0.1:10], 1, 1); 3 | KL = 0.1 * sum(Apdf .* (log(Apdf)-log(Bpdf))) 4 | KL = 5 | 2.000000000000000 -------------------------------------------------------------------------------- /kullback_Leibler.m: -------------------------------------------------------------------------------- 1 | Apdf = normpdf([-10:0.1:10], -1, 1); 2 | figure 3 | plot([-10:0.1:10],Apdf) 4 | Bpdf = normpdf([-10:0.1:10], 1, 1); 5 | hold on 6 | plot([-10:0.1:10],Bpdf) 7 | KL = 0.1 * sum(Apdf .* (log(Apdf)-log(Bpdf))) 8 | % KL = 2.000000000000000 -------------------------------------------------------------------------------- /license.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2009, Aboozar Taherkhani and Georgina Cosma All rights reserved. 2 | 3 | 4 | 5 | Redistribution and use in source and binary forms, 6 | with or without 7 | modification, are permitted provided that the following conditions are 8 | met: 9 | 10 | 11 | 12 | * Redistributions of source code must retain the above copyright 13 | notice, this list of 14 | conditions and the following disclaimer. 15 | 16 | 17 | * Redistributions in binary form must reproduce the above copyright 18 | 19 | notice, this list of conditions and the following disclaimer in 20 | the documentation and/or other materials provided 21 | with the distribution 22 | 23 | 24 | 25 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 26 | AND ANY EXPRESS OR 27 | IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 28 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A 29 | PARTICULAR PURPOSE 30 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 31 | LIABLE FOR ANY DIRECT, 32 | INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 33 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT 34 | OF 35 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 36 | INTERRUPTION) HOWEVER CAUSED AND ON 37 | ANY THEORY OF LIABILITY, WHETHER IN 38 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 39 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 40 | POSSIBILITY OF SUCH DAMAGE. 41 | -------------------------------------------------------------------------------- /makebatches.m: -------------------------------------------------------------------------------- 1 | % Version 1.000 2 | % 3 | % Code provided by Ruslan Salakhutdinov 4 | % 5 | % Permission is granted for anyone to copy, use, modify, or distribute this 6 | % program and accompanying programs and documents for any purpose, provided 7 | % this copyright notice is retained and prominently displayed, along with 8 | % a note saying that the original programs are available from our 9 | % web page. 10 | % The programs and documents are distributed without any warranty, express or 11 | % implied. As the programs were written for research purposes only, they have 12 | % not been tested to the degree that would be advisable in any important 13 | % application. All use of these programs is entirely at the user's own risk. 14 | 15 | digitdata=[]; 16 | targets=[]; 17 | load digit0; digitdata = [digitdata; D]; targets = [targets; repmat([1 0 0 0 0 0 0 0 0 0], size(D,1), 1)]; 18 | load digit1; digitdata = [digitdata; D]; targets = [targets; repmat([0 1 0 0 0 0 0 0 0 0], size(D,1), 1)]; 19 | load digit2; digitdata = [digitdata; D]; targets = [targets; repmat([0 0 1 0 0 0 0 0 0 0], size(D,1), 1)]; 20 | load digit3; digitdata = [digitdata; D]; targets = [targets; repmat([0 0 0 1 0 0 0 0 0 0], size(D,1), 1)]; 21 | load digit4; digitdata = [digitdata; D]; targets = [targets; repmat([0 0 0 0 1 0 0 0 0 0], size(D,1), 1)]; 22 | load digit5; digitdata = [digitdata; D]; targets = [targets; repmat([0 0 0 0 0 1 0 0 0 0], size(D,1), 1)]; 23 | load digit6; digitdata = [digitdata; D]; targets = [targets; repmat([0 0 0 0 0 0 1 0 0 0], size(D,1), 1)]; 24 | load digit7; digitdata = [digitdata; D]; targets = [targets; repmat([0 0 0 0 0 0 0 1 0 0], size(D,1), 1)]; 25 | load digit8; digitdata = [digitdata; D]; targets = [targets; repmat([0 0 0 0 0 0 0 0 1 0], size(D,1), 1)]; 26 | load digit9; digitdata = [digitdata; D]; targets = [targets; repmat([0 0 0 0 0 0 0 0 0 1], size(D,1), 1)]; 27 | digitdata = digitdata/255; 28 | 29 | totnum=size(digitdata,1); 30 | fprintf(1, 'Size of the training dataset= %5d \n', totnum); 31 | 32 | rand('state',0); %so we know the permutation of the training data 33 | randomorder=randperm(totnum); 34 | 35 | numbatches=totnum/100; 36 | numdims = size(digitdata,2); 37 | batchsize = 100; 38 | batchdata = zeros(batchsize, numdims, numbatches); 39 | batchtargets = zeros(batchsize, 10, numbatches); 40 | 41 | for b=1:numbatches 42 | batchdata(:,:,b) = digitdata(randomorder(1+(b-1)*batchsize:b*batchsize), :); 43 | batchtargets(:,:,b) = targets(randomorder(1+(b-1)*batchsize:b*batchsize), :); 44 | end; 45 | clear digitdata targets; 46 | 47 | digitdata=[]; 48 | targets=[]; 49 | load test0; digitdata = [digitdata; D]; targets = [targets; repmat([1 0 0 0 0 0 0 0 0 0], size(D,1), 1)]; 50 | load test1; digitdata = [digitdata; D]; targets = [targets; repmat([0 1 0 0 0 0 0 0 0 0], size(D,1), 1)]; 51 | load test2; digitdata = [digitdata; D]; targets = [targets; repmat([0 0 1 0 0 0 0 0 0 0], size(D,1), 1)]; 52 | load test3; digitdata = [digitdata; D]; targets = [targets; repmat([0 0 0 1 0 0 0 0 0 0], size(D,1), 1)]; 53 | load test4; digitdata = [digitdata; D]; targets = [targets; repmat([0 0 0 0 1 0 0 0 0 0], size(D,1), 1)]; 54 | load test5; digitdata = [digitdata; D]; targets = [targets; repmat([0 0 0 0 0 1 0 0 0 0], size(D,1), 1)]; 55 | load test6; digitdata = [digitdata; D]; targets = [targets; repmat([0 0 0 0 0 0 1 0 0 0], size(D,1), 1)]; 56 | load test7; digitdata = [digitdata; D]; targets = [targets; repmat([0 0 0 0 0 0 0 1 0 0], size(D,1), 1)]; 57 | load test8; digitdata = [digitdata; D]; targets = [targets; repmat([0 0 0 0 0 0 0 0 1 0], size(D,1), 1)]; 58 | load test9; digitdata = [digitdata; D]; targets = [targets; repmat([0 0 0 0 0 0 0 0 0 1], size(D,1), 1)]; 59 | digitdata = digitdata/255; 60 | 61 | totnum=size(digitdata,1); 62 | fprintf(1, 'Size of the test dataset= %5d \n', totnum); 63 | 64 | rand('state',0); %so we know the permutation of the training data 65 | randomorder=randperm(totnum); 66 | 67 | numbatches=totnum/100; 68 | numdims = size(digitdata,2); 69 | batchsize = 100; 70 | testbatchdata = zeros(batchsize, numdims, numbatches); 71 | testbatchtargets = zeros(batchsize, 10, numbatches); 72 | 73 | for b=1:numbatches 74 | testbatchdata(:,:,b) = digitdata(randomorder(1+(b-1)*batchsize:b*batchsize), :); 75 | testbatchtargets(:,:,b) = targets(randomorder(1+(b-1)*batchsize:b*batchsize), :); 76 | end; 77 | clear digitdata targets; 78 | 79 | 80 | %%% Reset random seeds 81 | rand('state',sum(100*clock)); 82 | randn('state',sum(100*clock)); 83 | 84 | 85 | 86 | -------------------------------------------------------------------------------- /makebatches_all.m: -------------------------------------------------------------------------------- 1 | % Version 1.000 2 | % Permission is granted for anyone to copy, use, modify, or distribute this 3 | % program and accompanying programs and documents for any purpose, provided 4 | % this copyright notice is retained and prominently displayed, along with 5 | % a note saying that the original programs are available from our 6 | % web page. 7 | % The programs and documents are distributed without any warranty, express or 8 | % implied. As the programs were written for research purposes only, they have 9 | % not been tested to the degree that would be advisable in any important 10 | % application. All use of these programs is entirely at the user's own risk. 11 | 12 | %%%%%%%%%%%%%%%load dexter: 1) training dataset 13 | % [classes, digitdata] = load_dexter();%load train dat 14 | % load('dexter_train.mat')% classes, digitdata 15 | filname=[filenamSave '_train.mat']; 16 | load(filname) 17 | digitdata =X_train; 18 | classes = Y_train; 19 | 20 | maxV=max(max(digitdata)); 21 | digitdata = digitdata/maxV; 22 | 23 | targets = tocategorical(classes); 24 | 25 | totnum=size(digitdata,1); 26 | fprintf(1, 'Size of the training dataset= %5d \n', totnum); 27 | 28 | rand('state',0); %so we know the permutation of the training data 29 | % batchsize = 100 30 | [batchtargets, batchdata] = puInBatch(targets, digitdata, batchsize); 31 | clear digitdata targets classes; 32 | 33 | %%%%%%%%%%%%%%%2) validation dataset 34 | % [classes, digitdata] = load_dexter_valid();% load validation data 35 | filname=[filenamSave '_validation.mat']; 36 | load(filname) 37 | classes=Y_valid; digitdata = X_valid; 38 | % load('dexter_validation.mat'); classes=val_classes; digitdata = val_digitdata; 39 | maxV=max(max(digitdata)); 40 | digitdata = digitdata/maxV; 41 | 42 | targets = tocategorical(classes); 43 | 44 | totnum=size(digitdata,1); 45 | fprintf(1, 'Size of the validation dataset= %5d \n', totnum); 46 | [testbatchtargets, testbatchdata] = puInBatch(targets, digitdata, batchsize); 47 | clear digitdata classes val_digitdata val_classes; 48 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 49 | 50 | %%% Reset random seeds 51 | rand('state',sum(100*clock)); 52 | randn('state',sum(100*clock)); 53 | 54 | % function [classes, vectors] = load_dexter() 55 | % % fprintf('\nNO PARAMETERS GIVEN! Loading & evaluating DEXTER data set.\n\n'); 56 | % fprintf('\n Loading DEXTER data set.\n\n'); 57 | % fprintf('DEXTER is a text classification problem in a bag-of-word\n'); 58 | % fprintf('representation. This is a two-class classification problem\n'); 59 | % fprintf('with sparse continuous input variables.\n'); 60 | % fprintf('This dataset is one of five datasets of the NIPS 2003 feature\n'); 61 | % fprintf('selection challenge.\n\n'); 62 | % 63 | % fprintf('http://archive.ics.uci.edu/ml/datasets/Dexter\n\n'); 64 | % 65 | % n = 300; 66 | % dim = 20000; 67 | % 68 | % vectors = zeros(n, dim); 69 | % classes = zeros(n, 1); 70 | % 71 | % 72 | % % fid = fopen('example_datasets/dexter_train.data', 'r'); 73 | % fid = fopen('C:\Users\cmp3tahera\OneDrive - Nottingham Trent University\Deep_FS_MATLA_Neurocomputing\MATLAB\hub-toolbox-matlab-master\DEXTER\DEXTER\dexter_train.data', 'r'); 74 | % 75 | % for i=1:n 76 | % tline = fgetl(fid); 77 | % d = sscanf(tline, '%d:%d'); 78 | % vectors(i,d(1:2:end)) = d(2:2:end); 79 | % end 80 | % fclose(fid); 81 | % 82 | % % fid = fopen('example_datasets/dexter_train.labels', 'r'); 83 | % fid = fopen('C:\Users\cmp3tahera\OneDrive - Nottingham Trent University\Deep_FS_MATLA_Neurocomputing\MATLAB\hub-toolbox-matlab-master\DEXTER\DEXTER\dexter_train.labels', 'r'); 84 | % 85 | % for i=1:n 86 | % tline = fgetl(fid); 87 | % d = sscanf(tline, '%d'); 88 | % classes(i) = d(1); 89 | % end 90 | % fclose(fid); 91 | % 92 | % % D = cosine_distance(vectors); 93 | % 94 | % end 95 | 96 | function [targets] = tocategorical(classes) 97 | targets = []; 98 | for i=1: length(classes) 99 | if classes(i) == -1 100 | targets = [targets;[1 0]]; 101 | elseif classes(i) == 1 102 | targets = [targets; [0 1]]; 103 | end 104 | end 105 | end 106 | 107 | function [batchtargets, batchdata] = puInBatch(targets, digitdata, batchsize) 108 | %get the tardet, data and batch size and put in in batches with the size of 109 | %batchsize 110 | totnum=size(digitdata,1); 111 | randomorder=randperm(totnum); 112 | 113 | % batchsize = 100; 114 | numbatches=totnum/batchsize; 115 | numdims = size(digitdata,2); 116 | batchdata = zeros(batchsize, numdims, numbatches); 117 | batchtargets = zeros(batchsize,size(targets, 2), numbatches); 118 | 119 | for b=1:numbatches 120 | batchdata(:,:,b) = digitdata(randomorder(1+(b-1)*batchsize:b*batchsize), :); 121 | batchtargets(:,:,b) = targets(randomorder(1+(b-1)*batchsize:b*batchsize), :); 122 | end 123 | % clear digitdata targets; 124 | end 125 | 126 | % function [classes, vectors] = load_dexter_valid() 127 | % % fprintf('\nNO PARAMETERS GIVEN! Loading & evaluating DEXTER data set.\n\n'); 128 | % fprintf('\n Loading DEXTER data set.\n\n'); 129 | % fprintf('DEXTER is a text classification problem in a bag-of-word\n'); 130 | % fprintf('representation. This is a two-class classification problem\n'); 131 | % fprintf('with sparse continuous input variables.\n'); 132 | % fprintf('This dataset is one of five datasets of the NIPS 2003 feature\n'); 133 | % fprintf('selection challenge.\n\n'); 134 | % 135 | % fprintf('http://archive.ics.uci.edu/ml/datasets/Dexter\n\n'); 136 | % 137 | % n = 300; 138 | % dim = 20000; 139 | % 140 | % vectors = zeros(n, dim); 141 | % classes = zeros(n, 1); 142 | % 143 | % 144 | % % fid = fopen('example_datasets/dexter_train.data', 'r'); 145 | % % fid = fopen('C:\Users\cmp3tahera\OneDrive - Nottingham Trent University\Deep_FS_MATLA_Neurocomputing\MATLAB\hub-toolbox-matlab-master\DEXTER\DEXTER\dexter_train.data', 'r'); 146 | % %vlidation dat: 147 | % fid = fopen('C:\Users\cmp3tahera\OneDrive - Nottingham Trent University\Deep_FS_MATLA_Neurocomputing\MATLAB\hub-toolbox-matlab-master\DEXTER\DEXTER\dexter_valid.data', 'r'); 148 | % 149 | % for i=1:n 150 | % tline = fgetl(fid); 151 | % d = sscanf(tline, '%d:%d'); 152 | % vectors(i,d(1:2:end)) = d(2:2:end); 153 | % end 154 | % fclose(fid); 155 | % 156 | % % fid = fopen('example_datasets/dexter_train.labels', 'r'); 157 | % fid = fopen('C:\Users\cmp3tahera\OneDrive - Nottingham Trent University\Deep_FS_MATLA_Neurocomputing\MATLAB\hub-toolbox-matlab-master\DEXTER\dexter_valid.labels', 'r'); 158 | % 159 | % for i=1:n 160 | % tline = fgetl(fid); 161 | % d = sscanf(tline, '%d'); 162 | % classes(i) = d(1); 163 | % end 164 | % fclose(fid); 165 | % 166 | % % D = cosine_distance(vectors); 167 | % 168 | % end 169 | 170 | 171 | 172 | -------------------------------------------------------------------------------- /makebatches_dexter.m: -------------------------------------------------------------------------------- 1 | % Version 1.000 2 | % Permission is granted for anyone to copy, use, modify, or distribute this 3 | % program and accompanying programs and documents for any purpose, provided 4 | % this copyright notice is retained and prominently displayed, along with 5 | % a note saying that the original programs are available from our 6 | % web page. 7 | % The programs and documents are distributed without any warranty, express or 8 | % implied. As the programs were written for research purposes only, they have 9 | % not been tested to the degree that would be advisable in any important 10 | % application. All use of these programs is entirely at the user's own risk. 11 | 12 | %%%%%%%%%%%%%%%load dexter: 1) training dataset 13 | % [classes, digitdata] = load_dexter();%load train dat 14 | load('dexter_train.mat')% classes, digitdata 15 | maxV=max(max(digitdata)); 16 | digitdata = digitdata/maxV; 17 | 18 | targets = tocategorical(classes); 19 | 20 | totnum=size(digitdata,1); 21 | fprintf(1, 'Size of the training dataset= %5d \n', totnum); 22 | 23 | rand('state',0); %so we know the permutation of the training data 24 | % batchsize = 100 25 | [batchtargets, batchdata] = puInBatch(targets, digitdata, batchsize); 26 | clear digitdata targets classes; 27 | 28 | %%%%%%%%%%%%%%%2) validation dataset 29 | % [classes, digitdata] = load_dexter_valid();% load validation data 30 | load('dexter_validation.mat'); classes=val_classes; digitdata = val_digitdata; 31 | maxV=max(max(digitdata)); 32 | digitdata = digitdata/maxV; 33 | 34 | targets = tocategorical(classes); 35 | 36 | totnum=size(digitdata,1); 37 | fprintf(1, 'Size of the validation dataset= %5d \n', totnum); 38 | [testbatchtargets, testbatchdata] = puInBatch(targets, digitdata, batchsize); 39 | clear digitdata classes val_digitdata val_classes; 40 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 41 | 42 | %%% Reset random seeds 43 | rand('state',sum(100*clock)); 44 | randn('state',sum(100*clock)); 45 | 46 | % function [classes, vectors] = load_dexter() 47 | % % fprintf('\nNO PARAMETERS GIVEN! Loading & evaluating DEXTER data set.\n\n'); 48 | % fprintf('\n Loading DEXTER data set.\n\n'); 49 | % fprintf('DEXTER is a text classification problem in a bag-of-word\n'); 50 | % fprintf('representation. This is a two-class classification problem\n'); 51 | % fprintf('with sparse continuous input variables.\n'); 52 | % fprintf('This dataset is one of five datasets of the NIPS 2003 feature\n'); 53 | % fprintf('selection challenge.\n\n'); 54 | % 55 | % fprintf('http://archive.ics.uci.edu/ml/datasets/Dexter\n\n'); 56 | % 57 | % n = 300; 58 | % dim = 20000; 59 | % 60 | % vectors = zeros(n, dim); 61 | % classes = zeros(n, 1); 62 | % 63 | % 64 | % % fid = fopen('example_datasets/dexter_train.data', 'r'); 65 | % fid = fopen('C:\Users\cmp3tahera\OneDrive - Nottingham Trent University\Deep_FS_MATLA_Neurocomputing\MATLAB\hub-toolbox-matlab-master\DEXTER\DEXTER\dexter_train.data', 'r'); 66 | % 67 | % for i=1:n 68 | % tline = fgetl(fid); 69 | % d = sscanf(tline, '%d:%d'); 70 | % vectors(i,d(1:2:end)) = d(2:2:end); 71 | % end 72 | % fclose(fid); 73 | % 74 | % % fid = fopen('example_datasets/dexter_train.labels', 'r'); 75 | % fid = fopen('C:\Users\cmp3tahera\OneDrive - Nottingham Trent University\Deep_FS_MATLA_Neurocomputing\MATLAB\hub-toolbox-matlab-master\DEXTER\DEXTER\dexter_train.labels', 'r'); 76 | % 77 | % for i=1:n 78 | % tline = fgetl(fid); 79 | % d = sscanf(tline, '%d'); 80 | % classes(i) = d(1); 81 | % end 82 | % fclose(fid); 83 | % 84 | % % D = cosine_distance(vectors); 85 | % 86 | % end 87 | 88 | function [targets] = tocategorical(classes) 89 | targets = []; 90 | for i=1: length(classes) 91 | if classes(i) == -1 92 | targets = [targets;[1 0]]; 93 | elseif classes(i) == 1 94 | targets = [targets; [0 1]]; 95 | end 96 | end 97 | end 98 | 99 | function [batchtargets, batchdata] = puInBatch(targets, digitdata, batchsize) 100 | %get the tardet, data and batch size and put in in batches with the size of 101 | %batchsize 102 | totnum=size(digitdata,1); 103 | randomorder=randperm(totnum); 104 | 105 | % batchsize = 100; 106 | numbatches=totnum/batchsize; 107 | numdims = size(digitdata,2); 108 | batchdata = zeros(batchsize, numdims, numbatches); 109 | batchtargets = zeros(batchsize,size(targets, 2), numbatches); 110 | 111 | for b=1:numbatches 112 | batchdata(:,:,b) = digitdata(randomorder(1+(b-1)*batchsize:b*batchsize), :); 113 | batchtargets(:,:,b) = targets(randomorder(1+(b-1)*batchsize:b*batchsize), :); 114 | end 115 | % clear digitdata targets; 116 | end 117 | 118 | % function [classes, vectors] = load_dexter_valid() 119 | % % fprintf('\nNO PARAMETERS GIVEN! Loading & evaluating DEXTER data set.\n\n'); 120 | % fprintf('\n Loading DEXTER data set.\n\n'); 121 | % fprintf('DEXTER is a text classification problem in a bag-of-word\n'); 122 | % fprintf('representation. This is a two-class classification problem\n'); 123 | % fprintf('with sparse continuous input variables.\n'); 124 | % fprintf('This dataset is one of five datasets of the NIPS 2003 feature\n'); 125 | % fprintf('selection challenge.\n\n'); 126 | % 127 | % fprintf('http://archive.ics.uci.edu/ml/datasets/Dexter\n\n'); 128 | % 129 | % n = 300; 130 | % dim = 20000; 131 | % 132 | % vectors = zeros(n, dim); 133 | % classes = zeros(n, 1); 134 | % 135 | % 136 | % % fid = fopen('example_datasets/dexter_train.data', 'r'); 137 | % % fid = fopen('C:\Users\cmp3tahera\OneDrive - Nottingham Trent University\Deep_FS_MATLA_Neurocomputing\MATLAB\hub-toolbox-matlab-master\DEXTER\DEXTER\dexter_train.data', 'r'); 138 | % %vlidation dat: 139 | % fid = fopen('C:\Users\cmp3tahera\OneDrive - Nottingham Trent University\Deep_FS_MATLA_Neurocomputing\MATLAB\hub-toolbox-matlab-master\DEXTER\DEXTER\dexter_valid.data', 'r'); 140 | % 141 | % for i=1:n 142 | % tline = fgetl(fid); 143 | % d = sscanf(tline, '%d:%d'); 144 | % vectors(i,d(1:2:end)) = d(2:2:end); 145 | % end 146 | % fclose(fid); 147 | % 148 | % % fid = fopen('example_datasets/dexter_train.labels', 'r'); 149 | % fid = fopen('C:\Users\cmp3tahera\OneDrive - Nottingham Trent University\Deep_FS_MATLA_Neurocomputing\MATLAB\hub-toolbox-matlab-master\DEXTER\dexter_valid.labels', 'r'); 150 | % 151 | % for i=1:n 152 | % tline = fgetl(fid); 153 | % d = sscanf(tline, '%d'); 154 | % classes(i) = d(1); 155 | % end 156 | % fclose(fid); 157 | % 158 | % % D = cosine_distance(vectors); 159 | % 160 | % end 161 | 162 | 163 | 164 | -------------------------------------------------------------------------------- /mf.m: -------------------------------------------------------------------------------- 1 | % Version 1.000 2 | % 3 | % Code provided by Ruslan Salakhutdinov 4 | % 5 | % Permission is granted for anyone to copy, use, modify, or distribute this 6 | % program and accompanying programs and documents for any purpose, provided 7 | % this copyright notice is retained and prominently displayed, along with 8 | % a note saying that the original programs are available from our 9 | % web page. 10 | % The programs and documents are distributed without any warranty, express or 11 | % implied. As the programs were written for research purposes only, they have 12 | % not been tested to the degree that would be advisable in any important 13 | % application. All use of these programs is entirely at the user's own risk. 14 | 15 | 16 | function [temp_h1, temp_h2] = ... 17 | mf(data,targets,vishid,hidbiases,visbiases,hidpen,penbiases,labpen,hidrecbiases); 18 | 19 | [numdim numhid]=size(vishid); 20 | [numhid numpen]=size(hidpen); 21 | 22 | numcases = size(data,1); 23 | bias_hid= repmat(hidbiases,numcases,1); 24 | bias_pen = repmat(penbiases,numcases,1); 25 | big_bias = data*vishid; 26 | lab_bias = targets*labpen; 27 | 28 | temp_h1 = 1./(1 + exp(-data*(2*vishid) - repmat(hidbiases,numcases,1))); 29 | temp_h2 = 1./(1 + exp(-temp_h1*hidpen - targets*labpen - bias_pen)); 30 | 31 | temp_h1_old = temp_h1; 32 | temp_h2_old = temp_h2; 33 | 34 | for ii= 1:10 % Number of the mean-field updates. I also used 30 MF updates. 35 | totin_h1 = big_bias + bias_hid + (temp_h2*hidpen'); 36 | temp_h1_new = 1./(1 + exp(-totin_h1)); 37 | 38 | totin_h2 = (temp_h1_new*hidpen + bias_pen + lab_bias); 39 | temp_h2_new = 1./(1 + exp(-totin_h2)); 40 | 41 | diff_h1 = sum(sum(abs(temp_h1_new - temp_h1),2))/(numcases*numhid); 42 | diff_h2 = sum(sum(abs(temp_h2_new - temp_h2),2))/(numcases*numpen); 43 | % fprintf(1,'\t\t\t\tii=%d Mean-Field: h1=%f h2=%f\r',ii,diff_h1,diff_h2); 44 | if (diff_h1 < 0.0000001 & diff_h2 < 0.0000001) 45 | break; 46 | end 47 | temp_h1 = temp_h1_new; 48 | temp_h2 = temp_h2_new; 49 | end 50 | 51 | temp_h1 = temp_h1_new; 52 | temp_h2 = temp_h2_new; 53 | 54 | 55 | 56 | -------------------------------------------------------------------------------- /mf_class.m: -------------------------------------------------------------------------------- 1 | % Version 1.000 2 | % 3 | % Code provided by Ruslan Salakhutdinov 4 | % 5 | % Permission is granted for anyone to copy, use, modify, or distribute this 6 | % program and accompanying programs and documents for any purpose, provided 7 | % this copyright notice is retained and prominently displayed, along with 8 | % a note saying that the original programs are available from our 9 | % web page. 10 | % The programs and documents are distributed without any warranty, express or 11 | % implied. As the programs were written for research purposes only, they have 12 | % not been tested to the degree that would be advisable in any important 13 | % application. All use of these programs is entirely at the user's own risk. 14 | 15 | 16 | function [temp_h1, temp_h2] = ... 17 | mf_class(data,vishid,hidbiases,visbiases,hidpen,penbiases); 18 | 19 | [numdim numhid]=size(vishid); 20 | [numhid numpen]=size(hidpen); 21 | 22 | numcases = size(data,1); 23 | bias_hid= repmat(hidbiases,numcases,1); 24 | bias_pen = repmat(penbiases,numcases,1); 25 | big_bias = data*vishid; 26 | 27 | temp_h1 = 1./(1 + exp(-data*(2*vishid) - repmat(hidbiases,numcases,1))); 28 | temp_h2 = 1./(1 + exp(-temp_h1*hidpen - bias_pen)); 29 | 30 | for ii= 1:50 31 | totin_h1 = big_bias + bias_hid + (temp_h2*hidpen'); 32 | temp_h1_new = 1./(1 + exp(-totin_h1)); 33 | 34 | totin_h2 = (temp_h1_new*hidpen + bias_pen); 35 | temp_h2_new = 1./(1 + exp(-totin_h2)); 36 | 37 | diff_h1 = sum(sum(abs(temp_h1_new - temp_h1),2))/(numcases*numhid); 38 | diff_h2 = sum(sum(abs(temp_h2_new - temp_h2),2))/(numcases*numpen); 39 | fprintf(1,'\t\t\t\tii=%d h1=%f h2=%f\r',ii,diff_h1,diff_h2); 40 | if (diff_h1 < 0.0000001 & diff_h2 < 0.0000001) 41 | break; 42 | end 43 | temp_h1 = temp_h1_new; 44 | temp_h2 = temp_h2_new; 45 | end 46 | 47 | temp_h1 = temp_h1_new; 48 | temp_h2 = temp_h2_new; 49 | 50 | 51 | 52 | -------------------------------------------------------------------------------- /minimize.m: -------------------------------------------------------------------------------- 1 | function [X, fX, i, XX] = minimize(X, f, length, P1, P2, P3, P4, P5); 2 | 3 | % Written by Carl E. Rasmussen 4 | 5 | 6 | % Minimize a continuous differentialble multivariate function. Starting point 7 | % is given by "X" (D by 1), and the function named in the string "f", must 8 | % return a function value and a vector of partial derivatives. The Polack- 9 | % Ribiere flavour of conjugate gradients is used to compute search directions, 10 | % and a line search using quadratic and cubic polynomial approximations and the 11 | % Wolfe-Powell stopping criteria is used together with the slope ratio method 12 | % for guessing initial step sizes. Additionally a bunch of checks are made to 13 | % make sure that exploration is taking place and that extrapolation will not 14 | % be unboundedly large. The "length" gives the length of the run: if it is 15 | % positive, it gives the maximum number of line searches, if negative its 16 | % absolute gives the maximum allowed number of function evaluations. The 17 | % function returns when either its length is up, or if no further progress can 18 | % be made (ie, we are at a minimum, or so close that due to numerical problems, 19 | % we cannot get any closer). If the function terminates within a few 20 | % iterations, it could be an indication that the function value and derivatives 21 | % are not consistent (ie, there may be a bug in the implementation of your "f" 22 | % function). The function returns the found solution "X", a vector of function 23 | % values "fX" indicating the progress made and "i" the number of iterations 24 | % (line searches or function evaluations, depending on the sign of "length") 25 | % used. 26 | % 27 | % Usage: [X, fX, i] = minimize(X, f, length, P1, P2, P3, P4, P5) 28 | % 29 | % See also: checkgrad 30 | % 31 | % Copyright (C) 2001 by Carl Edward Rasmussen. Date 2001-07-18 32 | 33 | RHO = 0.01; % a bunch of constants for line searches 34 | SIG = 0.5; % RHO and SIG are the constants in the Wolfe-Powell conditions 35 | INT = 0.1; % don't reevaluate within 0.1 of the limit of the current bracket 36 | EXT = 3.0; % extrapolate maximum 3 times the current bracket 37 | MAX = 20; % max 20 function evaluations per line search 38 | RATIO = 100; % maximum allowed slope ratio 39 | 40 | argstr = [f, '(X']; % compose string used to call function 41 | for i = 1:(nargin - 3) 42 | argstr = [argstr, ',P', int2str(i)]; 43 | end 44 | argstr = [argstr, ')']; 45 | 46 | if length>0, S=['Linesearch']; else S=['Function evaluation']; end 47 | 48 | i = 0; % zero the run length counter 49 | ls_failed = 0; % no previous line search has failed 50 | fX = []; 51 | XX = []; 52 | [f1 df1] = eval(argstr); % get function value and gradient 53 | XX=vertcat(XX,[1,-f1]); %by Russ, get first value 54 | i = i + (length<0); % count epochs?! 55 | s = -df1; % search direction is steepest 56 | d1 = -s'*s; % this is the slope 57 | z1 = 1/(1-d1); % initial step is 1/(|s|+1) 58 | 59 | while i < abs(length) % while not finished 60 | i = i + (length>0); % count iterations?! 61 | 62 | X0 = X; f0 = f1; df0 = df1; % make a copy of current values 63 | X = X + z1*s; % begin line search 64 | [f2 df2] = eval(argstr); 65 | i = i + (length<0); % count epochs?! 66 | d2 = df2'*s; 67 | f3 = f1; d3 = d1; z3 = -z1; % initialize point 3 equal to point 1 68 | if length>0, M = MAX; else M = min(MAX, -length-i); end 69 | success = 0; limit = -1; % initialize quanteties 70 | while 1 71 | while ((f2 > f1+z1*RHO*d1) | (d2 > -SIG*d1)) & (M > 0) 72 | limit = z1; % tighten the bracket 73 | if f2 > f1 74 | z2 = z3 - (0.5*d3*z3*z3)/(d3*z3+f2-f3); % quadratic fit 75 | else 76 | A = 6*(f2-f3)/z3+3*(d2+d3); % cubic fit 77 | B = 3*(f3-f2)-z3*(d3+2*d2); 78 | z2 = (sqrt(B*B-A*d2*z3*z3)-B)/A; % numerical error possible - ok! 79 | end 80 | if isnan(z2) | isinf(z2) 81 | z2 = z3/2; % if we had a numerical problem then bisect 82 | end 83 | z2 = max(min(z2, INT*z3),(1-INT)*z3); % don't accept too close to limits 84 | z1 = z1 + z2; % update the step 85 | X = X + z2*s; 86 | [f2 df2] = eval(argstr); 87 | M = M - 1; i = i + (length<0); % count epochs?! 88 | d2 = df2'*s; 89 | z3 = z3-z2; % z3 is now relative to the location of z2 90 | end 91 | if f2 > f1+z1*RHO*d1 | d2 > -SIG*d1 92 | break; % this is a failure 93 | elseif d2 > SIG*d1 94 | success = 1; break; % success 95 | elseif M == 0 96 | break; % failure 97 | end 98 | A = 6*(f2-f3)/z3+3*(d2+d3); % make cubic extrapolation 99 | B = 3*(f3-f2)-z3*(d3+2*d2); 100 | z2 = -d2*z3*z3/(B+sqrt(B*B-A*d2*z3*z3)); % num. error possible - ok! 101 | if ~isreal(z2) | isnan(z2) | isinf(z2) | z2 < 0 % num prob or wrong sign? 102 | if limit < -0.5 % if we have no upper limit 103 | z2 = z1 * (EXT-1); % the extrapolate the maximum amount 104 | else 105 | z2 = (limit-z1)/2; % otherwise bisect 106 | end 107 | elseif (limit > -0.5) & (z2+z1 > limit) % extraplation beyond max? 108 | z2 = (limit-z1)/2; % bisect 109 | elseif (limit < -0.5) & (z2+z1 > z1*EXT) % extrapolation beyond limit 110 | z2 = z1*(EXT-1.0); % set to extrapolation limit 111 | elseif z2 < -z3*INT 112 | z2 = -z3*INT; 113 | elseif (limit > -0.5) & (z2 < (limit-z1)*(1.0-INT)) % too close to limit? 114 | z2 = (limit-z1)*(1.0-INT); 115 | end 116 | f3 = f2; d3 = d2; z3 = -z2; % set point 3 equal to point 2 117 | z1 = z1 + z2; X = X + z2*s; % update current estimates 118 | [f2 df2] = eval(argstr); 119 | M = M - 1; i = i + (length<0); % count epochs?! 120 | d2 = df2'*s; 121 | end % end of line search 122 | 123 | if success % if line search succeeded 124 | f1 = f2; fX = [fX' f1]'; 125 | fprintf('%s %6i; Value %4.6e\r', S, i, f1); 126 | XX=vertcat(XX,[i,-f1]); %minus here for our purpuses only 127 | s = (df2'*df2-df1'*df2)/(df1'*df1)*s - df2; % Polack-Ribiere direction 128 | tmp = df1; df1 = df2; df2 = tmp; % swap derivatives 129 | d2 = df1'*s; 130 | if d2 > 0 % new slope must be negative 131 | s = -df1; % otherwise use steepest direction 132 | d2 = -s'*s; 133 | end 134 | z1 = z1 * min(RATIO, d1/(d2-realmin)); % slope ratio but max RATIO 135 | d1 = d2; 136 | ls_failed = 0; % this line search did not fail 137 | else 138 | X = X0; f1 = f0; df1 = df0; % restore point from before failed line search 139 | if ls_failed | i > abs(length) % line search failed twice in a row 140 | break; % or we ran out of time, so we give up 141 | end 142 | tmp = df1; df1 = df2; df2 = tmp; % swap derivatives 143 | s = -df1; % try steepest 144 | d1 = -s'*s; 145 | z1 = 1/(1-d1); 146 | ls_failed = 1; % this line search failed 147 | end 148 | end 149 | fprintf('\r'); 150 | -------------------------------------------------------------------------------- /pvh.m: -------------------------------------------------------------------------------- 1 | %%%%%%% p(h|v), -log(P(vi=1|h)):poshidprobs_rec/ in numdims 2 | poshidprobs=[]; 3 | poshidprobs_rec=[]; 4 | % errsum=0; 5 | errsum_analog=zeros(size(1,numdims)); 6 | negdata_analog_rec=[]; 7 | for batch = 1:numbatches, 8 | % fprintf(1,'epoch %d batch %d\r',epoch,batch); 9 | visbias = repmat(visbiases,numcases,1); 10 | hidbias = repmat(2*hidbiases,numcases,1); 11 | %%%%%%%%% START POSITIVE PHASE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 12 | data = batchdata(:,:,batch); 13 | data_analog=data; 14 | data = data > rand(numcases,numdims); 15 | poshidprobs = 1./(1 + exp(-data*(2*vishid) - hidbias));%p(h|v) 16 | %MBF 17 | poshidprobs_rec(:,:,batch)=poshidprobs;%%p(h|v) 18 | % batchposhidprobs(:,:,batch)=poshidprobs; 19 | % posprods = data' * poshidprobs; 20 | % poshidact = sum(poshidprobs); 21 | % posvisact = sum(data); 22 | %%%%%%%%% END OF POSITIVE PHASE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 23 | 24 | %%%%% START NEGATIVE PHASE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 25 | poshidstates = poshidprobs > rand(numcases,numhid); 26 | negdata = 1./(1 + exp(-poshidstates*vishid' - visbias));%P(vi=1?h) 27 | negdata_analog=negdata; 28 | % MBF 29 | %negdata_analog_rec(:,:,batch)=negdata;%P(vi=1?h) 30 | % negdata_analog_rec(:,batch)=-sum(log(negdata)');%-log(P(vi=1|h)) 31 | negdata_analog_rec(:,batch)=-sum(log(negdata)')/numdims;%-log(P(vi=1|h)) 32 | 33 | %%% 34 | % negdata = negdata > rand(numcases,numdims); 35 | % neghidprobs = 1./(1 + exp(-negdata*(2*vishid) - hidbias)); 36 | % negprods = negdata'*neghidprobs; 37 | % neghidact = sum(neghidprobs); 38 | % negvisact = sum(negdata); 39 | 40 | %%%%%%%%% END OF NEGATIVE PHASE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 41 | % err= sum(sum( (data-negdata).^2 )); 42 | err_analog=sum((data_analog-negdata_analog).^2)/numcases; 43 | 44 | % errsum = err + errsum; 45 | errsum_analog = err_analog + errsum_analog; 46 | % if rem(batch,600)==0 47 | % figure(1); 48 | % dispims(negdata',28,28); 49 | % drawnow 50 | % end 51 | end 52 | % fprintf(1, 'epoch %4i error %6.1f \n', epoch, errsum); 53 | % fprintf(1, 'error %6.1f \n', errsum); 54 | % errsum0=errsum; 55 | 56 | errmean_analog=errsum_analog/numbatches; 57 | -------------------------------------------------------------------------------- /pvh0.m: -------------------------------------------------------------------------------- 1 | 2 | poshidprobs = zeros(numcases,numhid); 3 | % errsum=0; 4 | errsum_analog=zeros(1,numdims); 5 | for batch = 1:numbatches, 6 | % fprintf(1,'epoch %d batch %d\r',epoch,batch); 7 | visbias = repmat(visbiases,numcases,1); 8 | hidbias = repmat(2*hidbiases,numcases,1); 9 | %%%%%%%%% START POSITIVE PHASE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 10 | data = batchdata(:,:,batch); 11 | data_analog=data; 12 | data = data > rand(numcases,numdims); 13 | poshidprobs = 1./(1 + exp(-data*(2*vishid) - hidbias)); 14 | % batchposhidprobs(:,:,batch)=poshidprobs; 15 | % posprods = data' * poshidprobs; 16 | % poshidact = sum(poshidprobs); 17 | % posvisact = sum(data); 18 | %%%%%%%%% END OF POSITIVE PHASE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 19 | 20 | %%%%% START NEGATIVE PHASE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 21 | poshidstates = poshidprobs > rand(numcases,numhid); 22 | negdata = 1./(1 + exp(-poshidstates*vishid' - visbias)); 23 | negdata_analog=negdata; 24 | negdata = negdata > rand(numcases,numdims); 25 | % neghidprobs = 1./(1 + exp(-negdata*(2*vishid) - hidbias)); 26 | % negprods = negdata'*neghidprobs; 27 | % neghidact = sum(neghidprobs); 28 | % negvisact = sum(negdata); 29 | 30 | %%%%%%%%% END OF NEGATIVE PHASE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 31 | % err= sum(sum( (data-negdata).^2 )); 32 | err_analog=sum((data_analog-negdata_analog).^2)/numcases; 33 | 34 | % errsum = err + errsum; 35 | errsum_analog = err_analog + errsum_analog; 36 | % if rem(batch,600)==0 37 | % figure(1); 38 | % dispims(negdata',28,28); 39 | % drawnow 40 | % end 41 | end 42 | % fprintf(1, 'epoch %4i error %6.1f \n', epoch, errsum); 43 | % fprintf(1, 'error %6.1f \n', errsum); 44 | % errsum0=errsum; 45 | 46 | errmean_analog=errsum_analog/numbatches; -------------------------------------------------------------------------------- /rbm.m: -------------------------------------------------------------------------------- 1 | % Version 1.000 2 | % 3 | % Code provided by Ruslan Salakhutdinov 4 | % 5 | % Permission is granted for anyone to copy, use, modify, or distribute this 6 | % program and accompanying programs and documents for any purpose, provided 7 | % this copyright notice is retained and prominently displayed, along with 8 | % a note saying that the original programs are available from our 9 | % web page. 10 | % The programs and documents are distributed without any warranty, express or 11 | % implied. As the programs were written for research purposes only, they have 12 | % not been tested to the degree that would be advisable in any important 13 | % application. All use of these programs is entirely at the user's own risk. 14 | 15 | % This program trains Restricted Boltzmann Machine in which 16 | % visible, binary, stochastic pixels are connected to 17 | % hidden, binary, stochastic feature detectors using symmetrically 18 | % weighted connections. Learning is done with 1-step Contrastive Divergence. 19 | % The program assumes that the following variables are set externally: 20 | % maxepoch -- maximum number of epochs 21 | % numhid -- number of hidden units 22 | % batchdata -- the data that is divided into batches (numcases numdims numbatches) 23 | % restart -- set to 1 if learning starts from beginning 24 | 25 | if restart ==1, 26 | restart=0; 27 | 28 | epsilonw = 0.05; % Learning rate for weights 29 | epsilonvb = 0.05; % Learning rate for biases of visible units 30 | epsilonhb = 0.05; % Learning rate for biases of hidden units 31 | 32 | CD=1; 33 | weightcost = 0.001; 34 | initialmomentum = 0.5; 35 | finalmomentum = 0.9; 36 | 37 | [numcases numdims numbatches]=size(batchdata); 38 | epoch=1; 39 | 40 | % Initializing symmetric weights and biases. 41 | vishid = 0.001*randn(numdims, numhid); 42 | hidbiases = zeros(1,numhid); 43 | visbiases = zeros(1,numdims); 44 | 45 | poshidprobs = zeros(numcases,numhid); 46 | neghidprobs = zeros(numcases,numhid); 47 | posprods = zeros(numdims,numhid); 48 | negprods = zeros(numdims,numhid); 49 | vishidinc = zeros(numdims,numhid); 50 | hidbiasinc = zeros(1,numhid); 51 | visbiasinc = zeros(1,numdims); 52 | batchposhidprobs=zeros(numcases,numhid,numbatches); 53 | end 54 | 55 | 56 | for epoch = epoch:maxepoch 57 | %fprintf(1,'epoch %d\r',epoch); 58 | errsum=0; 59 | for batch = 1:numbatches, 60 | %fprintf(1,'epoch %d batch %d\r',epoch,batch); 61 | 62 | visbias = repmat(visbiases,numcases,1); 63 | hidbias = repmat(2*hidbiases,numcases,1); 64 | %%%%%%%%% START POSITIVE PHASE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 65 | data = batchdata(:,:,batch); 66 | data = data > rand(numcases,numdims); 67 | 68 | poshidprobs = 1./(1 + exp(-data*(2*vishid) - hidbias)); 69 | batchposhidprobs(:,:,batch)=poshidprobs; 70 | posprods = data' * poshidprobs; 71 | poshidact = sum(poshidprobs); 72 | posvisact = sum(data); 73 | 74 | %%%%%%%%% END OF POSITIVE PHASE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 75 | 76 | %%%%% START NEGATIVE PHASE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 77 | poshidstates = poshidprobs > rand(numcases,numhid); 78 | negdata = 1./(1 + exp(-poshidstates*vishid' - visbias)); 79 | negdata = negdata > rand(numcases,numdims); 80 | neghidprobs = 1./(1 + exp(-negdata*(2*vishid) - hidbias)); 81 | 82 | negprods = negdata'*neghidprobs; 83 | neghidact = sum(neghidprobs); 84 | negvisact = sum(negdata); 85 | 86 | %%%%%%%%% END OF NEGATIVE PHASE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 87 | err= sum(sum( (data-negdata).^2 )); 88 | errsum = err + errsum; 89 | 90 | if epoch>5, 91 | momentum=finalmomentum; 92 | else 93 | momentum=initialmomentum; 94 | end; 95 | 96 | %%%%%%%%% UPDATE WEIGHTS AND BIASES %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 97 | vishidinc = momentum*vishidinc + ... 98 | epsilonw*( (posprods-negprods)/numcases - weightcost*vishid); 99 | visbiasinc = momentum*visbiasinc + (epsilonvb/numcases)*(posvisact-negvisact); 100 | hidbiasinc = momentum*hidbiasinc + (epsilonhb/numcases)*(poshidact-neghidact); 101 | 102 | vishid = vishid + vishidinc; 103 | visbiases = visbiases + visbiasinc; 104 | hidbiases = hidbiases + hidbiasinc; 105 | %%%%%%%%%%%%%%%% END OF UPDATES %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 106 | % if rem(batch,600)==0 107 | % figure(1); 108 | % dispims(negdata',28,28); 109 | % drawnow 110 | % end 111 | end 112 | % fprintf(1, 'epoch %4i error %6.1f \n', epoch, errsum); 113 | 114 | end; 115 | 116 | save fullmnistvh vishid visbiases hidbiases epoch 117 | 118 | 119 | -------------------------------------------------------------------------------- /rbm_MBF.m: -------------------------------------------------------------------------------- 1 | % Version 1.000 2 | % 3 | % Code provided by Ruslan Salakhutdinov 4 | % 5 | % Permission is granted for anyone to copy, use, modify, or distribute this 6 | % program and accompanying programs and documents for any purpose, provided 7 | % this copyright notice is retained and prominently displayed, along with 8 | % a note saying that the original programs are available from our 9 | % web page. 10 | % The programs and documents are distributed without any warranty, express or 11 | % implied. As the programs were written for research purposes only, they have 12 | % not been tested to the degree that would be advisable in any important 13 | % application. All use of these programs is entirely at the user's own risk. 14 | 15 | % This program trains Restricted Boltzmann Machine in which 16 | % visible, binary, stochastic pixels are connected to 17 | % hidden, binary, stochastic feature detectors using symmetrically 18 | % weighted connections. Learning is done with 1-step Contrastive Divergence. 19 | % The program assumes that the following variables are set externally: 20 | % maxepoch -- maximum number of epochs 21 | % numhid -- number of hidden units 22 | % batchdata -- the data that is divided into batches (numcases numdims numbatches) 23 | % restart -- set to 1 if learning starts from beginning 24 | 25 | 26 | clc 27 | clear all 28 | close all 29 | load('demo_pix-data') 30 | 31 | % save('demo_pix-data') 32 | fprintf(1, ' start to removed pixels by rbm_test9 \n') 33 | 34 | [numcases numdims numbatches]=size(batchdata); 35 | % poshidprobs = zeros(numcases,numhid); 36 | 37 | %%%%%%% p(h|v), -log(P(vi=1|h)):poshidprobs_rec/ in: numdims 38 | pvh 39 | 40 | % save fullmnistvh vishid visbiases hidbiases %epoch 41 | %% remove diferent pixels 42 | err2_rec=[]; 43 | 44 | % related to Removing extra pix 45 | batchdata0=batchdata; 46 | visbiases0=visbiases; 47 | vishid0=vishid; 48 | numdims0=numdims; 49 | ind_re=[]; 50 | % ind_a0=[1:numdims]; 51 | % ind_a=ind_a0; %indix of adding 52 | %%% 53 | % numdims0=10 54 | % for ipix=1:numdims0 55 | batchdata1=batchdata0; 56 | vishid1=vishid0; 57 | visbiases1=visbiases0; 58 | ipix=0; 59 | ipix_o=0; 60 | %learning 61 | vishidinc0=vishidinc; 62 | visbiasinc0=visbiasinc; 63 | %%%MBF 64 | delta_rec=[]; 65 | %%%% 66 | % 67 | while ipix < numdims 68 | % for ipix=1:numdims 69 | ipix=ipix+1; 70 | ipix_o=ipix_o+1; 71 | poshidprobs = zeros(numcases,numhid); 72 | % errsum=0; 73 | errsum_analog=zeros(1,numdims); 74 | %MBF 75 | delta=0; 76 | %%% 77 | for batch = 1:numbatches, 78 | visbias = repmat(visbiases,numcases,1); 79 | hidbias = repmat(2*hidbiases,numcases,1); 80 | %%%%%%%%% START POSITIVE PHASE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 81 | data = batchdata(:,:,batch); 82 | data_analog=data; 83 | data = data > rand(numcases,numdims); 84 | %%%% 85 | poshidprobs = 1./(1 + exp(-data*(2*vishid) - hidbias));%p(h|v-) 86 | 87 | %MBF 88 | KL = sum((poshidprobs_rec(:,:,batch) .* (log(poshidprobs_rec(:,:,batch))-log(poshidprobs)))')'; 89 | delta=delta+sum(negdata_analog_rec(:,batch).*KL)/numcases; 90 | %%%%%%%%% END OF POSITIVE PHASE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 91 | 92 | %%%%% START NEGATIVE PHASE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 93 | poshidstates = poshidprobs > rand(numcases,numhid); 94 | negdata = 1./(1 + exp(-poshidstates*vishid' - visbias)); 95 | negdata_analog=negdata; 96 | negdata = negdata > rand(numcases,numdims); 97 | %%%%%%%%% END OF NEGATIVE PHASE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 98 | % err= sum(sum( (data0-negdata).^2 )); 99 | % err= sum(sum( (data-negdata).^2 )); 100 | err_analog=(sum((data_analog-negdata_analog).^2))/numcases; 101 | % err_analog=(sum((data_analog(:,ipix)-negdata_analog(:,ipix)).^2))/numcases; 102 | 103 | % errsum = err + errsum; 104 | errsum_analog = err_analog + errsum_analog; 105 | % if rem(batch,600)==0 106 | % figure(1); 107 | % dispims(negdata',28,28); 108 | % drawnow 109 | % end 110 | end 111 | %MBF 112 | delta=delta/numbatches; 113 | delta_rec=[delta_rec, delta] 114 | %%% 115 | % figure(2); plot(errsum_analog/numbatches) 116 | errmean_analog2=errsum_analog(ipix)/numbatches; 117 | 118 | 119 | %%remove extra pixel 120 | % 121 | % err2t=(errmean_analog(ipix_o)-errmean_analog2); 122 | % err2_rec=[err2_rec, err2t/errmean_analog(ipix_o)]; 123 | 124 | err2t=(errmean_analog(ipix)-errmean_analog2);%irad:by removing a pixel the original error should be updated atleast after each training 125 | err2_rec=[err2_rec, err2t/errmean_analog(ipix)]; 126 | %%%%%Remove extra pixel 127 | 128 | % % if err2_rec(ipix_o)>0 129 | 130 | if delta< 135.1908 131 | 132 | ind_re=[ind_re ipix_o]; 133 | numdims=numdims-1;%size(ind_a,2); 134 | fprintf(1, 'ipix_o is %3.0f \n', ipix_o) 135 | fprintf(1, ' #removed pixels is %3.0f \n', 784-numdims) 136 | 137 | batchdata(:,ipix,:)=[]; 138 | vishid(ipix,:)=[]; 139 | visbiases(ipix)=[]; 140 | 141 | vishidinc(ipix,:)=[]; 142 | visbiasinc (ipix)=[]; 143 | 144 | ipix=ipix-1; 145 | 146 | 147 | %%%learning 148 | if (mod(size(ind_re,2),10)==0) && maxepoch<60 149 | % vishidinc=vishidinc0; 150 | % vishidinc(ind_re, :)=[]; 151 | % visbiasinc=visbiasinc0;%irad: 152 | % visbiasinc (ind_re)=[]; 153 | 154 | maxepoch=maxepoch+1; 155 | epoch=epoch+1; 156 | rbm% train with reduced data 157 | end 158 | 159 | %%%%%%% p(h|v), -log(P(vi=1|h)):poshidprobs_rec/ in: numdims 160 | pvh 161 | 162 | end 163 | 164 | %%%% 165 | 166 | 167 | end%ipix 168 | % vishidinc(ind_re, :) = []; 169 | % visbiasinc (ind_re) = []; 170 | % vishidinc=vishidinc0; 171 | % vishidinc(ind_re, :)=[]; 172 | % visbiasinc=visbiasinc0; 173 | % visbiasinc (ind_re)=[]; 174 | save fullmnistvh vishid visbiases hidbiases epoch 175 | 176 | %%%remove: save the new data 177 | 178 | load digit0; D(:,ind_re)=[]; save ('digit0','D'); 179 | 180 | load digit1; D(:,ind_re)=[]; save ('digit1','D'); 181 | 182 | load digit2; D(:,ind_re)=[]; save ('digit2','D'); 183 | 184 | load digit3; D(:,ind_re)=[]; save ('digit3','D'); 185 | 186 | load digit4; D(:,ind_re)=[]; save ('digit4','D'); 187 | load digit5; D(:,ind_re)=[]; save ('digit5','D'); 188 | load digit6; D(:,ind_re)=[]; save ('digit6','D'); 189 | load digit7; D(:,ind_re)=[]; save ('digit7','D'); 190 | load digit8; D(:,ind_re)=[]; save ('digit8','D'); 191 | load digit9; D(:,ind_re)=[]; save ('digit9','D'); 192 | 193 | load test0;D(:,ind_re)=[]; save ('test0','D'); 194 | load test1;D(:,ind_re)=[]; save ('test1','D'); 195 | load test2;D(:,ind_re)=[]; save ('test2','D'); 196 | load test3;D(:,ind_re)=[]; save ('test3','D'); 197 | load test4;D(:,ind_re)=[]; save ('test4','D'); 198 | load test5;D(:,ind_re)=[]; save ('test5','D'); 199 | load test6;D(:,ind_re)=[]; save ('test6','D'); 200 | load test7;D(:,ind_re)=[]; save ('test7','D'); 201 | load test8;D(:,ind_re)=[]; save ('test8','D'); 202 | load test9;D(:,ind_re)=[]; save ('test9','D'); 203 | % ind_re=find(errsum_rec