├── ML-OSELMmanual.docx ├── SigActFun.m ├── README.md ├── procrustNew.m ├── OSELM_UMIST.m └── MELM_UMISTseq.m /ML-OSELMmanual.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bilalmirza8519/Multi-layer-OSELM/HEAD/ML-OSELMmanual.docx -------------------------------------------------------------------------------- /SigActFun.m: -------------------------------------------------------------------------------- 1 | function H = SigActFun(P,IW,Bias); 2 | 3 | %%%%%%%% Feedforward neural network using sigmoidal activation function 4 | V=P*IW'; ind=ones(1,size(P,1)); 5 | BiasMatrix=Bias(ind,:); 6 | V=V+BiasMatrix; 7 | H = 1./(1+exp(-V)); -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Multi-layer-OSELM 2 | This is the code for ML-OSELM method for image classification. Please cite the following paper if you use his code 3 | Mirza B, Kok S, Dong F. Multi-layer online sequential extreme learning machine for image classification. InProceedings of ELM-2015 Volume 1 2016 (pp. 39-49). Springer, Cham. 4 | -------------------------------------------------------------------------------- /procrustNew.m: -------------------------------------------------------------------------------- 1 | function [A2, Q, r] = procrustNew( A, B ) 2 | 3 | % PROCRUST Orthogonal Procrustes problem 4 | % A2 = PROCRUST( A, B ) applies an orthogonal transformation to matrix B 5 | % by multiplication with Q such that A-B*Q has minimum Frobenius norm. The 6 | % results B*Q is returned as A2. 7 | % 8 | % [A2, Q] = PROCRUST( A, B ) also returns the orthogonal matrix Q that was 9 | % used for the transformation. 10 | % 11 | % [A2, Q, R] = PROCRUST( A, B ) also returns the Frobenius norm of A-B*Q. 12 | 13 | % Author : E. Larsen 14 | % Date : 12/22/03 15 | % Email : erik.larsen@ieee.org 16 | 17 | % Reference: Golub and van Loan, p. 601. 18 | 19 | % Error checking 20 | msg = nargchk( 2, 2, nargin ); 21 | if ~isempty( msg ) 22 | error( msg ) 23 | end 24 | 25 | % Do the computation 26 | C = B.'*A; 27 | [U1, S1, V1] = svd( C'*C ); 28 | [U2, S2, V2] = svd( C*C' ); 29 | 30 | Q = U2*V1'; 31 | A2 = B*Q; 32 | 33 | % Optional output of norm 34 | if nargout > 2 35 | r = norm( A-A2, 'fro' ); 36 | end 37 | -------------------------------------------------------------------------------- /OSELM_UMIST.m: -------------------------------------------------------------------------------- 1 | function [TrainingTime, TestingTime, TrainingAccuracy, TestingAccuracy] = OSELM(TrainingData_File, TestingData_File, Elm_Type, nHiddenNeurons, ActivationFunction, N0, Block) 2 | 3 | % Usage: OSELM(TrainingData_File, TestingData_File, Elm_Type, NumberofHiddenNeurons, ActivationFunction, N0, Block) 4 | % OR: [TrainingTime, TestingTime, TrainingAccuracy, TestingAccuracy] = OSELM(TrainingData_File, TestingData_File, Elm_Type, NumberofHiddenNeurons, ActivationFunction, N0, Block) 5 | % 6 | % Input: 7 | % TrainingData_File - Filename of training data set 8 | % TestingData_File - Filename of testing data set 9 | % Elm_Type - 0 for regression; 1 for (both binary and multi-classes) classification 10 | % nHiddenNeurons - Number of hidden neurons assigned to the OSELM 11 | % ActivationFunction - Type of activation function: 12 | % 'rbf' for radial basis function, G(a,b,x) = exp(-b||x-a||^2) 13 | % 'sig' for sigmoidal function, G(a,b,x) = 1/(1+exp(-(ax+b))) 14 | % 'sin' for sine function, G(a,b,x) = sin(ax+b) 15 | % 'hardlim' for hardlim function, G(a,b,x) = hardlim(ax+b) 16 | % N0 - Number of initial training data used in the initial phase of OSLEM, which is not less than the number of hidden neurons 17 | % Block - Size of block of data learned by OSELM in each step 18 | % 19 | % Output: 20 | % TrainingTime - Time (seconds) spent on training OSELM 21 | % TestingTime - Time (seconds) spent on predicting all testing data 22 | % TrainingAccuracy - Training accuracy: 23 | % RMSE for regression or correct classification rate for classifcation 24 | % TestingAccuracy - Testing accuracy: 25 | % RMSE for regression or correct classification rate for classifcation 26 | % 27 | % MULTI-CLASSE CLASSIFICATION: NUMBER OF OUTPUT NEURONS WILL BE AUTOMATICALLY SET EQUAL TO NUMBER OF CLASSES 28 | % FOR EXAMPLE, if there are 7 classes in all, there will have 7 output 29 | % neurons; neuron 5 has the highest output means input belongs to 5-th class 30 | % 31 | % Sample1 regression: OSELM('mpg_train', 'mpg_test', 0, 25, 'rbf', 75, 1); 32 | % Sample2 classification: OSELM('segment_train', 'segment_test', 1, 180, 'sig', 280, 20); 33 | 34 | %%%% Authors: 35 | %%%% NANYANG TECHNOLOGICAL UNIVERSITY, SINGAPORE 36 | %%%% EMAIL: 37 | %%%% WEBSITE: 38 | %%%% DATE: 39 | 40 | %%%%%%%%%%% Macro definition 41 | REGRESSION=0; 42 | CLASSIFICATION=1; 43 | 44 | %%%%%%%%%%% Load dataset 45 | %train_data=load(TrainingData_File); test_data=load(TestingData_File); 46 | %T=train_data(:,1); P=train_data(:,2:size(train_data,2)); 47 | %TV.T=test_data(:,1); TV.P=test_data(:,2:size(test_data,2)); 48 | %clear train_data test_data; 49 | 50 | %load cifar_kmeans_train1 51 | %load cifar_kmeans_train2 52 | %load cifar_kmeans_train3 53 | %train=vertcat(train1,train2,train3); 54 | %clear train1 train2 train3 55 | SPECTF = csvread('UMIST.csv'); 56 | P=SPECTF(:,2:size(SPECTF,2))/255; 57 | T=SPECTF(:,1); 58 | 59 | %clear SPECTF 60 | 61 | %P=train(:,1:(size(train,2)-1)); 62 | %T=train(:,size(train,2)); 63 | %clear train 64 | 65 | 66 | rand_sequence=randperm(size(P,1)); 67 | temp_P=P; 68 | temp_T=T; 69 | clear P; 70 | clear T; 71 | PP=temp_P(rand_sequence, :); 72 | TT=temp_T(rand_sequence, :); 73 | clear temp_P; 74 | clear temp_T; 75 | 76 | %load cifar_kmeans_test 77 | P=PP(1:400,:); 78 | T=TT(1:400,:); 79 | TV.P=PP(401:575,:); 80 | TV.T=TT(401:575,:); 81 | %TV.T=test(:,size(test,2)); 82 | %clear test 83 | %SPECTF1 = csvread('mnist_test_norm.csv'); 84 | %P=SPECTF1(:,2:size(SPECTF1,2))'; 85 | %TV.T=SPECTF1(:,1); 86 | 87 | %clear SPECTF1 88 | 89 | 90 | nTrainingData=size(P,1); 91 | nTestingData=size(TV.T,1); 92 | nInputNeurons=size(P,2); 93 | 94 | %%%%%%%%%%%% Preprocessing T in the case of CLASSIFICATION 95 | if Elm_Type==CLASSIFICATION 96 | sorted_target=sort(cat(1,T,TV.T),1); 97 | label=zeros(1,1); % Find and save in 'label' class label from training and testing data sets 98 | label(1,1)=sorted_target(1,1); 99 | j=1; 100 | for i = 2:(nTrainingData+nTestingData) 101 | if sorted_target(i,1) ~= label(j,1) 102 | j=j+1; 103 | label(j,1) = sorted_target(i,1); 104 | end 105 | end 106 | nClass=j; 107 | nOutputNeurons=nClass; 108 | 109 | %%%%%%%%%% Processing the targets of training 110 | temp_T=zeros(nTrainingData,nClass); 111 | for i = 1:nTrainingData 112 | for j = 1:nClass 113 | if label(j,1) == T(i,1) 114 | break; 115 | end 116 | end 117 | temp_T(i,j)=1; 118 | end 119 | T=temp_T*2-1; 120 | 121 | %%%%%%%%%% Processing the targets of testing 122 | temp_TV_T=zeros(nTestingData,nClass); 123 | for i = 1:nTestingData 124 | for j = 1:nClass 125 | if label(j,1) == TV.T(i,1) 126 | break; 127 | end 128 | end 129 | temp_TV_T(i,j)=1; 130 | end 131 | TV.T=temp_TV_T*2-1; 132 | end 133 | clear temp_T temp_TV_T sorted_target 134 | 135 | start_time_train=cputime; 136 | %%%%%%%%%%% step 1 Initialization Phase 137 | P0=P(1:N0,:); 138 | T0=T(1:N0,:); 139 | 140 | %P0 = gpuArray(P00); 141 | 142 | IW = rand(nHiddenNeurons,nInputNeurons)*2-1; 143 | switch lower(ActivationFunction) 144 | case{'rbf'} 145 | Bias = rand(1,nHiddenNeurons); 146 | % Bias = rand(1,nHiddenNeurons)*1/3+1/11; %%%%%%%%%%%%% for the cases of Image Segment and Satellite Image 147 | % Bias = rand(1,nHiddenNeurons)*1/20+1/60; %%%%%%%%%%%%% for the case of DNA 148 | H0 = RBFun(P0,IW,Bias); 149 | case{'sig'} 150 | Bias = rand(1,nHiddenNeurons)*2-1; 151 | H0 = SigActFun(P0,IW,Bias); 152 | case{'sin'} 153 | Bias = rand(1,nHiddenNeurons)*2-1; 154 | H0 = SinActFun(P0,IW,Bias); 155 | case{'hardlim'} 156 | Bias = rand(1,nHiddenNeurons)*2-1; 157 | H0 = HardlimActFun(P0,IW,Bias); 158 | H0 = double(H0); 159 | end 160 | 161 | M = pinv(H0' * H0); 162 | beta = pinv(H0) * T0; 163 | clear P0 T0 H0; 164 | 165 | %%%%%%%%%%%%% step 2 Sequential Learning Phase 166 | for n = N0 : Block : nTrainingData 167 | if (n+Block-1) > nTrainingData 168 | Pn = P(n:nTrainingData,:); Tn = T(n:nTrainingData,:); 169 | Block = size(Pn,1); %%%% correct the block size 170 | clear V; %%%% correct the first dimention of V 171 | else 172 | Pn = P(n:(n+Block-1),:); Tn = T(n:(n+Block-1),:); 173 | end 174 | 175 | 176 | %Pn = gpuArray(Pnn); 177 | switch lower(ActivationFunction) 178 | case{'rbf'} 179 | H = RBFun(Pn,IW,Bias); 180 | case{'sig'} 181 | H = SigActFun(Pn,IW,Bias); 182 | case{'sin'} 183 | H = SinActFun(Pn,IW,Bias); 184 | case{'hardlim'} 185 | H = HardlimActFun(Pn,IW,Bias); 186 | end 187 | M = M - M * H' * (eye(Block) + H * M * H')^(-1) * H * M; 188 | beta = beta + M * H' * (Tn - H * beta); 189 | end 190 | end_time_train=cputime; 191 | TrainingTime=end_time_train-start_time_train 192 | clear Pn Tn H M; 193 | 194 | switch lower(ActivationFunction) 195 | case{'rbf'} 196 | HTrain = RBFun(P, IW, Bias); 197 | case{'sig'} 198 | HTrain = SigActFun(P, IW, Bias); 199 | case{'sin'} 200 | HTrain = SinActFun(P, IW, Bias); 201 | case{'hardlim'} 202 | HTrain = HardlimActFun(P, IW, Bias); 203 | end 204 | Y=HTrain * beta; 205 | clear HTrain; 206 | 207 | %load cifar_kmeans_test 208 | %SPECTF1 = csvread('mnist_test_norm.csv'); 209 | %TV.P=SPECTF1(:,2:size(SPECTF1,2)); 210 | %TV.T=SPECTF1(:,1)'; 211 | 212 | 213 | %TV.P=test(:,1:(size(test,2)-1)); 214 | %clear SPECTF1 215 | %%%%%%%%%%% Performance Evaluation 216 | start_time_test=cputime; 217 | switch lower(ActivationFunction) 218 | case{'rbf'} 219 | HTest = RBFun(TV.P, IW, Bias); 220 | case{'sig'} 221 | HTest = SigActFun(TV.P, IW, Bias); 222 | case{'sin'} 223 | HTest = SinActFun(TV.P, IW, Bias); 224 | case{'hardlim'} 225 | HTest = HardlimActFun(TV.P, IW, Bias); 226 | end 227 | TY=HTest * beta; 228 | clear HTest; 229 | end_time_test=cputime; 230 | TestingTime=end_time_test-start_time_test 231 | 232 | if Elm_Type == REGRESSION 233 | %%%%%%%%%%%%%% Calculate RMSE in the case of REGRESSION 234 | TrainingAccuracy=sqrt(mse(T - Y)) 235 | TestingAccuracy=sqrt(mse(TV.T - TY)) 236 | elseif Elm_Type == CLASSIFICATION 237 | %%%%%%%%%% Calculate correct classification rate in the case of CLASSIFICATION 238 | MissClassificationRate_Training=0; 239 | MissClassificationRate_Testing=0; 240 | 241 | for i = 1 : nTrainingData 242 | [x, label_index_expected]=max(T(i,:)); 243 | [x, label_index_actual]=max(Y(i,:)); 244 | if label_index_actual~=label_index_expected 245 | MissClassificationRate_Training=MissClassificationRate_Training+1; 246 | end 247 | end 248 | TrainingAccuracy=1-MissClassificationRate_Training/nTrainingData 249 | for i = 1 : nTestingData 250 | [x, label_index_expected]=max(TV.T(i,:)); 251 | [x, label_index_actual]=max(TY(i,:)); 252 | if label_index_actual~=label_index_expected 253 | MissClassificationRate_Testing=MissClassificationRate_Testing+1; 254 | end 255 | end 256 | TestingAccuracy=1-MissClassificationRate_Testing/nTestingData 257 | end 258 | -------------------------------------------------------------------------------- /MELM_UMISTseq.m: -------------------------------------------------------------------------------- 1 | function [TrainingTime, TestingTime, TrainingAccuracy, TestingAccuracy] = MELM_MNISTseq(Testcode, TrainDataSize, TotalLayers, HiddernNeurons, C1, rhoValue, sigpara, sigpara1) 2 | 3 | % Usage: elm(TrainingData_File, TestingData_File, Elm_Type, NumberofHiddenNeurons, ActivationFunction) 4 | % OR: [TrainingTime, TestingTime, TrainingAccuracy, TestingAccuracy] = elm(TrainingData_File, TestingData_File, Elm_Type, NumberofHiddenNeurons, ActivationFunction) 5 | % 6 | % Input: 7 | % TrainingData_File - Filename of training data set 8 | % TestingData_File - Filename of testing data set 9 | % Elm_Type - 0 for regression; 1 for (both binary and multi-classes) classification 10 | % NumberofHiddenNeurons - Number of hidden neurons assigned to the ELM 11 | % ActivationFunction - Type of activation function: 12 | % 'sig' for Sigmoidal function 13 | % 'sin' for Sine function 14 | % 'hardlim' for Hardlim function 15 | % 'tribas' for Triangular basis function 16 | % 'radbas' for Radial basis function (for additive type of SLFNs instead of RBF type of SLFNs) 17 | % 18 | % Output: 19 | % TrainingTime - Time (seconds) spent on training ELM 20 | % TestingTime - Time (seconds) spent on predicting ALL testing data 21 | % TrainingAccuracy - Training accuracy: 22 | % RMSE for regression or correct classification rate for classification 23 | % TestingAccuracy - Testing accuracy: 24 | % RMSE for regression or correct classification rate for classification 25 | % 26 | % MULTI-CLASSE CLASSIFICATION: NUMBER OF OUTPUT NEURONS WILL BE AUTOMATICALLY SET EQUAL TO NUMBER OF CLASSES 27 | % FOR EXAMPLE, if there are 7 classes in all, there will have 7 output 28 | % neurons; neuron 5 has the highest output means input belongs to 5-th class 29 | % 30 | % Sample1 regression: [TrainingTime, TestingTime, TrainingAccuracy, TestingAccuracy] = elm('sinc_train', 'sinc_test', 0, 20, 'sig') 31 | % Sample2 classification: elm('diabetes_train', 'diabetes_test', 1, 20, 'sig') 32 | % 33 | %%%% Authors: MR QIN-YU ZHU AND DR GUANG-BIN HUANG 34 | %%%% NANYANG TECHNOLOGICAL UNIVERSITY, SINGAPORE 35 | %%%% EMAIL: EGBHUANG@NTU.EDU.SG; GBHUANG@IEEE.ORG 36 | %%%% WEBSITE: http://www.ntu.edu.sg/eee/icis/cv/egbhuang.htm 37 | %%%% DATE: APRIL 2004 38 | 39 | 40 | N0=150; %%use NO=400 for batch learning% 41 | Block=50; 42 | 43 | 44 | SPECTF = csvread('UMIST.csv'); 45 | P=SPECTF(:,2:size(SPECTF,2))'/255; 46 | T=SPECTF(:,1)'; 47 | 48 | 49 | clear SPECTF 50 | 51 | 52 | if TrainDataSize ~= 0 53 | rand_sequence=randperm(size(P,2)); 54 | temp_P=P'; 55 | temp_T=T'; 56 | clear P; 57 | clear T; 58 | PP=temp_P(rand_sequence, :)'; 59 | TT=temp_T(rand_sequence, :)'; 60 | clear temp_P; 61 | clear temp_T; 62 | end 63 | 64 | %%%%%%%%%%% Load testing dataset 65 | % TV.T = loadMNISTLabels('mnist/t10k-labels.idx1-ubyte'); 66 | % TV.T = TV.T'; % Release raw testing data array 67 | %SPECTF1 = csvread('mnist_test_zs.csv'); 68 | 69 | %TV.T=SPECTF1(:,1)'; 70 | %clear SPECTF1 71 | P=PP(:,1:400); 72 | T=TT(:,1:400); 73 | TV.P=PP(:,401:575); 74 | TV.T=TT(:,401:575); 75 | 76 | 77 | NumberofTrainingData=size(P,2); 78 | NumberofTestingData=size(TV.T,2); 79 | NumberofInputNeurons=size(P,1); 80 | 81 | 82 | 83 | %%%%%%%%%%%% Preprocessing the data of classification 84 | sorted_target=sort(cat(2,T,TV.T),2); 85 | label=zeros(1,1); % Find and save in 'label' class label from training and testing data sets 86 | label(1,1)=sorted_target(1,1); 87 | j=1; 88 | for i = 2:(NumberofTrainingData+NumberofTestingData) 89 | if sorted_target(1,i) ~= label(1,j) 90 | j=j+1; 91 | label(1,j) = sorted_target(1,i); 92 | end 93 | end 94 | number_class=j; 95 | NumberofOutputNeurons=number_class; 96 | 97 | %%%%%%%%%% Processing the targets of training 98 | temp_T=zeros(NumberofOutputNeurons, NumberofTrainingData); 99 | for i = 1:NumberofTrainingData 100 | for j = 1:number_class 101 | if label(1,j) == T(1,i) 102 | break; 103 | end 104 | end 105 | temp_T(j,i)=1; 106 | end 107 | T=temp_T*2-1; 108 | 109 | %%%%%%%%%% Processing the targets of testing 110 | temp_TV_T=zeros(NumberofOutputNeurons, NumberofTestingData); 111 | for i = 1:NumberofTestingData 112 | for j = 1:number_class 113 | if label(1,j) == TV.T(1,i) 114 | break; 115 | end 116 | end 117 | temp_TV_T(j,i)=1; 118 | end 119 | TV.T=temp_TV_T*2-1; 120 | 121 | 122 | 123 | %%%%%%%%%%% Calculate weights & biases 124 | train_time=tic; 125 | no_Layers = TotalLayers; 126 | stack = cell(no_Layers+1,1); 127 | % stack1= cell(no_Layers+1,1); 128 | % stacki=cell(no_Layers,1); 129 | % stackb=cell(no_Layers,1); 130 | lenHN = length(HiddernNeurons); 131 | lenC1 = length(C1); 132 | lensig = length(sigpara); 133 | lensig1 = length(sigpara1); 134 | HN_temp = [NumberofInputNeurons,HiddernNeurons(1:lenHN-1)]; 135 | if length(HN_temp) < no_Layers 136 | % HN = [ HN_temp, repmat( HN_temp( length( HN_temp ) ),1,no_Layers-length(HN_temp) ), HiddernNeurons(lenHN) ]; 137 | HN = [ HN_temp,4000, HiddernNeurons(lenHN) ]; 138 | 139 | C = [C1(1:lenC1-2), zeros(1,no_Layers - length(HN_temp) ), C1(lenC1-1:lenC1) ]; 140 | % sigscale = [sigpara(1:lensig-1),repmat(sigpara(lensig-1),1,no_Layers - length(HN_temp)),sigpara(lensig)]; 141 | sigscale = [sigpara(1:lensig-1),ones(1,no_Layers - length(HN_temp)),sigpara(lensig)]; 142 | sigscale1 = [sigpara1(1:lensig1-1),ones(1,no_Layers - length(HN_temp)),sigpara1(lensig1)]; 143 | else 144 | HN = [NumberofInputNeurons,HiddernNeurons]; 145 | C = C1; 146 | sigscale = sigpara; 147 | sigscale1 = sigpara1; 148 | end 149 | clear HN_temp; 150 | 151 | P0=P(:,1:N0); 152 | T0=T(:,1:N0); 153 | 154 | % InputDataLayer = zscore(P0); 155 | InputDataLayer = P0; 156 | clear P0; 157 | 158 | % s=rng; 159 | if Testcode == 1 160 | rng('default'); 161 | % randomWeightRng = 0; 162 | % else 163 | % randomWeightRng = s.Seed; 164 | end 165 | % prevRowWeight = -1; 166 | % prevColWeight = -1; 167 | for i=1:1:no_Layers 168 | 169 | % if prevRowWeight ~= HN(i+1) || prevColWeight ~= HN(i) 170 | % randomWeightRng = randomWeightRng +1; 171 | % end 172 | % prevRowWeight= HN(i+1); 173 | % prevColWeight = HN(i); 174 | % rng(randomWeightRng); 175 | InputWeight=rand(HN(i+1),HN(i))*2 -1; 176 | if HN(i+1) > HN(i) 177 | InputWeight = orth(InputWeight); 178 | stack{i}.ipw=InputWeight; 179 | else 180 | InputWeight = orth(InputWeight')'; 181 | stack{i}.ipw =InputWeight; 182 | end 183 | % rng(randomWeightRng); 184 | BiasofHiddenNeurons=rand(HN(i+1),1)*2 -1; 185 | BiasofHiddenNeurons=orth(BiasofHiddenNeurons); 186 | stack{i}.bias=BiasofHiddenNeurons; 187 | tempH=InputWeight*InputDataLayer; % Release input of training data 188 | clear InputWeight; 189 | 190 | 191 | 192 | 193 | % H0 = SigActFun(InputDataLayer',InputWeight,BiasofHiddenNeurons')'; 194 | ind=ones(1,size(T0,2)); 195 | BiasMatrix=BiasofHiddenNeurons(:,ind); % Extend the bias matrix BiasofHiddenNeurons to match the demention of H 196 | tempH=tempH+BiasMatrix; 197 | clear BiasMatrix; 198 | clear BiasofHiddenNeurons; 199 | fprintf(1,'AutoEncorder Max Val %f Min Val %f\n',max(tempH(:)),min(tempH(:))); 200 | H0 = 1 ./ (1 + exp(-sigscale1(i)*tempH)); 201 | clear tempH; % Release the temparary array for calculation of hidden neuron output matrix H 202 | 203 | %%%%%%%%%%% Calculate output weights OutputWeight (beta_i) 204 | %if HN(i+1) == HN(i) 205 | % [~,stack{i}.w,~] = procrustNew( InputDataLayer',H0'); 206 | %else 207 | if C(i) == 0 208 | stack{i}.m = pinv(H0 * H0'); 209 | stack{i}.w =pinv(H0') * InputDataLayer'; % implementation without regularization factor //refer to 2006 Neurocomputing paper 210 | else 211 | rhohats = mean(H,2); 212 | rho = rhoValue; 213 | KLsum = sum(rho * log(rho ./ rhohats) + (1-rho) * log((1-rho) ./ (1-rhohats))); 214 | 215 | Hsquare = H * H'; 216 | HsquareL = diag(max(Hsquare,[],2)); 217 | stack{i}.w=( ( eye(size(H,1)).*KLsum +HsquareL )*(1/C(i))+Hsquare) \ (H * InputDataLayer'); 218 | 219 | clear Hsquare; 220 | clear HsquareL; 221 | end 222 | %end 223 | 224 | tempH=(stack{i}.w) *(InputDataLayer); 225 | clear InputDataLayer; 226 | 227 | if HN(i+1) == HN(i) 228 | InputDataLayer = tempH; 229 | else 230 | fprintf(1,'Layered Max Val %f Min Val %f\n',max(tempH(:)),min(tempH(:))); 231 | InputDataLayer = 1 ./ (1 + exp(-sigscale(i)*tempH)); 232 | end 233 | 234 | clear tempH; 235 | clear H; 236 | end 237 | 238 | 239 | if C(no_Layers+1) == 0 240 | stack{no_Layers+1}.m=pinv(InputDataLayer * InputDataLayer') ; 241 | stack{no_Layers+1}.w=pinv(InputDataLayer') * T0'; 242 | else 243 | stack{no_Layers+1}.w=(eye(size(InputDataLayer,1))/C(no_Layers+1)+InputDataLayer * InputDataLayer') \ ( InputDataLayer * T0'); 244 | end 245 | %N0=N0+1; uncomment for batch learning 246 | 247 | for ii = N0 : Block : NumberofTrainingData 248 | if (ii+Block-1) > NumberofTrainingData 249 | Pn = P(:,ii:NumberofTrainingData); Tn = T(:,ii:NumberofTrainingData); 250 | Block = size(Pn,2); %%%% correct the block size 251 | % clear V; %%%% correct the first dimention of V 252 | else 253 | Pn = P(:,ii:(ii+Block-1)); Tn = T(:,ii:(ii+Block-1)); 254 | end 255 | 256 | % InputDataLayer=zscore(Pn); 257 | InputDataLayer=Pn; 258 | clear Pn 259 | for i=1:1:no_Layers 260 | 261 | tempH=stack{i}.ipw*InputDataLayer; 262 | ind=ones(1,size(Tn,2)); 263 | BiasMatrix=stack{i}.bias(:,ind); 264 | tempH=tempH+BiasMatrix; 265 | clear BiasMatrix 266 | H = 1 ./ (1 + exp(-sigscale1(i)*tempH)); 267 | clear tempH; % Release the temparary array for calculation of hidden neuron output matrix H 268 | %H = SigActFun(InputDataLayer',stack{i}.ipw,stack{i}.bias')'; 269 | % stack{i}.m = pinv(H0 * H0'); 270 | % stack{i}.w =pinv(H0') * InputDataLayer'; 271 | stack{i}.m=stack{i}.m - stack{i}.m * H * (eye(Block) +H'*stack{i}.m*H)^(-1)*H'*stack{i}.m; 272 | % M = M - M * H' * (eye(Block) + H * M * H')^(-1) * H * M; 273 | 274 | stack{i}.w = stack{i}.w + stack{i}.m*H*(InputDataLayer'-H'*stack{i}.w); 275 | %beta = beta + M * H' * (Tn - H * beta); 276 | tempH=(stack{i}.w) *(InputDataLayer); 277 | clear InputDataLayer; 278 | if HN(i+1) == HN(i) 279 | InputDataLayer = tempH; 280 | else 281 | fprintf(1,'Layered Max Val %f Min Val %f\n',max(tempH(:)),min(tempH(:))); 282 | InputDataLayer = 1 ./ (1 + exp(-sigscale(i)*tempH)); 283 | end 284 | 285 | clear tempH; 286 | clear H; 287 | end 288 | %stack{no_Layers+1}.m=pinv(InputDataLayer * InputDataLayer') ; 289 | stack{no_Layers+1}.m=stack{no_Layers+1}.m - stack{no_Layers+1}.m * InputDataLayer * (eye(Block) +InputDataLayer'*stack{no_Layers+1}.m*InputDataLayer)^(-1)*InputDataLayer'*stack{no_Layers+1}.m; 290 | 291 | % stack{no_Layers+1}.w=pinv(InputDataLayer') * T'; 292 | 293 | stack{no_Layers+1}.w = stack{no_Layers+1}.w + stack{no_Layers+1}.m*InputDataLayer*(Tn'-InputDataLayer'*stack{no_Layers+1}.w); 294 | 295 | end 296 | 297 | 298 | TrainingTime=toc(train_time); % Calculate CPU time (seconds) spent for training ELM 299 | % display_network(orth(stack{1}.w')); 300 | %%%%%%%%%%% Calculate the training accuracy 301 | Y=(InputDataLayer' * stack{no_Layers+1}.w)'; % Y: the actual output of the training data 302 | 303 | clear InputDataLayer; 304 | clear H; 305 | 306 | 307 | 308 | 309 | %SPECTF1 = csvread('mnist_test_zs.csv'); 310 | 311 | %TV.P=SPECTF1(:,2:size(SPECTF1,2))'; 312 | 313 | clear SPECTF1 314 | 315 | %%%%%%%%%%% Calculate the output of testing input 316 | test_time=tic; 317 | % InputDataLayer = zscore(TV.P); 318 | InputDataLayer = TV.P; 319 | clear TV.P; 320 | 321 | for i=1:1:no_Layers 322 | tempH_test=(stack{i}.w)*(InputDataLayer); 323 | clear TV.P; % Release input of testing data 324 | 325 | if HN(i+1) == HN(i) 326 | InputDataLayer = tempH_test; 327 | else 328 | InputDataLayer = 1 ./ (1 + exp(-sigscale(i)*tempH_test)); 329 | end 330 | clear tempH_test; 331 | 332 | end 333 | 334 | TY=(InputDataLayer' * stack{no_Layers+1}.w)'; % TY: the actual output of the testing data 335 | 336 | TestingTime=toc(test_time); % Calculate CPU time (seconds) spent by ELM predicting the whole testing data 337 | 338 | 339 | 340 | %%%%%%%%%% Calculate training & testing classification accuracy 341 | MissClassificationRate_Training=0; 342 | MissClassificationRate_Testing=0; 343 | 344 | 345 | TrainingAccuracy=HN; 346 | for i = 1 : size(TV.T, 2) 347 | [x, label_index_expected]=max(TV.T(:,i)); 348 | [x, label_index_actual]=max(TY(:,i)); 349 | if label_index_actual~=label_index_expected 350 | MissClassificationRate_Testing=MissClassificationRate_Testing+1; 351 | end 352 | end 353 | TestingAccuracy=1-MissClassificationRate_Testing/size(TV.T,2); 354 | 355 | 356 | 357 | 358 | 359 | --------------------------------------------------------------------------------