├── OutlierIX.m ├── CauchySCL.m ├── CauchyLpzCC.mexw64 ├── CauchyMedFilter.m ├── GetStopCriterion.m ├── CauchyLpzConstt.m ├── CauchyOGM.m ├── CauchyOutlIndex.m ├── README.md ├── CauchyNNLS.m ├── CauchyNLS.m ├── CauchyNMF.m └── CauchyNMF_basis.m /OutlierIX.m: -------------------------------------------------------------------------------- 1 | function IX=OutlierIX(Z) 2 | x=Z(Z<=median(Z(:))); 3 | IX=(abs(Z-mean(x))>=3*std(x)); 4 | return; -------------------------------------------------------------------------------- /CauchySCL.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NaiyangGuan/Truncated-Cauchy-Non-Negative-Matrix-Factorization/HEAD/CauchySCL.m -------------------------------------------------------------------------------- /CauchyLpzCC.mexw64: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NaiyangGuan/Truncated-Cauchy-Non-Negative-Matrix-Factorization/HEAD/CauchyLpzCC.mexw64 -------------------------------------------------------------------------------- /CauchyMedFilter.m: -------------------------------------------------------------------------------- 1 | function P=CauchyMedFilter(Q,im_h,im_w) 2 | P=Q; 3 | for num=1:size(Q,2), 4 | I=reshape(Q(:,num),im_h,im_w)*255; 5 | I=medfilt2(I); 6 | P(:,num)=reshape(double(I)/255,im_h*im_w,1); 7 | end 8 | return; -------------------------------------------------------------------------------- /GetStopCriterion.m: -------------------------------------------------------------------------------- 1 | function retVal=GetStopCriterion(stop_rule,X,gradX) 2 | % Stopping Criterions 3 | % Written by Naiyang (ny.guan@gmail.com) 4 | 5 | switch stop_rule 6 | case 1 7 | pGrad=gradX(gradX<0|X>0); 8 | retVal=norm(pGrad); 9 | case 2 10 | pGrad=gradX(gradX<0|X>0); 11 | pGradNorm=norm(pGrad); 12 | retVal=pGradNorm/length(pGrad); 13 | case 3 14 | resmat=min(X,gradX); resvec=resmat(:); 15 | deltao=norm(resvec,1); %L1-norm 16 | num_notconv=length(find(abs(resvec)>0)); 17 | retVal=deltao/num_notconv; 18 | end -------------------------------------------------------------------------------- /CauchyLpzConstt.m: -------------------------------------------------------------------------------- 1 | function LpzH=CauchyLpzConstt(LPZ_TYPE,W,Q) 2 | n=size(Q,2); 3 | r=size(W,2); 4 | switch upper(LPZ_TYPE), 5 | case 'PLAIN', % Exact Lipschitz constants 6 | LpzH=ones(1,n); 7 | for i=1:n, 8 | LpzH(i)=norm(W'*((Q(:,i)*ones(1,r)).*W)); 9 | end 10 | case 'CCOMP', % Exact Lipschitz constants 11 | LpzH=CauchyLpzCC(W,Q); 12 | case 'RELAX', % Inexact Lipschitz constants 13 | LpzH=norm(W')*sqrt(sum(Q.^2))*max(sqrt(sum(W.^2,2))); 14 | otherwise, 15 | error(['Unrecognized Lipschitz constant type: ',LPZ_TYPE]); 16 | end 17 | return; -------------------------------------------------------------------------------- /CauchyOGM.m: -------------------------------------------------------------------------------- 1 | function [H,iterH]=CauchyOGM(V,W,H0,Q,iterMin,tolH,iterMax,LPZ_TYPE) 2 | [r,n]=size(H0); 3 | L=CauchyLpzConstt(LPZ_TYPE,W,Q); 4 | Y=H0; % Search Point 5 | Grad=W'*(Q.*(W*Y-V)); % Gradient at Y 6 | alpha0=1; 7 | init_pgn=GetStopCriterion(1,Y,Grad); 8 | for k=0:iterMax-1, 9 | % Gradient Descent and Update 10 | H1=max(0,Y-Grad./(ones(r,1)*L)); 11 | alpha1=(1+sqrt(4*alpha0^2+1))/2; 12 | Y=H1+(alpha0-1)*(H1-H0)/alpha1; 13 | % Update for Next Recursion 14 | H0=H1; 15 | alpha0=alpha1; 16 | Grad=W'*(Q.*(W*Y-V)); 17 | % Stopping Criteria 18 | if k>=iterMin-1, 19 | % Lin's stopping condition 20 | pgn=GetStopCriterion(1,Y,Grad); 21 | if pgn=3*std(x)); 6 | case 'LOCAL', 7 | n=size(R,2); % Number of samples 8 | medn=median(R,2)*ones(1,n); % Median of each dimension 9 | id=(R<=medn); % Indicator of entries under median 10 | nn=sum(id,2); % Number of entries under median 11 | mn=(sum(R.*id,2)./nn)*ones(1,n); % Mean of each dimension 12 | stdn=sqrt((sum((((R-mn).*id).^2),2))./(nn-1))*ones(1,n); % Standard deviation of each dimension 13 | Index=(abs(R-mn)>=3*stdn); % Outlier indicator by three-sigma theory 14 | otherwise, 15 | error(['Unknown viewpoint: ',view]); 16 | end 17 | return; -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Truncated-Cauchy-Non-Negative-Matrix-Factorization 2 | Non-negative matrix factorization (NMF) minimizes the Euclidean distance between the data matrix and its low rank approximation, and it fails when applied to corrupted data because the loss function is sensitive to outliers. In this paper, we propose a Truncated CauchyNMF loss that handle outliers by truncating large errors, and develop a Truncated CauchyNMF to robustly learn the subspace on noisy datasets contaminated by outliers. We theoretically analyze the robustness of Truncated CauchyNMF comparing with the competing models and theoretically prove that Truncated CauchyNMF has a generalization bound which converges at a rate of order $O(\sqrt{{\ln n}/{n}})$, where $n$ is the sample size. We evaluate Truncated CauchyNMF by image clustering on both simulated and real datasets. The experimental results on the datasets containing gross corruptions validate the effectiveness and robustness of Truncated CauchyNMF for learning robust subspaces. 3 | -------------------------------------------------------------------------------- /CauchyNNLS.m: -------------------------------------------------------------------------------- 1 | function H=CauchyNNLS(W,V,gamma,tol) 2 | 3 | n=size(V,2); 4 | r=size(W,2); 5 | H=rand(r,n); 6 | tolH=max(tol,1e-2*ones(1,n)); 7 | iterH=zeros(1,n); 8 | E=abs(V-W*H); 9 | % scale parameter 10 | if gamma<0, 11 | gamma=sqrt(sum(sum(E.^2))/(2*m*n)); 12 | end 13 | Z=(E/gamma).^2+1; 14 | obj=sum(log(Z(:))); 15 | verbose=0; 16 | 17 | if verbose, 18 | fprintf('********** CauchyNNLS **********\n'); 19 | end 20 | 21 | for iter=1:1000, 22 | % Update H 23 | Q=1./Z; 24 | IX=OutlierIX(E); 25 | nn=sum(IX,2); 26 | IX(nn<=median(nn),:)=0; 27 | Q(IX)=0; 28 | 29 | for j=1:n, 30 | [Hj,iterH(j)]=OGM(V(:,j),W,H(:,j),Q(:,j),10,tolH(j),1000); 31 | H(:,j)=Hj; 32 | if iterH(j)<=10, 33 | tolH(j)=tolH(j)/10; 34 | end 35 | end 36 | E=abs(V-W*H); 37 | if gamma<0, 38 | gamma=sqrt(sum(sum(E.^2))/(2*m*n)); 39 | end 40 | Z=(E/gamma).^2+1; 41 | 42 | obj(iter+1)=sum(log(Z(:))); 43 | if verbose, 44 | fprintf('iter=%d, objective function=%f, gamma=%f.\n',iter,obj(iter+1),gamma); 45 | end 46 | 47 | stop=abs(obj(end-1)-obj(end))/abs(obj(1)-obj(end)); 48 | if stop<=tol && iter>=10, 49 | break; 50 | end 51 | end 52 | 53 | return; 54 | 55 | function [H,iterH]=OGM(V,W,H0,Q,iterMin,tolH,iterMax) 56 | WtW=W'*((Q*ones(1,size(W,2))).*W); 57 | L=norm(WtW); 58 | WtV=W'*((Q*ones(1,size(V,2))).*V); 59 | Y=H0; % Search Point 60 | Grad=WtW*Y-WtV; % Gradient at Y 61 | alpha0=1; 62 | init_pgn=GetStopCriterion(1,Y,Grad); 63 | for k=0:iterMax-1, 64 | % Gradient Descent and Update 65 | H1=max(0,Y-Grad/L); 66 | alpha1=(1+sqrt(4*alpha0^2+1))/2; 67 | Y=H1+(alpha0-1)*(H1-H0)/alpha1; 68 | % Update for Next Recursion 69 | H0=H1; 70 | alpha0=alpha1; 71 | Grad=WtW*Y-WtV; 72 | % Stopping Criteria 73 | if k>=iterMin-1, 74 | % Lin's stopping condition 75 | pgn=GetStopCriterion(1,Y,Grad); 76 | if pgn=MIN_ITER, 136 | break; 137 | end 138 | end 139 | % Support of error, 1 for clean entry, and 0 for dirty entry 140 | switch upper(WEI_TYPE), 141 | case {'ROBUSTG', 'ROBUSTL'}, 142 | Suppt=~IX; 143 | otherwise, 144 | Suppt=true(m,n); 145 | end 146 | return; -------------------------------------------------------------------------------- /CauchyNMF.m: -------------------------------------------------------------------------------- 1 | function [W,H,HIS]=CauchyNMF(V,r,varargin) 2 | 3 | % Cauchy Nonnegative Matrix Factorization via Nesterov Based Half-Quadratic Programming. 4 | 5 | % The model is V \approx WH, where V, W, and H are defined as follows: 6 | % V (m x n): data matrix including n samples in m-dimensional space; 7 | % W (m x r): basis matrix including r bases in m-dimensional space; 8 | % H (r x n): coefficients matrix including n encodings in r-dimensional space. 9 | 10 | % Written by Naiyang Guan (ny.guan@gmail.com). 11 | % Copyright @ 2013-2023 by Naiyang Guan and Dacheng Tao. 12 | % Last Modified at 13 | % Sep. 06 2014; 14 | % Nov. 13 2014; 15 | % Aug. 07 2015; 16 | % Aug. 12 2015; 17 | % Jan. 06 2016; 18 | % Jan. 16 2017. 19 | 20 | % 21 | % V : Input data matrix (m x n), should be scaled to [0,1] in the 22 | % default sense 23 | % r : Target low-rank 24 | % 25 | % (Below are optional arguments: can be set by providing name-value 26 | % pairs). 27 | % MAX_ITER : Maximum number of iterations. Default is 1,000. 28 | % MIN_ITER : Minimum number of iterations. Default is 10. 29 | % MAX_TIME : Maximum amount of time in seconds. Default is 10,000. 30 | % ALG_TYPE : Type of algorithms. 31 | % 'OGM' for Nesterov's optimal gradient method, 32 | % 'MUR' for multiplicative update rule, 33 | % 'HALS' for weighted HALS algorithm, 34 | % 'BCD' for linear-proximal block coordinate descent. 35 | % WEI_TYPE : Type of weighting. 36 | % 'plain' for original data-adapative weighting, 37 | % 'tgv' for robustly trimmed weighting in global viewpoint, 38 | % 'tlv' for robustly trimmed weighting in local viewpoint, 39 | % 'mrf' for trimmed weighting by detecting supports by MRF, 40 | % 'med' for trimmed weighting with median filtering. 41 | % LPZ_TYPE : Type of Lipschitz constant. 42 | % 'plain' for calculating by Matlab, 43 | % 'ccomp' for calculating by MEX-C and OpenMP, 44 | % 'relax' for approximation. 45 | % IMG_INFO : Struct format, contains two records, i.e., height and 46 | % width, formatted by [height,width]. 47 | % SCALE : Scale parameter. Default is 0.1. If gamma < 0, automatic. 48 | % -1 : Nagy algorithm for scale estimation, 49 | % -2 : Newton algorithm for scale estimation. 50 | % W_INIT : (m x r) Initial value for W. 51 | % H_INIT : (r x n) Initial value for H. 52 | % TOL : Stopping tolerance, [outer one, inner one]. Default is [1e-4,1e-2]. 53 | % If you want to obtain a more accurate solution, decrease TOL and increase MAX_TIME simultanuously. 54 | % VERBOSE : 0 (default) - No debugging information. 55 | % 1 (debugging purpose) - History of computation is printed on screen. 56 | % 57 | % W : Obtained basis matrix (m x r), 58 | % H : Obtained coefficients matrix (r x n), 59 | % HIS : (debugging purpose) History of computation. 60 | % 61 | % 62 | % >>A=rand(100); 63 | % >>CauchyNMF(A,10); 64 | % >>CauchyNMF(A,20,'verbose',1); 65 | % >>CauchyNMF(A,30,'verbose',2,'w_init',rand(100,30)); 66 | % >>CauchyNMF(A,5,'verbose',2,'tol',1e-5); 67 | 68 | [m,n]=size(V); 69 | if ~isa(V,'double'), V=double(V); end 70 | if ~exist('V','var'), error('please input the sample matrix.\n'); end 71 | if ~exist('r','var'), error('please input the low rank.\n'); end 72 | if n<=r || m<=r, error('Too high low-rank (n<=r or m<=r).\n'); end 73 | 74 | % Default setting 75 | MIN_ITER=20; % minimum number of outer iterations 76 | MAX_ITER=5000; % maximum number of outer iterations 77 | INN_MIN=20; % minimum number of inner iterations 78 | INN_MAX=500; % maximum number of inner iterations 79 | MAX_TIME=10000; % maximum running time 80 | ALG_TYPE='ogm'; % algorithm's type, default is OGM 81 | WEI_TYPE='plain'; % weighting's type, default is PLAIN 82 | LPZ_TYPE='plain'; % Lipschitz constant's type, default is PLAIN 83 | SCL_TYPE='plain'; % algorithm's type of scale estimation 84 | gamma=0.1; % scale parameter 85 | tol=[1e-4,1e-2]; % tolerance of outer and inner loops 86 | verbose=0; 87 | img_info=[sqrt(m),sqrt(m)]; % information [height, width] for images 88 | W0=rand(m,r); 89 | H0=rand(r,n); 90 | 91 | % Read optional parameters 92 | if (rem(length(varargin),2)==1), 93 | error('Optional parameters should always go by pairs'); 94 | else 95 | for i=1:2:(length(varargin)-1), 96 | switch upper(varargin{i}), 97 | case 'MAX_ITER', MAX_ITER=varargin{i+1}; 98 | case 'MIN_ITER', MIN_ITER=varargin{i+1}; 99 | case 'MAX_TIME', MAX_TIME=varargin{i+1}; 100 | case 'ALG_TYPE', ALG_TYPE=varargin{i+1}; 101 | case 'WEI_TYPE', WEI_TYPE=varargin{i+1}; 102 | case 'LPZ_TYPE', LPZ_TYPE=varargin{i+1}; 103 | case 'IMG_INFO', img_info=varargin{i+1}; 104 | case 'SCALE', gamma=varargin{i+1}; 105 | case 'W_INIT', W0=varargin{i+1}; 106 | case 'H_INIT', H0=varargin{i+1}; 107 | case 'TOL', tol=varargin{i+1}; 108 | case 'VERBOSE', verbose=varargin{i+1}; 109 | otherwise 110 | error(['Unrecognized option: ',varargin{i}]); 111 | end 112 | end 113 | end 114 | 115 | % Initialization 116 | switch upper(WEI_TYPE), 117 | case 'PLAIN', 118 | Z=W0*H0; 119 | s=max(Z(:))/max(median(V(:)),min(Z(:))); 120 | W0=W0/sqrt(s); 121 | H0=H0/sqrt(s); 122 | case {'TGV', 'TLV', 'MED'}, 123 | R=V-W0*H0; 124 | delta=sum(sum(R.^2))/(2*m*n); 125 | Q=exp(-(R.^2)/(2*delta)); 126 | W0=W0.*((Q.*V)*H0')./((Q.*(W0*H0))*H0'+eps); 127 | H0=H0.*(W0'*(Q.*V))./(W0'*(Q.*(W0*H0))+eps); 128 | case 'MRF', 129 | R=V-W0*H0; 130 | delta=sum(sum(R.^2))/(2*m*n); 131 | Q=exp(-(R.^2)/(2*delta)); 132 | W0=W0.*((Q.*V)*H0')./((Q.*(W0*H0))*H0'+eps); 133 | H0=H0.*(W0'*(Q.*V))./(W0'*(Q.*(W0*H0))+eps); 134 | height=img_info(1); 135 | width=img_info(2); 136 | NS=edges4connected(height,width); 137 | tau=0.17; 138 | lambda=3; 139 | otherwise, 140 | error(['Unknown weighting type: ',WEI_TYPE]); 141 | end 142 | Z=W0*H0; 143 | 144 | % Algorithmic Settings 145 | switch upper(ALG_TYPE), 146 | case 'OGM', 147 | tolH=max(tol(2),1e-3); 148 | tolW=max(tol(2),1e-3); 149 | case 'BCD', 150 | alpha0=1; 151 | end 152 | iterH=0; 153 | iterW=0; 154 | W=W0; % In case of LPRBCD, 'W' is current solution, 'W0' stores search point. 155 | H=H0; % In case of LPRBCD, 'H' is current solution, 'H0' stores search point. 156 | R=(V-W*H); % Residual Error 157 | if (gamma<0), 158 | switch gamma, 159 | case -1, 160 | SCL_TYPE='nagy'; 161 | case -2, 162 | SCL_TYPE='newton'; 163 | otherwise, 164 | error(['Unknown gamma value: ',int2str(gamma)]); 165 | end 166 | end 167 | switch upper(SCL_TYPE), 168 | case {'NAGY','NEWTON'}, 169 | gamma=CauchySCL(10,R,SCL_TYPE); 170 | case 'PLAIN', 171 | otherwise, 172 | error(['Unrecognized scale type: ',SCL_TYPE]); 173 | end 174 | Z=(R/gamma).^2+1; 175 | HIS.obj=sum(log(Z(:)))+m*n*log(gamma); 176 | HIS.sec=0; 177 | HIS.gamma=gamma; 178 | if verbose, 179 | fprintf('********** Cauchy NMF **********\n'); 180 | fprintf('%d: iterH=%d, iterW=%d, scale=%.3f, obj=%f.\n',0,iterH,iterW,gamma,HIS.obj); 181 | end 182 | tic; 183 | for iter=1:MAX_ITER, 184 | % Update Weights 185 | switch upper(WEI_TYPE), 186 | case 'PLAIN', 187 | Q=1./Z; 188 | case 'TGV', % For small samples 189 | Q=1./Z; 190 | IX=CauchyOutlIndex(abs(R),'global'); 191 | Q(IX)=0; 192 | case 'TLV', % For large samples 193 | Q=1./Z; 194 | IX=CauchyOutlIndex(abs(R),'local'); 195 | Q(IX)=0; 196 | case 'MED', 197 | Q=1./Z; 198 | Q=CauchyMedFilter(Q,img_info(1),img_info(2)); 199 | case 'MRF', 200 | Q=1./Z; 201 | IX=true(m,n); 202 | for j=1:n, 203 | s=MRF_ESuppt(R(:,j),img_info,NS,tau,lambda); 204 | IX(:,j)=(s==1); 205 | end 206 | Q(IX)=0; 207 | otherwise, 208 | error(['Unrecognized weighting type: ',WET_TYPE]); 209 | end 210 | % Update H 211 | switch upper(ALG_TYPE), 212 | case 'OGM', 213 | [H,iterH]=CauchyOGM(V,W,H,Q,INN_MIN,tolH,INN_MAX,LPZ_TYPE); 214 | if iterH<=10, 215 | tolH=tolH/10; 216 | end 217 | R=(V-W*H); 218 | case 'MUR', 219 | H=H.*(W'*(Q.*V))./(W'*(Q.*(W*H))+eps); 220 | R=(V-W*H); 221 | case 'HALS', 222 | for k=1:r, 223 | R=R+W(:,k)*H(k,:); 224 | Wk=W(:,k)*ones(1,n); % Update row of H 225 | H(k,:)=max(0,sum(Q.*R.*Wk)./(sum(Q.*Wk.*Wk)+eps)); 226 | Hk=ones(m,1)*H(k,:); % Update column of W 227 | W(:,k)=max(0,sum(Q.*R.*Hk,2)./(sum(Q.*Hk.*Hk,2)+eps)); 228 | R=R-W(:,k)*H(k,:); 229 | end 230 | case 'BCD', 231 | GradH0=W'*(Q.*(W*H0-V)); % Gradient at search point 232 | LpzH=CauchyLpzConstt(LPZ_TYPE,W,Q); 233 | H1=max(0,H0-GradH0./(ones(r,1)*LpzH)); % New 'H' solution 234 | alpha1=(1+sqrt(4*alpha0^2+1))/2; 235 | H0=H1+(alpha0-1)*(H1-H)/alpha1; 236 | H=H1; % Update for Next Recursion 237 | R=(V-W*H); 238 | otherwise, 239 | error(['Unrecognized algorithm type: ',ALG_TYPE]); 240 | end 241 | % Update W 242 | switch upper(ALG_TYPE), 243 | case 'OGM', 244 | [W,iterW]=CauchyOGM(V',H',W',Q',INN_MIN,tolW,INN_MAX,LPZ_TYPE); 245 | W=W'; 246 | if iterW<=10, 247 | tolW=tolW/10; 248 | end 249 | R=(V-W*H); 250 | case 'MUR', 251 | W=W.*((Q.*V)*H')./((Q.*(W*H))*H'+eps); 252 | R=(V-W*H); 253 | case 'HALS', 254 | case 'BCD', 255 | GradW0=(Q.*(W0*H-V))*H'; % Gradient at search point 256 | LpzW=CauchyLpzConstt(LPZ_TYPE,H',Q'); 257 | LpzW=LpzW'; 258 | W1=max(0,W0-GradW0./(LpzW*ones(1,r))); % New 'W' solution 259 | W0=W1+(alpha0-1)*(W1-W)/alpha1; 260 | W=W1; % Update for Next Recursion 261 | alpha0=alpha1; 262 | R=(V-W*H); 263 | otherwise, 264 | error(['Unrecognized algorithm type: ',ALG_TYPE]); 265 | end 266 | % Normalization 267 | H=H.*(max(W)'*ones(1,n)); 268 | W=W./(ones(m,1)*max(W)); 269 | switch upper(SCL_TYPE), 270 | case {'NAGY','NEWTON'}, 271 | gamma=CauchySCL(gamma,R,SCL_TYPE); 272 | case 'PLAIN', 273 | otherwise, 274 | error(['Unrecognized scale type: ',SCL_TYPE]); 275 | end 276 | Z=(R/gamma).^2+1; 277 | % Objective Function 278 | HIS.obj(iter+1)=sum(log(Z(:)))+m*n*log(gamma); 279 | HIS.sec(iter+1)=toc; 280 | HIS.gamma(iter+1)=gamma; 281 | if (rem(iter,10)==0) && verbose, 282 | fprintf('%d: iterH=%d, iterW=%d, scale=%.3f, obj=%f.\n',iter,iterH,iterW,gamma,HIS.obj(end)); 283 | end 284 | stop=abs(HIS.obj(end-1)-HIS.obj(end))/abs(HIS.obj(1)-HIS.obj(end)); 285 | if (stop<=tol(1) && iter>=MIN_ITER) || (HIS.sec(end)>=MAX_TIME), 286 | break; 287 | end 288 | end 289 | % Support for residual error, 1 for clean entry, and 0 for dirty entry 290 | switch upper(WEI_TYPE), 291 | case {'TGV', 'TLV'}, 292 | HIS.suppt=~IX; 293 | otherwise, 294 | HIS.suppt=true(m,n); 295 | end 296 | return; -------------------------------------------------------------------------------- /CauchyNMF_basis.m: -------------------------------------------------------------------------------- 1 | function [W,H,HIS]=CauchyNMF_basis(V,r,varargin) 2 | 3 | % Cauchy Nonnegative Matrix Factorization via Nesterov Based Half-Quadratic Programming. 4 | 5 | % The model is V \approx WH, where V, W, and H are defined as follows: 6 | % V (m x n): data matrix including n samples in m-dimensional space; 7 | % W (m x r): basis matrix including r bases in m-dimensional space; 8 | % H (r x n): coefficients matrix including n encodings in r-dimensional space. 9 | 10 | % Written by Naiyang Guan (ny.guan@gmail.com). 11 | % Copyright @ 2013-2023 by Naiyang Guan and Dacheng Tao. 12 | % Last Modified at 13 | % Sep. 06 2014; 14 | % Nov. 13 2014; 15 | % Aug. 07 2015; 16 | % Aug. 12 2015; 17 | % Jan. 06 2016; 18 | % Jan. 16 2017. 19 | 20 | % 21 | % V : Input data matrix (m x n), should be scaled to [0,1] in the 22 | % default sense 23 | % r : Target low-rank 24 | % 25 | % (Below are optional arguments: can be set by providing name-value 26 | % pairs). 27 | % MAX_ITER : Maximum number of iterations. Default is 1,000. 28 | % MIN_ITER : Minimum number of iterations. Default is 10. 29 | % MAX_TIME : Maximum amount of time in seconds. Default is 10,000. 30 | % ALG_TYPE : Type of algorithms. 31 | % 'OGM' for Nesterov's optimal gradient method, 32 | % 'MUR' for multiplicative update rule, 33 | % 'HALS' for weighted HALS algorithm, 34 | % 'BCD' for linear-proximal block coordinate descent. 35 | % WEI_TYPE : Type of weighting. 36 | % 'plain' for original data-adapative weighting, 37 | % 'tgv' for robustly trimmed weighting in global viewpoint, 38 | % 'tlv' for robustly trimmed weighting in local viewpoint, 39 | % 'mrf' for trimmed weighting by detecting supports by MRF, 40 | % 'med' for trimmed weighting with median filtering. 41 | % LPZ_TYPE : Type of Lipschitz constant. 42 | % 'plain' for calculating by Matlab, 43 | % 'ccomp' for calculating by MEX-C and OpenMP, 44 | % 'relax' for approximation. 45 | % IMG_INFO : Struct format, contains two records, i.e., height and 46 | % width, formatted by [height,width]. 47 | % SCALE : Scale parameter. Default is 0.1. If gamma < 0, automatic. 48 | % -1 : Nagy algorithm for scale estimation, 49 | % -2 : Newton algorithm for scale estimation. 50 | % W_INIT : (m x r) Initial value for W. 51 | % H_INIT : (r x n) Initial value for H. 52 | % TOL : Stopping tolerance, [outer one, inner one]. Default is [1e-4,1e-2]. 53 | % If you want to obtain a more accurate solution, decrease TOL and increase MAX_TIME simultanuously. 54 | % VERBOSE : 0 (default) - No debugging information. 55 | % 1 (debugging purpose) - History of computation is printed on screen. 56 | % 2 - History of basis. 57 | % 58 | % W : Obtained basis matrix (m x r), 59 | % H : Obtained coefficients matrix (r x n), 60 | % HIS : (debugging purpose) History of computation. 61 | % 62 | % 63 | % >>A=rand(100); 64 | % >>CauchyNMF(A,10); 65 | % >>CauchyNMF(A,20,'verbose',1); 66 | % >>CauchyNMF(A,30,'verbose',2,'w_init',rand(100,30)); 67 | % >>CauchyNMF(A,5,'verbose',2,'tol',1e-5); 68 | 69 | [m,n]=size(V); 70 | if ~isa(V,'double'), V=double(V); end 71 | if ~exist('V','var'), error('please input the sample matrix.\n'); end 72 | if ~exist('r','var'), error('please input the low rank.\n'); end 73 | % if n<=r || m<=r, error('Too high low-rank (n<=r or m<=r).\n'); end 74 | 75 | % Default setting 76 | MIN_ITER=20; % minimum number of outer iterations 77 | MAX_ITER=500; % maximum number of outer iterations 78 | INN_MIN=20; % minimum number of inner iterations 79 | INN_MAX=300; % maximum number of inner iterations 80 | MAX_TIME=10000; % maximum running time 81 | ALG_TYPE='ogm'; % algorithm's type, default is OGM 82 | WEI_TYPE='plain'; % weighting's type, default is PLAIN 83 | LPZ_TYPE='plain'; % Lipschitz constant's type, default is PLAIN 84 | SCL_TYPE='plain'; % algorithm's type of scale estimation 85 | gamma=0.1; % scale parameter 86 | tol=[1e-4,1e-2]; % tolerance of outer and inner loops 87 | verbose=0; 88 | img_info=[sqrt(m),sqrt(m)]; % information [height, width] for images 89 | W0=rand(m,r); 90 | H0=rand(r,n); 91 | 92 | % Read optional parameters 93 | if (rem(length(varargin),2)==1), 94 | error('Optional parameters should always go by pairs'); 95 | else 96 | for i=1:2:(length(varargin)-1), 97 | switch upper(varargin{i}), 98 | case 'MAX_ITER', MAX_ITER=varargin{i+1}; 99 | case 'MIN_ITER', MIN_ITER=varargin{i+1}; 100 | case 'MAX_TIME', MAX_TIME=varargin{i+1}; 101 | case 'ALG_TYPE', ALG_TYPE=varargin{i+1}; 102 | case 'WEI_TYPE', WEI_TYPE=varargin{i+1}; 103 | case 'LPZ_TYPE', LPZ_TYPE=varargin{i+1}; 104 | case 'IMG_INFO', img_info=varargin{i+1}; 105 | case 'SCALE', gamma=varargin{i+1}; 106 | case 'W_INIT', W0=varargin{i+1}; 107 | case 'H_INIT', H0=varargin{i+1}; 108 | case 'TOL', tol=varargin{i+1}; 109 | case 'VERBOSE', verbose=varargin{i+1}; 110 | otherwise 111 | error(['Unrecognized option: ',varargin{i}]); 112 | end 113 | end 114 | end 115 | 116 | % Initialization 117 | switch upper(WEI_TYPE), 118 | case 'PLAIN', 119 | Z=W0*H0; 120 | s=max(Z(:))/max(median(V(:)),min(Z(:))); 121 | W0=W0/sqrt(s); 122 | H0=H0/sqrt(s); 123 | case {'TGV', 'TLV', 'MED'}, 124 | R=V-W0*H0; 125 | delta=sum(sum(R.^2))/(2*m*n); 126 | Q=exp(-(R.^2)/(2*delta)); 127 | W0=W0.*((Q.*V)*H0')./((Q.*(W0*H0))*H0'+eps); 128 | H0=H0.*(W0'*(Q.*V))./(W0'*(Q.*(W0*H0))+eps); 129 | case 'MRF', 130 | R=V-W0*H0; 131 | delta=sum(sum(R.^2))/(2*m*n); 132 | Q=exp(-(R.^2)/(2*delta)); 133 | W0=W0.*((Q.*V)*H0')./((Q.*(W0*H0))*H0'+eps); 134 | H0=H0.*(W0'*(Q.*V))./(W0'*(Q.*(W0*H0))+eps); 135 | height=img_info(1); 136 | width=img_info(2); 137 | NS=edges4connected(height,width); 138 | tau=0.17; 139 | lambda=3; 140 | otherwise, 141 | error(['Unknown weighting type: ',WEI_TYPE]); 142 | end 143 | Z=W0*H0; 144 | 145 | % Algorithmic Settings 146 | switch upper(ALG_TYPE), 147 | case 'OGM', 148 | tolH=max(tol(2),1e-3); 149 | tolW=max(tol(2),1e-3); 150 | case 'BCD', 151 | alpha0=1; 152 | end 153 | iterH=0; 154 | iterW=0; 155 | W=W0; % In case of LPRBCD, 'W' is current solution, 'W0' stores search point. 156 | H=H0; % In case of LPRBCD, 'H' is current solution, 'H0' stores search point. 157 | R=(V-W*H); % Residual Error 158 | if (gamma<0), 159 | switch gamma, 160 | case -1, 161 | SCL_TYPE='nagy'; 162 | case -2, 163 | SCL_TYPE='newton'; 164 | otherwise, 165 | error(['Unknown gamma value: ',int2str(gamma)]); 166 | end 167 | end 168 | switch upper(SCL_TYPE), 169 | case {'NAGY','NEWTON'}, 170 | gamma=CauchySCL(10,R,SCL_TYPE); 171 | case 'PLAIN', 172 | otherwise, 173 | error(['Unrecognized scale type: ',SCL_TYPE]); 174 | end 175 | Z=(R/gamma).^2+1; 176 | HIS.obj=sum(log(Z(:)))+m*n*log(gamma); 177 | HIS.sec=0; 178 | HIS.gamma=gamma; 179 | if verbose, 180 | fprintf('********** Cauchy NMF **********\n'); 181 | fprintf('%d: iterH=%d, iterW=%d, scale=%.3f, obj=%f.\n',0,iterH,iterW,gamma,HIS.obj); 182 | if verbose==2, 183 | H=H.*(sum(W)'*ones(1,n)); 184 | W=W./(ones(m,1)*sum(W)); 185 | HIS.basis=num2cell(W0); 186 | HIS.basis={HIS.basis,num2cell(W0)}; 187 | end 188 | end 189 | tic; 190 | for iter=1:MAX_ITER, 191 | % Update Weights 192 | switch upper(WEI_TYPE), 193 | case 'PLAIN', 194 | Q=1./Z; 195 | case 'TGV', % For small samples 196 | Q=1./Z; 197 | IX=CauchyOutlIndex(abs(R),'global'); 198 | Q(IX)=0; 199 | case 'TLV', % For large samples 200 | Q=1./Z; 201 | IX=CauchyOutlIndex(abs(R),'local'); 202 | Q(IX)=0; 203 | case 'MED', 204 | Q=1./Z; 205 | Q=CauchyMedFilter(Q,img_info(1),img_info(2)); 206 | case 'MRF', 207 | Q=1./Z; 208 | IX=true(m,n); 209 | for j=1:n, 210 | s=MRF_ESuppt(R(:,j),img_info,NS,tau,lambda); 211 | IX(:,j)=(s==1); 212 | end 213 | Q(IX)=0; 214 | otherwise, 215 | error(['Unrecognized weighting type: ',WET_TYPE]); 216 | end 217 | % Update H 218 | switch upper(ALG_TYPE), 219 | case 'OGM', 220 | [H,iterH]=CauchyOGM(V,W,H,Q,INN_MIN,tolH,INN_MAX,LPZ_TYPE); 221 | if iterH<=10, 222 | tolH=tolH/10; 223 | end 224 | R=(V-W*H); 225 | case 'MUR', 226 | H=H.*(W'*(Q.*V))./(W'*(Q.*(W*H))+eps); 227 | R=(V-W*H); 228 | case 'HALS', 229 | for k=1:r, 230 | R=R+W(:,k)*H(k,:); 231 | Wk=W(:,k)*ones(1,n); % Update row of H 232 | H(k,:)=max(0,sum(Q.*R.*Wk)./(sum(Q.*Wk.*Wk)+eps)); 233 | Hk=ones(m,1)*H(k,:); % Update column of W 234 | W(:,k)=max(0,sum(Q.*R.*Hk,2)./(sum(Q.*Hk.*Hk,2)+eps)); 235 | R=R-W(:,k)*H(k,:); 236 | end 237 | case 'BCD', 238 | GradH0=W'*(Q.*(W*H0-V)); % Gradient at search point 239 | LpzH=CauchyLpzConstt(LPZ_TYPE,W,Q); 240 | H1=max(0,H0-GradH0./(ones(r,1)*LpzH)); % New 'H' solution 241 | alpha1=(1+sqrt(4*alpha0^2+1))/2; 242 | H0=H1+(alpha0-1)*(H1-H)/alpha1; 243 | H=H1; % Update for Next Recursion 244 | R=(V-W*H); 245 | otherwise, 246 | error(['Unrecognized algorithm type: ',ALG_TYPE]); 247 | end 248 | % Update W 249 | switch upper(ALG_TYPE), 250 | case 'OGM', 251 | [W,iterW]=CauchyOGM(V',H',W',Q',INN_MIN,tolW,INN_MAX,LPZ_TYPE); 252 | W=W'; 253 | if iterW<=10, 254 | tolW=tolW/10; 255 | end 256 | R=(V-W*H); 257 | case 'MUR', 258 | W=W.*((Q.*V)*H')./((Q.*(W*H))*H'+eps); 259 | R=(V-W*H); 260 | case 'HALS', 261 | case 'BCD', 262 | GradW0=(Q.*(W0*H-V))*H'; % Gradient at search point 263 | LpzW=CauchyLpzConstt(LPZ_TYPE,H',Q'); 264 | LpzW=LpzW'; 265 | W1=max(0,W0-GradW0./(LpzW*ones(1,r))); % New 'W' solution 266 | W0=W1+(alpha0-1)*(W1-W)/alpha1; 267 | W=W1; % Update for Next Recursion 268 | alpha0=alpha1; 269 | R=(V-W*H); 270 | otherwise, 271 | error(['Unrecognized algorithm type: ',ALG_TYPE]); 272 | end 273 | % Normalization 274 | if verbose==2, 275 | H=H.*(sum(W)'*ones(1,n)); 276 | W=W./(ones(m,1)*sum(W)); 277 | else 278 | H=H.*(max(W)'*ones(1,n)); 279 | W=W./(ones(m,1)*max(W)); 280 | end 281 | switch upper(SCL_TYPE), 282 | case {'NAGY','NEWTON'}, 283 | gamma=CauchySCL(gamma,R,SCL_TYPE); 284 | case 'PLAIN', 285 | otherwise, 286 | error(['Unrecognized scale type: ',SCL_TYPE]); 287 | end 288 | Z=(R/gamma).^2+1; 289 | % Objective Function 290 | HIS.obj(iter+1)=sum(log(Z(:)))+m*n*log(gamma); 291 | HIS.sec(iter+1)=toc; 292 | HIS.gamma(iter+1)=gamma; 293 | if (rem(iter,10)==0) && verbose, 294 | fprintf('%d: iterH=%d, iterW=%d, scale=%.3f, obj=%f.\n',iter,iterH,iterW,gamma,HIS.obj(end)); 295 | end 296 | if verbose==2, 297 | HIS.basis{iter+1}=num2cell(W); 298 | end 299 | stop=abs(HIS.obj(end-1)-HIS.obj(end))/abs(HIS.obj(1)-HIS.obj(end)); 300 | if (stop<=tol(1) && iter>=MIN_ITER) || (HIS.sec(end)>=MAX_TIME), 301 | break; 302 | end 303 | end 304 | % Support for residual error, 1 for clean entry, and 0 for dirty entry 305 | switch upper(WEI_TYPE), 306 | case {'TGV', 'TLV'}, 307 | HIS.suppt=~IX; 308 | otherwise, 309 | HIS.suppt=true(m,n); 310 | end 311 | return; --------------------------------------------------------------------------------