├── LICENSE ├── README.md ├── bfc ├── bfc_params.m ├── bfc_sampler.m └── test.m └── ibfc ├── c_update_ibfc.m ├── ibfc_pf.m ├── ibfc_pf_params.m ├── ll_ibfc.m ├── test.m └── u_update_ibfc.m /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 TigerSense 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # BayesianFuzzyClustering 2 | Matlab implementation of the Bayesian Fuzzy Clustering algorithms. Please cite this code when you use it: [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.2638099.svg)](https://doi.org/10.5281/zenodo.2638099) 3 | Taylor Glenn, Alina Zare & Paul Gader. (2019, April 12). GatorSense/BayesianFuzzyClustering: Initial Release (Version v1.0). Zenodo. http://doi.org/10.5281/zenodo.2638099 4 | 5 | *** 6 | See related paper, doi: 10.1109/TFUZZ.2014.2370676, http://ieeexplore.ieee.org/xpl/login.jsp?tp=&arnumber=6955803 7 | 8 | NOTE: If Bayesian Fuzzy Clustering is used in any publication or presentation, the following reference must be cited: 9 | Glenn, T.; Zare, A.; Gader, P., "Bayesian Fuzzy Clustering," IEEE Transactions on Fuzzy Systems, vol.23, no.5, pp.1545-1561 10 | doi: 10.1109/TFUZZ.2014.2370676 11 | *** 12 | 13 | Requirements: 14 | This code uses the excellent Lightspeed and Fastfit toolboxes by Tom Minka: 15 | Lightspeed toolbox - http://research.microsoft.com/en-us/um/people/minka/software/lightspeed/ 16 | Fastfit toolbox - http://research.microsoft.com/en-us/um/people/minka/software/fastfit/ 17 | 18 | This code also uses the Matlab Fuzzy Logic Toolbox for its fcm impleSeementation 19 | 20 | Contents: 21 | 22 | bfc/ % code for Bayesian Fuzzy Clustering MCMC sampler 23 | |- bfc_params.m % generate default parameters structure 24 | |- bfc_sampler.m % bayesian fuzzy clustering sampler 25 | |- test.m % test script - run this 26 | 27 | ibfc/ % code for Infinite Bayesian Fuzzy Clustering particle filter 28 | |- ibfc_pf_params.m % generate defaults parameters structure 29 | |- ibfc_pf.m % IBFC particle filter 30 | |- u_update_ibfc.m % membership update function 31 | |- c_update_ibfc.m % cluster prototype update function 32 | |- ll_ibfc.m % ibfc log-likelihood evaluation function 33 | |- test.m % test script - run this 34 | 35 | -------------------------------------------------------------------------------- /bfc/bfc_params.m: -------------------------------------------------------------------------------- 1 | % bfc_params.m 2 | % default parameters structure for bfc sampler 3 | % 4 | % Copyright 2013 Taylor C. Glenn 5 | % tcg@cise.ufl.edu 6 | % see ../LICENSE.txt for license terms 7 | 8 | function p = bfc_params 9 | 10 | % parameters for Bayesian Fuzzy Clustering sampler 11 | 12 | p = struct(); 13 | 14 | p.n_comp = 2; 15 | p.m = 2; 16 | p.n_iter = 1000; 17 | p.alpha = 1; 18 | 19 | p.mem_prop_alpha = 1; % dirichlet parameter for membership proposal distribution 20 | 21 | p.rpt_ival = 50; % reporting interval 22 | 23 | p.re_seed_rng = false; % reset random number generator to fixed seed to get same answer every run 24 | 25 | p.fcm_init = false; % if true, run fcm for a few iterations with m=2 to init centers and memberships 26 | 27 | p.do_trace = false; % record likelihood at each iteration 28 | 29 | p.figno = 12; % output figure number 30 | 31 | end -------------------------------------------------------------------------------- /bfc/bfc_sampler.m: -------------------------------------------------------------------------------- 1 | % bfc_sampler.m 2 | % Bayesian Fuzzy Clustering MCMC sampler 3 | % 4 | % Copyright 2013 Taylor C. Glenn 5 | % tcg@cise.ufl.edu 6 | % see ../LICENSE.txt for license terms 7 | 8 | function [C,U,ll,trace] = bfc_sampler(X,params) 9 | 10 | n_comp = params.n_comp; 11 | m = params.m; 12 | n_iter = params.n_iter; 13 | alpha = params.alpha; 14 | 15 | if params.re_seed_rng 16 | RandStream.setDefaultStream(RandStream('mt19937ar','Seed', 5489)); 17 | randomseed([101,1001,10001]); % set random seed for lightspeed library 18 | end 19 | 20 | [n_pts,n_dim] = size(X); 21 | 22 | % proposal parameters 23 | sigma_prop = .25*cov(X); 24 | sqrt_sigma_prop = chol(sigma_prop); 25 | 26 | % hyperparameters 27 | mu_c = mean(X,1); 28 | sigma_c = 3*cov(X); 29 | siginv_c = pinv(sigma_c); 30 | 31 | % initialize centers, memberships 32 | if params.fcm_init 33 | [C,Ut] = fcm(X,n_comp,[2,20,1e-5,1]); 34 | U = Ut'; 35 | else 36 | rp = randperm(n_pts); 37 | C = X(rp(1:n_comp),:); 38 | 39 | D = zeros(n_pts,n_comp); 40 | U = zeros(n_pts,n_comp); 41 | for i=1:n_comp 42 | D(:,i) = 1./(sum((X - repmat(C(i,:),n_pts,1)).^2, 2)+eps); 43 | end 44 | sumD = sum(D,2); 45 | for i=1:n_comp 46 | U(:,i) = D(:,i)./sumD; 47 | end 48 | end 49 | 50 | 51 | drawfig(X,C,U,m,alpha,params.figno); 52 | 53 | C_max = C; 54 | U_max = U; 55 | 56 | 57 | ll_fun = @(X,C,U,m,mu_c,siginv_c)(fcm_ll_dirichlet(X,C,U,m,mu_c,siginv_c,alpha)); 58 | u_ll_fun = @(X,C,U,m)(u_lls_dirichlet(X,C,U,m,alpha)); 59 | 60 | % ll_fun = @(X,C,U,m,mu_c,siginv_c)(fcm_ll_invgamma(X,C,U,m,mu_c,siginv_c,alpha)); 61 | % u_ll_fun = @(X,C,U,m)(u_lls_invgamma(X,C,U,m,alpha)); 62 | 63 | % ll_fun = @fcm_ll_noprior; 64 | % u_ll_fun = @u_lls_noprior; 65 | 66 | %ll_fun = @fcm_ll; 67 | %u_ll_fun = @u_lls; 68 | 69 | ll = ll_fun(X,C,U,m,mu_c,siginv_c); 70 | ll_max = ll; 71 | ll_max_display = ll_max; 72 | 73 | if params.do_trace 74 | trace = zeros(1,n_iter+1); 75 | trace(1) = ll; 76 | else 77 | trace = []; 78 | end 79 | 80 | u_accepted = 0; 81 | c_accepted = 0; 82 | 83 | for iter=1:n_iter 84 | 85 | %generate new U's 86 | % use uniform proposal 87 | U_samp = dirichlet_sample(params.mem_prop_alpha*ones(1,n_comp),n_pts); 88 | 89 | lls_old = u_ll_fun(X,C,U,m); 90 | lls_proposed = u_ll_fun(X,C,U_samp,m); 91 | 92 | % decide to accept each u vector or not 93 | accept_U = exp(lls_proposed-lls_old) > rand(n_pts,1); 94 | U(accept_U,:) = U_samp(accept_U,:); 95 | 96 | u_accepted = u_accepted + sum(accept_U); 97 | 98 | % check each sample for inclusion in the max likelihood set 99 | lls_new_max_c = u_ll_fun(X,C_max,U_samp,m); 100 | lls_old_max_c = u_ll_fun(X,C_max,U_max,m); 101 | accept = lls_new_max_c > lls_old_max_c; 102 | U_max(accept,:) = U_samp(accept,:); 103 | 104 | %generate new C's 105 | C_samp = randn(n_comp,n_dim)*sqrt_sigma_prop + C; 106 | new_C = C; 107 | new_C_max = C_max; 108 | for i=1:n_comp 109 | for d=1:n_dim 110 | %test each sample center for acceptance 111 | old_ll = ll_fun(X,new_C,U,m,mu_c,siginv_c); 112 | 113 | new_C(i,d) = C_samp(i,d); 114 | 115 | new_ll = ll_fun(X,new_C,U,m,mu_c,siginv_c); 116 | 117 | if (exp(new_ll-old_ll) > rand(1)) 118 | c_accepted = c_accepted + 1; 119 | else 120 | new_C(i,d) = C(i,d); 121 | end 122 | 123 | %check sample for inclusion in max set 124 | new_C_max(i,d) = C_samp(i,d); 125 | 126 | ll_new_c = ll_fun(X,new_C_max,U_max,m,mu_c,siginv_c); 127 | ll_old_c = ll_fun(X,C_max,U_max,m,mu_c,siginv_c); 128 | if ll_new_c > ll_old_c 129 | C_max(i,d) = C_samp(i,d); 130 | else 131 | new_C_max(i,d) = C_max(i,d); 132 | end 133 | end 134 | end 135 | C = new_C; 136 | 137 | % check full sample for new max 138 | ll_full = ll_fun(X,C,U,m,mu_c,siginv_c); 139 | ll_max = ll_fun(X,C_max,U_max,m,mu_c,siginv_c); 140 | if ll_full > ll_max 141 | ll_max = ll_full; 142 | C_max = C; 143 | U_max = U; 144 | end 145 | 146 | %fprintf('.'); 147 | if mod(iter,params.rpt_ival) == 0 148 | u_rate = u_accepted/(params.rpt_ival*n_pts); 149 | c_rate = c_accepted/(params.rpt_ival*n_comp*n_dim); 150 | u_accepted = 0; 151 | c_accepted = 0; 152 | 153 | if ll_max > ll_max_display 154 | ll_max_display = ll_max; 155 | drawfig(X,C_max,U_max,m,alpha,params.figno); 156 | end 157 | %drawfig(X,C,U); 158 | 159 | fprintf('iter %d: U rate: %.3f C rate: %.3f max: %f\n',iter,u_rate,c_rate,ll_max); 160 | end 161 | 162 | if params.do_trace 163 | trace(iter+1) = ll_max; 164 | end 165 | end 166 | 167 | C = C_max; 168 | U = U_max'; 169 | ll = ll_max; 170 | 171 | end 172 | 173 | function ll = fcm_ll(X,C,U,m,mu_c,siginv_c) 174 | 175 | [n_pts,n_dim] = size(X); 176 | n_comp = size(C,1); 177 | 178 | %sum of gaussian lls with u_ij precision 179 | ll = 0; 180 | 181 | % sum of exponent terms 182 | Z = zeros(n_pts,n_comp); 183 | for i=1:n_comp 184 | Z(:,i) = sum((X - repmat(C(i,:),n_pts,1)).^2, 2); 185 | end 186 | ll = ll + -.5* sum((U(:).^m).*Z(:)); 187 | 188 | % with fuzzy membership prior, base term of normal and 189 | % prior likelihoods cancel 190 | 191 | %sum of C lls in guassian with mu_c,siginv_c 192 | for i=1:n_comp 193 | z = C(i,:)-mu_c; 194 | ll = ll + -.5* z*siginv_c*z'; 195 | end 196 | 197 | end 198 | 199 | function ll = fcm_ll_invx2(X,C,U,m,mu_c,siginv_c) 200 | 201 | [n_pts,n_dim] = size(X); 202 | n_comp = size(C,1); 203 | 204 | %sum of gaussian lls with u_ij precision 205 | ll = 0; 206 | 207 | % sum of exponent terms 208 | Z = zeros(n_pts,n_comp); 209 | for i=1:n_comp 210 | Z(:,i) = sum((X - repmat(C(i,:),n_pts,1)).^2, 2); 211 | end 212 | ll = ll + -.5* sum((U(:).^m).*Z(:)); 213 | 214 | % with fuzzy membership prior, base term of normal and 215 | % first term of inv chi-sq prior cancel, leaves exponent term 216 | ll = ll - 1/2*sum(sum(1./U)); 217 | 218 | %sum of C lls in guassian with mu_c,siginv_c 219 | for i=1:n_comp 220 | z = C(i,:)-mu_c; 221 | ll = ll + -.5* z*siginv_c*z'; 222 | end 223 | 224 | end 225 | 226 | function ll = fcm_ll_invgamma(X,C,U,m,mu_c,siginv_c,beta) 227 | 228 | [n_pts,n_dim] = size(X); 229 | n_comp = size(C,1); 230 | 231 | %sum of gaussian lls with u_ij precision 232 | ll = 0; 233 | 234 | % sum of exponent terms 235 | Z = zeros(n_pts,n_comp); 236 | for i=1:n_comp 237 | Z(:,i) = sum((X - repmat(C(i,:),n_pts,1)).^2, 2); 238 | end 239 | ll = ll + -.5* sum((U(:).^m).*Z(:)); 240 | 241 | % with fuzzy membership prior, base term of normal and 242 | % first term of inv chi-sq prior cancel, leaves exponent term 243 | %beta = 0.00001; 244 | ll = ll - beta*sum(sum(1./U)); 245 | 246 | %sum of C lls in guassian with mu_c,siginv_c 247 | for i=1:n_comp 248 | z = C(i,:)-mu_c; 249 | ll = ll + -.5* z*siginv_c*z'; 250 | end 251 | 252 | end 253 | 254 | function ll = fcm_ll_noprior(X,C,U,m,mu_c,siginv_c) 255 | 256 | [n_pts,n_dim] = size(X); 257 | n_comp = size(C,1); 258 | 259 | %sum of gaussian lls with u_ij precision 260 | ll = 0; 261 | 262 | % sum of exponent terms 263 | Z = zeros(n_pts,n_comp); 264 | for i=1:n_comp 265 | Z(:,i) = sum((X - repmat(C(i,:),n_pts,1)).^2, 2); 266 | end 267 | ll = ll + m*n_dim/2*sum(log(U(:))) -.5* sum((U(:).^m).*Z(:)); 268 | 269 | 270 | %sum of C lls in guassian with mu_c,siginv_c 271 | for i=1:n_comp 272 | z = C(i,:)-mu_c; 273 | ll = ll + -.5* z*siginv_c*z'; 274 | end 275 | 276 | end 277 | 278 | function ll = fcm_ll_dirichlet(X,C,U,m,mu_c,siginv_c,alpha) 279 | 280 | [n_pts,n_dim] = size(X); 281 | n_comp = size(C,1); 282 | 283 | %sum of gaussian lls with u_ij precision 284 | ll = 0; 285 | 286 | % sum of exponent terms 287 | Z = zeros(n_pts,n_comp); 288 | for i=1:n_comp 289 | Z(:,i) = sum((X - repmat(C(i,:),n_pts,1)).^2, 2); 290 | end 291 | ll = ll + -.5* sum((U(:).^m).*Z(:)); 292 | 293 | % cancel normalizing term in normal with fuzzy cluster prior 294 | % then add an additional symmetric dirichlet(alpha) distribution 295 | 296 | ll = ll + (alpha-1)*sum(log(U(:))); 297 | 298 | %sum of C lls in guassian with mu_c,siginv_c 299 | for i=1:n_comp 300 | z = C(i,:)-mu_c; 301 | ll = ll + -.5* z*siginv_c*z'; 302 | end 303 | 304 | end 305 | 306 | 307 | function lls = u_lls(X,C,U,m) 308 | 309 | [n_pts,n_dim] = size(X); 310 | n_comp = size(C,1); 311 | 312 | lls = zeros(n_pts,1); 313 | 314 | % exponent terms 315 | Z = zeros(n_pts,n_comp); 316 | for i=1:n_comp 317 | Z(:,i) = sum((X - repmat(C(i,:),n_pts,1)).^2, 2); 318 | end 319 | lls = lls + -.5 *sum((U.^m).*Z,2); 320 | 321 | % with fuzzy membership prior, base term of normal and 322 | % prior likelihoods cancel 323 | 324 | end 325 | 326 | function lls = u_lls_invx2(X,C,U,m) 327 | 328 | [n_pts,n_dim] = size(X); 329 | n_comp = size(C,1); 330 | 331 | lls = zeros(n_pts,1); 332 | 333 | % exponent terms 334 | Z = zeros(n_pts,n_comp); 335 | for i=1:n_comp 336 | Z(:,i) = sum((X - repmat(C(i,:),n_pts,1)).^2, 2); 337 | end 338 | lls = lls + -.5 *sum((U.^m).*Z,2); 339 | 340 | % with fuzzy membership prior, base term of normal and 341 | % first term of inv chi-sq prior cancel, leaves exponent term 342 | lls = lls - 1/2*sum(1./U,2); 343 | 344 | end 345 | 346 | function lls = u_lls_invgamma(X,C,U,m,beta) 347 | 348 | [n_pts,n_dim] = size(X); 349 | n_comp = size(C,1); 350 | 351 | lls = zeros(n_pts,1); 352 | 353 | % exponent terms 354 | Z = zeros(n_pts,n_comp); 355 | for i=1:n_comp 356 | Z(:,i) = sum((X - repmat(C(i,:),n_pts,1)).^2, 2); 357 | end 358 | lls = lls + -.5 *sum((U.^m).*Z,2); 359 | 360 | % with fuzzy membership prior, base term of normal and 361 | % first term of inv chi-sq prior cancel, leaves exponent term 362 | %beta = 0.00001; 363 | lls = lls - beta*sum(1./U,2); 364 | 365 | end 366 | 367 | function lls = u_lls_dirichlet(X,C,U,m,alpha) 368 | 369 | [n_pts,n_dim] = size(X); 370 | n_comp = size(C,1); 371 | 372 | lls = zeros(n_pts,1); 373 | 374 | % exponent terms 375 | Z = zeros(n_pts,n_comp); 376 | for i=1:n_comp 377 | Z(:,i) = sum((X - repmat(C(i,:),n_pts,1)).^2, 2); 378 | end 379 | lls = lls + -.5 *sum((U.^m).*Z,2); 380 | 381 | % cancel normalizing term in normal with fuzzy cluster prior 382 | % then add an additional symmetric dirichlet(alpha) distribution 383 | 384 | lls = lls + (alpha-1)*sum(log(U),2); 385 | 386 | end 387 | 388 | function lls = u_lls_noprior(X,C,U,m) 389 | 390 | [n_pts,n_dim] = size(X); 391 | n_comp = size(C,1); 392 | 393 | lls = zeros(n_pts,1); 394 | 395 | % exponent terms 396 | Z = zeros(n_pts,n_comp); 397 | for i=1:n_comp 398 | Z(:,i) = sum((X - repmat(C(i,:),n_pts,1)).^2, 2); 399 | end 400 | lls = lls + m*n_dim/2*sum(log(U),2) -.5 *sum((U.^m).*Z,2); 401 | 402 | 403 | end 404 | 405 | 406 | function drawfig(X,C,U,m,alpha,figno) 407 | 408 | if ~exist('figno','var') 409 | figno = 12; 410 | end 411 | 412 | n_pts = size(X,1); 413 | n_comp = size(C,1); 414 | 415 | if n_pts > 10000 416 | return; 417 | end 418 | 419 | specs = {'r-','b-','g-','c-','m-','y-','k-'}; 420 | rgb = [U'; zeros(1,n_pts)]; 421 | 422 | figure(figno); 423 | clf; 424 | subplot(2,1,1); 425 | scatter(X(:,1),X(:,2),[],U(:,1)'); %rgb'); 426 | hold on; 427 | for i=1:n_comp 428 | text(C(i,1),C(i,2),num2str(i),'BackgroundColor','w','Color','k'); 429 | end 430 | xlabel('D1'); 431 | ylabel('D2'); 432 | %title({'FCM+Dirichlet sampler',... 433 | % sprintf('m = %.1f, \\alpha = %.2f',m,alpha)}); 434 | title('membership in cluster 1'); 435 | axis equal; 436 | 437 | subplot(2,1,2); 438 | plot(U(:,1),specs{1}); 439 | hold on; 440 | % for i=2:n_comp 441 | % plot(U(:,i),specs{i}); 442 | % end 443 | 444 | xlabel('index'); 445 | ylabel('membership'); 446 | ylim([-0.01,1.01]); 447 | drawnow; 448 | 449 | end 450 | 451 | 452 | -------------------------------------------------------------------------------- /bfc/test.m: -------------------------------------------------------------------------------- 1 | % test.m 2 | % test script comparing FCM and Bayesian Fuzzy Clustering Sampler 3 | % 4 | % Copyright 2013 Taylor C. Glenn 5 | % tcg@cise.ufl.edu 6 | % see ../LICENSE.txt for license terms 7 | 8 | RandStream.setDefaultStream(RandStream('mt19937ar','Seed', 5489)); 9 | 10 | % generate some data 11 | n_dim = 2; 12 | n_pts = 500; 13 | n_comp = 2; 14 | 15 | c1 = [1 0;0 1]; 16 | mu1 = [1; 2]; 17 | 18 | c2 = [1 0; 0 1]; 19 | mu2 = [4; 4]; 20 | 21 | x1 = chol(c1)*randn(n_dim,n_pts/2) + repmat(mu1,1,n_pts/2); 22 | x2 = chol(c2)*randn(n_dim,n_pts/2) + repmat(mu2,1,n_pts/2); 23 | 24 | X = [x1 x2]; 25 | 26 | figure(10); 27 | clf; 28 | scatter(x1(1,:),x1(2,:),'bo'); 29 | hold on; 30 | scatter(x2(1,:),x2(2,:),'rs'); 31 | 32 | %--- fcm m=2 33 | m = 2; 34 | [C,U] = fcm(X',2,[m 100 1e-5 1]); 35 | 36 | figure(11); 37 | clf; 38 | subplot(2,1,1); 39 | scatter(X(1,:),X(2,:),[],U(1,:)'); 40 | hold on; 41 | text(C(1,1),C(1,2),'1','BackgroundColor','w','Color','k'); 42 | text(C(2,1),C(2,2),'2','BackgroundColor','w','Color','k'); 43 | title({'FCM m=2','membership in cluster 1'}); 44 | xlabel('D1'); 45 | ylabel('D2'); 46 | axis equal; 47 | 48 | subplot(2,1,2); 49 | plot(U(1,:),'r-'); 50 | hold on; 51 | %plot(U(2,:),'b-'); 52 | xlabel('index'); 53 | ylabel('membership'); 54 | ylim([-0.01,1.01]); 55 | 56 | 57 | %--- fcm m=10 58 | m = 10; 59 | [C,U] = fcm(X',2,[m 100 1e-5 1]); 60 | 61 | 62 | figure(13); 63 | clf; 64 | subplot(2,1,1); 65 | scatter(X(1,:),X(2,:),[],U(1,:)'); 66 | hold on; 67 | text(C(1,1),C(1,2),'1','BackgroundColor','w','Color','k'); 68 | text(C(2,1),C(2,2),'2','BackgroundColor','w','Color','k'); 69 | title({'FCM m=10','membership in cluster 1'}); 70 | xlabel('D1'); 71 | ylabel('D2'); 72 | axis equal; 73 | 74 | subplot(2,1,2); 75 | plot(U(1,:),'r-'); 76 | hold on; 77 | %plot(U(2,:),'b-'); 78 | xlabel('index'); 79 | ylabel('membership'); 80 | ylim([-0.01,1.01]); 81 | 82 | 83 | %--- bfc m=2 84 | p1 = bfc_params; 85 | p1.re_seed_rng = true; 86 | p1.n_comp = 2; 87 | p1.m = 2; 88 | p1.n_iter = 1000; 89 | p1.alpha = 1; 90 | p1.fcm_init = false; 91 | 92 | [tC,tU] = bfc_sampler(X',p1); 93 | 94 | 95 | %--- bfc m=10 96 | p2 = p1; 97 | p2.m = 10; 98 | p2.figno = 14; 99 | [tC,tU] = bfc_sampler(X',p2); 100 | 101 | %--- bfc m=1 102 | p3 = p1; 103 | p3.m = 1; 104 | p3.figno = 15; 105 | [tC,tU] = bfc_sampler(X',p3); 106 | subplot(2,1,2); 107 | ylim([-0.1 1.1]); 108 | 109 | %--- bfc m=-10 110 | p4 = p1; 111 | p4.m = -10; 112 | p4.figno = 16; 113 | [tC,tU] = bfc_sampler(X',p4); 114 | 115 | %--- bfc m=2 alpha=3 116 | p5 = p1; 117 | p5.alpha = 3; 118 | p5.figno = 17; 119 | [tC,tU] = bfc_sampler(X',p5); 120 | 121 | %--- bfc m=4 alpha=.95 122 | p6 = p1; 123 | p6.m = 4; 124 | p6.alpha = 0.95; 125 | p6.figno = 18; 126 | [tC,tU] = bfc_sampler(X',p6); 127 | 128 | 129 | 130 | 131 | -------------------------------------------------------------------------------- /ibfc/c_update_ibfc.m: -------------------------------------------------------------------------------- 1 | % u_update_ibfc.m 2 | % ibfc cluster prototype update function 3 | % 4 | % Copyright 2013 Taylor C. Glenn 5 | % tcg@cise.ufl.edu 6 | % see ../LICENSE.txt for license terms 7 | 8 | function C = c_update_ibfc(X,U,p) 9 | 10 | K = size(U,2); 11 | D = p.D; 12 | 13 | C = zeros(K,D); 14 | 15 | Um = U.^p.m; 16 | 17 | sumUm = sum(Um,1); 18 | eD = eye(D); 19 | for k=1:K 20 | xbar = sum(bsxfun(@times,X,Um(:,k)),1); 21 | C(k,:) = (xbar+p.mu_c*p.siginv_c)/(sumUm(k)*eD+p.siginv_c); 22 | end 23 | 24 | end -------------------------------------------------------------------------------- /ibfc/ibfc_pf.m: -------------------------------------------------------------------------------- 1 | % ibfc_pf.m 2 | % Infinite Bayesian Fuzzy Clustering Particle Filter 3 | % 4 | % Copyright 2013 Taylor C. Glenn 5 | % tcg@cise.ufl.edu 6 | % see ../LICENSE.txt for license terms 7 | 8 | function [C,U,K,ll,mx,trace] = ibfc_pf(X,params) 9 | 10 | % ibfc, infinite bayesian fuzzy clustering 11 | % a particle filter based search for MAP values of C,U,K 12 | 13 | P = params.n_particles; 14 | m = params.m; 15 | alpha = params.beta; % model sparsity weight (called beta in paper, alpha here) 16 | n_iter = params.max_iter; 17 | 18 | [N,D] = size(X); 19 | 20 | p = struct(); %parameters 21 | p.N = N; 22 | p.D = D; 23 | p.m = m; 24 | p.mu_x = mean(X,1); 25 | p.sigma_c = 3*cov(X); 26 | p.rt_sigma_c = chol(p.sigma_c); 27 | p.siginv_c = pinv(p.sigma_c); 28 | p.logdet_sigma_c = logdet(p.sigma_c); 29 | p.mu_c = p.mu_x; 30 | p.C1 = -D/2*log(2*pi); % common gaussian constant 31 | p.alpha = alpha; 32 | 33 | %P = 5; % number of sample particles to use 34 | 35 | ll_fun = params.ll_fun; 36 | 37 | % initialize 38 | s = struct(); 39 | s.K = 1; 40 | s.C = randn(1,D)*p.rt_sigma_c + p.mu_c; 41 | s.U = u_update_ibfc(X,s.C,p); 42 | s.ll = ll_fun(X,s,p); 43 | 44 | for i=1:P 45 | pcl(i) = s; 46 | end 47 | 48 | mx = s; 49 | 50 | if params.do_trace 51 | trace = nan(1,n_iter+1); 52 | trace(1) = mx.ll; 53 | else 54 | trace = []; 55 | end 56 | 57 | n_update = 1; 58 | mxll = -inf; 59 | converge_ctr = 0; 60 | 61 | for iter=1:n_iter 62 | 63 | for i=1:P 64 | s = pcl(i); 65 | K = poissrnd(s.K); 66 | if K < s.K 67 | rp = randperm(s.K); 68 | inds = rp(1:K); 69 | pcl(i).K = K; 70 | pcl(i).C = s.C(inds,:); 71 | 72 | elseif K > s.K 73 | n_new = K-s.K; 74 | new_C = randn(n_new,D)*p.rt_sigma_c + repmat(p.mu_c,n_new,1); 75 | pcl(i).K = K; 76 | pcl(i).C = [s.C; new_C]; 77 | else 78 | pcl(i).K = s.K; 79 | pcl(i).C = s.C; 80 | end 81 | 82 | for update=1:n_update 83 | %update Us of each particle 84 | pcl(i).U = u_update_ibfc(X,pcl(i).C,p); 85 | 86 | %update C for each particle 87 | pcl(i).C = c_update_ibfc(X,pcl(i).U,p); 88 | end 89 | %compute likelihood of particle 90 | pcl(i).ll = ll_fun(X,pcl(i),p); 91 | mx = update_mx(pcl(i),mx); 92 | end 93 | 94 | %sample among particles by importance 95 | not_empty_mx = ~cellfun(@isempty,{mx.ll}); 96 | pcls = [pcl mx(not_empty_mx)]; 97 | lls = [pcls.ll]; 98 | prev_mxll = mxll; 99 | mxll = max(lls); 100 | 101 | probs = exp(lls - (log(sum(exp(lls - mxll)))+mxll)); 102 | inds = sample(probs,P); 103 | 104 | pcl = pcls(inds); 105 | 106 | %------ reporting --------- 107 | if mod(iter,50) == 0 108 | 109 | fprintf('------\n'); 110 | for sz = 1:numel(mx) 111 | fprintf('sz %d: %f\n',sz,mx(sz).ll); 112 | end 113 | fprintf('------\n'); 114 | end 115 | 116 | if mod(iter,10) == 0 117 | mx_lls = -inf(1,numel(mx)); 118 | for sz = 1:numel(mx) 119 | if ~isempty(mx(sz).ll), mx_lls(sz) = mx(sz).ll; end 120 | end 121 | [mx_ll,mx_ind] = max(mx_lls); 122 | fprintf('%d: max ll: %f (%d), curr: ',iter,mx_ll,mx_ind); 123 | fprintf('%d ',[pcl.K]); 124 | fprintf('\n'); 125 | end 126 | 127 | if params.do_trace 128 | trace(iter) = mxll; 129 | end 130 | 131 | %------ check for convergence ----- 132 | change = mxll-prev_mxll; 133 | if change < params.converge_thresh 134 | converge_ctr = converge_ctr+1; 135 | if converge_ctr >= params.converge_iter 136 | fprintf('Converged\n'); 137 | break; 138 | end 139 | else 140 | converge_ctr = 0; 141 | end 142 | end 143 | 144 | mx_lls = -inf(1,numel(mx)); 145 | for sz = 1:numel(mx) 146 | if ~isempty(mx(sz).ll), mx_lls(sz) = mx(sz).ll; end 147 | end 148 | [ll,mx_ind] = max(mx_lls); 149 | 150 | C = mx(mx_ind).C; 151 | U = mx(mx_ind).U; 152 | K = mx(mx_ind).K; 153 | 154 | end 155 | 156 | function mx = update_mx(s,mx) 157 | 158 | if s.K > 0 && (s.K > numel(mx) || isempty(mx(s.K).ll) || s.ll > mx(s.K).ll) 159 | mx(s.K) = s; 160 | end 161 | 162 | end -------------------------------------------------------------------------------- /ibfc/ibfc_pf_params.m: -------------------------------------------------------------------------------- 1 | % ibfc_pf_params.m 2 | % default parameters structure for ibfc_pf 3 | % 4 | % Copyright 2013 Taylor C. Glenn 5 | % tcg@cise.ufl.edu 6 | % see ../LICENSE.txt for license terms 7 | 8 | function params = ibfc_pf_params() 9 | 10 | params = struct(); 11 | 12 | params.m = 2; % fuzzifier 13 | params.beta = 2; % model sparsity weight (bigger is more sparse model) 14 | 15 | params.n_particles = 10; % number of particles in the particle filter 16 | 17 | params.max_iter = 500; % maximum total iterations 18 | 19 | params.converge_thresh = 1e-5; % converged if max loglike changes < this 20 | params.converge_iter = 10; % and has stayed like this for > this iterations 21 | 22 | params.do_trace = false; % record likelihood at each iteration 23 | 24 | params.ll_fun = @ll_ibfc; 25 | 26 | end -------------------------------------------------------------------------------- /ibfc/ll_ibfc.m: -------------------------------------------------------------------------------- 1 | % ll_ibfc.m 2 | % ibfc log-likelihood evaluation function(s) 3 | % 4 | % Copyright 2013 Taylor C. Glenn 5 | % tcg@cise.ufl.edu 6 | % see ../LICENSE.txt for license terms 7 | 8 | function ll = ll_ibfc(X,s,p) 9 | % full (joint) likelihood of data X and sample s given parameters p 10 | if isempty(s.U) 11 | ll = -inf; 12 | return; 13 | end 14 | llx = ll_X_given_UCK(X,s,p); 15 | llu = ll_U(s,p); 16 | llc = ll_C(s,p); 17 | llk = ll_K(s,p); 18 | 19 | ll = llx+llu+llc+llk; 20 | 21 | end 22 | 23 | function ll = ll_X_given_UCK(X,s,p) 24 | % data likelihood 25 | 26 | N = p.N; 27 | K = size(s.C,1); 28 | 29 | Um = s.U.^p.m; 30 | WW = zeros(N,K); 31 | for k=1:K 32 | W = X - repmat(s.C(k,:),N,1); 33 | WW(:,k) = sum(W.*W,2); 34 | end 35 | 36 | ll = N*p.C1 - .5/K*sum(sum(Um.*WW)); %average over K 37 | 38 | end 39 | 40 | function ll = ll_U(s,p) 41 | %memberships 42 | 43 | K = size(s.U,2); 44 | 45 | ll = p.alpha*p.N/(K); 46 | 47 | end 48 | 49 | function llc = ll_C(s,p) 50 | % cluster centers 51 | K = size(s.C,1); 52 | 53 | W = s.C - repmat(p.mu_c,K,1); 54 | WW = sum((W*p.siginv_c).*W,2); 55 | 56 | llc = sum(p.C1 - p.logdet_sigma_c - .5*WW)/K; 57 | 58 | end 59 | 60 | function llk = ll_K(s,p) 61 | % number of clusters 62 | 63 | % K ~ poisson(alpha * log(N)) 64 | % likelihood(k|lambda) = lambda^k/k! * exp(-lambda) 65 | % ll = k*log(lambda) - sum(log(1:k)) - lambda 66 | 67 | lambda = 1 * log(p.N); %fixme: maybe put a p.beta parameter to weight lambda 68 | llk = s.K* log(lambda) - sum(log(1:s.K)) - lambda; 69 | 70 | end 71 | 72 | 73 | -------------------------------------------------------------------------------- /ibfc/test.m: -------------------------------------------------------------------------------- 1 | % test.m 2 | % test script for ibfc, 4 clusters 3 | % two large spheres and two small spheres 4 | % 5 | % Copyright 2013 Taylor C. Glenn 6 | % tcg@cise.ufl.edu 7 | % see ../LICENSE.txt for license terms 8 | 9 | 10 | RandStream.setDefaultStream(RandStream('mt19937ar','Seed', 5489)); 11 | 12 | m = 2; 13 | % generate some data 14 | 15 | n_dim = 2; 16 | n_pts_per = 200; 17 | n_comp = 4; 18 | 19 | mu1 = [2; 2]; 20 | mu2 = [2; 9]; 21 | mu3 = [9; 2]; 22 | mu4 = [9; 9]; 23 | 24 | c1 = 2*[1 0;0 1]; 25 | c2 = .25*[1 0;0 1]; 26 | c3 = .75*[1 0;0 1]; 27 | c4 = 2*[1 0;0 1]; 28 | 29 | 30 | x1 = chol(c1)*randn(n_dim,n_pts_per) + repmat(mu1,1,n_pts_per); 31 | x2 = chol(c2)*randn(n_dim,n_pts_per) + repmat(mu2,1,n_pts_per); 32 | x3 = chol(c3)*randn(n_dim,n_pts_per) + repmat(mu3,1,n_pts_per); 33 | x4 = chol(c4)*randn(n_dim,n_pts_per) + repmat(mu4,1,n_pts_per); 34 | 35 | X = [x1 x2 x3 x4]; 36 | 37 | %--- fcm C=4 38 | [C,U] = fcm(X',n_comp); 39 | 40 | [~,label] = max(U,[],1); 41 | 42 | figure(11); 43 | clf; 44 | scatter(X(1,:),X(2,:),[],label); 45 | hold on; 46 | for i=1:n_comp 47 | text(C(i,1),C(i,2),num2str(i),'BackgroundColor','w','Color','k'); 48 | end 49 | title('FCM'); 50 | xlabel('D1'); 51 | ylabel('D2'); 52 | axis equal; 53 | 54 | 55 | %--- ibfc 56 | pr = ibfc_pf_params(); 57 | pr.m = m; 58 | pr.n_particles = 10; 59 | pr.max_iter = 50; 60 | pr.converge_iter = 50; 61 | pr.alpha = 2; 62 | [iC,iU,iK,ill,mx] = ibfc_pf(X',pr); 63 | 64 | %---- display max likelihood cluster output 65 | [~,l_iU] = max(iU,[],2); 66 | figure(12); 67 | clf; 68 | scatter(X(1,:),X(2,:),[],l_iU'); 69 | hold on; 70 | for i=1:size(iC,1) 71 | text(iC(i,1),iC(i,2),num2str(i),'BackgroundColor','w','Color','k'); 72 | end 73 | title({'IBFC',sprintf('K=%d LogLikelihood=%.3f',iK,ill)}); 74 | xlabel('D1'); 75 | ylabel('D2'); 76 | axis equal; 77 | 78 | 79 | %---- display other, less likely, cluster outputs 80 | sizes = [2 3 5 6]; 81 | for i=1:numel(sizes) 82 | 83 | s = mx(sizes(i)); 84 | [~,l_s] = max(s.U,[],2); 85 | 86 | figure(20+i); 87 | clf; 88 | scatter(X(1,:),X(2,:),[],l_s'); 89 | hold on; 90 | for k=1:size(s.C,1) 91 | text(s.C(k,1),s.C(k,2),num2str(k),'BackgroundColor','w','Color','k'); 92 | end 93 | title({'IBFC',sprintf('K=%d LogLikelihood=%.3f',s.K,s.ll)}); 94 | xlabel('D1'); 95 | ylabel('D2'); 96 | axis equal; 97 | 98 | end 99 | 100 | 101 | 102 | 103 | -------------------------------------------------------------------------------- /ibfc/u_update_ibfc.m: -------------------------------------------------------------------------------- 1 | % u_update_ibfc.m 2 | % ibfc membership update function 3 | % 4 | % Copyright 2013 Taylor C. Glenn 5 | % tcg@cise.ufl.edu 6 | % see ../LICENSE.txt for license terms 7 | 8 | function U = u_update_ibfc(X,C,p) 9 | 10 | K = size(C,1); 11 | N = p.N; 12 | 13 | U = zeros(N,K); 14 | 15 | idsq = zeros(N,K); 16 | for i=1:K 17 | W = X - repmat(C(i,:),N,1); 18 | idsq(:,i) = 1./sum(W.*W,2); 19 | end 20 | sum_idsq = sum(idsq,2); 21 | 22 | for i=1:K 23 | U(:,i) = (idsq(:,i)./sum_idsq).^(1/(p.m-1)); 24 | end 25 | 26 | end --------------------------------------------------------------------------------