├── README.md ├── adaptive_rejection_sampling.m ├── costFunction.m ├── dirichlet_process.m ├── gibbs.m ├── gmm_demo.m ├── hybrid_gmm.m ├── kalman_demo.m ├── kmeans_demo.m ├── lda_gibbs_example.m ├── soft_max.m ├── test_autocorrelation.m └── vb_normal_gamma.m /README.md: -------------------------------------------------------------------------------- 1 | # matlab_demos 2 | these are the MATLAB demo code associated with my machine learning notes 3 | please browse through [my machine leanring turtoria notes](https://github.com/roboticcam/machine-learning-notes/blob/master/README.md) to see which notes these MATLAB files are associated with 4 | 5 | 这些是我写的一系列MATLAB演示代码， 请到[我的机器学习讲义](https://github.com/roboticcam/machine-learning-notes/blob/master/README.md)看那些讲义用了这些代码 6 | 7 | 8 | -------------------------------------------------------------------------------- /adaptive_rejection_sampling.m: -------------------------------------------------------------------------------- 1 | %-------------------------------------------- 2 | % Demo to illustrate Adaptive Rejection Sampling 3 | % 4 | % written by Richard Xu 5 | % Yida.Xu@uts.edu.au 6 | % July,2012 7 | %-------------------------------------------- 8 | 9 | 10 | function s = adaptive_rejection_sampling() 11 | 12 | clc; 13 | clear; 14 | 15 | % -------------------------------------------------- 16 | % this is to sample Gaussian distribution 17 | % note that although gaussian is log-concave, ARS is an over-kill for 18 | % sampling it, this is for pure demo purpose 19 | % -------------------------------------------------- 20 | 21 | mu = 1; 22 | sigma = 3; 23 | log_func = @(x) (-1/2 * (x - mu).^2/sigma); 24 | log_dev = @(x) (- (x-mu)/sigma); 25 | %func = @(x) exp(-1/2 * (x - mu).^2/sigma); 26 | func = @(x) exp( (-1/2 * (x - mu).^2/sigma) ); 27 | init_points = [4 ; -1.5; 2; -3]; 28 | 29 | bounds(1) = -inf; 30 | bounds(2) = inf; 31 | 32 | plot_bounds(1) = -10; 33 | plot_bounds(2) = 10; 34 | 35 | 36 | % ------------------------------------------------------------------ 37 | % please adjust this parameter to see how the envelop was constructed 38 | % ------------------------------------------------------------------ 39 | 40 | MAX_ACCEPTED = 10; 41 | IS_PLOT = 1; 42 | 43 | 44 | % ------------------------------------------- 45 | % this is to sample posteior of DP's concentration parameter 46 | % ------------------------------------------- 47 | 48 | % w = 4; 49 | % s = ceil(rand([10 1]) * 10); 50 | % w_s_j_term_log = sum(log(s) + log(w) - s * w); 51 | % k = length(s); 52 | % 53 | % w_s_j_term_log = - 22; 54 | % %k = 20; 55 | % 56 | % offset = 0; 57 | % 58 | % log_func = @(b)( - k * gammaln(b/2) ... 59 | % - 1 ./(2*b)... 60 | % + (k * b - 3)/2 .* log(b/2)... 61 | % + b/2 * w_s_j_term_log + offset ); 62 | % 63 | % log_dev = @(b)( - k/2 * psi(b/2) ... 64 | % + 1/2 * b.^(-2) ... % 1/2 * b^-1 => -1/2 * b^-2 65 | % + k/2 * log(b/2)... 66 | % + (k * b - 3)./(2 * b)... log(b/2) => 2/b * 1/2 = 1/b 67 | % + w_s_j_term_log /2 ); 68 | % 69 | % func = @(b)(exp( - k * gammaln(b/2) ... 70 | % - 1 ./(2*b)... 71 | % + (k * b - 3)/2 .* log(b/2)... 72 | % + b/2 * w_s_j_term_log + offset)); 73 | % 74 | % log_func_neg = @(b)( -( - k * gammaln(b/2) ... 75 | % - 1 ./(2*b)... 76 | % + (k * b - 3)/2 .* log(b/2)... 77 | % + b/2 * w_s_j_term_log + offset)); 78 | % 79 | % max_pt = fminbnd(log_func_neg,0,100); 80 | % 81 | % bounds(1) = 0; 82 | % bounds(2) = inf; 83 | % 84 | % plot_bounds(1) = 0; 85 | % plot_bounds(2) = 4* max_pt; 86 | % 87 | % init_points = [max_pt/2; max_pt + max_pt/2]; 88 | % MAX_ACCEPTED = 100; 89 | % IS_PLOT = 1; 90 | 91 | % ------------------------------------------- 92 | 93 | [samples num_sampled] = ARS(func, log_func, log_dev, init_points, MAX_ACCEPTED, IS_PLOT, bounds, plot_bounds); 94 | fprintf('acceptance ratio = %f',MAX_ACCEPTED / num_sampled); 95 | 96 | 97 | function [samples num_sampled] = ARS(func, log_func, log_dev, init_points, MAX_ACCEPTED, IS_PLOT, bounds, plot_bounds) 98 | 99 | 100 | % (x,f) are the points on the log_func curve 101 | % z is the intersection points 102 | % m, b are gradient and y-intecept of tangenet 103 | x = init_points; 104 | 105 | [m b x z f ] = get_piecewise_exp (x, log_func, log_dev); 106 | 107 | left_bounds = [bounds(1); z']; 108 | right_bounds = [z'; bounds(2)]; 109 | cdf = exp(b) ./ m .* ( exp( m .* right_bounds) - exp( m .* left_bounds)); 110 | 111 | samples =[]; 112 | 113 | num_accepted = 0; 114 | num_sampled = 0; 115 | 116 | while num_accepted < MAX_ACCEPTED 117 | 118 | u_star = rand * sum(cdf,1); 119 | 120 | % compute inverse CDF 121 | c = cumsum(cdf); 122 | 123 | ind = find(c>=u_star, 1); 124 | 125 | if ind == 1 126 | L_int = 0; 127 | else 128 | L_int = c(ind-1); 129 | end 130 | 131 | L = left_bounds(ind); 132 | 133 | % int^x_{L} { exp (m x + b) } = exp(b)/m [ exp( m x) ]^x_{L} 134 | % = exp(b)/m [ exp( m x) - exp( mL) ] 135 | % u - L_int = exp(b)/m [ exp( m x) - exp( mL) ] 136 | 137 | u_k = u_star - L_int; 138 | 139 | % u_k = exp(b)/m [ exp( m x) - exp( mL) ] 140 | % exp(m x) - exp(m L ) = u_k * m / exp(b) 141 | % exp(m x) = u_k * m / exp(b) + exp(m L ) 142 | 143 | if m(ind) <0 144 | 145 | % there is log(-ve) problem, so we re-write m => - m, and m >0 146 | % exp(- mx) = - u_k * m /exp(b) + exp(- m L) 147 | % -mx = log [- u_k * m /exp(b) + exp(-mL) ] 148 | % mx = log {[- u_k * m /exp(b) + exp(-mL) ]^(-1)} 149 | % x = log {[- u_k * m /exp(b) + exp(-mL) ]^(-1)}/m 150 | 151 | m_pos = - m(ind); 152 | x_star = log ((- u_k * m_pos /exp(b(ind)) + exp(-m_pos * L) )^(-1))/m_pos; 153 | 154 | else 155 | 156 | % x = log [u_k * m / exp(b) + exp(m L )]/m 157 | 158 | %x_star = ( log(u+K) + log(m(ind)) - b(ind))/ m(ind); 159 | try 160 | x_star = log (u_k * m(ind) / exp(b(ind)) + exp(m(ind) * L ))/m(ind); 161 | catch 162 | h = 1; 163 | end 164 | end 165 | 166 | u_accept = rand; 167 | 168 | ratio = func(x_star) / exp(m(ind)*x_star + b(ind)); 169 | 170 | if u_accept < ratio 171 | samples(end+1) = x_star; 172 | num_accepted = num_accepted + 1; 173 | else 174 | [m b x z f ] = get_piecewise_exp ([x;x_star], log_func, log_dev); 175 | left_bounds = [-inf; z']; 176 | right_bounds = [z'; inf]; 177 | cdf = exp(b) ./ m .* ( exp( m .* right_bounds) - exp( m .* left_bounds)); 178 | end 179 | 180 | num_sampled = num_sampled + 1; 181 | 182 | end 183 | 184 | % ---------------------------------------------- 185 | % Debug - plot the final envelop 186 | % ---------------------------------------------- 187 | 188 | 189 | f1 = figure(1); 190 | set(f1,'Name','adapative rejection sampling'); 191 | 192 | if IS_PLOT 193 | 194 | step = (plot_bounds(2) - plot_bounds(1))/100; 195 | t = plot_bounds(1):step:plot_bounds(2); 196 | 197 | d = log_dev(x); 198 | for i=1:size(x,1)-1 199 | f_z(i) = (z(i) - x(i)) * d(i) + f(i); 200 | end 201 | 202 | subplot(1,2,1) 203 | 204 | y = log_func(t); 205 | plot(t,y); 206 | 207 | hold on; 208 | 209 | plot(x,f,'o'); 210 | 211 | hold on; 212 | 213 | plot(z,f_z,'o','color',[1 0 0]); 214 | 215 | hold on; 216 | 217 | border_x(1) = plot_bounds(1); 218 | border_y(1) = f(1) - (x(1) - border_x(1))*d(1); 219 | 220 | border_x(2) = plot_bounds(2); 221 | border_y(2) = f(end) - ( x(end) - border_x(2)) * d(end); 222 | 223 | % border_y(1) = -18; 224 | % border_x(1) = (border_y(1) - f(1))/d(1) + x(1); 225 | % border_y(2) = -18; 226 | % border_x(2) = (border_y(end) - f(end))/d(end) + x(end); 227 | 228 | plot(border_x,border_y,'o','color',[1 0 1]); 229 | 230 | hold on; 231 | 232 | l_line(1,:) = [border_x(1),border_y(1)]; 233 | l_line(2:2+size(x,1)-2,1) = z'; 234 | l_line(2:2+size(x,1)-2,2) = f_z; 235 | l_line(end+1,:) = [border_x(2),border_y(2)]; 236 | 237 | plot ( l_line(:,1), l_line(:,2), 'Color', [1 0 1], 'LineWidth',1); 238 | 239 | % for i = 1: size(l_line,1)-1 240 | % plot ( l_line(i:i+1,1), l_line(i:i+1,2), 'Color', [0 0 1], 'LineWidth',1); 241 | % hold on; 242 | % end 243 | 244 | hold off; 245 | title('log(P(x)) space') 246 | 247 | 248 | 249 | % --------------------------------------------- 250 | % plot q(.) in the original scale 251 | % --------------------------------------------- 252 | 253 | subplot(1,2,2) 254 | 255 | y = func(t); 256 | plot(t,y); 257 | 258 | hold on; 259 | plot(x,func(x),'o'); 260 | hold on; 261 | 262 | max_f = max(func(t)); 263 | min_f = min(func(t)); 264 | 265 | for i=1:size(x,1)-1 266 | plot([z(i) z(i)], [max_f min_f],'LineWidth',1, 'Color', [ 0 1 1]); 267 | end 268 | hold on; 269 | 270 | for i=1:size(x,1) 271 | if i == 1 272 | t = border_x(1):0.01:z(i); 273 | elseif i == size(x,1) 274 | t = z(end):0.01:border_x(2); 275 | else 276 | t = z(i-1):0.01:z(i); 277 | end 278 | 279 | plot(t, exp(m(i)*t + b(i)), 'color',[1 0 1]); 280 | hold on; 281 | end 282 | 283 | hold on; 284 | 285 | plot(samples, zeros(length(samples),1),'*'); 286 | 287 | hold off; 288 | title('P(x) space') 289 | 290 | end 291 | 292 | 293 | 294 | 295 | function [m b x z f_log] = get_piecewise_exp (x, log_func, log_dev) 296 | 297 | try 298 | f_log = log_func(x); 299 | catch 300 | h = 1; 301 | end 302 | 303 | d = log_dev(x); 304 | 305 | [u v] = sort(x); 306 | 307 | temp = f_log(v); 308 | f_log = temp; 309 | 310 | temp = d(v); 311 | d = temp; 312 | 313 | temp = x(v); 314 | x = temp; 315 | 316 | % get the meeting points 317 | for i=1:size(x,1)-1 318 | z(i) = ( f_log(i+1) - f_log(i) - x(i+1) * d(i+1) + x(i) * d(i) )/( d(i) - d(i+1) ); 319 | %f_log_z(i) = (z(i) - x(i)) * d(i) + f_log(i); 320 | end 321 | 322 | % u(x) = f_log(i) + (x - x(i)) * d(i) 323 | % = f_log(i) + d(i) x - x(i) * d(i) 324 | % = [f_log(i) - x(i) * d(i)] + d(i) * x 325 | % = b + m x 326 | 327 | 328 | for i=1:size(x,1) 329 | b(i) = f_log(i) - x(i) * d(i); 330 | m(i) = d(i); 331 | end 332 | 333 | b = b'; m = m'; 334 | 335 | -------------------------------------------------------------------------------- /costFunction.m: -------------------------------------------------------------------------------- 1 | % ------------------------------------------------------ 2 | % Demonstraton of loss function using linear regression 3 | % written by Richard Xu 4 | % Jan 2017 5 | % yida.xu@uts.edu.au 6 | % ------------------------------------------------------ 7 | 8 | function costFunction 9 | 10 | 11 | % ------------------------------------------------------------ 12 | % Generate linear regression dataset, adding noise to both x and y 13 | % ------------------------------------------------------------ 14 | 15 | m = 0.5; 16 | b = 2; 17 | 18 | 19 | X = 0:0.5:10; 20 | X = X + (randn(length(X),1))'; 21 | Y = m* X + b + (randn(length(X),1))'* 0.5; 22 | 23 | 24 | 25 | figure(1); 26 | 27 | % ------------------------------------------------------------ 28 | % Plot loss function across the whole range of values of (m,b) 29 | % ------------------------------------------------------------ 30 | 31 | [val_m,val_b] = meshgrid(0:0.02:1, -3:0.02:3); 32 | val_m_data = repmat(val_m,[1 1 length(X) ]); 33 | val_b_data = repmat(val_b,[1 1 length(X) ]); 34 | 35 | val_X = repmat(X', [1 size(val_m,1) size(val_m,2)]); 36 | 37 | % ------------------------------------------------------------ 38 | % put the dimensionality in the correct order 39 | % ------------------------------------------------------------ 40 | 41 | val_X=permute(val_X,[2 3 1]); 42 | 43 | 44 | val_Y = repmat(Y', [1 size(val_m,1) size(val_m,2)]); 45 | val_Y = permute(val_Y,[2 3 1]); 46 | 47 | 48 | cost_func = val_Y - (val_m_data .* val_X + val_b_data); 49 | cost_func = sum(cost_func.^2,3) / length(X); 50 | mesh(val_m,val_b,cost_func); 51 | 52 | xlabel('m'); 53 | ylabel('b'); 54 | zlabel('average loss'); 55 | 56 | 57 | figure(2); 58 | plot(X,Y,'+','color',[1 0 0]); 59 | 60 | xlabel('project duration'); 61 | ylabel('cost'); 62 | axis equal; 63 | axis( [ -5 15 0 10]); 64 | 65 | 66 | while(true) 67 | 68 | min_Y = m*min(X)+b; 69 | max_Y = m*max(X)+b; 70 | 71 | h1 = line([min(X) max(X)],[min_Y max_Y]); 72 | h1 = [h1; line([X' X']',[Y' (m*X+b)']','color',[0 1 0])]; 73 | 74 | average_loss = sum((Y- (m*X+b)).^2)/length(X); 75 | xlabel(['project duration: Parameter: m = ' num2str(m,'%0.1f') ', b = ' num2str(b,'%0.1f') ... 76 | ', average square loss = ' num2str(average_loss,'%0.1f') ]); 77 | 78 | m = rand; 79 | b = rand*5 - 2.5; 80 | 81 | waitforbuttonpress; 82 | delete(h1); 83 | end 84 | 85 | 86 | end -------------------------------------------------------------------------------- /dirichlet_process.m: -------------------------------------------------------------------------------- 1 | %-------------------------------------------------- 2 | % visualization of stick-breaking process 3 | % 4 | % 5 | % Written by Richard Xu: YiDa.Xu@uts.edu.au 6 | % July 2014 7 | %-------------------------------------------------- 8 | 9 | function dirichlet_process() 10 | 11 | 12 | % alpah : DP's concentration factor 13 | % K : number of sticks (Large K} 14 | % N_Iter : number of times G ~ DP(alpha, H) is drawn 15 | % init_parts : some initial partiion (just so partitions not too small 16 | % x_range : x range (for plotting) 17 | % max_p : max_p (maximum probability, for plotting) 18 | 19 | 20 | clear all; 21 | clc; 22 | 23 | alpha = 3; 24 | K = 50; 25 | N_Iter = 1000; 26 | 27 | CASE = 1; 28 | 29 | if CASE == 1 30 | 31 | %----------------------------------------------- 32 | % H is gaussian 33 | %----------------------------------------------- 34 | 35 | init_parts = -3:2:3; 36 | x_range = [-4 4]; 37 | max_p = 0.7; 38 | H_pdf = @normpdf; 39 | H_cdf = @normcdf; 40 | H_rand = @nrand_func; 41 | 42 | else 43 | 44 | %----------------------------------------------- 45 | % H is gamma 46 | %----------------------------------------------- 47 | 48 | init_parts = 0:2:8; 49 | x_range = [0 10]; 50 | max_p = 0.7; 51 | H_pdf = @gampdf_func; 52 | H_cdf = @gamcdf_func; 53 | H_rand = @gamrnd_func; 54 | 55 | end 56 | 57 | % ------------------------------------------------------------------ 58 | % function starts here 59 | % ------------------------------------------------------------------ 60 | 61 | % create random set of partitions 62 | r_parts = rand([1 length(init_parts)-1 ]) .* ( init_parts(2:end) - init_parts(1:end-1)) + init_parts(1:end-1); 63 | 64 | P = length(r_parts); 65 | 66 | mean_weights = zeros(N_Iter, P + 1); 67 | 68 | % theretical mean from CDF 69 | p = H_cdf(r_parts); 70 | cdf_regions = [p 1] - [0 p]; 71 | 72 | theory_mean = cdf_regions; 73 | 74 | % theretical variances 75 | theory_variance = theory_mean .* (ones(1, P+1) - theory_mean) / (alpha + 1); 76 | 77 | 78 | for i = 1:N_Iter 79 | 80 | % draw random partitions 81 | plot ( repmat(r_parts, [ 2 1]), [zeros(1,P); ones(1,P)*max_p], 'LineWidth',2,'color',[1 0 0]); 82 | hold on; 83 | 84 | % plot f(theta) 85 | x_data = x_range(1):0.01:x_range(2); 86 | plot(x_data,H_pdf(x_data),'.'); 87 | hold on; 88 | 89 | % sample G ~ DP (alpha, H) 90 | 91 | 92 | [sticks_weights, thetas] = Stick_breaking_process(alpha, K, H_rand); 93 | 94 | num_samples = length(thetas); 95 | 96 | 97 | % draw G 98 | lineX = repmat(thetas, [1 2]); 99 | lineY = [zeros([1 num_samples]); sticks_weights' ]; 100 | plot ( lineX', lineY, 'LineWidth',1,'color',[0 0 1]); 101 | hold on; 102 | 103 | for j=1:P+1 104 | if j == 1 105 | w_region = find(thetas < r_parts(j)); 106 | elseif j == P+1 107 | w_region = find(thetas > r_parts(j-1)); 108 | else 109 | w_region = find(thetas > r_parts(j-1) & thetas < r_parts(j) ); 110 | end 111 | 112 | if ~isempty(w_region) 113 | mean_weights(i,j) = sum(sticks_weights(w_region)); 114 | else 115 | mean_weights(i,j) = 0; 116 | end 117 | end 118 | 119 | if ~isempty(w_region) 120 | mean_weights(i,j) = sum(sticks_weights(w_region)); 121 | else 122 | mean_weights(i,j) = 0; 123 | end 124 | 125 | 126 | 127 | emperical_mean = mean(mean_weights(1:i,:),1); 128 | if i > 1 129 | emperical_variance = var(mean_weights(1:i,:),1); 130 | end 131 | 132 | for j=1:P+1 133 | if j == 1 134 | display_coord = (x_range(1)+ r_parts(j) )/2; 135 | elseif j == P+1 136 | display_coord = (r_parts(j-1) + x_range(2) )/2; 137 | else 138 | display_coord = (r_parts(j-1) + r_parts(j) )/2; 139 | end 140 | 141 | text( display_coord, max_p-0.1, num2str(theory_mean(j) ,'%.3f'), 'color', [1 0 0]); 142 | text( display_coord, max_p-0.2, num2str(emperical_mean(j) ,'%.3f'), 'color', [1 0 0]); 143 | text( display_coord, max_p-0.3, num2str(theory_variance(j) ,'%.3f'), 'color', [1 0 1]); 144 | if i > 1 145 | text( display_coord, max_p-0.4, num2str(emperical_variance(j) ,'%.3f'),'color', [1 0 1]); 146 | else 147 | text( display_coord, max_p-0.4, num2str(0 ,'%.3f'),'color', [1 0 1]); 148 | end 149 | end 150 | 151 | 152 | set(gca,'YTick',[0.5:0.1:max_p]); 153 | set(gca,'YTick',[max_p-0.4:0.1:max_p-0.1]); 154 | set(gca,'YTickLabel',[ 'emperical var '; 'theoretic var '; 'emperical mean'; 'theoretic mean']); 155 | 156 | 157 | hold on; 158 | 159 | axis([x_range(1) x_range(2) 0 max_p]); 160 | 161 | waitforbuttonpress; 162 | 163 | hold off; 164 | 165 | %sum(mean_weights(i,:)) 166 | end 167 | 168 | % emperical mean 169 | emperical_mean = mean(mean_weights); 170 | display(emperical_mean); 171 | 172 | 173 | % emperical variances 174 | emperical_variance = var(mean_weights); 175 | display(emperical_variance); 176 | 177 | end 178 | 179 | 180 | function probs = gampdf_func (x_data) 181 | a = 2; b = 2; 182 | probs = gampdf(x_data,a,b); 183 | end 184 | 185 | function probs = gamcdf_func (x_data) 186 | a = 2; b = 2; 187 | probs = gamcdf(x_data,a,b); 188 | end 189 | 190 | function x = gamrnd_func(N) 191 | a = 2; b = 2; 192 | x = gamrnd(a,b,[N 1]); 193 | end 194 | 195 | function x = nrand_func(N) 196 | x = randn([N 1]); 197 | end 198 | 199 | 200 | % stick-breaking process 201 | % when H_rand is a (continous) density function handel, it draws a sample, 202 | % otherwise, it performs a multinomial distribution draw 203 | 204 | function [sticks_weights, thetas] = Stick_breaking_process(alpha, N, H_rand) 205 | 206 | raw_sticks = betarnd(1,alpha,[N 1]); 207 | 208 | sticks_weights = zeros(N,1); 209 | 210 | sticks_weights(1) = raw_sticks(1); 211 | 212 | for i = 2:N 213 | sticks_weights(i) = raw_sticks(i) * (1 - sum(sticks_weights(1:i-1))); 214 | end 215 | 216 | % sampling thetas 217 | 218 | if isa(H_rand, 'function_handle') 219 | thetas = H_rand(N); 220 | else 221 | for t=1:N 222 | thetas(t) = find(rand < cumsum(H_rand),1); 223 | end 224 | h =1; 225 | end 226 | end 227 | -------------------------------------------------------------------------------- /gibbs.m: -------------------------------------------------------------------------------- 1 | % --------------------------- 2 | % Simulation of a Gibbs sampling 3 | % written by Richard Xu 4 | % March 2013 5 | % yida.xu@uts.edu.au 6 | % -------------------------- 7 | % clear; 8 | % m = 4; k=3; n = 5; A = randn(m,k); B = randn(k,n); % values 9 | % a1 = A * B; % method 1 10 | % display(a1); 11 | % 12 | % a2 = zeros(m,n); 13 | % for i=1:k; % method 2 14 | % a2 = a2+ A(:,i) * B(i,:); 15 | % end; 16 | % display(a2); 17 | % 18 | % return; 19 | 20 | 21 | clc; 22 | clf; 23 | clear; 24 | 25 | mu = [2 3]; 26 | Sigma = [ 3 2; 2 5]; 27 | 28 | 29 | figure(1); 30 | 31 | N = 2000; 32 | 33 | init_sample = [ 1 1]; 34 | 35 | directX = mvnrnd(mu,Sigma,N); 36 | 37 | minX = min(directX(:,1)); 38 | minY = min(directX(:,2)); 39 | maxX = max(directX(:,1)); 40 | maxY = max(directX(:,2)); 41 | 42 | figure(1); 43 | 44 | subplot(2,2,1); 45 | plot(directX(:,1),directX(:,2),'.','color',[1 0 0]); 46 | axis([minX maxX minY maxY]); 47 | 48 | subplot(2,2,2); 49 | autocorr(mvnpdf(directX,mu,Sigma)); 50 | 51 | 52 | 53 | HIDE_GIBBS_STEP = false; 54 | 55 | if HIDE_GIBBS_STEP == true 56 | 57 | X = zeros(N,2); 58 | X(1,:) = init_sample; 59 | 60 | for i = 2:N 61 | 62 | mu_1 = mu(1) + Sigma(1,2)/Sigma(2,2)*( X(i-1,2) - mu(2)); 63 | Sigma_1 = Sigma(1,1) - Sigma(1,2)/Sigma(2,2)*Sigma(2,1); 64 | 65 | X(i,1) = randn * sqrt(Sigma_1) + mu_1; 66 | 67 | mu_2 = mu(2) + Sigma(2,1)/Sigma(1,1)*(X(i,1) - mu(1)); 68 | Sigma_2 = Sigma(2,2) - Sigma(2,1)/Sigma(1,1)*Sigma(1,2); 69 | 70 | X(i,2) = randn * sqrt(Sigma_2) + mu_2; 71 | 72 | 73 | end 74 | 75 | subplot(2,2,3); 76 | plot(X(:,1),X(:,2),'.'); 77 | axis([minX maxX minY maxY]); 78 | %--------------------------------------- 79 | 80 | else 81 | 82 | 83 | subplot(2,2,3); 84 | 85 | 86 | 87 | X = zeros(N,2); 88 | X(1,:) = [1 1]; 89 | 90 | for i = 2:N 91 | 92 | mu_1 = mu(1) + Sigma(1,2)/Sigma(2,2)*( X(i-1,2) - mu(2)); 93 | Sigma_1 = Sigma(1,1) - Sigma(1,2)/Sigma(2,2)*Sigma(2,1); 94 | 95 | X(i,1) = randn * sqrt(Sigma_1) + mu_1; 96 | 97 | plot( [X(i-1,1) X(i,1)], [X(i-1,2) X(i-1,2)],'LineWidth',1); 98 | axis([minX maxX minY maxY]); 99 | hold on; 100 | 101 | waitforbuttonpress; 102 | %waitfor(100); 103 | mu_2 = mu(2) + Sigma(2,1)/Sigma(1,1)*(X(i,1) - mu(1)); 104 | Sigma_2 = Sigma(2,2) - Sigma(2,1)/Sigma(1,1)*Sigma(1,2); 105 | 106 | X(i,2) = randn * sqrt(Sigma_2) + mu_2; 107 | plot( [X(i,1) X(i,1)], [X(i-1,2) X(i,2)],'LineWidth',1); 108 | axis([minX maxX minY maxY]); 109 | 110 | hold on; 111 | 112 | waitforbuttonpress; 113 | %waitfor(100); 114 | end 115 | 116 | 117 | plot(X(:,1),X(:,2),'.'); 118 | 119 | end 120 | 121 | subplot(2,2,4); 122 | autocorr(mvnpdf(X,mu,Sigma)); 123 | 124 | -------------------------------------------------------------------------------- /gmm_demo.m: -------------------------------------------------------------------------------- 1 | % -------------------------------------- 2 | % Gausssian Mixture Model demos 3 | % written by Richard Xu 4 | % yida.xu@uts.edu.au, July, 2014 5 | % made it Octave-friendly 6 | % -------------------------------------- 7 | 8 | clc; 9 | clear; 10 | 11 | 12 | % ------------------------------------------------------ 13 | % Synthetically generate data from nominated parameters 14 | % ------------------------------------------------------ 15 | 16 | % N is the number of data 17 | N = 1000; 18 | gt_alpha = [0.3 0.3 0.4]; 19 | 20 | % M is K, means the number of clusters 21 | M = length(gt_alpha); 22 | 23 | 24 | % Sample Z ~ Mult(N,gt_alpha) 25 | 26 | Z = []; 27 | 28 | % drawing from a multinomial distribution, parameterised by gt_alpha 29 | % R1 = mnrnd(N,gt_alpha); 30 | 31 | R = mnrnd(N, gt_alpha); 32 | 33 | 34 | for t = 1:M 35 | Z = [Z; ones([R(t) 1])*t]; 36 | end 37 | 38 | 39 | % Simulate data from ground truth N ( theta_{z_1},....,theta_{z_n} 40 | 41 | mu = [ -3 4; 0 0; 3 4]; 42 | Sigma(:,:,1) = [ 2 0; 0 1]; 43 | Sigma(:,:,2) = [ 2 0; 0 1]; 44 | Sigma(:,:,3) = [ 2 0; 0 3]; 45 | 46 | % data is X = {x_1, .... x_n} 47 | data = zeros([N 2]); 48 | 49 | for t = 1:M 50 | comp_size = length(find(Z == t)); 51 | data(Z == t,:) = mvnrnd(mu(t,:), Sigma(:,:,t), comp_size); 52 | 53 | plot(data(:,1),data(:,2),'o'); 54 | end 55 | 56 | 57 | 58 | % ------------------------------------------------- 59 | % E-M algorithm 60 | % ------------------------------------------------- 61 | 62 | % initialize, i.e., nominate Theta^(1) 63 | prev_mu = [ -5 -1; -1 -1; 4 -1]; 64 | prev_Sigma(:,:,1) = [ 1 0; 0 1]; 65 | prev_Sigma(:,:,2) = [ 1 0; 0 1]; 66 | prev_Sigma(:,:,3) = [ 1 0; 0 1]; 67 | prev_alpha = ones([M 1])/M; 68 | 69 | 70 | % the iteration number 71 | MaxIter = 20; 72 | response = zeros([N M]); 73 | 74 | next_mu = zeros(size(prev_mu)); 75 | next_Sigma = zeros(size(prev_Sigma)); 76 | d = size(data,2); 77 | 78 | % ----------------------------- 79 | % The EM loop 80 | % ----------------------------- 81 | for t = 1:MaxIter 82 | 83 | if t > 1 84 | 85 | % ----------------------------- 86 | % Compute the responsibilities 87 | % ----------------------------- 88 | 89 | for l = 1:M 90 | response(:,l) = mvnpdf(data,prev_mu(l,:), prev_Sigma(:,:,l)); 91 | end 92 | 93 | response_sum = sum(response,2); 94 | response = response ./ repmat(response_sum,[1 3]); 95 | 96 | % --------------------------- 97 | % update alpha^(i+1) 98 | % --------------------------- 99 | 100 | next_alpha = sum(response)'/N; 101 | 102 | % --------------------------- 103 | % update mu^(i+1) 104 | % --------------------------- 105 | 106 | for l = 1:M 107 | next_mu(l,:) = sum(data .* repmat(response(:,l),[1 d])) / sum(response(:,l)); 108 | end 109 | 110 | % --------------------------- 111 | % update Sigma^(i+1) 112 | % --------------------------- 113 | 114 | for l = 1:M 115 | zero_mean_data = data - repmat(next_mu(l,:),[N 1]); 116 | %zero_mean_3d = reshape(zero_mean_data,[N 1 d]); 117 | 118 | covariances = zeros([d d N]); 119 | 120 | for i = 1:N 121 | covariances(:,:,i) = zero_mean_data(i,:)' * zero_mean_data(i,:) * response(i,l); 122 | end 123 | 124 | next_Sigma(:,:,l) = sum(covariances,3)/sum(response(:,l)); 125 | end 126 | 127 | prev_mu = next_mu; 128 | prev_Sigma = next_Sigma; 129 | prev_alpha = next_alpha; 130 | 131 | 132 | else 133 | 134 | % t = 1, just use initial parameters 135 | 136 | next_mu = prev_mu; 137 | next_Sigma = prev_Sigma; 138 | next_alpha = prev_alpha; 139 | 140 | end 141 | 142 | % ----------------------------- 143 | % plot GMM 144 | % ----------------------------- 145 | figure(1); 146 | plot(data(:,1), data(:,2), 'o'); 147 | axis([min(data(:,1)) max(data(:,1)) min(data(:,2)) max(data(:,2))]); 148 | hold on; 149 | 150 | gmm_pdf = gmdistribution(next_mu,next_Sigma,next_alpha); 151 | ezcontour(@(u,v)pdf(gmm_pdf,[u v])); 152 | 153 | 154 | % ----------------------------------------------- 155 | % Octave-friendly code 156 | % ----------------------------------------------- 157 | 158 | % x1 = min(data(:,1)):0.1:max(data(:,1)); 159 | % x2 = min(data(:,2)):0.1:max(data(:,2)); 160 | % [X1,X2] = meshgrid(x1,x2); 161 | % F = next_alpha(1) * mvnpdf([X1(:) X2(:)],next_mu(1,:),next_Sigma(:,:,1)); 162 | % for l = 2:M 163 | % F = F+ next_alpha(l) * mvnpdf([X1(:) X2(:)],next_mu(l,:),next_Sigma(:,:,l)); 164 | % end 165 | % F = reshape(F,length(x2),length(x1)); 166 | % contour(x1,x2,F); 167 | 168 | 169 | axis([min(data(:,1)) max(data(:,1)) min(data(:,2)) max(data(:,2))]); 170 | hold off; 171 | 172 | % this is to ask user to press a button 173 | waitforbuttonpress; 174 | 175 | end 176 | 177 | 178 | -------------------------------------------------------------------------------- /hybrid_gmm.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboticcam/matlab_demos/adf166783daf48d0deaa4ade194e14a4c55cb5d4/hybrid_gmm.m -------------------------------------------------------------------------------- /kalman_demo.m: -------------------------------------------------------------------------------- 1 | % -------------------------------------- 2 | % Kalman Filter demos 3 | % written by Richard Xu 4 | % yida.xu@uts.edu.au, July, 2014 5 | % -------------------------------------- 6 | 7 | clc; 8 | clear; 9 | 10 | 11 | % -------------------------------------------------------------------- 12 | % Synthetically generate data from nominated Kalman Filter parameters 13 | % -------------------------------------------------------------------- 14 | 15 | 16 | % for p(x_1) 17 | init_mu = [ 0 0]'; 18 | init_var = [ 5 0; 0 5]; 19 | 20 | 21 | % transition probability 22 | A = [1 0; 0 1]; 23 | 24 | B = [0.2 0.1]'; 25 | 26 | Q = [0.1 0.05; 0.05 0.1]; 27 | %Q = [1E-5 0; 0 1E-5]; 28 | 29 | % fixed parameters 30 | H = [1 0; 0 1]; 31 | R = [0.5 -0.05; -0.05 0.5]; 32 | %R = [1E-5 0 ; 0 1E-5]; 33 | 34 | T = 16; 35 | 36 | xs(1,:) = [0 0]; 37 | 38 | for t=2:T 39 | xs(t,:) = (A * xs (t-1,:)')' + B' + mvnrnd([0 0], Q ); 40 | end 41 | 42 | % data is a sequence of y 43 | data = xs + mvnrnd([0 0],R,T); 44 | 45 | data = data'; 46 | 47 | 48 | %update = Kalman_posterior (data, A, B, Q, H, R, T, init_mu, init_var); 49 | 50 | 51 | update_mu = zeros(2,T); 52 | update_Sigma = zeros(2,2,T); 53 | 54 | predict_mu = zeros(2,T); 55 | predict_Sigma = zeros(2,2,T); 56 | 57 | 58 | 59 | for t=1:T 60 | 61 | 62 | % -------------------------------- 63 | % Kalman Filter update 64 | % -------------------------------- 65 | 66 | if t == 1 67 | predict_mu(:,1) = init_mu; 68 | predict_Sigma(:,:,1) = init_var; 69 | else 70 | predict_mu(:,t) = A * update_mu(:,t-1) + B; 71 | predict_Sigma(:,:,t) = A * update_Sigma(:,:,t-1) * A' + Q; 72 | end 73 | 74 | S = H * predict_Sigma(:,:,t) * H' + R; 75 | K = predict_Sigma(:,:,t) * H' * inv(S); 76 | 77 | update_mu(:,t) = predict_mu(:,t) + K * (data(:,t) - H * predict_mu(:,t)); 78 | update_Sigma(:,:,t) = (eye(size(R,2)) - K * H ) * predict_Sigma(:,:,t); 79 | 80 | %display(update_mu(:,t)); 81 | display(update_Sigma(:,:,t)); 82 | 83 | 84 | % -------------------------------- 85 | % The plot 86 | % -------------------------------- 87 | 88 | 89 | gmm_pdf = gmdistribution(update_mu(:,t)',update_Sigma(:,:,t),1); 90 | ezcontour(@(u,v)pdf(gmm_pdf,[u v])); 91 | hold on; 92 | 93 | plot(data(1,1:t),data(2,1:t),'--','color',[0 0 1]); 94 | axis([min(data(1,:))-1 max(data(1,:))+1 min(data(2,:))-1 max(data(2,:))+1]); 95 | hold on; 96 | 97 | plot(update_mu(1,1:t),update_mu(2,1:t),'--','color',[1 0 0]); 98 | text(update_mu(1,t),update_mu(2,t), num2str(t), 'color', [ 1 0 0]); 99 | 100 | axis([min(data(1,:))-1 max(data(1,:))+1 min(data(2,:))-1 max(data(2,:))+1]); 101 | hold on; 102 | 103 | plot(data(1,t:T),data(2,t:T),'--','color',[0.7 0.7 0.7]); 104 | axis([min(data(1,:))-1 max(data(1,:))+1 min(data(2,:))-1 max(data(2,:))+1]); 105 | hold on; 106 | 107 | plot(data(1,t),data(2,t),'*'); 108 | 109 | for s=1:T 110 | text(data(1,s),data(2,s), num2str(s), 'color',[ 0 0 1]); 111 | hold on; 112 | end 113 | 114 | 115 | 116 | hold off; 117 | waitforbuttonpress; 118 | end 119 | 120 | 121 | 122 | -------------------------------------------------------------------------------- /kmeans_demo.m: -------------------------------------------------------------------------------- 1 | function kmeans_demo() 2 | 3 | % --------------------------------------------------- 4 | % K-means demos: This is a Octave friendly function 5 | % written by Richard Xu 6 | % yida.xu@uts.edu.au 7 | % --------------------------------------------------- 8 | 9 | clear all; 10 | clc; 11 | 12 | % -------------------------------------- 13 | % acts like a multinomial distribution 14 | % -------------------------------------- 15 | N = 1000; 16 | weights = [0.33 0.33 0.34]; 17 | M = length(weights); 18 | 19 | ind = rand([N 1]); 20 | Z = zeros([N 1]); 21 | acc_weights = cumsum(weights); 22 | 23 | for t = 1:M 24 | Z(ind > acc_weights(t)) = t; 25 | end 26 | Z = Z + 1; 27 | 28 | 29 | 30 | 31 | % ------------------------------------------ 32 | % Generate data from Gaussian Mixture model 33 | % ------------------------------------------ 34 | 35 | mu = [ -3 4; 0 0; 3 4]; 36 | Sigma(:,:,1) = [ 2 0; 0 1]; 37 | Sigma(:,:,2) = [ 2 0; 0 1]; 38 | Sigma(:,:,3) = [ 2 0; 0 3]; 39 | data = zeros([N 2]); 40 | 41 | for t = 1:M 42 | comp_size = length(find(Z == t)); 43 | data(Z == t,:) = mvnrnd(mu(t,:), Sigma(:,:,t), comp_size); 44 | end 45 | 46 | 47 | 48 | % ------------------------------------------ 49 | % K-means starts from here 50 | % ------------------------------------------ 51 | 52 | Max_Iter = 12; 53 | 54 | % ------------------------------------------ 55 | % draw before the first iteration 56 | % ------------------------------------------ 57 | 58 | plot(data(:,1), data(:,2),'.', 'color',[1 0 1]); 59 | hold on; 60 | ctrs = [1 3; 1 1; 0 -2]; 61 | plot(ctrs(:,1),ctrs(:,2),'kx','MarkerSize',12,'LineWidth',2) 62 | plot(ctrs(:,1),ctrs(:,2),'ko','MarkerSize',12,'LineWidth',2) 63 | hold off; 64 | waitforbuttonpress; 65 | 66 | 67 | colors = {'r.', 'g.', 'b.'}; 68 | 69 | for i = 1:Max_Iter 70 | 71 | % ---------------------------------------------------- 72 | % MATLAB has a kmeans(),but we implement ourselves 73 | % ---------------------------------------------------- 74 | 75 | % [idx,ctrs] = kmeans(data,3,'Distance','city','MaxIter',1,'start',ctrs); 76 | 77 | for j =1:size(ctrs,1) 78 | mean_val = repmat(ctrs(j,:),[size(data, 1) 1]); 79 | distance(j,:) = diag((data - mean_val)*(data - mean_val)'); 80 | end 81 | 82 | [temp idx] = min(distance); 83 | 84 | for j =1:size(ctrs,1) 85 | ctrs(j,:) = mean(data(idx==j,:)); 86 | plot(data(idx==j,1),data(idx==j,2),colors{j},'MarkerSize',12); 87 | hold on; 88 | end 89 | 90 | plot(ctrs(:,1),ctrs(:,2),'kx','MarkerSize',12,'LineWidth',2) 91 | plot(ctrs(:,1),ctrs(:,2),'ko','MarkerSize',12,'LineWidth',2) 92 | 93 | legend('type 1','type 2','type 3', 'Centroids', 'Location','NW') 94 | 95 | waitforbuttonpress; 96 | hold off; 97 | 98 | end 99 | end -------------------------------------------------------------------------------- /lda_gibbs_example.m: -------------------------------------------------------------------------------- 1 | % ---------------------------------- 2 | % A simple illustration of LDA 3 | % (1) generate W according to the LDA's generative model 4 | % (2) Perform Gibbs Sampling 5 | % (3) part of teaching for USTC computation statistics 6 | % 7 | % By Richard Xu 8 | % yida.xu@uts.edu.au 9 | % 10 | % 2017/04/10 11 | % 12 | % ---------------------------------- 13 | 14 | function lda_gibbs_example 15 | 16 | clc; 17 | clear; 18 | 19 | %check_dirrand 20 | 21 | % K is number of topics 22 | K = 5; 23 | 24 | % alpha = [2 ... 2] with K elements 25 | alphas = ones(1,K)*2; 26 | 27 | % V is the total number of words in a corpus. 28 | V = 20; 29 | 30 | % eta = [ 2 ... 2] with V elements 31 | etas = ones(1,V)*2; 32 | 33 | % D is the number of documents 34 | D = 6; 35 | 36 | % N is number of words in a document 37 | N = 10; 38 | 39 | % ------------------------------------------------ 40 | % Generate W, the words data 41 | % ------------------------------------------------ 42 | 43 | [betas, thetas, Z, W] = generate_LDA(K,V,D,N, alphas, etas); 44 | 45 | 46 | % ------------------------------------------------ 47 | % Now we have W, start Gibbs Sampling 48 | % ------------------------------------------------ 49 | 50 | [betas, thetas, Z, temp] = generate_LDA(K,V,D,N, alphas, etas); 51 | 52 | 53 | 54 | for g=1:100 55 | 56 | % sample theta 57 | 58 | for d = 1:D 59 | 60 | I = zeros(1,K); 61 | for i = 1:K 62 | I(i) = length(find(Z(d,:) == i)); 63 | end 64 | 65 | thetas(d,:) = dirrnd(alphas + I) ; 66 | 67 | end 68 | 69 | 70 | % sample beta 71 | I = zeros(1,V); 72 | for i = 1:V 73 | I(i) = length(find(W == i)); 74 | end 75 | 76 | for i = 1:K 77 | betas(i,:) = dirrnd(etas + I); 78 | end 79 | % sample Z 80 | 81 | for d =1:D 82 | 83 | for n =1:N 84 | 85 | probs = zeros(1,K); 86 | for k =1:K 87 | 88 | %Pr( Z_{d,n} = k) 89 | 90 | probs(k) = thetas(d,k) * betas(k,W(d,n)); 91 | end 92 | 93 | probs = probs / sum(probs); 94 | 95 | Z(d,n) = find(mnrnd(1,probs)); 96 | 97 | end 98 | 99 | end 100 | 101 | fprintf('at %d iterations\n',g); 102 | display(thetas); 103 | display(betas); 104 | display(Z); 105 | 106 | 107 | end 108 | 109 | end 110 | 111 | % ------------------------------------------------------------------ 112 | % Generate LDA according to the generative model 113 | % ------------------------------------------------------------------ 114 | function [betas, thetas, Z, W] = generate_LDA(K,V,D,N,alphas, etas) 115 | 116 | % beta is the word distribution per topic 117 | betas = zeros(K,V); 118 | for k = 1:K 119 | betas(k,:) = dirrnd(etas); 120 | end 121 | 122 | Z = zeros(D,N); 123 | W = zeros(D,N); 124 | thetas = zeros(D,K); 125 | 126 | for d = 1:D 127 | 128 | % for each document d, theta is the topic distribution for that document 129 | thetas(d,:) = dirrnd(alphas); 130 | 131 | temp = mnrnd(1,thetas(d,:),N); 132 | 133 | for n =1:N 134 | Z(d,n) = find(temp(n,:)); 135 | W(d,n) = find(mnrnd(1,betas(Z(d,n),:)),1); 136 | end 137 | 138 | end 139 | end 140 | 141 | % ------------------------------------------------------------------ 142 | % check to see if dirichlet distribution is generated properly 143 | % ------------------------------------------------------------------ 144 | function check_dirrand 145 | 146 | alphas = [1 2 3]; 147 | 148 | for i =1:10000 149 | g(i,:) = dirrnd(alphas); 150 | end 151 | 152 | % mean of Dirichlet distribution should be alpha/sum(alpha) 153 | display(mean(g)); 154 | 155 | % This is the emperical variance. 156 | display(var(g)); 157 | 158 | % This is the theoretical variance. 159 | emp_var = alphas .* ( repmat(sum(alphas),[1 3]) - alphas)/sum(alphas)^2/(sum(alphas) + 1); 160 | 161 | display(emp_var); 162 | 163 | end 164 | 165 | % ------------------------------------------------------------------ 166 | % Since MATLAB does not provide any method for sampling dirichlet 167 | % distribution, we need to sample it using Gamma distribution. 168 | % ------------------------------------------------------------------ 169 | 170 | 171 | function g = dirrnd(alphas) 172 | 173 | g = gamrnd(alphas,1); 174 | g = g/sum(g); 175 | 176 | end 177 | -------------------------------------------------------------------------------- /soft_max.m: -------------------------------------------------------------------------------- 1 | % -------------------------------------------- 2 | % Demo for softmax (or multinomial) regression 3 | % written by Richard Xu 4 | % yida.xu@uts.edu.au, 5 | % April, 2016 6 | % made it bare-minimum Octave-friendly 7 | % -------------------------------------------- 8 | 9 | function soft_max() 10 | 11 | clear all; 12 | clc; 13 | 14 | % -------------------------------------- 15 | % acts like a multinomial distribution 16 | % -------------------------------------- 17 | N = 1000; 18 | weights = [0.33 0.33 0.34]; 19 | M = length(weights); 20 | 21 | ind = rand([N 1]); 22 | Z = zeros([N 1]); 23 | acc_weights = cumsum(weights); 24 | 25 | for t = 1:M 26 | Z(ind > acc_weights(t)) = t; 27 | end 28 | Z = Z + 1; 29 | 30 | % ------------------------------------------ 31 | % Generate data from Gaussian Mixture model 32 | % ------------------------------------------ 33 | 34 | mu = [ -3 4; 0 0; 3 4]; 35 | Sigma(:,:,1) = [ 2 0; 0 1]; 36 | Sigma(:,:,2) = [ 2 0; 0 1]; 37 | Sigma(:,:,3) = [ 2 0; 0 3]; 38 | data = zeros([N 3]); 39 | 40 | for t = 1:M 41 | comp_size = length(find(Z == t)); 42 | data(Z == t,1:2) = mvnrnd(mu(t,:), Sigma(:,:,t), comp_size); 43 | % data(Z == t,1:2) = mvnrnd2( Sigma(:,:,t), mu(t,:),comp_size,2); 44 | data(Z == t,3) = t; 45 | end 46 | 47 | Max_Iter = 12; 48 | 49 | [x,y] = meshgrid( min(data(:,1)):0.3:max(data(:,1)), ... 50 | min(data(:,2)):0.3:max(data(:,2))); 51 | 52 | 53 | data_mesh = [reshape(x,[],1),reshape(y,[],1)]; 54 | data_mesh = [data_mesh ones(size(data_mesh,1),1) ]; 55 | 56 | 57 | colors = [ 1 0 1; 0 0 1; 0 1 0]; 58 | 59 | 60 | fig1 = figure('KeyPressFcn',@keyPress, 'WindowButtonDownFcn', @mouseClicked_fig1); 61 | clf(fig1); 62 | 63 | figure(fig1); 64 | 65 | 66 | for i =1:3 67 | 68 | subset = data(data(:,3) == i,:); 69 | hh(i) = plot(subset(:,1), subset(:,2),'o', 'MarkerFaceColor', colors(i,:),'MarkerEdgeColor', colors(i,:)); 70 | 71 | hold on; 72 | end 73 | 74 | legend(hh,{'class 1: [1 0 0]' 'class 2: [0 1 0]' 'class 3: [0 0 1]'}); 75 | 76 | 77 | 78 | h_sel = []; 79 | 80 | function mouseClicked_fig1(objectHandle , eventData ) 81 | 82 | if ~isempty(h_sel) 83 | delete(h_sel); 84 | end 85 | 86 | coord = get(gca,'CurrentPoint'); 87 | coord = coord(1,1:2); 88 | 89 | distances = sqrt(sum(bsxfun(@minus, data(:,1:2), coord).^2,2)); 90 | 91 | [m sel] = min(distances); 92 | 93 | h_sel = plot(data(sel,1), data(sel,2),'o', 'MarkerEdgeColor', [1 0 0], 'LineWidth',5); 94 | 95 | str = '[1 0 0]'; 96 | 97 | if data(sel,3) == 1 98 | str = '[1 0 0]'; 99 | elseif data(sel,3) == 2 100 | str = '[0 1 0]'; 101 | else 102 | str = '[0 0 1]'; 103 | end 104 | 105 | str = ['cost(' str ' , ']; 106 | 107 | a1 = exp([data(sel,1:2) 1]*thetas'); 108 | probs1 = a1 ./ repmat(sum(a1,2),[1 3]); 109 | 110 | str = [str [ '[' num2str(probs1(1),'%.3f') ' ' num2str(probs1(2),'%.3f') ' ' num2str(probs1(3),'%.3f')]]; 111 | str = [str '] )']; 112 | h_sel = [h_sel; text(min(data(:,1)), max(data(:,2)),str)]; 113 | 114 | end 115 | 116 | 117 | thetas = [7 1 -1; 8 2 2; 6 3 3]; 118 | 119 | 120 | alpha = 0.001; 121 | 122 | 123 | fig2 = figure(); 124 | clf(fig2); 125 | 126 | 127 | for n = 1:30 128 | 129 | % figure(2); 130 | % data_mesh = [reshape(x,[],1),reshape(y,[],1)]; 131 | % data_mesh = [data_mesh ones(size(data_mesh,1),1) ]; 132 | 133 | a = exp(data_mesh*thetas'); 134 | probs = a ./ repmat(sum(a,2),[1 3]); 135 | [val ind] = sort(probs','descend'); 136 | 137 | for i =1:3 138 | 139 | subset = data_mesh(ind(1,:)== i,:); 140 | plot(subset(:,1), subset(:,2),'o', 'color', colors(i,:)*0.9); 141 | hold on; 142 | subset = data(data(:,3) == i,:); 143 | hh(i) = plot(subset(:,1), subset(:,2),'o', 'MarkerFaceColor', colors(i,:),'MarkerEdgeColor', colors(i,:)); 144 | hold on; 145 | end 146 | 147 | legend(hh,{'class 1: [1 0 0]' 'class 2: [0 1 0]' 'class 3: [0 0 1]'}); 148 | 149 | 150 | % for i = 1:3 151 | % thetas_mesh = reshape(thetas)'; 152 | % a = exp([x,y]*thetas'); 153 | % 154 | % end 155 | % 156 | % h =1; 157 | 158 | % data_t = [data(:,1:2) ones(size(data,1),1)]; 159 | % a = exp(data_t*thetas'); 160 | % for i =1:3 161 | % prob = a(:,i)./sum(a,2); 162 | % [val ind] = sort(prob,'descend'); 163 | % ind = ind(1:length(find(data(:,3)==i))); 164 | % subset = data(ind,:); 165 | % gg(i) = plot(subset(:,1), subset(:,2),'o', 'color', colors(i,:)); 166 | % hold on; 167 | % end 168 | 169 | % legend(gg,{'class 1: [1 0 0]' 'class 2: [0 1 0]' 'class 3: [0 0 1]'}); 170 | 171 | % Cp = get(gca,'CurrentPoint'); 172 | 173 | waitforbuttonpress; 174 | 175 | 176 | for i =1:3 177 | 178 | r_data = data(data(:,3) == i,:); 179 | r_data(:,3) = 1; 180 | a = exp(r_data*thetas'); 181 | prob = a(:,i)./sum(a,2); 182 | thetas(i,:) = thetas(i,:) - alpha * ( sum(r_data .* repmat((prob -1),[1,3])) + thetas(i,:)); 183 | 184 | end 185 | 186 | 187 | end 188 | end 189 | 190 | -------------------------------------------------------------------------------- /test_autocorrelation.m: -------------------------------------------------------------------------------- 1 | % --------------------------- 2 | % Simulation of a Autocorrelation 3 | % written by Richard Xu 4 | % March 2013 5 | % yida.xu@uts.edu.au 6 | % -------------------------- 7 | % clear; 8 | % m = 4; k=3; n = 5; A = randn(m,k); B = randn(k,n); % values 9 | % a1 = A * B; % method 1 10 | % display(a1); 11 | % 12 | % a2 = zeros(m,n); 13 | % for i=1:k; % method 2 14 | % a2 = a2+ A(:,i) * B(i,:); 15 | % end; 16 | % display(a2); 17 | % 18 | % return; 19 | 20 | 21 | clc; 22 | clf; 23 | clear; 24 | 25 | 26 | % mu = [2 3]; 27 | % Sigma = [ 3 2; 2 5]; 28 | 29 | D = 5; 30 | 31 | mu_full = randn(1, D) * 2; 32 | Sigma_full = wishrnd(eye(D),5)/2; 33 | 34 | display(mu_full); 35 | display(Sigma_full); 36 | 37 | 38 | % Sigma = eye(D); 39 | N = 2000; 40 | 41 | X_full = mvnrnd(mu_full,Sigma_full,N); 42 | 43 | figure(1); 44 | 45 | for i = 2:D 46 | 47 | X = X_full(:,1:i); 48 | mu = mu_full(1:i); 49 | Sigma = Sigma_full(1:i,1:i); 50 | 51 | 52 | % -------------------------------- 53 | % sample it directly 54 | % -------------------------------- 55 | 56 | subplot(D-1,2,(i-1)*2); 57 | autocorr(mvnpdf(X,mu,Sigma)) 58 | title(['direct sampling of ' num2str(i) 'D Gaussian']); 59 | 60 | 61 | % -------------------------------- 62 | % using Gibbs sampling 63 | % -------------------------------- 64 | 65 | 66 | X = zeros(N,i); 67 | X(1,:) = mu; 68 | 69 | for j = 2:N 70 | 71 | X(j,:) = X(j-1,:); 72 | 73 | for k = 1:i 74 | 75 | 76 | Sigma_12 = Sigma(k,:); 77 | Sigma_12(k) = []; 78 | 79 | Sigma_21 = Sigma(:,k); 80 | Sigma_21(k) = []; 81 | 82 | Sigma_22 = Sigma; 83 | Sigma_22(k,:) = []; 84 | Sigma_22(:,k) = []; 85 | 86 | mu_2 = mu; 87 | mu_2(k) = []; 88 | 89 | 90 | X2 = X(j,:); 91 | X2(k) = []; 92 | 93 | mu_k = mu(k) + Sigma_12*inv(Sigma_22)*( X2 - mu_2)'; 94 | Sigma_k = Sigma(k,k) - Sigma_12*inv(Sigma_22)*Sigma_21; 95 | 96 | X(j,k) = randn * sqrt(Sigma_k) + mu_k; 97 | 98 | end 99 | 100 | end 101 | 102 | subplot(D-1,2,(i-1)*2-1); 103 | autocorr(mvnpdf(X,mu,Sigma)) 104 | title(['gibbs sampling of ' num2str(i) 'D Gaussian']); 105 | 106 | 107 | display(mean(X)); 108 | display(cov(X)); 109 | 110 | 111 | end 112 | 113 | 114 | return; 115 | 116 | 117 | figure(1); 118 | 119 | N = 2000; 120 | 121 | init_sample = [ 1 1]; 122 | 123 | directX = mvnrnd(mu,Sigma,N); 124 | 125 | figure(1); 126 | subplot(2,2,1); 127 | plot(directX(:,1),directX(:,2),'.','color',[1 0 0]); 128 | 129 | subplot(2,2,2); 130 | autocorr(mvnpdf(directX,mu,Sigma)); 131 | 132 | SHOW_STEP = true; 133 | 134 | if SHOW_STEP == true 135 | 136 | X = zeros(N,2); 137 | X(1,:) = init_sample; 138 | 139 | for i = 2:N 140 | 141 | mu_1 = mu(1) + Sigma(1,2)/Sigma(2,2)*( X(i-1,2) - mu(2)); 142 | Sigma_1 = Sigma(1,1) - Sigma(1,2)/Sigma(2,2)*Sigma(2,1); 143 | 144 | X(i,1) = randn * sqrt(Sigma_1) + mu_1; 145 | 146 | mu_2 = mu(2) + Sigma(2,1)/Sigma(1,1)*(X(i,1) - mu(1)); 147 | Sigma_2 = Sigma(2,2) - Sigma(2,1)/Sigma(1,1)*Sigma(1,2); 148 | 149 | X(i,2) = randn * sqrt(Sigma_2) + mu_2; 150 | 151 | 152 | end 153 | 154 | subplot(2,2,3); 155 | plot(X(:,1),X(:,2),'.'); 156 | 157 | %--------------------------------------- 158 | 159 | else 160 | 161 | 162 | subplot(2,2,3); 163 | 164 | X = zeros(N,2); 165 | X(1,:) = [1 1]; 166 | 167 | for i = 2:N 168 | 169 | mu_1 = mu(1) + Sigma(1,2)/Sigma(2,2)*( X(i-1,2) - mu(2)); 170 | Sigma_1 = Sigma(1,1) - Sigma(1,2)/Sigma(2,2)*Sigma(2,1); 171 | 172 | X(i,1) = randn * sqrt(Sigma_1) + mu_1; 173 | 174 | plot( [X(i-1,1) X(i,1)], [X(i-1,2) X(i-1,2)],'LineWidth',1); 175 | axis([-6 8 -6 12]); 176 | hold on; 177 | 178 | waitforbuttonpress; 179 | %waitfor(100); 180 | mu_2 = mu(2) + Sigma(2,1)/Sigma(1,1)*(X(i,1) - mu(1)); 181 | Sigma_2 = Sigma(2,2) - Sigma(2,1)/Sigma(1,1)*Sigma(1,2); 182 | 183 | X(i,2) = randn * sqrt(Sigma_2) + mu_2; 184 | plot( [X(i,1) X(i,1)], [X(i-1,2) X(i,2)],'LineWidth',1); 185 | axis([-6 8 -6 12]); 186 | hold on; 187 | 188 | waitforbuttonpress; 189 | %waitfor(100); 190 | end 191 | 192 | 193 | plot(X(:,1),X(:,2),'.'); 194 | 195 | end 196 | 197 | subplot(2,2,4); 198 | autocorr(mvnpdf(X,mu,Sigma)); 199 | 200 | -------------------------------------------------------------------------------- /vb_normal_gamma.m: -------------------------------------------------------------------------------- 1 | %-------------------------------------------- 2 | % Demo for Variational Bayes: 3 | % use VB to approximate Normal-Gamma 4 | % written by Richard Xu 5 | % July,2014 6 | %-------------------------------------------- 7 | 8 | clc; 9 | clf; 10 | clear; 11 | 12 | % --------------------------------------------- 13 | % ground-truth parameters 14 | % --------------------------------------------- 15 | 16 | mu_0 = 0; 17 | lambda_0 = 1; 18 | a_0 = 1.5; 19 | b_0 = 1; 20 | 21 | 22 | 23 | 24 | 25 | T = 10; 26 | n = 0; 27 | k = a_0; 28 | theta = 1/b_0; 29 | tau = gamrnd(k, theta,[n 1]); 30 | 31 | 32 | %Generate from a normal distribution with mu and standard deviation sigma: 33 | %r = mu + sigma.*randn; 34 | 35 | if n >0 36 | X = mu_0 + (tau * lambda_0).^(-0.5) .* randn(n,1); 37 | else 38 | X = mu_0; 39 | end 40 | 41 | mu_n = (lambda_0 * mu_0 + n * mean(X) )/ (lambda_0 + n); 42 | lambda_n = lambda_0 + n; 43 | 44 | a_n = a_0 + n/2; 45 | b_n = b_0 + 1/2 * sum((X - mean(X)).^2) + (lambda_0*n * (mean(X) - mu_0)^2)/(2*(lambda_0 + n)); 46 | 47 | 48 | S_t = 100; 49 | S_m = 100; 50 | 51 | tau_pdf = zeros(S_t,S_m); 52 | mu_pdf = zeros(S_t,S_m); 53 | [mu_axis, tau_axis] = meshgrid(linspace(-5,5,S_m), linspace(0, 3 ,S_t)); 54 | 55 | %posterior for Normal-Gamma 56 | %tau_n = gampdf(tau_axis, a_n,b_n); 57 | 58 | tau_pdf = tau_axis.^(a_n -1).*exp(-b_n * tau_axis); 59 | 60 | for i = 1:S_m 61 | mu_pdf (i,:) = normpdf(mu_axis(i,:),mu_n,(tau_pdf(i,1) * lambda_n).^(-0.5)); 62 | end 63 | 64 | Z = mu_pdf .* tau_pdf ; 65 | 66 | 67 | f1 = figure(1); 68 | set(f1,'Name','variational Inferece to approximate Normal-Gamma distribution'); 69 | 70 | % ----------------------------------------------------- 71 | % plot ground truth distribution 72 | % ------------------------------------------------------ 73 | 74 | subplot(1,2,1) 75 | contour(tau_axis, mu_axis, Z); 76 | title('ground truth distribution') 77 | xlabel('\tau') 78 | ylabel('\mu') 79 | 80 | % ----------------------------- 81 | % Variational Approximation 82 | % ----------------------------- 83 | 84 | init_mu_0 = -4; 85 | init_lambda_0 = 4; 86 | init_a_0 = 1; 87 | init_b_0 = 3; 88 | 89 | % ------------------------- 90 | 91 | mu_prev = init_mu_0; 92 | lambda_prev = init_lambda_0; 93 | a_prev = init_a_0; 94 | b_prev = init_b_0; 95 | 96 | 97 | for t =1:T 98 | 99 | if t ==1 100 | % the initial distribution for mu 101 | mu_current = mu_prev; 102 | lambda_current = lambda_prev; 103 | a_current = a_prev; 104 | b_current = b_prev; 105 | 106 | else 107 | 108 | %-------------------------------------- 109 | % update q_tau (a_current, b_current) 110 | %-------------------------------------- 111 | 112 | a_current = a_0 + n/2; 113 | 114 | % E[mu^2] = var(mu) + (E[mu])^2 115 | E_mu_square = inv(lambda_0) + mu_prev^2; 116 | % E[mu] 117 | E_mu = mu_prev; 118 | 119 | %sum [(x_i - mu)^2] 120 | first = sum( X.^2 - 2 * X .* repmat(E_mu,size(X)) + repmat(E_mu_square, size(X))); 121 | 122 | %lambda_0 (mu - mu_0)^2 123 | second = lambda_0 *(E_mu_square - 2*mu_0*E_mu + mu_0^2); 124 | 125 | b_current = b_0 + (first + second)/2; 126 | 127 | %---------------------------------------------- 128 | % update q_normal (mu_current, lambda_current) 129 | %---------------------------------------------- 130 | 131 | %E[tau] = a/b; 132 | 133 | E_tau = a_current/b_current; 134 | 135 | mu_current = (lambda_0 * mu_0 + n * mean(X))/(lambda_0 + n); 136 | lambda_current = (lambda_0 + n) * E_tau; 137 | 138 | % ------------------------------ 139 | 140 | mu_prev = mu_current; 141 | lambda_prev = lambda_current; 142 | a_prev = a_current; 143 | b_prev = b_current; 144 | 145 | end 146 | 147 | 148 | tau_pdf = tau_axis.^(a_current -1).*exp(-b_current * tau_axis); 149 | 150 | mu_pdf = normpdf(mu_axis,mu_current, lambda_current^(-0.5)); 151 | 152 | Z = mu_pdf .* tau_pdf ; 153 | 154 | subplot(1,2,2) 155 | contour(tau_axis, mu_axis, Z); 156 | title('Press to next iteration') 157 | xlabel('\tau') 158 | ylabel('\mu') 159 | 160 | waitforbuttonpress; 161 | 162 | end 163 | 164 | --------------------------------------------------------------------------------