├── DPEM_FA.py ├── README.md ├── compute_per_iter_budget.m ├── data ├── Gowalla.mat └── lifesci.csv ├── laprnd.m ├── pmkt3 ├── NICV.m ├── SimplexProj.m ├── compute_NICV.m ├── compute_ll_estep.m ├── condGaussCpdCreate.m ├── convergenceTest.m ├── emAlgo.m ├── gaussInvWishartLogprob.m ├── gaussLogprob.m ├── gaussPlot2d.m ├── gaussSample.m ├── invWishartLogprob.m ├── isposdef.m ├── kmeansFit.m ├── logdet.m ├── logsumexp.m ├── minidx.m ├── mixGaussCreate.m ├── mixGaussFit.m ├── mixGaussInferLatent.m ├── mkStochastic.m ├── mkUnitVariance.m ├── mvtGammaln.m ├── normalize.m ├── normalizeLogspace.m ├── partitionedMean.m ├── partitionedSum.m ├── pdinv.m ├── plotKmeans.m ├── plotfn.m ├── prepareArgs.m ├── printPmtkFigure.m ├── process_options.m ├── rowvec.m ├── setMixPrior.m ├── shadedErrorBar.m ├── sqDistance.m ├── standardizeCols.m └── structure.m ├── startup.m ├── testGowalla.m └── testLifesci.m /DPEM_FA.py: -------------------------------------------------------------------------------- 1 | """ simple FA code using EM """ 2 | 3 | import numpy as np 4 | from sklearn.datasets import fetch_olivetti_faces 5 | import matplotlib.pyplot as plt 6 | import numpy as np 7 | from numpy.random import RandomState 8 | import logging 9 | from time import time 10 | from sklearn import decomposition 11 | 12 | 13 | # ============================ 14 | # DPEM code for Factor Analysis 15 | # ============================ 16 | # 17 | # Mijung wrote this part, while below "Faces dataset decompositions" part is written by Vlad Niculae, Alexandre Gramfort as noted. 18 | 19 | def EM_FA(XX, n_compo, W_init, Psi_init_inv): 20 | # XX is data second moment matrix 21 | # n_compo is the number of components 22 | 23 | # (1) M-step given 24 | n_features, n_features = XX.shape 25 | G = np.linalg.inv(np.dot(np.dot(W_init.transpose(), Psi_init_inv), W_init) + np.eye(n_compo)) 26 | 27 | Psi_inv_W_G_trp = np.dot(np.dot(Psi_init_inv, W_init), G.transpose()) 28 | W_first_term = np.dot(XX, Psi_inv_W_G_trp) 29 | W_second_term = np.linalg.inv(G + np.dot(np.dot(Psi_inv_W_G_trp.transpose(), XX), Psi_inv_W_G_trp)) 30 | W = np.dot(W_first_term, W_second_term) 31 | 32 | Psi = np.diag(XX - np.dot(np.dot(W_init, Psi_inv_W_G_trp.transpose()),XX)) 33 | 34 | return W, Psi 35 | 36 | 37 | # ============================ 38 | # Faces dataset decompositions 39 | # ============================ 40 | # 41 | # This example applies to :ref:`olivetti_faces` different unsupervised 42 | # matrix decomposition (dimension reduction) methods from the module 43 | # :py:mod:`sklearn.decomposition` (see the documentation chapter 44 | # :ref:`decompositions`) . 45 | # 46 | # """ 47 | # print(__doc__) 48 | # 49 | # # Authors: Vlad Niculae, Alexandre Gramfort 50 | # # License: BSD 3 clause 51 | # 52 | # 53 | # 54 | # from sklearn.cluster import MiniBatchKMeans 55 | # 56 | ############################################################################### 57 | def plot_gallery(title, images, n_col=10, n_row=1): 58 | plt.figure(figsize=(2. * n_col, 2.26 * n_row)) 59 | plt.suptitle(title, size=16) 60 | for i, comp in enumerate(images): 61 | plt.subplot(n_row, n_col, i + 1) 62 | vmax = max(comp.max(), -comp.min()) 63 | plt.imshow(comp.reshape(image_shape), cmap=plt.cm.gray, 64 | interpolation='nearest', 65 | vmin=-vmax, vmax=vmax) 66 | plt.xticks(()) 67 | plt.yticks(()) 68 | plt.subplots_adjust(0.01, 0.05, 0.99, 0.93, 0.04, 0.) 69 | 70 | # 71 | # Display progress logs on stdout 72 | logging.basicConfig(level=logging.INFO, 73 | format='%(asctime)s %(levelname)s %(message)s') 74 | n_components = 10 75 | image_shape = (64, 64) 76 | rng = RandomState(0) 77 | # 78 | # ############################################################################### 79 | # Load faces data 80 | dataset = fetch_olivetti_faces(shuffle=True, random_state=rng) 81 | faces = dataset.data 82 | 83 | n_samples, n_features = faces.shape 84 | 85 | # global centering 86 | faces_centered = faces - faces.mean(axis=0) 87 | 88 | # local centering 89 | faces_centered -= faces_centered.mean(axis=1).reshape(n_samples, -1) 90 | 91 | print("Dataset consists of %d faces" % n_samples) 92 | 93 | 94 | print("Preprocess the face data such that max norm of each image is less than 1") 95 | X = faces_centered 96 | max_norm = max(np.linalg.norm(X, axis=0)) 97 | X = X/max_norm 98 | 99 | XX = np.dot(X.T, X)/float(n_samples) 100 | 101 | max_iter = 20 102 | 103 | epsilon = 1 104 | delta = 0.0001 105 | 106 | print("Private FA is being processed for epsilon= %f" % epsilon) 107 | 108 | # noise addition 109 | sensitivity = 2/float(n_samples) 110 | c2 = 2*np.log(1.25/delta) 111 | nsv = c2*(sensitivity**2)/(epsilon**2) 112 | # how_many = n_features*(n_features+1)*0.5 113 | 114 | nse_mat = np.random.normal(0,nsv,[n_features,n_features]) 115 | upper_nse_mat = np.triu(nse_mat, 0) 116 | 117 | print("noise generation for perturbing XX") 118 | 119 | for i in range(n_features): 120 | for j in range(i, n_features): 121 | upper_nse_mat[j][i] = upper_nse_mat[i][j] 122 | 123 | nse = upper_nse_mat 124 | 125 | XX_tile = XX + nse 126 | 127 | print("once we add noise to XX, we do svd to make sure the resulting matrix is still positive definite") 128 | print("this will take a while, because the image size is about 4000") 129 | # to ensure the matrix is positive definite 130 | w, v = np.linalg.eig(XX_tile) 131 | # remember: XX_tile = np.dot(v, np.dot(np.diag(w), v.transpose())) 132 | neg_idx = np.nonzero(w<0) 133 | w[neg_idx] = 0.0001 134 | 135 | XX_perturbed = np.dot(v, np.dot(np.diag(w), v.transpose())) 136 | 137 | print("now perturbed XX is positive definite!") 138 | 139 | # W_init = np.random.normal(0, 1, [n_features, n_components]) 140 | W_init = v[:, 0:n_components] 141 | Psi_init_inv = np.diag(0.1*np.ones(n_features)) 142 | 143 | for i in range(0, max_iter): 144 | print("%d th iteration of EM" %i) 145 | W_new, Psi_new = EM_FA(XX_perturbed, n_components, W_init, Psi_init_inv) 146 | W_init = W_new 147 | Psi_init_inv = np.diag(1/Psi_new) 148 | 149 | 150 | plot_gallery('privateFA with %f' % (epsilon), W_new.transpose()[:n_components]) 151 | plt.savefig('privFA_epsilon=%f.pdf' % epsilon) 152 | plt.show() 153 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # dpem_code 2 | 3 | Hi, thanks for your interests in our DPEM paper! 4 | Our paper was published at AISTATS 2017, which you can find http://proceedings.mlr.press/v54/park17c.html 5 | 6 | This code package consists of several folders 7 | 8 | (1) data: which contains Gowalla / lifescience datasets, which are from UCI data repository 9 | 10 | (2) pmkt3: contains code copied from pmtk3.googlecode.com, which I made some modifications for adding noise to moments 11 | 12 | (3) matfiles: once you run my code, the results will be saved here. 13 | 14 | 15 | I wrote code for test lifescience data and Gowalla data in Matlab, while my code for DP factor analysis is written in Python. 16 | 17 | So, if you want to run "testLifesci.m" or "testGowalla.m", open Matlab. The first thing you want to do is, "startup.m", which generates paths 18 | for running necessary sub-functions. Then run "testLifesci.m" or "testGowalla.m". 19 | 20 | If you want to run "DPEM_FA.py", open Python, then run "DPEM_FA.py". 21 | 22 | I haven't added my code for testing stroke data, because this dataset isn't publically available. Please contact IMEDS (http://imeds.reaganudall.org/ResearchLab) if you want to get access their data. 23 | 24 | If you have any questions on my implementation, find any bugs, errors, ways to improve, please contact me @ mijungi.p@gmail.com 25 | 26 | Have a wonderful day! 27 | 28 | May 19, 2017 @ Amsterdam, Netherlands. 29 | -------------------------------------------------------------------------------- /compute_per_iter_budget.m: -------------------------------------------------------------------------------- 1 | % test compositions with combinations of mechanisms 2 | 3 | function model = compute_per_iter_budget(epsilon) 4 | 5 | J = epsilon.J; 6 | K = epsilon.K; 7 | D = epsilon.D; 8 | delta_i = 1e-8; 9 | 10 | if epsilon.comp == 0 11 | % non-private 12 | model.eps_prime = 0; 13 | 14 | elseif epsilon.comp == 1 15 | % linear composition 16 | 17 | if epsilon.lap == 1 18 | model.eps_prime = epsilon.total_eps/(J*(2*K+1)); 19 | else % Gaussian 20 | delta_i = epsilon.total_del/(J*(K+1)); 21 | c2 = 2*log(1.25/delta_i); 22 | model.c2 = c2; 23 | model.eps_prime = epsilon.total_eps/(J*(2*K+1)); 24 | end 25 | 26 | elseif epsilon.comp == 2 27 | % advanced composition 28 | 29 | if epsilon.lap == 1 30 | JK = J*(2*K+1); 31 | myfun = @(x) (JK*x*(exp(x)-1) + sqrt(2*JK*log(1/epsilon.total_del))*x - epsilon.total_eps)^2; 32 | model.eps_prime = fsolve(myfun, 0.1); 33 | else % gaussian 34 | c2 = 2*log(1.25/delta_i); 35 | model.c2 = c2; 36 | delta_prime = epsilon.total_del - J*(K+1)*delta_i; 37 | JK = J*(2*K+1); 38 | myfun = @(x) (JK*x*(exp(x)-1) + sqrt(2*JK*log(1/delta_prime))*x - epsilon.total_eps)^2; 39 | model.eps_prime = fsolve(myfun, 0.1); 40 | end 41 | 42 | elseif epsilon.comp == 3 43 | % zCDP composition 44 | if epsilon.lap == 1 45 | JK = J*(2*K+1); 46 | myfun = @(x) (JK*(x^2)/2 + 2*sqrt(JK*(x^2)/2*log(1/epsilon.total_del)) - epsilon.total_eps)^2; 47 | A = []; 48 | b = []; 49 | Aeq = []; 50 | beq = []; 51 | lb = 0; 52 | ub = 1; 53 | model.eps_prime = fmincon(myfun, 0.1, A, b, Aeq, beq, lb, ub); 54 | else 55 | c2 = 2*log(1.25/delta_i); 56 | model.c2 = c2; 57 | myfun = @(x) (J*(x^2)/(2*c2) + J*K*(x^2)/(2*c2) + J*K*(x^2)/2 + 2*sqrt((J*(x^2)/(2*c2) + J*K*(x^2)/(2*c2) + J*K*(x^2)/2)*log(1/epsilon.total_del)) - epsilon.total_eps)^2; 58 | A = []; 59 | b = []; 60 | Aeq = []; 61 | beq = []; 62 | lb = 0; 63 | ub = 1; 64 | model.eps_prime = fmincon(myfun, 0.1, A, b, Aeq, beq, lb, ub); 65 | end 66 | 67 | else % MA composition 68 | if epsilon.lap == 1 69 | 70 | maxlam = 100; 71 | % howmanyeps = 100; 72 | % eps_i_mat = linspace(1e-6, 1-1e-6, howmanyeps); 73 | % del_val = zeros(maxlam, howmanyeps); 74 | eps_i_mat = zeros(maxlam,1); 75 | for lam=1:maxlam 76 | % for ep = 1:howmanyeps 77 | trm1 = (1+lam)/(2*lam+1); 78 | trm2 = lam/(2*lam+1); 79 | % myfun = @(x) J*(K+1)*log(trm1*exp(lam*x) + trm2*exp(-x*(lam+1))) + J*K*lam*x - lam*epsilon.total_eps; 80 | % del_val(lam, ep) = myfun(ep); 81 | % end 82 | % myfun = @(x) (-log(epsilon.total_del) + J*(K+1)*log(trm1*exp(lam*x) + trm2*exp(-x*(lam+1))) + J*K*lam*x - lam*epsilon.total_eps)^2; 83 | % myfun = @(x) (-log(epsilon.total_del) + J*K*log(trm1*exp(lam*(x/K)) + trm2*exp(-(x/K)*(lam+1))) ... 84 | % + J*K*D*log(trm1*exp(lam*(x/D)) + trm2*exp(-(x/D)*(lam+1))) ... 85 | % + J*K*lam*x - lam*epsilon.total_eps)^2; 86 | myfun = @(x) (-log(epsilon.total_del) + J*K*log(trm1*exp(lam*x) + trm2*exp(-x*(lam+1))) ... 87 | + J*K*D*log(trm1*exp(lam*x) + trm2*exp(-x*(lam+1))) ... 88 | + J*K*lam*D^2*x - lam*epsilon.total_eps)^2; 89 | 90 | A = []; 91 | b = []; 92 | Aeq = []; 93 | beq = []; 94 | lb = 0; 95 | ub = 1; 96 | eps_i_mat(lam) = fmincon(myfun, 0.1, A, b, Aeq, beq, lb, ub); 97 | end 98 | eps_i_mat_sorted = sort(eps_i_mat, 'descend'); 99 | model.eps_prime = eps_i_mat_sorted(1)*D^2; 100 | 101 | else 102 | c2 = 2*log(1.25/delta_i); 103 | model.c2 = c2; 104 | % c2_pi = 2*log(1.25/(delta_i/K)); 105 | % model.c2_pi = c2_pi; 106 | % c2_mu = 2*log(1.25/(delta_i/D)); 107 | % model.c2_mu = c2_mu; 108 | 109 | maxlam = 100; 110 | eps_i_mat = zeros(maxlam,1); 111 | for lam=1:maxlam 112 | trm1 = lam^2 + lam; 113 | % myfun = @(x) (-log(epsilon.total_del) + J*(K+1)*trm1*(x^2)/(2*c2) + J*K*lam*x - lam*epsilon.total_eps)^2; 114 | myfun = @(x) (-log(epsilon.total_del) ... 115 | + J*K*trm1*(x^2)/(2*c2) ... 116 | + J*K*D*trm1*(x^2)/(2*c2) ... 117 | + J*K*lam*D^2*x- lam*epsilon.total_eps)^2; 118 | A = []; 119 | b = []; 120 | Aeq = []; 121 | beq = []; 122 | lb = 0; 123 | ub = 1; 124 | eps_i_mat(lam) = fmincon(myfun, 0.1, A, b, Aeq, beq, lb, ub); 125 | end 126 | eps_i_mat_sorted = sort(eps_i_mat, 'descend'); 127 | model.eps_prime = eps_i_mat_sorted(1)*D^2; 128 | 129 | 130 | end 131 | 132 | end -------------------------------------------------------------------------------- /data/Gowalla.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mijungi/dpem_code/13c705c93a79d86cd290185e690916bcc7387385/data/Gowalla.mat -------------------------------------------------------------------------------- /laprnd.m: -------------------------------------------------------------------------------- 1 | function y = laprnd(m, n, mu, sigma) 2 | %LAPRND generate i.i.d. laplacian random number drawn from laplacian distribution 3 | % with mean mu and standard deviation sigma. 4 | % mu : mean 5 | % sigma : standard deviation 6 | % [m, n] : the dimension of y. 7 | % Default mu = 0, sigma = 1. 8 | % For more information, refer to 9 | % http://en.wikipedia.org./wiki/Laplace_distribution 10 | 11 | % Author : Elvis Chen (bee33@sjtu.edu.cn) 12 | % Date : 01/19/07 13 | 14 | %Check inputs 15 | if nargin < 2 16 | error('At least two inputs are required'); 17 | end 18 | 19 | if nargin == 2 20 | mu = 0; sigma = 1; 21 | end 22 | 23 | if nargin == 3 24 | sigma = 1; 25 | end 26 | 27 | % Generate Laplacian noise 28 | u = rand(m, n)-0.5; 29 | b = sigma / sqrt(2); 30 | y = mu - b * sign(u).* log(1- 2* abs(u)); 31 | -------------------------------------------------------------------------------- /pmkt3/NICV.m: -------------------------------------------------------------------------------- 1 | function [err, assign] = NICV(X, mu) 2 | 3 | dist = sqDistance(X, mu'); 4 | N = size(X, 1); 5 | assign = minidx(dist, [], 2); 6 | K = size(mu, 2); 7 | err = sum(sum(dist'.*bsxfun(@eq, (1:K).', assign.')))/N; -------------------------------------------------------------------------------- /pmkt3/SimplexProj.m: -------------------------------------------------------------------------------- 1 | 2 | function X = SimplexProj(Y) 3 | [N,D] = size(Y); 4 | X = sort(Y,2,'descend'); 5 | Xtmp = (cumsum(X,2)-1)*diag(sparse(1./(1:D))); 6 | X = max(bsxfun(@minus,Y,Xtmp(sub2ind([N,D],(1:N)',sum(X>Xtmp,2)))),0); 7 | end -------------------------------------------------------------------------------- /pmkt3/compute_NICV.m: -------------------------------------------------------------------------------- 1 | 2 | clear; 3 | clc; 4 | 5 | 6 | seednum = 100; 7 | rng(seednum, 'twister'); 8 | 9 | % load data 10 | load('data/Gowalla') 11 | X = Gowalla; 12 | normaliser = 1./sqrt(max((sum(X.^2,2)))); 13 | X = bsxfun(@times, X, normaliser); 14 | maxIter = 40; 15 | 16 | % for EM 17 | % filename = ['matfiles/gowalla_G=lap_' num2str(0) '_epsilon=' num2str(0) '_delta=' num2str(0) '_comp=' num2str(0) '.mat']; 18 | % load(filename); 19 | % 20 | % [err_EM, assign] = NICV(X, model.cpd.mu); 21 | % 22 | % 23 | N = size(X,1); 24 | rpidx = randperm(N); 25 | rpidx = rpidx(1:7000); 26 | % figure(1); 27 | % plotKmeans(X(rpidx,:), model.cpd.mu, assign(rpidx), err_EM, maxIter); 28 | 29 | 30 | K = 5; 31 | D = 2; 32 | 33 | epsilon.Lap = 1; 34 | epsilon.val = 0; 35 | epsilon.method = 0; 36 | epsilon.maxiter = maxIter; 37 | % epsilon.delta_i = 1e-6; 38 | epsilon.delta = 1e-4; 39 | 40 | mu_init = bsxfun(@plus, lhsdesign(K,2)', [-0.5; -0.5]) + 0.05*randn(D,K); 41 | mu_init = mu_init/normaliser; 42 | [mu, assign, errHist] = kmeansFit(epsilon, X, K, 'plotfn', [], ... 43 | 'maxIter', maxIter, 'mu', mu_init); 44 | 45 | figure(1); 46 | plotKmeans(X(rpidx,:), mu, assign(rpidx), errHist(end), maxIter); 47 | 48 | %% 49 | 50 | % ep_val = 0.01; 51 | % epsilon.val = ep_val; 52 | % % % for DPEM with zCDP 53 | % 54 | % % the full EM % 55 | % initParams.mu = bsxfun(@plus, lhsdesign(K,D)', -0.5*ones(D, 1)) + 0.01*randn(D,K); 56 | % initParams.mu = bsxfun(@times, initParams.mu, normaliser); 57 | % initParams.Sigma = repmat(eye(D),[1 1 K]); 58 | % initParams.mixWeight = normalize(ones(1,K)); 59 | % 60 | % % conventional EM 61 | % epsilon.total_eps = ep_val; 62 | % epsilon.total_del = 1e-4; 63 | % epsilon.lap = 1; 64 | % epsilon.comp = 3; 65 | % 66 | % [model, loglikHist] = mixGaussFit(epsilon, X, K, 'initParams', initParams, ... 67 | % 'maxIter', maxIter, 'plotfn', [], 'verbose', false); 68 | 69 | %% 70 | 71 | % [err_EM, assign] = NICV(X, model.cpd.mu); 72 | % 73 | % figure(2); 74 | % plotKmeans(X(rpidx,:), model.cpd.mu, assign(rpidx), err_EM, maxIter); 75 | 76 | ep_val = 0.01; 77 | epsilon.Lap = 1; 78 | epsilon.val = ep_val; 79 | epsilon.method = 0; 80 | epsilon.maxiter = maxIter; 81 | % epsilon.delta_i = 1e-6; 82 | epsilon.delta = 1e-4; 83 | 84 | % mu_init = bsxfun(@plus, lhsdesign(K,2)', [-0.5; -0.5]) + 0.01*randn(D,K); 85 | [mu, assign, errHist] = kmeansFit(epsilon, X, K, 'plotfn', [], ... 86 | 'maxIter', maxIter, 'mu', mu_init); 87 | 88 | figure(2); 89 | plotKmeans(X(rpidx,:), mu, assign(rpidx), errHist(end), maxIter); 90 | 91 | 92 | % 93 | % epsilon.Lap = 1; 94 | % epsilon.val = ep_val; 95 | % epsilon.method = 0; 96 | % epsilon.maxiter = maxIter; 97 | % % epsilon.delta_i = 1e-6; 98 | % epsilon.delta = 1e-4; 99 | % 100 | % mu_init = bsxfun(@plus, lhsdesign(K,2)', [-0.5; -0.5]) + 0.01*randn(D,K); 101 | % [mu, assign, errHist] = kmeansFit(epsilon, X, K, 'plotfn', [], ... 102 | % 'maxIter', maxIter, 'mu', mu_init); 103 | % 104 | % figure(3); 105 | % plotKmeans(X(rpidx,:), mu, assign(rpidx), errHist(end), maxIter); 106 | 107 | %% DPllyod 108 | 109 | epsilon.cdpDPlloyd = 1; 110 | epsilon.val = ep_val; 111 | epsilon.method = 1; 112 | epsilon.maxiter = maxIter; 113 | % epsilon.delta_i = 1e-6; 114 | epsilon.delta = 1e-4; 115 | 116 | % mu_init = bsxfun(@plus, lhsdesign(K,2)', [-0.5; -0.5]) + 0.01*randn(D,K); 117 | [mu, assign, errHist] = kmeansFit(epsilon, X, K, 'plotfn', [], ... 118 | 'maxIter', maxIter, 'mu', mu_init); 119 | 120 | figure(4); 121 | plotKmeans(X(rpidx,:), mu, assign(rpidx), errHist(end), maxIter); 122 | % set(gca, 'xlim', [-1 1], 'ylim', [-1 1]); 123 | 124 | 125 | % 126 | 127 | epsilon.cdpDPlloyd = 0; 128 | epsilon.val = ep_val; 129 | epsilon.method = 1; 130 | epsilon.maxiter = maxIter; 131 | % epsilon.delta_i = 1e-6; 132 | epsilon.delta = 1e-4; 133 | 134 | % mu_init = bsxfun(@plus, lhsdesign(K,2)', [-0.5; -0.5]) + 0.01*randn(D,K); 135 | [mu, assign, errHist] = kmeansFit(epsilon, X, K, 'plotfn', [], ... 136 | 'maxIter', maxIter, 'mu', mu_init); 137 | 138 | figure(5); 139 | plotKmeans(X(rpidx,:), mu, assign(rpidx), errHist(end), maxIter); 140 | % set(gca, 'xlim', [-1 1], 'ylim', [-1 1]); -------------------------------------------------------------------------------- /pmkt3/compute_ll_estep.m: -------------------------------------------------------------------------------- 1 | function [ess, loglik] = compute_ll_estep(model, data) 2 | % this is just a copy of [ess, loglik] = estep(model, data) 3 | %% Compute the expected sufficient statistics 4 | [weights, ll] = mixGaussInferLatent(model, data); 5 | cpd = model.cpd; 6 | ess = cpd.essFn(cpd, data, weights); 7 | ess.weights = weights; % useful for plottings 8 | loglik = sum(ll) + cpd.logPriorFn(cpd) + model.mixPriorFn(model); 9 | end -------------------------------------------------------------------------------- /pmkt3/condGaussCpdCreate.m: -------------------------------------------------------------------------------- 1 | function CPD = condGaussCpdCreate(mu, Sigma, varargin) 2 | %% Create a conditional Gaussian distribution for use in a graphical model 3 | % 4 | % mu is a matrix of size d-by-nstates 5 | % Sigma is of size d-by-d-by-nstates 6 | % 7 | % 'prior' is a Gauss-inverseWishart distribution, namely, a struct with 8 | % fields mu, Sigma, dof, k 9 | % Set 'prior' to 'none' to do mle. 10 | %% 11 | 12 | % This file is from pmtk3.googlecode.com 13 | 14 | prior = process_options(varargin, 'prior', []); 15 | d = size(Sigma, 1); 16 | if isempty(prior) 17 | prior.mu = zeros(1, d); 18 | % prior.mu = 0.1*ones(1,d); 19 | prior.Sigma = 0.1*eye(d); 20 | % prior.k = 0.01; 21 | prior.k = 1.01; 22 | prior.dof = d + 1; 23 | end 24 | if isvector(Sigma) 25 | Sigma = permute(rowvec(Sigma), [1 3 2]); 26 | end 27 | nstates = size(Sigma, 3); 28 | if size(mu, 2) ~= nstates && size(mu, 1) == nstates 29 | mu = mu'; 30 | end 31 | CPD = structure(mu, Sigma, nstates, d, prior); 32 | CPD.cpdType = 'condGauss'; 33 | CPD.fitFn = @condGaussCpdFit; 34 | CPD.fitFnEss = @condGaussCpdFitEss; 35 | CPD.essFn = @condGaussCpdComputeEss; 36 | CPD.logPriorFn = @logPriorFn; 37 | CPD.rndInitFn = @rndInit; 38 | end 39 | 40 | function cpd = rndInit(cpd) 41 | %% randomly initialize 42 | d = cpd.d; 43 | nstates = cpd.nstates; 44 | cpd.mu = randn(d, nstates); 45 | Sigma = zeros(d, d, nstates); 46 | for i=1:nstates 47 | Sigma(:, :, i) = randpd(d) + 2*eye(d); 48 | end 49 | cpd.Sigma = Sigma; 50 | end 51 | 52 | function logp = logPriorFn(cpd) 53 | %% calculate the logprior 54 | logp = 0; 55 | prior = cpd.prior; 56 | if ~isempty(prior) && isstruct(prior) 57 | nstates = cpd.nstates; 58 | mu = cpd.mu; 59 | Sigma = cpd.Sigma; 60 | for k = 1:nstates 61 | logp = logp + gaussInvWishartLogprob(prior, mu(:, k), Sigma(:, :, k)); 62 | end 63 | end 64 | end 65 | 66 | function ess = condGaussCpdComputeEss(cpd, data, weights, B) 67 | %% Compute the expected sufficient statistics for a condGaussCpd 68 | % data is nobs-by-d 69 | % weights is nobs-by-nstates; the marginal probability of the discrete 70 | % parent for each observation. 71 | % B is ignored, but required by the interface, (since mixture emissions use 72 | % it). 73 | %% 74 | d = cpd.d; 75 | nstates = cpd.nstates; 76 | wsum = sum(weights, 1); 77 | 78 | 79 | if min(wsum) <= 1 80 | wsum(wsum<=1)= 1.01; 81 | end 82 | 83 | % if min(wsum)<1 84 | % % for numerical stability 85 | % wsum = wsum + 2; 86 | % wsum_ratio = wsum./sum(wsum); 87 | % wsum = length(data)*wsum_ratio; 88 | % end 89 | 90 | xbar = bsxfun(@rdivide, data'*weights, wsum); %d-by-nstates 91 | XX = zeros(d, d, nstates); 92 | secondMoment = zeros(d, d, nstates); 93 | for j=1:nstates 94 | Xc = bsxfun(@minus, data, xbar(:, j)'); 95 | secondMoment(:,:,j) = bsxfun(@times, data, weights(:, j))'*data; 96 | XX(:, :, j) = bsxfun(@times, Xc, weights(:, j))'*Xc; 97 | end 98 | ess = structure(xbar, XX, wsum, secondMoment); 99 | end 100 | 101 | function cpd = condGaussCpdFitEss(cpd, ess, model) 102 | %% Fit a condGaussCpd given expected sufficient statistics 103 | % ess is a struct containing wsum, XX, and xbar 104 | % cpd is a condGaussCpd as created by e.g condGaussCpdCreate 105 | % 106 | %% 107 | wsum = ess.wsum; 108 | 109 | %%%%%%%%%%% 110 | 111 | if min(wsum) <= 1 112 | wsum(wsum<=1)= 1.01; 113 | end 114 | 115 | %%%%%%%%%%% 116 | 117 | XX = ess.XX; 118 | xbar = ess.xbar; 119 | d = cpd.d; 120 | nstates = cpd.nstates; 121 | prior = cpd.prior; 122 | if ~isstruct(prior) || isempty(prior) % do mle 123 | cpd.mu = reshape(xbar, d, nstates); 124 | cpd.Sigma = bsxfun(@rdivide, XX, reshape(wsum, [1 1 nstates])); 125 | else % do map 126 | kappa0 = prior.k; 127 | m0 = prior.mu(:); 128 | nu0 = prior.dof; 129 | S0 = prior.Sigma; 130 | mu = zeros(d, nstates); 131 | Sigma = zeros(d, d, nstates); 132 | 133 | if model.total_eps ~=0 134 | 135 | d = model.d; 136 | 137 | % sesntivity for mean 138 | Nk_plus_kappa0 = wsum + kappa0; 139 | L1sen_mean = 2*sqrt(d)*(1./Nk_plus_kappa0); 140 | const = nu0 + d + 2; 141 | Nk_plus_const = wsum + const; 142 | Sen_cov = 1./Nk_plus_const; 143 | 144 | for k = 1:nstates 145 | xbark = xbar(:, k); 146 | XXk = XX(:, :, k); 147 | wk = wsum(k); 148 | 149 | %% (1) add noise to mean 150 | 151 | mn = (wk*xbark + kappa0*m0)./(wk + kappa0); 152 | eps_prime = model.eps_prime; 153 | 154 | if model.lap 155 | noise_variance_for_mean_2 = L1sen_mean(k)/eps_prime; 156 | noise_for_mean = laprnd(d, 1, 0, sqrt(2)*noise_variance_for_mean_2); 157 | else 158 | % or Gaussian 159 | mudiff = L1sen_mean(k)/sqrt(d); 160 | c2 = model.c2; 161 | sigma = mudiff/eps_prime; 162 | noise_for_mean = mvnrnd(zeros(1,d), c2*sigma^2*eye(d))'; 163 | end 164 | 165 | new_mu = mn + noise_for_mean; 166 | if norm(new_mu)>1 167 | new_mu = new_mu./norm(new_mu); 168 | end 169 | mu(:, k) = new_mu; 170 | 171 | %% (2) add noise to Sigma 172 | 173 | a = (kappa0*wk)./(kappa0 + wk); 174 | b = nu0 + wk + d + 2; 175 | Sprior = (xbark-m0)*(xbark-m0)'; 176 | Sigma_org = (S0 + XXk + a*Sprior)./b; 177 | 178 | % Gaussian noise 179 | c2 = model.c2; 180 | sigma = Sen_cov(k)/eps_prime; 181 | noisemat = triu(normrnd(0, sqrt(c2)*sigma, d, d)); 182 | Z = noisemat + tril(noisemat', -1); 183 | 184 | noisedup_Sigma = Sigma_org + Z; 185 | 186 | % make sure the matrix is pd 187 | [V,D] = eig(noisedup_Sigma); 188 | diagD = diag(D); 189 | tol = 1e-3; 190 | diagD(diagD 1 31 | models = cell(1, nRandomRestarts); 32 | llhists = cell(1, nRandomRestarts); 33 | bestLL = zeros(1, nRandomRestarts); 34 | for i=1:nRandomRestarts 35 | if verbose 36 | fprintf('\n********** Random Restart %d **********\n', i); 37 | end 38 | [models{i}, llhists{i}] = emAlgo(model, data, init, estep,... 39 | mstep, 'verbose', verbose, 'restartNum', i, args{:}); 40 | bestLL(i) = llhists{i}(end); 41 | end 42 | bestndx = maxidx(bestLL); 43 | model = models{bestndx}; 44 | loglikHist = llhists{bestndx}; 45 | return 46 | end 47 | %% Perform EM 48 | [maxIter, convTol, plotfn, restartNum, computeLoglik] = process_options(args ,... 49 | 'maxIter' , 50 , ... 50 | 'convTol' , 1e-4 , ... 51 | 'plotfn' , [] , ... 52 | 'restartNum' , 1, .... 53 | 'computeLoglik', true); 54 | 55 | if verbose, fprintf('initializing model for EM\n'); end 56 | model = init(model, data, restartNum); 57 | iter = 1; 58 | done = false; 59 | loglikHist = zeros(maxIter + 1, 1); 60 | while ~done 61 | if computeLoglik 62 | [ess, ll] = estep(model, data); 63 | else 64 | [ess] = estep(model, data); 65 | ll = 0; 66 | end 67 | if verbose 68 | fprintf('%d\t loglik: %g\n', iter, ll ); 69 | end 70 | if ~isempty(plotfn) 71 | plotfn(model, data, ess, ll, iter); 72 | end 73 | loglikHist(iter) = ll; 74 | model = mstep(model, ess); 75 | if iter > maxIter 76 | done = true; 77 | elseif iter > 1 78 | if computeLoglik 79 | showWarning = true; 80 | done = convergenceTest(loglikHist(iter), loglikHist(iter-1), convTol, showWarning); 81 | end 82 | % could add convergence test based on params not changing 83 | end 84 | %done = (iter > maxIter) || ( (iter > 1) && ... 85 | % convergenceTest(loglikHist(iter), loglikHist(iter-1), convTol, true)); 86 | iter = iter + 1; 87 | end 88 | loglikHist = loglikHist(1:iter-1); 89 | llHists{1} = loglikHist; 90 | end 91 | -------------------------------------------------------------------------------- /pmkt3/gaussInvWishartLogprob.m: -------------------------------------------------------------------------------- 1 | function logp = gaussInvWishartLogprob(arg1, arg2, arg3, arg4, arg5, arg6) 2 | % logp(i) = p(m(:, i), S(:, :, i) | model) 3 | % logp = gaussInvWishartLogprob(model, m, S) OR 4 | % logp = gaussInvWishartLogprob(mu, Sigma, dof, k, m, S) 5 | % 6 | % logp(i) = p(m(:, i), S(:, :, i) | model) 7 | % = N(m(:, i) | model.mu, (1/model.k) * S(:, :, i)) * 8 | % IW(S(:, :, i) | model.dof, model.Sigma) 9 | % 10 | % OR if scalar 11 | % 12 | % logp(i) = p(m(i), S(i) | model) 13 | % = N(m(i) | model.mu, (1/model.k) * S(i)) * 14 | % IW(S(i) | model.dof, model.Sigma) 15 | % 16 | % 17 | % model has the following fields: mu, Sigma, dof, k 18 | % 19 | % *** Vectorized w.r.t. to both m and S *** 20 | %% 21 | 22 | % This file is from pmtk3.googlecode.com 23 | 24 | 25 | if isstruct(arg1) 26 | model = arg1; 27 | m = arg2; 28 | S = arg3; 29 | mu = model.mu; 30 | Sigma = model.Sigma; 31 | dof = model.dof; 32 | k = model.k; 33 | else 34 | mu = arg1; 35 | Sigma = arg2; 36 | dof = arg3; 37 | k = arg4; 38 | m = arg5; 39 | S = arg6; 40 | end 41 | 42 | 43 | 44 | d = min(size(S, 1), size(S, 2)); % take care of 1d vectorized case 45 | m = reshape(m, d, []); 46 | S = reshape(S, d, d, []); 47 | n = max(size(m, 2), size(S, 3)); 48 | if size(m, 2) < n, m = repmat(m, 1, n); end 49 | if size(S, 3) < n, S = repmat(S, [1, 1, n]); end 50 | 51 | pgauss = zeros(n, 1); 52 | gaussModel.mu = mu; 53 | for i=1:n 54 | gaussModel.Sigma = S(:, :, i) ./ k; 55 | pgauss(i) = gaussLogprob(gaussModel, m(:, i)'); 56 | end 57 | 58 | iwModel.Sigma = Sigma; 59 | iwModel.dof = dof; 60 | piw = invWishartLogprob(iwModel, S); 61 | 62 | logp = pgauss + piw; 63 | 64 | end 65 | -------------------------------------------------------------------------------- /pmkt3/gaussLogprob.m: -------------------------------------------------------------------------------- 1 | function logp = gaussLogprob(arg1, arg2, arg3) 2 | % Log pdf of multivariate Gaussian 3 | % L = gaussLogprob(model, X) 4 | % L = gaussLogprob(mu, Sigma, X) 5 | % L(i) = log prob X(i,:) is i'th case, can contain NaNs for missing values. 6 | % Sigma can be a vector; this is interpreted as a diagonal matrix. 7 | % *** In the univariate case, Sigma is the variance, not the standard 8 | % deviation! *** 9 | % 10 | % Examples 11 | % L = gaussLogprob(zeros(3,1), randpd(3), rand(10,3)) 12 | % L = gaussLogprob(zeros(3,1), diag(randpd(3)), rand(10,3)) 13 | % L = gaussLogprob(structure(mu, Sigma), X) 14 | 15 | % This file is from pmtk3.googlecode.com 16 | 17 | 18 | switch nargin 19 | case 3, mu = arg1; Sigma = arg2; X = arg3; 20 | case 2, model = arg1; mu = model.mu; Sigma = model.Sigma; X = arg2; 21 | otherwise 22 | error('bad num args') 23 | end 24 | 25 | mu = mu(:); 26 | if any(isnan(X(:))) 27 | logp = gaussLogprobMissingData(structure(mu, Sigma), X); 28 | return; 29 | end 30 | if isscalar(mu) 31 | X = X(:); 32 | end 33 | [N, d] = size(X); 34 | 35 | if d == 1 36 | X = X(:) - mu(:); % each data case evaluated under a different mu 37 | else 38 | X = bsxfun(@minus, X, rowvec(mu)); 39 | end 40 | if isvector(Sigma) && (numel(Sigma) > 1) % diagonal case 41 | sig2 = repmat(Sigma', N, 1); 42 | tmp = -(X.^2)./(2*sig2) - 0.5*log(2*pi*sig2); 43 | logp = sum(tmp, 2); 44 | else 45 | % Full covariance case 46 | % if 0 47 | logp2 = -0.5*sum((X/Sigma).*X, 2); 48 | logZ2 = (d/2)*log(2*pi) + 0.5*logdet(Sigma); 49 | logp = logp2 - logZ2; 50 | % end 51 | % % slightly faster version 52 | % R = chol(Sigma); 53 | % logp = -0.5*sum((X/R).^2, 2); 54 | % logZ = 0.5*d*log(2*pi) + sum(log(diag(R))); 55 | % logp = logp - logZ; 56 | %assert(approxeq(logp, logp2)) 57 | end 58 | 59 | end 60 | 61 | -------------------------------------------------------------------------------- /pmkt3/gaussPlot2d.m: -------------------------------------------------------------------------------- 1 | function h=gaussPlot2d(mu, Sigma, varargin) 2 | % Plot an ellipse representing the covariance matrix of a Gaussian 3 | % 4 | 5 | % This file is from pmtk3.googlecode.com 6 | 7 | if size(Sigma) ~= [2 2], error('Sigma must be a 2 by 2 matrix'); end 8 | [color, plotMarker] = process_options(varargin, 'color', 'b', 'plotMarker', true); 9 | 10 | mu = mu(:); 11 | [U, D] = eig(Sigma); 12 | n = 100; 13 | t = linspace(0, 2*pi, n); 14 | xy = [cos(t); sin(t)]; 15 | %k = sqrt(chi2inv(0.95, 2)); % 5.99 16 | k = sqrt(6); 17 | w = (k * U * sqrt(D)) * xy; 18 | z = repmat(mu, [1 n]) + w; 19 | h = plot(z(1, :), z(2, :), 'color', color, 'linewidth', 2); 20 | hold on 21 | if plotMarker 22 | hh=plot(mu(1), mu(2), 'x'); 23 | set(hh,'color',color, 'linewidth', 2, 'markersize', 13); 24 | end 25 | %axis('equal'); 26 | end 27 | -------------------------------------------------------------------------------- /pmkt3/gaussSample.m: -------------------------------------------------------------------------------- 1 | function S = gaussSample(arg1, arg2, arg3) 2 | % Returns n samples (in the rows) from a multivariate Gaussian distribution 3 | % 4 | % Examples: 5 | % S = gaussSample(mu, Sigma, 10) 6 | % S = gaussSample(model, 100) 7 | % S = gaussSample(struct('mu',[0], 'Sigma', eye(1)), 3) 8 | 9 | % This file is from pmtk3.googlecode.com 10 | 11 | 12 | switch nargin 13 | case 3, mu = arg1; Sigma = arg2; n = arg3; 14 | case 2, model = arg1; mu = model.mu; Sigma = model.Sigma; n = arg2; 15 | case 1, model = arg1; mu = model.mu; Sigma = model.Sigma; n = 1; 16 | otherwise 17 | error('bad num args') 18 | end 19 | 20 | A = chol(Sigma, 'lower'); 21 | Z = randn(length(mu), n); 22 | S = bsxfun(@plus, mu(:), A*Z)'; 23 | 24 | 25 | end 26 | -------------------------------------------------------------------------------- /pmkt3/invWishartLogprob.m: -------------------------------------------------------------------------------- 1 | function logp = invWishartLogprob(arg1, arg2, arg3) 2 | % logp(i) = log p(S(:, :, i) | model) 3 | % If model.Sigma is scalar, than logp(i) = log p(S(i) | model) 4 | % 5 | % model has fields Sigma and dof 6 | % model.dof must be > d-1 7 | % 8 | % logp = invWishartLogprob(model, S); OR 9 | % logp = invWishartLogprob(Sigma, dof, S); 10 | %% 11 | 12 | % This file is from pmtk3.googlecode.com 13 | 14 | 15 | if isstruct(arg1) 16 | model = arg1; 17 | Sigma = model.Sigma; 18 | dof = model.dof; 19 | S = arg2; 20 | else 21 | Sigma = arg1; 22 | dof = arg2; 23 | S = arg3; 24 | end 25 | 26 | d = size(Sigma, 1); 27 | S = reshape(S, d, d, []); 28 | n = size(S, 3); 29 | logZ = (dof*d/2)*log(2) + mvtGammaln(d, dof/2) - (dof/2)*logdet(Sigma); 30 | logp = zeros(n, 1); 31 | for i=1:n 32 | logp(i) = - (dof + d+1) / 2*logdet(S(:, :, i)) ... 33 | - 0.5*trace(Sigma / S(:, :, i)) ... 34 | - logZ; 35 | end 36 | 37 | 38 | 39 | end 40 | -------------------------------------------------------------------------------- /pmkt3/isposdef.m: -------------------------------------------------------------------------------- 1 | function b = isposdef(a) 2 | % ISPOSDEF Test for positive definite matrix. 3 | % ISPOSDEF(A) returns 1 if A is positive definite, 0 otherwise. 4 | % Using chol is much more efficient than computing eigenvectors. 5 | 6 | % Written by Tom Minka 7 | 8 | [R,p] = chol(a); 9 | b = (p == 0); 10 | -------------------------------------------------------------------------------- /pmkt3/kmeansFit.m: -------------------------------------------------------------------------------- 1 | function [mu, assign, errHist] = kmeansFit(epsilon, X, K, varargin) 2 | % Hard cluster data using kmeans. 3 | % 4 | %% Inputs 5 | % epsilon has all the quantities for DP or CDP setup 6 | % 7 | % X ... X(i, :) the ith data case 8 | % K ... the number of clusters to fit 9 | %% Outputs 10 | % 11 | % mu ... mu(:, k) is the k'th center 12 | % assign ... assign(i) is the cluster to which data point i is 13 | % assigned. 14 | %% Optional (named) Inputs 15 | % 16 | % 'maxIter' ... [100] maximum number of iterations to run 17 | % 'thresh' ... [1e-3] convergence threshold 18 | % 'plotFn' ... @plotfn(X, mu, assign, err, iter) called each iteration 19 | % 'verbose' ... [false] if true, display progress each iteration 20 | % 'mu' ... initial guess for the cluster centers 21 | %% Example 22 | % 23 | % [mu, assign, errHist] = kmeansFit(randn(1000, 10), 7, 'verbose', true); 24 | % 25 | %% Parse inputs 26 | 27 | % This file is from pmtk3.googlecode.com 28 | 29 | [maxIter, thresh, plotfn, verbose, mu] = process_options(varargin, ... 30 | 'maxIter' , 100 , ... 31 | 'thresh' , 1e-3 , ... 32 | 'plotfn' , @(varargin)[] , ... % default does nothing 33 | 'verbose' , false , ... 34 | 'mu' , [] ); 35 | [N,D] = size(X); 36 | %% Initialize 37 | % Initialize using K data points chosen at random 38 | if isempty(mu) 39 | perm = randperm(N); 40 | % in the unlikely event of a tie, 41 | % we want to ensure the means are different. 42 | v = var(X); 43 | noise = gaussSample(zeros(1, length(v)), 0.01*diag(v), K); 44 | mu = X(perm(1:K), :)' + noise'; 45 | end 46 | %% Setup loop 47 | iter = 1; 48 | errHist = zeros(maxIter, 1); 49 | prevErr = Inf; 50 | prevMu = zeros(size(mu)); 51 | 52 | while true 53 | dist = sqDistance(X, mu'); % dist(i, j) = sum((X(i, :)- mu(:, j)').^2) 54 | assign = minidx(dist, [], 2); 55 | %% DP stuff comes into this function 56 | mu = partitionedMean(epsilon, X, assign, K)'; 57 | currentErr = sum(sum(dist'.*bsxfun(@eq, (1:K).', assign.')))/N; 58 | % currentErr 59 | % if isnan(currentErr) 60 | % display; 61 | % end 62 | 63 | % currentErr 64 | % dbstop if isnan(currentErr) 65 | % S = bsxfun(@eq, sparse(1:K).', assign.')'.*X; 66 | %% Display progress 67 | 68 | errHist(iter) = currentErr; 69 | % if plotfn 70 | % plotfn(X, mu, assign, currentErr, iter); 71 | % end 72 | % if verbose, fprintf('iteration %d, err = %f\n', iter, currentErr); end 73 | %% Check convergence 74 | if convergenceTest(currentErr, prevErr, thresh) || (iter >= maxIter) 75 | mu = prevMu; 76 | break 77 | end 78 | iter = iter + 1; 79 | prevErr = currentErr; 80 | prevMu = mu; 81 | % % dbstop if isnan(prevErr) 82 | end 83 | errHist = errHist(1:iter); 84 | end 85 | -------------------------------------------------------------------------------- /pmkt3/logdet.m: -------------------------------------------------------------------------------- 1 | function y = logdet(A) 2 | % Compute log(det(A)) where A is positive-definite 3 | % This is faster and more stable than using log(det(A)). 4 | % 5 | %PMTKauthor Tom Minka 6 | % (c) Microsoft Corporation. All rights reserved. 7 | 8 | % This file is from pmtk3.googlecode.com 9 | 10 | try 11 | U = chol(A); 12 | y = 2*sum(log(diag(U))); 13 | catch %#ok 14 | y = 0; 15 | warning('logdet:posdef', 'Matrix is not positive definite'); 16 | end 17 | 18 | end 19 | -------------------------------------------------------------------------------- /pmkt3/logsumexp.m: -------------------------------------------------------------------------------- 1 | function s = logsumexp(x, dim) 2 | % Compute log(sum(exp(x),dim)) while avoiding numerical underflow. 3 | % By default dim = 1 (columns). 4 | % Written by Michael Chen (sth4nth@gmail.com). 5 | if nargin == 1, 6 | % Determine which dimension sum will use 7 | dim = find(size(x)~=1,1); 8 | if isempty(dim), dim = 1; end 9 | end 10 | 11 | % subtract the largest in each column 12 | y = max(x,[],dim); 13 | x = bsxfun(@minus,x,y); 14 | s = y + log(sum(exp(x),dim)); 15 | i = find(~isfinite(y)); 16 | if ~isempty(i) 17 | s(i) = y(i); 18 | end 19 | 20 | % function s = logsumexp(a, dim) 21 | % % Returns log(sum(exp(a),dim)) while avoiding numerical underflow. 22 | % % Default is dim = 1 (columns). 23 | % % logsumexp(a, 2) will sum across rows instead of columns. 24 | % % Unlike matlab's "sum", it will not switch the summing direction 25 | % % if you provide a row vector. 26 | % 27 | % % Written by Tom Minka 28 | % % (c) Microsoft Corporation. All rights reserved. 29 | % 30 | % if nargin < 2 31 | % dim = 1; 32 | % end 33 | % 34 | % % subtract the largest in each column 35 | % [y, i] = max(a,[],dim); 36 | % dims = ones(1,ndims(a)); 37 | % dims(dim) = size(a,dim); 38 | % a = a - repmat(y, dims); 39 | % s = y + log(sum(exp(a),dim)); 40 | % i = find(~isfinite(y)); 41 | % if ~isempty(i) 42 | % s(i) = y(i); 43 | % end -------------------------------------------------------------------------------- /pmkt3/minidx.m: -------------------------------------------------------------------------------- 1 | function [idx, val] = minidx(varargin) 2 | % Return the linear index of the minimum value 3 | % Same as built in min but with the order of the outputs reversed. 4 | 5 | % This file is from pmtk3.googlecode.com 6 | 7 | [val,idx] = min(varargin{:}); 8 | end 9 | -------------------------------------------------------------------------------- /pmkt3/mixGaussCreate.m: -------------------------------------------------------------------------------- 1 | function model = mixGaussCreate(mu, Sigma, mixWeight) 2 | 3 | model.cpd.mu = mu; 4 | model.cpd.Sigma = Sigma; 5 | model.mixWeight = mixWeight; 6 | model.nmix = numel(mixWeight); 7 | model.type = 'mixGauss'; 8 | end 9 | -------------------------------------------------------------------------------- /pmkt3/mixGaussFit.m: -------------------------------------------------------------------------------- 1 | function [model, loglikHist] = mixGaussFit(epsilon, data, nmix, varargin) 2 | %% Fit a mixture of Gaussians via MLE/MAP (using EM) 3 | % 4 | % 5 | %% Inputs 6 | % epsilon - 0 if this is non-DP. If non-zero, this is DPEM. 7 | % data - data(i, :) is the ith case, i.e. data is of size n-by-d 8 | % nmix - the number of mixture components to use 9 | % 10 | % This file is from pmtk3.googlecode.com 11 | 12 | 13 | [initParams, prior, mixPrior, EMargs] = ... 14 | process_options(varargin, ... 15 | 'initParams' , [], ... 16 | 'prior' , [], ... 17 | 'mixPrior' , []); 18 | [n, d] = size(data); 19 | model.type = 'mixGauss'; 20 | model.nmix = nmix; 21 | model.d = d; 22 | model = setMixPrior(model, mixPrior); 23 | model.maxIter = EMargs{2}; 24 | model.N = n; 25 | 26 | J = model.maxIter; 27 | K = nmix; 28 | 29 | model.total_eps = epsilon.total_eps; 30 | model.lap = epsilon.lap; 31 | model.comp = epsilon.comp; 32 | 33 | % epsilon.total_eps = total_eps; 34 | % epsilon.total_del = total_del; 35 | % epsilon.Lap = lap; 36 | % epsilon.comp = comp; 37 | 38 | delta_i = 1e-6; % this is for advanced, zCDP, and MA (per iteration tolerance) 39 | % test compositions with combinations of mechanisms 40 | 41 | if epsilon.comp == 0 42 | % non-private 43 | model.eps_prime = 0; 44 | 45 | elseif epsilon.comp == 1 46 | % linear composition 47 | 48 | if epsilon.lap == 1 49 | delta_i = epsilon.total_del/(J*K); 50 | c2 = 2*log(1.25/delta_i); 51 | model.c2 = c2; 52 | model.eps_prime = epsilon.total_eps/(J*(2*K+1)); 53 | else % Gaussian 54 | delta_i = epsilon.total_del/(J*(K+1)); 55 | c2 = 2*log(1.25/delta_i); 56 | model.c2 = c2; 57 | model.eps_prime = epsilon.total_eps/(J*(2*K+1)); 58 | end 59 | 60 | elseif epsilon.comp == 2 61 | % advanced composition 62 | 63 | if epsilon.lap == 1 64 | c2 = 2*log(1.25/delta_i); 65 | model.c2 = c2; 66 | delta_prime = epsilon.total_del - J*K*delta_i; 67 | JK = J*(2*K+1); 68 | myfun = @(x) (JK*x*(exp(x)-1) + sqrt(2*JK*log(1/delta_prime))*x - epsilon.total_eps)^2; 69 | model.eps_prime = real(fsolve(myfun, 0.1)); 70 | else % gaussian 71 | c2 = 2*log(1.25/delta_i); 72 | model.c2 = c2; 73 | delta_prime = epsilon.total_del - J*(K+1)*delta_i; 74 | JK = J*(2*K+1); 75 | myfun = @(x) (JK*x*(exp(x)-1) + sqrt(2*JK*log(1/delta_prime))*x - epsilon.total_eps)^2; 76 | model.eps_prime = real(fsolve(myfun, 0.1)); 77 | end 78 | 79 | elseif epsilon.comp == 3 80 | % zCDP composition 81 | if epsilon.lap == 1 82 | c2 = 2*log(1.25/delta_i); 83 | model.c2 = c2; 84 | myfun = @(x) (J*(K+1)*(x^2)/2 + 2*sqrt((J*(K+1)*(x^2)/2)*log(1/epsilon.total_del)) - epsilon.total_eps)^2; 85 | A = []; 86 | b = []; 87 | Aeq = []; 88 | beq = []; 89 | lb = 0; 90 | ub = 1; 91 | model.eps_prime = fmincon(myfun, 0.1, A, b, Aeq, beq, lb, ub); 92 | else 93 | c2 = 2*log(1.25/delta_i); 94 | model.c2 = c2; 95 | myfun = @(x) (J*(x^2)/(2*c2) + J*K*(x^2)/(2*c2) + J*K*(x^2)/2 + 2*sqrt((J*(x^2)/(2*c2) + J*K*(x^2)/(2*c2) + J*K*(x^2)/2)*log(1/epsilon.total_del)) - epsilon.total_eps)^2; 96 | A = []; 97 | b = []; 98 | Aeq = []; 99 | beq = []; 100 | lb = 0; 101 | ub = 1; 102 | model.eps_prime = fmincon(myfun, 0.1, A, b, Aeq, beq, lb, ub); 103 | end 104 | 105 | else % MA composition 106 | if epsilon.lap == 1 107 | c2 = 2*log(1.25/delta_i); 108 | model.c2 = c2; 109 | 110 | maxlam = 100; 111 | eps_i_mat = linspace(0.001, 0.999, 200); 112 | logdel = zeros(maxlam, length(eps_i_mat)); 113 | 114 | for lam=1:maxlam 115 | 116 | trm1_L = (1+lam)/(2*lam+1); 117 | trm2_L = lam/(2*lam+1); 118 | trm1_G = lam^2 + lam; 119 | 120 | myfun = @(x) J*(K+1)*(log(trm1_L + trm2_L*exp(-(2*lam+1)*x)) + lam*x) ... 121 | + J*K*trm1_G*(x^2)/(2*c2)- lam*epsilon.total_eps; 122 | 123 | for ep = 1:length(eps_i_mat) 124 | eps = eps_i_mat(ep); 125 | logdel(lam, ep) = myfun(eps); 126 | end 127 | end 128 | 129 | % Find the largest epsilon_i that satisfies delta constraint 130 | opt_del = zeros(length(eps_i_mat), 1); 131 | for i=1:length(eps_i_mat); 132 | idx = find(logdel(:,i)1) = 1; 245 | noisedup_numerator = normalize(noisedup_mixWeight).*N; 246 | [model.mixWeight]= normalize(noisedup_numerator + model.mixPrior - 1); 247 | 248 | else 249 | [model.mixWeight]= normalize(ess.wsum + model.mixPrior - 1); 250 | end 251 | 252 | end 253 | 254 | 255 | -------------------------------------------------------------------------------- /pmkt3/mixGaussInferLatent.m: -------------------------------------------------------------------------------- 1 | function [pZ, ll] = mixGaussInferLatent(model, X) 2 | % Infer latent mixture node from a set of data 3 | % pZ(i, k) = p( Z = k | X(i, :), model) 4 | % ll(i) = log p(X(i, :) | model) 5 | % X must be fully observed (no NaNs) 6 | 7 | % This file is from pmtk3.googlecode.com 8 | 9 | nmix = model.nmix; 10 | [n, d] = size(X); 11 | logMix = log(rowvec(model.mixWeight)); 12 | logPz = zeros(n, nmix); 13 | 14 | mu = model.cpd.mu; 15 | Sigma = model.cpd.Sigma; 16 | for k = 1:nmix 17 | logPz(:, k) = logMix(k) + gaussLogprob(mu(:, k), Sigma(:, :, k), X); 18 | end 19 | 20 | 21 | [logPz, ll] = normalizeLogspace(logPz); 22 | pZ = exp(logPz); 23 | end 24 | -------------------------------------------------------------------------------- /pmkt3/mkStochastic.m: -------------------------------------------------------------------------------- 1 | function [T,Z] = mkStochastic(T) 2 | % Make a multidimensional array sum to one along its last dimension 3 | % [T,Z] = mk_stochastic(T) 4 | % 5 | % If T is a vector, it will sum to 1. 6 | % If T is a matrix, each row will sum to 1. 7 | % If T is a 3D array, then sum_k T(i,j,k) = 1 for all i,j. 8 | 9 | % This file is from pmtk3.googlecode.com 10 | 11 | 12 | % Set zeros to 1 before dividing 13 | % This is valid since S(j) = 0 iff T(i,j) = 0 for all j 14 | 15 | if isvector(T) 16 | [T, Z] = normalize(T); 17 | else 18 | [T, Z] = normalize(T, ndims(T)); 19 | end 20 | 21 | 22 | % if (ndims(T)==2) & (size(T,1)==1 | size(T,2)==1) % isvector 23 | % [T,Z] = normalize(T); 24 | % elseif ndims(T)==2 % matrix 25 | % Z = sum(T,2); 26 | % S = Z + (Z==0); 27 | % norm = repmat(S, 1, size(T,2)); 28 | % T = T ./ norm; 29 | % else % multi-dimensional array 30 | % ns = size(T); 31 | % T = reshape(T, prod(ns(1:end-1)), ns(end)); 32 | % Z = sum(T,2); 33 | % S = Z + (Z==0); 34 | % norm = repmat(S, 1, ns(end)); 35 | % T = T ./ norm; 36 | % T = reshape(T, ns); 37 | % end 38 | 39 | end 40 | -------------------------------------------------------------------------------- /pmkt3/mkUnitVariance.m: -------------------------------------------------------------------------------- 1 | function [X, s] = mkUnitVariance(X, s) 2 | % Make each column of X be variance 1 3 | % ie., sum_i x(i,j)^2 = n (so var(X(:,j))=1) 4 | % If s is omitted, it computed from X and returned for use at test time 5 | 6 | % This file is from pmtk3.googlecode.com 7 | 8 | 9 | if nargin < 2, s = []; end 10 | if isempty(s) 11 | s = std(X); 12 | s((s (n-1)/2 5 | % See Muirhead pp 61-62. 6 | 7 | % This file is from pmtk3.googlecode.com 8 | 9 | logp = ((n*(n-1))/4)*log(pi)+sum(gammaln(alpha+0.5*(1-[1:n]))); 10 | 11 | end 12 | -------------------------------------------------------------------------------- /pmkt3/normalize.m: -------------------------------------------------------------------------------- 1 | function [A, z] = normalize(A, dim) 2 | % Make the entries of a (multidimensional) array sum to 1 3 | % [A, z] = normalize(A) normalize the whole array, where z is the normalizing constant 4 | % [A, z] = normalize(A, dim) 5 | % If dim is specified, we normalize the specified dimension only. 6 | % dim=1 means each column sums to one 7 | % dim=2 means each row sums to one 8 | % 9 | %% 10 | % Set any zeros to one before dividing. 11 | % This is valid, since s=0 iff all A(i)=0, so 12 | % we will get 0/1=0 13 | 14 | % This file is from pmtk3.googlecode.com 15 | 16 | if(nargin < 2) 17 | z = sum(A(:)); 18 | z(z==0) = 1; 19 | A = A./z; 20 | else 21 | z = sum(A, dim); 22 | z(z==0) = 1; 23 | A = bsxfun(@rdivide, A, z); 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /pmkt3/normalizeLogspace.m: -------------------------------------------------------------------------------- 1 | function [y, L] = normalizeLogspace(x) 2 | % Normalize in logspace while avoiding numerical underflow 3 | % Each *row* of x is a log discrete distribution. 4 | % y(i,:) = x(i,:) - logsumexp(x,2) = x(i) - log[sum_c exp(x(i,c)] 5 | % L is the log normalization constant 6 | % eg [logPost, L] = normalizeLogspace(logprior + loglik) 7 | % post = exp(logPost); 8 | %% 9 | 10 | % This file is from pmtk3.googlecode.com 11 | 12 | L = logsumexp(x, 2); 13 | %y = x - repmat(L, 1, size(x,2)); 14 | y = bsxfun(@minus, x, L); 15 | 16 | end 17 | -------------------------------------------------------------------------------- /pmkt3/partitionedMean.m: -------------------------------------------------------------------------------- 1 | function [M] = partitionedMean(epsilon, X, y, C) 2 | % Group the rows of X according to the class labels in y and take the mean of each group 3 | % 4 | % X - an n-by-d matrix of doubles 5 | % y - an n-by-1 vector of ints in 1:C 6 | % C - (optional) the number of classes, (calculated if not specified) 7 | % 8 | % M - a C-by-d matrix of means. 9 | % counts(i) = sum(y==i) 10 | % 11 | % See also partitionedSum 12 | 13 | % This file is from pmtk3.googlecode.com 14 | 15 | 16 | % epsilon.method = 1; % 1: DPLloyd, 2: ours 17 | % epsilon.val = 0.9; 18 | % epsilon.Lap = 1; 19 | % epsilon.cdp = 0; 20 | 21 | % if nargin < 3 22 | % C = nunique(y); 23 | % end 24 | % if 1 25 | d = size(X,2); 26 | % N = size(X, 1); 27 | M = zeros(C, d); 28 | % threshold = 0.1; 29 | 30 | %% 31 | ndx_mat = zeros(C,1); 32 | for c=1:C 33 | ndx_mat(c) = sum(y==c); 34 | end 35 | 36 | for c=1:C 37 | 38 | ndx = (y==c); 39 | if sum(ndx)==0 40 | rand_idx = randi(length(y), 2); 41 | rand_idx(rand_idx==1) = 1; 42 | ndx(rand_idx) = 1; 43 | end 44 | 45 | if epsilon.val==0 46 | M(c, :) = mean(X(ndx, :)); 47 | else 48 | if epsilon.method==1 49 | 50 | if epsilon.cdpDPlloyd ==0 51 | 52 | num_dpts_in_c = sum(ndx); 53 | 54 | b_Lap = (d+1)*epsilon.maxiter/epsilon.val; 55 | noise_num_dpts_in_c = laprnd(1, 1, 0, sqrt(2)*b_Lap); 56 | noisedup_num_dpts_in_c = num_dpts_in_c + noise_num_dpts_in_c; 57 | 58 | 59 | sum_before_noise = sum(X(ndx,:)); 60 | noise_for_sum = laprnd(1, d, 0, sqrt(2)*b_Lap); 61 | noisedup_sum = sum_before_noise + noise_for_sum; 62 | 63 | noisedup_mean = noisedup_sum./noisedup_num_dpts_in_c; 64 | 65 | noisedup_mean(noisedup_mean>1) = 1; 66 | noisedup_mean(noisedup_mean<-1)= -1; 67 | 68 | M(c, :) = noisedup_mean; 69 | 70 | else % epsilon.cdpDPlloyd ==1 71 | 72 | % this is for DPlloyd with Laplace noise and zCDP composition 73 | JK = epsilon.maxiter; % laplace mechanism for J times 74 | myfun = @(x) (JK*x^2/2 + 2*sqrt(JK*x^2/2*log(1/epsilon.delta)) - epsilon.val)^2; 75 | model.eps_prime = fsolve(myfun, 0.1); 76 | 77 | 78 | % mudiff = (d+1); 79 | b_Lap = (d+1)/model.eps_prime; 80 | noise_num_dpts_in_c = laprnd(1, 1, 0, sqrt(2)*b_Lap); 81 | 82 | num_dpts_in_c = sum(ndx); 83 | noisedup_num_dpts_in_c = num_dpts_in_c + noise_num_dpts_in_c; 84 | 85 | sum_before_noise = sum(X(ndx,:)); 86 | noise_for_sum = laprnd(1, d, 0, sqrt(2)*b_Lap); 87 | noisedup_sum = sum_before_noise + noise_for_sum; 88 | 89 | noisedup_mean = noisedup_sum./noisedup_num_dpts_in_c; 90 | 91 | noisedup_mean(noisedup_mean>1) = 1; 92 | noisedup_mean(noisedup_mean<-1)= -1; 93 | 94 | M(c, :) = noisedup_mean; 95 | end 96 | 97 | 98 | else % epsilon.method ==0 (our method) 99 | 100 | M_c = mean(X(ndx, :)); 101 | 102 | % ndx_mat(ndx_mat<=1) = 1.01; 103 | sorted_ndx = sort(ndx_mat); 104 | sorted_ndx(sorted_ndx==0)=2; 105 | L1sen = sqrt(d)*1./sorted_ndx; 106 | 107 | if epsilon.Lap==1 108 | 109 | JK = epsilon.maxiter; % laplace mechanism for J times 110 | myfun = @(x) (JK*x^2/2 + 2*sqrt(JK*x^2/2*log(1/epsilon.delta)) - epsilon.val)^2; 111 | model.eps_prime = fsolve(myfun, 0.1); 112 | b_Lap = L1sen(c)/model.eps_prime; 113 | noise_for_mean = laprnd(1, 2, 0, sqrt(2)*b_Lap); 114 | 115 | else 116 | 117 | mudiff = L1sen(c)/sqrt(d); 118 | delta = 1e-6; 119 | c2 = 2*log(1.25/delta); 120 | 121 | JK = epsilon.maxiter; % laplace mechanism for J times 122 | myfun = @(x) (JK*x^2/(2*c2) + 2*sqrt(JK*x^2/(2*c2)*log(1/epsilon.delta))*x - epsilon.val)^2; 123 | model.eps_prime = fsolve(myfun, 0.1); 124 | sigma = mudiff/model.eps_prime; 125 | 126 | noise_for_mean = mvnrnd(zeros(1,d), c2*sigma^2*eye(d)); 127 | end 128 | 129 | noisedup_mean = M_c + noise_for_mean; 130 | 131 | noisedup_mean(noisedup_mean>1) = 1; 132 | noisedup_mean(noisedup_mean<-1)= -1; 133 | 134 | % noisedup_mean 135 | 136 | M(c, :) = noisedup_mean; 137 | end 138 | end 139 | 140 | end 141 | 142 | end 143 | 144 | 145 | 146 | 147 | -------------------------------------------------------------------------------- /pmkt3/partitionedSum.m: -------------------------------------------------------------------------------- 1 | function S = partitionedSum(X, y, C) 2 | % Group the rows of X according to the class labels in y and sum each group 3 | % 4 | % X - an n-by-d matrix of doubles 5 | % y - an n-by-1 vector of ints in 1:C 6 | % C - (optional) the number of classes, (calculated if not specified) 7 | % 8 | % M - a C-by-d matrix of sums. 9 | % 10 | % See also - partitionedMean 11 | 12 | % This file is from pmtk3.googlecode.com 13 | 14 | 15 | 16 | if nargin < 3 17 | C = nunique(y); 18 | end 19 | % if isOctave 20 | % S = bsxfun(@eq, (1:C).', y.')*X; 21 | % else 22 | S = bsxfun(@eq, sparse(1:C).', y.')*X; 23 | % end 24 | 25 | 26 | end 27 | 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /pmkt3/pdinv.m: -------------------------------------------------------------------------------- 1 | function [Ainv, UC, jitter] = pdinv(A, UC) 2 | 3 | % PDINV Invert a positive definite matrix. 4 | % 5 | % Description: 6 | % 7 | % [AINV, U] = PDINV(A) inverts a positive definite matrix. If the 8 | % matrix isn't quite positive definite the function adds 'jitter' to 9 | % make it positive definite and gives out a warning message (this is 10 | % done through JITCHOL). 11 | % Returns: 12 | % AINV - the inverse of A computed using Cholesky decomposition. 13 | % U - the Cholesky decomposition of A. 14 | % Arguments: 15 | % A - the input positive definite matrix to be inverted. 16 | % 17 | % [AINV, U] = PDINV(A, U) inverts a positive definite matrix given the 18 | % Cholesky decomposition of A. 19 | % Returns: 20 | % AINV - the inverse of A computed using Cholesky decomposition. 21 | % U - the Cholesky decomposition of A. 22 | % Arguments: 23 | % A - the input positive definite matrix to be inverted. 24 | % U - the Cholesky decomposition of A. 25 | % 26 | % [AINV, U, JITTER] = PDINV(A, U) inverts a positive definite matrix 27 | % given the Cholesky decomposition of A. If jitter is used then the 28 | % amount of jitter used is returned. 29 | % Returns: 30 | % AINV - the inverse of A computed using Cholesky decomposition. 31 | % U - the Cholesky decomposition of A. 32 | % JITTER - the amount of jitter added. 33 | % Arguments: 34 | % A - the input positive definite matrix to be inverted. 35 | % U - the Cholesky decomposition of A. 36 | % 37 | % 38 | % See also 39 | % JITCHOL, LOGDET, CHOL 40 | 41 | 42 | % Copyright (c) 2003, 2004, 2005, 2006 Neil D. Lawrence 43 | 44 | 45 | 46 | if nargin < 2 47 | UC=[]; 48 | end 49 | 50 | % Obtain a Cholesky decomposition. 51 | if isempty(UC) 52 | if nargout > 2 53 | [UC, jitter] = jitChol(A); 54 | else 55 | UC = jitChol(A); 56 | end 57 | end 58 | 59 | invU = UC\eye(size(A, 1)); 60 | %invU = eye(size(A, 1))/UC; 61 | Ainv = invU*invU'; -------------------------------------------------------------------------------- /pmkt3/plotKmeans.m: -------------------------------------------------------------------------------- 1 | 2 | % This file is from pmtk3.googlecode.com 3 | 4 | function plotKmeans(data, mu, assign, err, iter) 5 | 6 | fprintf('iteration %d, error %5.4f\n', iter, err); 7 | % mu 8 | K = size(mu,2); 9 | % figure; 10 | symbols = {'r.', 'gx', 'bo', 'ko', 'm.'}; 11 | for k=1:K 12 | %subplot(2,2,iter) 13 | members = find(assign==k); 14 | plot(data(members,1), data(members, 2), symbols{k}, 'markersize', 10); 15 | hold on 16 | plot(mu(1,k), mu(2,k), sprintf('%sx', 'c'), 'markersize', 14, 'linewidth', 3) 17 | grid on 18 | end 19 | title(sprintf('iteration %d, error %5.4f', iter, err)) 20 | if iter==2, printPmtkFigure('kmeansDemoFaithfulIter2'); end 21 | 22 | end -------------------------------------------------------------------------------- /pmkt3/plotfn.m: -------------------------------------------------------------------------------- 1 | function plotfn(model, X, ess, loglik, iter) 2 | post = ess.weights; mu = model.cpd.mu; Sigma = model.cpd.Sigma; 3 | % str = sprintf('iteration %d, loglik %5.4f\n', iter, loglik); 4 | n = size(X, 1); 5 | % colors = [post(:,1), zeros(n, 1), post(:,2)]; % fraction of red and blue 6 | % figure(1); clf; 7 | hold on; 8 | for i=1:n 9 | plot(X(i, 1), X(i, 2), 'ko', 'MarkerSize', 6); 10 | end 11 | % classColors = 'rb'; 12 | K = size(mu,2); 13 | if K==2 14 | classColors = 'rb'; 15 | elseif K==3 16 | classColors = 'rbg'; 17 | end 18 | 19 | for k=1:K 20 | gaussPlot2d(mu(:,k), Sigma(:,:,k), 'color', classColors(k)); 21 | plot(mu(1,k), mu(2,k),'o','linewidth',2, 'color', classColors(k)); 22 | end 23 | % title(str) 24 | axis equal 25 | box on 26 | pause(0.1); 27 | %set(gca, 'YTick', -2:2); 28 | %pause 29 | % fname = sprintf('mixGaussDemo_%s_iter%d', dataset, iter); 30 | % printPmtkFigure(fname); 31 | end -------------------------------------------------------------------------------- /pmkt3/prepareArgs.m: -------------------------------------------------------------------------------- 1 | function out = prepareArgs(args) 2 | % Convert a struct into a name/value cell array for use by process_options 3 | % 4 | % Prepare varargin args for process_options by converting a struct in args{1} 5 | % into a name/value pair cell array. If args{1} is not a struct, args 6 | % is left unchanged. 7 | % Example: 8 | % opts.maxIter = 100; 9 | % opts.verbose = true; 10 | % foo(opts) 11 | % 12 | % This is equivalent to calling 13 | % foo('maxiter', 100, 'verbose', true) 14 | 15 | % This file is from pmtk3.googlecode.com 16 | 17 | 18 | if isstruct(args) 19 | out = interweave(fieldnames(args), struct2cell(args)); 20 | elseif ~isempty(args) && isstruct(args{1}) 21 | out = interweave(fieldnames(args{1}), struct2cell(args{1})); 22 | else 23 | out = args; 24 | end 25 | 26 | 27 | 28 | 29 | end 30 | -------------------------------------------------------------------------------- /pmkt3/printPmtkFigure.m: -------------------------------------------------------------------------------- 1 | function printPmtkFigure(filename, format, printFolder) %#ok 2 | % print current figure to specified file in .pdf (or other) format 3 | 4 | % This file is from pmtk3.googlecode.com 5 | 6 | return; % uncomment this to enable printing 7 | 8 | if nargin <2, format = 'pdf'; end 9 | if nargin < 3, printFolder = []; end 10 | if isempty(printFolder) 11 | if ismac 12 | %printFolder = '/Users/kpmurphy/Dropbox/MLbook/Figures/pdfFigures'; 13 | printFolder = '/Users/kpmurphy/GDrive/Backup/MLbook/book2.0/Figures/pdfFigures'; 14 | else 15 | error('need to specify printFolder') 16 | end 17 | end 18 | if strcmpi(format, 'pdf') 19 | pdfcrop(gcf, 0, 0); 20 | end 21 | fname = sprintf('%s/%s.%s', printFolder, filename, format); 22 | fprintf('printing to %s\n', fname); 23 | if exist(fname,'file'), delete(fname); end % prevent export_fig from appending 24 | if 0 25 | %opts = struct('Color', 'rgb', 'Resolution', 1200, 'fontsize', 12); 26 | opts = struct('Color', 'rgb', 'Resolution', 1200); 27 | exportfig(gcf, fname, opts, 'Format', 'pdf' ); 28 | end 29 | if 0 30 | set(gca,'Color','none') % turn off gray background 31 | set(gcf,'Color','none') 32 | export_fig(fname) 33 | end 34 | if 1 35 | print(gcf, '-dpdf', fname); 36 | end 37 | 38 | end 39 | 40 | -------------------------------------------------------------------------------- /pmkt3/process_options.m: -------------------------------------------------------------------------------- 1 | %% Process named arguments to a function 2 | % 3 | % This allows you to pass in arguments using name, value pairs 4 | % eg func(x, y, 'u', 0, 'v', 1) 5 | % Or you can pass in a struct with named fields 6 | % eg S.u = 0; S.v = 1; func(x, y, S) 7 | % 8 | % Usage: [var1, var2, ..., varn[, unused]] = ... 9 | % process_options(args, ... 10 | % str1, def1, str2, def2, ..., strn, defn) 11 | % 12 | % 13 | % Arguments: 14 | % args - a cell array of input arguments, such 15 | % as that provided by VARARGIN. Its contents 16 | % should alternate between strings and 17 | % values. 18 | % str1, ..., strn - Strings that are associated with a 19 | % particular variable 20 | % def1, ..., defn - Default values returned if no option 21 | % is supplied 22 | % 23 | % Returns: 24 | % var1, ..., varn - values to be assigned to variables 25 | % unused - an optional cell array of those 26 | % string-value pairs that were unused; 27 | % if this is not supplied, then a 28 | % warning will be issued for each 29 | % option in args that lacked a match. 30 | % 31 | % Examples: 32 | % 33 | % Suppose we wish to define a Matlab function 'func' that has 34 | % required parameters x and y, and optional arguments 'u' and 'v'. 35 | % With the definition 36 | % 37 | % function y = func(x, y, varargin) 38 | % 39 | % [u, v] = process_options(varargin, 'u', 0, 'v', 1); 40 | % 41 | % calling func(0, 1, 'v', 2) will assign 0 to x, 1 to y, 0 to u, and 2 42 | % to v. The parameter names are insensitive to case; calling 43 | % func(0, 1, 'V', 2) has the same effect. The function call 44 | % 45 | % func(0, 1, 'u', 5, 'z', 2); 46 | % 47 | % will result in u having the value 5 and v having value 1, but 48 | % will issue a warning that the 'z' option has not been used. On 49 | % the other hand, if func is defined as 50 | % 51 | % function y = func(x, y, varargin) 52 | % 53 | % [u, v, unused_args] = process_options(varargin, 'u', 0, 'v', 1); 54 | % 55 | % then the call func(0, 1, 'u', 5, 'z', 2) will yield no warning, 56 | % and unused_args will have the value {'z', 2}. This behaviour is 57 | % useful for functions with options that invoke other functions 58 | % with options; all options can be passed to the outer function and 59 | % its unprocessed arguments can be passed to the inner function. 60 | % 61 | 62 | % This file is from pmtk3.googlecode.com 63 | 64 | 65 | %PMTKauthor Mark Paskin 66 | %PMTKurl http://ai.stanford.edu/~paskin/software.html 67 | %PMTKmodified Matt Dunham 68 | 69 | % Copyright (C) 2002 Mark A. Paskin 70 | 71 | function [varargout] = process_options(args, varargin) 72 | 73 | args = prepareArgs(args); % added to support structured arguments 74 | % Check the number of input arguments 75 | n = length(varargin); 76 | if (mod(n, 2)) 77 | error('Each option must be a string/value pair.'); 78 | end 79 | 80 | % Check the number of supplied output arguments 81 | if (nargout < (n / 2)) 82 | error('Insufficient number of output arguments given'); 83 | elseif (nargout == (n / 2)) 84 | warn = 1; 85 | nout = n / 2; 86 | else 87 | warn = 0; 88 | nout = n / 2 + 1; 89 | end 90 | 91 | % Set outputs to be defaults 92 | varargout = cell(1, nout); 93 | for i=2:2:n 94 | varargout{i/2} = varargin{i}; 95 | end 96 | 97 | % Now process all arguments 98 | nunused = 0; 99 | for i=1:2:length(args) 100 | found = 0; 101 | for j=1:2:n 102 | if strcmpi(args{i}, varargin{j}) || strcmpi(args{i}(2:end),varargin{j}) 103 | varargout{(j + 1)/2} = args{i + 1}; 104 | found = 1; 105 | break; 106 | end 107 | end 108 | if (~found) 109 | if (warn) 110 | warning(sprintf('Option ''%s'' not used.', args{i})); 111 | args{i} 112 | else 113 | nunused = nunused + 1; 114 | unused{2 * nunused - 1} = args{i}; 115 | unused{2 * nunused} = args{i + 1}; 116 | end 117 | end 118 | end 119 | 120 | % Assign the unused arguments 121 | if (~warn) 122 | if (nunused) 123 | varargout{nout} = unused; 124 | else 125 | varargout{nout} = cell(0); 126 | end 127 | end 128 | 129 | end 130 | -------------------------------------------------------------------------------- /pmkt3/rowvec.m: -------------------------------------------------------------------------------- 1 | function x = rowvec(x) 2 | % Reshape a matrix into a row vector 3 | % Return x as a row vector. This function is useful when a function returns a 4 | % column vector or matrix and you want to immediately reshape it in a functional 5 | % way. Suppose f(a,b,c) returns a column vector, matlab will not let you write 6 | % f(a,b,c)(:)' - you would have to first store the result. With this function you 7 | % can write rowvec(f(a,b,c)) and be assured that the result is a row vector. 8 | 9 | % This file is from pmtk3.googlecode.com 10 | 11 | x = x(:)'; 12 | end 13 | -------------------------------------------------------------------------------- /pmkt3/setMixPrior.m: -------------------------------------------------------------------------------- 1 | 2 | function model = setMixPrior(model, mixPrior) 3 | %% Set the mixture prior 4 | nmix = model.nmix; 5 | if isempty(mixPrior) 6 | model.mixPrior = 2*ones(1, nmix); 7 | % model.mixPrior = 5*ones(1, nmix); 8 | end 9 | if ischar('none') && strcmpi(mixPrior, 'none'); 10 | model.mixPrior = ones(1, nmix); 11 | model.mixPriorFn = @(m)0; 12 | else 13 | model.mixPriorFn = @(m)log(m.mixWeight(:))'*(m.mixPrior(:)-1); 14 | end 15 | if isscalar(model.mixPrior) 16 | model.mixPrior = repmat(model.mixPrior, 1, nmix); 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /pmkt3/shadedErrorBar.m: -------------------------------------------------------------------------------- 1 | function varargout=shadedErrorBar(x,y,errBar,lineProps,transparent) 2 | % function H=shadedErrorBar(x,y,errBar,lineProps,transparent) 3 | % 4 | % Purpose 5 | % Makes a 2-d line plot with a pretty shaded error bar made 6 | % using patch. Error bar color is chosen automatically. 7 | % 8 | % Inputs 9 | % x - vector of x values [optional, can be left empty] 10 | % y - vector of y values or a matrix of n observations by m cases 11 | % where m has length(x); 12 | % errBar - if a vector we draw symmetric errorbars. If it has a size 13 | % of [2,length(x)] then we draw asymmetric error bars with 14 | % row 1 being the upper bar and row 2 being the lower bar 15 | % (with respect to y). ** alternatively ** errBar can be a 16 | % cellArray of two function handles. The first defines which 17 | % statistic the line should be and the second defines the 18 | % error bar. 19 | % lineProps - [optional,'-k' by default] defines the properties of 20 | % the data line. e.g.: 21 | % 'or-', or {'-or','markerfacecolor',[1,0.2,0.2]} 22 | % transparent - [optional, 0 by default] if ==1 the shaded error 23 | % bar is made transparent, which forces the renderer 24 | % to be openGl. However, if this is saved as .eps the 25 | % resulting file will contain a raster not a vector 26 | % image. 27 | % 28 | % Outputs 29 | % H - a structure of handles to the generated plot objects. 30 | % 31 | % 32 | % Examples 33 | % y=randn(30,80); x=1:size(y,2); 34 | % shadedErrorBar(x,mean(y,1),std(y),'g'); 35 | % shadedErrorBar(x,y,{@median,@std},{'r-o','markerfacecolor','r'}); 36 | % shadedErrorBar([],y,{@median,@std},{'r-o','markerfacecolor','r'}); 37 | % 38 | % Overlay two transparent lines 39 | % y=randn(30,80)*10; x=(1:size(y,2))-40; 40 | % shadedErrorBar(x,y,{@mean,@std},'-r',1); 41 | % hold on 42 | % y=ones(30,1)*x; y=y+0.06*y.^2+randn(size(y))*10; 43 | % shadedErrorBar(x,y,{@mean,@std},'-b',1); 44 | % hold off 45 | % 46 | % 47 | % Rob Campbell - November 2009 48 | 49 | 50 | 51 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 52 | % Error checking 53 | error(nargchk(3,5,nargin)) 54 | 55 | 56 | %Process y using function handles if needed to make the error bar 57 | %dynamically 58 | if iscell(errBar) 59 | fun1=errBar{1}; 60 | fun2=errBar{2}; 61 | errBar=fun2(y); 62 | y=fun1(y); 63 | else 64 | y=y(:)'; 65 | end 66 | 67 | if isempty(x) 68 | x=1:length(y); 69 | else 70 | x=x(:)'; 71 | end 72 | 73 | 74 | %Make upper and lower error bars if only one was specified 75 | if length(errBar)==length(errBar(:)) 76 | errBar=repmat(errBar(:)',2,1); 77 | else 78 | s=size(errBar); 79 | f=find(s==2); 80 | if isempty(f), error('errBar has the wrong size'), end 81 | if f==2, errBar=errBar'; end 82 | end 83 | 84 | if length(x) ~= length(errBar) 85 | error('length(x) must equal length(errBar)') 86 | end 87 | 88 | %Set default options 89 | defaultProps={'-k'}; 90 | if nargin<4, lineProps=defaultProps; end 91 | if isempty(lineProps), lineProps=defaultProps; end 92 | if ~iscell(lineProps), lineProps={lineProps}; end 93 | 94 | if nargin<5, transparent=0; end 95 | 96 | 97 | 98 | 99 | 100 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 101 | % Plot to get the parameters of the line 102 | H.mainLine=plot(x,y,lineProps{:}); 103 | 104 | 105 | % Work out the color of the shaded region and associated lines 106 | % Using alpha requires the render to be openGL and so you can't 107 | % save a vector image. On the other hand, you need alpha if you're 108 | % overlaying lines. There we have the option of choosing alpha or a 109 | % de-saturated solid colour for the patch surface . 110 | 111 | col=get(H.mainLine,'color'); 112 | edgeColor=col+(1-col)*0.55; 113 | patchSaturation=0.15; %How de-saturated or transparent to make patch 114 | if transparent 115 | faceAlpha=patchSaturation; 116 | patchColor=col; 117 | set(gcf,'renderer','openGL') 118 | else 119 | faceAlpha=1; 120 | patchColor=col+(1-col)*(1-patchSaturation); 121 | set(gcf,'renderer','painters') 122 | end 123 | 124 | 125 | %Calculate the error bars 126 | uE=y+errBar(1,:); 127 | lE=y-errBar(2,:); 128 | 129 | 130 | %Add the patch error bar 131 | holdStatus=ishold; 132 | if ~holdStatus, hold on, end 133 | 134 | 135 | %Make the patch 136 | yP=[lE,fliplr(uE)]; 137 | xP=[x,fliplr(x)]; 138 | 139 | %remove nans otherwise patch won't work 140 | xP(isnan(yP))=[]; 141 | yP(isnan(yP))=[]; 142 | 143 | 144 | H.patch=patch(xP,yP,1,'facecolor',patchColor,... 145 | 'edgecolor','none',... 146 | 'facealpha',faceAlpha); 147 | 148 | 149 | %Make pretty edges around the patch. 150 | H.edge(1)=plot(x,lE,'-','color',edgeColor); 151 | H.edge(2)=plot(x,uE,'-','color',edgeColor); 152 | 153 | %Now replace the line (this avoids having to bugger about with z coordinates) 154 | delete(H.mainLine) 155 | H.mainLine=plot(x,y,lineProps{:}); 156 | 157 | 158 | if ~holdStatus, hold off, end 159 | 160 | 161 | if nargout==1 162 | varargout{1}=H; 163 | end 164 | -------------------------------------------------------------------------------- /pmkt3/sqDistance.m: -------------------------------------------------------------------------------- 1 | function d = sqDistance(p, q, pSOS, qSOS) 2 | % Efficiently compute squared euclidean distances between sets of vectors 3 | % 4 | % Compute the squared Euclidean distances between every d-dimensional point 5 | % in p to every d-dimensional point in q. Both p and q are 6 | % npoints-by-ndimensions. 7 | % 8 | % d(i, j) = sum((p(i, :) - q(j, :)).^2)s 9 | % 10 | % pSOS = sum(p.^2, 2) and is calculated if not specified 11 | % qSOS = sum(q.^2, 2) and is calculated if not specified 12 | % 13 | %% 14 | 15 | % This file is from pmtk3.googlecode.com 16 | 17 | if(nargin < 4) 18 | pSOS = sum(p.^2, 2); 19 | qSOS = sum(q.^2, 2); 20 | end 21 | d = bsxfun(@plus, pSOS, qSOS') - 2*p*q'; 22 | end 23 | -------------------------------------------------------------------------------- /pmkt3/standardizeCols.m: -------------------------------------------------------------------------------- 1 | function [X, mu, s] = standardizeCols(X, mu, s) 2 | % Make each column of X be zero mean and unit variance 3 | % ie., var(X(:,j))=1 4 | % If mu and s are omitted, it computed from X and returned for use at test time 5 | 6 | % This file is from pmtk3.googlecode.com 7 | 8 | 9 | if nargin < 2, mu = []; s = []; end 10 | [X,mu] = centerCols(X,mu); 11 | [X,s] = mkUnitVariance(X,s); 12 | 13 | end 14 | -------------------------------------------------------------------------------- /pmkt3/structure.m: -------------------------------------------------------------------------------- 1 | function S = structure(varargin) 2 | % Create a struct directly from variables, without having to provide names 3 | % The current names of the variables are used as the structure fields. 4 | % 5 | % *** does not support anonymous variables as in structure(25, 2+3), etc *** 6 | % 7 | %% Example 8 | % 9 | % mu = zeros(1, 10); 10 | % Sigma = randpd(10); 11 | % pi = normalize(ones(1, 10)); 12 | % model = structure(mu, Sigma, pi); 13 | % model 14 | % model = 15 | % mu: [0 0 0 0 0 0 0 0 0 0] 16 | % Sigma: [10x10 double] 17 | % pi: [1x10 double] 18 | %% 19 | 20 | % This file is from pmtk3.googlecode.com 21 | 22 | 23 | for i=1:nargin 24 | S.(inputname(i)) = varargin{i}; 25 | end 26 | 27 | 28 | end 29 | 30 | -------------------------------------------------------------------------------- /startup.m: -------------------------------------------------------------------------------- 1 | % Matlab startup file. 2 | % Include necessary dependencies 3 | display('Running startup.m'); 4 | 5 | base = pwd(); 6 | fs = filesep(); 7 | addpath(pwd); 8 | 9 | % folders to be added by genpath 10 | % gfolders = {'pmkt3', 'data', 'drtoolbox'}; 11 | gfolders = {'pmkt3', 'data'}; 12 | for i=1:length(gfolders) 13 | fol = gfolders{i}; 14 | p = [base , fs, fol]; 15 | fprintf('add gen path: %s\n', p); 16 | addpath(genpath(p)); 17 | end 18 | 19 | % addpath(genpath(fullfile(base, 'real'))); 20 | % addpath(genpath(fullfile(base, 'other'))); 21 | 22 | clear base fs gfolders i fol p 23 | 24 | -------------------------------------------------------------------------------- /testGowalla.m: -------------------------------------------------------------------------------- 1 | % test DP kmeans with Gowalla dataset 2 | % mijung edits kmeansDemo from pmkt3 package 3 | % April 28, 2016 4 | 5 | % clear all; 6 | % close all; 7 | 8 | load('data/Gowalla') 9 | X = Gowalla; 10 | % X = Gowalla(1:107091,:); 11 | % X = standardizeCols(X); 12 | normaliser = 1./sqrt(max((sum(X.^2,2)))); 13 | X = bsxfun(@times, X, normaliser); 14 | 15 | %% 16 | 17 | % Nmat = floor(length(X)*0.01); 18 | Nmat = floor(length(X).*0.01); 19 | % maxseed= 1; 20 | maxseed = 100; 21 | D = 2; 22 | K = 3; 23 | nummethod = 5; 24 | epsilonmat = 10.^linspace(-2, -0.01, 8); 25 | % epsilonmat = 0.01; 26 | errmat_nonpriv = zeros(maxseed,1); 27 | errmat_dplloyd = zeros(maxseed, length(epsilonmat)); 28 | errmat_ourlap = zeros(maxseed, length(epsilonmat)); 29 | errmat_ourgau = zeros(maxseed, length(epsilonmat)); 30 | errmat_dplloyd_cdp = zeros(maxseed, length(epsilonmat)); 31 | epsilon.delta = 1e-4; 32 | epsilon.delta_i = 1e-6; 33 | 34 | % for numSeed= 56; 35 | for numSeed = 1:maxseed 36 | seednum = 2+numSeed; 37 | rng(seednum, 'twister'); 38 | 39 | X = X(randperm(length(X)),:); 40 | fprintf([num2str(numSeed) ' th iteration out of ' num2str(maxseed) ' \n']); 41 | 42 | 43 | for numN = 1: length(Nmat) 44 | N = Nmat(numN); 45 | Xtrain = X(1:N, :); 46 | %% 47 | mu = bsxfun(@plus, lhsdesign(K,2)', [-0.5; -0.5]) + 0.01*randn(D,K); 48 | mu_init = bsxfun(@times, mu, normaliser); 49 | %% 50 | maxIter = 10; 51 | 52 | for whichmeth = 1: nummethod 53 | 54 | epsilon.maxiter = maxIter; 55 | if whichmeth==1 % nonpriv 56 | %% 57 | % whichmeth 58 | epsilon.val=0; 59 | [mu, assign, errHist] = kmeansFit(epsilon, Xtrain, K, 'plotfn', [], ... 60 | 'maxIter', maxIter, 'mu', mu_init); 61 | errmat_nonpriv(numSeed) = errHist(end); 62 | % figure(1); clf; 63 | % subplot(221); plotKmeans(Xtrain, mu, assign, errHist(end), maxIter); 64 | % set(gca, 'xlim', [-1 1], 'ylim', [-1 1]); 65 | %% 66 | elseif whichmeth == 2 % DPLloy 67 | epsilon.method = 1; 68 | epsilon.cdpDPlloyd = 0; 69 | for whicheps = 1: length(epsilonmat) 70 | mu = bsxfun(@plus, lhsdesign(K,2)', [-0.5; -0.5]) + 0.01*randn(D,K); 71 | mu_init = bsxfun(@times, mu, normaliser); 72 | epsilon.val=epsilonmat(whicheps); 73 | [mu, assign, errHist] = kmeansFit(epsilon, Xtrain, K, 'plotfn', [], ... 74 | 'maxIter', maxIter, 'mu', mu_init); 75 | errmat_dplloyd(numSeed, whicheps) = errHist(end); 76 | % subplot(222); plotKmeans(Xtrain, mu, assign, errHist(end), maxIter); 77 | % set(gca, 'xlim', [-1 1], 'ylim', [-1 1]); 78 | end 79 | elseif whichmeth == 3 % ours with Laplace noise 80 | epsilon.method = 2; 81 | epsilon.Lap = 1; 82 | for whicheps = 1: length(epsilonmat) 83 | mu = bsxfun(@plus, lhsdesign(K,2)', [-0.5; -0.5]) + 0.01*randn(D,K); 84 | mu_init = bsxfun(@times, mu, normaliser); 85 | epsilon.val=epsilonmat(whicheps); 86 | [mu, assign, errHist] = kmeansFit(epsilon, Xtrain, K, 'plotfn', [], ... 87 | 'maxIter', maxIter, 'mu', mu_init); 88 | errmat_ourlap(numSeed, whicheps) = errHist(end); 89 | % subplot(223); plotKmeans(Xtrain, mu, assign, errHist(end), maxIter); 90 | % set(gca, 'xlim', [-1 1], 'ylim', [-1 1]); 91 | end 92 | 93 | elseif whichmeth == 4 % ours with Gaussian noise 94 | epsilon.method = 2; 95 | epsilon.Lap = 0; 96 | for whicheps = 1: length(epsilonmat) 97 | mu = bsxfun(@plus, lhsdesign(K,2)', [-0.5; -0.5]) + 0.01*randn(D,K); 98 | mu_init = bsxfun(@times, mu, normaliser); 99 | epsilon.val=epsilonmat(whicheps); 100 | [mu, assign, errHist] = kmeansFit(epsilon, Xtrain, K, 'plotfn', [], ... 101 | 'maxIter', maxIter, 'mu', mu_init); 102 | errmat_ourgau(numSeed, whicheps) = errHist(end); 103 | % subplot(224); plotKmeans(Xtrain, mu, assign, errHist(end), maxIter); 104 | % set(gca, 'xlim', [-1 1], 'ylim', [-1 1]); 105 | end 106 | else % DPLloyd with CDP 107 | 108 | epsilon.method = 1; 109 | epsilon.cdpDPlloyd = 1; 110 | for whicheps = 1: length(epsilonmat) 111 | epsilon.val=epsilonmat(whicheps); 112 | mu = bsxfun(@plus, lhsdesign(K,2)', [-0.5; -0.5]) + 0.01*randn(D,K); 113 | mu_init = bsxfun(@times, mu, normaliser); 114 | [mu, assign, errHist] = kmeansFit(epsilon, Xtrain, K, 'plotfn', [], ... 115 | 'maxIter', maxIter, 'mu', mu_init); 116 | errmat_dplloyd_cdp(numSeed, whicheps) = errHist(end); 117 | % subplot(222); plotKmeans(Xtrain, mu, assign, errHist(end), maxIter); 118 | % set(gca, 'xlim', [-1 1], 'ylim', [-1 1]); 119 | end 120 | end 121 | 122 | end 123 | 124 | % 125 | 126 | end 127 | 128 | end 129 | 130 | %% 131 | 132 | filename = ['matfiles/septestGowalla.mat']; 133 | save(filename, 'errmat_ourgau', 'errmat_ourlap', 'errmat_dplloyd', 'errmat_nonpriv', 'errmat_dplloyd_cdp'); 134 | 135 | -------------------------------------------------------------------------------- /testLifesci.m: -------------------------------------------------------------------------------- 1 | % life science data from UCI data repository 2 | % mijung wrote on October 3, 2016 for aistats submission 3 | 4 | % inputs: 5 | 6 | % (1) total_eps: total privacy loss (budget), e.g., 0.1 7 | % (2) total_del: total privacy tolerance 8 | 9 | % (3) lap: do you want to use laplace mechanism or Gaussian mechanism? 10 | % lap = 1, for LLG scenario 11 | % lap = 0, for GGG scenario 12 | 13 | % (4) comp: composition method 14 | % comp = 0, for non-private 15 | % comp = 1, for linear 16 | % comp = 2, for adv 17 | % comp = 3, for zcdp 18 | % comp = 4, for ma 19 | 20 | function testLifesci(total_eps, total_del, lap, comp) 21 | 22 | 23 | startup; 24 | 25 | seednum = 100; 26 | rng(seednum, 'twister'); 27 | 28 | load('data/lifesci.csv'); 29 | lifesci = lifesci(:, 1:10); 30 | normaliser = 1./max(sqrt((sum(lifesci.^2,2)))); 31 | X = bsxfun(@times, lifesci, normaliser); 32 | 33 | % split data into 10 training/set sets 34 | maxfold = 10; 35 | Indices = crossvalind('Kfold', length(X), maxfold); 36 | 37 | maxseed = 1; 38 | loglik_train = zeros(maxfold, maxseed); 39 | loglik_test = zeros(maxfold, maxseed); 40 | 41 | for iter = 1: maxfold 42 | 43 | Xtest = X(Indices==iter, :); 44 | Xtrain = X(Indices~=iter, :); 45 | 46 | fprintf([num2str(iter) ' th iteration out of ' num2str(maxfold) ' \n']); 47 | 48 | % generate data 49 | D = size(Xtest, 2); 50 | K = 3; 51 | 52 | %% 53 | for seednum = 1:maxseed 54 | 55 | [seednum maxseed] 56 | 57 | rng(seednum, 'twister'); 58 | 59 | %% initialisation of parameters 60 | 61 | initParams.mu = bsxfun(@plus, lhsdesign(K,D)', -0.5*ones(D, 1)) + 0.01*randn(D,K); 62 | initParams.mu = bsxfun(@times, initParams.mu, normaliser); 63 | initParams.Sigma = repmat(eye(D),[1 1 K]); 64 | initParams.mixWeight = normalize(ones(1,K)); 65 | 66 | % conventional EM 67 | maxIter = 10; 68 | epsilon.total_eps = total_eps; 69 | epsilon.total_del = total_del; 70 | epsilon.lap = lap; 71 | epsilon.comp = comp; 72 | 73 | warning('off','all') 74 | 75 | [model, loglikHist] = mixGaussFit(epsilon, Xtrain, K, 'initParams', initParams, ... 76 | 'maxIter', maxIter, 'plotfn', [], 'verbose', false); 77 | 78 | 79 | %% store results 80 | 81 | loglik_train(iter, seednum) = max(loglikHist(end), loglikHist(end-1))/length(Xtrain); 82 | [~, logliktst] = compute_ll_estep(model, Xtest); 83 | loglik_test(iter, seednum) = logliktst/length(Xtest); 84 | 85 | end 86 | 87 | end 88 | 89 | 90 | %% 91 | 92 | filename = ['matfiles/lifesci_G=lap_' num2str(lap) '_epsilon=' num2str(total_eps) '_delta=' num2str(total_del) '_comp=' num2str(comp) '.mat']; 93 | 94 | save(filename, 'loglik_test', 'model'); 95 | 96 | end 97 | 98 | 99 | 100 | 101 | --------------------------------------------------------------------------------