├── README.md ├── distribution.py ├── example.py └── gibbs.py /README.md: -------------------------------------------------------------------------------- 1 | # pydpmm 2 | Direct Gibbs sampling for DPMM using python. 3 | Based on technical report 'Gibbs Sampling Methods for Dirichlet Process Mixture Model' of Xiaodong Yu 4 | 5 | Ref: 6 | Escobar, M. and West, M. (1995), “Bayesian Density Estimation and Inference Using Mixtures,” Journal of the American Statistical Association, 90, pp. 577–588. 7 | Yee Whye Teh(2005), "Hierarchical Dirichlet Processes" 8 | Resamples the hyperparameters of an infinite hmm. https://searchcode.com/codesearch/view/9624224/ -------------------------------------------------------------------------------- /distribution.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2017, Danyang 'Frank' Li 2 | from __future__ import division 3 | import numpy as np 4 | 5 | #fix var = 1 standard normal distribution 6 | #with prior mu conjugate to a normal distribution 7 | class UnivariateGaussian(object): 8 | 9 | 10 | def __init__(self,mu=None): 11 | self.mu = mu 12 | 13 | def rvs(self, size=None): 14 | return np.random.normal(self.mu,1,size) 15 | 16 | def set_mu(self,mu=None): 17 | self.mu = mu 18 | 19 | 20 | def sample_new_mu(self,x): 21 | return np.random.normal(0.5*x, 0.5, 1) 22 | 23 | def log_likelihood(self,x): 24 | x = np.reshape(x, (-1, 1)) 25 | return (-0.5 * (x - self.mu) ** 2 - np.log(2 * np.pi )).ravel() 26 | 27 | @staticmethod 28 | def epsilon_log_univariate_normal(self, mu, sigma): 29 | return np.log(1/(sigma * np.sqrt(2*np.pi))) - mu**2/2*sigma**2 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /example.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2017, Danyang 'Frank' Li 2 | from gibbs import direct_dpmm_gibbs 3 | from gibbs import collapsed_dpmm_gibbs 4 | from matplotlib import pyplot as plt 5 | plt.style.use('ggplot') 6 | 7 | x = [4.0429277,10.71686209,10.73144389,5.05700962,4.70910861,1.38603028,-12.87114683,0.90842492,2.26485196,0.3287409, 1.85740593, -0.08981766, 0.11817958, 0.60973202, 1.88309994, 8 | 1.47112954, 0.77061995, 1.24543065, 1.92506892, 0.7578275, -30.12442321] 9 | 10 | 11 | # mu = np.empty(len(x)); 12 | # loglikelihood = np.empty(len(x)); 13 | # 14 | # gaussian = UnivariateGaussian(mu=2) 15 | # result = test.rvs(10) 16 | # 17 | # 18 | # plt.plot(result) 19 | # plt.show() 20 | 21 | ##SAMPLE THETA 22 | 23 | # for idx,xi in enumerate(x): 24 | # mu[idx] = gaussian.sample_new_mu(xi) 25 | # 26 | # for idx, (x_i, mu_i) in enumerate(zip(x, mu)): 27 | # loglikelihood[idx] = gaussian.log_likelihood(x_i,mu_i) 28 | # 29 | # plt.plot(x) 30 | # plt.show() 31 | 32 | # print(loglikelihood) 33 | 34 | 35 | #sample = gaussian.sample_discrete(loglikelihood) 36 | 37 | ##Direct Gibbs sampling for DPMM 38 | init_K = 5 39 | alpha_prior = {'a':1,'b':2} 40 | observation_prior = {'mu':0,'sigma':10} 41 | 42 | # gibbs = direct_dpmm_gibbs(init_K,x,alpha_prior) 43 | # 44 | # iter = 50 45 | # for i in range(0,iter): 46 | # print('Iter: '+ str(i)) 47 | # gibbs.sample_z() 48 | # gibbs.sample_mu() 49 | # gibbs.sample_alpha_0() 50 | 51 | collapsed_gibbs = collapsed_dpmm_gibbs(init_K,x,alpha_prior,observation_prior) 52 | iter = 50 53 | for i in range(0,iter): 54 | collapsed_gibbs.sample_z() 55 | collapsed_gibbs.sample_alpha_0() 56 | -------------------------------------------------------------------------------- /gibbs.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2017, Danyang 'Frank' Li 2 | 3 | from __future__ import division 4 | from distribution import UnivariateGaussian 5 | import numpy as np 6 | 7 | 8 | 9 | class dpmm_gibbs_base(object): 10 | def __init__(self, init_K=5, x=[], alpha_prior=None): 11 | #Convert python array to numpy array 12 | self.x = np.asarray(x) 13 | self.K = init_K 14 | 15 | self.nn = np.ones(self.K) 16 | self.alpha_prior = alpha_prior 17 | self.alpha_0 = 1 18 | #np.random.gamma(self.alpha_prior['a'],self.alpha_prior['b']) 19 | 20 | # init zz randomly 21 | self.zz = np.random.randint(init_K, size=len(self.x)) 22 | self.mu_0 = 1 23 | self.mu = np.ones(self.K) 24 | self.components = [mixture_component(ss=[], distn=UnivariateGaussian(mu=mu_i)) for mu_i in self.mu] 25 | 26 | for idx, c in enumerate(self.components): 27 | c.ss = self.x[self.zz == idx] 28 | self.nn[idx] = len(c.ss) 29 | 30 | self.n = len(self.x) 31 | 32 | 33 | #TODO Add variance parameter 34 | # fix var = 1 35 | class direct_dpmm_gibbs(dpmm_gibbs_base): 36 | def __init__(self, init_K=5, x=[], alpha_prior=None): 37 | super(direct_dpmm_gibbs, self).__init__(init_K, x, alpha_prior) 38 | 39 | 40 | def new_component_probability(self, x): 41 | # TODO check formula 42 | return (1 / (2 * np.sqrt(np.pi))) * np.exp(- x**2 / 4) 43 | 44 | def new_component_log_integral(self, x): 45 | # TODO check formula 46 | return np.log(2 * np.sqrt(np.pi)) - (x**2/4) 47 | 48 | def sample_z(self): 49 | # STEP 2(d) 50 | # add z_i = new to form a new multi dist 51 | 52 | # Start sample aux indication variable z 53 | for idx, x_i in enumerate(self.x): 54 | kk = self.zz[idx] 55 | # Clean mixture components 56 | temp_zz, = np.where(self.zz == kk) 57 | temp_zz = np.setdiff1d(temp_zz, np.array([idx])) 58 | self.nn[kk] -= 1 59 | temp_ss = self.x[temp_zz] 60 | self.components[kk].ss = temp_ss 61 | if (len(temp_ss) == 0): 62 | #print('component deleted') 63 | self.components = np.delete(self.components, kk) 64 | self.K = len(self.components) 65 | self.nn = np.delete(self.nn, kk) 66 | zz_to_minus_1 = np.where(self.zz > kk) 67 | self.zz[zz_to_minus_1] -= 1 68 | 69 | proportion = np.array([]) 70 | for k in range(0, self.K): 71 | # Calculate proportion for exist mixture component 72 | # Clean mixture components 73 | 74 | n_k = self.nn[k] 75 | #return exp 76 | _proportion = (n_k / (self.n + self.alpha_0 - 1)) * np.exp(self.components[k].distn.log_likelihood(x_i)) 77 | proportion = np.append(proportion, _proportion) 78 | 79 | new_proportion = (self.alpha_0 / (self.n + self.alpha_0 - 1)) * self.new_component_probability(x_i) 80 | 81 | all_propotion = np.append(proportion, new_proportion) 82 | 83 | normailizedAllPropotion = all_propotion / sum(all_propotion) 84 | 85 | sample_z = np.random.multinomial(1, normailizedAllPropotion, size=1) 86 | 87 | z_index = np.where(sample_z == 1)[1][0] 88 | 89 | self.zz[idx] = z_index 90 | 91 | # found new component 92 | if (z_index == self.K): 93 | self.K += 1 94 | # sample new mu for new component 95 | # G_0 = n(0,1) 96 | new_mu = np.random.normal(0.5 * x_i, 0.5, 1); 97 | 98 | new_component = mixture_component(ss=[x_i], distn=UnivariateGaussian(mu=new_mu)) 99 | 100 | self.components = np.append(self.components, new_component) 101 | self.nn = np.append(self.nn, 1) 102 | 103 | #print 'new component added' 104 | 105 | # add data to exist component 106 | else: 107 | self.components[z_index].ss = np.append(self.components[z_index].ss, x_i) 108 | self.nn[z_index] += 1 109 | 110 | for component in self.components: 111 | component.print_self() 112 | print('alpha -> ' + str(self.alpha_0)) 113 | 114 | def sample_mu(self): 115 | 116 | for k in range(0, self.K): 117 | x_k = self.components[k].ss 118 | mu_k = np.random.normal((self.mu_0 + sum(x_k))/(1+len(x_k)), 1/(1 + len(x_k)), 1) 119 | self.components[k].distn.set_mu(mu=mu_k) 120 | #print('new mu -> ' + str(mu_k[0])) 121 | 122 | def sample_alpha_0(self): 123 | #Escobar and West 1995 124 | eta = np.random.beta(self.alpha_0 + 1,self.n,1) 125 | #Teh HDP 2005 126 | #construct the mixture model 127 | pi = self.n/self.alpha_0 128 | pi = pi/(1+pi) 129 | s = np.random.binomial(1,pi,1) 130 | #sample from a two gamma mixture models 131 | self.alpha_0 = np.random.gamma(self.alpha_prior['a'] + self.K - s, 1/(self.alpha_prior['b'] - np.log(eta)), 1) 132 | 133 | 134 | class collapsed_dpmm_gibbs(dpmm_gibbs_base): 135 | def __init__(self, init_K=5, x=[], alpha_prior = None, observation_prior=None,): 136 | super(collapsed_dpmm_gibbs, self).__init__(init_K, x, alpha_prior) 137 | self.observation_prior = observation_prior 138 | 139 | #add a new empty component 140 | new_mu = np.random.normal(self.observation_prior['mu'], self.observation_prior['sigma'], 1); 141 | new_component = mixture_component(ss=[], distn=UnivariateGaussian(mu=new_mu)) 142 | self.components = np.append(self.components, new_component) 143 | 144 | #print (UnivariateGaussian.epsilon_log_univariate_normal(self,-12,2) - UnivariateGaussian.epsilon_log_univariate_normal(self,1,1)) 145 | 146 | 147 | def sample_z(self): 148 | for idx, x_i in enumerate(self.x): 149 | 150 | kk = self.zz[idx] 151 | # Clean mixture components 152 | temp_zz, = np.where(self.zz == kk) 153 | # print('----') 154 | # print len(self.components[kk].ss) 155 | temp_zz = np.setdiff1d(temp_zz,np.array([idx])) 156 | self.nn[kk] -= 1 157 | temp_ss = self.x[temp_zz] 158 | #print len(temp_ss) 159 | self.components[kk].ss = temp_ss 160 | if (len(temp_ss) == 0): 161 | #print('component deleted') 162 | #print(len(self.components)) 163 | self.components = np.delete(self.components, kk) 164 | #print(len(self.components)) 165 | self.K = len(self.components) 166 | self.nn = np.delete(self.nn,kk) 167 | zz_to_minus_1 = np.where(self.zz > kk) 168 | self.zz[zz_to_minus_1] -= 1 169 | 170 | pp = np.log(np.append(self.nn, self.alpha_0)) 171 | 172 | for k in range(0, self.K): 173 | pp[k] = pp[k] + self.log_predictive(self.components[k],x_i) 174 | print(self.log_predictive(self.components[k],x_i)) 175 | pp = np.exp(pp - np.max(pp)) 176 | pp = pp/np.sum(pp) 177 | sample_z = np.random.multinomial(1, pp, size=1) 178 | print x_i 179 | 180 | 181 | z_index = np.where(sample_z == 1)[1][0] 182 | self.zz[idx] = z_index 183 | 184 | if(z_index == len(self.components) - 1): 185 | print('component added') 186 | new_mu = np.random.normal(0.5 * x_i, 0.5, 1); 187 | 188 | new_component = mixture_component(ss=[x_i], distn=UnivariateGaussian(mu=new_mu)) 189 | 190 | self.components = np.append(self.components, new_component) 191 | self.K = len(self.components) 192 | self.nn = np.append(self.nn, 1) 193 | else: 194 | self.components[z_index].ss = np.append(self.components[z_index].ss, x_i) 195 | self.nn[z_index] += 1 196 | 197 | print '----Summary----' 198 | # print self.zz 199 | # print self.nn 200 | # for component in self.components: 201 | # component.print_self() 202 | 203 | def sample_alpha_0(self): 204 | #Escobar and West 1995 205 | eta = np.random.beta(self.alpha_0 + 1,self.n,1) 206 | #Teh HDP 2005 207 | #construct the mixture model 208 | pi = self.n/self.alpha_0 209 | pi = pi/(1+pi) 210 | s = np.random.binomial(1,pi,1) 211 | #sample from a two gamma mixture models 212 | self.alpha_0 = np.random.gamma(self.alpha_prior['a'] + self.K - s, 1/(self.alpha_prior['b'] - np.log(eta)), 1) 213 | print self.alpha_0 214 | 215 | def log_predictive(self,component, x_i): 216 | ll = UnivariateGaussian.epsilon_log_univariate_normal(self, self.observation_prior['mu'] + np.sum(component.get_ss()) + x_i ,\ 217 | self.observation_prior['sigma'] + component.get_n_k_minus_i() + 1) - \ 218 | UnivariateGaussian.epsilon_log_univariate_normal(self, self.observation_prior['mu'] + np.sum(component.get_ss()), \ 219 | self.observation_prior['sigma'] + component.get_n_k_minus_i()) 220 | return ll 221 | 222 | 223 | 224 | class mixture_component(object): 225 | def __init__(self, ss, distn): 226 | self.ss = ss 227 | self.distn = distn 228 | 229 | if(len(ss)> 0): 230 | self.n_k_minus_i = len(ss) - 1 231 | else: 232 | self.n_k_minus_i = 0 233 | 234 | def get_n_k_minus_i(self): 235 | if (len(self.ss) > 1): 236 | self.n_k_minus_i = len(self.ss) - 1 237 | else: 238 | self.n_k_minus_i = 0 239 | return self.n_k_minus_i 240 | def get_ss(self): 241 | return self.ss 242 | 243 | def print_self(self): 244 | print(self.ss) 245 | print('Mu: '+ str(self.distn.mu)) --------------------------------------------------------------------------------