├── .gitignore ├── GPclust ├── MOG.py ├── MOHGP.py ├── OMGP.py ├── __init__.py ├── collapsed_mixture.py ├── collapsed_vb.py ├── np_utilities.py └── utilities.py ├── LICENSE.txt ├── README.md ├── data ├── kalinka09.csv ├── kalinka09_pdata.csv └── split_data_test.csv ├── notebooks ├── MOG_demo.py ├── MOHGP_demo.ipynb ├── MOHGP_demo.py ├── MOHGP_sine_demo.py ├── OMGP_demo.ipynb ├── drosophila.ipynb ├── index.ipynb ├── mixture of Gaussians.ipynb └── twoD_clustering_example.numpy ├── setup.cfg └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | 3 | # C extensions 4 | *.so 5 | 6 | # Packages 7 | *.egg 8 | *.egg-info 9 | dist 10 | build 11 | eggs 12 | parts 13 | bin 14 | var 15 | sdist 16 | develop-eggs 17 | .installed.cfg 18 | lib 19 | lib64 20 | 21 | # Installer logs 22 | pip-log.txt 23 | 24 | # Unit test / coverage reports 25 | .coverage 26 | .tox 27 | nosetests.xml 28 | 29 | # Translations 30 | *.mo 31 | 32 | # Mr Developer 33 | .mr.developer.cfg 34 | .project 35 | .pydevproject 36 | 37 | #various editors, including Kate 38 | *~ 39 | -------------------------------------------------------------------------------- /GPclust/MOG.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2012, 2013, 2014 James Hensman 2 | # Licensed under the GPL v3 (see LICENSE.txt) 3 | 4 | import numpy as np 5 | 6 | try: 7 | from .utilities import multiple_pdinv, lngammad 8 | except ImportError: 9 | from .np_utilities import multiple_pdinv, lngammad 10 | 11 | from scipy.special import gammaln, digamma 12 | from scipy import stats 13 | from .collapsed_mixture import CollapsedMixture 14 | 15 | class MOG(CollapsedMixture): 16 | """ 17 | A Mixture of Gaussians, using the fast variational framework 18 | 19 | Arguments 20 | ========= 21 | X - a np.array of the observed data: each row contains one datum. 22 | K - the number of clusters (or initial number of clusters in the Dirichlet Process case) 23 | alpha - the A priori dirichlet concentrationn parameter (default 1.) 24 | prior_Z - either 'symmetric' or 'DP', specifies whether to use a symmetric Dirichlet prior for the clusters, or a (truncated) Dirichlet process. 25 | name - a convenient string for printing the model (default MOG) 26 | 27 | Optional arguments for the parameters of the Gaussian-Wishart priors on the clusters 28 | prior_m - prior mean (defaults to mean of the data) 29 | prior_kappa - prior connectivity (default 1e-6) 30 | prior_S - prior Wishart covariance (defaults to 1e-3 * I) 31 | prior_v - prior Wishart degrees of freedom (defaults to dimension of the problem +1.) 32 | 33 | """ 34 | def __init__(self, X, K=2, prior_Z='symmetric', alpha=1., prior_m=None, prior_kappa=1e-6, prior_S=None, prior_v=None, name='MOG'): 35 | self.X = X 36 | self.N, self.D = X.shape 37 | 38 | # store the prior cluster parameters 39 | self.m0 = self.X.mean(0) if prior_m is None else prior_m 40 | self.k0 = prior_kappa 41 | self.S0 = np.eye(self.D)*1e-3 if prior_S is None else prior_S 42 | self.v0 = prior_v or self.D+1. 43 | 44 | #precomputed stuff 45 | self.k0m0m0T = self.k0*self.m0[:,np.newaxis]*self.m0[np.newaxis,:] 46 | self.XXT = self.X[:,:,np.newaxis]*self.X[:,np.newaxis,:] 47 | self.S0_halflogdet = np.sum(np.log(np.sqrt(np.diag(np.linalg.cholesky(self.S0))))) 48 | 49 | CollapsedMixture.__init__(self, self.N, K, prior_Z, alpha, name=name) 50 | self.do_computations() 51 | 52 | def do_computations(self): 53 | #computations needed for bound, gradient and predictions 54 | self.kNs = self.phi_hat + self.k0 55 | self.vNs = self.phi_hat + self.v0 56 | self.Xsumk = np.tensordot(self.X,self.phi,((0),(0))) #D x K 57 | Ck = np.tensordot(self.phi, self.XXT,((0),(0))).T# D x D x K 58 | self.mun = (self.k0*self.m0[:,None] + self.Xsumk)/self.kNs[None,:] # D x K 59 | self.munmunT = self.mun[:,None,:]*self.mun[None,:,:] 60 | self.Sns = self.S0[:,:,None] + Ck + self.k0m0m0T[:,:,None] - self.kNs[None,None,:]*self.munmunT 61 | self.Sns_inv, self.Sns_halflogdet = multiple_pdinv(self.Sns) 62 | 63 | def bound(self): 64 | """Compute the lower bound on the model evidence. """ 65 | return -0.5*self.D*np.sum(np.log(self.kNs/self.k0))\ 66 | +self.K*self.v0*self.S0_halflogdet - np.sum(self.vNs*self.Sns_halflogdet)\ 67 | +np.sum(lngammad(self.vNs, self.D))- self.K*lngammad(self.v0, self.D)\ 68 | +self.mixing_prop_bound()\ 69 | +self.H\ 70 | -0.5*self.N*self.D*np.log(np.pi) 71 | 72 | def vb_grad_natgrad(self): 73 | """Gradients of the bound""" 74 | x_m = self.X[:,:,None]-self.mun[None,:,:] 75 | dS = x_m[:,:,None,:]*x_m[:,None,:,:] 76 | SnidS = self.Sns_inv[None,:,:,:]*dS 77 | dlndtS_dphi = np.dot(np.ones(self.D), np.dot(np.ones(self.D), SnidS)) 78 | 79 | grad_phi = (-0.5*self.D/self.kNs + 0.5*digamma((self.vNs-np.arange(self.D)[:,None])/2.).sum(0) + self.mixing_prop_bound_grad() - self.Sns_halflogdet -1.) + (self.Hgrad-0.5*dlndtS_dphi*self.vNs) 80 | 81 | natgrad = grad_phi - np.sum(self.phi*grad_phi, 1)[:,None] # corrects for softmax (over) parameterisation 82 | grad = natgrad*self.phi 83 | 84 | return grad.flatten(), natgrad.flatten() 85 | 86 | 87 | def predict_components_ln(self, Xnew): 88 | """The log predictive density under each component at Xnew""" 89 | Dist = Xnew[:,:,np.newaxis]-self.mun[np.newaxis,:,:] # Nnew x D x K 90 | tmp = np.sum(Dist[:,:,None,:]*self.Sns_inv[None,:,:,:],1)#*(kn+1.)/(kn*(vn-self.D+1.)) 91 | mahalanobis = np.sum(tmp*Dist, 1)/(self.kNs+1.)*self.kNs*(self.vNs-self.D+1.) 92 | halflndetSigma = self.Sns_halflogdet + 0.5*self.D*np.log((self.kNs+1.)/(self.kNs*(self.vNs-self.D+1.))) 93 | 94 | Z = gammaln(0.5*(self.vNs[np.newaxis,:]+1.))\ 95 | -gammaln(0.5*(self.vNs[np.newaxis,:]-self.D+1.))\ 96 | -(0.5*self.D)*(np.log(self.vNs[np.newaxis,:]-self.D+1.) + np.log(np.pi))\ 97 | - halflndetSigma \ 98 | - (0.5*(self.vNs[np.newaxis,:]+1.))*np.log(1.+mahalanobis/(self.vNs[np.newaxis,:]-self.D+1.)) 99 | return Z 100 | 101 | def predict_components(self, Xnew): 102 | """The predictive density under each component at Xnew""" 103 | return np.exp(self.predict_components_ln(Xnew)) 104 | 105 | def predict(self, Xnew): 106 | """The predictive density of the model at Xnew""" 107 | Z = self.predict_components(Xnew) 108 | #calculate the weights for each component 109 | phi_hat = self.phi.sum(0) 110 | pi = phi_hat+self.alpha 111 | pi /= pi.sum() 112 | Z *= pi[np.newaxis,:] 113 | return Z.sum(1) 114 | 115 | def plot(self, newfig=True): 116 | from matplotlib import pyplot as plt 117 | if self.X.shape[1]==2: 118 | if newfig:plt.figure() 119 | xmin, ymin = self.X.min(0) 120 | xmax, ymax = self.X.max(0) 121 | xmin, xmax = xmin-0.1*(xmax-xmin), xmax+0.1*(xmax-xmin) 122 | ymin, ymax = ymin-0.1*(ymax-ymin), ymax+0.1*(ymax-ymin) 123 | xx, yy = np.mgrid[xmin:xmax:100j, ymin:ymax:100j] 124 | Xgrid = np.vstack((xx.flatten(), yy.flatten())).T 125 | zz = self.predict(Xgrid).reshape(100, 100) 126 | zz_data = self.predict(self.X) 127 | plt.contour(xx, yy, zz, [stats.scoreatpercentile(zz_data, 5)], colors='k', linewidths=3) 128 | plt.scatter(self.X[:,0], self.X[:,1], 30, np.argmax(self.phi, 1), linewidth=0, cmap=plt.cm.gist_rainbow) 129 | 130 | zz_components = self.predict_components(Xgrid) 131 | phi_hat = self.phi.sum(0) 132 | pi = phi_hat+self.alpha 133 | pi /= pi.sum() 134 | zz_components *= pi[np.newaxis,:] 135 | [plt.contour(xx, yy, zz.reshape(100, 100), [stats.scoreatpercentile(zz_data, 5.)], colors='k', linewidths=1) for zz in zz_components.T] 136 | else: 137 | print("plotting only for 2D mixtures") 138 | 139 | -------------------------------------------------------------------------------- /GPclust/MOHGP.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2012, 2013, 2014 James Hensman 2 | # Licensed under the GPL v3 (see LICENSE.txt) 3 | 4 | import numpy as np 5 | from .collapsed_mixture import CollapsedMixture 6 | import GPy 7 | from GPy.util.linalg import mdot, pdinv, backsub_both_sides, dpotrs, jitchol, dtrtrs 8 | 9 | try: 10 | from .utilities import multiple_mahalanobis 11 | except ImportError: 12 | from .np_utilities import multiple_mahalanobis_numpy_loops as multiple_mahalanobis 13 | 14 | class MOHGP(CollapsedMixture): 15 | """ 16 | A Hierarchical Mixture of Gaussian Processes 17 | 18 | A hierarchy is formed by using a GP model the mean function of a cluster, 19 | and further GPs to model the deviation of each time-course in the cluster 20 | from the mean. 21 | 22 | Arguments 23 | ========= 24 | X - The times of observation of the time series in a (Tx1) np.array 25 | Y - A np.array of the observed time-course values: each row contains a time series, each column represents a unique time point 26 | kernF - A GPy kernel to model the mean function of each cluster 27 | kernY - A GPy kernel to model the deviation of each of the time courses from the mean fro teh cluster 28 | alpha - The a priori Dirichlet concentrationn parameter (default 1.) 29 | prior_Z - Either 'symmetric' or 'dp', specifies whether to use a symmetric dirichelt prior for the clusters, or a (truncated) Dirichlet Process. 30 | name - A convenient string for printing the model (default MOHGP) 31 | 32 | """ 33 | def __init__(self, X, kernF, kernY, Y, K=2, alpha=1., prior_Z='symmetric', name='MOHGP'): 34 | 35 | N, self.D = Y.shape 36 | self.Y = Y 37 | self.X = X 38 | assert X.shape[0]==self.D, "input data don't match observations" 39 | 40 | CollapsedMixture.__init__(self, N, K, prior_Z, alpha, name) 41 | 42 | self.kernF = kernF 43 | self.kernY = kernY 44 | self.link_parameters(self.kernF, self.kernY) 45 | 46 | #initialize kernels 47 | self.Sf = self.kernF.K(self.X) 48 | self.Sy = self.kernY.K(self.X) 49 | self.Sy_inv, self.Sy_chol, self.Sy_chol_inv, self.Sy_logdet = pdinv(self.Sy+np.eye(self.D)*1e-6) 50 | 51 | #Computations that can be done outside the optimisation loop 52 | self.YYT = self.Y[:,:,np.newaxis]*self.Y[:,np.newaxis,:] 53 | self.YTY = np.dot(self.Y.T,self.Y) 54 | 55 | self.do_computations() 56 | 57 | 58 | def parameters_changed(self): 59 | """ Set the kernel parameters. Note that the variational parameters are handled separately.""" 60 | #get the latest kernel matrices, decompose 61 | self.Sf = self.kernF.K(self.X) 62 | self.Sy = self.kernY.K(self.X) 63 | self.Sy_inv, self.Sy_chol, self.Sy_chol_inv, self.Sy_logdet = pdinv(self.Sy+np.eye(self.D)*1e-6) 64 | 65 | #update everything 66 | self.do_computations() 67 | self.update_kern_grads() 68 | 69 | 70 | def do_computations(self): 71 | """ 72 | Here we do all the computations that are required whenever the kernels 73 | or the variational parameters are changed. 74 | """ 75 | #sufficient stats. 76 | self.ybark = np.dot(self.phi.T,self.Y).T 77 | 78 | # compute posterior variances of each cluster (lambda_inv) 79 | tmp = backsub_both_sides(self.Sy_chol, self.Sf, transpose='right') 80 | self.Cs = [np.eye(self.D) + tmp*phi_hat_i for phi_hat_i in self.phi_hat] 81 | 82 | self._C_chols = [jitchol(C) for C in self.Cs] 83 | self.log_det_diff = np.array([2.*np.sum(np.log(np.diag(L))) for L in self._C_chols]) 84 | tmp = [dtrtrs(L, self.Sy_chol.T, lower=1)[0] for L in self._C_chols] 85 | self.Lambda_inv = np.array([( self.Sy - np.dot(tmp_i.T, tmp_i) )/phi_hat_i if (phi_hat_i>1e-6) else self.Sf for phi_hat_i, tmp_i in zip(self.phi_hat, tmp)]) 86 | 87 | #posterior mean and other useful quantities 88 | self.Syi_ybark, _ = dpotrs(self.Sy_chol, self.ybark, lower=1) 89 | self.Syi_ybarkybarkT_Syi = self.Syi_ybark.T[:,None,:]*self.Syi_ybark.T[:,:,None] 90 | self.muk = (self.Lambda_inv*self.Syi_ybark.T[:,:,None]).sum(1).T 91 | 92 | def update_kern_grads(self): 93 | """ 94 | Set the derivative of the lower bound wrt the (kernel) parameters 95 | """ 96 | 97 | tmp = [dtrtrs(L, self.Sy_chol_inv, lower=1)[0] for L in self._C_chols] 98 | B_invs = [phi_hat_i*np.dot(tmp_i.T, tmp_i) for phi_hat_i, tmp_i in zip(self.phi_hat, tmp)] 99 | #B_invs = [phi_hat_i*mdot(self.Sy_chol_inv.T,Ci,self.Sy_chol_inv) for phi_hat_i, Ci in zip(self.phi_hat,self.C_invs)] 100 | 101 | #heres the mukmukT*Lambda term 102 | LiSfi = [np.eye(self.D)-np.dot(self.Sf,Bi) for Bi in B_invs]#seems okay 103 | tmp1 = [np.dot(LiSfik.T,Sy_inv_ybark_k) for LiSfik, Sy_inv_ybark_k in zip(LiSfi,self.Syi_ybark.T)] 104 | tmp = 0.5*sum([np.dot(tmpi[:,None],tmpi[None,:]) for tmpi in tmp1]) 105 | 106 | #here's the difference in log determinants term 107 | tmp += -0.5*sum(B_invs) 108 | 109 | #kernF_grads = np.array([np.sum(tmp*g) for g in self.kernF.extract_gradients()]) # OKAY! 110 | self.kernF.update_gradients_full(dL_dK=tmp,X=self.X) 111 | 112 | #gradient wrt Sigma_Y 113 | ybarkybarkT = self.ybark.T[:,None,:]*self.ybark.T[:,:,None] 114 | Byks = [np.dot(Bi,yk) for Bi,yk in zip(B_invs,self.ybark.T)] 115 | tmp = sum([np.dot(Byk[:,None],Byk[None,:])/np.power(ph_k,3)\ 116 | -Syi_ybarkybarkT_Syi/ph_k -Bi/ph_k for Bi, Byk, yyT, ph_k, Syi_ybarkybarkT_Syi in zip(B_invs, Byks, ybarkybarkT, self.phi_hat, self.Syi_ybarkybarkT_Syi) if ph_k >1e-6]) 117 | tmp += (self.K - self.N) * self.Sy_inv 118 | tmp += mdot(self.Sy_inv,self.YTY,self.Sy_inv) 119 | tmp /= 2. 120 | 121 | #kernY_grads = np.array([np.sum(tmp*g) for g in self.kernY.extract_gradients()]) 122 | self.kernY.update_gradients_full(dL_dK=tmp, X=self.X) 123 | 124 | def bound(self): 125 | """ 126 | Compute the lower bound on the marginal likelihood (conditioned on the 127 | GP hyper parameters). 128 | """ 129 | return -0.5 * ( self.N * self.D * np.log(2. * np.pi) \ 130 | + self.log_det_diff.sum() \ 131 | + self.N * self.Sy_logdet \ 132 | + np.sum(self.YTY * self.Sy_inv) ) \ 133 | + 0.5 * np.sum(self.Syi_ybarkybarkT_Syi * self.Lambda_inv) \ 134 | + self.mixing_prop_bound() \ 135 | + self.H 136 | 137 | def vb_grad_natgrad(self): 138 | """ 139 | Natural Gradients of the bound with respect to phi, the variational 140 | parameters controlling assignment of the data to clusters 141 | """ 142 | #yn_mk = self.Y[:,:,None] - self.muk[None,:,:] 143 | #ynmk2 = np.sum(np.dot(self.Sy_inv, yn_mk) * np.rollaxis(yn_mk,0,2),0) 144 | ynmk2 = multiple_mahalanobis(self.Y, self.muk.T, self.Sy_chol) 145 | 146 | grad_phi = (self.mixing_prop_bound_grad() - 147 | 0.5 * np.sum(np.sum(self.Lambda_inv * self.Sy_inv[None, :, :], 1), 1)) + \ 148 | ( self.Hgrad - 0.5 * ynmk2 ) # parentheses are for operation ordering! 149 | 150 | natgrad = grad_phi - np.sum(self.phi*grad_phi,1)[:,None] 151 | grad = natgrad*self.phi 152 | 153 | return grad.flatten(), natgrad.flatten() 154 | 155 | 156 | def predict_components(self,Xnew): 157 | """The predictive density under each component""" 158 | 159 | tmp = [dtrtrs(L, self.Sy_chol_inv, lower=1)[0] for L in self._C_chols] 160 | B_invs = [phi_hat_i*np.dot(tmp_i.T, tmp_i) for phi_hat_i, tmp_i in zip(self.phi_hat, tmp)] 161 | kx= self.kernF.K(self.X,Xnew) 162 | try: 163 | kxx = self.kernF.K(Xnew) + self.kernY.K(Xnew) 164 | except TypeError: 165 | #kernY has a hierarchical structure that we should deal with 166 | con = np.ones((Xnew.shape[0],self.kernY.connections.shape[1])) 167 | kxx = self.kernF.K(Xnew) + self.kernY.K(Xnew,con) 168 | 169 | #prediction as per my notes 170 | tmp = [np.eye(self.D) - np.dot(Bi,self.Sf) for Bi in B_invs] 171 | mu = [mdot(kx.T,tmpi,self.Sy_inv,ybark) for tmpi,ybark in zip(tmp,self.ybark.T)] 172 | var = [kxx - mdot(kx.T,Bi,kx) for Bi in B_invs] 173 | 174 | return mu,var 175 | 176 | def plot_simple(self): 177 | from matplotlib import pyplot as plt 178 | assert self.X.shape[1]==1, "can only plot mixtures of 1D functions" 179 | #plt.figure() 180 | plt.plot(self.Y.T,'k',linewidth=0.5,alpha=0.4) 181 | plt.plot(self.muk[:,self.phi_hat>1e-3],'k',linewidth=2) 182 | 183 | def plot(self, on_subplots=True, colour=False, newfig=True, errorbars=False, in_a_row=False, joined=True, gpplot=True,min_in_cluster=1e-3, data_in_grey=False, numbered=True, data_in_replicate=False, fixed_inputs=[], ylim=None): 184 | """ 185 | Plot the mixture of Gaussian processes. Some of these arguments are rather esoteric! The defaults should be okay for most cases. 186 | 187 | Arguments 188 | --------- 189 | 190 | on_subplots (bool) whether to plot all the clusters on separate subplots (True), or all on the same plot (False) 191 | colour (bool) to cycle through colours (True) or plot in black nd white (False) 192 | newfig (bool) whether to make a new matplotlib figure(True) or use the current figure (False) 193 | in_a_row (bool) if true, plot the subplots (if using) in a single row. Else make the subplots approximately square. 194 | joined (bool) if true, connect the data points with lines 195 | gpplot (bool) if true, plto the posterior of the GP for each cluster. 196 | min_in_cluster (float) ignore clusterse with less total assignemnt than this 197 | data_in_grey (bool) whether the data should be plotted in black and white. 198 | numbered (bool) whether to include numbers on the top-right of each subplot. 199 | data_in_replicate (bool) whether to assume the data are in replicate, and plot the mean of each replicate instead of each data point 200 | fixed_inputs (list of tuples, as GPy.GP.plot) for GPs defined on more that one input, we'll plot a slice of the GP. this list defines how to fix the remaining inputs. 201 | ylim (tuple) the limits to set on the y-axes. 202 | 203 | """ 204 | 205 | from matplotlib import pyplot as plt 206 | 207 | #work out what input dimensions to plot 208 | fixed_dims = np.array([i for i,v in fixed_inputs]) 209 | free_dims = np.setdiff1d(np.arange(self.X.shape[1]),fixed_dims) 210 | assert len(free_dims)==1, "can only plot mixtures of 1D functions" 211 | 212 | #figure, subplots 213 | if newfig: 214 | fig = plt.figure() 215 | else: 216 | fig = plt.gcf() 217 | Tango = GPy.plotting.Tango 218 | Tango.reset() 219 | 220 | if data_in_replicate: 221 | X_ = self.X[:, free_dims].flatten() 222 | X = np.unique(X_).reshape(-1,1) 223 | Y = np.vstack([self.Y[:,X_==x].mean(1) for x in X.flatten()]).T 224 | 225 | else: 226 | Y = self.Y 227 | X = self.X[:,free_dims] 228 | 229 | #find some sensible y-limits for the plotting 230 | if ylim is None: 231 | ymin,ymax = Y.min(), Y.max() 232 | ymin,ymax = ymin-0.1*(ymax-ymin), ymax+0.1*(ymax-ymin) 233 | else: 234 | ymin, ymax = ylim 235 | 236 | #work out how many clusters we're going to plot. 237 | Ntotal = np.sum(self.phi_hat > min_in_cluster) 238 | if on_subplots: 239 | if in_a_row: 240 | Nx = 1 241 | Ny = Ntotal 242 | else: 243 | Nx = np.floor(np.sqrt(Ntotal)) 244 | Ny = int(np.ceil(Ntotal/Nx)) 245 | Nx = int(Nx) 246 | else: 247 | ax = plt.gca() # this seems to make new ax if needed 248 | 249 | #limits of GPs 250 | xmin,xmax = X.min(), X.max() 251 | xmin,xmax = xmin-0.1*(xmax-xmin), xmax+0.1*(xmax-xmin) 252 | 253 | Xgrid = np.empty((300,self.X.shape[1])) 254 | Xgrid[:,free_dims] = np.linspace(xmin,xmax,300)[:,None] 255 | for i,v in fixed_inputs: 256 | Xgrid[:,i] = v 257 | 258 | 259 | subplot_count = 0 260 | for i, ph, mu, var in zip(range(self.K), self.phi_hat, *self.predict_components(Xgrid)): 261 | if ph>(min_in_cluster): 262 | ii = np.argmax(self.phi,1)==i 263 | num_in_clust = np.sum(ii) 264 | if not np.any(ii): 265 | continue 266 | if on_subplots: 267 | ax = fig.add_subplot(Nx,Ny,subplot_count+1) 268 | subplot_count += 1 269 | if colour: 270 | col = Tango.nextMedium() 271 | else: 272 | col='k' 273 | if joined: 274 | if data_in_grey: 275 | ax.plot(X,Y[ii].T,'k',marker=None, linewidth=0.2,alpha=0.4) 276 | else: 277 | ax.plot(X,Y[ii].T,col,marker=None, linewidth=0.2,alpha=1) 278 | else: 279 | if data_in_grey: 280 | ax.plot(X,Y[ii].T,'k',marker='.', linewidth=0.0,alpha=0.4) 281 | else: 282 | ax.plot(X,Y[ii].T,col,marker='.', linewidth=0.0,alpha=1) 283 | 284 | if gpplot: 285 | _ = None 286 | helper_data = [_, free_dims, Xgrid, _, _, _, _, _] 287 | helper_prediction = [mu[:,None], [(mu-2.*np.sqrt(np.diag(var)))[:,None],(mu+2.*np.sqrt(np.diag(var)))[:,None]], _] 288 | GPy.plotting.gpy_plot.gp_plots._plot_mean(None, ax, helper_data, helper_prediction, color=col) 289 | GPy.plotting.gpy_plot.gp_plots._plot_confidence(None, ax, helper_data, helper_prediction, None, color=col) 290 | 291 | 292 | if numbered and on_subplots: 293 | ax.text(1,1,str(int(num_in_clust)),transform=ax.transAxes,ha='right',va='top',bbox={'ec':'k','lw':1.3,'fc':'w'}) 294 | 295 | err = 2*np.sqrt(np.diag(self.Lambda_inv[i,:,:])) 296 | if errorbars:ax.errorbar(self.X.flatten(), self.muk[:,i], yerr=err,ecolor=col, elinewidth=2, linewidth=0) 297 | 298 | ax.set_ylim(ymin, ymax) 299 | 300 | if on_subplots: 301 | GPy.plotting.matplot_dep.util.align_subplots(Nx,Ny,xlim=(xmin,xmax), ylim=(ymin, ymax)) 302 | else: 303 | ax.set_xlim(xmin,xmax) 304 | -------------------------------------------------------------------------------- /GPclust/OMGP.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2012, 2013, 2014 James Hensman 2 | # Licensed under the GPL v3 (see LICENSE.txt) 3 | 4 | import numpy as np 5 | from .collapsed_mixture import CollapsedMixture 6 | import GPy 7 | from GPy.util.linalg import mdot, pdinv, backsub_both_sides, dpotrs, jitchol, dtrtrs 8 | from GPy.util.linalg import tdot_numpy as tdot 9 | 10 | class OMGP(CollapsedMixture): 11 | """ 12 | Overlapping mixtures of Gaussian processes 13 | """ 14 | def __init__(self, X, Y, K=2, kernels=None, variance=1., alpha=1., prior_Z='symmetric', name='OMGP'): 15 | 16 | N, self.D = Y.shape 17 | self.Y = Y 18 | self.YYT = tdot(self.Y) 19 | 20 | self.X = X 21 | 22 | if kernels == None: 23 | self.kern = [] 24 | for i in range(K): 25 | self.kern.append(GPy.kern.RBF(input_dim=1)) 26 | else: 27 | self.kern = kernels 28 | 29 | CollapsedMixture.__init__(self, N, K, prior_Z, alpha, name) 30 | 31 | self.link_parameter(GPy.core.parameterization.param.Param('variance', variance, GPy.core.parameterization.transformations.Logexp())) 32 | self.link_parameters(*self.kern) 33 | 34 | 35 | def parameters_changed(self): 36 | """ Set the kernel parameters 37 | """ 38 | self.update_kern_grads() 39 | 40 | def do_computations(self): 41 | """ 42 | Here we do all the computations that are required whenever the kernels 43 | or the variational parameters are changed. 44 | """ 45 | if len(self.kern) < self.K: 46 | self.kern.append(self.kern[-1].copy()) 47 | self.link_parameter(self.kern[-1]) 48 | 49 | if len(self.kern) > self.K: 50 | for kern in self.kern[self.K:]: 51 | self.unlink_parameter(kern) 52 | 53 | self.kern = self.kern[:self.K] 54 | 55 | def update_kern_grads(self): 56 | """ 57 | Set the derivative of the lower bound wrt the (kernel) parameters 58 | """ 59 | grad_Lm_variance = 0.0 60 | 61 | for i, kern in enumerate(self.kern): 62 | K = kern.K(self.X) 63 | B_inv = np.diag(1. / (self.phi[:, i] / self.variance)) 64 | 65 | # Numerically more stable version using cholesky decomposition 66 | #alpha = linalg.cho_solve(linalg.cho_factor(K + B_inv), self.Y) 67 | #K_B_inv = pdinv(K + B_inv)[0] 68 | #dL_dK = .5*(tdot(alpha) - K_B_inv) 69 | 70 | # Make more stable using cholesky factorization: 71 | Bi, LB, LBi, Blogdet = pdinv(K+B_inv) 72 | 73 | tmp = dpotrs(LB, self.YYT)[0] 74 | GPy.util.diag.subtract(tmp, 1) 75 | dL_dB = dpotrs(LB, tmp.T)[0] 76 | 77 | kern.update_gradients_full(dL_dK=.5*dL_dB, X=self.X) 78 | 79 | # variance gradient 80 | 81 | #for i, kern in enumerate(self.kern): 82 | K = kern.K(self.X) 83 | #I = np.eye(self.N) 84 | 85 | B_inv = np.diag(1. / ((self.phi[:, i] + 1e-6) / self.variance)) 86 | #alpha = np.linalg.solve(K + B_inv, self.Y) 87 | #K_B_inv = pdinv(K + B_inv)[0] 88 | #dL_dB = tdot(alpha) - K_B_inv 89 | grad_B_inv = np.diag(1. / (self.phi[:, i] + 1e-6)) 90 | 91 | grad_Lm_variance += 0.5 * np.trace(np.dot(dL_dB, grad_B_inv)) 92 | grad_Lm_variance -= .5*self.D * np.einsum('j,j->',self.phi[:, i], 1./self.variance) 93 | 94 | self.variance.gradient = grad_Lm_variance 95 | 96 | def bound(self): 97 | """ 98 | Compute the lower bound on the marginal likelihood (conditioned on the 99 | GP hyper parameters). 100 | """ 101 | GP_bound = 0.0 102 | 103 | for i, kern in enumerate(self.kern): 104 | K = kern.K(self.X) 105 | B_inv = np.diag(1. / ((self.phi[:, i] + 1e-6) / self.variance)) 106 | 107 | # Make more stable using cholesky factorization: 108 | Bi, LB, LBi, Blogdet = pdinv(K+B_inv) 109 | 110 | # Data fit 111 | # alpha = linalg.cho_solve(linalg.cho_factor(K + B_inv), self.Y) 112 | # GP_bound += -0.5 * np.dot(self.Y.T, alpha).trace() 113 | GP_bound -= .5 * dpotrs(LB, self.YYT)[0].trace() 114 | 115 | # Penalty 116 | # GP_bound += -0.5 * np.linalg.slogdet(K + B_inv)[1] 117 | GP_bound -= 0.5 * Blogdet 118 | 119 | # Constant, weighted by model assignment per point 120 | #GP_bound += -0.5 * (self.phi[:, i] * np.log(2 * np.pi * self.variance)).sum() 121 | GP_bound -= .5*self.D * np.einsum('j,j->',self.phi[:, i], np.log(2 * np.pi * self.variance)) 122 | 123 | return GP_bound + self.mixing_prop_bound() + self.H 124 | 125 | def vb_grad_natgrad(self): 126 | """ 127 | Natural Gradients of the bound with respect to phi, the variational 128 | parameters controlling assignment of the data to GPs 129 | """ 130 | grad_Lm = np.zeros_like(self.phi) 131 | for i, kern in enumerate(self.kern): 132 | K = kern.K(self.X) 133 | I = np.eye(self.N) 134 | 135 | B_inv = np.diag(1. / ((self.phi[:, i] + 1e-6) / self.variance)) 136 | K_B_inv, L_B, _, _ = pdinv(K + B_inv) 137 | alpha, _ = dpotrs(L_B, self.Y) 138 | dL_dB_diag = np.sum(np.square(alpha), 1) - np.diag(K_B_inv) 139 | 140 | grad_Lm[:,i] = -0.5 * self.variance * dL_dB_diag / (self.phi[:,i]**2 + 1e-6) 141 | 142 | grad_phi = grad_Lm + self.mixing_prop_bound_grad() + self.Hgrad 143 | 144 | natgrad = grad_phi - np.sum(self.phi * grad_phi, 1)[:, None] 145 | grad = natgrad * self.phi 146 | 147 | return grad.flatten(), natgrad.flatten() 148 | 149 | def predict(self, Xnew, i): 150 | """ Predictive mean for a given component 151 | """ 152 | kern = self.kern[i] 153 | K = kern.K(self.X) 154 | kx = kern.K(self.X, Xnew) 155 | 156 | # Predict mean 157 | # This works but should Cholesky for stability 158 | B_inv = np.diag(1. / (self.phi[:, i] / self.variance)) 159 | K_B_inv = pdinv(K + B_inv)[0] 160 | mu = kx.T.dot(np.dot(K_B_inv, self.Y)) 161 | 162 | # Predict variance 163 | kxx = kern.K(Xnew, Xnew) 164 | va = self.variance + kxx - kx.T.dot(np.dot(K_B_inv, kx)) 165 | 166 | return mu, va 167 | 168 | def predict_components(self, Xnew): 169 | """The predictive density under each component""" 170 | mus = [] 171 | vas = [] 172 | for i in range(len(self.kern)): 173 | mu, va = self.predict(Xnew, i) 174 | mus.append(mu) 175 | vas.append(va) 176 | 177 | return np.array(mus)[:, :, 0].T, np.array(vas)[:, :, 0].T 178 | 179 | def sample(self, Xnew, gp=0, size=10, full_cov=True): 180 | ''' Sample the posterior of a component 181 | ''' 182 | mu, va = self.predict(Xnew, gp) 183 | 184 | samples = [] 185 | for i in range(mu.shape[1]): 186 | if full_cov: 187 | smp = np.random.multivariate_normal(mean=mu[:, i], cov=va, size=size) 188 | else: 189 | smp = np.random.multivariate_normal(mean=mu[:, i], cov=np.diag(np.diag(va)), size=size) 190 | 191 | samples.append(smp) 192 | 193 | return np.stack(samples, -1) 194 | 195 | def plot(self, gp_num=0): 196 | """ 197 | Plot the mixture of Gaussian Processes. 198 | 199 | Supports plotting 1d and 2d regression. 200 | """ 201 | from matplotlib import pylab as plt 202 | from matplotlib import cm 203 | 204 | XX = np.linspace(self.X.min(), self.X.max())[:, None] 205 | 206 | if self.Y.shape[1] == 1: 207 | plt.scatter(self.X, self.Y, c=self.phi[:, gp_num], cmap=cm.RdBu, vmin=0., vmax=1., lw=0.5) 208 | plt.colorbar(label='GP {} assignment probability'.format(gp_num)) 209 | 210 | try: 211 | Tango = GPy.plotting.Tango 212 | except: 213 | Tango = GPy.plotting.matplot_dep.Tango 214 | Tango.reset() 215 | 216 | 217 | for i in range(self.phi.shape[1]): 218 | YY_mu, YY_var = self.predict(XX, i) 219 | col = Tango.nextMedium() 220 | plt.fill_between(XX[:, 0], 221 | YY_mu[:, 0] - 2 * np.sqrt(YY_var[:, 0]), 222 | YY_mu[:, 0] + 2 * np.sqrt(YY_var[:, 0]), 223 | alpha=0.1, 224 | facecolor=col) 225 | plt.plot(XX, YY_mu[:, 0], c=col, lw=2); 226 | 227 | elif self.Y.shape[1] == 2: 228 | plt.scatter(self.Y[:, 0], self.Y[:, 1], c=self.phi[:, gp_num], cmap=cm.RdBu, vmin=0., vmax=1., lw=0.5) 229 | plt.colorbar(label='GP {} assignment probability'.format(gp_num)) 230 | 231 | try: 232 | Tango = GPy.plotting.Tango 233 | except: 234 | Tango = GPy.plotting.matplot_dep.Tango 235 | Tango.reset() 236 | 237 | for i in range(self.phi.shape[1]): 238 | YY_mu, YY_var = self.predict(XX, i) 239 | col = Tango.nextMedium() 240 | plt.plot(YY_mu[:, 0], YY_mu[:, 1], c=col, lw=2); 241 | 242 | else: 243 | raise NotImplementedError('Only 1d and 2d regression can be plotted') 244 | 245 | def plot_probs(self, gp_num=0): 246 | """ 247 | Plot assignment probabilities for each data point of the OMGP model 248 | """ 249 | from matplotlib import pylab as plt 250 | plt.scatter(self.X, self.phi[:, gp_num]) 251 | plt.ylim(-0.1, 1.1) 252 | plt.ylabel('GP {} assignment probability'.format(gp_num)) 253 | -------------------------------------------------------------------------------- /GPclust/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2012, 2013, 2014 James Hensman 2 | # Licensed under the GPL v3 (see LICENSE.txt) 3 | 4 | from .MOG import MOG 5 | from .MOHGP import MOHGP 6 | from .OMGP import OMGP 7 | 8 | -------------------------------------------------------------------------------- /GPclust/collapsed_mixture.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2012, 2013, 2014 James Hensman 2 | # Licensed under the GPL v3 (see LICENSE.txt) 3 | 4 | import numpy as np 5 | 6 | try: 7 | from .utilities import ln_dirichlet_C, softmax_weave 8 | except ImportError: 9 | from .np_utilities import ln_dirichlet_C 10 | from .np_utilities import softmax_numpy as softmax_weave 11 | 12 | from scipy.special import gammaln, digamma 13 | from .collapsed_vb import CollapsedVB 14 | 15 | class CollapsedMixture(CollapsedVB): 16 | """ 17 | A base class for collapsed mixture models based on the CollapsedVB class 18 | 19 | We inherrit from this to build mixtures of Gaussians, mixures of GPs etc. 20 | 21 | This handles the mixing proportion part of the model, 22 | as well as providing generic functions for a merge-split approach 23 | """ 24 | def __init__(self, N, K, prior_Z='symmetric', alpha=1.0, name='col_mix'): 25 | """ 26 | Arguments 27 | ========= 28 | N: the number of data 29 | K: the (initial) number of cluster (or truncation) 30 | prior_Z - either 'symmetric' or 'DP', specifies whether to use a symmetric Dirichlet prior for the clusters, or a (truncated) Dirichlet Process. 31 | alpha: parameter of the Dirichelt (process) 32 | 33 | """ 34 | CollapsedVB.__init__(self, name) 35 | self.N, self.K = N, K 36 | assert prior_Z in ['symmetric','DP'] 37 | self.prior_Z = prior_Z 38 | self.alpha = alpha 39 | 40 | #random initial conditions for the vb parameters 41 | self.phi_ = np.random.randn(self.N, self.K) 42 | self.phi, logphi, self.H = softmax_weave(self.phi_) 43 | self.phi_hat = self.phi.sum(0) 44 | self.Hgrad = -logphi 45 | if self.prior_Z == 'DP': 46 | self.phi_tilde_plus_hat = self.phi_hat[::-1].cumsum()[::-1] 47 | self.phi_tilde = self.phi_tilde_plus_hat - self.phi_hat 48 | 49 | def set_vb_param(self,phi_): 50 | """ 51 | Accept a vector representing the variatinoal parameters, and reshape it into self.phi 52 | """ 53 | self.phi_ = phi_.reshape(self.N, self.K) 54 | self.phi, logphi, self.H = softmax_weave(self.phi_) 55 | self.phi_hat = self.phi.sum(0) 56 | self.Hgrad = -logphi 57 | if self.prior_Z == 'DP': 58 | self.phi_tilde_plus_hat = self.phi_hat[::-1].cumsum()[::-1] 59 | self.phi_tilde = self.phi_tilde_plus_hat - self.phi_hat 60 | 61 | self.do_computations() 62 | 63 | def get_vb_param(self): 64 | return self.phi_.flatten() 65 | 66 | def mixing_prop_bound(self): 67 | """ 68 | The portion of the bound which is provided by the mixing proportions 69 | """ 70 | if self.prior_Z=='symmetric': 71 | return ln_dirichlet_C(np.ones(self.K)*self.alpha) -ln_dirichlet_C(self.alpha + self.phi_hat) 72 | elif self.prior_Z=='DP': 73 | A = gammaln(1. + self.phi_hat) 74 | B = gammaln(self.alpha + self.phi_tilde) 75 | C = gammaln(self.alpha + 1. + self.phi_tilde_plus_hat) 76 | D = self.K*(gammaln(1.+self.alpha) - gammaln(self.alpha)) 77 | return A.sum() + B.sum() - C.sum() + D 78 | else: 79 | raise NotImplementedError("invalid mixing proportion prior type: %s" % self.prior_Z) 80 | 81 | def mixing_prop_bound_grad(self): 82 | """ 83 | The gradient of the portion of the bound which arises from the mixing 84 | proportions, with respect to self.phi 85 | """ 86 | if self.prior_Z=='symmetric': 87 | return digamma(self.alpha + self.phi_hat) 88 | elif self.prior_Z=='DP': 89 | A = digamma(self.phi_hat + 1.) 90 | B = np.hstack((0, digamma(self.phi_tilde + self.alpha)[:-1].cumsum())) 91 | C = digamma(self.phi_tilde_plus_hat + self.alpha + 1.).cumsum() 92 | return A + B - C 93 | else: 94 | raise NotImplementedError("invalid mixing proportion prior type: %s"%self.prior_Z) 95 | 96 | def reorder(self): 97 | """ 98 | Re-order the clusters so that the biggest one is first. This increases 99 | the bound if the prior type is a DP. 100 | """ 101 | if self.prior_Z=='DP': 102 | i = np.argsort(self.phi_hat)[::-1] 103 | self.set_vb_param(self.phi_[:,i]) 104 | 105 | def remove_empty_clusters(self,threshold=1e-6): 106 | """Remove any cluster which has no data assigned to it""" 107 | i = self.phi_hat>threshold 108 | phi_ = self.phi_[:,i] 109 | self.K = i.sum() 110 | self.set_vb_param(phi_) 111 | 112 | def try_split(self, indexK=None, threshold=0.9, verbose=True, maxiter=100, optimize_params=None): 113 | """ 114 | Re-initialize one of the clusters as two clusters, optimize, and keep 115 | the solution if the bound is increased. Kernel parameters stay constant. 116 | 117 | Arguments 118 | --------- 119 | indexK: (int) the index of the cluster to split 120 | threshold: float [0,1], to assign data to the splitting cluster 121 | verbose: whether to print status 122 | 123 | Returns 124 | ------- 125 | Success: (bool) 126 | 127 | """ 128 | if indexK is None: 129 | indexK = np.random.multinomial(1,self.phi_hat/self.N).argmax() 130 | if indexK > (self.K-1): 131 | return False #index exceed no. clusters 132 | elif self.phi_hat[indexK]<1: 133 | return False # no data to split 134 | 135 | #ensure there's something to split 136 | if np.sum(self.phi[:,indexK]>threshold) <2: 137 | return False 138 | 139 | if verbose:print("\nattempting to split cluster ", indexK) 140 | 141 | bound_old = self.bound() 142 | phi_old = self.get_vb_param().copy() 143 | self._optimizer_copy_transformed = False # Redo transform in case parameters have been unlinked 144 | param_old = self.optimizer_array.copy() 145 | old_K = self.K 146 | 147 | #re-initalize 148 | self.K += 1 149 | self.phi_ = np.hstack((self.phi_,self.phi_.min(1)[:,None])) 150 | indexN = np.nonzero(self.phi[:,indexK] > threshold)[0] 151 | 152 | #this procedure randomly assigns data to the new and old clusters 153 | #rand_indexN = np.random.permutation(indexN) 154 | #n = np.floor(indexN.size/2.) 155 | #i1 = rand_indexN[:n] 156 | #self.phi_[i1,indexK], self.phi_[i1,-1] = self.phi_[i1,-1], self.phi_[i1,indexK] 157 | #self.set_vb_param(self.get_vb_param()) 158 | 159 | #this procedure equally assigns data to the new and old clusters, aside from one random point, which is in the new cluster 160 | special = np.random.permutation(indexN)[0] 161 | self.phi_[indexN,-1] = self.phi_[indexN,indexK].copy() 162 | self.phi_[special,-1] = np.max(self.phi_[special])+10 163 | self.set_vb_param(self.get_vb_param()) 164 | 165 | 166 | self.optimize(maxiter=maxiter, verbose=verbose) 167 | self.remove_empty_clusters() 168 | bound_new = self.bound() 169 | 170 | bound_increase = bound_new-bound_old 171 | if (bound_increase < 1e-3): 172 | self.K = old_K 173 | self.set_vb_param(phi_old) 174 | self.optimizer_array = param_old 175 | if verbose:print("split failed, bound changed by: ",bound_increase, '(K=%s)' % self.K) 176 | 177 | return False 178 | 179 | else: 180 | if verbose:print("split suceeded, bound changed by: ", bound_increase, ',', self.K-old_K,' new clusters', '(K=%s)' % self.K) 181 | if verbose:print("optimizing new split to convergence:") 182 | if optimize_params: 183 | self.optimize(**optimize_params) 184 | 185 | else: 186 | self.optimize(maxiter=5000, verbose=verbose) 187 | 188 | return True 189 | 190 | def systematic_splits(self, verbose=True): 191 | """ 192 | perform recursive splits on each of the existing clusters 193 | """ 194 | for kk in range(self.K): 195 | self.recursive_splits(kk, verbose=verbose) 196 | 197 | def recursive_splits(self,k=0, verbose=True, optimize_params=None): 198 | """ 199 | A recursive function which attempts to split a cluster (indexed by k), and if sucessful attempts to split the resulting clusters 200 | """ 201 | success = self.try_split(k, verbose=verbose, optimize_params=optimize_params) 202 | if success: 203 | if not k==(self.K-1): 204 | self.recursive_splits(self.K-1, verbose=verbose, optimize_params=optimize_params) 205 | self.recursive_splits(k, verbose=verbose, optimize_params=optimize_params) 206 | 207 | -------------------------------------------------------------------------------- /GPclust/collapsed_vb.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2012, 2013, 2014 James Hensman 2 | # Licensed under the GPL v3 (see LICENSE.txt) 3 | 4 | import numpy as np 5 | import GPy 6 | import time 7 | import sys #for flushing 8 | from numpy.linalg.linalg import LinAlgError 9 | class LinAlgWarning(Warning): 10 | pass 11 | 12 | class CollapsedVB(GPy.core.Model): 13 | """ 14 | A base class for collapsed variational models, using the GPy framework for 15 | non-variational parameters. 16 | 17 | Optimisation of the (collapsed) variational paremters is performed by 18 | Riemannian conjugate-gradient ascent, interleaved with optimisation of the 19 | non-variational parameters 20 | 21 | This class specifies a scheme for implementing the model, as well as 22 | providing the optimisation routine. 23 | """ 24 | 25 | def __init__(self, name): 26 | """""" 27 | GPy.core.Model.__init__(self, name) 28 | 29 | self.hyperparam_interval=50 30 | 31 | self.default_method = 'HS' 32 | self.hyperparam_opt_args = { 33 | 'max_iters': 20, 34 | 'messages': 1, 35 | 'clear_after_finish': True} 36 | 37 | def randomize(self): 38 | GPy.core.Model.randomize(self) 39 | self.set_vb_param(np.random.randn(self.get_vb_param().size)) 40 | 41 | def get_vb_param(self): 42 | """Return a vector of variational parameters""" 43 | raise NotImplementedError 44 | 45 | def set_vb_param(self,x): 46 | """Expand a vector of variational parameters into the model""" 47 | raise NotImplementedError 48 | 49 | def bound(self): 50 | """Returns the lower bound on the marginal likelihood""" 51 | raise NotImplementedError 52 | 53 | def vb_grad_natgrad(self): 54 | """Returns the gradient and natural gradient of the variational parameters""" 55 | 56 | def log_likelihood(self): 57 | """ 58 | In optimising the non variational (e.g. kernel) parameters, use the 59 | bound as a proxy for the likelihood 60 | """ 61 | return self.bound() 62 | 63 | def optimize(self, method='HS', maxiter=500, ftol=1e-6, gtol=1e-6, step_length=1., callback=None, verbose=True): 64 | """ 65 | Optimize the model. 66 | 67 | The strategy is to run conjugate natural gradients on the variational 68 | parameters, interleaved with gradient based optimization of any 69 | non-variational parameters. self.hyperparam_interval dictates how 70 | often this happens. 71 | 72 | Arguments 73 | --------- 74 | :method: ['FR', 'PR','HS','steepest'] -- conjugate gradient method 75 | :maxiter: int 76 | :ftol: float 77 | :gtol: float 78 | :step_length: float 79 | 80 | """ 81 | 82 | assert method in ['FR', 'PR','HS','steepest'], 'invalid conjugate gradient method specified.' 83 | 84 | ## For GPy style notebook verbosity 85 | 86 | if verbose: 87 | try: 88 | from IPython.display import display 89 | from IPython.html.widgets import IntProgress, HTML, Box, VBox, HBox, FlexBox 90 | self.text = HTML(width='100%') 91 | self.progress = IntProgress(min=0, max=maxiter) 92 | self.progress.bar_style = 'info' 93 | self.status = 'Running' 94 | 95 | html_begin = """ 102 | """ 103 | 104 | html_end = "
" 105 | 106 | self.ipython_notebook = True 107 | except: 108 | # Not in Ipython notebook 109 | self.ipython_notebook = False 110 | else: 111 | self.ipython_notebook = False 112 | 113 | if self.ipython_notebook: 114 | left_col = VBox(children=[self.progress, self.text], padding=2, width='100%') 115 | self.hor_align = FlexBox(children = [left_col], width='100%', orientation='horizontal') 116 | 117 | display(self.hor_align) 118 | 119 | try: 120 | self.text.set_css('width', '100%') 121 | left_col.set_css({ 122 | 'padding': '2px', 123 | 'width': "100%", 124 | }) 125 | 126 | self.hor_align.set_css({ 127 | 'width': "100%", 128 | }) 129 | 130 | self.hor_align.remove_class('vbox') 131 | self.hor_align.add_class('hbox') 132 | 133 | left_col.add_class("box-flex1") 134 | 135 | except: 136 | pass 137 | 138 | self.start = time.time() 139 | self._time = self.start 140 | 141 | ## --- 142 | 143 | iteration = 0 144 | bound_old = self.bound() 145 | searchDir_old = 0. 146 | iteration_failed = False 147 | while True: 148 | 149 | if callback is not None: 150 | callback() 151 | 152 | grad,natgrad = self.vb_grad_natgrad() 153 | grad,natgrad = -grad,-natgrad 154 | squareNorm = np.dot(natgrad,grad) # used to monitor convergence 155 | 156 | #find search direction 157 | if (method=='steepest') or not iteration: 158 | beta = 0 159 | elif (method=='PR'): 160 | beta = np.dot((natgrad-natgrad_old),grad)/squareNorm_old 161 | elif (method=='FR'): 162 | beta = squareNorm/squareNorm_old 163 | elif (method=='HS'): 164 | beta = np.dot((natgrad-natgrad_old),grad)/np.dot((natgrad-natgrad_old),grad_old) 165 | if np.isnan(beta) or (beta < 0.): 166 | beta = 0. 167 | searchDir = -natgrad + beta*searchDir_old 168 | 169 | # Try a conjugate step 170 | phi_old = self.get_vb_param().copy() 171 | try: 172 | self.set_vb_param(phi_old + step_length*searchDir) 173 | bound = self.bound() 174 | except LinAlgError: 175 | self.set_vb_param(phi_old) 176 | bound = bound_old-1 177 | 178 | iteration += 1 179 | 180 | # Make sure there's an increase in the bound, else revert to steepest, which is guaranteed to increase the bound. 181 | # (It's the same as VBEM.) 182 | if bound < bound_old: 183 | searchDir = -natgrad 184 | try: 185 | self.set_vb_param(phi_old + step_length*searchDir) 186 | bound = self.bound() 187 | except LinAlgError: 188 | import warnings 189 | warnings.warn("Caught LinalgError in setting variational parameters, trying to continue with old parameter settings", LinAlgWarning) 190 | self.set_vb_param(phi_old) 191 | bound = self.bound() 192 | iteration_failed = False 193 | iteration += 1 194 | 195 | 196 | if verbose: 197 | if self.ipython_notebook: 198 | 199 | t = time.time() 200 | seconds = t-self.start 201 | 202 | self.status = 'Running' 203 | self.progress.bar_style = 'info' 204 | 205 | names_vals = [['conjugate gradient method', "{:s}".format(method)], 206 | ['runtime', "{:.1f}s".format(seconds)], 207 | ['evaluation', "{}".format(iteration)], 208 | ['objective', "{:12.5f}".format(-bound)], 209 | ['||gradient||', "{:12.5f}".format(float(squareNorm))], 210 | ['beta', "{:12.5f}".format(beta)], 211 | ['status', "{:s}".format(self.status)], 212 | ] 213 | 214 | html_body = "" 215 | for name, val in names_vals: 216 | html_body += "" 217 | html_body += "{}".format(name) 218 | html_body += "{}".format(val) 219 | html_body += "" 220 | 221 | self.progress.value = iteration 222 | self.text.value = html_begin + html_body + html_end 223 | 224 | else: 225 | print('\riteration '+str(iteration)+' bound='+str(bound) + ' grad='+str(squareNorm) + ', beta='+str(beta)) 226 | sys.stdout.flush() 227 | 228 | # Converged yet? try the parameters if so 229 | if np.abs(bound-bound_old) <= ftol: 230 | if verbose: 231 | if self.ipython_notebook: 232 | self.status = 'vb converged (ftol)' 233 | names_vals[-1] = ['status', "{:s}".format(self.status)] 234 | 235 | html_body = "" 236 | for name, val in names_vals: 237 | html_body += "" 238 | html_body += "{}".format(name) 239 | html_body += "{}".format(val) 240 | html_body += "" 241 | 242 | self.text.value = html_begin + html_body + html_end 243 | self.progress.bar_style = 'success' 244 | 245 | else: 246 | print('vb converged (ftol)') 247 | 248 | if self.optimize_parameters() < 1e-1: 249 | break 250 | 251 | if squareNorm <= gtol: 252 | if verbose: 253 | if self.ipython_notebook: 254 | self.status = 'vb converged (gtol)' 255 | names_vals[-1] = ['status', "{:s}".format(self.status)] 256 | 257 | html_body = "" 258 | for name, val in names_vals: 259 | html_body += "" 260 | html_body += "{}".format(name) 261 | html_body += "{}".format(val) 262 | html_body += "" 263 | 264 | self.text.value = html_begin + html_body + html_end 265 | self.progress.bar_style = 'success' 266 | 267 | else: 268 | print('vb converged (gtol)') 269 | 270 | if self.optimize_parameters() < 1e-1: 271 | break 272 | 273 | if iteration >= maxiter: 274 | if verbose: 275 | if self.ipython_notebook: 276 | self.status = 'maxiter exceeded' 277 | names_vals[-1] = ['status', "{:s}".format(self.status)] 278 | 279 | html_body = "" 280 | for name, val in names_vals: 281 | html_body += "" 282 | html_body += "{}".format(name) 283 | html_body += "{}".format(val) 284 | html_body += "" 285 | 286 | self.text.value = html_begin + html_body + html_end 287 | self.progress.bar_style = 'warning' 288 | 289 | else: 290 | print('maxiter exceeded') 291 | break 292 | 293 | #store essentials of previous iteration 294 | natgrad_old = natgrad.copy() # copy: better safe than sorry. 295 | grad_old = grad.copy() 296 | searchDir_old = searchDir.copy() 297 | squareNorm_old = squareNorm 298 | 299 | # hyper param_optimisation 300 | if ((iteration >1) and not (iteration%self.hyperparam_interval)) or iteration_failed: 301 | self.optimize_parameters() 302 | 303 | bound_old = bound 304 | 305 | # Clean up temporary fields after optimization 306 | if self.ipython_notebook: 307 | del self.text 308 | del self.progress 309 | del self.hor_align 310 | 311 | 312 | 313 | def optimize_parameters(self): 314 | """ 315 | Optimises the model parameters (non variational parameters) 316 | Returns the increment in the bound acheived 317 | """ 318 | if self.optimizer_array.size>0: 319 | start = self.bound() 320 | GPy.core.model.Model.optimize(self,**self.hyperparam_opt_args) 321 | return self.bound()-start 322 | else: 323 | return 0. 324 | -------------------------------------------------------------------------------- /GPclust/np_utilities.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from scipy import linalg, special, sparse 3 | import sys # for flushing 4 | import GPy 5 | 6 | def multiple_pdinv(A): 7 | """ 8 | Arguments 9 | --------- 10 | A : A DxDxN numpy array (each A[:,:,i] is pd) 11 | 12 | Returns 13 | ------- 14 | invs : the inverses of A 15 | hld: 0.5* the log of the determinants of A 16 | """ 17 | N = A.shape[-1] 18 | chols = [GPy.util.linalg.jitchol(A[:,:,i]) for i in range(N)] 19 | halflogdets = [np.sum(np.log(np.diag(L))) for L in chols] 20 | invs = [GPy.util.linalg.dpotri(L,True)[0] for L in chols] 21 | invs = [np.triu(I)+np.triu(I,1).T for I in invs] 22 | return np.dstack(invs),np.array(halflogdets) 23 | 24 | 25 | def softmax_numpy(X): 26 | log_phi = X.copy() 27 | max_x_i = X.max(1) 28 | 29 | log_phi -= max_x_i[:, None] 30 | phi = np.exp(log_phi) 31 | norm_i = phi.sum(1) 32 | 33 | log_norm_i = np.log(norm_i) 34 | phi /= norm_i[:, None] 35 | log_phi -= log_norm_i[:, None] 36 | entropy_i = -(phi * log_phi).sum(1) 37 | 38 | entropy = entropy_i.sum() 39 | 40 | return phi, log_phi, entropy 41 | 42 | 43 | def multiple_mahalanobis_numpy_loops(X1, X2, L): 44 | N1, D = X1.shape 45 | N2, D = X2.shape 46 | 47 | LLT = L.dot(L.T) 48 | result = np.zeros(shape=(N1, N2), dtype=np.float64) 49 | n = 0 50 | while n < N1: 51 | m = 0 52 | while m < N2: 53 | x1x2 = X1[n] - X2[m] 54 | result[n, m] = x1x2.dot(np.linalg.solve(LLT, x1x2)) 55 | m += 1 56 | 57 | n += 1 58 | 59 | return result 60 | 61 | 62 | def lngammad(v,D): 63 | """sum of log gamma functions, as appears in a Wishart Distribution""" 64 | return np.sum([special.gammaln((v+1.-d)/2.) for d in range(1,D+1)],0) 65 | 66 | 67 | def ln_dirichlet_C(a): 68 | """the log-normalizer of a Dirichlet distribution""" 69 | return special.gammaln(a.sum())-np.sum(special.gammaln(a)) 70 | -------------------------------------------------------------------------------- /GPclust/utilities.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2012, 2013, 2014 James Hensman 2 | # Licensed under the GPL v3 (see LICENSE.txt) 3 | 4 | import numpy as np 5 | from scipy import linalg, special, sparse, weave 6 | import sys # for flushing 7 | import GPy 8 | 9 | def multiple_pdinv(A): 10 | """ 11 | Arguments 12 | --------- 13 | A : A DxDxN numpy array (each A[:,:,i] is pd) 14 | 15 | Returns 16 | ------- 17 | invs : the inverses of A 18 | hld: 0.5* the log of the determinants of A 19 | """ 20 | N = A.shape[-1] 21 | chols = [GPy.util.linalg.jitchol(A[:,:,i]) for i in range(N)] 22 | halflogdets = [np.sum(np.log(np.diag(L))) for L in chols] 23 | invs = [GPy.util.linalg.dpotri(L,True)[0] for L in chols] 24 | invs = [np.triu(I)+np.triu(I,1).T for I in invs] 25 | return np.dstack(invs),np.array(halflogdets) 26 | 27 | def softmax_weave(X): 28 | """ 29 | Use weave to compute the softmax of the (rows of) matrix X. 30 | 31 | Uses omp to parallelise the loop 32 | 33 | Also returns the log of the softmaxed result 34 | """ 35 | 36 | #configure weave for parallel (or not) 37 | weave_options_openmp = {'headers' : [''], 38 | 'extra_compile_args': ['-fopenmp -O3'], 39 | 'extra_link_args' : ['-lgomp'], 40 | 'libraries': ['gomp']} 41 | weave_options_noopenmp = {'extra_compile_args': ['-O3']} 42 | 43 | if GPy.util.config.config.getboolean('parallel', 'openmp'): 44 | weave_options = weave_options_openmp 45 | weave_support_code = """ 46 | #include 47 | #include 48 | """ 49 | else: 50 | weave_options = weave_options_noopenmp 51 | weave_support_code = "#include " 52 | 53 | if GPy.util.config.config.getboolean('parallel', 'openmp'): 54 | pragma_string = '#pragma omp parallel for private(i, j, max_x, norm, log_norm, entropy)' 55 | else: 56 | pragma_string = '' 57 | 58 | N,D = X.shape 59 | phi = np.zeros_like(X) 60 | log_phi = X.copy() 61 | code = """ 62 | int i, j; 63 | double max_x, norm, log_norm, entropy; 64 | double entropy_i [N]; 65 | {pragma} 66 | for(i=0;imax_x){{ 72 | max_x = X(i,j); 73 | }} 74 | }} 75 | 76 | //compute un-normalised phi, normaliser 77 | norm = 0.0; 78 | for(j=0;j'], 127 | 'extra_compile_args': ['-fopenmp -O3'], 128 | 'extra_link_args' : ['-lgomp'], 129 | 'libraries': ['gomp']} 130 | weave_options_noopenmp = {'extra_compile_args': ['-O3']} 131 | 132 | if GPy.util.config.config.getboolean('parallel', 'openmp'): 133 | weave_options = weave_options_openmp 134 | weave_support_code = """ 135 | #include 136 | #include 137 | """ 138 | else: 139 | weave_options = weave_options_noopenmp 140 | weave_support_code = "#include " 141 | 142 | if GPy.util.config.config.getboolean('parallel', 'openmp'): 143 | pragma_string = '#pragma omp parallel for private(n,m,i,j,tmp)' 144 | else: 145 | pragma_string = '' 146 | 147 | code = """ 148 | double tmp [D]; 149 | //two loops over the N1 x N2 vectors 150 | int n, m, i, j; 151 | {pragma} 152 | for(n=0; n 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU General Public License is a free, copyleft license for 11 | software and other kinds of works. 12 | 13 | The licenses for most software and other practical works are designed 14 | to take away your freedom to share and change the works. By contrast, 15 | the GNU General Public License is intended to guarantee your freedom to 16 | share and change all versions of a program--to make sure it remains free 17 | software for all its users. We, the Free Software Foundation, use the 18 | GNU General Public License for most of our software; it applies also to 19 | any other work released this way by its authors. You can apply it to 20 | your programs, too. 21 | 22 | When we speak of free software, we are referring to freedom, not 23 | price. Our General Public Licenses are designed to make sure that you 24 | have the freedom to distribute copies of free software (and charge for 25 | them if you wish), that you receive source code or can get it if you 26 | want it, that you can change the software or use pieces of it in new 27 | free programs, and that you know you can do these things. 28 | 29 | To protect your rights, we need to prevent others from denying you 30 | these rights or asking you to surrender the rights. Therefore, you have 31 | certain responsibilities if you distribute copies of the software, or if 32 | you modify it: responsibilities to respect the freedom of others. 33 | 34 | For example, if you distribute copies of such a program, whether 35 | gratis or for a fee, you must pass on to the recipients the same 36 | freedoms that you received. You must make sure that they, too, receive 37 | or can get the source code. And you must show them these terms so they 38 | know their rights. 39 | 40 | Developers that use the GNU GPL protect your rights with two steps: 41 | (1) assert copyright on the software, and (2) offer you this License 42 | giving you legal permission to copy, distribute and/or modify it. 43 | 44 | For the developers' and authors' protection, the GPL clearly explains 45 | that there is no warranty for this free software. For both users' and 46 | authors' sake, the GPL requires that modified versions be marked as 47 | changed, so that their problems will not be attributed erroneously to 48 | authors of previous versions. 49 | 50 | Some devices are designed to deny users access to install or run 51 | modified versions of the software inside them, although the manufacturer 52 | can do so. This is fundamentally incompatible with the aim of 53 | protecting users' freedom to change the software. The systematic 54 | pattern of such abuse occurs in the area of products for individuals to 55 | use, which is precisely where it is most unacceptable. Therefore, we 56 | have designed this version of the GPL to prohibit the practice for those 57 | products. If such problems arise substantially in other domains, we 58 | stand ready to extend this provision to those domains in future versions 59 | of the GPL, as needed to protect the freedom of users. 60 | 61 | Finally, every program is threatened constantly by software patents. 62 | States should not allow patents to restrict development and use of 63 | software on general-purpose computers, but in those that do, we wish to 64 | avoid the special danger that patents applied to a free program could 65 | make it effectively proprietary. To prevent this, the GPL assures that 66 | patents cannot be used to render the program non-free. 67 | 68 | The precise terms and conditions for copying, distribution and 69 | modification follow. 70 | 71 | TERMS AND CONDITIONS 72 | 73 | 0. Definitions. 74 | 75 | "This License" refers to version 3 of the GNU General Public License. 76 | 77 | "Copyright" also means copyright-like laws that apply to other kinds of 78 | works, such as semiconductor masks. 79 | 80 | "The Program" refers to any copyrightable work licensed under this 81 | License. Each licensee is addressed as "you". "Licensees" and 82 | "recipients" may be individuals or organizations. 83 | 84 | To "modify" a work means to copy from or adapt all or part of the work 85 | in a fashion requiring copyright permission, other than the making of an 86 | exact copy. The resulting work is called a "modified version" of the 87 | earlier work or a work "based on" the earlier work. 88 | 89 | A "covered work" means either the unmodified Program or a work based 90 | on the Program. 91 | 92 | To "propagate" a work means to do anything with it that, without 93 | permission, would make you directly or secondarily liable for 94 | infringement under applicable copyright law, except executing it on a 95 | computer or modifying a private copy. Propagation includes copying, 96 | distribution (with or without modification), making available to the 97 | public, and in some countries other activities as well. 98 | 99 | To "convey" a work means any kind of propagation that enables other 100 | parties to make or receive copies. Mere interaction with a user through 101 | a computer network, with no transfer of a copy, is not conveying. 102 | 103 | An interactive user interface displays "Appropriate Legal Notices" 104 | to the extent that it includes a convenient and prominently visible 105 | feature that (1) displays an appropriate copyright notice, and (2) 106 | tells the user that there is no warranty for the work (except to the 107 | extent that warranties are provided), that licensees may convey the 108 | work under this License, and how to view a copy of this License. If 109 | the interface presents a list of user commands or options, such as a 110 | menu, a prominent item in the list meets this criterion. 111 | 112 | 1. Source Code. 113 | 114 | The "source code" for a work means the preferred form of the work 115 | for making modifications to it. "Object code" means any non-source 116 | form of a work. 117 | 118 | A "Standard Interface" means an interface that either is an official 119 | standard defined by a recognized standards body, or, in the case of 120 | interfaces specified for a particular programming language, one that 121 | is widely used among developers working in that language. 122 | 123 | The "System Libraries" of an executable work include anything, other 124 | than the work as a whole, that (a) is included in the normal form of 125 | packaging a Major Component, but which is not part of that Major 126 | Component, and (b) serves only to enable use of the work with that 127 | Major Component, or to implement a Standard Interface for which an 128 | implementation is available to the public in source code form. A 129 | "Major Component", in this context, means a major essential component 130 | (kernel, window system, and so on) of the specific operating system 131 | (if any) on which the executable work runs, or a compiler used to 132 | produce the work, or an object code interpreter used to run it. 133 | 134 | The "Corresponding Source" for a work in object code form means all 135 | the source code needed to generate, install, and (for an executable 136 | work) run the object code and to modify the work, including scripts to 137 | control those activities. However, it does not include the work's 138 | System Libraries, or general-purpose tools or generally available free 139 | programs which are used unmodified in performing those activities but 140 | which are not part of the work. For example, Corresponding Source 141 | includes interface definition files associated with source files for 142 | the work, and the source code for shared libraries and dynamically 143 | linked subprograms that the work is specifically designed to require, 144 | such as by intimate data communication or control flow between those 145 | subprograms and other parts of the work. 146 | 147 | The Corresponding Source need not include anything that users 148 | can regenerate automatically from other parts of the Corresponding 149 | Source. 150 | 151 | The Corresponding Source for a work in source code form is that 152 | same work. 153 | 154 | 2. Basic Permissions. 155 | 156 | All rights granted under this License are granted for the term of 157 | copyright on the Program, and are irrevocable provided the stated 158 | conditions are met. This License explicitly affirms your unlimited 159 | permission to run the unmodified Program. The output from running a 160 | covered work is covered by this License only if the output, given its 161 | content, constitutes a covered work. This License acknowledges your 162 | rights of fair use or other equivalent, as provided by copyright law. 163 | 164 | You may make, run and propagate covered works that you do not 165 | convey, without conditions so long as your license otherwise remains 166 | in force. You may convey covered works to others for the sole purpose 167 | of having them make modifications exclusively for you, or provide you 168 | with facilities for running those works, provided that you comply with 169 | the terms of this License in conveying all material for which you do 170 | not control copyright. Those thus making or running the covered works 171 | for you must do so exclusively on your behalf, under your direction 172 | and control, on terms that prohibit them from making any copies of 173 | your copyrighted material outside their relationship with you. 174 | 175 | Conveying under any other circumstances is permitted solely under 176 | the conditions stated below. Sublicensing is not allowed; section 10 177 | makes it unnecessary. 178 | 179 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 180 | 181 | No covered work shall be deemed part of an effective technological 182 | measure under any applicable law fulfilling obligations under article 183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 184 | similar laws prohibiting or restricting circumvention of such 185 | measures. 186 | 187 | When you convey a covered work, you waive any legal power to forbid 188 | circumvention of technological measures to the extent such circumvention 189 | is effected by exercising rights under this License with respect to 190 | the covered work, and you disclaim any intention to limit operation or 191 | modification of the work as a means of enforcing, against the work's 192 | users, your or third parties' legal rights to forbid circumvention of 193 | technological measures. 194 | 195 | 4. Conveying Verbatim Copies. 196 | 197 | You may convey verbatim copies of the Program's source code as you 198 | receive it, in any medium, provided that you conspicuously and 199 | appropriately publish on each copy an appropriate copyright notice; 200 | keep intact all notices stating that this License and any 201 | non-permissive terms added in accord with section 7 apply to the code; 202 | keep intact all notices of the absence of any warranty; and give all 203 | recipients a copy of this License along with the Program. 204 | 205 | You may charge any price or no price for each copy that you convey, 206 | and you may offer support or warranty protection for a fee. 207 | 208 | 5. Conveying Modified Source Versions. 209 | 210 | You may convey a work based on the Program, or the modifications to 211 | produce it from the Program, in the form of source code under the 212 | terms of section 4, provided that you also meet all of these conditions: 213 | 214 | a) The work must carry prominent notices stating that you modified 215 | it, and giving a relevant date. 216 | 217 | b) The work must carry prominent notices stating that it is 218 | released under this License and any conditions added under section 219 | 7. This requirement modifies the requirement in section 4 to 220 | "keep intact all notices". 221 | 222 | c) You must license the entire work, as a whole, under this 223 | License to anyone who comes into possession of a copy. This 224 | License will therefore apply, along with any applicable section 7 225 | additional terms, to the whole of the work, and all its parts, 226 | regardless of how they are packaged. This License gives no 227 | permission to license the work in any other way, but it does not 228 | invalidate such permission if you have separately received it. 229 | 230 | d) If the work has interactive user interfaces, each must display 231 | Appropriate Legal Notices; however, if the Program has interactive 232 | interfaces that do not display Appropriate Legal Notices, your 233 | work need not make them do so. 234 | 235 | A compilation of a covered work with other separate and independent 236 | works, which are not by their nature extensions of the covered work, 237 | and which are not combined with it such as to form a larger program, 238 | in or on a volume of a storage or distribution medium, is called an 239 | "aggregate" if the compilation and its resulting copyright are not 240 | used to limit the access or legal rights of the compilation's users 241 | beyond what the individual works permit. Inclusion of a covered work 242 | in an aggregate does not cause this License to apply to the other 243 | parts of the aggregate. 244 | 245 | 6. Conveying Non-Source Forms. 246 | 247 | You may convey a covered work in object code form under the terms 248 | of sections 4 and 5, provided that you also convey the 249 | machine-readable Corresponding Source under the terms of this License, 250 | in one of these ways: 251 | 252 | a) Convey the object code in, or embodied in, a physical product 253 | (including a physical distribution medium), accompanied by the 254 | Corresponding Source fixed on a durable physical medium 255 | customarily used for software interchange. 256 | 257 | b) Convey the object code in, or embodied in, a physical product 258 | (including a physical distribution medium), accompanied by a 259 | written offer, valid for at least three years and valid for as 260 | long as you offer spare parts or customer support for that product 261 | model, to give anyone who possesses the object code either (1) a 262 | copy of the Corresponding Source for all the software in the 263 | product that is covered by this License, on a durable physical 264 | medium customarily used for software interchange, for a price no 265 | more than your reasonable cost of physically performing this 266 | conveying of source, or (2) access to copy the 267 | Corresponding Source from a network server at no charge. 268 | 269 | c) Convey individual copies of the object code with a copy of the 270 | written offer to provide the Corresponding Source. This 271 | alternative is allowed only occasionally and noncommercially, and 272 | only if you received the object code with such an offer, in accord 273 | with subsection 6b. 274 | 275 | d) Convey the object code by offering access from a designated 276 | place (gratis or for a charge), and offer equivalent access to the 277 | Corresponding Source in the same way through the same place at no 278 | further charge. You need not require recipients to copy the 279 | Corresponding Source along with the object code. If the place to 280 | copy the object code is a network server, the Corresponding Source 281 | may be on a different server (operated by you or a third party) 282 | that supports equivalent copying facilities, provided you maintain 283 | clear directions next to the object code saying where to find the 284 | Corresponding Source. Regardless of what server hosts the 285 | Corresponding Source, you remain obligated to ensure that it is 286 | available for as long as needed to satisfy these requirements. 287 | 288 | e) Convey the object code using peer-to-peer transmission, provided 289 | you inform other peers where the object code and Corresponding 290 | Source of the work are being offered to the general public at no 291 | charge under subsection 6d. 292 | 293 | A separable portion of the object code, whose source code is excluded 294 | from the Corresponding Source as a System Library, need not be 295 | included in conveying the object code work. 296 | 297 | A "User Product" is either (1) a "consumer product", which means any 298 | tangible personal property which is normally used for personal, family, 299 | or household purposes, or (2) anything designed or sold for incorporation 300 | into a dwelling. In determining whether a product is a consumer product, 301 | doubtful cases shall be resolved in favor of coverage. For a particular 302 | product received by a particular user, "normally used" refers to a 303 | typical or common use of that class of product, regardless of the status 304 | of the particular user or of the way in which the particular user 305 | actually uses, or expects or is expected to use, the product. A product 306 | is a consumer product regardless of whether the product has substantial 307 | commercial, industrial or non-consumer uses, unless such uses represent 308 | the only significant mode of use of the product. 309 | 310 | "Installation Information" for a User Product means any methods, 311 | procedures, authorization keys, or other information required to install 312 | and execute modified versions of a covered work in that User Product from 313 | a modified version of its Corresponding Source. The information must 314 | suffice to ensure that the continued functioning of the modified object 315 | code is in no case prevented or interfered with solely because 316 | modification has been made. 317 | 318 | If you convey an object code work under this section in, or with, or 319 | specifically for use in, a User Product, and the conveying occurs as 320 | part of a transaction in which the right of possession and use of the 321 | User Product is transferred to the recipient in perpetuity or for a 322 | fixed term (regardless of how the transaction is characterized), the 323 | Corresponding Source conveyed under this section must be accompanied 324 | by the Installation Information. But this requirement does not apply 325 | if neither you nor any third party retains the ability to install 326 | modified object code on the User Product (for example, the work has 327 | been installed in ROM). 328 | 329 | The requirement to provide Installation Information does not include a 330 | requirement to continue to provide support service, warranty, or updates 331 | for a work that has been modified or installed by the recipient, or for 332 | the User Product in which it has been modified or installed. Access to a 333 | network may be denied when the modification itself materially and 334 | adversely affects the operation of the network or violates the rules and 335 | protocols for communication across the network. 336 | 337 | Corresponding Source conveyed, and Installation Information provided, 338 | in accord with this section must be in a format that is publicly 339 | documented (and with an implementation available to the public in 340 | source code form), and must require no special password or key for 341 | unpacking, reading or copying. 342 | 343 | 7. Additional Terms. 344 | 345 | "Additional permissions" are terms that supplement the terms of this 346 | License by making exceptions from one or more of its conditions. 347 | Additional permissions that are applicable to the entire Program shall 348 | be treated as though they were included in this License, to the extent 349 | that they are valid under applicable law. If additional permissions 350 | apply only to part of the Program, that part may be used separately 351 | under those permissions, but the entire Program remains governed by 352 | this License without regard to the additional permissions. 353 | 354 | When you convey a copy of a covered work, you may at your option 355 | remove any additional permissions from that copy, or from any part of 356 | it. (Additional permissions may be written to require their own 357 | removal in certain cases when you modify the work.) You may place 358 | additional permissions on material, added by you to a covered work, 359 | for which you have or can give appropriate copyright permission. 360 | 361 | Notwithstanding any other provision of this License, for material you 362 | add to a covered work, you may (if authorized by the copyright holders of 363 | that material) supplement the terms of this License with terms: 364 | 365 | a) Disclaiming warranty or limiting liability differently from the 366 | terms of sections 15 and 16 of this License; or 367 | 368 | b) Requiring preservation of specified reasonable legal notices or 369 | author attributions in that material or in the Appropriate Legal 370 | Notices displayed by works containing it; or 371 | 372 | c) Prohibiting misrepresentation of the origin of that material, or 373 | requiring that modified versions of such material be marked in 374 | reasonable ways as different from the original version; or 375 | 376 | d) Limiting the use for publicity purposes of names of licensors or 377 | authors of the material; or 378 | 379 | e) Declining to grant rights under trademark law for use of some 380 | trade names, trademarks, or service marks; or 381 | 382 | f) Requiring indemnification of licensors and authors of that 383 | material by anyone who conveys the material (or modified versions of 384 | it) with contractual assumptions of liability to the recipient, for 385 | any liability that these contractual assumptions directly impose on 386 | those licensors and authors. 387 | 388 | All other non-permissive additional terms are considered "further 389 | restrictions" within the meaning of section 10. If the Program as you 390 | received it, or any part of it, contains a notice stating that it is 391 | governed by this License along with a term that is a further 392 | restriction, you may remove that term. If a license document contains 393 | a further restriction but permits relicensing or conveying under this 394 | License, you may add to a covered work material governed by the terms 395 | of that license document, provided that the further restriction does 396 | not survive such relicensing or conveying. 397 | 398 | If you add terms to a covered work in accord with this section, you 399 | must place, in the relevant source files, a statement of the 400 | additional terms that apply to those files, or a notice indicating 401 | where to find the applicable terms. 402 | 403 | Additional terms, permissive or non-permissive, may be stated in the 404 | form of a separately written license, or stated as exceptions; 405 | the above requirements apply either way. 406 | 407 | 8. Termination. 408 | 409 | You may not propagate or modify a covered work except as expressly 410 | provided under this License. Any attempt otherwise to propagate or 411 | modify it is void, and will automatically terminate your rights under 412 | this License (including any patent licenses granted under the third 413 | paragraph of section 11). 414 | 415 | However, if you cease all violation of this License, then your 416 | license from a particular copyright holder is reinstated (a) 417 | provisionally, unless and until the copyright holder explicitly and 418 | finally terminates your license, and (b) permanently, if the copyright 419 | holder fails to notify you of the violation by some reasonable means 420 | prior to 60 days after the cessation. 421 | 422 | Moreover, your license from a particular copyright holder is 423 | reinstated permanently if the copyright holder notifies you of the 424 | violation by some reasonable means, this is the first time you have 425 | received notice of violation of this License (for any work) from that 426 | copyright holder, and you cure the violation prior to 30 days after 427 | your receipt of the notice. 428 | 429 | Termination of your rights under this section does not terminate the 430 | licenses of parties who have received copies or rights from you under 431 | this License. If your rights have been terminated and not permanently 432 | reinstated, you do not qualify to receive new licenses for the same 433 | material under section 10. 434 | 435 | 9. Acceptance Not Required for Having Copies. 436 | 437 | You are not required to accept this License in order to receive or 438 | run a copy of the Program. Ancillary propagation of a covered work 439 | occurring solely as a consequence of using peer-to-peer transmission 440 | to receive a copy likewise does not require acceptance. However, 441 | nothing other than this License grants you permission to propagate or 442 | modify any covered work. These actions infringe copyright if you do 443 | not accept this License. Therefore, by modifying or propagating a 444 | covered work, you indicate your acceptance of this License to do so. 445 | 446 | 10. Automatic Licensing of Downstream Recipients. 447 | 448 | Each time you convey a covered work, the recipient automatically 449 | receives a license from the original licensors, to run, modify and 450 | propagate that work, subject to this License. You are not responsible 451 | for enforcing compliance by third parties with this License. 452 | 453 | An "entity transaction" is a transaction transferring control of an 454 | organization, or substantially all assets of one, or subdividing an 455 | organization, or merging organizations. If propagation of a covered 456 | work results from an entity transaction, each party to that 457 | transaction who receives a copy of the work also receives whatever 458 | licenses to the work the party's predecessor in interest had or could 459 | give under the previous paragraph, plus a right to possession of the 460 | Corresponding Source of the work from the predecessor in interest, if 461 | the predecessor has it or can get it with reasonable efforts. 462 | 463 | You may not impose any further restrictions on the exercise of the 464 | rights granted or affirmed under this License. For example, you may 465 | not impose a license fee, royalty, or other charge for exercise of 466 | rights granted under this License, and you may not initiate litigation 467 | (including a cross-claim or counterclaim in a lawsuit) alleging that 468 | any patent claim is infringed by making, using, selling, offering for 469 | sale, or importing the Program or any portion of it. 470 | 471 | 11. Patents. 472 | 473 | A "contributor" is a copyright holder who authorizes use under this 474 | License of the Program or a work on which the Program is based. The 475 | work thus licensed is called the contributor's "contributor version". 476 | 477 | A contributor's "essential patent claims" are all patent claims 478 | owned or controlled by the contributor, whether already acquired or 479 | hereafter acquired, that would be infringed by some manner, permitted 480 | by this License, of making, using, or selling its contributor version, 481 | but do not include claims that would be infringed only as a 482 | consequence of further modification of the contributor version. For 483 | purposes of this definition, "control" includes the right to grant 484 | patent sublicenses in a manner consistent with the requirements of 485 | this License. 486 | 487 | Each contributor grants you a non-exclusive, worldwide, royalty-free 488 | patent license under the contributor's essential patent claims, to 489 | make, use, sell, offer for sale, import and otherwise run, modify and 490 | propagate the contents of its contributor version. 491 | 492 | In the following three paragraphs, a "patent license" is any express 493 | agreement or commitment, however denominated, not to enforce a patent 494 | (such as an express permission to practice a patent or covenant not to 495 | sue for patent infringement). To "grant" such a patent license to a 496 | party means to make such an agreement or commitment not to enforce a 497 | patent against the party. 498 | 499 | If you convey a covered work, knowingly relying on a patent license, 500 | and the Corresponding Source of the work is not available for anyone 501 | to copy, free of charge and under the terms of this License, through a 502 | publicly available network server or other readily accessible means, 503 | then you must either (1) cause the Corresponding Source to be so 504 | available, or (2) arrange to deprive yourself of the benefit of the 505 | patent license for this particular work, or (3) arrange, in a manner 506 | consistent with the requirements of this License, to extend the patent 507 | license to downstream recipients. "Knowingly relying" means you have 508 | actual knowledge that, but for the patent license, your conveying the 509 | covered work in a country, or your recipient's use of the covered work 510 | in a country, would infringe one or more identifiable patents in that 511 | country that you have reason to believe are valid. 512 | 513 | If, pursuant to or in connection with a single transaction or 514 | arrangement, you convey, or propagate by procuring conveyance of, a 515 | covered work, and grant a patent license to some of the parties 516 | receiving the covered work authorizing them to use, propagate, modify 517 | or convey a specific copy of the covered work, then the patent license 518 | you grant is automatically extended to all recipients of the covered 519 | work and works based on it. 520 | 521 | A patent license is "discriminatory" if it does not include within 522 | the scope of its coverage, prohibits the exercise of, or is 523 | conditioned on the non-exercise of one or more of the rights that are 524 | specifically granted under this License. You may not convey a covered 525 | work if you are a party to an arrangement with a third party that is 526 | in the business of distributing software, under which you make payment 527 | to the third party based on the extent of your activity of conveying 528 | the work, and under which the third party grants, to any of the 529 | parties who would receive the covered work from you, a discriminatory 530 | patent license (a) in connection with copies of the covered work 531 | conveyed by you (or copies made from those copies), or (b) primarily 532 | for and in connection with specific products or compilations that 533 | contain the covered work, unless you entered into that arrangement, 534 | or that patent license was granted, prior to 28 March 2007. 535 | 536 | Nothing in this License shall be construed as excluding or limiting 537 | any implied license or other defenses to infringement that may 538 | otherwise be available to you under applicable patent law. 539 | 540 | 12. No Surrender of Others' Freedom. 541 | 542 | If conditions are imposed on you (whether by court order, agreement or 543 | otherwise) that contradict the conditions of this License, they do not 544 | excuse you from the conditions of this License. If you cannot convey a 545 | covered work so as to satisfy simultaneously your obligations under this 546 | License and any other pertinent obligations, then as a consequence you may 547 | not convey it at all. For example, if you agree to terms that obligate you 548 | to collect a royalty for further conveying from those to whom you convey 549 | the Program, the only way you could satisfy both those terms and this 550 | License would be to refrain entirely from conveying the Program. 551 | 552 | 13. Use with the GNU Affero General Public License. 553 | 554 | Notwithstanding any other provision of this License, you have 555 | permission to link or combine any covered work with a work licensed 556 | under version 3 of the GNU Affero General Public License into a single 557 | combined work, and to convey the resulting work. The terms of this 558 | License will continue to apply to the part which is the covered work, 559 | but the special requirements of the GNU Affero General Public License, 560 | section 13, concerning interaction through a network will apply to the 561 | combination as such. 562 | 563 | 14. Revised Versions of this License. 564 | 565 | The Free Software Foundation may publish revised and/or new versions of 566 | the GNU General Public License from time to time. Such new versions will 567 | be similar in spirit to the present version, but may differ in detail to 568 | address new problems or concerns. 569 | 570 | Each version is given a distinguishing version number. If the 571 | Program specifies that a certain numbered version of the GNU General 572 | Public License "or any later version" applies to it, you have the 573 | option of following the terms and conditions either of that numbered 574 | version or of any later version published by the Free Software 575 | Foundation. If the Program does not specify a version number of the 576 | GNU General Public License, you may choose any version ever published 577 | by the Free Software Foundation. 578 | 579 | If the Program specifies that a proxy can decide which future 580 | versions of the GNU General Public License can be used, that proxy's 581 | public statement of acceptance of a version permanently authorizes you 582 | to choose that version for the Program. 583 | 584 | Later license versions may give you additional or different 585 | permissions. However, no additional obligations are imposed on any 586 | author or copyright holder as a result of your choosing to follow a 587 | later version. 588 | 589 | 15. Disclaimer of Warranty. 590 | 591 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 592 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 596 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 597 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 599 | 600 | 16. Limitation of Liability. 601 | 602 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 610 | SUCH DAMAGES. 611 | 612 | 17. Interpretation of Sections 15 and 16. 613 | 614 | If the disclaimer of warranty and limitation of liability provided 615 | above cannot be given local legal effect according to their terms, 616 | reviewing courts shall apply local law that most closely approximates 617 | an absolute waiver of all civil liability in connection with the 618 | Program, unless a warranty or assumption of liability accompanies a 619 | copy of the Program in return for a fee. 620 | 621 | END OF TERMS AND CONDITIONS 622 | 623 | How to Apply These Terms to Your New Programs 624 | 625 | If you develop a new program, and you want it to be of the greatest 626 | possible use to the public, the best way to achieve this is to make it 627 | free software which everyone can redistribute and change under these terms. 628 | 629 | To do so, attach the following notices to the program. It is safest 630 | to attach them to the start of each source file to most effectively 631 | state the exclusion of warranty; and each file should have at least 632 | the "copyright" line and a pointer to where the full notice is found. 633 | 634 | 635 | Copyright (C) 636 | 637 | This program is free software: you can redistribute it and/or modify 638 | it under the terms of the GNU General Public License as published by 639 | the Free Software Foundation, either version 3 of the License, or 640 | (at your option) any later version. 641 | 642 | This program is distributed in the hope that it will be useful, 643 | but WITHOUT ANY WARRANTY; without even the implied warranty of 644 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 645 | GNU General Public License for more details. 646 | 647 | You should have received a copy of the GNU General Public License 648 | along with this program. If not, see . 649 | 650 | Also add information on how to contact you by electronic and paper mail. 651 | 652 | If the program does terminal interaction, make it output a short 653 | notice like this when it starts in an interactive mode: 654 | 655 | Copyright (C) 656 | This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 657 | This is free software, and you are welcome to redistribute it 658 | under certain conditions; type `show c' for details. 659 | 660 | The hypothetical commands `show w' and `show c' should show the appropriate 661 | parts of the General Public License. Of course, your program's commands 662 | might be different; for a GUI interface, you would use an "about box". 663 | 664 | You should also get your employer (if you work as a programmer) or school, 665 | if any, to sign a "copyright disclaimer" for the program, if necessary. 666 | For more information on this, and how to apply and follow the GNU GPL, see 667 | . 668 | 669 | The GNU General Public License does not permit incorporating your program 670 | into proprietary programs. If your program is a subroutine library, you 671 | may consider it more useful to permit linking proprietary applications with 672 | the library. If this is what you want to do, use the GNU Lesser General 673 | Public License instead of this License. But first, please read 674 | . 675 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | GPclust 2 | ===== 3 | 4 | Clustering time series using Gaussian processes and variational Bayes. 5 | 6 | User guide and tutorials are available via the included [notebooks](https://github.com/jameshensman/GPclust/blob/master/notebooks/index.ipynb). 7 | 8 | Currently implemented models are 9 | 10 | * MOG - Mixture of Gaussians 11 | * MOHGP - Mixtures of Hierarchical Gaussian processes 12 | * OMGP - Overlapping mixtures of Gaussian processes 13 | 14 | Citation 15 | ======== 16 | 17 | The underlying algorithm is based on the 2012 NIPS paper: 18 | 19 | 20 | http://books.nips.cc/papers/files/nips25/NIPS2012_1314.pdf 21 | ```TeX 22 | @article{hensman2012fast, 23 | title={Fast variational inference in the conjugate exponential family}, 24 | author={Hensman, James and Rattray, Magnus and Lawrence, Neil D}, 25 | journal={Advances in Neural Information Processing Systems}, 26 | year={2012} 27 | } 28 | ``` 29 | 30 | The code also implements clustering of Hierachical Gaussian Processes using that inference framework, detailed in the two following works: 31 | 32 | http://ieeexplore.ieee.org/xpls/abs_all.jsp?arnumber=6802369 33 | ```TeX 34 | @article{hensman2014fast, 35 | author={Hensman, J. and Rattray, M. and Lawrence, N.}, 36 | journal={Pattern Analysis and Machine Intelligence, IEEE Transactions on}, 37 | title={Fast nonparametric clustering of structured time-series}, 38 | year={2014}, 39 | volume={PP}, 40 | number={99}, 41 | keywords={Biological system modeling;Computational modeling;Data models;Gaussian processes;Optimization;Time series analysis}, 42 | doi={10.1109/TPAMI.2014.2318711}, 43 | ISSN={0162-8828} 44 | } 45 | ``` 46 | 47 | http://www.biomedcentral.com/1471-2105/14/252 48 | ```TeX 49 | @article{hensman2013hierarchical, 50 | title={Hierarchical Bayesian modelling of gene expression time series across irregularly sampled replicates and clusters}, 51 | author={Hensman, James and Lawrence, Neil D and Rattray, Magnus}, 52 | journal={BMC bioinformatics}, 53 | volume={14}, 54 | number={1}, 55 | pages={1--12}, 56 | year={2013}, 57 | publisher={BioMed Central} 58 | } 59 | ``` 60 | 61 | 62 | Additionally Overlapping Mixtures of Gaussian Processes model is implemented (using the variational methods described in the above), which was published in this paper: 63 | 64 | ```TeX 65 | @article{Lazaro-Gredilla2012, 66 | title = {{Overlapping Mixtures of Gaussian Processes for the data association problem}}, 67 | author = {L{\'{a}}zaro-Gredilla, Miguel and {Van Vaerenbergh}, Steven and Lawrence, Neil D.}, 68 | doi = {10.1016/j.patcog.2011.10.004}, 69 | journal = {Pattern Recognition}, 70 | month = {apr}, 71 | number = {4}, 72 | pages = {1386--1395}, 73 | url = {}, 74 | volume = {45}, 75 | year = {2012} 76 | } 77 | ``` 78 | 79 | 80 | 81 | Dependencies 82 | ------------ 83 | 84 | This work depends on the [GPy project](https://github.com/SheffieldML/GPy), as well as the numpy/scipy stack. matplotlib is optional for plotting. 85 | 86 | I've tested the demos with GPy v0.8, but it should work with later versions also. 87 | 88 | 89 | Contributors 90 | ------------ 91 | 92 | - James Hensman 93 | - Valentine Svensson 94 | - Max Zwiessele 95 | -------------------------------------------------------------------------------- /data/kalinka09_pdata.csv: -------------------------------------------------------------------------------- 1 | an,1,2 2 | an,1,4 3 | an,1,6 4 | an,1,8 5 | an,1,10 6 | an,1,12 7 | an,1,14 8 | an,1,16 9 | an,1,18 10 | an,2,2 11 | an,2,4 12 | an,2,6 13 | an,2,8 14 | an,2,10 15 | an,2,12 16 | an,2,14 17 | an,2,16 18 | an,2,18 19 | an,3,2 20 | an,3,4 21 | an,3,6 22 | an,3,8 23 | an,3,10 24 | an,3,12 25 | an,3,14 26 | an,3,16 27 | an,3,18 28 | me,1,2 29 | me,1,4 30 | me,1,6 31 | me,1,8 32 | me,1,10 33 | me,1,12 34 | me,1,14 35 | me,1,16 36 | me,2,2 37 | me,2,4 38 | me,2,6 39 | me,2,8 40 | me,2,10 41 | me,2,12 42 | me,2,14 43 | me,2,16 44 | me,3,2 45 | me,3,4 46 | me,3,6 47 | me,3,8 48 | me,3,10 49 | me,3,12 50 | me,3,14 51 | me,3,16 52 | me,4,2 53 | me,4,4 54 | me,4,6 55 | me,4,8 56 | me,4,10 57 | me,4,12 58 | me,4,14 59 | me,4,16 60 | me,5,2 61 | me,5,4 62 | me,5,6 63 | me,5,8 64 | me,5,10 65 | me,5,12 66 | me,5,14 67 | me,5,16 68 | me,5,18 69 | me,5,20 70 | me,6,12 71 | me,6,14 72 | me,6,16 73 | me,6,18 74 | me,6,20 75 | me,7,12 76 | me,7,14 77 | me,7,16 78 | me,7,18 79 | me,8,12 80 | me,8,14 81 | me,8,16 82 | me,8,18 83 | me,8,20 84 | pr,1,2 85 | pr,1,4 86 | pr,1,6 87 | pr,1,8 88 | pr,1,10 89 | pr,1,12 90 | pr,1,14 91 | pr,1,16 92 | pr,1,18 93 | pr,2,2 94 | pr,2,4 95 | pr,2,6 96 | pr,2,8 97 | pr,2,10 98 | pr,2,12 99 | pr,2,14 100 | pr,2,16 101 | pr,2,18 102 | pr,3,2 103 | pr,3,4 104 | pr,3,6 105 | pr,3,8 106 | pr,3,10 107 | pr,3,12 108 | pr,3,14 109 | pr,3,16 110 | pr,3,18 111 | ps,1,2 112 | ps,1,4 113 | ps,1,6 114 | ps,1,8 115 | ps,1,10 116 | ps,1,12 117 | ps,1,14 118 | ps,1,16 119 | ps,1,18 120 | ps,2,2 121 | ps,2,4 122 | ps,2,6 123 | ps,2,8 124 | ps,2,10 125 | ps,2,12 126 | ps,2,14 127 | ps,2,16 128 | ps,2,18 129 | ps,3,2 130 | ps,3,4 131 | ps,3,6 132 | ps,3,8 133 | ps,3,10 134 | ps,3,12 135 | ps,3,14 136 | ps,3,16 137 | ps,3,18 138 | si,1,2 139 | si,1,4 140 | si,1,6 141 | si,1,8 142 | si,1,10 143 | si,1,12 144 | si,1,14 145 | si,1,16 146 | si,1,18 147 | si,2,2 148 | si,2,4 149 | si,2,6 150 | si,2,8 151 | si,2,10 152 | si,2,12 153 | si,2,14 154 | si,2,16 155 | si,2,18 156 | si,3,2 157 | si,3,4 158 | si,3,6 159 | si,3,8 160 | si,3,10 161 | si,3,12 162 | si,3,14 163 | si,3,16 164 | si,3,18 165 | vi,1,2 166 | vi,1,4 167 | vi,1,6 168 | vi,1,8 169 | vi,1,10 170 | vi,1,12 171 | vi,1,14 172 | vi,1,16 173 | vi,1,18 174 | vi,1,20 175 | vi,2,2 176 | vi,2,4 177 | vi,2,6 178 | vi,2,8 179 | vi,2,10 180 | vi,2,12 181 | vi,2,14 182 | vi,2,16 183 | vi,2,18 184 | vi,2,20 185 | vi,2,22 186 | vi,2,24 187 | vi,2,26 188 | vi,3,2 189 | vi,3,4 190 | vi,3,6 191 | vi,3,8 192 | vi,3,10 193 | vi,3,12 194 | vi,3,14 195 | vi,3,16 196 | vi,3,18 197 | vi,3,20 198 | vi,3,22 199 | vi,3,24 200 | vi,3,26 201 | -------------------------------------------------------------------------------- /data/split_data_test.csv: -------------------------------------------------------------------------------- 1 | true_assignment,x,y 2 | 0,0.26944731816407386,0.10613021313174695 3 | 1,0.3080905459463089,0.00768973634856418 4 | 0,0.45754941572622965,0.20571035756392977 5 | 0,0.29031668723629356,0.09558820173680577 6 | 1,0.17724843483737573,0.10976595316236898 7 | 1,0.9060625365740629,0.12744777012573377 8 | 1,0.758249677804775,0.23468805511063173 9 | 0,0.9851901896240195,0.8956037002390912 10 | 0,0.27786343679926495,0.0592154831157746 11 | 1,0.6937536230254309,0.21722742465579803 12 | 1,0.40406566215961626,0.1735473592669341 13 | 1,0.5004498652058665,0.23712479939472147 14 | 0,0.6159397061362147,0.3503320095711528 15 | 0,0.39056722822891154,0.19172158510021903 16 | 1,0.7905568676423207,0.30526930543982567 17 | 0,0.977437550236215,0.8433563860163114 18 | 0,0.30304707529034003,0.054783596586953455 19 | 0,0.7250228843698485,0.4204827649621661 20 | 1,0.08002849302662907,-0.01278941177378163 21 | 1,0.9871891817128337,0.0348358172745965 22 | 1,0.5489786984015347,0.20989346455555494 23 | 0,0.8777788853841478,0.627828618892456 24 | 1,0.4349126062507641,0.10297013635312939 25 | 1,0.6075580437923078,0.21654514189347251 26 | 0,0.7765860913640333,0.4583336439386213 27 | 1,0.4284863484198774,0.16999619229769594 28 | 1,0.624758449317914,0.30816353523164963 29 | 0,0.3651996005906146,0.1606172189551325 30 | 1,0.025777626846202217,-0.00186646761068353 31 | 1,0.49522656618417205,0.1736037473264947 32 | 0,0.8343381097481938,0.5629192905096188 33 | 0,0.99086360135717,1.039047939120119 34 | 0,0.6709128750364762,0.33902252909393993 35 | 0,0.9451574416087752,0.7955180296879012 36 | 1,0.792610455498458,0.17216949440868964 37 | 0,0.3120996999953315,0.19765725468501183 38 | 0,0.30369770632891013,0.10700118454010378 39 | 1,0.9682161550126684,0.0010760871753122472 40 | 1,0.8940361613449848,0.1419593035338989 41 | 1,0.020255057127560927,-0.007365716747847513 42 | 1,0.464829895570563,0.23393242477514758 43 | 1,0.03676651318278423,0.05195772420916275 44 | 0,0.7157333966064577,0.44842542450624884 45 | 1,0.03328474753372168,0.0459411602893534 46 | 1,0.10935666824209456,0.0008327353259498117 47 | 0,0.39020866338227667,0.12747174366936195 48 | 0,0.7403078869637189,0.4665334125794267 49 | 1,0.9538793259940255,0.02706700530047418 50 | 0,0.6923420930329253,0.35829914602150015 51 | 1,0.608792332113599,0.2875305808386123 52 | 1,0.30924426861360066,0.14634381153058207 53 | 0,0.005836888952079566,0.04532806579876944 54 | 0,0.45228628067301824,0.19627938568588982 55 | 0,0.3795598997420837,0.09733928082271925 56 | 0,0.5830204928984701,0.30396816655882053 57 | 0,0.5252653933610343,0.1743911151969374 58 | 0,0.7218264993093295,0.41753792751446334 59 | 0,0.6474305074589235,0.3063082343583381 60 | 0,0.3071903404809774,0.023060759395148706 61 | 1,0.15381552385227526,-0.009380004321090717 62 | 1,0.6397240126779914,0.2779284918481963 63 | 0,0.11632120631369003,0.03615492937447649 64 | 1,0.8645206910318733,0.20201279325852997 65 | 1,0.5550485561305949,0.20078590217621342 66 | 0,0.8543529296014675,0.5556491696263306 67 | 0,0.6681891864431283,0.3180113886212027 68 | 1,0.6940854325101692,0.21961220289055006 69 | 0,0.29225843131146656,0.057739123840681375 70 | 0,0.5972740555020588,0.2810685959411986 71 | 0,0.36265277710289623,0.09105943476972465 72 | 0,0.0630022422263351,-0.04695357659079026 73 | 0,0.2211016352962356,0.004589507856537979 74 | 0,0.16886961561962488,0.0005482130520842836 75 | 0,0.7685768025168216,0.4547253108273449 76 | 0,0.10447113781982131,0.04338776615673046 77 | 1,0.5155153751962936,0.1875120648709179 78 | 0,0.6902176202367271,0.3528232062363659 79 | 0,0.8309607871772322,0.5838869959027081 80 | 1,0.21437575393198482,0.05540056193868539 81 | 0,0.7279300160901631,0.38369588209283095 82 | 0,0.5807093910505056,0.32739668900370633 83 | 0,0.08140132300670633,-0.03935002148870382 84 | 0,0.32996900823702047,0.13767511319869435 85 | 0,0.7365782849619551,0.34325077239339075 86 | 0,0.4545558507135433,0.1326265397645032 87 | 0,0.816897343567411,0.5883694996798241 88 | 1,0.21585600162941376,0.035163130312515084 89 | 0,0.16544579752901445,0.12578910168079055 90 | 0,0.33406418239580804,0.18817887757154894 91 | 0,0.04492838998308357,-0.0039166079875242055 92 | 1,0.45834676252172646,0.12172804084757342 93 | 1,0.3887629715979901,0.12589484987335306 94 | 0,0.6958275454451119,0.4062979796663833 95 | 0,0.5736769872034806,0.24466896130599627 96 | 0,0.5936945578278469,0.2172670336687902 97 | 0,0.9074731382226368,0.7224547735102202 98 | 0,0.53570148790686,0.18030765377963032 99 | 0,0.572109518254989,0.2639716090727371 100 | 0,0.7591084549923985,0.4296342955941084 101 | 0,0.8387933386839894,0.5394564473173437 102 | 0,0.8370448690675084,0.4739293745024031 103 | 1,0.038270549303624724,0.019578017169448434 104 | 1,0.8743259664530618,0.18495371511700656 105 | 0,0.6478263836846848,0.28275733196268094 106 | 1,0.1935461538894534,0.020681758629480407 107 | 0,0.1619669714527313,-0.014800909935878986 108 | 1,0.9076712844397925,0.1559130122978471 109 | 0,0.04308635604706934,-0.057183743389338 110 | 1,0.8507187852993945,0.14366242192919046 111 | 1,0.2927261550159391,0.08059136328276911 112 | 0,0.43834247317186714,0.18726587410650658 113 | 1,0.39337214044510627,0.15058036054331542 114 | 0,0.24892868544249602,0.08935864260581847 115 | 1,0.484973449651514,0.1501720827645334 116 | 0,0.24104536073518523,0.112029929298041 117 | 0,0.4922794326542468,0.13122489968520493 118 | 1,0.054853412914733224,0.03319534331051651 119 | 0,0.658632169407443,0.4319138698140551 120 | 0,0.4852359424921997,0.1900915233167908 121 | 0,0.3106851725305172,0.17549752446643266 122 | 0,0.8907940756346614,0.6523798902952731 123 | 0,0.6186244287487508,0.35197779554806585 124 | 1,0.932723524018418,0.12335835061439412 125 | 1,0.7166366212680887,0.30564955173398156 126 | 0,0.9008318832370299,0.7135220801551403 127 | 1,0.9194528671715895,0.14019664308049584 128 | 1,0.1878988156452457,-0.05613392509995276 129 | 1,0.07997008111327375,0.0710352556939032 130 | 1,0.9627496174897492,0.12410551817229179 131 | 0,0.495281279917581,0.18823964939760446 132 | 0,0.6124298154451867,0.17229230473336957 133 | 1,0.7068840746188141,0.20681655438802085 134 | 0,0.06749001127365473,0.021896989994948556 135 | 1,0.3322582806248032,0.05214986372504314 136 | 0,0.8556139273633735,0.5378820850791795 137 | 1,0.13099806415090298,0.13224271968558962 138 | 1,0.640118196248423,0.2713308755118142 139 | 1,0.16333178313761,0.08088912883848982 140 | 1,0.3187368619934512,0.019096999083299718 141 | 0,0.9919688489670457,1.0028803099913965 142 | 1,0.927666094290259,0.12438832744588786 143 | 0,0.9300288380784921,0.7797691929893837 144 | 1,0.3592229601194792,0.21170837584676172 145 | 1,0.19761454326498873,0.0461528969730953 146 | 0,0.09590169771142787,0.008569784181562628 147 | 0,0.3490579767886961,0.08577878051354759 148 | 1,0.14656296639383404,0.009723295022983507 149 | 0,0.03804097931534278,0.03483989358801512 150 | 1,0.9294715745368798,0.06407365163877042 151 | 1,0.7641033663830963,0.2934459957842875 152 | 0,0.49869680238285874,0.18248155969084104 153 | 0,0.8965235130418845,0.7047303787847109 154 | 1,0.7771918658781837,0.23975830767686362 155 | 1,0.35641454999943567,0.037747753923473404 156 | 1,0.9500618646908954,0.04453791064951886 157 | 0,0.20748018066768448,0.05406473552534324 158 | 1,0.11538507507102413,0.030969669679137956 159 | 0,0.1759143211994364,0.08102157224545231 160 | 0,0.924651615468185,0.8305065969986027 161 | 1,0.07069149996048774,-0.008638804997579286 162 | 1,0.5276027211062466,0.16103637673917875 163 | 0,0.07310351491442413,-0.027450238371271276 164 | 1,0.6216147842492896,0.26148520035819317 165 | 1,0.2873078780444983,0.09990839043927181 166 | 1,0.2840616410174287,0.15498036720287273 167 | 1,0.25394601797983263,0.08525271423558789 168 | 1,0.020603896884660133,-0.09434443483174106 169 | 0,0.8496500731215241,0.546851632007451 170 | 1,0.8282163421179117,0.20734257912313245 171 | 0,0.8135730019488471,0.4648351278280812 172 | 0,0.28825492107043793,0.027424684924698524 173 | 1,0.01999444998209776,0.08093315805100537 174 | 1,0.5812600309402733,0.24062245350013892 175 | 0,0.9956305804050609,1.0788402223587195 176 | 1,0.8895010016989473,0.24801069156404468 177 | 0,0.5646392937057493,0.1792575307574406 178 | 0,0.6012932941415747,0.14878592484207612 179 | 1,0.8088547199433307,0.16115068041085923 180 | 0,0.05233718064963777,0.03869112390917457 181 | 0,0.01985577631160007,-0.01643243597898435 182 | 0,0.20591372133872676,0.014902348703527173 183 | 1,0.8852662267460125,0.2266407334619766 184 | 1,0.8010545536933499,0.23701251930510872 185 | 0,0.8010638110936322,0.49215649309385134 186 | 1,0.27803793696011103,0.1262734430151728 187 | 0,0.4289795065398454,0.23326150443580057 188 | 1,0.7771748525674183,0.2437930701268423 189 | 1,0.13278995345756595,-0.003918683175591107 190 | 1,0.6965646052403289,0.3056064144003644 191 | 0,0.4766218870614254,0.17988650608486073 192 | 1,0.913269165078939,0.07453940606933637 193 | 0,0.7439723116413347,0.3912765193342829 194 | 1,0.2610016190707468,0.12579512650577776 195 | 0,0.22352808811285374,0.05207005428647828 196 | 1,0.016256864152359607,-0.005625205833047342 197 | 0,0.9160354971724081,0.7160226644811193 198 | 0,0.014513865606277765,-0.09272430992608217 199 | 0,0.18457740792993038,0.06702938538839545 200 | 1,0.5128806619878862,0.21389409808711968 201 | 0,0.7471454260348308,0.34434585501508164 202 | 1,0.5017818876946541,0.20726785251382496 203 | 0,0.1297205334575563,0.033729472334926694 204 | 0,0.003234720482852338,-0.002524522345720779 205 | 0,0.4328218389104529,0.2223350407680289 206 | 1,0.20574020602786913,-0.0002315999324248863 207 | 1,0.3711360592825701,0.17769694833840632 208 | 0,0.806897683911629,0.5019200465205015 209 | 1,0.7080777639490293,0.25480001743724545 210 | 0,0.6972563519751388,0.41708564995873254 211 | 1,0.18568770134621282,-0.04286170707715712 212 | 0,0.6415044573771345,0.34030611195324795 213 | 1,0.9928133285113511,0.0220564249881334 214 | 0,0.847317053768947,0.6115242506305621 215 | 0,0.37658974396607536,0.14053649724405637 216 | 0,0.3986402817239665,0.1158317619060288 217 | 1,0.6575074910338058,0.1996922824252083 218 | 0,0.14014445085439986,0.013082278010295333 219 | 1,0.6300657231514306,0.2893605775383225 220 | 0,0.09859562396778532,0.03383518220575525 221 | 0,0.5916408914354747,0.34276390068841234 222 | 0,0.2624300009903654,0.05603083789068776 223 | 1,0.2277246320538846,0.002359685719498253 224 | 1,0.42502857220263035,0.0982730720681538 225 | 1,0.574353799266372,0.2737818892680943 226 | 0,0.138374557789682,-0.008477812023701344 227 | 1,0.8833689375727819,0.1189636067896693 228 | 0,0.724608374715159,0.42488000142814325 229 | 1,0.3321432427926464,0.15370526645882573 230 | 0,0.22877281749995393,0.024441751147991162 231 | 1,0.6461324781311847,0.2857182192157706 232 | 0,0.5756723195050207,0.26108489226706616 233 | 1,0.0027676334005887204,0.00531800371304411 234 | 0,0.8485087910533532,0.5590760531589241 235 | 1,0.02615009336249241,-0.11343381147526692 236 | 1,0.5231583100393972,0.2139630732529224 237 | 1,0.7804064167485047,0.26714156409753576 238 | 0,0.19908953423683606,0.06792398093307078 239 | 1,0.8285742488912631,0.196276288084756 240 | 1,0.2734071818223661,0.03722628303284128 241 | 1,0.8110463253350259,0.21264780121136811 242 | 1,0.8372197240967527,0.20625491866351578 243 | 1,0.5004753896775483,0.16549404013509098 244 | 0,0.8363723472209383,0.5147283944913454 245 | 0,0.9225682246308973,0.7503974285826336 246 | 0,0.1605803591802668,-0.052348398798046494 247 | 1,0.8359948425900717,0.20485773207650967 248 | 1,0.5461409268771638,0.15111904100098097 249 | 0,0.9328032391161373,0.723568948323747 250 | 1,0.9375423751484272,0.1930798095800813 251 | 0,0.5723161346044439,0.19172284078751237 252 | -------------------------------------------------------------------------------- /notebooks/MOG_demo.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2012 James Hensman 2 | # Licensed under the GPL v3 (see LICENSE.txt) 3 | 4 | import numpy as np 5 | import pylab as pb 6 | import sys 7 | sys.path.append('..') 8 | from colvb import MOG 9 | pb.ion() 10 | 11 | np.random.seed(0) 12 | 13 | #make some Data which appears in clusters: 14 | Nclust = 15 15 | dim = 2 16 | Nmin = 25 17 | Nmax = 50 18 | Ndata = np.random.randint(Nmin, Nmax, Nclust) 19 | means = np.random.randn(Nclust, dim)*5 20 | aa = [np.random.randn(dim, dim+1) for i in range(Nclust)] 21 | Sigmas = [np.dot(a, a.T) for a in aa] 22 | X = np.vstack([np.random.multivariate_normal(mu, cov, (n,)) for mu, cov, n in zip(means, Sigmas, Ndata)])/100 23 | Nrestarts=3 24 | Nclust = 15 25 | 26 | m = MOG(X, Nclust, prior_Z='DP') 27 | 28 | #starts = [np.random.randn(m.N*m.K) for i in range(Nrestarts)] 29 | from scipy.cluster import vq 30 | starts = [] 31 | for i in range(Nrestarts): 32 | means = X[np.random.permutation(X.shape[0])[:Nclust]] 33 | dists = np.square(X[:,:,None]-means.T[None,:,:]).sum(1) 34 | starts.append(dists) 35 | 36 | for method in ['steepest', 'PR', 'FR', 'HS']: 37 | for st in starts: 38 | m.set_vb_param(st) 39 | m.optimize(method=method, maxiter=1e4) 40 | 41 | m.plot_tracks() 42 | m.plot() 43 | 44 | 45 | 46 | 47 | 48 | -------------------------------------------------------------------------------- /notebooks/MOHGP_demo.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pylab as pb 3 | import sys 4 | import GPy 5 | from colvb import MOHGP 6 | np.random.seed(1) 7 | pb.close('all') 8 | 9 | #cool structed GP demo 10 | Nclust = 20 11 | Nx = 12 12 | Nobs = [np.random.randint(20,21) for i in range(Nclust)] 13 | X = np.random.rand(Nx,1)*5 14 | X.sort(0) 15 | 16 | Kf = GPy.kern.RBF(1) + GPy.kern.White(1, 1e-6) 17 | S = Kf.K(X) 18 | means = np.vstack([np.tile(np.random.multivariate_normal(np.zeros(Nx),S,1),(N,1)) for N in Nobs]) # GP draws for mean of each cluster 19 | 20 | #add GP draw for noise 21 | Ky = GPy.kern.RBF(1,0.3,1) + GPy.kern.White(1,0.001) 22 | Y = means + np.random.multivariate_normal(np.zeros(Nx),Ky.K(X),means.shape[0]) 23 | 24 | #construct model 25 | m = MOHGP(X, Kf.copy(), Ky.copy(), Y, K=Nclust) 26 | m.constrain_positive('') 27 | 28 | m.optimize() 29 | m.preferred_optimizer='bfgs' 30 | m.systematic_splits() 31 | m.remove_empty_clusters(1e-3) 32 | m.plot(1,1,1,0,0,1) 33 | raw_input('press enter to continue ...') 34 | 35 | #and again without structure 36 | Y -= Y.mean(1)[:,None] 37 | Y /= Y.std(1)[:,None] 38 | m2 = MOHGP(X, Kf, GPy.kern.White(1), Y, K=Nclust) 39 | m2.constrain_positive('') 40 | m2.preferred_optimizer='bfgs' 41 | m2.optimize() 42 | m2.systematic_splits() 43 | m2.remove_empty_clusters(1e-3) 44 | m2.plot(1,1,1,0,0,1) 45 | -------------------------------------------------------------------------------- /notebooks/MOHGP_sine_demo.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pylab as pb 3 | pb.close('all') 4 | pb.ion() 5 | from GPy import kern 6 | import sys 7 | sys.path.append('..') 8 | from colvb import MOHGP, MOG, utilities 9 | np.random.seed(0) 10 | 11 | #cool structed GP demo 12 | Nclust = 10 13 | Nx = 12 14 | Nobs = [np.random.randint(20,31) for i in range(Nclust)] 15 | X = np.random.rand(Nx,1) 16 | X.sort(0) 17 | ground_truth_phi = utilities.blockdiag([np.ones((n,1)) for n in Nobs]) 18 | alpha = 2. #(should give approximately 10 clusters) 19 | 20 | freqs = 2*np.pi + 0.3*(np.random.rand(Nclust)-.5) 21 | phases = 2*np.pi*np.random.rand(Nclust) 22 | means = np.vstack([np.tile(np.sin(f*X+p).T,(Ni,1)) for f,p,Ni in zip(freqs,phases,Nobs)]) 23 | 24 | #add a lower freq sin for the noise 25 | freqs = .4*np.pi + 0.01*(np.random.rand(means.shape[0])-.5) 26 | phases = 2*np.pi*np.random.rand(means.shape[0]) 27 | offsets = 0.3*np.vstack([np.sin(f*X+p).T for f,p in zip(freqs,phases)]) 28 | Y = means + offsets + np.random.randn(*means.shape)*0.05 29 | 30 | 31 | #construct full model 32 | Kf = kern.rbf(1,0.01,0.001) 33 | Ky1 = kern.rbf(1,0.1,0.001) 34 | Ky2 = kern.white(1,0.01) 35 | Ky = Ky1 + Ky2 36 | m = MOHGP(X,Kf,Ky,Y, K=Nclust, prior_Z = 'DP', alpha=alpha) 37 | m.ensure_default_constraints() 38 | m.checkgrad(verbose=1) 39 | 40 | m.randomize() 41 | m.optimize() 42 | m.systematic_splits() 43 | m.systematic_splits() 44 | m.plot(1,1,1,0,0,1) 45 | 46 | #construct model without structure 47 | #give it a fighting chance by normalising signals first 48 | Y = Y.copy() 49 | Y -= Y.mean(1)[:,None] 50 | Y /= Y.std(1)[:,None] 51 | Kf = kern.rbf(1,0.01,0.001) 52 | Ky = kern.white(1,0.01) 53 | m2 = MOHGP(X,Kf,Ky,Y, K=Nclust, prior_Z = 'DP', alpha=alpha) 54 | m2.ensure_default_constraints() 55 | m2.checkgrad(verbose=1) 56 | 57 | m2.randomize() 58 | m2.optimize() 59 | m2.systematic_splits() 60 | m2.systematic_splits() 61 | m2.plot(1,1,1,0,0,1) 62 | 63 | #construct a MOG model (can't recover the clusters) 64 | Y_ = Y.copy() 65 | Y_ -= Y_.mean(0) 66 | Y_ /= Y_.std(0) 67 | m3 = MOG(Y_, prior_Z='DP', alpha=alpha) 68 | m3.randomize() 69 | m3.optimize() 70 | m3.systematic_splits() 71 | m3.systematic_splits() 72 | pb.figure() 73 | pb.subplot(2,2,1) 74 | pb.imshow(ground_truth_phi,aspect='auto',cmap=pb.cm.gray) 75 | pb.title('ground truth') 76 | pb.subplot(2,2,2) 77 | pb.imshow(m.phi,aspect='auto',cmap=pb.cm.gray) 78 | pb.title('structured GP-DP') 79 | pb.subplot(2,2,3) 80 | pb.imshow(m2.phi,aspect='auto',cmap=pb.cm.gray) 81 | pb.title('unstructured GP-DP') 82 | pb.subplot(2,2,4) 83 | pb.imshow(m3.phi,aspect='auto',cmap=pb.cm.gray) 84 | pb.title('DP mixture model') 85 | 86 | 87 | 88 | 89 | 90 | 91 | -------------------------------------------------------------------------------- /notebooks/index.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Mixtures of Gaussian Processes with GPclust\n", 8 | "\n", 9 | "*James Hensman, November 2014
Valentine Svensson, November 2015*\n", 10 | "\n", 11 | "Here are a few notebooks outlining the functionality of GPclust.\n", 12 | "\n", 13 | "You may also be interested in reading some of the tutorials on [GPy](http://sheffieldml.github.io/GPy/), the Gaussian process framework that GPclust is built on." 14 | ] 15 | }, 16 | { 17 | "cell_type": "markdown", 18 | "metadata": {}, 19 | "source": [ 20 | "## Mixtures of Gaussians\n", 21 | "The [MOG notebook](./mixture of Gaussians.ipynb) described how to use GPclust to infer a basic mixture of Gaussians model. This also illustrates some of the workings of GPclust such as the merge-split method, truncated Dirichlet process approximations and the setting of parameters.\n", 22 | "\n" 23 | ] 24 | }, 25 | { 26 | "cell_type": "markdown", 27 | "metadata": {}, 28 | "source": [ 29 | "## Mixtures of Gaussian Processes\n", 30 | "The [first MOHGP notebook](./MOHGP_demo.ipynb) demonstrates using GPclust's MOHGP (Mixture of Hierarchical Gaussian Processes) class on a synthetic dataset. We show how to build the model, how to deal with the kernel (covariance function) parameteres, and basic plotting. \n", 31 | "\n", 32 | "The MOHGP model is hierarchical because is uses a hierachy of GPs to model first the mean of the cluster, and then the deviation of each time-course in the cluster from that mean. You may be interested in the notebook on [hierarchical GPs in GPy](https://github.com/SheffieldML/notebook/blob/master/compbio/Hierarchical.ipynb)\n", 33 | "\n", 34 | "The next notebook illustates using [GPclust on the Drosphila](./drosophila.ipynb) data of Kalinka et al (2009)." 35 | ] 36 | }, 37 | { 38 | "cell_type": "markdown", 39 | "metadata": { 40 | "collapsed": false 41 | }, 42 | "source": [ 43 | "## Overlapping Mixtures of Gaussian Processes\n", 44 | "\n", 45 | "A [notebook](./OMGP_demo.ipynb) describing a couple of applications of the OMGP model is included. The OMGP model is generally useful for heteroscedastic trends. The illustratory examples shows how to model a time series with diverging populations over time, and how to separate noise from signal. " 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": null, 51 | "metadata": { 52 | "collapsed": true 53 | }, 54 | "outputs": [], 55 | "source": [] 56 | } 57 | ], 58 | "metadata": { 59 | "kernelspec": { 60 | "display_name": "Python 2", 61 | "language": "python", 62 | "name": "python2" 63 | }, 64 | "language_info": { 65 | "codemirror_mode": { 66 | "name": "ipython", 67 | "version": 2 68 | }, 69 | "file_extension": ".py", 70 | "mimetype": "text/x-python", 71 | "name": "python", 72 | "nbconvert_exporter": "python", 73 | "pygments_lexer": "ipython2", 74 | "version": "2.7.10" 75 | } 76 | }, 77 | "nbformat": 4, 78 | "nbformat_minor": 0 79 | } 80 | -------------------------------------------------------------------------------- /notebooks/twoD_clustering_example.numpy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SheffieldML/GPclust/9f746598f2e383db5a4e89ef3bfc0270451c7873/notebooks/twoD_clustering_example.numpy -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.md 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name = "GPclust", 5 | version = "0.1.0", 6 | author = "James Hensman", 7 | author_email = "james.hensman@sheffield.ac.uk", 8 | url = "http://staffwww.dcs.sheffield.ac.uk/people/J.Hensman/gpclust.html", 9 | description = ("Clustering of time series using Gaussian processes and variational Bayes"), 10 | license = "GPL v3", 11 | keywords = " clustering Gaussian-process machine-learning", 12 | download_url = 'https://github.com/jameshensman/gpclust/tarball/0.1', 13 | packages=['GPclust'], 14 | install_requires=['GPy>=0.6'], 15 | classifiers=[] 16 | ) 17 | --------------------------------------------------------------------------------