├── LapSVM.py ├── README.md └── results.png /LapSVM.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from scipy.optimize import minimize 3 | from scipy.spatial.distance import cdist 4 | from sklearn.neighbors import kneighbors_graph 5 | from scipy import sparse 6 | 7 | 8 | 9 | class LapSVM(object): 10 | def __init__(self,opt): 11 | self.opt=opt 12 | 13 | 14 | def fit(self,X,Y,X_u): 15 | #construct graph 16 | self.X=np.vstack([X,X_u]) 17 | Y=np.diag(Y) 18 | if self.opt['neighbor_mode']=='connectivity': 19 | W = kneighbors_graph(self.X, self.opt['n_neighbor'], mode='connectivity',include_self=False) 20 | W = (((W + W.T) > 0) * 1) 21 | elif self.opt['neighbor_mode']=='distance': 22 | W = kneighbors_graph(self.X, self.opt['n_neighbor'], mode='distance',include_self=False) 23 | W = W.maximum(W.T) 24 | W = sparse.csr_matrix((np.exp(-W.data**2/4/self.opt['t']),W.indices,W.indptr),shape=(self.X.shape[0],self.X.shape[0])) 25 | else: 26 | raise Exception() 27 | 28 | # Computing Graph Laplacian 29 | L = sparse.diags(np.array(W.sum(0))[0]).tocsr() - W 30 | 31 | # Computing K with k(i,j) = kernel(i, j) 32 | K = self.opt['kernel_function'](self.X,self.X,**self.opt['kernel_parameters']) 33 | 34 | l=X.shape[0] 35 | u=X_u.shape[0] 36 | # Creating matrix J [I (l x l), 0 (l x (l+u))] 37 | J = np.concatenate([np.identity(l), np.zeros(l * u).reshape(l, u)], axis=1) 38 | 39 | # Computing "almost" alpha 40 | almost_alpha = np.linalg.inv(2 * self.opt['gamma_A'] * np.identity(l + u) \ 41 | + ((2 * self.opt['gamma_I']) / (l + u) ** 2) * L.dot(K)).dot(J.T).dot(Y) 42 | 43 | # Computing Q 44 | Q = Y.dot(J).dot(K).dot(almost_alpha) 45 | Q = (Q+Q.T)/2 46 | 47 | del W, L, K, J 48 | 49 | e = np.ones(l) 50 | q = -e 51 | 52 | # ===== Objectives ===== 53 | def objective_func(beta): 54 | return (1 / 2) * beta.dot(Q).dot(beta) + q.dot(beta) 55 | 56 | def objective_grad(beta): 57 | return np.squeeze(np.array(beta.T.dot(Q) + q)) 58 | 59 | # =====Constraint(1)===== 60 | # 0 <= beta_i <= 1 / l 61 | bounds = [(0, 1 / l) for _ in range(l)] 62 | 63 | # =====Constraint(2)===== 64 | # Y.dot(beta) = 0 65 | def constraint_func(beta): 66 | return beta.dot(np.diag(Y)) 67 | 68 | def constraint_grad(beta): 69 | return np.diag(Y) 70 | 71 | cons = {'type': 'eq', 'fun': constraint_func, 'jac': constraint_grad} 72 | 73 | # ===== Solving ===== 74 | x0 = np.zeros(l) 75 | 76 | beta_hat = minimize(objective_func, x0, jac=objective_grad, constraints=cons, bounds=bounds)['x'] 77 | 78 | # Computing final alpha 79 | self.alpha = almost_alpha.dot(beta_hat) 80 | 81 | del almost_alpha, Q 82 | 83 | # Finding optimal decision boundary b using labeled data 84 | new_K = self.opt['kernel_function'](self.X,X,**self.opt['kernel_parameters']) 85 | f = np.squeeze(np.array(self.alpha)).dot(new_K) 86 | 87 | self.sv_ind=np.nonzero((beta_hat>1e-7)*(beta_hat<(1/l-1e-7)))[0] 88 | 89 | ind=self.sv_ind[0] 90 | self.b=np.diag(Y)[ind]-f[ind] 91 | 92 | 93 | def decision_function(self,X): 94 | new_K = self.opt['kernel_function'](self.X, X, **self.opt['kernel_parameters']) 95 | f = np.squeeze(np.array(self.alpha)).dot(new_K) 96 | return f+self.b 97 | 98 | def rbf(X1,X2,**kwargs): 99 | return np.exp(-cdist(X1,X2)**2*kwargs['gamma']) 100 | 101 | if __name__ == "__main__": 102 | from sklearn.datasets import make_moons 103 | import matplotlib.pyplot as plt 104 | np.random.seed(5) 105 | 106 | X, Y = make_moons(n_samples=200, noise=0.05) 107 | ind_0 = np.nonzero(Y == 0)[0] 108 | ind_1 = np.nonzero(Y == 1)[0] 109 | Y[ind_0] = -1 110 | 111 | ind_l0=np.random.choice(ind_0,1,False) 112 | ind_u0=np.setdiff1d(ind_0,ind_l0) 113 | 114 | ind_l1 = np.random.choice(ind_1, 1, False) 115 | ind_u1 = np.setdiff1d(ind_1, ind_l1) 116 | 117 | Xl=np.vstack([X[ind_l0,:],X[ind_l1,:]]) 118 | Yl=np.hstack([Y[ind_l0],Y[ind_l1]]) 119 | Xu=np.vstack([X[ind_u0,:],X[ind_u1,:]]) 120 | 121 | 122 | plt.subplot(1,2,1) 123 | 124 | plt.scatter(Xl[:,0],Xl[:,1],marker='+',c=Yl) 125 | plt.scatter(Xu[:,0],Xu[:,1],marker='.') 126 | 127 | 128 | opt={'neighbor_mode':'connectivity', 129 | 'n_neighbor' : 5, 130 | 't': 1, 131 | 'kernel_function':rbf, 132 | 'kernel_parameters':{'gamma':10}, 133 | 'gamma_A':0.03125, 134 | 'gamma_I':10000} 135 | 136 | s=LapSVM(opt) 137 | s.fit(Xl,Yl,Xu) 138 | plt.scatter(Xl[s.sv_ind,0],Xl[s.sv_ind,1],marker='o',c=Yl[s.sv_ind]) 139 | 140 | Y_=s.decision_function(X) 141 | Y_pre=np.ones(X.shape[0]) 142 | Y_pre[Y_<0]=-1 143 | 144 | print(np.nonzero(Y_pre==Y)[0].shape[0]/X.shape[0]*100.) 145 | 146 | xv, yv = np.meshgrid(np.linspace(X[:,0].min(),X[:,0].max(),100), np.linspace(X[:,1].min(),X[:,1].max(),100)) 147 | XX=s.decision_function(np.hstack([xv.reshape([-1,1]),yv.reshape([-1,1])])).reshape([xv.shape[0],yv.shape[0]]) 148 | plt.contour(xv,yv,XX,[-1,0,1]) 149 | 150 | 151 | 152 | 153 | 154 | plt.subplot(1,2,2) 155 | 156 | plt.scatter(Xl[:, 0], Xl[:, 1], marker='+', c=Yl) 157 | plt.scatter(Xu[:, 0], Xu[:, 1], marker='.') 158 | 159 | opt = {'neighbor_mode': 'connectivity', 160 | 'n_neighbor': 5, 161 | 't': 1, 162 | 'kernel_function': rbf, 163 | 'kernel_parameters': {'gamma': 10}, 164 | 'gamma_A': 0.03125, 165 | 'gamma_I': 0} 166 | 167 | s = LapSVM(opt) 168 | s.fit(Xl, Yl, Xu) 169 | plt.scatter(Xl[s.sv_ind, 0], Xl[s.sv_ind, 1], marker='o', c=Yl[s.sv_ind]) 170 | 171 | Y_ = s.decision_function(X) 172 | Y_pre = np.ones(X.shape[0]) 173 | Y_pre[Y_ < 0] = -1 174 | 175 | print(np.nonzero(Y_pre == Y)[0].shape[0]/X.shape[0]*100.) 176 | 177 | xv, yv = np.meshgrid(np.linspace(X[:, 0].min(), X[:, 0].max(), 100), np.linspace(X[:, 1].min(), X[:, 1].max(), 100)) 178 | XX = s.decision_function(np.hstack([xv.reshape([-1, 1]), yv.reshape([-1, 1])])).reshape([xv.shape[0], yv.shape[0]]) 179 | plt.contour(xv, yv, XX, [-1, 0, 1]) 180 | 181 | 182 | plt.show() 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # LapSVM-python 2 | Laplacian Support Vector Machines 3 | 4 | Manifold Regularization: A Geometric Framework for Learning from Labeled and Unlabeled Examples 5 | 6 | ![Alt text](https://github.com/GuHongyang/LapSVM-python/blob/master/results.png) 7 | -------------------------------------------------------------------------------- /results.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GuHY777/LapSVM-python/be410fc2f31510bbe14353635ba29970f92625dc/results.png --------------------------------------------------------------------------------