├── README.md ├── SBtest.py └── SparseBayesian.py /README.md: -------------------------------------------------------------------------------- 1 | # Sparse-Bayesian-Learning 2 | Based on M.E.Tipping. Sparse Bayesian Learning and the Relevance Vector Machine. JMLR, 1:211-244, 2001 3 | -------------------------------------------------------------------------------- /SBtest.py: -------------------------------------------------------------------------------- 1 | from matplotlib import pyplot as plt 2 | from numpy import random as rd 3 | from scipy import io as sio 4 | import SparseBayesian as sb 5 | import numpy as np 6 | import time 7 | 8 | def regression(noise, basis, targets, test_basis, test_targets): 9 | options = sb.OptionSet(max_iter=500, diagnosticLevel=2, monitor=10) 10 | settings = sb.ParamSet(noiseStdDev=0.1) 11 | t_start = time.time() 12 | Param, HyperParam, D = sb.SparseBayesian('gaussian', basis, targets, options, settings) 13 | t_end = time.time() 14 | N,M = basis.shape 15 | w_infer = np.zeros((M,1)) 16 | w_inder[Param.relevant] = Param.value 17 | y = test_basis*w_infer 18 | error = np.mean(np.power(test_targets-y,2)) 19 | w_num = len(Param.relevant) 20 | 21 | f_Rows = 1 22 | f_Cols = 2 23 | SP_Likely = 1 24 | SP_WEIGHTS = 2 25 | 26 | fig = plt.figure(1) 27 | ax = fig.add_subplot(f_Rows, f_Cols, SP_LIKELY) 28 | lsteps = D.likelihood.shape[0] 29 | plt.plot(range(1,lsteps+1), D.likelihood, 'g-') 30 | ax.set_xlim(0, lstep+1) 31 | ax.set_xlabel('Iteration') 32 | ax.set_ylabel('Log Marginal Likelihood') 33 | ax.title('Log Marginal Likelihood Trace',fontsize=12) 34 | lim = ax.axis() 35 | dx = lim[1]-lim[0] 36 | dy = lim[3]-lim[2] 37 | s = 'Actural Noise: %.5f'%noise 38 | ax.text(lim[0]+0.1*dx,lim[2]+0.6*dy, s) 39 | s = 'Inferred Noise: %.5f'%(1/np.sqrt(HyperParam.beta)) 40 | ax.text(lim[0]+0.1*dx,lim[2]+0.5*dy, s) 41 | 42 | ax = fig.add_subplot(f_Rows, f_Cols, SP_WEIGHTS) 43 | ax.set_xlim(0, N+1) 44 | ax.set_ylabel('Inferred Weights') 45 | ax.stem(w_infer) 46 | ax.title('Inferred Weights (%d)'%len(Param.relevant),fontsize=12) 47 | return fig, error, w_num, t_end-t_start 48 | 49 | def classification(X, basis, targets): 50 | options = sb.OptionSet(max_iter=500, diagnosticLevel=2, monitor=10) 51 | t_start = time.time() 52 | Param, HyperParam, D = SparseBayesian('Bernoulli', basis, targets, options) 53 | N,M = basis.shape 54 | w_infer = np.zeros((M,1)) 55 | w_infer[Param.relevant] = Param.value 56 | y = np.divide(1,1+np.exp(-basis*w_infer)) 57 | c = np.double(y>=0.5) 58 | mis = c[c!=targets].shape[1]/N 59 | t_end = time.time() 60 | 61 | f_Rows = 1 62 | f_Cols = 2 63 | SP_LIKELY = 1 64 | SP_SCATTER = 2 65 | sqrtN = sqrt(size(X,1)) 66 | 67 | fig = plt.figure(1) 68 | ax = fig.add_subplot(f_Rows, f_Cols, SP_LIKELY) 69 | lsteps = D.likelihood.shape[0] 70 | ax.plot(range(1,lsteps+1), D.lieklihood, 'g-') 71 | ax.set_xlim(0, lsteps+1) 72 | ax.title('Log Marginal Likelihood Trace', fontsize=12) 73 | 74 | ax = subplot(f_Rows, f_Cols, SP_SCATTER) 75 | x_min = np.min(X[:,0]) 76 | x_max = np.max(X[:,0]) 77 | x_tick = (x_max-x_min)/sqrtN 78 | ax.set_xlim(x_min-x_tick, x_max+x_tick) 79 | y_min = np.min(X[:,1]) 80 | y_max = np.max(X[:,1]) 81 | y_tick = (y_max-y_min)/sqrtN 82 | ax.set_ylim(y_max-y_min)/sqrtN 83 | ax.scatter(X[:,0][targets==1],X[:,1][targets==1], 'r.') 84 | ax.scatter(X[:,0][targets==0],X[:,1][targets==0], 'b.') 85 | ax.scatter(X[:,0][c==1], X[:,1][c==1], 'ro') 86 | ax.scatter(X[:,0][c==0], X[:,1][c==0], 'bo') 87 | ax.title('Distribution of Classified Points', fontsize=12) 88 | 89 | return fig, mis, t_end-t_start 90 | 91 | def run(datafile, it, batch): 92 | raw = np.matrix(sio.loadmat(datafile)) 93 | error = np.zeros((it,1)) 94 | w_num = np.zeros((it,1)) 95 | time = np.zeros((it,1)) 96 | for i in range(it): 97 | N = batch*(i+1) 98 | basis = raw[range(N),0:-1] 99 | targets = raw[range(N),-1] 100 | noise = np.std(targets)*0.2 101 | test_basis = raw[N+1:,0:-1] 102 | test_targets = raw[N+1:,-1] 103 | fig1, error[i,0], w_num[i,0], time[i,0] = regression(noise, basis, targets, test_basis, test_targets) 104 | 105 | fig2 = plt.figure(2) 106 | ax = fig2.add_subplot(121) 107 | ax.plot(range(batch, batch*(it+1), batch), error) 108 | ax.set_xlabel('Number of Samples') 109 | ax.set_ylabel('MSE') 110 | ax.title('Regression Error') 111 | ax = fig2.add_subplot(122) 112 | ax.plot(range(batch, batch*(it+1), batch), w_num) 113 | ax.set_xlabel('Number of Samples') 114 | ax.set_ylabel('Number of Relevant Vectors') 115 | ax.title('Number of Utilized Basis', fontszie=12) 116 | 117 | 118 | -------------------------------------------------------------------------------- /SparseBayesian.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from os import _exit 4 | import sys 5 | import time 6 | import numpy as np 7 | import scipy as sp 8 | import random as rd 9 | 10 | def sigmoid(x): 11 | return 1/(1+np.exp(-x)) 12 | 13 | def PreProcessBasis(basis): 14 | ''' preprocess basis matrix to normalize column vectors''' 15 | N,M = np.shape(basis) 16 | scale = np.sqrt(np.sum(np.multiply(basis, basis), 0)) 17 | scale[scale==0] = 1 18 | for m in range(M): 19 | basis[:,m] = basis[:,m]/scale[0,m] 20 | return basis, scale 21 | 22 | Likelihoods = {'GAUSSIAN':1, 'BERNOULLI':2, 'POISSON':3} 23 | def getLikelihood(likelihood_str): 24 | ''' set likelihood type''' 25 | try: 26 | Likelihood = Likelihoods[likelihood_str.upper()] 27 | except KeyError: 28 | print('Unknown Likelihood Type') 29 | _exit(1) 30 | return Likelihood 31 | 32 | def ParamSet(**argin): 33 | ''' if user defines some parameters, set them here ''' 34 | settings = {} 35 | settings['relevant'] = set() 36 | settings['mu'] = None 37 | settings['alpha'] = None 38 | for property_str in argin: 39 | upper_str = property_str.upper() 40 | if upper_str == 'BETA': 41 | settings['beta'] = argin[property_str] 42 | elif upper_str == 'NOISESTDDEV': 43 | settings['noiseStdDev'] = argin[property_str] 44 | elif upper_str == 'ALPHA': 45 | settings['alpha'] = argin[property_str] 46 | elif upper_str == 'WEIGHTS': 47 | settings['mu'] = argin[property_str] 48 | elif upper_str == 'RELEVANT': 49 | settings['relevant'] = set(argin[property_str]) 50 | else: 51 | print('Unrecognized Parameter Type of "%s"' % property_str) 52 | _exit(1) 53 | return settings 54 | 55 | def OptionSet(**argin): 56 | ''' user options to restrict the model and computation ''' 57 | options = {} 58 | options['fixedNoise'] = False 59 | options['freeBasis'] = set() 60 | options['max_iter'] = 500 61 | options['max_time'] = 1000 #seconds 62 | options['monitor'] = 0 63 | options['diagnosticLevel'] = 3 64 | options['diagnosticFID'] = 1 65 | options['diagnosticFile'] = None 66 | 67 | def timeFormat(time): 68 | s = time.split(' ') 69 | if len(s) == 2: 70 | v = int(s[0]) 71 | r = s[1].upper() 72 | if r in ('SECONDS', 'SECOND'): 73 | s = v 74 | elif r in ('MINUTES', 'MINUTE'): 75 | s = v*60 76 | elif r in ('HOURS', 'HOUR'): 77 | s = v*3600 78 | else: 79 | print('Badly formed time argument') 80 | _exit(1) 81 | return s 82 | 83 | for option_str in argin: 84 | upper_str = option_str.upper() 85 | if upper_str == 'FIXEDNOISE': 86 | options['fixedNoise'] = argin[option_str] 87 | elif upper_str == 'FREEBASIS': 88 | options['freeBasis'].update(argin[option_str]) 89 | elif upper_str == 'ITERATIONS': 90 | options['max_iter'] = argin[option_str] 91 | elif upper_str == 'TIME': 92 | options['max_time'] = timeFormat(argin[option_str]) 93 | elif upper_str == 'MONITOR': 94 | options['monitor'] = argin[option_str] 95 | elif upper_str == 'DIAGNOSTICLEVEL': 96 | if argin[option_str] >= 0 and argin[option_str] <=4: 97 | options['diagnosticLevel'] = argin[option_str] 98 | else: 99 | print('Illegal Assignment of Diagnostic Level: %d'% 100 | argin[option_str]) 101 | _exit(1) 102 | elif upper_str == 'DIAGNOSTICFILE': 103 | options.diagnosticFID = 0 104 | options.diagnosticFile = argin[option_str] 105 | else: 106 | print('Unrecognized Option Item') 107 | _exit(1) 108 | return options 109 | 110 | def Diagnostics(options, level, msg = None, **argin): 111 | ''' print the situation of model during training ''' 112 | def isnum(value): 113 | try: 114 | value=value+1 115 | except TypeError: 116 | return False 117 | else: 118 | return True 119 | 120 | f = sys.stdout 121 | if isnum(level): 122 | if level <= options['diagnosticLevel']: 123 | print(msg, file=f) 124 | else: 125 | if level.upper() in set(['OPEN', 'START']): 126 | if options['diagnosticFID'] != 1: 127 | try: 128 | f = open(options['diagnosticFile'], 'w') 129 | except FileNotFoundError: 130 | print('Could not open diagnostic file %s' %\ 131 | options['diagnosticFile']) 132 | _exit(1) 133 | elif level.upper() in set(['CLOSE', 'END', 'FINISH']): 134 | if options['diagnosticFID'] != 1: 135 | f.close() 136 | #return options 137 | 138 | def Initialization(likelihood, basis, targets, settings, options): 139 | ''' initialize parameters and hyper parameters of the model ''' 140 | GAUSSIAN_SNR_INIT = 0.1 141 | INIT_ALPHA_MAX = 1e3 142 | INIT_ALPHA_MIN = 1e-3 143 | 144 | Basis, Scale = PreProcessBasis(basis) 145 | Likelihood = getLikelihood(likelihood) 146 | if Likelihood == Likelihoods['GAUSSIAN']: 147 | if 'beta' in settings: 148 | beta = settings['beta'] 149 | elif 'noiseStdDev' in settings: 150 | beta = 1/(settings['noiseStdDev']**2) 151 | else: 152 | beta = 1/(np.max([1e-6, np.std(targets, ddof=1)])*GAUSSIAN_SNR_INIT) ** 2 153 | else: 154 | beta = np.array([]) 155 | 156 | targetsPseudoLinear = targets 157 | if Likelihood == Likelihoods['BERNOULLI']: 158 | targetsPseudoLinear = targets*2-1 159 | elif Likelihood == Likelihoods['POISSON']: 160 | targetsPseudoLinear = np.log(targets+1e-3) 161 | 162 | extra = options['freeBasis'] - settings['relevant'] 163 | used = settings['relevant'] | extra 164 | if not used: 165 | proj = Basis.T * targetsPseudoLinear 166 | used = np.argmax(np.abs(proj)) 167 | Diagnostics(options, 2,\ 168 | 'Initializing with the maximally aligned basis vector(%d)'% used) 169 | used = set([used]) 170 | else: 171 | Diagnostics(options, 2,\ 172 | 'Initializing with supplied vectors with size=%d'% len(used)) 173 | Phi = Basis[:,list(used)] 174 | Mt = len(used) 175 | order = [item for item in used] 176 | 177 | 178 | if not settings['mu']: 179 | if Likelihood == Likelihoods['GAUSSIAN']: 180 | mu = np.array([], dtype=float) 181 | elif Likelihood == Likelihoods['BERNOULLI']: 182 | tmp = (targetsPseudoLinear*0.9+1)/2 183 | mu = np.linalg.lstsq(Phi, np.log(np.divide(tmp,1-tmp)))[0] 184 | tmp = np.log(np.divide(tmp,1-tmp)) 185 | elif Likelihood == Likelihoods['POISSON']: 186 | mu = np.linalg.lstsq(Phi, targetsPseudoLinear) 187 | else: 188 | if len(settings['mu']) != len(settings['relevent']): 189 | print('Basis length (%d) should equal weight vector length (%d)'\ 190 | %(len(settings['mu']), len(settings['relevant']))) 191 | _exit(1) 192 | Diagnostics(options, 2, 'Initializing with supplied weights') 193 | mu = np.vstack([settings['mu'], np.zeros((len(extra), 1))]) 194 | 195 | if not settings['alpha']: 196 | if Likelihood == Likelihoods['GAUSSIAN']: 197 | p = np.diag(Phi.T*Phi)*beta 198 | q = (Phi.T*targets)*beta 199 | alpha = np.power(p,2)/(np.power(q,2)-p) 200 | if np.all(alpha<0): 201 | Diagnostics(options, 1,\ 202 | 'Warning: no relevant basis function at initialization!') 203 | alpha[alpha<0] = INIT_ALPHA_MAX 204 | elif Likelihood == Likelihoods['BERNOULLI'] or\ 205 | Likelihood == Likelihoods['POISSON']: 206 | alpha = 1/np.power(mu+np.array((mu==0),dtype=float), 2) 207 | alpha[alphaINIT_ALPHA_MAX] = INIT_ALPHA_MAX 209 | else: 210 | if len(settings['alpha']) != len(settings['relevant']): 211 | print('Basis length (%d) should equal alpha vector length (%d)' %\ 212 | (len(settings['relevant']), len(settings['alpha']))) 213 | _exit(1) 214 | alpha = np.vstack(alpha, np.zeros((len(extra), 1))) 215 | 216 | alpha[list(options['freeBasis']-used)] = 1e-6 217 | 218 | return Likelihood, Basis, Scale, alpha, beta, mu, Phi, used, order 219 | 220 | def ControlSet(): 221 | ''' 222 | some threshold values related to iteration stopping conditions and 223 | update necessasities 224 | ''' 225 | controls = {} 226 | controls['ZeroFactor'] = 1e-12 227 | controls['MinDeltaLogAlpha'] = 1e-3 228 | controls['MinDeltaLogBeta'] = 1e-6 229 | controls['AdditionPriority'] = False 230 | controls['DeletionPriority'] = True 231 | controls['betaUpdateStart'] = 10 232 | controls['betaUpdateFrequency'] = 5 233 | controls['betaMaxFactor'] = 1e6 234 | controls['PosteriorModeFrequency'] = 1 235 | controls['BasisAlignmentTest'] = True 236 | controls['AlignmentMax'] = 1-1e-3 237 | 238 | return controls 239 | 240 | def PosteriorMode(Likelihood, Basis, targets, alpha, mu, iterMax,\ 241 | options): 242 | def DataError(Likelihood, Basis_mu, targets): 243 | if Likelihood == Likelihoods['BERNOULLI']: 244 | y = sigmoid(Basis_mu) 245 | if (y==0)[targets>0].any() or (y==1)[targets<1].any(): 246 | e = float('inf') 247 | else: 248 | y0 = y!=0 249 | y1 = y!=1 250 | e = -(targets[y0].T*np.log(y[y0].reshape(-1,1))+\ 251 | (1-targets[y1]).T*np.log(1-y[y1].reshape(-1,1))) 252 | elif Likelihood == Likelihoods['POISSON']: 253 | y = np.exp(Basis_mu) 254 | e = -np.sum(np.multiply(targets, Basis_mu)-y) 255 | 256 | return e, y 257 | 258 | GRAD_MIN = 1e-6 259 | STEP_MIN = 1/(2**8) 260 | Mt = np.shape(Basis)[1] 261 | A = np.diag(alpha.ravel().tolist()) 262 | Basis_Mu = Basis*mu 263 | dataError, y = DataError(Likelihood, Basis_Mu, targets) 264 | regularizer = (alpha.T*np.power(mu, 2))/2 265 | totalError = dataError + regularizer 266 | badHess = False 267 | errorLog = np.zeros((iterMax, 1)) 268 | 269 | for it in range(iterMax): 270 | errorLog[it, 0] = totalError 271 | Diagnostics(options, 4, 'PosteriorMode Cycle:%2d\terror:%.6f'\ 272 | %(it, totalError)) 273 | #solve for gradient of objective function 274 | e = targets-y 275 | g = Basis.T*e - np.multiply(alpha,mu) 276 | if Likelihood == Likelihoods['BERNOULLI']: 277 | beta = np.multiply(y,1-y) 278 | elif Likelihood == Likelihoods['POISSON']: 279 | beta = y 280 | Basis_b = np.multiply(Basis, beta*np.ones((1,Mt))) 281 | Hess = Basis_b.T*Basis + A 282 | try: 283 | U = np.linalg.cholesky(Hess) 284 | except np.linalg.linalg.LinAlgError: 285 | Diagnostics(options, 1, 'Warning: ill-conditioned Hessian') 286 | badHess = True 287 | U = np.matrix([]) 288 | beta = np.array([]) 289 | LikelihoodMode = np.array([]) 290 | break 291 | if (np.abs(g) STEP_MIN: 301 | mu_new = mu + step*delta_mu 302 | Basis_mu = Basis*mu_new 303 | [dataError, y] = DataError(Likelihood, Basis_mu, targets) 304 | regularizer = (alpha.T*np.power(mu, 2))/2 305 | totalError = dataError + regularizer 306 | if totalError >= errorLog[it, 1]: 307 | step = step/2 308 | Diagnostics(options, 4, 'PosteriorMode Error increase! Backing\ 309 | off to l = %.3f' % step) 310 | else: 311 | mu = mu_new 312 | step = 0.0 313 | 314 | if step>0: 315 | Diagnostics(options, 4, 'PosteriorMode stopping due to back-off\ 316 | limie (|g|=%.3f)'%np.max(np.abs(g))) 317 | break 318 | LikelihoodMode = -dataError 319 | 320 | return mu, U, beta, LikelihoodMode, badHess 321 | 322 | def updateStats(Likelihood, Basis, Phi, targets, order, alpha, beta, mu,\ 323 | Basis_Phi, Basis_targets, options): 324 | ''' update related statistics after a modification is performed ''' 325 | MAX_POSTMODE_ITER = 25 326 | N = np.shape(Basis)[0] 327 | Mt = np.shape(Phi)[1] 328 | if Likelihood == Likelihoods['GAUSSIAN']: 329 | U = np.linalg.cholesky(Phi.T*Phi*beta +\ 330 | np.diag(alpha.ravel().tolist()[0])) 331 | U_inv = U.I 332 | Sigma = U_inv.T*U_inv 333 | mu = beta*(Sigma*(Phi.T*targets)) 334 | y = Phi*mu 335 | e = targets - y 336 | dataLikelihood = (N*np.log(beta)-beta*(e.T*e))/2 337 | else: 338 | mu, U, beta, dataLikelihood, badHess =\ 339 | PosteriorMode(Likelihood, Phi, targets, alpha, mu, MAX_POSTMODE_ITER, options) 340 | U_inv = U.I 341 | Sigma = U_inv.T*U_inv 342 | if Likelihood == Likelihoods['BERNOULLI']: 343 | y = sigmoid(Phi*mu) 344 | elif Likelihood == Likelihoods['POISSON']: 345 | y = np.exp(Phi*mu) 346 | e = targets - y 347 | 348 | logML = dataLikelihood - np.power(mu, 2).T*alpha/2 +\ 349 | np.sum(np.log(alpha))/2-np.sum(np.log(np.diag(U))) 350 | diagSigma = np.matrix(np.diag(Sigma)).reshape(-1,1) 351 | gamma = 1 - np.multiply(alpha, diagSigma) 352 | 353 | if Likelihood == Likelihoods['GAUSSIAN']: 354 | b_Basis_Phi = beta*Basis_Phi 355 | tmp = b_Basis_Phi*U_inv.T 356 | S_in = (beta - np.diag(tmp*tmp.T)).reshape(-1,1) 357 | Q_in = beta*(Basis_targets - Basis_Phi*mu) 358 | else: 359 | b_Basis_Phi = Basis.T * (np.multiply(Phi, beta*np.ones((1, Mt)))) 360 | tmp = b_Basis_Phi*U_inv.T 361 | S_in = ((beta.T*np.power(Basis,2))-np.diag(tmp*tmp.T)).reshape(-1, 1) 362 | Q_in = Basis.T*e 363 | 364 | S_out = S_in.copy() 365 | Q_out = Q_in.copy() 366 | index = order 367 | tmp = alpha-S_in[index] 368 | S_out[index] = np.divide(np.multiply(alpha,S_in[index]), tmp) 369 | Q_out[index] = np.divide(np.multiply(alpha,Q_in[index]), tmp) 370 | factor = np.power(Q_out,2) - S_out 371 | 372 | return Sigma, mu, S_in, Q_in, S_out, Q_out, factor, logML, \ 373 | gamma, b_Basis_Phi, beta 374 | 375 | def Bayesian(likelihood, basis, targets, settings = None, options = None): 376 | ''' main model configuration and training ''' 377 | if not settings: 378 | settings = ParamSet() 379 | if not options: 380 | options = OptionSet() 381 | if (options['fixedNoise']) and ('beta' not in settings)\ 382 | and ('noiseStdDev' not in settings): 383 | print('Options to fix noise variance but no value supplied') 384 | _exit(1) 385 | 386 | Res = {} 387 | controls = ControlSet() 388 | t_start = time.time() 389 | 390 | Diagnostics(options, 'start') 391 | Likelihood, Basis, Scale, alpha, beta, mu, Phi, used, order = \ 392 | Initialization(likelihood, basis, targets, settings, options) 393 | if Likelihood == Likelihoods['GAUSSIAN']: 394 | Basis_Phi = Basis.T*Phi 395 | else: 396 | Basis_Phi = np.matrix([]) 397 | Basis_targets = Basis.T*targets 398 | 399 | Sigma, mu, S_in, Q_in, S_out, Q_out, factor, logML, gamma, b_Basis_Phi,\ 400 | beta = updateStats(Likelihood, Basis, Phi, targets, order, alpha,\ 401 | beta, mu, Basis_Phi, Basis_targets, options) 402 | if options['max_iter'] == 0: 403 | Param = np.array([]) 404 | HyperParam = np.array([]) 405 | Res['Likelihood'] = logML 406 | return Param, HyperParam, Res 407 | N,M_f = np.shape(Basis) 408 | Mt = np.shape(Phi)[1] 409 | add_count = delete_count = update_count = 0 410 | maxLogSize = options['max_iter']+controls['betaUpdateStart']+\ 411 | options['max_iter']//controls['betaUpdateFrequency'] 412 | logMarginalLog = np.zeros((maxLogSize, 1)) 413 | count = 0 414 | 415 | if controls['BasisAlignmentTest']: 416 | aligned_in = np.array([]).reshape(-1,1) 417 | aligned_out = np.array([]).reshape(-1,1) 418 | alignDeferCount = 0 419 | 420 | ACTION = {'reestimate':0, 'add':1, 'delete':-1, 'terminate':2,\ 421 | 'noise_only':11, 'alignment_skip':12} 422 | 423 | ''' MAIN LOOP ''' 424 | it = 0 425 | LAST_ITER = False 426 | while not LAST_ITER: 427 | it = it+1 428 | update_iter = Likelihood==Likelihoods['GAUSSIAN'] or\ 429 | it%controls['PosteriorModeFrequency'] 430 | deltaML = np.zeros((M_f, 1)) 431 | Action = np.zeros((M_f, 1)) 432 | 433 | ''' find vectors needing re-estimation ''' 434 | 435 | iu = [i for i in range(len(order)) if factor[order[i]]>controls['ZeroFactor']] 436 | index = [order[i] for i in iu] 437 | new_alpha = np.divide(np.power(S_out[index],2),factor[index]) 438 | delta = (1/new_alpha - 1/alpha[iu]) 439 | tmp1 = np.multiply(delta, np.power(Q_in[index],2)) 440 | tmp2 = np.multiply(delta, S_in[index])+1 441 | deltaML[index] = (np.divide(tmp1,tmp2)-np.log(tmp2))/2 442 | 443 | ''' find vectors needing deletion ''' 444 | 445 | iu = [i for i in range(len(order)) if factor[order[i]]<=controls['ZeroFactor']] 446 | index = [order[i] for i in iu] 447 | any_to_delete = Mt>1 and not (set(index)-options['freeBasis']).issubset({}) 448 | if any_to_delete: 449 | tmp1 = np.divide(np.power(Q_out[index], 2), alpha[iu]) 450 | tmp2 = np.divide(S_out[index],alpha[iu])-1 451 | deltaML[index] = np.divide(tmp1, tmp2)-np.log(-tmp2) 452 | Action[index] = ACTION['delete'] 453 | 454 | ''' find vectors needing addition ''' 455 | 456 | index = set([i for i in range(M_f) if factor[i]>controls['ZeroFactor']])-used 457 | if controls['BasisAlignmentTest']: 458 | index -= set(aligned_out.ravel().tolist()) 459 | index = list(index) 460 | any_to_add = len(index) > 0 461 | if any_to_add: 462 | tmp = np.divide(np.power(Q_in[index],2), S_in[index]) 463 | deltaML[index] = (tmp-1-np.log(tmp))/2 464 | Action[index] = ACTION['add'] 465 | deltaML[list(options['freeBasis'])] = 0 466 | flag_add = any_to_add and controls['AdditionPriority'] 467 | flag_delete = any_to_delete and controls['DeletionPriority'] 468 | if flag_add or flag_delete: 469 | deltaML[Action==ACTION['reestimate']] = 0 470 | if flag_add and not controls['DeletionPriority']: 471 | deltaML[Action==ACTION['delete']] = 0 472 | if flag_delete and not controls['AdditionPriority']: 473 | deltaML[Action==ACTION['add']] = 0 474 | 475 | ''' find the most significant update to decide next step ''' 476 | 477 | deltaLogMarginal = np.max(deltaML) 478 | nu = np.argmax(deltaML) 479 | selectedAction = Action[nu] 480 | action_worth = deltaLogMarginal>0 481 | if selectedAction in [ACTION['reestimate'], ACTION['delete']]: 482 | try: 483 | j = order.index(nu) 484 | except ValueError: 485 | if not action_worth: 486 | j = 0 487 | else: 488 | print('Selected vector %d is not in basis but operated!'% nu) 489 | _exit(1) 490 | phi = Basis[:, nu] 491 | new_alpha = S_out[nu]**2/factor[nu] 492 | 493 | if (not action_worth) or\ 494 | (selectedAction == ACTION['reestimate'] and \ 495 | np.abs(np.log(new_alpha/alpha[j]))controls['AlignmentMax']] 508 | aligned_num = len(aligned_pos) 509 | if aligned_num > 0: 510 | selectedAction = ACTION['alignment_skip'] 511 | act = 'alignment-deferred addition' 512 | alignDeferCount = alignDeferCount+1 513 | aligned_out = np.vstack((aligned_out,\ 514 | nu*np.ones((aligned_num,1)))) 515 | aligned_in = np.vstack((aligned_in,\ 516 | np.array(aligned_pos).reshape(-1,1))) 517 | elif selectedAction == ACTION['delete']: 518 | aligned_pos = (aligned_in == nu).nonzero()[0] 519 | aligned_num = len(aligned_pos) 520 | if aligned_num>0: 521 | aligned_in = np.delete(aligned_in,aligned_pos,axis=0) 522 | aligned_out = np.delete(aligned_out, aligned_pos, axis=0) 523 | Diagnostics(options, 3, 'Alignment reinstated') 524 | 525 | ''' perform modification according to selected action ''' 526 | 527 | update_required = False 528 | if selectedAction == ACTION['reestimate']: 529 | old_alpha = alpha[j].copy() 530 | alpha[j] = new_alpha 531 | s_j = Sigma[:,j] 532 | delta = 1/(new_alpha-old_alpha) 533 | kappa = 1/(Sigma[j,j]+delta) 534 | tmp = s_j*kappa 535 | new_Sigma = Sigma - tmp*s_j.T 536 | delta_mu = -mu[j,0]*tmp 537 | mu = mu + delta_mu 538 | if update_iter: 539 | S_in = S_in + np.power(b_Basis_Phi*s_j,2)*kappa 540 | Q_in = Q_in - b_Basis_Phi*delta_mu 541 | update_count = update_count+1 542 | act = 're-estimation' 543 | update_required = True 544 | elif selectedAction == ACTION['add']: 545 | if Likelihood == Likelihoods['GAUSSIAN']: 546 | Basis_phi = Basis.T*phi 547 | Basis_Phi = np.hstack((Basis_Phi, Basis_phi)) 548 | beta_phi = beta*phi 549 | b_Basis_phi = beta*Basis_phi 550 | else: 551 | beta_phi = np.multiply(beta, phi) 552 | b_Basis_phi = Basis.T*beta_phi 553 | tmp = ((beta_phi.T*Phi)*Sigma).T 554 | alpha = np.vstack((alpha, new_alpha)) 555 | Phi = np.hstack((Phi, phi)) 556 | s_ii = 1/(new_alpha+S_in[nu,0]) 557 | s_i = -tmp*s_ii 558 | tau = -s_i*tmp.T 559 | new_Sigma = np.vstack((np.hstack((Sigma+tau, s_i)),\ 560 | np.hstack((s_i.T, s_ii)))) 561 | mu_i = s_ii*Q_in[nu,0] 562 | delta_mu = np.vstack((-tmp*mu_i, mu_i)) 563 | mu = np.vstack((mu, 0)) + delta_mu 564 | if update_iter: 565 | mCi = b_Basis_phi - b_Basis_Phi*tmp 566 | S_in = S_in-np.power(mCi,2)*s_ii 567 | Q_in = Q_in - mCi*mu_i 568 | used |= {nu} 569 | order.append(nu) 570 | add_count = add_count+1 571 | act = 'addition' 572 | update_required = True 573 | elif selectedAction == ACTION['delete']: 574 | if Likelihood == Likelihoods['GAUSSIAN']: 575 | Basis_Phi = np.delete(Basis_Phi, j, axis=1) 576 | Phi = np.delete(Phi, j, axis=1) 577 | alpha = np.delete(alpha, j, axis=0) 578 | s_jj = Sigma[j,j].copy() 579 | s_j = Sigma[:,j].copy() 580 | tmp = s_j/s_jj 581 | new_Sigma = Sigma-tmp*s_j.T 582 | new_Sigma = np.delete(new_Sigma, j, axis=0) 583 | new_Sigma = np.delete(new_Sigma, j, axis=1) 584 | delte_mu = -mu[j,0]*tmp 585 | mu_j = mu[j,0].copy() 586 | mu = mu + delta_mu 587 | mu = np.delete(mu, j, axis=0) 588 | if update_iter: 589 | jPm = b_Basis_Phi * s_j 590 | S_in = S_in + np.power(jPm,2)/s_jj 591 | Q_in = Q_in + mu_j*jPm/s_jj 592 | used -= {nu} 593 | order.remove(nu) 594 | delete_count = delete_count+1 595 | act = 'deletion' 596 | update_required = True 597 | Mt = len(order) 598 | Diagnostics(options, 3, 'Action: %s of %d (%g)'\ 599 | %(act, nu, deltaLogMarginal)) 600 | 601 | ''' update main statistics ''' 602 | 603 | if update_required: 604 | if update_iter: 605 | S_out = S_in.copy() 606 | Q_out = Q_in.copy() 607 | tmp = np.divide(alpha, alpha-S_in[order]) 608 | S_out[order] = np.multiply(tmp, S_in[order]) 609 | Q_out[order] = np.multiply(tmp, Q_in[order]) 610 | factor = np.power(Q_out, 2)-S_out 611 | Sigma = new_Sigma.copy() 612 | gamma = 1 - np.multiply(alpha, np.diag(Sigma).reshape(-1,1)) 613 | if Likelihood == Likelihoods['GAUSSIAN']: 614 | b_Basis_Phi = beta * Basis_Phi 615 | else: 616 | b_Basis_Phi = np.multiply(Phi, beta*np.ones(1,Mt)).T*Basis 617 | else: 618 | Sigma, mu, S_in, Q_in, S_out, Q_out, factor,\ 619 | newLogML, gamma, b_Basis_Phi, beta = \ 620 | updateStats(Likelihood, Basis, Phi, \ 621 | targets, order, alpha, beta, mu, Basis_Phi,\ 622 | Basis_targets, options) 623 | deltaLogMarginal = newLogML - logML 624 | if update_iter and deltaLogMarginal<0: 625 | Diagnostics(options, 1, 'Warning: Marginal Likelihood decreases\ 626 | %g!'% deltaLogMarginal) 627 | logML = logML + deltaLogMarginal 628 | logMarginalLog[count, 0] = logML 629 | count = count+1 630 | 631 | ''' update Noise Parameter beta ''' 632 | 633 | if Likelihood == Likelihoods['GAUSSIAN'] and not options['fixedNoise']\ 634 | and (selectedAction==ACTION['terminate'] or\ 635 | it<=controls['betaUpdateStart']\ 636 | or not it%controls['betaUpdateFrequency']): 637 | beta_old = beta 638 | y = Phi*mu 639 | e = targets-y 640 | beta = (N-np.sum(gamma))/np.power(e,2).sum() 641 | beta = np.min([beta, controls['betaMaxFactor']/targets.var()]) 642 | delta_logbeta = np.log(beta/beta_old) 643 | if np.abs(delta_logbeta)>controls['MinDeltaLogBeta']: 644 | Sigma, mu, S_in, Q_in, S_out, Q_out, factor,\ 645 | logML, gamma, b_Basis_Phi, beta = \ 646 | updateStats(Likelihood, Basis, Phi, \ 647 | targets, order, alpha, beta, mu, Basis_Phi,\ 648 | Basis_targets, options) 649 | count = count+1 650 | logMarginalLog[count,0] = logML 651 | if selectedAction == ACTION['terminate']: 652 | selectedAction = ACTION['noise_only'] 653 | Diagnostics(options, 3, 'Noise update and termination defferd!') 654 | 655 | if selectedAction == ACTION['terminate']: 656 | Diagnostics(options, 2, 'Stop at iteration %d (max deltaML = %.3f)\ 657 | '%(it, np.max(deltaLogMarginal))) 658 | if Likelihood == Likelihoods['GAUSSIAN']: 659 | Diagnostics(options, 2, '%4d>\tL=%.6f\tgamma=%.2f(M=%d)\tsigma=%.3f'\ 660 | %(it, logML/N, np.sum(gamma), Mt, np.sqrt(1/beta))) 661 | else: 662 | Diagnostics(options, 2, '%4d>\tL=%.6f\tgamma=%.2f(M=%d)'\ 663 | %(it, logML/N, np.sum(gamma), Mt)) 664 | break 665 | 666 | ''' check the stop condition of main iteration ''' 667 | 668 | ITER_LIMIT = it==options['max_iter'] 669 | TIME_LIMIT = (time.time()-t_start)>=options['max_time'] 670 | LAST_ITER = ITER_LIMIT or TIME_LIMIT 671 | if (options['monitor'] and not i%options['monitor']) or LAST_ITER: 672 | if Likelihood == Likelihoods['GAUSSIAN']: 673 | Diagnostics(options, 2, '%4d>\tL=%.6f\tgamma=%.2f(M=%d)\tsigma=%.3f'\ 674 | %(it, logML/N, np.sum(gamma), Mt, np.sqrt(1/beta))) 675 | else: 676 | Diagnositcs(options, 2, '%4d>\tL=%.6f\tgamma=%.2f(M=%d)'\ 677 | %(it, logML/N, np.sum(gamma), Mt)) 678 | 679 | ''' END OF MAIN LOOP ''' 680 | 681 | if selectedAction != ACTION['terminate']: 682 | if ITER_LIMIT: 683 | Diagnostics(options, 1, 'Iteration Limit: Algorithm did not converge!') 684 | elif TIME_LIMIT: 685 | Diagnostics(options, 1, 'Iteration Limit: Algorithm did not converge!') 686 | 687 | if options['diagnosticLevel']>1: 688 | t_stop = time.time() 689 | total = add_count + delete_count + update_count 690 | if controls['BasisAlignmentTest']: 691 | total = total + alignDeferCount 692 | total = 1 if total == 0 else total 693 | Diagnostics(options, 2, 'Action Summary\n===============') 694 | Diagnostics(options, 2, 'Added\t\t%6d(%.0f%%)'%(add_count,100*add_count/total)) 695 | Diagnostics(options, 2, 'Deleted\t\t%6d(%.0f%%)'%(delete_count,100*delete_count/total)) 696 | Diagnostics(options, 2, 'Reesimated\t%6d(%.0f%%)'%(update_count,100*update_count/total)) 697 | if controls['BasisAlignmentTest'] and alignDeferCount: 698 | Diagnostics(options, 2, '--------------') 699 | Diagnostics(options, 2, 'Deferred\t%6d(%.0f%%)'%(alignDeferCount,100*alignDeferCount/total)) 700 | Diagnostics(options, 2, '==============') 701 | Diagnostics(options, 2, 'Total of %d likelihood updates'%count) 702 | Diagnostics(options, 2, 'Time to run: %.2f seconds'%(t_stop-t_start)) 703 | Diagnostics(options, 'end') 704 | 705 | Param = {} 706 | HyperParam = {} 707 | Diagnostic = {} 708 | argorder = np.argsort(order) 709 | Param['relevant'] = list(used) 710 | Param['value'] = np.divide(mu[argorder],Scale[:,list(used)].T) 711 | HyperParam['alpha'] = np.divide(alpha[argorder],\ 712 | np.power(Scale[:,list(used)],2).T) 713 | HyperParam['beta'] = beta 714 | Diagnostic['gamma'] = gamma[argorder] 715 | Diagnostic['likelihood'] = logMarginalLog[1:count,0] 716 | Diagnostic['iterations'] = it 717 | Diagnostic['sparse_factor'] = S_out 718 | Diagnostic['quality_factor'] = Q_out 719 | 720 | return Param, HyperParam, Diagnostic 721 | 722 | if __name__ == '__main__': 723 | a = [rd.uniform(0, 100) for i in range(20*20)] 724 | basis = np.matrix(a).reshape(20,20) 725 | w = [rd.uniform(0, 100) for i in range(20)] 726 | w = np.matrix(w).reshape(-1, 1) 727 | n = [rd.uniform(0,1) for i in range(20)] 728 | noise = np.matrix(n).reshape(-1, 1) 729 | targets = basis*w+noise 730 | Param, HyperParam, Diagnostic = Bayesian('gaussian', basis, targets) 731 | --------------------------------------------------------------------------------