├── SB2_Manual.pdf ├── SB2_Sigmoid.py ├── SB2_FormatTime.py ├── SB2_PreProcessBasis.py ├── SB2_Likelihoods.py ├── Readme.txt ├── SB2_ParameterSettings.py ├── SB2_Diagnostic.py ├── SB2_ControlSettings.py ├── SB2_UserOptions.py ├── SB2_FullStatistics.py ├── SB2_PosteriorMode.py ├── SB2_Initialisation.py ├── SparseBayesDemo.py ├── licence.txt └── SparseBayes.py /SB2_Manual.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jhallock7/SparseBayes-Python/HEAD/SB2_Manual.pdf -------------------------------------------------------------------------------- /SB2_Sigmoid.py: -------------------------------------------------------------------------------- 1 | 2 | # The following is a Python translation of a MATLAB file originally written principally by Mike Tipping 3 | # as part of his SparseBayes software library. Initially published on GitHub on July 21st, 2015. 4 | 5 | # SB2_SIGMOID Generic sigmoid function for SPARSEBAYES computations 6 | # 7 | # Y = SB2_SIGMOID(X) 8 | # 9 | # Implements the function f(x) = 1 / (1+exp(-x)) 10 | # 11 | 12 | # 13 | # Copyright 2009, Vector Anomaly Ltd 14 | # 15 | # This file is part of the SPARSEBAYES library for Matlab (V2.0). 16 | # 17 | # SPARSEBAYES is free software; you can redistribute it and/or modify it 18 | # under the terms of the GNU General Public License as published by the Free 19 | # Software Foundation; either version 2 of the License, or (at your option) 20 | # any later version. 21 | # 22 | # SPARSEBAYES is distributed in the hope that it will be useful, but WITHOUT 23 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 24 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 25 | # more details. 26 | # 27 | # You should have received a copy of the GNU General Public License along 28 | # with SPARSEBAYES in the accompanying file "licence.txt"; if not, write to 29 | # the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, 30 | # MA 02110-1301 USA 31 | # 32 | # Contact the author: m a i l [at] m i k e t i p p i n g . c o m 33 | # 34 | 35 | import numpy as np 36 | 37 | def SB2_Sigmoid(x): 38 | 39 | y = 1 / (1 + np.exp(-x)) 40 | 41 | return y 42 | -------------------------------------------------------------------------------- /SB2_FormatTime.py: -------------------------------------------------------------------------------- 1 | 2 | # The following is a Python translation of a MATLAB file originally written principally by Mike Tipping 3 | # as part of his SparseBayes software library. Initially published on GitHub on July 21st, 2015. 4 | 5 | # SB2_FORMATTIME Pretty output of diagnostic SPARSEBAYES time information 6 | # 7 | # STRING = SB2_FORMATTIME(ELAPSEDTIMEINSECONDS) 8 | # 9 | 10 | # 11 | # Copyright 2009, Vector Anomaly Ltd 12 | # 13 | # This file is part of the SPARSEBAYES library for Matlab (V2.0). 14 | # 15 | # SPARSEBAYES is free software; you can redistribute it and/or modify it 16 | # under the terms of the GNU General Public License as published by the Free 17 | # Software Foundation; either version 2 of the License, or (at your option) 18 | # any later version. 19 | # 20 | # SPARSEBAYES is distributed in the hope that it will be useful, but WITHOUT 21 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 22 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 23 | # more details. 24 | # 25 | # You should have received a copy of the GNU General Public License along 26 | # with SPARSEBAYES in the accompanying file "licence.txt"; if not, write to 27 | # the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, 28 | # MA 02110-1301 USA 29 | # 30 | # Contact the author: m a i l [at] m i k e t i p p i n g . c o m 31 | # 32 | 33 | import numpy as np 34 | import math 35 | 36 | def SB2_FormatTime(elapsedTime): 37 | if elapsedTime >= 3600: 38 | # More than an hour... 39 | h = math.floor(elapsedTime/float(3600)) 40 | m = math.floor(np.remainder(elapsedTime,3600)/float(60)) 41 | s = math.floor(np.remainder(elapsedTime,60)) 42 | 43 | timeString_ = '{0}h {1}m {2}s'.format(h,m,s) 44 | 45 | elif elapsedTime >= 60: 46 | # More than one minute (but less than an hour) 47 | m = math.floor(elapsedTime/float(60)) 48 | s = math.floor(np.remainder(elapsedTime,60)) 49 | 50 | timeString_ = '{0}m {1}s'.format(m,s) 51 | 52 | else: 53 | # Under a minute 54 | s = elapsedTime 55 | 56 | timeString_ = '{0} secs'.format(s) 57 | 58 | return timeString_ 59 | -------------------------------------------------------------------------------- /SB2_PreProcessBasis.py: -------------------------------------------------------------------------------- 1 | 2 | # The following is a Python translation of a MATLAB file originally written principally by Mike Tipping 3 | # as part of his SparseBayes software library. Initially published on GitHub on July 21st, 2015. 4 | 5 | # SB2_PREPROCESSBASIS Do basis matrix pre-processing for SPARSEBAYES 6 | # 7 | # [BASIS, SCALES] = SB2_PREPROCESSBASIS(BASIS) 8 | # 9 | # OUTPUT ARGUMENTS: 10 | # 11 | # BASIS NxM matrix of basis vectors appropriately pre-processed 12 | # (scaled to unit length per column) 13 | # 14 | # SCALES Vector of scaling factors applied 15 | # 16 | # 17 | # INPUT ARGUMENTS: 18 | # 19 | # BASIS NxM matrix of basis vectors (one column per basis function) 20 | # 21 | # NOTES: 22 | # 23 | # Simply normalises the basis vectors to unit length, returning the 24 | # original lengths so that the weights can be rescaled appropriately 25 | # before returning to the user. 26 | # 27 | 28 | # 29 | # Copyright 2009, Vector Anomaly Ltd 30 | # 31 | # This file is part of the SPARSEBAYES library for Matlab (V2.0). 32 | # 33 | # SPARSEBAYES is free software; you can redistribute it and/or modify it 34 | # under the terms of the GNU General Public License as published by the Free 35 | # Software Foundation; either version 2 of the License, or (at your option) 36 | # any later version. 37 | # 38 | # SPARSEBAYES is distributed in the hope that it will be useful, but WITHOUT 39 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 40 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 41 | # more details. 42 | # 43 | # You should have received a copy of the GNU General Public License along 44 | # with SPARSEBAYES in the accompanying file "licence.txt"; if not, write to 45 | # the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, 46 | # MA 02110-1301 USA 47 | # 48 | # Contact the author: m a i l [at] m i k e t i p p i n g . c o m 49 | # 50 | 51 | import numpy as np 52 | 53 | def SB2_PreProcessBasis(BASIS): 54 | 55 | [N,M] = BASIS.shape 56 | 57 | # Compute "lengths" of basis vectors (columns of BASIS) 58 | 59 | Scales = np.power(BASIS, 2) #Square all elements of BASIS 60 | Scales = np.sum(Scales, axis=0) #Sum columns 61 | Scales = np.power(Scales, 0.5) #Take sqrt 62 | 63 | # Work-around divide-by-zero inconvenience 64 | 65 | Scales[Scales == 0] = 1 66 | 67 | for m in range(M): 68 | BASIS[:,m] /= Scales[0,m] 69 | 70 | return BASIS, Scales 71 | -------------------------------------------------------------------------------- /SB2_Likelihoods.py: -------------------------------------------------------------------------------- 1 | 2 | # The following is a Python translation of a MATLAB file originally written principally by Mike Tipping 3 | # as part of his SparseBayes software library. Initially published on GitHub on July 21st, 2015. 4 | 5 | # SB2_LIKELIHOODS Convenience function to encapsulate likelihood types 6 | # 7 | # LIKELIHOOD = SB2_LIKELIHOODS(TYPE) 8 | # 9 | # OUTPUT ARGUMENTS: 10 | # 11 | # LIKELIHOOD A structure containing an enumeration of possible 12 | # likelihood types, and specification of the type in use. 13 | # 14 | # INPUT ARGUMENTS: 15 | # 16 | # TYPE A string. One of: 17 | # 'Gaussian' 18 | # 'Bernoulli' 19 | # 'Poisson' 20 | # 21 | # EXAMPLE: 22 | # 23 | # LIKELIHOOD = SB2_Likelihoods('Gaussian') 24 | # 25 | # NOTES: 26 | # 27 | # This function is "for internal use only" and exists purely for convenience 28 | # of coding and to facilitate potential future expansion. 29 | # 30 | 31 | # 32 | # Copyright 2009, Vector Anomaly Ltd 33 | # 34 | # This file is part of the SPARSEBAYES library for Matlab (V2.0). 35 | # 36 | # SPARSEBAYES is free software; you can redistribute it and/or modify it 37 | # under the terms of the GNU General Public License as published by the Free 38 | # Software Foundation; either version 2 of the License, or (at your option) 39 | # any later version. 40 | # 41 | # SPARSEBAYES is distributed in the hope that it will be useful, but WITHOUT 42 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 43 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 44 | # more details. 45 | # 46 | # You should have received a copy of the GNU General Public License along 47 | # with SPARSEBAYES in the accompanying file "licence.txt"; if not, write to 48 | # the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, 49 | # MA 02110-1301 USA 50 | # 51 | # Contact the author: m a i l [at] m i k e t i p p i n g . c o m 52 | # 53 | 54 | def SB2_Likelihoods(likelihood_): 55 | 56 | LIKELIHOOD = {} 57 | 58 | # For convenience in code, and later expandability, 59 | # we "enum" the available likelihood models 60 | 61 | # Continuous 62 | LIKELIHOOD['Gaussian'] = 1 63 | 64 | # Discrete 65 | LIKELIHOOD['Bernoulli'] = 2 66 | LIKELIHOOD['Poisson'] = 3 67 | 68 | # Feel free to add your own ... and work out the maths :-) 69 | 70 | # Determine the likelihood to be used 71 | 72 | if likelihood_[0:5].upper() == 'GAUSS': 73 | LIKELIHOOD['InUse'] = LIKELIHOOD['Gaussian'] 74 | 75 | elif likelihood_[0:5].upper() == 'BERNO': 76 | LIKELIHOOD['InUse'] = LIKELIHOOD['Bernoulli'] 77 | 78 | elif likelihood_[0:5].upper() == 'POISS': 79 | LIKELIHOOD['InUse'] = LIKELIHOOD['Poisson'] 80 | 81 | else: 82 | raise Exception('Unknown likelihood {0} specified\n'.format(likelihood_)) 83 | 84 | return LIKELIHOOD 85 | -------------------------------------------------------------------------------- /Readme.txt: -------------------------------------------------------------------------------- 1 | SPARSEBAYES Matlab Toolbox 2 | 3 | Version 2.00 4 | 5 | -- INTRODUCTION -- 6 | 7 | "SparseBayes" is a package of Matlab functions designed to implement 8 | an efficient learning algorithm for "Sparse Bayesian" models. 9 | 10 | The "Version 2" package is an expanded implementation of the algorithm 11 | detailed in: 12 | 13 | Tipping, M. E. and A. C. Faul (2003). "Fast marginal likelihood 14 | maximisation for sparse Bayesian models." In C. M. Bishop and 15 | B. J. Frey (Eds.), Proceedings of the Ninth International Workshop 16 | on Artificial Intelligence and Statistics, Key West, FL, Jan 3-6. 17 | 18 | This paper, the accompanying code, and further information regarding 19 | Sparse Bayesian models and the related "relevance vector machine" may 20 | be obtained from: 21 | 22 | http://www.relevancevector.com 23 | 24 | 25 | -- CODE STATUS -- 26 | 27 | March 2009: The code is currently at Version 2.0, and has seen limited 28 | testing under Matlab Version 7.4. 29 | 30 | 31 | -- GETTING STARTED -- 32 | 33 | The SparseBayes distribution comes with a basic user manual: see 34 | SB2_Manual.pdf. 35 | 36 | In summary, there are a number of files supplied in the SB2 37 | distribution, but the user should only need to call a subset of them. 38 | 39 | To set up options and parameters: 40 | 41 | SB2_UserOptions.m 42 | SB2_ParameterSettings.m 43 | 44 | The main algorithm 45 | 46 | SparseBayes.m 47 | 48 | Type "help SparseBayes" etc. at the Matlab prompt for further details 49 | on how to use these functions. 50 | 51 | There is a also a simple demonstration program, showing how 52 | SparseBayes may be used, in: 53 | 54 | SparseBayesDemo.m 55 | 56 | 57 | -- LICENCE -- 58 | 59 | Note that the "SparseBayes" software is supplied subject to version 2 60 | of the "GNU General Public License" (detailed in the file "licence.txt"). 61 | 62 | SPARSEBAYES is free software; you can redistribute it and/or modify it 63 | under the terms of the GNU General Public License as published by the 64 | Free Software Foundation; either version 2 of the License, or (at your 65 | option) any later version. 66 | 67 | SPARSEBAYES is distributed in the hope that it will be useful, but 68 | WITHOUT ANY WARRANTY; without even the implied warranty of 69 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 70 | General Public License for more details. 71 | 72 | You should have received a copy of the GNU General Public License 73 | along with SPARSEBAYES in the accompanying file "licence.txt"; if not, 74 | write to the Free Software Foundation, Inc., 51 Franklin St, Fifth 75 | Floor, Boston, MA 02110-1301 USA 76 | 77 | 78 | -- ACKNOWLEDGEMENTS -- 79 | 80 | The author would like to thank Mark Hatton, Anita Faul, Ian Nabney, 81 | Arnulf Graf and Gavin Cawley for their assistance in producing this 82 | code. 83 | 84 | 85 | -- 86 | 87 | Mike Tipping 88 | www.relevancevector.com 89 | m a i l [at] m i k e t i p p i n g . c o m 90 | 91 | This README file (Readme.txt) was created on 13-Mar-2009 at 8:58 AM. 92 | -------------------------------------------------------------------------------- /SB2_ParameterSettings.py: -------------------------------------------------------------------------------- 1 | 2 | # The following is a Python translation of a MATLAB file originally written principally by Mike Tipping 3 | # as part of his SparseBayes software library. Initially published on GitHub on July 21st, 2015. 4 | 5 | # SB2_PARAMETERSETTINGS User parameter initialisation for SPARSEBAYES 6 | # 7 | # SETTINGS = SB2_PARAMETERSETTINGS(parameter1, value1, parameter2, value2,...) 8 | # 9 | # OUTPUT ARGUMENTS: 10 | # 11 | # SETTINGS An initialisation structure to pass to SPARSEBAYES 12 | # 13 | # INPUT ARGUMENTS: 14 | # 15 | # Optional number of parameter-value pairs to specify some, all, or 16 | # none of the following: 17 | # 18 | # BETA (Gaussian) noise precision (inverse variance) 19 | # NOISESTD (Gaussian) noise standard deviation 20 | # RELEVANT Indices of columns of basis matrix to use at start-up 21 | # MU (WEIGHTS) Corresponding vector of weights to RELEVANT 22 | # ALPHA Corresponding vector of hyperparameter values to RELEVANT 23 | # 24 | # EXAMPLE: 25 | # 26 | # SETTINGS = SB2_ParameterSettings('NoiseStd',0.1) 27 | # 28 | # NOTES: 29 | # 30 | # 1. If no input arguments are supplied, defaults (effectively an 31 | # empty structure) will be returned. 32 | # 33 | # 2. If both BETA and NOISESTD are specified, BETA will take 34 | # precedence. 35 | # 36 | # 3. RELEVANT may be specified without WEIGHTS or ALPHA (these will be 37 | # sensibly initialised later). 38 | # 39 | # 4. If RELEVANT is specified, WEIGHTS may be specified also without ALPHA. 40 | # 41 | 42 | # 43 | # Copyright 2009, Vector Anomaly Ltd 44 | # 45 | # This file is part of the SPARSEBAYES library for Matlab (V2.0). 46 | # 47 | # SPARSEBAYES is free software; you can redistribute it and/or modify it 48 | # under the terms of the GNU General Public License as published by the Free 49 | # Software Foundation; either version 2 of the License, or (at your option) 50 | # any later version. 51 | # 52 | # SPARSEBAYES is distributed in the hope that it will be useful, but WITHOUT 53 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 54 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 55 | # more details. 56 | # 57 | # You should have received a copy of the GNU General Public License along 58 | # with SPARSEBAYES in the accompanying file "licence.txt"; if not, write to 59 | # the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, 60 | # MA 02110-1301 USA 61 | # 62 | # Contact the author: m a i l [at] m i k e t i p p i n g . c o m 63 | # 64 | 65 | def SB2_ParameterSettings(*args): 66 | 67 | # Ensure arguments are supplied in pairs 68 | 69 | if len(args) % 2 != 0: 70 | raise Exception('Arguments to SB2_ParameterSettings should be (property, value) pairs') 71 | 72 | 73 | # Any settings specified? 74 | 75 | numSettings = len(args)/2 76 | 77 | ## Defaults - over-ridden later if requested 78 | 79 | # Two options for setting noise level (purely for convenience) 80 | # - if 'beta' set, 'noiseStdDev' will be over-ridden 81 | 82 | SETTINGS = { 83 | 'BETA' : [], 84 | 'NOISESTD' : [], 85 | 86 | 'RELEVANT' : [], 87 | 'MU' : [], 88 | 'ALPHA' : [] 89 | } 90 | 91 | ## Requested overrides 92 | 93 | # Parse string/variable pairs 94 | 95 | for n in range(numSettings): 96 | property_ = args[n*2] 97 | value = args[n*2 + 1] 98 | 99 | if property_ not in SETTINGS: 100 | raise Exception('Unrecognised initialisation property: {0}'.format(property_)) 101 | else: 102 | SETTINGS[property_] = value 103 | 104 | return SETTINGS 105 | -------------------------------------------------------------------------------- /SB2_Diagnostic.py: -------------------------------------------------------------------------------- 1 | 2 | # The following is a Python translation of a MATLAB file originally written principally by Mike Tipping 3 | # as part of his SparseBayes software library. Initially published on GitHub on July 21st, 2015. 4 | 5 | # SB2_DIAGNOSTIC Helper function to output diagnostics in SPARSEBAYES 6 | # 7 | # USAGE (1): 8 | # 9 | # OPTIONS = SB2_DIAGNOSTIC(OPTIONS, ACTION) 10 | # 11 | # OUTPUT ARGUMENTS: 12 | # 13 | # OPTIONS Returned OPTIONS settings, possibly updated 14 | # (e.g. with file handle after 'open' action) 15 | # 16 | # INPUT ARGUMENTS: 17 | # 18 | # OPTIONS Options settings (SB2_USEROPTIONS) 19 | # 20 | # ACTION One of: 21 | # 'OPEN' or 'START' Begin diagnostics 22 | # 'CLOSE', 'END' or 'FINISH' End diagnostics 23 | # 24 | # 25 | # USAGE (2): 26 | # 27 | # SB2_DIAGNOSTIC(OPTIONS, LEVEL, MESSAGE, VARARGIN ...) 28 | # 29 | # OUTPUT ARGUMENTS: None 30 | # 31 | # INPUT ARGUMENTS: 32 | # 33 | # OPTIONS Options settings (SB2_USEROPTIONS) 34 | # 35 | # LEVEL Importance level of message (0 to 4) 36 | # NB: LOWER is MORE IMPORTANT 37 | # 38 | # MESSAGE Message string 39 | # 40 | # VARARGIN Optional arguments to fill in string placeholders 41 | # 42 | # EXAMPLE: 43 | # 44 | # OPTIONS = SB2_Diagnostic(OPTIONS, 'start'); 45 | # SB2_Diagnostic(OPTIONS,2,'Total of #d likelihood updates\n', 25); 46 | # OPTIONS = SB2_Diagnostic(OPTIONS, 'end'); 47 | # 48 | # NOTES: 49 | # 50 | # This function offers a convenient way to output diagnostic information, 51 | # either to screen or to a file. 52 | # 53 | # It will "filter" the incoming messages and only display/write them if 54 | # their LEVEL is less than or equal to the DIAGNOSTICLEVEL as set by 55 | # SB2_USEROPTIONS. 56 | # 57 | # e.g. after 58 | # 59 | # OPTIONS = SB2_UserOptions('diagnosticLevel',2) 60 | # 61 | # This message won't display: 62 | # 63 | # SB2_Diagnostic(OPTIONS, 3, 'Pointless message') 64 | # 65 | # But this will: 66 | # 67 | # SB2_Diagnostic(OPTIONS, 1, 'Really important message') 68 | # 69 | # Messages of level 0 are therefore most "important", and those of level 70 | # 4 least important. Level-0 messages are always displayed, since the 71 | # DIAGNOSTICLEVEL will always be at least zero. 72 | # 73 | 74 | # 75 | # Copyright 2009, Vector Anomaly Ltd 76 | # 77 | # This file is part of the SPARSEBAYES library for Matlab (V2.0). 78 | # 79 | # SPARSEBAYES is free software; you can redistribute it and/or modify it 80 | # under the terms of the GNU General Public License as published by the Free 81 | # Software Foundation; either version 2 of the License, or (at your option) 82 | # any later version. 83 | # 84 | # SPARSEBAYES is distributed in the hope that it will be useful, but WITHOUT 85 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 86 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 87 | # more details. 88 | # 89 | # You should have received a copy of the GNU General Public License along 90 | # with SPARSEBAYES in the accompanying file "licence.txt"; if not, write to 91 | # the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, 92 | # MA 02110-1301 USA 93 | # 94 | # Contact the author: m a i l [at] m i k e t i p p i n g . c o m 95 | # 96 | 97 | import numpy as np 98 | 99 | def SB2_Diagnostic(OPTIONS, level, *args): 100 | 101 | # Check if "level" is actually a string (this is a control call) 102 | 103 | if type(level) == str: 104 | level = level.upper() 105 | if level == 'OPEN' or level == 'START': 106 | if OPTIONS['DIAGNOSTICFID'] != 1: 107 | 108 | # If diagnosticFID is not 1, we must have previously set a 109 | # diagnostics file via SB2_USEROPTIONS 110 | 111 | OPTIONS['DIAGNOSTICFID'] = open(OPTIONS['DIAGNOSTICFILE'],'w') 112 | 113 | if OPTIONS['DIAGNOSTICFILE']: 114 | 115 | OPTIONS['DIAGNOSTICFID'] = open(OPTIONS['DIAGNOSTICFILE'],'w') 116 | 117 | if level == 'CLOSE' or level == 'END' or level == 'FINISH': 118 | try: 119 | OPTIONS['DIAGNOSTICFID'].close() 120 | except AttributeError: 121 | pass 122 | 123 | else: 124 | 125 | # Its a message call. 126 | # 127 | # Only output diagnostic messages if the "diagnostic level", 128 | # set via SB2_USEROPTIONS, is equal to or greater than the level of 129 | # the message to be displayed. 130 | 131 | message_ = args[0] 132 | 133 | if level <= OPTIONS['DIAGNOSTICLEVEL']: 134 | try: # Write to file 135 | if not OPTIONS['DIAGNOSTICFID'].closed: 136 | #OPTIONS['DIAGNOSTICFID'].write('\n'*level) 137 | OPTIONS['DIAGNOSTICFID'].write(message_) 138 | else: 139 | #print('\n'*level) 140 | print(message_) 141 | 142 | except AttributeError: #Print to screen 143 | #print('\n'*level) 144 | print(message_) 145 | 146 | 147 | return OPTIONS 148 | -------------------------------------------------------------------------------- /SB2_ControlSettings.py: -------------------------------------------------------------------------------- 1 | 2 | # The following is a Python translation of a MATLAB file originally written principally by Mike Tipping 3 | # as part of his SparseBayes software library. Initially published on GitHub on July 21st, 2015. 4 | 5 | # SB2_CONTROLSETTINGS Set parameters to control the SPARSEBAYES algorithm 6 | # 7 | # CONTROLS = SB2_CONTROLSETTINGS 8 | # 9 | # OUTPUT ARGUMENTS: 10 | # 11 | # CONTROLS A structure whose fields control various aspects of the 12 | # running of the SPARSEBAYES algorithm. 13 | # 14 | # .ZeroFactor Small number equivalent to zero for Q^2-S 15 | # .MinDeltaLogAlpha Termination criterion for changes in log-alpha 16 | # .MinDeltaLogBeta Termination criterion for changes in log-beta 17 | # 18 | # .PriorityAddition Prefer "addition" operations 19 | # .PriorityDeletion Prefer "deletion" operations 20 | # 21 | # .BetaUpdateStart How many "fast start" beta updates 22 | # .BetaUpdateFrequency 23 | # How regularly to update beta after the above 24 | # .BetaMaxFactor Minimum value control for noise estimate 25 | # 26 | # .PosteriorModeFrequency 27 | # How regularly to re-find the posterior mode 28 | # 29 | # .BasisAlignmentTest Test for redundant basis vectors? 30 | # .AlignmentMax Basis redundancy criterion 31 | # 32 | # NOTES: 33 | # 34 | # The various definitions in the file are effectively "fixed" and not 35 | # modified elsewhere. 36 | # 37 | # The interested user may wish to experiment with the operation of the 38 | # SPARSEBAYES algorithm by modifying the values the file directly. See the 39 | # inline comments for hints on the various control settings. 40 | # 41 | 42 | # 43 | # Copyright 2009, Vector Anomaly Ltd 44 | # 45 | # This file is part of the SPARSEBAYES library for Matlab (V2.0). 46 | # 47 | # SPARSEBAYES is free software; you can redistribute it and/or modify it 48 | # under the terms of the GNU General Public License as published by the Free 49 | # Software Foundation; either version 2 of the License, or (at your option) 50 | # any later version. 51 | # 52 | # SPARSEBAYES is distributed in the hope that it will be useful, but WITHOUT 53 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 54 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 55 | # more details. 56 | # 57 | # You should have received a copy of the GNU General Public License along 58 | # with SPARSEBAYES in the accompanying file "licence.txt"; if not, write to 59 | # the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, 60 | # MA 02110-1301 USA 61 | # 62 | # Contact the author: m a i l [at] m i k e t i p p i n g . c o m 63 | # 64 | 65 | def SB2_ControlSettings(): 66 | 67 | ## Define parameters which influence the underlying operation of the 68 | ## SparseBayes inference algorithm 69 | 70 | CONTROLS = {} 71 | 72 | # TOLERANCES 73 | 74 | # Any Q^2-S "relevance factor" less than this is considered to be zero 75 | 76 | CONTROLS['ZeroFactor'] = 1e-12 77 | 78 | # If the change in log-alpha for the best re-estimation is less than this, 79 | # we consider termination 80 | 81 | CONTROLS['MinDeltaLogAlpha'] = 1e-3 82 | 83 | # In the Gaussian case, we also require a beta update to change the value 84 | # of log-beta (inverse noise variance) less than this to terminate 85 | 86 | CONTROLS['MinDeltaLogBeta'] = 1e-6 87 | 88 | # ADD/DELETE 89 | 90 | # - preferring addition where possible will probably make the algorithm a 91 | # little slower and perhaps less "greedy" 92 | 93 | # - preferring deletion may make the model a little more sparse and the 94 | # algorithm may run slightly quicker 95 | 96 | # Note: both these can be set to 'true' at the same time, in which case 97 | # both take equal priority over re-estimation. 98 | 99 | CONTROLS['PriorityAddition'] = False 100 | CONTROLS['PriorityDeletion'] = True 101 | 102 | # (GAUSSIAN) NOISE 103 | 104 | # When to update the noise estimate 105 | 106 | # The number of iterations from the start for which we update it every 107 | # iteration (to get in the right ball-park to begin with) 108 | 109 | CONTROLS['BetaUpdateStart'] = 10 110 | 111 | # After the above, we only regularly update it after 112 | # a given number of iterations 113 | 114 | CONTROLS['BetaUpdateFrequency'] = 5 115 | 116 | # Prevent zero-noise estimate (perfect fit) problem 117 | # - effectively says the noise variance estimate is clamped to be no 118 | # lower than variance-of-targets / BetaMaxFactor. 119 | 120 | CONTROLS['BetaMaxFactor'] = 1e6 121 | 122 | # POSTERIORMODE 123 | 124 | # How many alpha updates to do in between each full posterior mode 125 | # computation in the non-Gaussian case 126 | 127 | # In principle, this should be set to one (to update the posterior every 128 | # iteration) but it may be more efficient to do several alpha updates before 129 | # re-finding the posterior mode. 130 | 131 | CONTROLS['PosteriorModeFrequency'] = 1 132 | 133 | # REDUNDANT BASIS 134 | 135 | # Check for basis vector alignment/correlation redundancy 136 | 137 | CONTROLS['BasisAlignmentTest'] = True 138 | 139 | ALIGNMENT_ZERO = 1e-3 140 | 141 | # If BasisAlignmentTest is true, any basis vector with inner product more 142 | # than MAX_ALIGNMENT with any existing model vector will not be added 143 | 144 | CONTROLS['AlignmentMax'] = 1 - ALIGNMENT_ZERO 145 | 146 | 147 | return CONTROLS 148 | -------------------------------------------------------------------------------- /SB2_UserOptions.py: -------------------------------------------------------------------------------- 1 | 2 | # The following is a Python translation of a MATLAB file originally written principally by Mike Tipping 3 | # as part of his SparseBayes software library. Initially published on GitHub on July 21st, 2015. 4 | 5 | # SB2_USEROPTIONS User option specification for SPARSEBAYES 6 | # 7 | # OPTIONS = SB2_USEROPTIONS(parameter1, value1, parameter2, value2,...) 8 | # 9 | # OUTPUT ARGUMENTS: 10 | # 11 | # OPTIONS An options structure to pass to SPARSEBAYES 12 | # 13 | # INPUT ARGUMENTS: 14 | # 15 | # Optional number of parameter-value pairs to specify the following: 16 | # 17 | # ITERATIONS Number of interations to run for. 18 | # 19 | # TIME Time limit to run for, expressed as a space-separated 20 | # string. e.g. '1.5 hours', '30 minutes', '1 second'. 21 | # 22 | # DIAGNOSTICLEVEL Integer [0,4] or string to determine the verbosity of 23 | # diagnostic output. 24 | # 0 or 'ZERO' or 'NONE' No output 25 | # 1 or 'LOW' Low level of output 26 | # 2 or 'MEDIUM' etc... 27 | # 3 or 'HIGH' 28 | # 4 or 'ULTRA' 29 | # 30 | # DIAGNOSTICFILE Filename to write diagnostics to file instead of 31 | # the default stdout. 32 | # 33 | # MONITOR Integer number: diagnostic information is output 34 | # every MONITOR iterations. 35 | # 36 | # FIXEDNOISE True/false whether the Gaussian noise is to be fixed 37 | # (default: false. 38 | # 39 | # FREEBASIS Indices of basis vectors considered "free" and not 40 | # constrained by the Bayesian prior (e.g. the "bias"). 41 | # 42 | # CALLBACK External function to call each iteration of the algorithm 43 | # (string). Intended to facilitate graphical demos etc. 44 | # 45 | # CALLBACKDATA Arbitrary additional data to pass to the CALLBACK 46 | # function. 47 | # 48 | # EXAMPLE: 49 | # 50 | # OPTIONS = SB2_UserOptions('diagnosticLevel','medium',... 51 | # 'monitor',25,... 52 | # 'diagnosticFile', 'logfile.txt'); 53 | # 54 | # NOTES: 55 | # 56 | # Each option (field of OPTIONS) is given a default value in 57 | # SB2_USEROPTIONS. Any supplied property-value pairs over-ride those 58 | # defaults. 59 | # 60 | 61 | # 62 | # Copyright 2009, Vector Anomaly Ltd 63 | # 64 | # This file is part of the SPARSEBAYES library for Matlab (V2.0). 65 | # 66 | # SPARSEBAYES is free software; you can redistribute it and/or modify it 67 | # under the terms of the GNU General Public License as published by the Free 68 | # Software Foundation; either version 2 of the License, or (at your option) 69 | # any later version. 70 | # 71 | # SPARSEBAYES is distributed in the hope that it will be useful, but WITHOUT 72 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 73 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 74 | # more details. 75 | # 76 | # You should have received a copy of the GNU General Public License along 77 | # with SPARSEBAYES in the accompanying file "licence.txt"; if not, write to 78 | # the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, 79 | # MA 02110-1301 USA 80 | # 81 | # Contact the author: m a i l [at] m i k e t i p p i n g . c o m 82 | # 83 | 84 | def SB2_UserOptions(*args): 85 | 86 | # Ensure arguments are supplied in pairs 87 | 88 | if len(args) % 2 != 0: 89 | raise Exception('Arguments to SB2_UserOptions should be (property, value) pairs') 90 | 91 | # Any options specified? 92 | 93 | numSettings = len(args)/2 94 | 95 | 96 | ########################################################################### 97 | 98 | # Set defaults 99 | 100 | OPTIONS = {} 101 | 102 | # Assume we will infer the noise in the Gaussian case 103 | 104 | OPTIONS['FIXEDNOISE'] = False 105 | 106 | # Option to allow subset of the basis (e.g. bias) to be unregularised 107 | 108 | OPTIONS['FREEBASIS'] = [] 109 | 110 | # Option to set max iterations to run for 111 | 112 | OPTIONS['ITERATIONS'] = 10000 113 | 114 | # Option to set max time to run for 115 | 116 | OPTIONS['TIME'] = 10000 # seconds 117 | 118 | # Set options for monitoring and recording the algorithm's progress 119 | 120 | OPTIONS['MONITOR'] = 0 121 | OPTIONS['DIAGNOSTICLEVEL'] = 0 122 | OPTIONS['DIAGNOSTICFID'] = 1 # stdout 123 | OPTIONS['DIAGNOSTICFILE'] = [] 124 | 125 | # Option to call a function during each iteration (to create demos etc) 126 | 127 | OPTIONS['CALLBACK'] = False 128 | OPTIONS['CALLBACKFUNC'] = [] 129 | OPTIONS['CALLBACKDATA'] = {} 130 | 131 | 132 | ########################################################################### 133 | 134 | # Parse string/variable pairs 135 | 136 | for n in range(numSettings): 137 | property_ = args[n*2] 138 | value = args[n*2 + 1] 139 | 140 | if property_ not in OPTIONS: 141 | raise Exception('Unrecognised user option: {0}'.format(property_)) 142 | 143 | OPTIONS[property_] = value 144 | 145 | if property_ == 'DIAGNOSTICLEVEL': 146 | if type(value) is str: 147 | if value == 'ZERO' or value == 'NONE': 148 | OPTIONS['DIAGNOSTICLEVEL'] = 0 149 | elif value == 'LOW': 150 | OPTIONS['DIAGNOSTICLEVEL'] = 1 151 | elif value == 'MEDIUM': 152 | OPTIONS['DIAGNOSTICLEVEL'] = 2 153 | elif value == 'HIGH': 154 | OPTIONS['DIAGNOSTICLEVEL'] = 3 155 | elif value == 'ULTRA': 156 | OPTIONS['DIAGNOSTICLEVEL'] = 4 157 | else: 158 | raise Exception('Unrecognised textual diagnostic level: {0}'.format(value)) 159 | elif type(value) is int: 160 | if value < 0 or value > 4: 161 | raise Exception('Supplied level should be integer in [0,4], or one of ZERO/LOW/MEDIUM/HIGH/ULTRA') 162 | 163 | if property_ == 'DIAGNOSTICFILE': 164 | OPTIONS['DIAGNOSTICFID'] = -1 # "It will be opened later" 165 | 166 | if property_ == 'CALLBACK': 167 | OPTIONS['CALLBACK'] = True 168 | OPTIONS['CALLBACKFUNC'] = value 169 | if OPTIONS['CALLBACKFUNC'] not in locals(): #UNCERTAIN ABOUT THIS 170 | raise Exception('Callback function {0} does not appear to exist'.format(value)) 171 | 172 | if property_ == 'TIME': 173 | OPTIONS['TIME'] = timeInSeconds(value) 174 | 175 | 176 | return OPTIONS 177 | 178 | 179 | 180 | 181 | ##### Support function: parse time specification 182 | 183 | def timeInSeconds(value_): 184 | 185 | args = value_.split() 186 | args[1] = args[1].upper() 187 | 188 | v = int(args[0]) 189 | 190 | if args[1] == 'SECONDS' or args[1] == 'SECOND': 191 | pass 192 | 193 | elif args[1] == 'MINUTES' or args[1] == 'MINUTE': 194 | v *= 60 195 | 196 | elif args[1] == 'HOURS' or args[1] == 'HOUR': 197 | v *= 3600 198 | 199 | else: 200 | raise Exception('Badly formed time string: {0}'.format(value_)) 201 | 202 | return v # Returns time in seconds 203 | -------------------------------------------------------------------------------- /SB2_FullStatistics.py: -------------------------------------------------------------------------------- 1 | 2 | # The following is a Python translation of a MATLAB file originally written principally by Mike Tipping 3 | # as part of his SparseBayes software library. Initially published on GitHub on July 21st, 2015. 4 | 5 | # SB2_FULLSTATISTICS Compute all relevant statistics in full for SPARSEBAYES 6 | # 7 | # [SIGMA,MU,S_IN,Q_IN,S_OUT,Q_OUT,FACTOR,LOGML,GAMMA,... 8 | # BETABASIS_PHI,BETA] = ... 9 | # SB2_FULLSTATISTICS(LIKELIHOOD,BASIS,PHI,TARGETS,USED,ALPHA,BETA,... 10 | # MU,BASIS_PHI,BASIS_TARGETS,OPTIONS) 11 | # 12 | # OUTPUT ARGUMENTS: 13 | # 14 | # SIGMA Posterior covariance matrix for relevant bases 15 | # MU Posterior mean 16 | # S_IN S-factors for in-model (relevant) basis vectors 17 | # Q_IN Q-factors for in-model (relevant) basis vectors 18 | # S_OUT S-factors for all basis vectors 19 | # Q_OUT Q-factors for all basis vectors 20 | # FACTOR Q^2-S relevance factors 21 | # LOGML Log-marginal-likelihood 22 | # GAMMA "Well-determinedness" factors 23 | # BETABASIS_PHI Cached value of BASIS'*B*PHI matrix 24 | # BETA Inverse noise variance (vector of beta 25 | # approximations in non-Gaussian case) 26 | # 27 | # INPUT ARGUMENTS: 28 | # 29 | # LIKELIHOOD LIKELIHOOD structure 30 | # BASIS Full NxM basis matrix 31 | # PHI Current relevant basis matrix 32 | # TARGETS N-vector with target output values 33 | # USED Relevant basis vector indices 34 | # ALPHA Hyperparameters 35 | # BETA Inverse noise variance (ignored in non-Gauss case) 36 | # MU Current posterior mean (for non-Gauss) 37 | # BASIS_PHI Cached value of BASIS'*PHI (for Gauss only) 38 | # BASIS_TARGETS Cached value of BASIS'*TARGETS (for Gauss only) 39 | # OPTIONS Standard OPTIONS structure (see SB2_USEROPTIONS) 40 | # 41 | # NOTES: 42 | # 43 | # This function computes the posterior, and other, statistics for the 44 | # SPARSEBAYES algorithm in "long-hand" fashion. i.e. it does not use the 45 | # more effecient "iterative" updates. 46 | # 47 | # It is required on every iteration (unless approximating) in the 48 | # non-Gaussian case, and on iterations in the Gaussian case where the noise 49 | # estimate (BETA) is updated. 50 | # 51 | # This function is intended for internal use by SPARSEBAYES only. 52 | # 53 | 54 | 55 | # 56 | # Copyright 2009, Vector Anomaly Ltd 57 | # 58 | # This file is part of the SPARSEBAYES library for Matlab (V2.0). 59 | # 60 | # SPARSEBAYES is free software; you can redistribute it and/or modify it 61 | # under the terms of the GNU General Public License as published by the Free 62 | # Software Foundation; either version 2 of the License, or (at your option) 63 | # any later version. 64 | # 65 | # SPARSEBAYES is distributed in the hope that it will be useful, but WITHOUT 66 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 67 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 68 | # more details. 69 | # 70 | # You should have received a copy of the GNU General Public License along 71 | # with SPARSEBAYES in the accompanying file "licence.txt"; if not, write to 72 | # the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, 73 | # MA 02110-1301 USA 74 | # 75 | # Contact the author: m a i l [at] m i k e t i p p i n g . c o m 76 | # 77 | 78 | import numpy as np 79 | from SB2_PosteriorMode import SB2_PosteriorMode 80 | from SB2_Sigmoid import SB2_Sigmoid 81 | 82 | def SB2_FullStatistics(LIKELIHOOD, BASIS, PHI, Targets, Used, Alpha, beta, Mu, BASIS_PHI, BASIS_Targets, OPTIONS): 83 | 84 | # Mu is only required for non-Gauss 85 | # BASIS_PHI and BASIS_Targets are only required for Gauss 86 | 87 | # beta (a vector for non-Gauss) is returned only for non-Gauss 88 | 89 | MAX_POSTMODE_ITS = 25 # More than enough 90 | 91 | [N, M_full] = BASIS.shape 92 | M = PHI.shape[1] 93 | 94 | 95 | ######################################################################## 96 | 97 | # COMPUTE POSTERIOR 98 | 99 | # 100 | # Compute full relevant statistics 101 | 102 | if LIKELIHOOD['InUse'] == LIKELIHOOD['Gaussian']: 103 | # 104 | # Posterior is analytic: use linear algebra here 105 | # 106 | # Compute posterior covariance SIGMA (inverse Hessian) 107 | # via Cholesky decomposition 108 | # 109 | 110 | U = np.linalg.cholesky(PHI.T * PHI * beta + np.diagflat(Alpha)) 111 | Ui = np.linalg.inv(U).T 112 | SIGMA = Ui * Ui.T 113 | 114 | # Posterior mean Mu 115 | Mu = (SIGMA * (PHI.T * Targets)) * beta 116 | 117 | # Data error and likelihood 118 | y = PHI * Mu 119 | e = (Targets - y) 120 | ED = e.T * e 121 | 122 | dataLikely = (N*np.log(beta) - beta*ED) / 2 123 | 124 | else: 125 | # Posterior must be approximated: find the posterior mode as the basis 126 | # for the Laplace approximation 127 | 128 | [Mu, U, beta, dataLikely, a] = SB2_PosteriorMode(LIKELIHOOD,PHI,Targets,Alpha,Mu, MAX_POSTMODE_ITS,OPTIONS) 129 | 130 | # Compute covariance approximation 131 | 132 | Ui = np.linalg.inv(U) 133 | SIGMA = Ui * Ui.T 134 | 135 | # Compute posterior-mean-based outputs and errors 136 | if LIKELIHOOD['InUse'] == LIKELIHOOD['Bernoulli']: 137 | y = SB2_Sigmoid(PHI * Mu) 138 | 139 | if LIKELIHOOD['InUse'] == LIKELIHOOD['Poisson']: 140 | y = np.exp(PHI*Mu) 141 | 142 | e = (Targets-y) 143 | 144 | ######################################################################## 145 | 146 | # COMPUTE LOG MARGINAL LIKELIHOOD 147 | 148 | logdetHOver2 = np.sum(np.log(U.diagonal())) 149 | logML = dataLikely - (np.power(Mu,2)).T*Alpha/2 + np.sum(np.log(Alpha))/2 - logdetHOver2 150 | 151 | # Well-determinedness factors 152 | DiagC = np.sum(np.power(Ui,2),1) 153 | Gamma = 1 - np.multiply(Alpha,DiagC) 154 | 155 | ######################################################################## 156 | 157 | # COMPUTE THE Q & S VALUES 158 | 159 | # Q: "quality" factor - related to how well the basis function contributes 160 | # to reducing the error 161 | # 162 | # S: "sparsity factor" - related to how orthogonal a given basis function 163 | # is to the currently used set of basis functions 164 | 165 | if LIKELIHOOD['InUse'] == LIKELIHOOD['Gaussian']: 166 | 167 | # Gaussian case simple: beta a scalar 168 | 169 | betaBASIS_PHI = beta*BASIS_PHI 170 | 171 | # The S_in calculation exploits the fact that BASIS is normalised, 172 | # i.e. sum(BASIS.^2,1)==ones(1,M_full) 173 | 174 | S_in = beta - np.sum(np.power(betaBASIS_PHI*Ui, 2), 1) 175 | Q_in = beta*(BASIS_Targets - BASIS_PHI*Mu) 176 | 177 | else: 178 | 179 | # Non-Gaussian case: beta an N-vector 180 | 181 | betaBASIS_PHI = BASIS.T*(np.multiply(PHI, beta*np.ones((1,M)))) 182 | S_in = (beta.T * np.power(BASIS,2)).T - np.sum(np.power(betaBASIS_PHI*Ui,2), 1) 183 | Q_in = BASIS.T*e 184 | 185 | S_out = np.matrix(S_in) 186 | Q_out = np.matrix(Q_in) 187 | 188 | # S,Q with that basis excluded: equations (23) 189 | S_out[Used] = np.divide(np.multiply(Alpha, S_in[Used]) , (Alpha - S_in[Used])) 190 | Q_out[Used] = np.divide(np.multiply(Alpha, Q_in[Used]) , (Alpha - S_in[Used])) 191 | 192 | # Pre-compute the "relevance factor" for ongoing convenience 193 | 194 | Factor = np.multiply(Q_out, Q_out) - S_out 195 | 196 | return [SIGMA, Mu, S_in, Q_in, S_out, Q_out, Factor, logML, Gamma, betaBASIS_PHI, beta] 197 | -------------------------------------------------------------------------------- /SB2_PosteriorMode.py: -------------------------------------------------------------------------------- 1 | 2 | # The following is a Python translation of a MATLAB file originally written principally by Mike Tipping 3 | # as part of his SparseBayes software library. Initially published on GitHub on July 21st, 2015. 4 | 5 | # SB2_POSTERIORMODE Posterior mode-finder for the SPARSEBAYES algorithm 6 | # 7 | # [MU, U, BETA, LIKELIHOODMODE, BADHESS] = ... 8 | # SB2_POSTERIORMODE(LIKELIHOOD,BASIS,TARGETS,ALPHA,MU,ITSMAX,OPTIONS) 9 | # 10 | # OUTPUT ARGUMENTS: 11 | # 12 | # MU Parameter values at the mode 13 | # U Cholesky factor of the covariance at the mode 14 | # BETA Vector of pseudo-noise variances at the mode 15 | # LIKELIHOODMODE Data likelihood at the mode 16 | # BADHESS Returns true if Hessian is "bad" (becomes 17 | # non-positive-definite during maximisation) 18 | # 19 | # INPUT ARGUMENTS: 20 | # 21 | # LIKELIHOOD LIKELIHOOD structure 22 | # BASIS Current relevant basis matrix 23 | # TARGETS N-vector with target output values 24 | # ALPHA Current hyperparameters 25 | # MU Current weights 26 | # ITSMAX Maximum number of iterations to run 27 | # OPTIONS Standard OPTIONS structure (only affects diagnostics) 28 | # 29 | # NOTES: 30 | # 31 | # SB2_POSTERIORMODE finds the posterior mode (with respect to the 32 | # weights) of the likelihood function in the non-Gaussian case to 33 | # facilitate subsequent Laplace approximation. 34 | # 35 | # This function is intended for internal use by SPARSEBAYES only (within 36 | # SB2_FullStatistics). 37 | # 38 | 39 | # 40 | # Copyright 2009, Vector Anomaly Ltd 41 | # 42 | # This file is part of the SPARSEBAYES library for Matlab (V2.0). 43 | # 44 | # SPARSEBAYES is free software; you can redistribute it and/or modify it 45 | # under the terms of the GNU General Public License as published by the Free 46 | # Software Foundation; either version 2 of the License, or (at your option) 47 | # any later version. 48 | # 49 | # SPARSEBAYES is distributed in the hope that it will be useful, but WITHOUT 50 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 51 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 52 | # more details. 53 | # 54 | # You should have received a copy of the GNU General Public License along 55 | # with SPARSEBAYES in the accompanying file "licence.txt"; if not, write to 56 | # the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, 57 | # MA 02110-1301 USA 58 | # 59 | # Contact the author: m a i l [at] m i k e t i p p i n g . c o m 60 | # 61 | 62 | import numpy as np 63 | from SB2_Sigmoid import SB2_Sigmoid 64 | from SB2_Diagnostic import SB2_Diagnostic 65 | 66 | def SB2_PosteriorMode(LIKELIHOOD, BASIS, Targets, Alpha, Mu, itsMax, OPTIONS): 67 | 68 | # TOLERANCES 69 | 70 | # Termination criterion for each gradient dimension 71 | 72 | GRADIENT_MIN = 1e-6 73 | 74 | # Minimum fraction of the full Newton step considered 75 | 76 | STEP_MIN = 1/float(2^8) 77 | 78 | 79 | [N, M] = BASIS.shape 80 | A = np.diagflat(Alpha) 81 | 82 | # NB: for historical reasons, we work in term of error here (negative 83 | # log-liklihood) and minimise 84 | 85 | 86 | # Get current model output and data error 87 | 88 | BASIS_Mu = BASIS * Mu # Linear output 89 | [dataError, y] = SB2_DataError(LIKELIHOOD, BASIS_Mu, Targets) 90 | 91 | # Add on the weight penalty 92 | 93 | regulariser = (Alpha.T * np.power(Mu, 2))/2 94 | newTotalError = dataError + regulariser 95 | 96 | badHess = False 97 | errorLog = np.zeros((itsMax,1)) 98 | 99 | for iteration in range(0, itsMax): 100 | 101 | # Log the error value each iteration 102 | 103 | errorLog[iteration] = newTotalError 104 | 105 | SB2_Diagnostic(OPTIONS,4,'PM cycle: {:d}\t error: {%:.6f}\n', iteration, errorLog[iteration]) 106 | 107 | # Construct the gradient 108 | 109 | e = (Targets-y) 110 | g = BASIS.T*e - np.multiply(Alpha, Mu) 111 | 112 | # Compute the likelihood-dependent analogue of the noise precision. 113 | # NB: Beta now a vector. 114 | 115 | if LIKELIHOOD['InUse'] == LIKELIHOOD['Bernoulli']: 116 | Beta = np.multiply(y,(1-y)) 117 | 118 | if LIKELIHOOD['InUse'] == LIKELIHOOD['Poisson']: 119 | Beta = y 120 | 121 | # Compute the Hessian 122 | 123 | BASIS_B = np.multiply(BASIS, (Beta * np.ones((1,M)))) 124 | H = (BASIS_B.T*BASIS + A) 125 | # Invert Hessian via Cholesky, watching out for ill-conditioning 126 | pdErr = False 127 | try: 128 | U = np.linalg.cholesky(H).T 129 | 130 | except np.linalg.linalg.LinAlgError: 131 | pdErr = True 132 | 133 | if pdErr: 134 | # If you see this, it's *probably* the result of a bad choice of 135 | # basis. e.g. a kernel with too large a "width" 136 | SB2_Diagnostic(OPTIONS, 1, '** Warning ** Ill-conditioned Hessian ({0})\n'.format(1/np.linalg.cond(H))) 137 | badHess = True 138 | U = [] 139 | Beta = [] 140 | likelihoodMode = [] 141 | return 142 | 143 | # Before progressing, check for termination based on the gradient norm 144 | if np.all(np.abs(g) STEP_MIN: 155 | # Follow gradient to get new value of parameters 156 | Mu_new = Mu + step*DeltaMu 157 | BASIS_Mu = BASIS*Mu_new 158 | 159 | # Compute outputs and error at new point 160 | 161 | [dataError,y] = SB2_DataError(LIKELIHOOD,BASIS_Mu,Targets) 162 | regulariser = (Alpha.T * (np.power(Mu_new,2)))/2 163 | newTotalError = dataError + regulariser 164 | 165 | # Test that we haven't made things worse 166 | 167 | if newTotalError >= errorLog[iteration]: 168 | # If so, back off! 169 | step = step/2 170 | SB2_Diagnostic(OPTIONS, 4,['PM error increase! Backing off to l= {0}\n'.format(step)]) 171 | else: 172 | Mu = Mu_new 173 | step = 0 # this will force exit from the "while" loop 174 | 175 | 176 | # If we get here with non-zero "step", it means that the smallest 177 | # offset from the current point along the "downhill" direction did not 178 | # lead to a decrease in error. In other words, we must be 179 | # infinitesimally close to a minimum (which is OK). 180 | 181 | if step != 0: 182 | SB2_Diagnostic(OPTIONS, 4, 'PM stopping due to back-off limit (|g| = {0})\n'.format(np.max(np.abs(g)))) 183 | break 184 | 185 | # Simple computation of return value of log likelihood at mode 186 | 187 | likelihoodMode = -dataError 188 | 189 | return [Mu, U, Beta, likelihoodMode, badHess] 190 | 191 | 192 | 193 | 194 | 195 | ###################### 196 | # 197 | # Support function 198 | # 199 | ###################### 200 | 201 | 202 | def SB2_DataError(LIKELIHOOD, BASIS_Mu,Targets): 203 | 204 | if LIKELIHOOD['InUse'] == LIKELIHOOD['Bernoulli']: 205 | y = SB2_Sigmoid(BASIS_Mu) 206 | 207 | # Handle probability zero cases 208 | y0 = (y==0) 209 | y1 = (y==1) 210 | 211 | if np.any(y0[Targets>0]) or np.any(y1[Targets<1]): 212 | # Error infinite when model gives zero probability in 213 | # contradiction to data 214 | e = np.inf 215 | 216 | else: 217 | # Any y=0 or y=1 cases must now be accompanied by appropriate 218 | # output=0 or output=1 values, so can be excluded. 219 | e = -(Targets[np.logical_not(y0)] * np.log(y[np.logical_not(y0)].T) + (1-Targets[np.logical_not(y1)].T).T * np.log(1-y[np.logical_not(y1)].T))[0,0] 220 | 221 | if LIKELIHOOD['InUse'] == LIKELIHOOD['Poisson']: 222 | y = np.exp(BASIS_Mu) 223 | e = -np.sum(np.multiply(Targets, BASIS_Mu) - y) 224 | 225 | return [e, y] 226 | -------------------------------------------------------------------------------- /SB2_Initialisation.py: -------------------------------------------------------------------------------- 1 | 2 | # The following is a Python translation of a MATLAB file originally written principally by Mike Tipping 3 | # as part of his SparseBayes software library. Initially published on GitHub on July 21st, 2015. 4 | 5 | # SB2_INITIALISATION Initialise everything for the SPARSEBAYES algorithm 6 | # 7 | # [LIKELIHOOD, BASIS, BASISSCALES, ALPHA, BETA, MU, PHI, USED] = ... 8 | # SB2_INITIALISATION(TYPE, BASIS, TARGETS, SETTINGS, OPTIONS) 9 | # 10 | # OUTPUT ARGUMENTS: 11 | # 12 | # LIKELIHOOD Likelihood structure (from SB2_LIKELIHOODS) 13 | # BASIS Pre-processed full basis matrix 14 | # BASISSCALES Scaling factors from full basis pre-processing 15 | # ALPHA Initial hyperparameter alpha values 16 | # BETA Initial noise level (Gaussian) 17 | # MU Initial weight values 18 | # PHI Initial "relevant" basis 19 | # USED Indices of initial basis vectors (columns of BASIS) 20 | # 21 | # INPUT ARGUMENTS: 22 | # 23 | # TYPE Likelihood: one of 'Gaussian', 'Bernoulli', 'Poisson' 24 | # 25 | # BASIS NxM matrix of all possible basis vectors 26 | # (one column per basis function) 27 | # 28 | # TARGETS N-vector with target output values 29 | # 30 | # SETTINGS Initialisation structure for main parameter values via 31 | # SB2_PARAMETERSETTINGS 32 | # 33 | # OPTIONS User options structure from SB2_USEROPTIONS 34 | # 35 | # NOTES: 36 | # 37 | # This function initialises all necessary model parameters appropriately 38 | # before entering the main loop of the SPARSEBAYES inference algorithm. 39 | # 40 | # This function is intended for internal use by SPARSEBAYES only. 41 | # 42 | 43 | # 44 | # Copyright 2009, Vector Anomaly Ltd 45 | # 46 | # This file is part of the SPARSEBAYES library for Matlab (V2.0). 47 | # 48 | # SPARSEBAYES is free software; you can redistribute it and/or modify it 49 | # under the terms of the GNU General Public License as published by the Free 50 | # Software Foundation; either version 2 of the License, or (at your option) 51 | # any later version. 52 | # 53 | # SPARSEBAYES is distributed in the hope that it will be useful, but WITHOUT 54 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 55 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 56 | # more details. 57 | # 58 | # You should have received a copy of the GNU General Public License along 59 | # with SPARSEBAYES in the accompanying file "licence.txt"; if not, write to 60 | # the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, 61 | # MA 02110-1301 USA 62 | # 63 | # Contact the author: m a i l [at] m i k e t i p p i n g . c o m 64 | # 65 | 66 | from SB2_PreProcessBasis import SB2_PreProcessBasis 67 | from SB2_Likelihoods import SB2_Likelihoods 68 | from SB2_Diagnostic import SB2_Diagnostic 69 | import numpy as np 70 | 71 | def SB2_Initialisation(likelihood_, BASIS, Targets, SETTINGS, OPTIONS): 72 | 73 | # A "reasonable" initial value for the noise in the Gaussian case 74 | 75 | GAUSSIAN_SNR_INIT = 0.1 76 | 77 | # "Reasonable" initial alpha bounds 78 | INIT_ALPHA_MAX = 1e3 79 | INIT_ALPHA_MIN = 1e-3 80 | 81 | # BASIS PREPROCESSING: 82 | 83 | # Scale basis vectors to unit norm. This eases some calculations and 84 | # will improve numerical robustness later. 85 | 86 | [BASIS, BasisScales] = SB2_PreProcessBasis(BASIS) 87 | 88 | ## Noise model considerations 89 | 90 | # Validate the likelihood model specification 91 | 92 | LIKELIHOOD = SB2_Likelihoods(likelihood_) 93 | 94 | # Default beta value for non-Gaussian likelihood 95 | 96 | beta = [] 97 | 98 | # In the Gaussian case, initialise "sensibly" to a proportion of the 99 | # signal level (e.g. 10 percent) 100 | 101 | if LIKELIHOOD['InUse'] == LIKELIHOOD['Gaussian']: 102 | # Noise initialisation 103 | if not not SETTINGS['BETA']: 104 | # The user set "beta" 105 | beta = SETTINGS['beta'] 106 | 107 | elif not not SETTINGS['NOISESTD'] and SETTINGS['NOISESTD'] > 0: 108 | # The user set the "noise std dev" 109 | beta = 1/float(SETTINGS['NOISESTD']) ** 2 110 | 111 | else: 112 | # catch the pathological case where all outputs are zero 113 | # (although we're probably doomed anyway if that's true) 114 | stdt = max(1e-6, np.std(Targets)) 115 | 116 | # Initialise automatically approximately according to "SNR" 117 | beta = 1 / (float(stdt*GAUSSIAN_SNR_INIT) ** 2) 118 | 119 | 120 | # Initialise basis (PHI), mu and alpha 121 | 122 | # Either as specified by the SETTINGS structure, or heuristically 123 | 124 | 125 | # First, compute 'linearised' output for use in heuristic initialisation 126 | 127 | TargetsPseudoLinear = Targets # standard linear case 128 | TargetsPseudoLinear = np.matrix(TargetsPseudoLinear) 129 | 130 | if LIKELIHOOD['InUse'] == LIKELIHOOD['Bernoulli']: 131 | TargetsPseudoLinear = (2*Targets - 1) 132 | 133 | if LIKELIHOOD['InUse'] == LIKELIHOOD['Poisson']: 134 | TargetsPseudoLinear = np.log(Targets + 1e-3) 135 | 136 | # 1) the starting basis, PHI 137 | 138 | # Take account of "free basis": it needs to be included from the outset 139 | 140 | Extra = np.setdiff1d(OPTIONS['FREEBASIS'], SETTINGS['RELEVANT']) 141 | 142 | # [TODO:] More elegant to put "Extra" first in Used list? 143 | 144 | Used = np.hstack([SETTINGS['RELEVANT'], Extra]) 145 | 146 | # At this point Used will contain both pre-specified relevant basis 147 | # functions (SB2_PARAMETERSETTINGS) and any "free" ones (SB2_USEROPTIONS). 148 | # If neither are active, Used will be empty. 149 | 150 | if not Used: 151 | # Set initial basis to be the largest projection with the targets 152 | proj = (BASIS.T * TargetsPseudoLinear) 153 | foo, Used = np.abs(proj).max(0), np.abs(proj).argmax(0) 154 | SB2_Diagnostic(OPTIONS, 2, 'Initialising with maximally aligned basis vector ({0})'.format(Used[0,0])) 155 | 156 | else: 157 | SB2_Diagnostic(OPTIONS, 2, 'Initialising with supplied basis of size M = {0}'.format(len(Used))) 158 | 159 | Used = np.array(Used).flatten() 160 | PHI = BASIS[:, Used] # If Used is a vector, not an integer, this may break 161 | 162 | if type(Used) == int: 163 | M = 1 164 | else: 165 | M = len(Used) 166 | 167 | 168 | # 2) the most probable weights 169 | 170 | if not SETTINGS['MU']: 171 | 172 | if LIKELIHOOD['InUse'] == LIKELIHOOD['Gaussian']: 173 | # MU (WEIGHTS) will be analytically calculated later in the Gaussian case 174 | 175 | Mu = [] 176 | 177 | if LIKELIHOOD['InUse'] == LIKELIHOOD['Bernoulli']: 178 | # Heuristic initialisation based on log-odds 179 | LogOut = (TargetsPseudoLinear*0.9 + 1)/float(2) 180 | Mu = np.linalg.lstsq(PHI, np.log(LogOut / (1-LogOut)) )[0] 181 | 182 | if LIKELIHOOD['InUse'] == LIKELIHOOD['Poisson']: 183 | # Heuristic initialisation based on log 184 | Mu = np.linalg.solve(PHI, TargetsPseudoLinear) 185 | 186 | else: 187 | if len(SETTINGS['Mu']) != len(SETTINGS['Relevant']): 188 | raise Exception('Basis length {0} should equal weight vector length {1}'.format( len(SETTINGS['Relevant']), len(SETTINGS['Mu']))) 189 | 190 | SB2_Diagnostic(OPTIONS, 2,'Initialising with supplied weights') 191 | # We may have extra "freebasis" functions without supplied weights 192 | # - set those to zero for now 193 | Mu = np.concatenate( (Mu, np.zeros((max(Extra.shape), 1)) ), axis=1) 194 | 195 | 196 | # 3) the hyperparameters 197 | 198 | if not SETTINGS['ALPHA']: 199 | 200 | if LIKELIHOOD['InUse'] == LIKELIHOOD['Gaussian']: 201 | # Exact for single basis function case (diag irrelevant), 202 | # heuristic in the multiple case 203 | 204 | p = np.matrix(np.diag(PHI.T * PHI)).T * beta 205 | q = (PHI.T * Targets) * beta 206 | Alpha = p**2 / (q ** 2 - p) 207 | 208 | # Its possible that one of the basis functions could already be 209 | # irrelevant (alpha<0), so trap that 210 | if (Alpha<0).all(): 211 | SB2_Diagnostic(OPTIONS, 1, 'WARNING: no relevant basis function at initialisation') 212 | 213 | # The main algorithm will handle these automatically shortly 214 | # (i.e. prune them) 215 | Alpha[Alpha<0] = INIT_ALPHA_MAX 216 | 217 | if LIKELIHOOD['InUse'] == LIKELIHOOD['Bernoulli'] or LIKELIHOOD['InUse'] == LIKELIHOOD['Poisson']: 218 | # Heuristic initialisation, trapping Mu==0 219 | Alpha = 1 / (Mu + (Mu==0)) ** 2 220 | # Limit max/min alpha 221 | Alpha[Alpha < INIT_ALPHA_MIN] = INIT_ALPHA_MIN 222 | Alpha[Alpha > INIT_ALPHA_MAX] = INIT_ALPHA_MAX 223 | 224 | if M == 1: 225 | SB2_Diagnostic(OPTIONS,2,'Initial alpha = {0}'.format(Alpha[0,0])) 226 | 227 | else: 228 | if len(SETTINGS['ALPHA']) != len(SETTINGS['RELEVANT']): # Replace len with max( .shape) ? 229 | raise Exception('Basis length {0} should equal alpha vector length {1}'.format(len(SETTINGS['Relevant']), len(SETTINGS['Alpha']))) 230 | 231 | # We may have extra "freebasis" functions without supplied alpha 232 | # - set those to zero for now 233 | Alpha = np.concatenate(SETTINGS['Alpha'], np.zeros((max(Extra.shape),1))) 234 | 235 | 236 | # Finally - set Alphas to zero for free basis 237 | 238 | ALPHA_ZERO = np.spacing(1) 239 | Alpha[np.in1d(Used,OPTIONS['FREEBASIS'])] = ALPHA_ZERO 240 | 241 | for s in np.nditer(Alpha, op_flags=['readwrite']): 242 | if s in OPTIONS['FREEBASIS']: 243 | s[...] = ALPHA_ZERO 244 | 245 | return [LIKELIHOOD, BASIS, BasisScales, Alpha, beta, Mu, PHI, Used] 246 | -------------------------------------------------------------------------------- /SparseBayesDemo.py: -------------------------------------------------------------------------------- 1 | 2 | # The following is a Python translation of a MATLAB file originally written principally by Mike Tipping 3 | # as part of his SparseBayes software library. Initially published on GitHub on July 21st, 2015. 4 | 5 | # SPARSEBAYESDEMO Simple demonstration of the SPARSEBAYES algorithm 6 | # 7 | # SPARSEBAYESDEMO(LIKELIHOOD, DIMENSION, NOISETOSIGNAL) 8 | # 9 | # OUTPUT ARGUMENTS: None 10 | # 11 | # INPUT ARGUMENTS: 12 | # 13 | # LIKELIHOOD Text string, one of 'Gaussian' or 'Bernoulli' 14 | # DIMENSION Integer, 1 or 2 15 | # NOISETOSIGNAL An optional positive number to specify the 16 | # noise-to-signal (standard deviation) fraction. 17 | # (Optional: default value is 0.2). 18 | # 19 | # EXAMPLES: 20 | # 21 | # SPARSEBAYESDEMO("Bernoulli",2) 22 | # SPARSEBAYESDEMO("Gaussian",1,0.5) 23 | # 24 | # NOTES: 25 | # 26 | # This program offers a simple demonstration of how to use the 27 | # SPARSEBAYES (V2) Matlab software. 28 | # 29 | # Synthetic data is generated from an underlying linear model based 30 | # on a set of "Gaussian" basis functions, with the generator being 31 | # "sparse" such that 10% of potential weights are non-zero. Data may be 32 | # generated in an input space of one or two dimensions. 33 | # 34 | # This generator is then used either as the basis for real-valued data with 35 | # additive Gaussian noise (whose level may be varied), or for binary 36 | # class-labelled data based on probabilities given by a sigmoid link 37 | # function. 38 | # 39 | # The SPARSEBAYES algorithm is then run on the data, and results and 40 | # diagnostic information are graphed. 41 | # 42 | 43 | # 44 | # Copyright 2009, Vector Anomaly Ltd 45 | # 46 | # This file is part of the SPARSEBAYES library for Matlab (V2.0). 47 | # 48 | # SPARSEBAYES is free software; you can redistribute it and/or modify it 49 | # under the terms of the GNU General Public License as published by the Free 50 | # Software Foundation; either version 2 of the License, or (at your option) 51 | # any later version. 52 | # 53 | # SPARSEBAYES is distributed in the hope that it will be useful, but WITHOUT 54 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 55 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 56 | # more details. 57 | # 58 | # You should have received a copy of the GNU General Public License along 59 | # with SPARSEBAYES in the accompanying file "licence.txt"; if not, write to 60 | # the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, 61 | # MA 02110-1301 USA 62 | # 63 | # Contact the author: m a i l [at] m i k e t i p p i n g . c o m 64 | # 65 | 66 | import numpy as np 67 | import matplotlib.pyplot as plt 68 | import math 69 | from SB2_Likelihoods import SB2_Likelihoods 70 | from SB2_UserOptions import SB2_UserOptions 71 | from SB2_ParameterSettings import SB2_ParameterSettings 72 | from SparseBayes import SparseBayes 73 | from SB2_Sigmoid import SB2_Sigmoid 74 | 75 | def SparseBayesDemo(*args): 76 | likelihood_ = args[0] 77 | dimension = args[1] 78 | if len(args) == 3: 79 | noiseToSignal = args[2] 80 | 81 | # Fix the random seed for reproducibility of results 82 | 83 | rseed = 1 84 | np.random.seed(rseed) 85 | 86 | ####################################################################### 87 | # 88 | # --- VALIDATE INPUT ARGUMENTS --- 89 | # 90 | ####################################################################### 91 | 92 | # (1) likelihood 93 | 94 | LIKELIHOOD = SB2_Likelihoods(likelihood_) 95 | 96 | # (2) dimension 97 | 98 | if (dimension != 1) and (dimension != 2): 99 | raise Exception('Specified dimension should be 1 or 2') 100 | 101 | # Set up default for "noiseToSignal" variable. 102 | # For ease of use, we'll just ignore it in the case of a non-Gaussian 103 | # likelihood model. 104 | 105 | if 'noiseToSignal' not in locals(): 106 | noiseToSignal = 0.2 107 | 108 | ####################################################################### 109 | # 110 | # --- SET UP DEMO PARAMETERS --- 111 | # 112 | ####################################################################### 113 | # 114 | # Experiment with these values to vary the demo if you wish 115 | # 116 | 117 | if dimension == 1: 118 | N = 100 # Number of points 119 | else: 120 | N = 900 # Gives a nice square grid of decent size 121 | 122 | basisWidth = 0.05 # NB: data is in [0,1] 123 | 124 | # Define probability of a basis function NOT being used by the generative 125 | # model. i.e. if pSparse=0.90, only 10% of basis functions (on average) will 126 | # be used to synthesise the data. 127 | 128 | pSparse = 0.90 129 | iterations = 500 130 | 131 | # Heuristically adjust basis width to account for 132 | # distance scaling with dimension. 133 | 134 | basisWidth = basisWidth ** (1/float(dimension)) 135 | 136 | ####################################################################### 137 | # 138 | # --- SYNTHETIC DATA GENERATION --- 139 | # 140 | ####################################################################### 141 | # 142 | # First define the input data over a regular grid 143 | # 144 | 145 | 146 | if (dimension == 1): 147 | X = (np.matrix(range(N)) / float(N)).T 148 | 149 | else: # dimension is 2 150 | sqrtN = math.floor(math.sqrt(N)) 151 | N = sqrtN*sqrtN 152 | x = (np.matrix(range(int(sqrtN))) / sqrtN).T 153 | X = np.zeros((sqrtN ** 2, 2), float) 154 | for i in range(int(sqrtN)): 155 | for j in range(int(sqrtN)): 156 | X[i*int(sqrtN) + j, 0] = x[i] 157 | X[i*int(sqrtN) + j, 1] = x[j] 158 | X = np.matrix(X) 159 | 160 | 161 | # Now define the basis 162 | # 163 | # Locate basis functions at data points 164 | 165 | C = X 166 | 167 | # Compute ("Gaussian") basis (design) matrix 168 | 169 | BASIS = np.exp(-distSquared(X,C)/(basisWidth**2)) 170 | 171 | # Randomise some weights, then make each weight sparse with probability 172 | # pSparse 173 | 174 | M = BASIS.shape[1] 175 | w = np.random.randn(M,1)*100 / float((M*(1-pSparse))) 176 | sparse = np.random.rand(M,1) 0.5 315 | 316 | 317 | 318 | 319 | ####################################################################### 320 | # 321 | # --- PLOT THE RESULTS --- 322 | # 323 | ####################################################################### 324 | 325 | 326 | # Likelihood trace (and Gaussian noise info) 327 | 328 | plt.subplot(fRows,fCols,SP_LIKELY) 329 | lsteps = np.size(DIAGNOSTIC['LIKELIHOOD']) 330 | plt.plot(range(0,lsteps), DIAGNOSTIC['LIKELIHOOD'], 'g-') 331 | plt.xlim(0, lsteps+1) 332 | plt.title('Log marginal likelihood trace',fontsize=TITLE_SIZE) 333 | 334 | if LIKELIHOOD['InUse'] == LIKELIHOOD['Gaussian']: 335 | ax = plt.axis() 336 | dx = ax[1]-ax[0] 337 | dy = ax[3]-ax[2] 338 | t_ = 'Actual noise: {:.5f}'.format(noise) 339 | plt.text(ax[0]+0.1*dx,ax[2]+0.6*dy,t_,fontname='Courier') 340 | t_ = 'Inferred noise: {:.5f}'.format( 1/math.sqrt(HYPERPARAMETER['BETA']) ) 341 | plt.text(ax[0]+0.1*dx,ax[2]+0.5*dy,t_,fontname='Courier') 342 | 343 | 344 | # Compare the generative and predictive linear models 345 | if dimension == 1: 346 | plt.subplot(fRows,fCols,SP_LINEAR) 347 | 348 | if dimension == 1: 349 | plt.plot(X,z,'b-', linewidth=4, label='Actual') 350 | plt.plot(X,y,'r-', linewidth=3, label='Model') 351 | else: 352 | pass 353 | plt.title('Generative function and linear model',fontsize=TITLE_SIZE) 354 | legend = plt.legend(loc=2, shadow=False, fontsize='small', frameon=False) 355 | 356 | 357 | # Compare the data and the predictive model (post link-function) 358 | if dimension == 1: 359 | plt.subplot(fRows,fCols,SP_COMPARE) 360 | if dimension == 1: 361 | plt.plot(X,Outputs,'k.', linewidth=4) 362 | plt.plot(X,y_l,'r-', linewidth=3) 363 | plt.plot(X[PARAMETER['RELEVANT']],Outputs[PARAMETER['RELEVANT']],'yo', markersize=8, clip_on=False) 364 | else: 365 | pass 366 | plt.title('Data and predictor',fontsize=TITLE_SIZE) 367 | 368 | 369 | # Show the inferred weights 370 | 371 | plt.subplot(fRows,fCols,SP_WEIGHTS) 372 | markerline, stemlines, baseline = plt.stem(w_infer, 'r-.') 373 | plt.setp(markerline, markerfacecolor='r') 374 | plt.setp(baseline, color='k', linewidth=1) 375 | plt.xlim(0, N+1) 376 | t_ = 'Inferred weights ({:d})'.format(len(PARAMETER['RELEVANT'])) 377 | plt.title(t_,fontsize=TITLE_SIZE) 378 | 379 | 380 | # Show the "well-determinedness" factors 381 | 382 | plt.subplot(fRows,fCols,SP_GAMMA) 383 | ind = np.arange(len(DIAGNOSTIC['GAMMA'])) + 1 384 | plt.bar(ind, DIAGNOSTIC['GAMMA'], 0.7, color='g', align = 'center') 385 | plt.xlim(0, len(PARAMETER['RELEVANT'])+1) 386 | plt.ylim(0, 1.1) 387 | plt.title('Well-determinedness (gamma)',fontsize=TITLE_SIZE) 388 | 389 | plt.show() 390 | 391 | 392 | 393 | ####################################################################### 394 | # 395 | # Support function to compute basis 396 | # 397 | 398 | 399 | def distSquared(X,Y): 400 | nx = np.size(X, 0) 401 | ny = np.size(Y, 0) 402 | D2 = ( np.multiply(X, X).sum(1) * np.ones((1, ny)) ) + ( np.ones((nx, 1)) * np.multiply(Y, Y).sum(1).T ) - 2*X*Y.T 403 | return D2 404 | -------------------------------------------------------------------------------- /licence.txt: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 2, June 1991 3 | 4 | Copyright (C) 1989, 1991 Free Software Foundation, Inc., 5 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | Preamble 10 | 11 | The licenses for most software are designed to take away your 12 | freedom to share and change it. By contrast, the GNU General Public 13 | License is intended to guarantee your freedom to share and change free 14 | software--to make sure the software is free for all its users. This 15 | General Public License applies to most of the Free Software 16 | Foundation's software and to any other program whose authors commit to 17 | using it. (Some other Free Software Foundation software is covered by 18 | the GNU Lesser General Public License instead.) You can apply it to 19 | your programs, too. 20 | 21 | When we speak of free software, we are referring to freedom, not 22 | price. Our General Public Licenses are designed to make sure that you 23 | have the freedom to distribute copies of free software (and charge for 24 | this service if you wish), that you receive source code or can get it 25 | if you want it, that you can change the software or use pieces of it 26 | in new free programs; and that you know you can do these things. 27 | 28 | To protect your rights, we need to make restrictions that forbid 29 | anyone to deny you these rights or to ask you to surrender the rights. 30 | These restrictions translate to certain responsibilities for you if you 31 | distribute copies of the software, or if you modify it. 32 | 33 | For example, if you distribute copies of such a program, whether 34 | gratis or for a fee, you must give the recipients all the rights that 35 | you have. You must make sure that they, too, receive or can get the 36 | source code. And you must show them these terms so they know their 37 | rights. 38 | 39 | We protect your rights with two steps: (1) copyright the software, and 40 | (2) offer you this license which gives you legal permission to copy, 41 | distribute and/or modify the software. 42 | 43 | Also, for each author's protection and ours, we want to make certain 44 | that everyone understands that there is no warranty for this free 45 | software. If the software is modified by someone else and passed on, we 46 | want its recipients to know that what they have is not the original, so 47 | that any problems introduced by others will not reflect on the original 48 | authors' reputations. 49 | 50 | Finally, any free program is threatened constantly by software 51 | patents. We wish to avoid the danger that redistributors of a free 52 | program will individually obtain patent licenses, in effect making the 53 | program proprietary. To prevent this, we have made it clear that any 54 | patent must be licensed for everyone's free use or not licensed at all. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | GNU GENERAL PUBLIC LICENSE 60 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 61 | 62 | 0. This License applies to any program or other work which contains 63 | a notice placed by the copyright holder saying it may be distributed 64 | under the terms of this General Public License. The "Program", below, 65 | refers to any such program or work, and a "work based on the Program" 66 | means either the Program or any derivative work under copyright law: 67 | that is to say, a work containing the Program or a portion of it, 68 | either verbatim or with modifications and/or translated into another 69 | language. (Hereinafter, translation is included without limitation in 70 | the term "modification".) Each licensee is addressed as "you". 71 | 72 | Activities other than copying, distribution and modification are not 73 | covered by this License; they are outside its scope. The act of 74 | running the Program is not restricted, and the output from the Program 75 | is covered only if its contents constitute a work based on the 76 | Program (independent of having been made by running the Program). 77 | Whether that is true depends on what the Program does. 78 | 79 | 1. You may copy and distribute verbatim copies of the Program's 80 | source code as you receive it, in any medium, provided that you 81 | conspicuously and appropriately publish on each copy an appropriate 82 | copyright notice and disclaimer of warranty; keep intact all the 83 | notices that refer to this License and to the absence of any warranty; 84 | and give any other recipients of the Program a copy of this License 85 | along with the Program. 86 | 87 | You may charge a fee for the physical act of transferring a copy, and 88 | you may at your option offer warranty protection in exchange for a fee. 89 | 90 | 2. You may modify your copy or copies of the Program or any portion 91 | of it, thus forming a work based on the Program, and copy and 92 | distribute such modifications or work under the terms of Section 1 93 | above, provided that you also meet all of these conditions: 94 | 95 | a) You must cause the modified files to carry prominent notices 96 | stating that you changed the files and the date of any change. 97 | 98 | b) You must cause any work that you distribute or publish, that in 99 | whole or in part contains or is derived from the Program or any 100 | part thereof, to be licensed as a whole at no charge to all third 101 | parties under the terms of this License. 102 | 103 | c) If the modified program normally reads commands interactively 104 | when run, you must cause it, when started running for such 105 | interactive use in the most ordinary way, to print or display an 106 | announcement including an appropriate copyright notice and a 107 | notice that there is no warranty (or else, saying that you provide 108 | a warranty) and that users may redistribute the program under 109 | these conditions, and telling the user how to view a copy of this 110 | License. (Exception: if the Program itself is interactive but 111 | does not normally print such an announcement, your work based on 112 | the Program is not required to print an announcement.) 113 | 114 | These requirements apply to the modified work as a whole. If 115 | identifiable sections of that work are not derived from the Program, 116 | and can be reasonably considered independent and separate works in 117 | themselves, then this License, and its terms, do not apply to those 118 | sections when you distribute them as separate works. But when you 119 | distribute the same sections as part of a whole which is a work based 120 | on the Program, the distribution of the whole must be on the terms of 121 | this License, whose permissions for other licensees extend to the 122 | entire whole, and thus to each and every part regardless of who wrote it. 123 | 124 | Thus, it is not the intent of this section to claim rights or contest 125 | your rights to work written entirely by you; rather, the intent is to 126 | exercise the right to control the distribution of derivative or 127 | collective works based on the Program. 128 | 129 | In addition, mere aggregation of another work not based on the Program 130 | with the Program (or with a work based on the Program) on a volume of 131 | a storage or distribution medium does not bring the other work under 132 | the scope of this License. 133 | 134 | 3. You may copy and distribute the Program (or a work based on it, 135 | under Section 2) in object code or executable form under the terms of 136 | Sections 1 and 2 above provided that you also do one of the following: 137 | 138 | a) Accompany it with the complete corresponding machine-readable 139 | source code, which must be distributed under the terms of Sections 140 | 1 and 2 above on a medium customarily used for software interchange; or, 141 | 142 | b) Accompany it with a written offer, valid for at least three 143 | years, to give any third party, for a charge no more than your 144 | cost of physically performing source distribution, a complete 145 | machine-readable copy of the corresponding source code, to be 146 | distributed under the terms of Sections 1 and 2 above on a medium 147 | customarily used for software interchange; or, 148 | 149 | c) Accompany it with the information you received as to the offer 150 | to distribute corresponding source code. (This alternative is 151 | allowed only for noncommercial distribution and only if you 152 | received the program in object code or executable form with such 153 | an offer, in accord with Subsection b above.) 154 | 155 | The source code for a work means the preferred form of the work for 156 | making modifications to it. For an executable work, complete source 157 | code means all the source code for all modules it contains, plus any 158 | associated interface definition files, plus the scripts used to 159 | control compilation and installation of the executable. However, as a 160 | special exception, the source code distributed need not include 161 | anything that is normally distributed (in either source or binary 162 | form) with the major components (compiler, kernel, and so on) of the 163 | operating system on which the executable runs, unless that component 164 | itself accompanies the executable. 165 | 166 | If distribution of executable or object code is made by offering 167 | access to copy from a designated place, then offering equivalent 168 | access to copy the source code from the same place counts as 169 | distribution of the source code, even though third parties are not 170 | compelled to copy the source along with the object code. 171 | 172 | 4. You may not copy, modify, sublicense, or distribute the Program 173 | except as expressly provided under this License. Any attempt 174 | otherwise to copy, modify, sublicense or distribute the Program is 175 | void, and will automatically terminate your rights under this License. 176 | However, parties who have received copies, or rights, from you under 177 | this License will not have their licenses terminated so long as such 178 | parties remain in full compliance. 179 | 180 | 5. You are not required to accept this License, since you have not 181 | signed it. However, nothing else grants you permission to modify or 182 | distribute the Program or its derivative works. These actions are 183 | prohibited by law if you do not accept this License. Therefore, by 184 | modifying or distributing the Program (or any work based on the 185 | Program), you indicate your acceptance of this License to do so, and 186 | all its terms and conditions for copying, distributing or modifying 187 | the Program or works based on it. 188 | 189 | 6. Each time you redistribute the Program (or any work based on the 190 | Program), the recipient automatically receives a license from the 191 | original licensor to copy, distribute or modify the Program subject to 192 | these terms and conditions. You may not impose any further 193 | restrictions on the recipients' exercise of the rights granted herein. 194 | You are not responsible for enforcing compliance by third parties to 195 | this License. 196 | 197 | 7. If, as a consequence of a court judgment or allegation of patent 198 | infringement or for any other reason (not limited to patent issues), 199 | conditions are imposed on you (whether by court order, agreement or 200 | otherwise) that contradict the conditions of this License, they do not 201 | excuse you from the conditions of this License. If you cannot 202 | distribute so as to satisfy simultaneously your obligations under this 203 | License and any other pertinent obligations, then as a consequence you 204 | may not distribute the Program at all. For example, if a patent 205 | license would not permit royalty-free redistribution of the Program by 206 | all those who receive copies directly or indirectly through you, then 207 | the only way you could satisfy both it and this License would be to 208 | refrain entirely from distribution of the Program. 209 | 210 | If any portion of this section is held invalid or unenforceable under 211 | any particular circumstance, the balance of the section is intended to 212 | apply and the section as a whole is intended to apply in other 213 | circumstances. 214 | 215 | It is not the purpose of this section to induce you to infringe any 216 | patents or other property right claims or to contest validity of any 217 | such claims; this section has the sole purpose of protecting the 218 | integrity of the free software distribution system, which is 219 | implemented by public license practices. Many people have made 220 | generous contributions to the wide range of software distributed 221 | through that system in reliance on consistent application of that 222 | system; it is up to the author/donor to decide if he or she is willing 223 | to distribute software through any other system and a licensee cannot 224 | impose that choice. 225 | 226 | This section is intended to make thoroughly clear what is believed to 227 | be a consequence of the rest of this License. 228 | 229 | 8. If the distribution and/or use of the Program is restricted in 230 | certain countries either by patents or by copyrighted interfaces, the 231 | original copyright holder who places the Program under this License 232 | may add an explicit geographical distribution limitation excluding 233 | those countries, so that distribution is permitted only in or among 234 | countries not thus excluded. In such case, this License incorporates 235 | the limitation as if written in the body of this License. 236 | 237 | 9. The Free Software Foundation may publish revised and/or new versions 238 | of the General Public License from time to time. Such new versions will 239 | be similar in spirit to the present version, but may differ in detail to 240 | address new problems or concerns. 241 | 242 | Each version is given a distinguishing version number. If the Program 243 | specifies a version number of this License which applies to it and "any 244 | later version", you have the option of following the terms and conditions 245 | either of that version or of any later version published by the Free 246 | Software Foundation. If the Program does not specify a version number of 247 | this License, you may choose any version ever published by the Free Software 248 | Foundation. 249 | 250 | 10. If you wish to incorporate parts of the Program into other free 251 | programs whose distribution conditions are different, write to the author 252 | to ask for permission. For software which is copyrighted by the Free 253 | Software Foundation, write to the Free Software Foundation; we sometimes 254 | make exceptions for this. Our decision will be guided by the two goals 255 | of preserving the free status of all derivatives of our free software and 256 | of promoting the sharing and reuse of software generally. 257 | 258 | NO WARRANTY 259 | 260 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 268 | REPAIR OR CORRECTION. 269 | 270 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 278 | POSSIBILITY OF SUCH DAMAGES. 279 | 280 | END OF TERMS AND CONDITIONS 281 | -------------------------------------------------------------------------------- /SparseBayes.py: -------------------------------------------------------------------------------- 1 | 2 | # The following is a Python translation of a MATLAB file originally written principally by Mike Tipping 3 | # as part of his SparseBayes software library. Initially published on GitHub on July 21st, 2015. 4 | 5 | # SPARSEBAYES Sparse Bayesian modelling: main estimation algorithm 6 | # 7 | # [PARAMETER, HYPERPARAMETER, DIAGNOSTIC] = ... 8 | # SPARSEBAYES(LIKELIHOOD, BASIS, TARGETS, OPTIONS, SETTINGS) 9 | # 10 | # OUTPUT ARGUMENTS: 11 | # 12 | # PARAMETER Structure specifying inferred primary parameters: 13 | # 14 | # .Value Vector of weight values 15 | # .Relevant Vector of corresponding indices of relevant 16 | # columns of BASIS matrix (sorted ascending) 17 | # 18 | # HYPERPARAMETER Structure specifying inferred hyperparameters: 19 | # 20 | # .Alpha Vector of weight precision values 21 | # .beta Noise precision (Gaussian likelihood case) 22 | # 23 | # DIAGNOSTIC Structure containing various diagnostics: 24 | # 25 | # .Gamma Vector of "well-determined" factors [0,1] for 26 | # relevant weights 27 | # .Likelihood Vector of evolving log-marginal-likelihood 28 | # .iterations Number of iterations run 29 | # .S_Factor Vector of S ("Sparsity") values for relevant weights 30 | # .Q_Factor Vector of Q ("Quality") values for relevant weights 31 | # 32 | # INPUT ARGUMENTS: 33 | # 34 | # LIKELIHOOD String comprising one of 'Gaussian', 'Bernoulli' or 'Poisson' 35 | # 36 | # BASIS NxM matrix of basis vectors (one column per basis function) 37 | # 38 | # TARGETS N-vector with target output values 39 | # 40 | # OPTIONS User-specified settings via SB2_USEROPTIONS [Optional] 41 | # 42 | # SETTINGS Initialisation of main parameter values via 43 | # SB2_PARAMETERSETTINGS [Optional] 44 | # 45 | # NOTES: 46 | # 47 | # SPARSEBAYES is the implementation of the main algorithm for parameter 48 | # inference in "sparse Bayesian" models. 49 | # 50 | # Given inputs (BASIS), desired outputs (TARGETS) and an appropriate 51 | # LIKELIHOOD function, SPARSEBAYES will optimise the log-marginal-likelihood 52 | # of the corresponding sparse Bayesian model and should return (given 53 | # reasonable choice of basis) a sparse vector of model parameters. 54 | # 55 | # OPTIONS and SETTINGS arguments may be omitted, and if so, will assume 56 | # sensible default values. 57 | # 58 | # SEE ALSO: 59 | # 60 | # SB2_USEROPTIONS: available user-definable options, such as the 61 | # number of iterations to run for, level of diagnostic output etc. 62 | # 63 | # SB2_PARAMETERSETTINGS: facility to change default initial values for 64 | # parameters and hyperparameters. 65 | # 66 | # SB2_CONTROLSETTINGS: hard-wired internal algorithm settings. 67 | # 68 | # The main algorithm is based upon that outlined in "Fast marginal 69 | # likelihood maximisation for sparse Bayesian models", by Tipping & Faul, in 70 | # Proceedings of AISTATS 2003. That paper may be downloaded from 71 | # www.relevancevector.com or via the conference online proceedings site at 72 | # http://research.microsoft.com/conferences/aistats2003/proceedings/. 73 | # 74 | 75 | # 76 | # Copyright 2009, Vector Anomaly Ltd 77 | # 78 | # This file is part of the SPARSEBAYES library for Matlab (V2.0). 79 | # 80 | # SPARSEBAYES is free software; you can redistribute it and/or modify it 81 | # under the terms of the GNU General Public License as published by the Free 82 | # Software Foundation; either version 2 of the License, or (at your option) 83 | # any later version. 84 | # 85 | # SPARSEBAYES is distributed in the hope that it will be useful, but WITHOUT 86 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 87 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 88 | # more details. 89 | # 90 | # You should have received a copy of the GNU General Public License along 91 | # with SPARSEBAYES in the accompanying file "licence.txt"; if not, write to 92 | # the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, 93 | # MA 02110-1301 USA 94 | # 95 | # Contact the author: m a i l [at] m i k e t i p p i n g . c o m 96 | # 97 | 98 | 99 | import numpy as np 100 | import math 101 | import time 102 | from SB2_ParameterSettings import SB2_ParameterSettings 103 | from SB2_UserOptions import SB2_UserOptions 104 | from SB2_ControlSettings import SB2_ControlSettings 105 | from SB2_Diagnostic import SB2_Diagnostic 106 | from SB2_Initialisation import SB2_Initialisation 107 | from SB2_FullStatistics import SB2_FullStatistics 108 | from SB2_FormatTime import SB2_FormatTime 109 | 110 | 111 | def SparseBayes(*args): 112 | 113 | likelihood_ = args[0] 114 | BASIS = args[1] 115 | Targets = args[2] 116 | if len(args) > 3: 117 | OPTIONS = args[3] 118 | if len(args) > 4: 119 | SETTINGS = args[4] 120 | 121 | 122 | 123 | ########################################################################### 124 | ## 125 | ## SET UP INITIAL PARAMETERS, USER OPTIONS AND ALGORITHM CONTROL SETTINGS 126 | ## 127 | ########################################################################### 128 | 129 | 130 | ## If no initial parameter setting structure passed, import the defaults 131 | 132 | if 'SETTINGS' not in locals(): 133 | SETTINGS = SB2_ParameterSettings() 134 | 135 | ## If no user options passed, import defaults 136 | 137 | if 'OPTIONS' not in locals(): 138 | OPTIONS = SB2_UserOptions() 139 | 140 | 141 | ## Any sanity checks on options and initialisation here 142 | 143 | # Error if fixed noise specified but value not set 144 | 145 | if OPTIONS['FIXEDNOISE'] and not SETTINGS['BETA'] and not SETTINGS['NOISESTD']: 146 | raise Exception('Option to fix noise variance is set but value is not supplied.') 147 | 148 | 149 | ## Get the default algorithm control settings 150 | 151 | CONTROLS = SB2_ControlSettings() 152 | 153 | # Start the clock now for diagnostic purposes (and to allow the algorithm to 154 | # run for a fixed time) 155 | 156 | t0 = time.time() 157 | 158 | ########################################################################### 159 | 160 | ## 161 | ## INITIALISATION 162 | ## 163 | ## Pre-process basis, set up internal parameters according to initial 164 | ## settings and likelihod specification 165 | ## 166 | 167 | ########################################################################### 168 | 169 | 170 | # Kick off diagnostics (primarily, open log file if specified) 171 | 172 | OPTIONS = SB2_Diagnostic(OPTIONS, 'start') 173 | 174 | # Initialise everything, based on SETTINGS and OPTIONS 175 | 176 | [LIKELIHOOD, BASIS, BasisScales, Alpha, beta, Mu, PHI, Used] = SB2_Initialisation(likelihood_, BASIS, Targets, SETTINGS, OPTIONS) 177 | 178 | # Cache some values for later efficiency 179 | 180 | if LIKELIHOOD['InUse'] == LIKELIHOOD['Gaussian']: 181 | # It will be computationally advantageous to "cache" this quantity 182 | # in the Gaussian case 183 | BASIS_PHI = BASIS.T * PHI 184 | else: 185 | BASIS_PHI = [] 186 | 187 | BASIS_Targets = BASIS.T * Targets 188 | 189 | # FULL COMPUTATION 190 | # 191 | # Initialise with a full explicit computation of the statistics 192 | # 193 | # NOTE: The AISTATS paper uses "S/Q" (upper case) to denote the key 194 | # "sparsity/quality" Factors for "included" basis functions, and "s/q" 195 | # (lower case) for the factors calculated when the relevant basis 196 | # functions are "excluded". 197 | # 198 | # Here, for greater clarity: 199 | # 200 | # All S/Q are denoted by vectors S_in, Q_in 201 | # All s/q are denoted by vectors S_out, Q_out 202 | 203 | [SIGMA,Mu,S_in,Q_in,S_out,Q_out,Factor,logML,Gamma,BASIS_B_PHI,beta] = SB2_FullStatistics(LIKELIHOOD,BASIS,PHI,Targets,Used,Alpha,beta,Mu,BASIS_PHI,BASIS_Targets,OPTIONS) 204 | 205 | # Avoid falling over in pathological case of zero iterations 206 | 207 | if OPTIONS['ITERATIONS'] == 0: 208 | PARAMETER = [] 209 | HYPERPARAMETER = [] 210 | #DIAGNOSTIC['Likelihood'] = logML THIS IS A BUG IN TH ORIGINAL PROGRAM 211 | return 212 | 213 | [N, M_full] = BASIS.shape 214 | M = PHI.shape[1] 215 | 216 | # Some diagnostics 217 | 218 | addCount = 0 219 | deleteCount = 0 220 | updateCount = 0 221 | 222 | # Create storage to record evolution of log marginal likelihood 223 | 224 | maxLogSize = OPTIONS['ITERATIONS'] + CONTROLS['BetaUpdateStart'] + math.ceil(OPTIONS['ITERATIONS'] / float(CONTROLS['BetaUpdateFrequency'])) 225 | logMarginalLog = np.zeros((maxLogSize,1)) 226 | count = 0 227 | 228 | # If we're doing basis alignment testing, we'll need to maintain lists of 229 | # those functions that are near identical, both in and out of the current 230 | # model. 231 | 232 | if CONTROLS['BasisAlignmentTest']: 233 | Aligned_out = [] 234 | Aligned_in = [] 235 | alignDeferCount = 0 236 | 237 | # ACTION CODES 238 | # 239 | # Assign an integer code to the basic action types 240 | 241 | ACTION_REESTIMATE = 0 242 | ACTION_ADD = 1 243 | ACTION_DELETE = -1 244 | 245 | # Some extra types 246 | 247 | ACTION_TERMINATE = 10 248 | ACTION_NOISE_ONLY = 11 249 | ACTION_ALIGNMENT_SKIP = 12 250 | 251 | # Before kicking off the main loop, call the specified "callback" function 252 | # with "ACTION_ADD" to take account of the initialisation of the model 253 | 254 | if OPTIONS['CALLBACK']: 255 | eval(OPTIONS['CALLBACKFUNC'])(0, ACTION_ADD,logML/N, Used, Mu / BasisScales[Used].T, SIGMA, Alpha, beta, Gamma, PHI, OPTIONS['CALLBACKDATA'].flatten()) 256 | 257 | 258 | ########################################################################### 259 | ## 260 | ## MAIN LOOP 261 | ## 262 | ########################################################################### 263 | 264 | i = 0 # Iteration number 265 | LAST_ITERATION = False 266 | 267 | while not LAST_ITERATION: 268 | 269 | i = i+1 270 | 271 | # "UpdateIteration": set to true if this is an iteration where fast matrix 272 | # update rules can be used compute the appropriate quantities 273 | 274 | # This can be done if: 275 | 276 | # - we are using a Gaussian likelihood 277 | # - we are using other likelihoods and we have not specified a full 278 | # posterior mode computation this cycle 279 | 280 | UpdateIteration = LIKELIHOOD['InUse'] == LIKELIHOOD['Gaussian'] or np.remainder(i,CONTROLS['PosteriorModeFrequency']) 281 | 282 | ####################################################################### 283 | 284 | ## DECISION PHASE 285 | 286 | ## Assess all potential actions 287 | 288 | 289 | # Compute change in likelihood for all possible updates 290 | 291 | DeltaML = np.zeros((M_full, 1)) 292 | Action = ACTION_REESTIMATE*np.ones((M_full, 1)) # Default 293 | 294 | # 'Relevance Factor' (Q^S-S) values for basis functions in model 295 | UsedFactor = Factor[Used] 296 | 297 | # RE-ESTIMATION: must be a POSITIVE 'factor' and already IN the model 298 | 299 | iu = np.array(UsedFactor > CONTROLS['ZeroFactor']).flatten() 300 | index = Used[iu] 301 | NewAlpha = np.divide(np.power(S_out[index],2), Factor[index].astype(float)) 302 | Delta = np.divide(float(1), NewAlpha) - np.divide(float(1), Alpha[iu]) # Temp vector 303 | 304 | # Quick computation of change in log-likelihood given all re-estimations 305 | 306 | DeltaML[index] = ( np.divide(np.multiply(Delta, (np.power(Q_in[index], 2))) , ( np.multiply(Delta, S_in[index]) + 1)) - np.log(1 + np.multiply(S_in[index], Delta)) )/2 307 | 308 | # DELETION: if NEGATIVE factor and IN model 309 | 310 | # But don't delete: 311 | # - any "free" basis functions (e.g. the "bias") 312 | # - if there is only one basis function (M=1) 313 | 314 | # (In practice, this latter event ought only to happen with the Gaussian 315 | # likelihood when initial noise is too high. In that case, a later beta 316 | # update should 'cure' this. 317 | 318 | iu = np.logical_not(iu) # iu = UsedFactor <= CONTROLS.ZeroFactor 319 | index = Used[iu] 320 | anyToDelete = ( np.setdiff1d(index, OPTIONS['FREEBASIS'])).size != 0 and M > 1 321 | 322 | if anyToDelete: 323 | # Quick computation of change in log-likelihood given all deletions 324 | DeltaML[index] = -(np.divide(np.power(Q_out[index], 2), (S_out[index] + Alpha[iu])) - np.log(1 + np.divide(S_out[index], Alpha[iu])))/2 325 | Action[index] = ACTION_DELETE 326 | # Note: if M==1, DeltaML will be left as zero, which is fine 327 | 328 | # ADDITION: must be a POSITIVE factor and OUT of the model 329 | 330 | # Find ALL good factors ... 331 | GoodFactor = Factor > CONTROLS['ZeroFactor'] 332 | # ... then mask out those already in model 333 | GoodFactor[Used] = 0 334 | # ... and then mask out any that are aligned with those in the model 335 | if CONTROLS['BasisAlignmentTest']: 336 | GoodFactor[Aligned_out] = 0 337 | index = np.array(GoodFactor).squeeze().nonzero() 338 | anyToAdd = np.size(index) != 0 339 | 340 | if anyToAdd: 341 | # Quick computation of change in log-likelihood given all additions 342 | quot = np.divide(np.power(Q_in[index], 2) , S_in[index]) 343 | DeltaML[index] = (quot - 1 - np.log(quot))/2 344 | Action[index] = ACTION_ADD 345 | 346 | 347 | ####################################################################### 348 | 349 | # Post-process action results to take account of preferences 350 | 351 | # Ensure that nothing happens with "free basis" functions 352 | 353 | DeltaML[OPTIONS['FREEBASIS']] = 0 354 | 355 | # If we prefer ADD or DELETE actions over RE-ESTIMATION 356 | 357 | if (anyToAdd and CONTROLS['PriorityAddition']) or (anyToDelete and CONTROLS['PriorityDeletion']): 358 | # We won't perform re-estimation this iteration, which we achieve by 359 | # zero-ing out the delta 360 | DeltaML[Action == ACTION_REESTIMATE] = 0 361 | 362 | # Furthermore, we should enforce ADD if preferred and DELETE is not 363 | # - and vice-versa 364 | 365 | if anyToAdd and CONTROLS['PriorityAddition'] and not CONTROLS['PriorityDeletion']: 366 | DeltaML[Action == ACTION_DELETE] = 0 367 | 368 | if anyToDelete and CONTROLS['PriorityDeletion'] and not CONTROLS['PriorityAddition']: 369 | DeltaML[Action == ACTION_ADD] = 0 370 | 371 | # Finally...we choose the action that results 372 | # in the greatest change in likelihood 373 | 374 | deltaLogMarginal, nu = DeltaML.max(), DeltaML.argmax() 375 | selectedAction = Action[nu] 376 | anyWorthwhileAction = deltaLogMarginal > 0 377 | 378 | # We need to note if basis nu is already in the model, and if so, 379 | # find its interior index, denoted by "j" 380 | 381 | if selectedAction == ACTION_REESTIMATE or selectedAction == ACTION_DELETE: 382 | j = np.array(Used == nu).squeeze().nonzero()[0] 383 | 384 | # Get the individual basis vector for update and compute its optimal alpha, 385 | # according to equation (20): alpha = S_out^2 / (Q_out^2 - S_out) 386 | 387 | Phi = BASIS[:,nu] 388 | newAlpha = S_out[nu]**2 / float(Factor[nu]) 389 | 390 | ######################################################################### 391 | 392 | # TERMINATION CONDITIONS 393 | # 394 | # Propose to terminate if: 395 | # 396 | # 1. there is no worthwhile (likelihood-increasing) action, OR 397 | # 398 | # 2a. the best action is an ACTION_REESTIMATE but this would only lead to 399 | # an infinitesimal alpha change, AND 400 | # 2b. at the same time there are no potential awaiting deletions 401 | 402 | if not anyWorthwhileAction or (selectedAction == ACTION_REESTIMATE and abs( np.log(newAlpha) - np.log(Alpha[j]) ) < CONTROLS['MinDeltaLogAlpha'] and not anyToDelete): 403 | 404 | selectedAction = ACTION_TERMINATE 405 | act_ = 'potential termination' 406 | 407 | ######################################################################### 408 | 409 | # ALIGNMENT CHECKS 410 | # 411 | # If we're checking "alignment", we may have further processing to do 412 | # on addition and deletion 413 | 414 | if CONTROLS['BasisAlignmentTest']: 415 | 416 | # Addition - rule out addition (from now onwards) if the new basis 417 | # vector is aligned too closely to one or more already in the model 418 | 419 | if selectedAction == ACTION_ADD: 420 | # Basic test for correlated basis vectors 421 | # (note, Phi and columns of PHI are normalised) 422 | 423 | p = Phi.T * PHI 424 | findAligned = np.array(p > CONTROLS['AlignmentMax']).squeeze().nonzero() 425 | numAligned = np.size(findAligned) 426 | 427 | if numAligned > 0: 428 | # The added basis function is effectively indistinguishable from 429 | # one present already 430 | selectedAction = ACTION_ALIGNMENT_SKIP 431 | act_ = 'alignment-deferred addition' 432 | alignDeferCount = alignDeferCount + 1 433 | 434 | # Make a note so we don't try this next time 435 | # May be more than one in the model, which we need to note was 436 | # the cause of function 'nu' being rejected 437 | 438 | Aligned_out = np.vstack([Aligned_out , nu * np.ones((numAligned,1))]) 439 | Aligned_in = np.vstack([Aligned_in , Used[findAligned]]) 440 | 441 | # Deletion: reinstate any previously deferred basis functions 442 | # resulting from this basis function 443 | 444 | if selectedAction == ACTION_DELETE: 445 | findAligned = np.array(Aligned_in == nu).squeeze().nonzero() 446 | numAligned = np.size(findAligned) 447 | 448 | if numAligned > 0: 449 | reinstated = Aligned_out[findAligned] 450 | Aligned_in[findAligned] = [] 451 | Aligned_out[findAligned] = [] 452 | 453 | 454 | r_ = '' 455 | 456 | for i in np.nditer(reinstated): #Mimics MATLAB's sprintf function 457 | if (len(r_) == 0): 458 | r_ = r_ + str(i) 459 | else: 460 | r_ = r_ + ' ' + str(i) 461 | 462 | SB2_Diagnostic(OPTIONS,3,'Alignment reinstatement of %s', r_) 463 | 464 | ######################################################################### 465 | 466 | ## ACTION PHASE 467 | ## 468 | ## Implement above decision 469 | 470 | # We'll want to note if we've made a change which necessitates later 471 | # updating of the statistics 472 | 473 | UPDATE_REQUIRED = False 474 | 475 | if selectedAction == ACTION_REESTIMATE: 476 | 477 | # Basis function 'nu' is already in the model, 478 | # and we're re-estimating its corresponding alpha 479 | # 480 | # - should correspond to Appendix A.3 481 | 482 | oldAlpha = Alpha[j] 483 | Alpha[j] = newAlpha 484 | s_j = SIGMA[:,j] 485 | deltaInv = 1 / float((newAlpha - oldAlpha)) 486 | kappa = 1 / float((SIGMA[j,j] + deltaInv)) 487 | tmp = kappa * s_j 488 | SIGMANEW = SIGMA - tmp * s_j.T 489 | deltaMu = -Mu[j][0,0] * tmp 490 | Mu = Mu + deltaMu 491 | 492 | if UpdateIteration: 493 | S_in = S_in + kappa * np.power((BASIS_B_PHI * s_j), 2) 494 | Q_in = Q_in - (BASIS_B_PHI * deltaMu) 495 | 496 | updateCount = updateCount + 1 497 | act_ = 're-estimation' 498 | 499 | UPDATE_REQUIRED = True 500 | 501 | if selectedAction == ACTION_ADD: 502 | 503 | # Basis function nu is not in the model, and we're adding it in 504 | # 505 | # - should correspond to Appendix A.2 506 | 507 | if LIKELIHOOD['InUse'] == LIKELIHOOD['Gaussian']: 508 | BASIS_Phi = BASIS.T * Phi 509 | BASIS_PHI = np.hstack([BASIS_PHI, BASIS_Phi]) 510 | B_Phi = beta * Phi 511 | BASIS_B_Phi = beta * BASIS_Phi 512 | 513 | else: 514 | B_Phi = np.multiply(Phi, beta) 515 | BASIS_B_phi = BASIS.T * B_Phi 516 | 517 | tmp = ((B_Phi.T * PHI) * SIGMA).T 518 | 519 | Alpha = np.vstack([Alpha , newAlpha]) 520 | PHI = np.hstack([PHI, Phi]) 521 | 522 | s_ii = np.matrix(1 / float((newAlpha + S_in[nu]))) 523 | s_i = -s_ii[0,0] * tmp 524 | TAU = -s_i * tmp.T 525 | SIGMANEW = np.vstack([ np.hstack([SIGMA+TAU, s_i]) , np.hstack([s_i.T, s_ii]) ]) 526 | mu_i = s_ii[0,0] * Q_in[nu] 527 | deltaMu = np.vstack([-mu_i[0,0]*tmp , mu_i]) 528 | Mu = np.vstack([Mu , 0]) + deltaMu 529 | 530 | if UpdateIteration: 531 | mCi = BASIS_B_Phi - BASIS_B_PHI*tmp 532 | S_in = S_in - s_ii[0,0] * np.power(mCi, 2) 533 | Q_in = Q_in - mu_i[0,0] * mCi 534 | 535 | Used = np.hstack([Used, nu]) 536 | addCount = addCount + 1 537 | act_ = 'addition' 538 | 539 | UPDATE_REQUIRED = True 540 | 541 | 542 | if selectedAction == ACTION_DELETE: 543 | 544 | # Basis function nu is in the model, but we're removing it 545 | # 546 | # - should correspond to Appendix A.4 547 | 548 | if LIKELIHOOD['InUse'] == LIKELIHOOD['Gaussian']: 549 | BASIS_PHI = np.delete(BASIS_PHI,j,1) #Deletes jth column 550 | 551 | 552 | PHI = np.delete(PHI,j,1) 553 | Alpha = np.delete(Alpha,j,0) 554 | 555 | s_jj = SIGMA[j,j] 556 | s_j = SIGMA[:,j] 557 | tmp = s_j / float(s_jj) 558 | SIGMANEW = SIGMA - tmp * s_j.T 559 | SIGMANEW = np.delete(SIGMANEW,j,0) 560 | SIGMANEW = np.delete(SIGMANEW,j,1) 561 | deltaMu = -Mu[j][0,0] * tmp 562 | mu_j = Mu[j] 563 | Mu = Mu + deltaMu 564 | Mu = np.delete(Mu,j,0) 565 | 566 | if UpdateIteration: 567 | jPm = BASIS_B_PHI * s_j 568 | S_in = S_in + np.power(jPm, 2) / float(s_jj) 569 | Q_in = Q_in + jPm * mu_j / float(s_jj) 570 | 571 | Used = np.delete(Used,j) 572 | deleteCount = deleteCount + 1 573 | act_ = 'deletion' 574 | 575 | UPDATE_REQUIRED = True 576 | 577 | M = np.size(Used) 578 | 579 | SB2_Diagnostic(OPTIONS, 3, 'ACTION: %s of %d (%g)', act_, nu, deltaLogMarginal) 580 | 581 | 582 | ######################################################################### 583 | 584 | ## UPDATE STATISTICS 585 | 586 | # If we've performed a meaningful action, 587 | # update the relevant variables 588 | 589 | if UPDATE_REQUIRED: 590 | 591 | # S_in & Q_in values were calculated earlier 592 | # 593 | # Here: update the S_out/Q_out values and relevance factors 594 | 595 | if UpdateIteration: 596 | 597 | # Previous "update" statisics calculated earlier are valid 598 | 599 | S_out = np.matrix(S_in) 600 | Q_out = np.matrix(Q_in) 601 | tmp = np.divide(Alpha, (Alpha - S_in[Used])) 602 | S_out[Used] = np.multiply(tmp, S_in[Used]) 603 | Q_out[Used] = np.multiply(tmp, Q_in[Used]) 604 | Factor = np.multiply(Q_out, Q_out) - S_out 605 | SIGMA = SIGMANEW 606 | Gamma = 1 - np.multiply(Alpha, SIGMA.diagonal().T) 607 | 608 | if LIKELIHOOD['InUse'] == LIKELIHOOD['Gaussian']: 609 | BASIS_B_PHI = beta * BASIS_PHI 610 | else: 611 | BASIS_B_PHI = ((np.multiply(PHI , (beta * np.ones((1,M))))).T *BASIS).T 612 | 613 | else: 614 | # Compute all statistics in "full" form (non-Gaussian likelihoods) 615 | 616 | [SIGMA,Mu,S_in,Q_in,S_out,Q_out,Factor,newLogM,Gamma,BASIS_B_PHI,beta] = SB2_FullStatistics(LIKELIHOOD,BASIS,PHI,Targets,Used,Alpha,beta,Mu,BASIS_PHI,BASIS_Targets,OPTIONS) 617 | 618 | deltaLogMarginal = newLogM - logML 619 | 620 | if (UpdateIteration and deltaLogMarginal < 0): 621 | SB2_Diagnostic(OPTIONS, 1, '** Alert ** DECREASE IN LIKELIHOOD !! (%g)', deltaLogMarginal) 622 | 623 | logML = logML + deltaLogMarginal 624 | count = count + 1 625 | logMarginalLog[count - 1] = logML 626 | 627 | 628 | # GAUSSIAN NOISE ESTIMATE 629 | # 630 | # For Gaussian likelihood, re-estimate noise variance if: 631 | # 632 | # - not fixed, AND 633 | # - an update is specified this cycle as normal, OR 634 | # - we're considering termination 635 | 636 | if LIKELIHOOD['InUse'] == LIKELIHOOD['Gaussian'] and not OPTIONS['FIXEDNOISE'] and (selectedAction == ACTION_TERMINATE or i<=CONTROLS['BetaUpdateStart'] or np.remainder(i,CONTROLS['BetaUpdateFrequency'])==0): 637 | 638 | betaZ1 = beta 639 | y = PHI * Mu 640 | e = Targets - y 641 | beta = (N - np.sum(Gamma)) / float(e.T*e) 642 | 643 | # Work-around zero-noise issue 644 | beta = np.min(np.hstack([beta , (CONTROLS['BetaMaxFactor'] / float(np.var(Targets))) ])) 645 | 646 | deltaLogBeta = np.log(beta) - np.log(betaZ1) 647 | 648 | if abs(deltaLogBeta) > CONTROLS['MinDeltaLogBeta']: 649 | 650 | # Full re-computation of statistics required after beta update 651 | 652 | [SIGMA,Mu,S_in,Q_in,S_out,Q_out,Factor,logML,Gamma,BASIS_B_PHI] = SB2_FullStatistics(LIKELIHOOD,BASIS,PHI,Targets,Used,Alpha,beta,Mu,BASIS_PHI,BASIS_Targets,OPTIONS)[0:10] 653 | count = count + 1 654 | logMarginalLog[count - 1] = logML 655 | 656 | if selectedAction == ACTION_TERMINATE: 657 | 658 | # We considered terminating above as no alpha update seemed 659 | # worthwhile. However, a beta update has made a non-trivial 660 | # increase in the likelihood, so we continue. 661 | 662 | selectedAction = ACTION_NOISE_ONLY 663 | 664 | SB2_Diagnostic(OPTIONS,3,'Noise update (termination deferred)') 665 | 666 | 667 | # CALLBACK 668 | # 669 | # Call callback function if specified 670 | # - this can be useful for demos etc where it is desired to display 671 | # graphical information at each iteration 672 | 673 | if OPTIONS['CALLBACK']: 674 | 675 | eval(OPTIONS['CALLBACKFUNC'])(i,selectedAction,logMarginalLog[1:count]/float(N),Used,np.divide(Mu, BasisScales[Used].T),SIGMA,Alpha,beta,Gamma,np.multiply(PHI,(np.ones((N,1))*BasisScales[Used])),OPTIONS['CALLBACKDATA'].flatten()) 676 | 677 | 678 | ######################################################################### 679 | 680 | # END OF CYCLE PROCESSING 681 | # 682 | # Check if termination still specified, and output diagnostics 683 | 684 | if selectedAction == ACTION_TERMINATE: 685 | 686 | # If we're here, then no update to the model was considered worthwhile 687 | 688 | SB2_Diagnostic(OPTIONS,2, '** Stopping at iteration {:d} (Max_delta_ml={:g}) **'.format(i, deltaLogMarginal)) 689 | 690 | if LIKELIHOOD['InUse'] != LIKELIHOOD['Gaussian']: 691 | SB2_Diagnostic(OPTIONS,2,'{:4d}> L = {:.6f}\t Gamma = {:.2f} (M = {:d})'.format( i, logML[0,0]/float(N), np.sum(Gamma), M)) 692 | else: 693 | SB2_Diagnostic(OPTIONS,2, '{:4d}> L = {:.6f}\t Gamma = {:.2f} (M = {:d})\t s={:.3f}'.format(i,logML[0,0]/float(N),np.sum(Gamma),M,np.sqrt(1/float(beta)))) 694 | break 695 | 696 | # Check for "natural" termination 697 | 698 | ITERATION_LIMIT = i == OPTIONS['ITERATIONS'] 699 | TIME_LIMIT = time.time() - t0 > OPTIONS['TIME'] 700 | LAST_ITERATION = ITERATION_LIMIT or TIME_LIMIT 701 | 702 | if ( OPTIONS['MONITOR'] and not np.remainder(i,OPTIONS['MONITOR']) ) or LAST_ITERATION: 703 | 704 | # Output diagnostic progress info if desired 705 | 706 | if LIKELIHOOD['InUse'] != LIKELIHOOD['Gaussian']: 707 | SB2_Diagnostic(OPTIONS,2,'{:5d}> L = {:.6f}\t Gamma = {:.2f} (M = {:d})'.format(i,logML[0,0]/float(N),np.sum(Gamma),M)) 708 | else: 709 | SB2_Diagnostic(OPTIONS,2,'{:5d}> L = {:.6f}\t Gamma = {:.2f} (M = {:d})\t s={:.3f}'.format(i,logML[0,0]/float(N),np.sum(Gamma),M,np.sqrt(1/float(beta)))) 710 | 711 | 712 | ########################################################################### 713 | 714 | ## 715 | ## POST-PROCESSING 716 | ## 717 | 718 | # 719 | # Warn if we exited the main loop without terminating automatically 720 | 721 | if selectedAction != ACTION_TERMINATE: 722 | 723 | if ITERATION_LIMIT: 724 | SB2_Diagnostic(OPTIONS,1,'** Iteration limit: algorithm did not converge') 725 | elif TIME_LIMIT: 726 | SB2_Diagnostic(OPTIONS,1,'** Time limit: algorithm did not converge') 727 | 728 | 729 | 730 | # Output action summary if desired 731 | 732 | if OPTIONS['DIAGNOSTICLEVEL'] > 1: 733 | # Stop timer 734 | t1 = time.time() - t0 735 | total = addCount + deleteCount + updateCount 736 | if CONTROLS['BasisAlignmentTest']: 737 | total = total + alignDeferCount 738 | 739 | SB2_Diagnostic(OPTIONS,2,'Action Summary') 740 | SB2_Diagnostic(OPTIONS,2,'==============') 741 | SB2_Diagnostic(OPTIONS,2,'Added\t\t{:6d} ({:.0f}%)'.format(addCount,100*addCount/float(total))) 742 | SB2_Diagnostic(OPTIONS,2,'Deleted\t\t{:6d} ({:.0f}%)'.format(deleteCount,100*deleteCount/float(total))) 743 | SB2_Diagnostic(OPTIONS,2,'Reestimated\t{:6d} ({:.0f}%)'.format(updateCount, 100*updateCount/float(total))) 744 | 745 | if CONTROLS['BasisAlignmentTest'] and alignDeferCount: 746 | SB2_Diagnostic(OPTIONS,2,'--------------') 747 | SB2_Diagnostic(OPTIONS,2,'Deferred\t{:6d} (%.0f%)'.format(alignDeferCount,100*alignDeferCount/float(total))) 748 | 749 | SB2_Diagnostic(OPTIONS,2,'==============') 750 | SB2_Diagnostic(OPTIONS,2,'Total of {:d} likelihood updates'.format(count)) 751 | SB2_Diagnostic(OPTIONS,2,'Time to run: {:s}'.format(SB2_FormatTime(t1))) 752 | 753 | 754 | # Terminate diagnostics 755 | OPTIONS = SB2_Diagnostic(OPTIONS, 'end') 756 | 757 | ############################################################ 758 | 759 | ## 760 | ## OUTPUT VARIABLES 761 | ## 762 | 763 | PARAMETER = {} 764 | HYPERPARAMETER = {} 765 | DIAGNOSTIC = {} 766 | 767 | # We also choose to sort here - it can't hurt and may help later 768 | 769 | PARAMETER['RELEVANT'], index = np.sort(Used), np.argsort(Used) 770 | 771 | # Not forgetting to correct for normalisation too 772 | 773 | PARAMETER['VALUE'] = np.divide(Mu[index], BasisScales[:,Used[index]].T) 774 | 775 | HYPERPARAMETER['ALPHA'] = np.divide(Alpha[index], (np.power(BasisScales[:,Used[index]].T, 2))) 776 | HYPERPARAMETER['BETA'] = beta 777 | 778 | DIAGNOSTIC['GAMMA'] = Gamma[index] 779 | DIAGNOSTIC['LIKELIHOOD'] = logMarginalLog[0:count] 780 | DIAGNOSTIC['ITERATIONS'] = i 781 | DIAGNOSTIC['S_FACTOR'] = S_out 782 | DIAGNOSTIC['Q_FACTOR'] = Q_out 783 | DIAGNOSTIC['M_FULL'] = M_full 784 | 785 | 786 | return [PARAMETER, HYPERPARAMETER, DIAGNOSTIC] 787 | --------------------------------------------------------------------------------