├── jbopt ├── __init__.py ├── problem.py ├── mn.py ├── de.py ├── mcmc.py ├── classic.py ├── independent.py └── optimize1d.py ├── .gitignore ├── CHANGES.txt ├── doc ├── classic.rst ├── doc.rst ├── index.rst ├── example.rst ├── Makefile └── conf.py ├── MANIFEST.in ├── README.rst ├── setup.py ├── LICENSE.txt └── test ├── test.py ├── test_rosenbrock.py ├── test_all.py └── test_ga.py /jbopt/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | -------------------------------------------------------------------------------- /CHANGES.txt: -------------------------------------------------------------------------------- 1 | v0.1, 2013 -- Initial release. 2 | -------------------------------------------------------------------------------- /doc/classic.rst: -------------------------------------------------------------------------------- 1 | .. automodule:: jbopt.classic 2 | 3 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.txt 2 | recursive-include docs *.txt 3 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Find out about jbopt at ``_! 2 | 3 | -------------------------------------------------------------------------------- /jbopt/problem.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | class Problem(object): 4 | start = None 5 | ndim = None 6 | def transform(self, cube): 7 | params = cube 8 | return params 9 | def likelihood(self, params): 10 | return -params**2 11 | def prior(self, params): 12 | return 0 13 | def viz(self, params): 14 | return 15 | def output(self, params): 16 | print 'candidate:', params 17 | 18 | 19 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from distutils.core import setup 2 | 3 | setup( 4 | name='jbopt', 5 | version='0.2', 6 | author='Johannes Buchner', 7 | author_email='buchner.johannes@gmx.at', 8 | packages=['jbopt'], 9 | scripts=[], 10 | url='http://johannesbuchner.github.io/jbopt/', 11 | license='LICENSE.txt', 12 | description='Parameter space exploration toolbox', 13 | install_requires=[ 14 | "scipy>=0.7.0", 15 | ], 16 | ) 17 | 18 | -------------------------------------------------------------------------------- /doc/doc.rst: -------------------------------------------------------------------------------- 1 | API documentation 2 | =========================== 3 | 4 | .. toctree:: 5 | 6 | .. _opt: 7 | 8 | Heuristic optimization algorithms 9 | ----------------------------------- 10 | 11 | .. automodule:: jbopt.classic 12 | :members: 13 | 14 | .. _de: 15 | 16 | Differential evolution 17 | ----------------------------------- 18 | 19 | .. automodule:: jbopt.de 20 | :members: 21 | 22 | .. _mcmc: 23 | 24 | Markov Chain Monte Carlo methods 25 | ----------------------------------- 26 | 27 | .. automodule:: jbopt.mcmc 28 | :members: 29 | 30 | .. _mn: 31 | 32 | Nested Sampling 33 | ----------------------------------- 34 | 35 | .. automodule:: jbopt.mn 36 | :members: 37 | 38 | .. _1d: 39 | 40 | Simple, custom 1d methods 41 | ----------------------------------- 42 | 43 | 2 custom algorithms are implemented: 44 | 45 | * :func:`jbopt.independent.opt_normalizations` 46 | * :func:`jbopt.optimize1d.optimize` (used by :func:`jbopt.classic.onebyone`) 47 | 48 | .. automodule:: jbopt.optimize1d 49 | :members: optimize 50 | 51 | .. automodule:: jbopt.independent 52 | :members: 53 | 54 | 55 | 56 | 57 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013, Johannes Buchner 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 5 | 6 | Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 7 | Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 8 | 9 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 10 | 11 | -------------------------------------------------------------------------------- /test/test.py: -------------------------------------------------------------------------------- 1 | import scipy 2 | import numpy 3 | 4 | """ 5 | likelihood is multivariate, independent gaussian 6 | optimize each param in turn 7 | """ 8 | 9 | 10 | centers = numpy.array([0.1, 15, 3.3, 4.1, 0]) 11 | sigmas = numpy.array([0.01, 0.1, 3, 10, 10]) 12 | 13 | eval_cache = [] 14 | 15 | def like(params): 16 | eval_cache.append(params) 17 | return (((params - centers) / sigmas)**2).sum() 18 | 19 | from jbopt.independent import * 20 | 21 | limits = numpy.array([(0, 1000)]*len(centers)) 22 | start = numpy.array([0.1]*len(centers)) 23 | 24 | def test_normalizations(): 25 | print 'TEST normalization step method' 26 | print opt_normalizations(start, like, limits, disp=0) #, abandon_threshold=1) 27 | print 'TEST normalization step method: neval:', 28 | print len(eval_cache) 29 | 30 | while len(eval_cache) > 0: 31 | eval_cache.pop() 32 | 33 | def test_grid(): 34 | print 'TEST grid using BRENT' 35 | print opt_grid(start, like, limits, ftol=0.01, disp=0) 36 | print 'TEST grid using BRENT: neval:', 37 | print len(eval_cache) 38 | 39 | def test_grid_parallel(): 40 | print 'TEST grid using BRENT --- parallel' 41 | print opt_grid_parallel(start, like, limits, ftol=0.01, disp=0) 42 | print 'TEST grid using BRENT: neval:', 43 | print len(eval_cache) 44 | 45 | if __name__ == '__main__': 46 | test_grid() 47 | test_grid_parallel() 48 | 49 | -------------------------------------------------------------------------------- /jbopt/mn.py: -------------------------------------------------------------------------------- 1 | """ 2 | MultiNest 3 | """ 4 | 5 | def multinest(parameter_names, transform, loglikelihood, output_basename, **problem): 6 | """ 7 | **MultiNest Nested Sampling** 8 | 9 | via `PyMultiNest `_. 10 | 11 | :param parameter_names: name of parameters; not directly used here, 12 | but for multinest_marginal.py plotting tool. 13 | 14 | """ 15 | import numpy 16 | from numpy import log, exp 17 | import pymultinest 18 | 19 | # n observations 20 | # number of dimensions our problem has 21 | parameters = parameter_names 22 | n_params = len(parameters) 23 | 24 | def myprior(cube, ndim, nparams): 25 | params = transform([cube[i] for i in range(ndim)]) 26 | for i in range(ndim): 27 | cube[i] = params[i] 28 | 29 | def myloglike(cube, ndim, nparams): 30 | l = loglikelihood([cube[i] for i in range(ndim)]) 31 | return l 32 | 33 | # run MultiNest 34 | mn_args = dict( 35 | importance_nested_sampling = False, 36 | outputfiles_basename = output_basename, 37 | resume = problem.get('resume', False), 38 | verbose = True, 39 | n_live_points = problem.get('n_live_points', 400), 40 | const_efficiency_mode = False) 41 | if 'seed' in problem: 42 | mn_args['seed'] = problem['seed'] 43 | pymultinest.run(myloglike, myprior, n_params, **mn_args) 44 | 45 | import json 46 | # store name of parameters, always useful 47 | with file('%sparams.json' % output_basename, 'w') as f: 48 | json.dump(parameters, f, indent=2) 49 | # analyse 50 | a = pymultinest.Analyzer(n_params = n_params, 51 | outputfiles_basename = output_basename) 52 | s = a.get_stats() 53 | with open('%sstats.json' % a.outputfiles_basename, mode='w') as f: 54 | json.dump(s, f, indent=2) 55 | 56 | chain = a.get_equal_weighted_posterior()[:,:-1] 57 | lower = [m['1sigma'][0] for m in s['marginals']] 58 | upper = [m['1sigma'][1] for m in s['marginals']] 59 | stdev = (numpy.array(upper) - numpy.array(lower)) / 2 60 | center = [m['median'] for m in s['marginals']] 61 | 62 | #final = a.get_best_fit()['parameters'] # is already transformed 63 | data = numpy.loadtxt('%slive.points' % output_basename) 64 | i = data[:,-1].argmax() 65 | final = data[i,:-1] # untransformed 66 | 67 | return dict(start=final, chain=chain, 68 | stdev=stdev, upper=upper, lower=lower, 69 | method='MultiNest') 70 | 71 | 72 | 73 | 74 | -------------------------------------------------------------------------------- /test/test_rosenbrock.py: -------------------------------------------------------------------------------- 1 | import scipy 2 | import numpy 3 | 4 | """ 5 | problem is: 6 | 7 | Rosenbrocks valley 8 | 9 | we will compare finding the optimum 10 | """ 11 | 12 | 13 | n_params = 2 14 | def like(params): 15 | x, y = params 16 | f = (1 - x)**2 + 100*(y - x**2)**2 17 | return -0.5 * f 18 | 19 | def transform(cube): 20 | return numpy.asarray(cube) * 10 - 2 21 | 22 | def prior(params): 23 | return 0 24 | 25 | from jbopt.classic import * 26 | from jbopt.mcmc import * 27 | from jbopt.mn import * 28 | from jbopt.de import * 29 | 30 | logfile = open('test_rosen.log', 'w') 31 | def knowledge(ret, loglikelihood, transform, prior, **args): 32 | if 'method' in ret: 33 | logfile.write('### %10s ###\n' % ret['method']) 34 | if 'start' in ret: 35 | params = transform(ret['start']) 36 | logfile.write('# new starting point: %s\n' % params) 37 | logfile.write('# value there : %s\n' % (prior(params) + loglikelihood(params))) 38 | if 'maximum' in ret: 39 | logfile.write('# maximum a post. : %s\n' % ret['maximum']) 40 | if 'median' in ret: 41 | logfile.write('# median estimate : %s\n' % ret['median']) 42 | if 'stdev' in ret: 43 | logfile.write('# std estimate : %s\n' % ret['stdev']) 44 | if 'chain' in ret: 45 | logfile.write('# chain means : %s\n' % ret['chain'].mean(axis=0)) 46 | logfile.write('# chain std : %s\n' % ret['chain'].std(axis=0)) 47 | if 'neval' in ret: 48 | logfile.write('# num. of eval : %s\n' % ret['neval']) 49 | logfile.flush() 50 | return ret 51 | 52 | if __name__ == '__main__': 53 | args = dict( 54 | loglikelihood=like, transform=transform, prior=prior, 55 | parameter_names = ['c%d' % i for i in range(n_params)], 56 | nsteps=2000, 57 | seed = 0, 58 | ) 59 | 60 | for method in 'neldermead', 'cobyla', 'bobyqa', 'ralg', 'mma', 'auglag': 61 | print 'next method:', method 62 | knowledge(classical(method=method, **args), **args) 63 | 64 | ret = knowledge(onebyone(**args), **args) 65 | 66 | knowledge(onebyone(parallel=True, find_uncertainties=True, **args), **args) 67 | 68 | knowledge(de(output_basename='test_rosenbrock_de', **args), **args) 69 | 70 | knowledge(mcmc(output_basename='test_rosenbrock_mcmc', **args), **args) 71 | 72 | knowledge(ensemble(output_basename='test_rosenbrock_mcmc', **args), **args) 73 | 74 | knowledge(multinest(output_basename='test_rosenbrock_mn', **args), **args) 75 | 76 | 77 | 78 | -------------------------------------------------------------------------------- /doc/index.rst: -------------------------------------------------------------------------------- 1 | ======================================= 2 | Welcome to jbopt's documentation! 3 | ======================================= 4 | 5 | About 6 | ------------------------------ 7 | 8 | *jbopt* is a suite of powerful parameter space exploration methods. 9 | 10 | Methods of likelihood maximization, estimation of uncertainty for parameter estimation 11 | in a maximum likelihood and Bayesian way are available. 12 | The problem only has to be stated once, then the various methods can called 13 | interchangably, or in combination, by a common call interface. 14 | 15 | Take a look at the :doc:`instructive code example `. 16 | 17 | Methods 18 | --------- 19 | 20 | 1. Optimization methods 21 | 22 | * :ref:`jbopt.classic` 23 | 24 | * **Nelder-Mead**, **COBYLA** (via `scipy.optimize `_) 25 | * **ralg**, **auglag** and 100s others from the OpenOpt framework (via `openopt.NLP `_) 26 | * **Minuit** (via `PyMinuit `_) 27 | * Custom minimization methods (see :ref:`here<1d>`) 28 | 29 | * :ref:`jbopt.de` 30 | 31 | * **Differential evolution**, specially preconfigured (via `inspyred `_) 32 | 33 | 2. Parameter estimation methods 34 | 35 | * :ref:`jbopt.mcmc` 36 | 37 | * **Metropolis Hastings MCMC** with automatic step width adaption 38 | * **Ensemble MCMC** (via `emcee `_) 39 | 40 | * :ref:`jbopt.mn` 41 | 42 | * **MultiNest Nested Sampling** (via `PyMultiNest `_) 43 | 44 | 3. Integration methods 45 | 46 | * see the `pymultinest/pycuba package `_ 47 | 48 | Documentation 49 | ------------------------------- 50 | 51 | .. toctree:: 52 | example 53 | doc 54 | :maxdepth: -1 55 | 56 | 57 | Installation 58 | ------------------------------- 59 | 60 | #. using pip:: 61 | 62 | $ pip install jbopt # also consider the --user option 63 | 64 | #. Get source using git: *jbopt* is hosted at ``_ 65 | :: 66 | 67 | $ git clone git://github.com/JohannesBuchner/jbopt 68 | $ cd jbopt 69 | $ python setup.py install # also consider the --user option 70 | 71 | Be aware that you will have to install the various dependencies for the 72 | algorithms you would like to use. 73 | 74 | Indices and tables 75 | ------------------------------- 76 | 77 | * :ref:`genindex` 78 | * :ref:`modindex` 79 | * :ref:`search` 80 | 81 | -------------------------------------------------------------------------------- /test/test_all.py: -------------------------------------------------------------------------------- 1 | import scipy 2 | import numpy 3 | 4 | """ 5 | problem is: 6 | 7 | gaussian in 5 dimensions 8 | 9 | problem is: 10 | 11 | x/y data 12 | find k, d, sigma-sys 13 | 14 | """ 15 | 16 | centers = numpy.array([0.1, 15, 3.3, 4.1, 0]) 17 | sigmas = numpy.array([0.01, 0.1, 3, 10, 10]) 18 | 19 | def like(params): 20 | return -0.5 * (((params - centers) / sigmas)**2).sum() 21 | 22 | def transform(cube): 23 | return numpy.asarray(cube) * 100 - 50 24 | 25 | def prior(params): 26 | return 0 27 | 28 | from jbopt.classic import * 29 | from jbopt.mcmc import * 30 | from jbopt.mn import * 31 | from jbopt.de import * 32 | 33 | logfile = open('test_all.log', 'w') 34 | def knowledge(ret, loglikelihood, transform, prior, **args): 35 | if 'method' in ret: 36 | logfile.write('### %10s ###\n' % ret['method']) 37 | if 'start' in ret: 38 | params = transform(ret['start']) 39 | logfile.write('# new starting point: %s\n' % params) 40 | logfile.write('# value there : %s\n' % (prior(params) + loglikelihood(params))) 41 | if 'maximum' in ret: 42 | logfile.write('# maximum a post. : %s\n' % ret['maximum']) 43 | if 'median' in ret: 44 | logfile.write('# median estimate : %s\n' % ret['median']) 45 | if 'stdev' in ret: 46 | logfile.write('# std estimate : %s\n' % ret['stdev']) 47 | if 'chain' in ret: 48 | logfile.write('# chain means : %s\n' % ret['chain'].mean(axis=0)) 49 | logfile.write('# chain std : %s\n' % ret['chain'].std(axis=0)) 50 | if 'neval' in ret: 51 | logfile.write('# num. of eval : %s\n' % ret['neval']) 52 | logfile.flush() 53 | return ret 54 | 55 | if __name__ == '__main__': 56 | args = dict( 57 | loglikelihood=like, transform=transform, prior=prior, 58 | parameter_names = ['c%d' % i for i in range(len(centers))], 59 | nsteps=2000, 60 | seed = 0, 61 | ) 62 | 63 | #knowledge(classical(method='neldermead', **args), **args) 64 | 65 | #knowledge(classical(method='cobyla', **args), **args) 66 | for method in 'neldermead', 'cobyla', 'ralg', 'mma', 'auglag', 'minuit': 67 | print 'next method:', method 68 | knowledge(classical(method=method, **args), **args) 69 | 70 | ret = knowledge(onebyone(**args), **args) 71 | 72 | knowledge(onebyone(parallel=True, find_uncertainties=True, **args), **args) 73 | 74 | knowledge(de(output_basename='test_all_de', **args), **args) 75 | 76 | knowledge(mcmc(output_basename='test_all_mcmc', **args), **args) 77 | 78 | knowledge(ensemble(output_basename='test_all_mcmc', **args), **args) 79 | 80 | knowledge(multinest(output_basename='test_all_mn', **args), **args) 81 | 82 | 83 | 84 | -------------------------------------------------------------------------------- /doc/example.rst: -------------------------------------------------------------------------------- 1 | Example code 2 | ===================== 3 | 4 | First, lets define the problem. 5 | 6 | We will use the **log-likelihood function** of a 5-dimensional gaussian likelihood surface:: 7 | 8 | centers = numpy.array([0.1, 15, 3.3, 4.1, 0]) 9 | sigmas = numpy.array([0.01, 0.1, 3, 10, 10]) 10 | 11 | def loglikelihood(params): 12 | return -0.5 * (((params - centers) / sigmas)**2).sum() 13 | 14 | Next, we define the **parameter space**, in this case between -50 and +50. 15 | This function is also used in Bayesian methods for prior weighting:: 16 | 17 | def transform(cube): 18 | return numpy.asarray(cube) * 100 - 50 19 | 20 | We also define the **prior weighting** explicitly for some of the Bayesian methods; 21 | Other methods will not use it. Again, in this case we consider a uniform prior:: 22 | 23 | def prior(params): 24 | return 0 25 | 26 | Some imports:: 27 | 28 | from jbopt.classic import * 29 | from jbopt.mcmc import * 30 | from jbopt.mn import * 31 | from jbopt.de import * 32 | 33 | All methods have arguments in common, namely the loglikelihood, transform and prior 34 | defined above. Also, the maximum number of steps is obeyed by most algorithms. 35 | So lets define the common arguments for re-use:: 36 | 37 | # the common parameters 38 | args = dict( 39 | loglikelihood=loglikelihood, transform=transform, prior=prior, 40 | parameter_names = ['c%d' % i for i in range(len(centers))], 41 | nsteps=2000, # maximum number of likelihood evaluations 42 | # start = [0.5, 0.3, 0.5, 0.7, 0.9], # set starting point 43 | # seed = 0, # set for reproducible output 44 | # disp = 1, # for more verbose output 45 | ) 46 | 47 | Now we **call the various methods**, starting with the simple Nelder-Mead. 48 | See :doc:`doc` for which methods are available and how to call them:: 49 | 50 | ret = classical(method='neldermead', **args) 51 | 52 | Minuit, COBYLA and some methods through OpenOpt:: 53 | 54 | for method in 'cobyla', 'ralg', 'mma', 'auglag', 'minuit': 55 | print 'next method:', method 56 | ret = classical(method=method, **args) 57 | 58 | A custom algorithm:: 59 | 60 | ret = onebyone(**args) 61 | ret = onebyone(parallel=True, find_uncertainties=True, **args) 62 | 63 | Differential evolution, MCMC, Ensemble MCMC and MultiNest:: 64 | 65 | ret = de(output_basename='test_all_de', **args) 66 | ret = mcmc(output_basename='test_all_mcmc', **args) 67 | ret = ensemble(output_basename='test_all_mcmc', **args) 68 | ret = multinest(output_basename='test_all_mn', **args) 69 | 70 | For each, we can analyse the return value, which gives information about what was 71 | learned about the parameter space. This of course varies between the methods:: 72 | 73 | print ret['method'] # method used 74 | print ret['start'] # new maximum likelihood value 75 | print ret['neval'] # number of evaluations used 76 | # some methods also provide the following 77 | print ret['chain'] # Markov Chain, samples drawn from posterior 78 | print ret['median'] # medians of the parameters 79 | print ret['stdev'] # standard deviation uncertainty of the parameters 80 | 81 | Read more in the :doc:`doc`. 82 | -------------------------------------------------------------------------------- /doc/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 14 | 15 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest 16 | 17 | help: 18 | @echo "Please use \`make ' where is one of" 19 | @echo " html to make standalone HTML files" 20 | @echo " dirhtml to make HTML files named index.html in directories" 21 | @echo " singlehtml to make a single large HTML file" 22 | @echo " pickle to make pickle files" 23 | @echo " json to make JSON files" 24 | @echo " htmlhelp to make HTML files and a HTML help project" 25 | @echo " qthelp to make HTML files and a qthelp project" 26 | @echo " devhelp to make HTML files and a Devhelp project" 27 | @echo " epub to make an epub" 28 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 29 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 30 | @echo " text to make text files" 31 | @echo " man to make manual pages" 32 | @echo " changes to make an overview of all changed/added/deprecated items" 33 | @echo " linkcheck to check all external links for integrity" 34 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 35 | 36 | clean: 37 | -rm -rf $(BUILDDIR)/* 38 | 39 | html: 40 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 41 | @echo 42 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 43 | 44 | dirhtml: 45 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 46 | @echo 47 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 48 | 49 | singlehtml: 50 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 51 | @echo 52 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 53 | 54 | pickle: 55 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 56 | @echo 57 | @echo "Build finished; now you can process the pickle files." 58 | 59 | json: 60 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 61 | @echo 62 | @echo "Build finished; now you can process the JSON files." 63 | 64 | htmlhelp: 65 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 66 | @echo 67 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 68 | ".hhp project file in $(BUILDDIR)/htmlhelp." 69 | 70 | qthelp: 71 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 72 | @echo 73 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 74 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 75 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/pymultinest.qhcp" 76 | @echo "To view the help file:" 77 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/pymultinest.qhc" 78 | 79 | devhelp: 80 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 81 | @echo 82 | @echo "Build finished." 83 | @echo "To view the help file:" 84 | @echo "# mkdir -p $$HOME/.local/share/devhelp/pymultinest" 85 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/pymultinest" 86 | @echo "# devhelp" 87 | 88 | epub: 89 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 90 | @echo 91 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 92 | 93 | latex: 94 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 95 | @echo 96 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 97 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 98 | "(use \`make latexpdf' here to do that automatically)." 99 | 100 | latexpdf: 101 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 102 | @echo "Running LaTeX files through pdflatex..." 103 | make -C $(BUILDDIR)/latex all-pdf 104 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 105 | 106 | text: 107 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 108 | @echo 109 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 110 | 111 | man: 112 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 113 | @echo 114 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 115 | 116 | changes: 117 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 118 | @echo 119 | @echo "The overview file is in $(BUILDDIR)/changes." 120 | 121 | linkcheck: 122 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 123 | @echo 124 | @echo "Link check complete; look for any errors in the above output " \ 125 | "or in $(BUILDDIR)/linkcheck/output.txt." 126 | 127 | doctest: 128 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 129 | @echo "Testing of doctests in the sources finished, look at the " \ 130 | "results in $(BUILDDIR)/doctest/output.txt." 131 | -------------------------------------------------------------------------------- /jbopt/de.py: -------------------------------------------------------------------------------- 1 | """ 2 | Differential evolution 3 | """ 4 | import numpy 5 | 6 | def de(output_basename, parameter_names, transform, loglikelihood, prior, nsteps=40000, vizfunc=None, printfunc=None, **problem): 7 | """ 8 | **Differential evolution** 9 | 10 | via `inspyred `_ 11 | 12 | specially tuned. steady state replacement, n-point crossover, 13 | pop size 20, gaussian mutation noise 0.01 & 1e-6. 14 | stores intermediate results (can be used for resume, see seeds) 15 | 16 | :param start: start point 17 | :param seeds: list of start points 18 | :param vizfunc: callback to do visualization of current best solution 19 | :param printfunc: callback to summarize current best solution 20 | :param seed: RNG initialization (if set) 21 | 22 | """ 23 | import json 24 | import inspyred 25 | import random 26 | prng = random.Random() 27 | if 'seed' in problem: 28 | prng.seed(problem['seed']) 29 | 30 | n_params = len(parameter_names) 31 | seeds = problem.get('seeds', []) 32 | if 'start' in problem: 33 | seeds.append(problem['start']) 34 | prefix = output_basename 35 | 36 | def viz(candidate, args): 37 | if vizfunc is not None: 38 | vizfunc(candidate) 39 | def print_candidate(candidate, l, args): 40 | if printfunc is not None: 41 | printfunc(cube=candidate, loglikelihood=l) 42 | else: 43 | print l, candidate 44 | def eval_candidate(candidate): 45 | params = transform(candidate) 46 | l = loglikelihood(params) 47 | p = prior(params) 48 | if numpy.isinf(p) and p < 0: 49 | print ' prior rejection' 50 | return -1e300 51 | if numpy.isnan(l): 52 | return -1e300 53 | return l, p 54 | @inspyred.ec.utilities.memoize 55 | @inspyred.ec.evaluators.evaluator 56 | def fitness(candidate, args): 57 | l, p = eval_candidate(candidate) 58 | #print_candidate(candidate, (l + p), args) 59 | return (l + p) 60 | 61 | cutoff_store = 10 62 | def solution_archiver(random, population, archive, args): 63 | psize = len(population) 64 | population.sort(reverse=True) 65 | best = population[0].fitness 66 | #print 'BEST: ', best, 67 | all_candidates = sorted(population + archive, reverse=True) 68 | all_fitness = numpy.array([c.fitness for c in all_candidates]) 69 | mask = best - all_fitness > cutoff_store / 3 70 | if mask.sum() < 20: 71 | mask = best - all_fitness > cutoff_store 72 | newarchive = [c for i, c in enumerate(all_candidates) if i == 0 or all_fitness[i - 1] != c.fitness] 73 | print 'ARCHIVE: ', len(archive), len(newarchive) 74 | json.dump([{'candidate': [float(f) for f in c.candidate], 'fitness':c.fitness} for c in newarchive], 75 | open(prefix + '_values.json', 'w'), indent=4) 76 | return newarchive 77 | 78 | def observer(population, num_generations, num_evaluations, args): 79 | population.sort(reverse=True) 80 | candidate = population[0] 81 | print ('{0} evaluations'.format(num_evaluations)), ' best:', 82 | print_candidate(candidate.candidate, candidate.fitness, args) 83 | if num_evaluations % len(population) == 0 or num_evaluations < len(population) or args.get('force_viz', False): 84 | # for each turnaround of a full generation 85 | viz(candidate.candidate, args) 86 | 87 | def generator(random, args): 88 | u = [random.uniform(0, 1) for _ in range(n_params)] 89 | u = [random.gauss(0.5, 0.1) for _ in range(n_params)] 90 | return bounder(u, args) 91 | 92 | ea = inspyred.ec.DEA(prng) 93 | ea.terminator = inspyred.ec.terminators.evaluation_termination 94 | ea.archiver = solution_archiver 95 | bounder = inspyred.ec.Bounder(lower_bound=1e-10, upper_bound=1-1e-10) 96 | #bounder = inspyred.ec.Bounder(lower_bound=-20, upper_bound=20) 97 | import copy 98 | from math import log 99 | @inspyred.ec.variators.mutator 100 | def double_exponential_mutation(random, candidate, args): 101 | mut_rate = args.setdefault('mutation_rate', 0.1) 102 | mean = args.setdefault('gaussian_mean', 0.0) 103 | stdev = args.setdefault('gaussian_stdev', 1.0) 104 | scale = log(0.5) / - (stdev) 105 | bounder = args['_ec'].bounder 106 | mutant = copy.copy(candidate) 107 | for i, m in enumerate(mutant): 108 | dice = random.random() 109 | if dice < mut_rate: 110 | sign = (dice < mut_rate / 2) * 2 - 1 111 | delta = -log(random.random()) / scale 112 | mutant[i] += delta * sign 113 | mutant = bounder(mutant, args) 114 | return mutant 115 | 116 | def minute_gaussian_mutation(random, candidates, args): 117 | args = dict(args) 118 | args['mutation_rate'] = 1 119 | args['gaussian_stdev'] = 1e-6 120 | return inspyred.ec.variators.gaussian_mutation(random, candidates, args) 121 | ea.variator = [inspyred.ec.variators.n_point_crossover, inspyred.ec.variators.gaussian_mutation, minute_gaussian_mutation] 122 | #ea.variator = [inspyred.ec.variators.n_point_crossover, double_exponential_mutation] 123 | 124 | ea.replacer = inspyred.ec.replacers.steady_state_replacement 125 | ea.observer = observer 126 | 127 | pop_size = 20 128 | 129 | final_pop = ea.evolve(pop_size=pop_size, 130 | max_evaluations=nsteps, maximize=True, seeds=seeds, 131 | gaussian_stdev=0.01, #mutation_rate=0.3, 132 | bounder=bounder, generator=generator, evaluator=fitness, 133 | ) 134 | 135 | best = max(final_pop) 136 | seeds = [c.candidate for c in ea.archive] 137 | print 'final candidate:', best 138 | 139 | 140 | return {'start': best.candidate, 'value': best.fitness, 141 | 'seeds': seeds, 'method': 'DE'} 142 | 143 | 144 | -------------------------------------------------------------------------------- /jbopt/mcmc.py: -------------------------------------------------------------------------------- 1 | """ 2 | MCMC 3 | """ 4 | import numpy 5 | 6 | def mcmc_advance(start, stdevs, logp, nsteps = 1e300, adapt=True, callback=None): 7 | """ 8 | Generic Metropolis MCMC. Advances the chain by nsteps. 9 | Called by :func:`mcmc` 10 | 11 | :param adapt: enables adaptive stepwidth alteration (converges). 12 | """ 13 | import scipy 14 | from numpy import log 15 | import progressbar 16 | 17 | prob = logp(start) 18 | chain = [start] 19 | accepts = [True] 20 | probs = [prob] 21 | assert not numpy.isnan(start).any() 22 | assert not numpy.isnan(stdevs).any() 23 | 24 | i = 0 25 | widgets=['AR', progressbar.Percentage(), progressbar.Counter('%5d'), 26 | progressbar.Bar(), progressbar.ETA()] 27 | pbar = progressbar.ProgressBar(widgets=widgets, 28 | maxval=nsteps).start() 29 | 30 | prev = start 31 | prev_prob = prob 32 | print 'MCMC: start at prob', prob 33 | stepchange = 0.1 34 | while len(chain) < nsteps: 35 | i = i + 1 36 | next = scipy.random.normal(prev, stdevs) 37 | next[next > 1] = 1 38 | next[next < 0] = 0 39 | next_prob = logp(next) 40 | assert not numpy.isnan(next).any() 41 | assert not numpy.isnan(next_prob).any() 42 | delta = next_prob - prev_prob 43 | dice = log(scipy.random.uniform(0, 1)) 44 | accept = delta > dice 45 | if accept: 46 | prev = next 47 | prev_prob = next_prob 48 | if adapt: stdevs *= (1 + stepchange) 49 | else: 50 | if adapt: stdevs *= (1 + stepchange)**(-0.4) # aiming for 40% acceptance 51 | if callback: callback(prev_prob, prev, accept) 52 | chain.append(prev) 53 | accepts.append(accept) 54 | probs.append(prev_prob) 55 | if adapt: stepchange = min(0.1, 10. / i) 56 | #print 'STDEV', stdevs[:5], stepchange 57 | 58 | # compute stats 59 | widgets[0] = 'AR: %.03f' % numpy.mean(numpy.array(accepts[len(accepts)/3:])+0) 60 | pbar.update(pbar.currval + 1) 61 | pbar.finish() 62 | 63 | return chain, probs, accepts, stdevs 64 | 65 | def mcmc(transform, loglikelihood, parameter_names, nsteps=40000, nburn=400, 66 | stdevs=0.1, start = 0.5, **problem): 67 | """ 68 | **Metropolis Hastings MCMC** 69 | 70 | with automatic step width adaption. 71 | Burnin period is also used to guess steps. 72 | 73 | :param nburn: number of burnin steps 74 | :param stdevs: step widths to start with 75 | """ 76 | 77 | if 'seed' in problem: 78 | numpy.random.seed(problem['seed']) 79 | n_params = len(parameter_names) 80 | 81 | def like(cube): 82 | cube = numpy.array(cube) 83 | if (cube <= 1e-10).any() or (cube >= 1-1e-10).any(): 84 | return -1e100 85 | params = transform(cube) 86 | return loglikelihood(params) 87 | 88 | start = start + numpy.zeros(n_params) 89 | stdevs = stdevs + numpy.zeros(n_params) 90 | 91 | def compute_stepwidths(chain): 92 | return numpy.std(chain, axis=0) / 3 93 | 94 | import matplotlib.pyplot as plt 95 | plt.figure(figsize=(7, 7)) 96 | steps = numpy.array([0.1]*(n_params)) 97 | print 'burn-in (1/2)...' 98 | chain, prob, _, steps_ = mcmc_advance(start, steps, like, nsteps=nburn / 2, adapt=True) 99 | steps = compute_stepwidths(chain) 100 | print 'burn-in (2/2)...' 101 | chain, prob, _, steps_ = mcmc_advance(chain[-1], steps, like, nsteps=nburn / 2, adapt=True) 102 | steps = compute_stepwidths(chain) 103 | print 'recording chain ...' 104 | chain, prob, _, steps_ = mcmc_advance(chain[-1], steps, like, nsteps=nsteps) 105 | chain = numpy.array(chain) 106 | 107 | i = numpy.argmax(prob) 108 | final = chain[-1] 109 | print 'postprocessing...' 110 | chain = numpy.array([transform(params) for params in chain]) 111 | 112 | return dict(start=chain[-1], maximum=chain[i], seeds=[final, chain[i]], chain=chain, method='Metropolis MCMC') 113 | 114 | def ensemble(transform, loglikelihood, parameter_names, nsteps=40000, nburn=400, 115 | start=0.5, **problem): 116 | """ 117 | **Ensemble MCMC** 118 | 119 | via `emcee `_ 120 | """ 121 | import emcee 122 | import progressbar 123 | if 'seed' in problem: 124 | numpy.random.seed(problem['seed']) 125 | n_params = len(parameter_names) 126 | nwalkers = 50 + n_params * 2 127 | if nwalkers > 200: 128 | nwalkers = 200 129 | p0 = [numpy.random.rand(n_params) for i in xrange(nwalkers)] 130 | start = start + numpy.zeros(n_params) 131 | p0[0] = start 132 | 133 | def like(cube): 134 | cube = numpy.array(cube) 135 | if (cube <= 1e-10).any() or (cube >= 1-1e-10).any(): 136 | return -1e100 137 | params = transform(cube) 138 | return loglikelihood(params) 139 | 140 | sampler = emcee.EnsembleSampler(nwalkers, n_params, like, 141 | live_dangerously=True) 142 | 143 | print 'burn-in...' 144 | pos, prob, state = sampler.run_mcmc(p0, nburn / nwalkers) 145 | 146 | # Reset the chain to remove the burn-in samples. 147 | sampler.reset() 148 | 149 | print 'running ...' 150 | # Starting from the final position in the burn-in chain, sample 151 | pbar = progressbar.ProgressBar( 152 | widgets=[progressbar.Percentage(), progressbar.Counter('%5d'), 153 | progressbar.Bar(), progressbar.ETA()], 154 | maxval=nsteps).start() 155 | for results in sampler.sample(pos, iterations=nsteps / nwalkers, rstate0=state): 156 | pbar.update(pbar.currval + 1) 157 | pbar.finish() 158 | 159 | print "Mean acceptance fraction:", numpy.mean(sampler.acceptance_fraction) 160 | 161 | chain = sampler.flatchain 162 | 163 | final = chain[-1] 164 | print 'postprocessing...' 165 | chain_post = numpy.array([transform(params) for params in chain]) 166 | chain_prob = sampler.flatlnprobability 167 | 168 | return dict(start=final, chain=chain_post, chain_prior=chain, 169 | chain_prob=chain_prob, 170 | method='Ensemble MCMC') 171 | 172 | 173 | -------------------------------------------------------------------------------- /jbopt/classic.py: -------------------------------------------------------------------------------- 1 | """ 2 | Classic optimization methods 3 | """ 4 | import numpy 5 | from independent import opt_grid_parallel, opt_grid 6 | 7 | def classical(transform, loglikelihood, parameter_names, prior, 8 | start = 0.5, ftol=0.1, disp=0, nsteps=40000, 9 | method='neldermead', **args): 10 | """ 11 | **Classic optimization methods** 12 | 13 | :param start: start position vector (before transform) 14 | :param ftol: accuracy required to stop at optimum 15 | :param disp: verbosity 16 | :param nsteps: number of steps 17 | :param method: string 18 | neldermead, cobyla (via `scipy.optimize `_) 19 | bobyqa, ralg, algencan, ipopt, mma, auglag and many others from the OpenOpt framework (via `openopt.NLP `_) 20 | minuit (via `PyMinuit `_) 21 | """ 22 | import scipy.optimize 23 | 24 | n_params = len(parameter_names) 25 | 26 | def minfunc(params): 27 | l = loglikelihood(params) 28 | p = prior(params) 29 | if numpy.isinf(p) and p < 0: 30 | print ' prior rejection' 31 | return -1e300 32 | if numpy.isnan(l): 33 | return -1e300 34 | return -l - p 35 | def minfunc_cube(cube): 36 | cube = numpy.array(cube) 37 | if (cube <= 1e-10).any() or (cube >= 1-1e-10).any(): 38 | return 1e100 39 | params = transform(cube) 40 | l = loglikelihood(params) 41 | p = prior(params) 42 | if numpy.isinf(p) and p < 0: 43 | print ' prior rejection' 44 | return -1e300 45 | if numpy.isnan(l): 46 | return -1e300 47 | return -l - p 48 | 49 | start = start + numpy.zeros(n_params) 50 | ret = {} 51 | if method == 'neldermead': 52 | final, value, _niter, neval, warnflag = scipy.optimize.fmin(minfunc_cube, start, ftol=ftol, disp=disp, maxfun=nsteps, full_output=True) 53 | elif method == 'cobyla': 54 | cons = [lambda params: params[i] for i in range(n_params)] 55 | cons += [lambda params: 1 - params[i] for i in range(n_params)] 56 | final = scipy.optimize.fmin_cobyla(minfunc_cube, start, cons, rhoend=ftol / 10, disp=disp, maxfun=nsteps) 57 | neval = nsteps 58 | elif method == 'minuit' or method == 'hesse': 59 | """ 60 | We use eval here, and it is a dangerous thing to do. 61 | But Minuit probes the objective function for parameter names, 62 | and there is no way to create the objective function 63 | dynamically with an unknown number of parameters other than 64 | through eval. 65 | """ 66 | s = ', '.join(parameter_names) 67 | s = """lambda %s: minfunc([%s])""" % (s, s) 68 | if method == 'hesse': 69 | f = eval(s, dict(minfunc=minfunc, numpy=numpy)) 70 | start = transform(start) 71 | else: 72 | f = eval(s, dict(minfunc=minfunc_cube, numpy=numpy)) 73 | import minuit 74 | m = minuit.Minuit(f) 75 | for i, p in enumerate(parameter_names): 76 | m.values[p] = start[i] 77 | if method == 'minuit': 78 | m.limits[p] = (1e-10, 1 - 1e-10) 79 | m.up = 0.5 80 | m.tol = ftol * 100 81 | m.printMode = disp 82 | if method == 'minuit': 83 | m.migrad() 84 | elif method == 'hesse': 85 | m.hesse() 86 | final = [m.values[p] for p in parameter_names] 87 | neval = m.ncalls 88 | errors = [m.errors[p] for p in parameter_names] 89 | 90 | if method == 'minuit': 91 | c0 = final 92 | p0 = transform(c0) 93 | stdev = numpy.zeros(n_params) 94 | lower = numpy.zeros(n_params) 95 | upper = numpy.zeros(n_params) 96 | for i, w in enumerate(errors): 97 | c1 = numpy.copy(c0) 98 | c1[i] -= w 99 | c2 = numpy.copy(c0) 100 | c2[i] += w 101 | p1 = transform(c1) 102 | p2 = transform(c2) 103 | stdev[i] = numpy.abs(p2[i] - p1[i]) / 2 104 | lower[i] = min(p2[i], p1[i]) 105 | upper[i] = max(p2[i], p1[i]) 106 | ret['stdev'] = stdev 107 | ret['upper'] = upper 108 | ret['lower'] = lower 109 | elif method == 'hesse': 110 | ret['stdev'] = errors 111 | ret['cov'] = numpy.matrix([[m.covariance[(a, b)] for b in parameter_names] for a in parameter_names]) 112 | 113 | else: 114 | from openopt import NLP 115 | lo = [1e-10] * n_params 116 | hi = [1-1e-10] * n_params 117 | iprint = 0 if disp == 0 else 10 if disp == 1 else 1 118 | p = NLP(f=minfunc_cube, x0=start, lb=lo, ub=hi, 119 | maxFunEvals=nsteps, ftol=ftol, iprint=iprint) 120 | r = p.solve(method) 121 | final = r.xf 122 | neval = r.evals['f'] 123 | 124 | ret.update(dict(start=final, maximum=transform(final), method=method, neval=neval)) 125 | return ret 126 | 127 | 128 | def onebyone(transform, loglikelihood, parameter_names, prior, start = 0.5, ftol=0.1, disp=0, nsteps=40000, 129 | parallel=False, find_uncertainties=False, **args): 130 | """ 131 | **Convex optimization based on Brent's method** 132 | 133 | A strict assumption of one optimum between the parameter limits is used. 134 | The bounds are narrowed until it is found, i.e. the likelihood function is flat 135 | within the bounds. 136 | * If optimum outside bracket, expands bracket until contained. 137 | * Thus guaranteed to return local optimum. 138 | * Supports parallelization (multiple parameters are treated independently) 139 | * Supports finding ML uncertainties (Delta-Chi^2=1) 140 | 141 | Very useful for 1-3d problems. 142 | Otherwise useful, reproducible/deterministic algorithm for finding the minimum in 143 | well-behaved likelihoods, where the parameters are weakly independent, 144 | or to find a good starting point. 145 | Optimizes each parameter in order, assuming they are largely independent. 146 | 147 | For 1-dimensional algorithm used, see :func:`jbopt.opt_grid` 148 | 149 | :param ftol: difference in values at which the function can be considered flat 150 | :param compute_errors: compute standard deviation of gaussian around optimum 151 | """ 152 | 153 | def minfunc(cube): 154 | cube = numpy.array(cube) 155 | if (cube <= 1e-10).any() or (cube >= 1-1e-10).any(): 156 | return 1e100 157 | params = transform(cube) 158 | l = loglikelihood(params) 159 | p = prior(params) 160 | if numpy.isinf(p) and p < 0: 161 | print ' prior rejection' 162 | return -1e300 163 | if numpy.isnan(l): 164 | return -1e300 165 | return -l - p 166 | 167 | if parallel: 168 | func = opt_grid_parallel 169 | else: 170 | func = opt_grid 171 | 172 | n_params = len(parameter_names) 173 | start = start + numpy.zeros(n_params) 174 | ret = func(start, minfunc, [(1e-10, 1-1e-10)] * n_params, ftol=ftol, disp=disp, compute_errors=find_uncertainties) 175 | 176 | if find_uncertainties: 177 | c0 = ret[0] 178 | p0 = transform(c0) 179 | stdev = numpy.zeros(n_params) 180 | lower = numpy.zeros(n_params) 181 | upper = numpy.zeros(n_params) 182 | for i, (lo, hi) in enumerate(ret[1]): 183 | c1 = numpy.copy(c0) 184 | c1[i] = lo 185 | c2 = numpy.copy(c0) 186 | c2[i] = hi 187 | p1 = transform(c1) 188 | p2 = transform(c2) 189 | stdev[i] = numpy.abs(p2[i] - p1[i]) / 2 190 | lower[i] = min(p2[i], p1[i]) 191 | upper[i] = max(p2[i], p1[i]) 192 | return dict(start=ret[0], maximum=p0, 193 | stdev=stdev, upper=upper, lower=lower, 194 | method='opt_grid') 195 | else: 196 | return dict(start=ret, maximum=transform(ret), method='opt_grid') 197 | 198 | 199 | 200 | 201 | 202 | 203 | -------------------------------------------------------------------------------- /jbopt/independent.py: -------------------------------------------------------------------------------- 1 | """ 2 | Custom minimization algorithms in jbopt 3 | 4 | minimization routines that assume the parameters to be mostly independent from 5 | each other, optimizing each parameter in turn. 6 | """ 7 | 8 | import numpy 9 | 10 | def opt_normalizations(params, func, limits, abandon_threshold=100, noimprovement_threshold=1e-3, 11 | disp=0): 12 | """ 13 | **optimization algorithm for scale variables (positive value of unknown magnitude)** 14 | 15 | Each parameter is a normalization of a feature, and its value is sought. 16 | The parameters are handled in order (assumed to be independent), 17 | but a second round can be run. 18 | Various magnitudes of the normalization are tried. If the normalization converges 19 | to zero, the largest value yielding a comparable value is used. 20 | 21 | Optimizes each normalization parameter in rough steps 22 | using multiples of 3 of start point 23 | to find reasonable starting values for another algorithm. 24 | 25 | parameters, minimization function, parameter space definition [(lo, hi) for i in params] 26 | 27 | :param abandon_threshold: 28 | if in one direction the function increases by this much over the best value, 29 | abort search in this direction 30 | :param noimprovement_threshold: 31 | when decreasing the normalization, if the function increases by less than 32 | this amount, abort search in this direction 33 | :param disp: 34 | verbosity 35 | """ 36 | newparams = numpy.copy(params) 37 | lower = [lo for lo, hi in limits] 38 | upper = [hi for lo, hi in limits] 39 | for i, p in enumerate(params): 40 | startval = p 41 | beststat = func(newparams) 42 | bestval = startval 43 | if disp > 0: print '\t\tstart val = %e: %e' % (startval, beststat) 44 | go_up = True 45 | go_down = True 46 | # go up and down in multiples of 3 47 | # once that is done, refine in multiples of 1.1 48 | for n in list(3.**numpy.arange(1, 20)) + [None] + list(1.1**numpy.arange(1, 13)): 49 | if n is None: 50 | startval = bestval 51 | if disp > 0: print '\t\trefining from %e' % (startval) 52 | go_up = True 53 | go_down = True 54 | continue 55 | if go_up and startval * n > upper[i]: 56 | if disp > 0: print '\t\thit upper border (%e * %e > %e)' % (startval, n, upper[i]) 57 | go_up = False 58 | if go_down and startval / n < lower[i]: 59 | if disp > 0: print '\t\thit lower border (%e / %e > %e)' % (startval, n, lower[i]) 60 | go_down = False 61 | if go_up: 62 | if disp > 1: print '\t\ttrying %e ^' % (startval * n) 63 | newparams[i] = startval * n 64 | newstat = func(newparams) 65 | if disp > 1: print '\t\tval = %e: %e' % (newparams[i], newstat) 66 | if newstat <= beststat: 67 | bestval = newparams[i] 68 | beststat = newstat 69 | if disp > 0: print '\t\t\timprovement: %e' % newparams[i] 70 | if newstat > beststat + abandon_threshold: 71 | go_up = False 72 | if go_down: 73 | if disp > 1: print '\t\ttrying %e v' % (startval / n) 74 | newparams[i] = startval / n 75 | newstat = func(newparams) 76 | if disp > 1: print '\t\tval = %e: %e' % (newparams[i], newstat) 77 | if newstat + noimprovement_threshold < beststat: # avoid zeros in normalizations 78 | bestval = newparams[i] 79 | beststat = newstat 80 | if disp > 0: print '\t\t\timprovement: %e' % newparams[i] 81 | if newstat > beststat + abandon_threshold: 82 | go_down = False 83 | newparams[i] = bestval 84 | print '\tnew normalization of %d: %e' % (i, newparams[i]) 85 | print 'optimization done, reached %.3f' % (beststat) 86 | return newparams 87 | 88 | from optimize1d import * 89 | 90 | def opt_grid(params, func, limits, ftol=0.01, disp=0, compute_errors=True): 91 | """ 92 | see :func:`optimize1d.optimize`, considers each parameter in order 93 | 94 | :param ftol: 95 | difference in values at which the function can be considered flat 96 | :param compute_errors: 97 | compute standard deviation of gaussian around optimum 98 | """ 99 | caches = [[] for p in params] 100 | newparams = numpy.copy(params) 101 | errors = [[] for p in params] 102 | for i, p in enumerate(params): 103 | cache = [] 104 | def func1(x0): 105 | newparams[i] = x0 106 | v = func(newparams) 107 | cache.append([x0, v]) 108 | return v 109 | lo, hi = limits[i] 110 | bestval = optimize(func1, x0=p, 111 | cons=[lambda x: x - lo, lambda x: hi - x], 112 | ftol=ftol, disp=disp - 1) 113 | beststat = func1(bestval) 114 | if compute_errors: 115 | errors[i] = cache2errors(func1, cache, disp=disp - 1) 116 | 117 | newparams[i] = bestval 118 | caches[i] = cache 119 | if disp > 0: 120 | if compute_errors: 121 | print '\tnew value of %d: %e [%e .. %e] yielded %e' % (i, bestval, errors[i][0], errors[i][1], beststat) 122 | else: 123 | print '\tnew value of %d: %e yielded %e' % (i, bestval, beststat) 124 | beststat = func(newparams) 125 | if disp > 0: 126 | print 'optimization done, reached %.3f' % (beststat) 127 | 128 | if compute_errors: 129 | return newparams, errors 130 | else: 131 | return newparams 132 | 133 | def opt_grid_parallel(params, func, limits, ftol=0.01, disp=0, compute_errors=True): 134 | """ 135 | parallelized version of :func:`opt_grid` 136 | """ 137 | 138 | import multiprocessing 139 | 140 | def spawn(f): 141 | def fun(q_in,q_out): 142 | while True: 143 | i,x = q_in.get() 144 | if i == None: 145 | break 146 | q_out.put((i,f(x))) 147 | return fun 148 | 149 | def parmap(f, X, nprocs = multiprocessing.cpu_count()): 150 | q_in = multiprocessing.Queue(1) 151 | q_out = multiprocessing.Queue() 152 | 153 | proc = [multiprocessing.Process(target=spawn(f),args=(q_in,q_out)) for _ in range(nprocs)] 154 | for p in proc: 155 | p.daemon = True 156 | p.start() 157 | 158 | sent = [q_in.put((i,x)) for i,x in enumerate(X)] 159 | [q_in.put((None,None)) for _ in range(nprocs)] 160 | res = [q_out.get() for _ in range(len(sent))] 161 | 162 | [p.join() for p in proc] 163 | 164 | return [x for i,x in sorted(res)] 165 | 166 | nthreads = multiprocessing.cpu_count() 167 | 168 | caches = [[] for p in params] 169 | newparams = numpy.copy(params) 170 | errors = [[] for p in params] 171 | indices = range(0, len(params), nthreads) 172 | k = 0 173 | while k < len(params): 174 | j = min(len(params), k + nthreads * 2) 175 | def run1d((i, curparams, curlimits)): 176 | cache = [] 177 | def func1(x0): 178 | curparams[i] = x0 179 | v = func(curparams) 180 | cache.append([x0, v]) 181 | return v 182 | lo, hi = curlimits 183 | bestval = optimize(func1, x0=p, 184 | cons=[lambda x: x - lo, lambda x: hi - x], 185 | ftol=ftol, disp=disp - 1) 186 | beststat = func1(bestval) 187 | if compute_errors: 188 | errors = cache2errors(func1, cache, disp=disp - 1) 189 | return bestval, beststat, errors, cache 190 | return bestval, beststat, cache 191 | results = parmap(run1d, [(i, numpy.copy(newparams), limits[i]) for i in range(k, j)]) 192 | for i, r in enumerate(results): 193 | if compute_errors: 194 | v, s, e, c = r 195 | if disp > 0: 196 | print '\tnew value of %d: %e [%e .. %e] yielded %e' % (i + k, v, e[0], e[1], s) 197 | else: 198 | v, s, c = r 199 | e = [] 200 | if disp > 0: 201 | print '\tnew value of %d: %e yielded %e' % (i + k, v, s) 202 | newparams[i + k] = v 203 | caches[i + k] = c 204 | errors[i + k] = e 205 | 206 | k = j 207 | beststat = func(newparams) 208 | if disp > 0: 209 | print 'optimization done, reached %e' % (beststat) 210 | 211 | if compute_errors: 212 | return newparams, errors 213 | else: 214 | return newparams 215 | 216 | 217 | 218 | 219 | -------------------------------------------------------------------------------- /doc/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # pymultinest documentation build configuration file, created by 4 | # sphinx-quickstart on Thu Jan 5 04:13:31 2012. 5 | # 6 | # This file is execfile()d with the current directory set to its containing dir. 7 | # 8 | # Note that not all possible configuration values are present in this 9 | # autogenerated file. 10 | # 11 | # All configuration values have a default; values that are commented out 12 | # serve to show the default. 13 | 14 | from __future__ import absolute_import, unicode_literals, print_function 15 | import sys, os 16 | 17 | # If extensions (or modules to document with autodoc) are in another directory, 18 | # add these directories to sys.path here. If the directory is relative to the 19 | # documentation root, use os.path.abspath to make it absolute, like shown here. 20 | sys.path.insert(0, os.path.abspath('..')) 21 | 22 | # -- General configuration ----------------------------------------------------- 23 | 24 | # If your documentation needs a minimal Sphinx version, state it here. 25 | #needs_sphinx = '1.0' 26 | 27 | # Add any Sphinx extension module names here, as strings. They can be extensions 28 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 29 | extensions = ['sphinx.ext.autodoc', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode'] 30 | 31 | # Add any paths that contain templates here, relative to this directory. 32 | templates_path = ['templates'] 33 | 34 | # The suffix of source filenames. 35 | source_suffix = '.rst' 36 | 37 | # The encoding of source files. 38 | #source_encoding = 'utf-8-sig' 39 | 40 | # The master toctree document. 41 | master_doc = 'index' 42 | 43 | # General information about the project. 44 | project = 'jbopt' 45 | copyright = '2013, Johannes Buchner' 46 | 47 | # The version info for the project you're documenting, acts as replacement for 48 | # |version| and |release|, also used in various other places throughout the 49 | # built documents. 50 | # 51 | # The short X.Y version. 52 | version = '0.1' 53 | # The full version, including alpha/beta/rc tags. 54 | release = '0.1' 55 | 56 | # The language for content autogenerated by Sphinx. Refer to documentation 57 | # for a list of supported languages. 58 | #language = None 59 | 60 | # There are two options for replacing |today|: either, you set today to some 61 | # non-false value, then it is used: 62 | #today = '' 63 | # Else, today_fmt is used as the format for a strftime call. 64 | #today_fmt = '%B %d, %Y' 65 | 66 | # List of patterns, relative to source directory, that match files and 67 | # directories to ignore when looking for source files. 68 | exclude_patterns = ['_build'] 69 | 70 | # The reST default role (used for this markup: `text`) to use for all documents. 71 | #default_role = None 72 | 73 | # If true, '()' will be appended to :func: etc. cross-reference text. 74 | #add_function_parentheses = True 75 | 76 | # If true, the current module name will be prepended to all description 77 | # unit titles (such as .. function::). 78 | #add_module_names = True 79 | 80 | # If true, sectionauthor and moduleauthor directives will be shown in the 81 | # output. They are ignored by default. 82 | #show_authors = False 83 | 84 | # The name of the Pygments (syntax highlighting) style to use. 85 | pygments_style = 'sphinx' 86 | 87 | # A list of ignored prefixes for module index sorting. 88 | #modindex_common_prefix = [] 89 | 90 | 91 | # -- Options for HTML output --------------------------------------------------- 92 | 93 | # The theme to use for HTML and HTML Help pages. See the documentation for 94 | # a list of builtin themes. 95 | html_theme = 'nature' 96 | 97 | # Theme options are theme-specific and customize the look and feel of a theme 98 | # further. For a list of options available for each theme, see the 99 | # documentation. 100 | #html_theme_options = {} 101 | 102 | # Add any paths that contain custom themes here, relative to this directory. 103 | #html_theme_path = [] 104 | 105 | # The name for this set of Sphinx documents. If None, it defaults to 106 | # " v documentation". 107 | #html_title = None 108 | 109 | # A shorter title for the navigation bar. Default is the same as html_title. 110 | #html_short_title = None 111 | 112 | # The name of an image file (relative to this directory) to place at the top 113 | # of the sidebar. 114 | #html_logo = None 115 | 116 | # The name of an image file (within the static path) to use as favicon of the 117 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 118 | # pixels large. 119 | #html_favicon = None 120 | 121 | # Add any paths that contain custom static files (such as style sheets) here, 122 | # relative to this directory. They are copied after the builtin static files, 123 | # so a file named "default.css" will overwrite the builtin "default.css". 124 | html_static_path = ['static'] 125 | 126 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 127 | # using the given strftime format. 128 | #html_last_updated_fmt = '%b %d, %Y' 129 | 130 | # If true, SmartyPants will be used to convert quotes and dashes to 131 | # typographically correct entities. 132 | #html_use_smartypants = True 133 | 134 | # Custom sidebar templates, maps document names to template names. 135 | #html_sidebars = {} 136 | 137 | # Additional templates that should be rendered to pages, maps page names to 138 | # template names. 139 | #html_additional_pages = {} 140 | 141 | # If false, no module index is generated. 142 | #html_domain_indices = True 143 | 144 | # If false, no index is generated. 145 | #html_use_index = True 146 | 147 | # If true, the index is split into individual pages for each letter. 148 | #html_split_index = False 149 | 150 | # If true, links to the reST sources are added to the pages. 151 | #html_show_sourcelink = True 152 | 153 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 154 | #html_show_sphinx = True 155 | 156 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 157 | #html_show_copyright = True 158 | 159 | # If true, an OpenSearch description file will be output, and all pages will 160 | # contain a tag referring to it. The value of this option must be the 161 | # base URL from which the finished HTML is served. 162 | #html_use_opensearch = '' 163 | 164 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 165 | #html_file_suffix = None 166 | 167 | # Output file base name for HTML help builder. 168 | htmlhelp_basename = 'jboptdoc' 169 | 170 | 171 | # -- Options for LaTeX output -------------------------------------------------- 172 | 173 | # The paper size ('letter' or 'a4'). 174 | #latex_paper_size = 'letter' 175 | 176 | # The font size ('10pt', '11pt' or '12pt'). 177 | #latex_font_size = '10pt' 178 | 179 | # Grouping the document tree into LaTeX files. List of tuples 180 | # (source start file, target name, title, author, documentclass [howto/manual]). 181 | latex_documents = [ 182 | ('index', 'jbopt.tex', 'jbopt Documentation', 183 | 'Johannes Buchner', 'manual'), 184 | ] 185 | 186 | # The name of an image file (relative to this directory) to place at the top of 187 | # the title page. 188 | #latex_logo = None 189 | 190 | # For "manual" documents, if this is true, then toplevel headings are parts, 191 | # not chapters. 192 | #latex_use_parts = False 193 | 194 | # If true, show page references after internal links. 195 | #latex_show_pagerefs = False 196 | 197 | # If true, show URL addresses after external links. 198 | #latex_show_urls = False 199 | 200 | # Additional stuff for the LaTeX preamble. 201 | #latex_preamble = '' 202 | 203 | # Documents to append as an appendix to all manuals. 204 | #latex_appendices = [] 205 | 206 | # If false, no module index is generated. 207 | #latex_domain_indices = True 208 | 209 | 210 | # -- Options for manual page output -------------------------------------------- 211 | 212 | # One entry per manual page. List of tuples 213 | # (source start file, name, description, authors, manual section). 214 | man_pages = [ 215 | ('index', 'jbopt', 'jbopt Documentation', 216 | ['Johannes Buchner'], 1) 217 | ] 218 | -------------------------------------------------------------------------------- /test/test_ga.py: -------------------------------------------------------------------------------- 1 | import os 2 | import copy 3 | import inspyred 4 | import random 5 | import numpy 6 | import json 7 | from joblib import delayed, Parallel, Memory 8 | cachedir = '.cache' 9 | if not os.path.isdir(cachedir): os.mkdir(cachedir) 10 | mem = Memory(cachedir=cachedir, verbose=False) 11 | 12 | from numpy import log, exp, log10, loadtxt, array, linspace, pi, sin, cos 13 | prng = random.Random() 14 | prng.seed(0) 15 | 16 | nvar = 500 17 | centers = numpy.array([prng.gauss(0.5, 0.1) for _ in range(nvar)]) 18 | widths = numpy.array([(prng.gauss(0, 0.5) + 0.01)**2 for _ in range(nvar)]) 19 | widths = numpy.array([0.01, 0.05, 0.001, 0.02, 0.1] * (nvar / 5)) 20 | #widths = numpy.array([0.1 for _ in range(nvar)]) 21 | 22 | def like(params): 23 | return -0.5 * (((numpy.asarray(params) - centers) / widths)**2).sum() 24 | 25 | def print_candidate(candidate, l, args): 26 | params = numpy.asarray(candidate) 27 | print params[nvar / 2], params.mean(), params.std(), l 28 | 29 | def eval_candidate(candidate): 30 | l = like(candidate) 31 | if numpy.isnan(l): 32 | return -1e300 33 | return l 34 | 35 | from jbopt.independent import opt_grid 36 | def func(par): 37 | params = [par[0]] * nvar 38 | print 'flat func:', par, '-'*20, 39 | l = eval_candidate(params) 40 | print 'likelihood', ' '*30, l 41 | return -l 42 | 43 | val = opt_grid([0.5], func, [(-20, 20)], ftol=1, disp=0, compute_errors=False) 44 | l = -func(val) 45 | startcandidate = [float(val) for _ in range(nvar)] 46 | seeds = [startcandidate] 47 | json.dump([{'candidate': startcandidate, 'fitness':l}], 48 | open('test_ga_values.json', 'w'), indent=4) 49 | 50 | 51 | 52 | @inspyred.ec.utilities.memoize 53 | @inspyred.ec.evaluators.evaluator 54 | def fitness(candidate, args): 55 | l = eval_candidate(candidate) 56 | #print_candidate(candidate, l, args) 57 | return l 58 | 59 | cutoff_store = 10 60 | def solution_archiver(random, population, archive, args): 61 | psize = len(population) 62 | population.sort(reverse=True) 63 | best = population[0].fitness 64 | print 'BEST: ', best, 'cutoff', cutoff_store 65 | newarchive = [] 66 | for c in population + archive: 67 | if c.fitness > best - cutoff_store and c not in newarchive: 68 | newarchive.append(c) 69 | print 'ARCHIVE: ', len(archive), len(newarchive) 70 | json.dump([{'candidate': [float(f) for f in c.candidate], 'fitness':c.fitness} for c in newarchive], 71 | open('test_ga_values.json', 'w'), indent=4) 72 | return newarchive 73 | 74 | def observer(population, num_generations, num_evaluations, args): 75 | population.sort(reverse=True) 76 | candidate = population[0] 77 | if num_evaluations % 1000 == 0: 78 | print ('{0} evaluations'.format(num_evaluations)), ' best:', 79 | print_candidate(candidate.candidate, candidate.fitness, args) 80 | 81 | 82 | bounder = inspyred.ec.Bounder(lower_bound=-20, upper_bound=20) 83 | def generator(random, args): 84 | center = random.normalvariate(0.5, 0.1) 85 | candidate = [random.uniform(0, 1) + center for _ in range(nvar)] 86 | return bounder(candidate, args) 87 | 88 | import sys 89 | eval_log = sys.stdout 90 | 91 | neval = 10000 92 | 93 | @mem.cache 94 | def run_variants(choices): 95 | eval_log.write('run %s\n' % choices) 96 | choices = copy.copy(choices)[::-1] 97 | if choices.pop(): 98 | eval_log.write('\tDEA\n') 99 | ea = inspyred.ec.DEA(prng) 100 | else: 101 | eval_log.write('\tGA\n') 102 | ea = inspyred.ec.GA(prng) 103 | ea.terminator = inspyred.ec.terminators.evaluation_termination 104 | if choices.pop(): 105 | eval_log.write('\tvariators: npoint\n') 106 | variators = [inspyred.ec.variators.n_point_crossover, inspyred.ec.variators.gaussian_mutation] 107 | else: 108 | eval_log.write('\tvariators: heuristic\n') 109 | variators = [inspyred.ec.variators.heuristic_crossover, inspyred.ec.variators.gaussian_mutation] 110 | #ea.archiver = solution_archiver 111 | if choices.pop(): 112 | eval_log.write('\treplacements: steady_state\n') 113 | ea.replacer = inspyred.ec.replacers.steady_state_replacement 114 | else: 115 | eval_log.write('\treplacements: generational\n') 116 | ea.replacer = inspyred.ec.replacers.generational_replacement 117 | ea.observer = observer 118 | 119 | pop_size = 4 120 | if choices.pop(): 121 | pop_size = 20 122 | if choices.pop(): 123 | pop_size = 100 124 | elif choices.pop(): 125 | pop_size = 50 126 | eval_log.write('\tpop-size: %d\n' % pop_size) 127 | 128 | args = dict(pop_size=pop_size, 129 | bounder=bounder, generator=generator, evaluator=fitness, 130 | max_evaluations=neval, maximize=True, seeds=seeds) 131 | if choices.pop(): 132 | args['mutation_rate'] = 0.3 133 | eval_log.write('\t extra: mutation_rate: 0.3\n') 134 | if choices.pop(): 135 | args['gaussian_stdev'] = 0.01 136 | eval_log.write('\t extra: gaussian_stdev: 0.01\n') 137 | if choices.pop(): 138 | args['num_crossover_points'] = nvar / 10 139 | eval_log.write('\t extra: num_crossover_points: %d\n' % (nvar / 10)) 140 | if choices.pop(): 141 | args['num_elites'] = 1 142 | eval_log.write('\t extra: num_elites: 1\n') 143 | eval_log.flush() 144 | prng.seed(0) 145 | final_pop = ea.evolve(**args) 146 | best = max(final_pop) 147 | return best 148 | 149 | def evaluate_best(candidate, fitness): 150 | #print 'final candidate:', candidate 151 | 152 | #print 'distance:', candidate 153 | print 'distance:', ((numpy.array(candidate) - centers)**2).mean() 154 | print 'final likelihood:', fitness 155 | eval_log.write('\tresult: likelihood: %.1f, distance: %.3f\n' % ( 156 | fitness, ((numpy.array(candidate) - centers)**2).mean())) 157 | return fitness 158 | 159 | def run_classical(): 160 | import scipy.optimize 161 | 162 | def func(params): 163 | return -eval_candidate(params) 164 | 165 | """print 'fmin ...' 166 | result = scipy.optimize.fmin(func, candidate, maxfun=10000, retall=True) 167 | print 'fmin', result 168 | eval_log.write('fmin\n') 169 | evaluate_best(result[0], result[1])""" 170 | 171 | print 'cobyla ...' 172 | result = scipy.optimize.fmin_cobyla(func, candidate, cons=[], maxfun=neval) 173 | print 'cobyla', result 174 | eval_log.write('cobyla\n') 175 | evaluate_best(result, func(result)) 176 | 177 | def run_mcmc(): 178 | import emcee 179 | nwalkers = 2*nvar + 2 180 | sampler = emcee.EnsembleSampler(nwalkers, nvar, eval_candidate) 181 | p0 = startcandidate 182 | pos, prob, state = sampler.run_mcmc(p0, neval) 183 | print "Mean acceptance fraction:", numpy.mean(sampler.acceptance_fraction) 184 | print 'pos', pos 185 | print 'prob', prob 186 | 187 | def run_indep(): 188 | candidate = copy.copy(startcandidate) 189 | global neval_count 190 | neval_count = 0 191 | 192 | def func(params): 193 | l = eval_candidate(params) 194 | global neval_count 195 | neval_count = neval_count + 1 196 | return -l 197 | 198 | print 'optgrid' 199 | candidate = opt_grid(startcandidate, func, [(-20, 20)] * nvar, 200 | ftol=1, disp=0, compute_errors=False) 201 | l = -func(candidate) 202 | print 'Best after %d evaluations' % neval_count, 203 | print_candidate(candidate, l, {}) 204 | evaluate_best(candidate, l) 205 | 206 | if __name__ == '__main__': 207 | import sys 208 | eval_log = open('test_ga%s.log' % sys.argv[1], 'w') 209 | if sys.argv[1] == 'mcmc': 210 | run_mcmc() 211 | elif sys.argv[1] == 'classical': 212 | run_classical() 213 | elif sys.argv[1] == 'indep': 214 | run_indep() 215 | elif sys.argv[1] == 'ga': 216 | nchoices = 9 217 | 218 | prev_choices = [False for _ in range(nchoices)] 219 | best = run_variants(prev_choices) 220 | last_val = evaluate_best(best.candidate, best.fitness) 221 | import scipy 222 | 223 | while True: 224 | flipbits = [scipy.random.poisson(1./nchoices) for i in range(nchoices)] 225 | if sum(flipbits) == 0: 226 | continue 227 | choices = [not c if f else c for c, f in zip(prev_choices, flipbits)] 228 | #i = random.choice(range(nchoices)) 229 | #choices[i] = not choices[i] 230 | eval_log.write('flipping %s\n' % (flipbits)) 231 | best = run_variants(choices) 232 | val = evaluate_best(best.candidate, best.fitness) 233 | if val > last_val: 234 | eval_log.write('GOOD CHOICE! %d=%s; now at %f\n' %(i, choices[i], val)) 235 | last_val = val 236 | prev_choices = choices 237 | else: # go back 238 | #choices[i] = not choices[i] 239 | choices = prev_choices 240 | eval_log.flush() 241 | 242 | 243 | 244 | -------------------------------------------------------------------------------- /jbopt/optimize1d.py: -------------------------------------------------------------------------------- 1 | """ 2 | 1D optimization, which should be robust against local flatness in the target function, 3 | but also estimate errors on the final parameter. 4 | 5 | We know that there is only one minimum, and it is far from the constraints. 6 | """ 7 | 8 | import scipy.optimize, scipy.interpolate 9 | import numpy 10 | import matplotlib.pyplot as plt 11 | 12 | def pause(): 13 | import sys 14 | print 'please press enter: >> ', 15 | sys.stdin.readline() 16 | 17 | def plot_values(values, points, lastpoint, ymax=numpy.nan, ftol=0.05): 18 | #print 'values:', zip(points, values) 19 | #print 'last value', points[lastpoint], values[lastpoint] 20 | plt.figure() 21 | plt.plot(points, values, 'o--', color='blue') 22 | plt.plot(points[lastpoint], values[lastpoint], 'o', color='red') 23 | if numpy.isnan(ymax): 24 | worst = [v for v in values if v < 1e100] 25 | ymax = max(worst) 26 | plt.ylim(min(values)-ftol, ymax+ftol) 27 | plt.savefig('optimize1d.pdf') 28 | plt.close() 29 | 30 | phi = (1 + 5**0.5) / 2 31 | resphi = 2 - phi 32 | 33 | """ 34 | Is this a line within the tolerance --> return False 35 | """ 36 | def has_curvature(a, b, c, va, vb, vc, ftol, disp): 37 | ftol = 10000 * ftol 38 | grad = (vc - va) / (c - a) 39 | vbpred = grad * (b - a) + va 40 | curvcrit = numpy.abs(vbpred - vb) > ftol 41 | if disp > 0: print '\tCurvature checking: %f (tol=%f): ' % (vbpred - vb, ftol), curvcrit 42 | return curvcrit 43 | 44 | def escalate_left(function, a, b, va, vb, cons, lstep, ftol, disp, plot): 45 | assert va < vb, (va, vb) 46 | while va < vb: # we have to go left 47 | if disp > 0: print ' <<< fast forwarding to LEFT <<< ' 48 | b, c = a, b 49 | vb, vc = va, vb 50 | lstep += 1 51 | while any([con(b - lstep) < 0 for con in cons]): 52 | lstep /= 3 53 | a = b - lstep 54 | va = function(a) 55 | if disp > 5: 56 | if plot: 57 | plot_values([va, vb, vc], [a, b, c], lastpoint=0, ftol=ftol) 58 | if disp > 0: print ' left %f [%f]' % (a, va) 59 | if va > vb or not has_curvature(a, b, c, va, vb, vc, ftol, disp): # finally, we found the border 60 | if disp > 0: print ' found left border' 61 | return [a, b, c], [va, vb, vc] 62 | if lstep < 1e-4 and c - a < 1e-4 and numpy.abs(vb - va) < 1e-4: 63 | if disp > 0: print ' WARNING: hit the lower limit of the parameter', lstep, a, b, va, vb 64 | return [a, b, c], [va, vb, vc] 65 | return [a, b, c], [va, vb, vc] 66 | def escalate_right(function, b, c, vb, vc, cons, rstep, ftol, disp, plot): 67 | assert vc < vb, (vc, vb) 68 | while vc < vb: # we have to go right 69 | if disp > 0: print ' >>> fast forwarding to RIGHT >>> ' 70 | a, b = b, c 71 | va, vb = vb, vc 72 | rstep += 1 73 | while any([con(b + rstep) < 0 for con in cons]): 74 | rstep /= 3 75 | c = b + rstep 76 | vc = function(c) 77 | if disp > 5: 78 | if plot: 79 | plot_values([va, vb, vc], [a, b, c], lastpoint=2, ftol=ftol) 80 | if disp > 0: print ' right %f [%f]' % (c, vc) 81 | if vc > vb: # finally, we found the border 82 | if disp > 0: print ' found right border' 83 | return [a, b, c], [va, vb, vc] 84 | if rstep < 1e-4 and c - a < 1e-4 and numpy.abs(vc - vb) < 1e-4: 85 | if disp > 0: print ' WARNING: hit the upper limit of the parameter', rstep, b, c, vb, vc 86 | return [a, b, c], [va, vb, vc] 87 | return [a, b, c], [va, vb, vc] 88 | 89 | def seek_minimum_bracket(function, b, cons, ftol, disp, plot): 90 | # we want to bracket the minimum first 91 | lstep = 0.7 92 | rstep = 0.7 93 | 94 | assert not any([c(b) < 0 for c in cons]), [b] 95 | vb = function(b) 96 | if disp > 0: print 'starting at %f [%f]' % (b, vb) 97 | while any([c(b - lstep) < 0 for c in cons]): 98 | lstep /= 3 99 | if disp > 0: print 'reducing lstep for constraint' 100 | a = b - lstep 101 | va = function(a) 102 | if disp > 0: print 'left %f [%f]' % (a, va) 103 | #plot([va, vb], [a, b], lastpoint=0, ftol=ftol) 104 | 105 | if va <= vb: # we have to go left 106 | return escalate_left(function, a, b, va, vb, cons, lstep, ftol, disp, plot) 107 | 108 | while any([c(b + rstep) < 0 for c in cons]): 109 | rstep /= 3 110 | if disp > 0: print 'reducing rstep for constraint' 111 | c = b + rstep 112 | vc = function(c) 113 | #plot([va, vb, vc], [a, b, c], lastpoint=2, ftol=ftol) 114 | if disp > 0: print 'right %f [%f]' % (c, vc) 115 | 116 | if vc <= vb: # we have to go right 117 | return escalate_right(function, b, c, vb, vc, cons, rstep, ftol, disp, plot) 118 | return [a, b, c], [va, vb, vc] 119 | 120 | def brent(function, a, b, c, va, vb, vc, cons, ftol, disp=0, plot=False): 121 | while True: 122 | if disp > 0: print ' BRENT', a, b, c, va, vb, vc 123 | if vb <= va and vb <= vc: 124 | if numpy.abs(vb - va) + numpy.abs(vb - vc) <= ftol: 125 | if disp > 0: print ' ===> found minimum at %f, %f' % (b, vb) 126 | return b 127 | if numpy.abs(vb - va) + numpy.abs(vb - vc) <= ftol: 128 | print 'Potentially problematic case. Increasing verbosity!' 129 | print ' Narrowing to ftol:', numpy.abs(vb - va) + numpy.abs(vb - vc) 130 | disp = 4 131 | x = b - 0.5 * ((b - a)**2 * (vb - vc) - (b - c)**2*(vb - va)) / ((b - a) * (vb - vc) - (b - c) * (vb - va)) 132 | if disp > 0: print 'suggested point:', x 133 | safety = 10. 134 | if x < b and x > a and c - b >= (b - a) * safety: 135 | if disp > 0: print 'we want to go left, but right side is getting too mighty' 136 | x = (c + (safety - 1) * b) / safety 137 | if x < c and x > b and b - a >= (c - b) * safety: 138 | if disp > 0: print 'we want to go right, but left side is getting too mighty' 139 | x = ((safety - 1) * b + a) / safety 140 | 141 | safety2 = 10. 142 | if x <= b: 143 | if x - a <= numpy.abs(b - a) / safety2: 144 | if disp > 0: print 'we want to go left, but are too close to left side' 145 | x = a + (b - a) / safety2 146 | if b - x <= (b - a) / safety2: 147 | if disp > 0: print 'we want to go left, but are too close to the center' 148 | if (b - a) * numpy.abs(va - vb) >= (c - b) * numpy.abs(vc - vb) * safety**2: 149 | if disp > 0: print 'left side is very mighty' 150 | x = (b + a) / 2. 151 | else: 152 | x = b - (b - a) / safety2 153 | if x >= b: 154 | if c - x <= numpy.abs(c - b) / safety2: 155 | if disp > 0: print 'we want to go right, but are too close to right side' 156 | x = c - (c - b) / safety2 157 | if x - b <= (c - b) / safety2: 158 | if disp > 0: print 'we want to go right, but are too close to the center' 159 | if (c - b) * numpy.abs(vc - vb) >= (b - a) * numpy.abs(va - vb) * safety**2: 160 | if disp > 0: print 'right side is very mighty' 161 | x = (b + c) / 2. 162 | else: 163 | x = b + (c - b) / safety2 164 | 165 | if va < vb: # should go left 166 | if x > a: 167 | if disp > 0: print 'I think we should go further left, to bracket the minimum' 168 | (a, b, c), (va, vb, vc) = escalate_left(function, a, b, va, vb, cons=cons, lstep=c - a, ftol=ftol, disp=disp, plot=plot) 169 | if numpy.abs(vb - va) + numpy.abs(vb - vc) <= ftol: 170 | if disp > 0: print ' ===> found minimum at left border %f, %f' % (b, vb) 171 | return b 172 | #disp = 4 173 | continue 174 | x = a - (c - b) 175 | 176 | if vc < vb: # should go right 177 | if x < c: 178 | if disp > 0: print 'I think we should go further right, to bracket the minimum' 179 | (a, b, c), (va, vb, vc) = escalate_right(function, b, c, vb, vc, cons=cons, rstep=c - a, ftol=ftol, disp=disp, plot=plot) 180 | if numpy.abs(vb - va) + numpy.abs(vb - vc) <= ftol: 181 | if disp > 0: print ' ===> found minimum at right border %f, %f' % (b, vb) 182 | return b 183 | #disp = 4 184 | continue 185 | x = c + (b - a) 186 | 187 | 188 | if disp > 0: print 'next point:', x 189 | v = function(x) 190 | if disp > 0: print 'next value:', v 191 | if disp > 3: 192 | if plot: 193 | plot_values([va, vb, vc, v], [a, b, c, x], lastpoint=3, ftol=ftol) 194 | pause() 195 | 196 | if disp > 0: print ' deciding on next bracket' 197 | if v < min(va, vb, vc): 198 | # improvement was made. 199 | if x < a: # go to very left 200 | if disp > 0: print ' <<<< ' 201 | a, b, c, va, vb, vc = x, a, b, v, va, vb 202 | continue 203 | elif x < b: # go to left 204 | if disp > 0: print ' << ' 205 | a, b, c, va, vb, vc = a, x, b, va, v, vb 206 | continue 207 | elif x > c: # go to very right 208 | if disp > 0: print ' >>>> ' 209 | a, b, c, va, vb, vc = b, c, x, vb, vc, v 210 | continue 211 | else: # go to right 212 | if disp > 0: print ' >> ' 213 | a, b, c, va, vb, vc = b, x, c, vb, v, vc 214 | continue 215 | # no improvement 216 | if disp > 0: print ' no improvement made' 217 | # did we try to move to the outer edges? 218 | if va < vb and x < a: 219 | # we tried to go very left, but hit the wall 220 | if disp > 0: print ' |<< ' 221 | a, b, c, va, vb, vc = x, a, b, v, va, vb 222 | continue 223 | elif vc < vb and x > c: 224 | # we tried to go very right, but hit the wall 225 | if disp > 0: print ' >>| ' 226 | a, b, c, va, vb, vc = b, c, x, vb, vc, v 227 | continue 228 | 229 | if disp > 0: print ' subdividing side' 230 | # go to the other side 231 | if not (v < va or v < vc): 232 | if plot: 233 | plot_values([va, vb, vc, v], [a, b, c, x], lastpoint=3, ftol=ftol) 234 | if disp > 0: print 'warning: found flat bit!' 235 | return b 236 | assert False, [v < va, v, va, v < vc, v, vc] 237 | 238 | if x < b: # on the left, go to right side 239 | if v > va and v < vb: 240 | if disp > 0: print ' . | x | sequence, going left' 241 | a, b, c, va, vb, vc = a, x, b, va, v, vb 242 | elif v > va: 243 | if plot: 244 | plot_values([va, vb, vc, v], [a, b, c, x], lastpoint=3, ftol=ftol) 245 | disp = 4 246 | if disp > 0: print 'warning: found flat bit on the right!' 247 | return b 248 | else: 249 | if disp > 0: print ' . | x | going right' 250 | a, b, c, va, vb, vc = x, b, c, v, vb, vc 251 | continue 252 | else: # on the right, go to left side 253 | if v > vc and v < vb: 254 | if disp > 0: print ' . | x | sequence, going right' 255 | a, b, c, va, vb, vc = b, x, c, vb, v, vc 256 | elif v > vc: 257 | if plot: 258 | plot_values([va, vb, vc, v], [a, b, c, x], lastpoint=3, ftol=ftol) 259 | disp = 4 260 | if disp > 0: print 'warning: found flat bit on the left!' 261 | return b 262 | else: 263 | if disp > 0: print ' | x | . going left' 264 | a, b, c, va, vb, vc = a, b, x, va, vb, v 265 | continue 266 | assert False, [a, b, c, x, va, vb, vc, v] 267 | 268 | neval = 0 269 | 270 | def optimize(function, x0, cons=[], ftol=0.2, disp=0, plot=False): 271 | """ 272 | **Optimization method based on Brent's method** 273 | 274 | First, a bracket (a b c) is sought that contains the minimum (b value is 275 | smaller than both a or c). 276 | 277 | The bracket is then recursively halfed. Here we apply some modifications 278 | to ensure our suggested point is not too close to either a or c, 279 | because that could be problematic with the local approximation. 280 | Also, if the bracket does not seem to include the minimum, 281 | it is expanded generously in the right direction until it covers it. 282 | 283 | Thus, this function is fail safe, and will always find a local minimum. 284 | """ 285 | if disp > 0: 286 | print 287 | print ' ===== custom 1d optimization routine ==== ' 288 | print 289 | print 'initial suggestion on', function, ':', x0 290 | points = [] 291 | values = [] 292 | def recordfunction(x): 293 | v = function(x) 294 | points.append(x) 295 | values.append(v) 296 | return v 297 | (a, b, c), (va, vb, vc) = seek_minimum_bracket(recordfunction, x0, cons=cons, ftol=ftol, disp=disp, plot=plot) 298 | if disp > 0: 299 | print '---------------------------------------------------' 300 | print 'found useable minimum bracker after %d evaluations:' % len(points), (a, b, c), (va, vb, vc) 301 | if disp > 2: 302 | if plot: 303 | plot_values(values, points, lastpoint=-1, ftol=ftol) 304 | pause() 305 | 306 | result = brent(recordfunction, a, b, c, va, vb, vc, cons=cons, ftol=ftol, disp=disp, plot=plot) 307 | if disp > 0: 308 | print '---------------------------------------------------' 309 | print 'found minimum after %d evaluations:' % len(points), result 310 | if disp > 1 or len(points) > 20: 311 | if plot: 312 | plot_values(values, points, lastpoint=-1, ftol=ftol) 313 | if disp > 2: 314 | pause() 315 | if disp > 0: 316 | print '---------------------------------------------------' 317 | print 318 | print ' ===== end of custom 1d optimization routine ==== ' 319 | print 320 | global neval 321 | neval += len(points) 322 | return result 323 | 324 | def cache2errors(function, cache, disp=0, ftol=0.05): 325 | """ 326 | This function will attempt to identify 1 sigma errors, assuming your 327 | function is a chi^2. For this, the 1-sigma is bracketed. 328 | 329 | If you were smart enough to build a cache list of [x,y] into your function, 330 | you can pass it here. The values bracketing 1 sigma will be used as 331 | starting values. 332 | If no such values exist, e.g. because all values were very close to the 333 | optimum (good starting values), the bracket is expanded. 334 | """ 335 | 336 | vals = numpy.array(sorted(cache, key=lambda x: x[0])) 337 | if disp > 0: print ' --- cache2errors --- ', vals 338 | vi = vals[:,1].min() 339 | def renormedfunc(x): 340 | y = function(x) 341 | cache.append([x, y]) 342 | if disp > 1: print ' renormed:', x, y, y - (vi + 1) 343 | return y - (vi + 1) 344 | vals[:,1] -= vi + 1 345 | lowmask = vals[:,1] < 0 346 | highmask = vals[:,1] > 0 347 | indices = numpy.arange(len(vals)) 348 | b, vb = vals[indices[lowmask][ 0],:] 349 | c, vc = vals[indices[lowmask][-1],:] 350 | if any(vals[:,0][highmask] < b): 351 | if disp > 0: print 'already have bracket' 352 | a, va = vals[indices[highmask][vals[:,0][highmask] < b][-1],:] 353 | else: 354 | a = b 355 | va = vb 356 | while b > -50: 357 | a = b - max(vals[-1,0] - vals[0,0], 1) 358 | va = renormedfunc(a) 359 | if disp > 0: print 'going further left: %.1f [%.1f] --> %.1f [%.1f]' % (b, vb, a, va) 360 | if va > 0: 361 | if disp > 0: print 'found outer part' 362 | break 363 | else: 364 | # need to go further 365 | b = a 366 | vb = va 367 | 368 | if disp > 0: print 'left bracket', a, b, va, vb 369 | if va > 0 and vb < 0: 370 | leftroot = scipy.optimize.brentq(renormedfunc, a, b, rtol=ftol) 371 | else: 372 | if disp > 0: print 'WARNING: border problem found.' 373 | leftroot = a 374 | if disp > 0: print 'left root', leftroot 375 | 376 | if any(vals[:,0][highmask] > c): 377 | if disp > 0: print 'already have bracket' 378 | d, vd = vals[indices[highmask][vals[:,0][highmask] > c][ 0],:] 379 | else: 380 | d = c 381 | vd = vc 382 | while c < 50: 383 | d = c + max(vals[-1,0] - vals[0,0], 1) 384 | vd = renormedfunc(d) 385 | if disp > 0: print 'going further right: %.1f [%.1f] --> %.1f [%.1f]' % (c, vc, d, vd) 386 | if vd > 0: 387 | if disp > 0: print 'found outer part' 388 | break 389 | else: 390 | # need to go further 391 | c = d 392 | vc = vd 393 | if disp > 0: print 'right bracket', c, d, vc, vd 394 | if vd > 0 and vc < 0: 395 | rightroot = scipy.optimize.brentq(renormedfunc, c, d, rtol=ftol) 396 | else: 397 | if disp > 0: print 'WARNING: border problem found.' 398 | rightroot = d 399 | if disp > 0: print 'right root', rightroot 400 | 401 | assert leftroot < rightroot 402 | 403 | if disp > 2: 404 | fullvals = numpy.array(sorted(cache, key=lambda x: x[0])) 405 | fullvals[:,1] -= vi + 1 406 | plt.figure() 407 | plt.plot(fullvals[:,0], fullvals[:,1], 's') 408 | plt.plot(vals[:,0], vals[:,1], 'o') 409 | plt.xlim(a, d) 410 | plt.ylim(min(va, vb, vc, vd), max(va, vb, vc, vd)) 411 | ymin, ymax = plt.ylim() 412 | plt.vlines([leftroot, rightroot], ymin, ymax, linestyles='dotted') 413 | plt.savefig('cache_brent.pdf') 414 | 415 | return leftroot, rightroot 416 | 417 | 418 | 419 | 420 | 421 | --------------------------------------------------------------------------------