├── .gitignore ├── LICENSE.txt ├── MANIFEST ├── README.md ├── doc ├── reference │ ├── Makefile │ ├── README │ └── source │ │ ├── .templates │ │ ├── index.html │ │ ├── indexsidebar.html │ │ └── layout.html │ │ ├── api.rst │ │ ├── conf.py │ │ ├── contents.rst │ │ ├── installation.rst │ │ ├── opal.rst │ │ ├── sphinxext │ │ ├── inheritance_diagram.py │ │ ├── ipython_console_highlighting.py │ │ └── mathjax.py │ │ └── tutorial.rst └── technical-report │ ├── analysis-design.pdf │ └── analysis-design.tex ├── examples ├── coopsort │ ├── benchmark_algorithms.py │ ├── cleanup │ ├── coopsort.py │ ├── coopsort_declaration.py │ ├── coopsort_optimize.py │ ├── coopsort_optimize_surrogate.py │ ├── coopsort_run.py │ ├── generate_performance_profiles.py │ ├── sort.py │ ├── test.py │ ├── test_coopsort.py │ ├── test_generate_list.py │ ├── test_sort.py │ └── transform_iteration_log.py ├── dfo │ ├── dfo_declaration.py │ ├── dfo_optimize.py │ └── dfo_run.py ├── fd │ ├── README.rst │ ├── fd.py │ ├── fd_declaration.py │ ├── fd_optimize.py │ ├── fd_run.py │ └── test_fd.py ├── ipopt │ ├── ipopt_composite_measures.py │ ├── ipopt_declaration.py │ ├── ipopt_optimize.py │ ├── ipopt_run.py │ └── ipopt_test_problems.py ├── test_examples.py └── trunk │ ├── test_trunk.py │ ├── trunk_declaration.py │ ├── trunk_optimize.py │ ├── trunk_optimize_lsf.py │ ├── trunk_optimize_mpi.py │ ├── trunk_optimize_opalmpi.py │ ├── trunk_optimize_opalsmp_nomadmpi.py │ ├── trunk_optimize_surrogate.py │ └── trunk_run.py ├── nosetests.py ├── opal ├── Platforms │ ├── __init__.py │ ├── linux.py │ ├── lsf.py │ ├── mpi.py │ ├── smp.py │ ├── sungrid.py │ └── test_platforms.py ├── Solvers │ ├── __init__.py │ └── nomad.py ├── TestProblemCollections │ ├── CUTEr.data │ ├── CUTEr.py │ ├── __init__.py │ └── cuterfactory.py ├── __init__.py ├── config.py ├── core │ ├── __init__.py │ ├── algorithm.py │ ├── data.py │ ├── datagenerator.py │ ├── datamanager.py │ ├── experimentmanager.py │ ├── io.py │ ├── log.py │ ├── mafrw.py │ ├── measure.py │ ├── model.py │ ├── modeldata.py │ ├── modelevaluator.py │ ├── modelstructure.py │ ├── opalproblem.py │ ├── parameter.py │ ├── platform.py │ ├── savablefunction.py │ ├── set.py │ ├── solver.py │ ├── statsmeasure.py │ ├── structureevaluator.py │ ├── test_core.py │ ├── testproblem.py │ └── tools.py └── test_opal.py ├── setup.py └── test.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.py~ 3 | *~ 4 | build 5 | -------------------------------------------------------------------------------- /MANIFEST: -------------------------------------------------------------------------------- 1 | README 2 | setup.py 3 | paropt/__init__.py 4 | paropt/Algorithms/DFO.py 5 | paropt/Algorithms/__init__.py 6 | paropt/Drivers/DFO.py 7 | paropt/Drivers/__init__.py 8 | paropt/Measures/__init__.py 9 | paropt/Measures/basics.py 10 | paropt/Platforms/LINUX.py 11 | paropt/Platforms/LSF.py 12 | paropt/Platforms/__init__.py 13 | paropt/Solvers/NOMAD.py 14 | paropt/Solvers/__init__.py 15 | paropt/TestProblemCollections/CUTEr.py 16 | paropt/TestProblemCollections/__init__.py 17 | paropt/core/__init__.py 18 | paropt/core/algorithm.py 19 | paropt/core/blackbox.py 20 | paropt/core/driver.py 21 | paropt/core/measure.py 22 | paropt/core/optmodel.py 23 | paropt/core/parameter.py 24 | paropt/core/paramopt.py 25 | paropt/core/platform.py 26 | paropt/core/poproblem.py 27 | paropt/core/popsolver.py 28 | paropt/core/posolver.py 29 | paropt/core/solver.py 30 | paropt/core/test.py 31 | paropt/core/testproblem.py 32 | paropt/core/utility.py 33 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # OPAL—A Framework for Optimization of Algorithms 2 | 3 | ## What is OPAL? 4 | 5 | OPAL is a Python modeling language for algorithmic optimization. Most 6 | algorithms depend on parameters. Although changing the values of those 7 | parameters doesn't affect the correctness of the algorithm, it typically 8 | affects its performance, where *performance* is understood broadly. How can we 9 | best choose those parameter values so as to maximize a certain measure of 10 | performance? 11 | 12 | OPAL is a framework that allows to easily declare algorithms and the parameters 13 | on which they depend along with representative test cases. It provides a 14 | convenient syntax to formulate the optimization problem to be solved. A 15 | black-box optimization solver takes care of the rest. 16 | 17 | ## Requirements 18 | 19 | + Python version 2.6 or 2.7 (not tested with 3.x) 20 | + [NOMAD](http://www.gerad.ca/NOMAD) 21 | 22 | ## Install 23 | 24 | + Unzip the package or clone the git repository 25 | + Go to the source directory 26 | + Run `python setup.py install` 27 | 28 | ## Testing 29 | 30 | To run the following test, [Numpy](http://www.numpy.org) is required. Install 31 | it with `pip install numpy`. 32 | 33 | Assuming your `PYTHONPATH` is set correctly, you should be able to do: 34 | 35 | cd examples/fd 36 | python fd_optimize.py 37 | 38 | ## References 39 | 40 | + [Optimization of Algorithms with OPAL](http://dx.doi.org/10.1007/s12532-014-0067-x) 41 | + [Templating and Automatic Code Generation for Performance with Python](https://gerad.ca/en/papers/G-2011-30/view) 42 | + [Taking Advantage of Parallelism in Algorithmic Parameter Optimization](http://dx.doi.org/10.1007/s11590-011-0428-6) 43 | + [Algorithmic Parameter Optimization of the DFO Method with the OPAL Framework](http://dx.doi.org/10.1007/978-1-4419-6935-4_15) 44 | 45 | ## Licensing 46 | 47 | [![LGPL-3.0](http://www.gnu.org/graphics/lgplv3-88x31.png)](http://www.gnu.org/licenses/lgpl-3.0.html) 48 | -------------------------------------------------------------------------------- /doc/reference/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = build 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 14 | 15 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest 16 | 17 | help: 18 | @echo "Please use \`make ' where is one of" 19 | @echo " html to make standalone HTML files" 20 | @echo " dirhtml to make HTML files named index.html in directories" 21 | @echo " singlehtml to make a single large HTML file" 22 | @echo " pickle to make pickle files" 23 | @echo " json to make JSON files" 24 | @echo " htmlhelp to make HTML files and a HTML help project" 25 | @echo " qthelp to make HTML files and a qthelp project" 26 | @echo " devhelp to make HTML files and a Devhelp project" 27 | @echo " epub to make an epub" 28 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 29 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 30 | @echo " text to make text files" 31 | @echo " man to make manual pages" 32 | @echo " changes to make an overview of all changed/added/deprecated items" 33 | @echo " linkcheck to check all external links for integrity" 34 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 35 | 36 | clean: 37 | -rm -rf $(BUILDDIR)/* 38 | 39 | html: 40 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 41 | @echo 42 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 43 | 44 | dirhtml: 45 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 46 | @echo 47 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 48 | 49 | singlehtml: 50 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 51 | @echo 52 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 53 | 54 | pickle: 55 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 56 | @echo 57 | @echo "Build finished; now you can process the pickle files." 58 | 59 | json: 60 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 61 | @echo 62 | @echo "Build finished; now you can process the JSON files." 63 | 64 | htmlhelp: 65 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 66 | @echo 67 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 68 | ".hhp project file in $(BUILDDIR)/htmlhelp." 69 | 70 | qthelp: 71 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 72 | @echo 73 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 74 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 75 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/bla.qhcp" 76 | @echo "To view the help file:" 77 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/bla.qhc" 78 | 79 | devhelp: 80 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 81 | @echo 82 | @echo "Build finished." 83 | @echo "To view the help file:" 84 | @echo "# mkdir -p $$HOME/.local/share/devhelp/bla" 85 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/bla" 86 | @echo "# devhelp" 87 | 88 | epub: 89 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 90 | @echo 91 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 92 | 93 | latex: 94 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 95 | @echo 96 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 97 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 98 | "(use \`make latexpdf' here to do that automatically)." 99 | 100 | latexpdf: 101 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 102 | @echo "Running LaTeX files through pdflatex..." 103 | make -C $(BUILDDIR)/latex all-pdf 104 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 105 | 106 | text: 107 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 108 | @echo 109 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 110 | 111 | man: 112 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 113 | @echo 114 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 115 | 116 | changes: 117 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 118 | @echo 119 | @echo "The overview file is in $(BUILDDIR)/changes." 120 | 121 | linkcheck: 122 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 123 | @echo 124 | @echo "Link check complete; look for any errors in the above output " \ 125 | "or in $(BUILDDIR)/linkcheck/output.txt." 126 | 127 | doctest: 128 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 129 | @echo "Testing of doctests in the sources finished, look at the " \ 130 | "results in $(BUILDDIR)/doctest/output.txt." 131 | -------------------------------------------------------------------------------- /doc/reference/README: -------------------------------------------------------------------------------- 1 | To build documentation in html format, use 'make html'. The entry point is in 2 | ./build/html/index.html. 3 | 4 | To build documentation in PDF format, use 'make latex'. Change directory to 5 | ./build/latex and use 'make all-pdf'. The documentation file is ParOpt.pdf. 6 | -------------------------------------------------------------------------------- /doc/reference/source/.templates/index.html: -------------------------------------------------------------------------------- 1 | {% extends "layout.html" %} 2 | {% set title = 'Overview' %} 3 | {% block body %} 4 | 5 |

Welcome

6 | 7 |
8 |

What authors say:

9 |

“Enjoy trying a tool that optimizes an algorithm in a snap!”

10 |
11 | 12 |

13 | OPAL stands for Optimization of Algorithm. OPAL provides a framework in 14 | which algorithmic parameter optimization problems can be defined and solved 15 | in a programmatic way. 16 |

17 | 28 | 38 |

39 | OPAL is written in Python and uses Python as the language in which the parameter 40 | optimization problem is described. The procedure consists in three main steps: 41 | declare the target algorithm, define an optimization problem and choose a 42 | suitable solver. 43 |

44 | 45 |

Documentation

46 | 47 | 48 | 59 |
49 |

Very simple tutorial
50 | overview of basic tasks

51 |

Contents
52 | for a complete overview

53 | 		54 |

Search page
55 | search the documentation

56 |

General Index
57 | all functions, classes, terms

58 |
60 | 61 | 62 |

Examples

63 |

Links to some examples can be found in the 64 | Tutorial section. 65 |

66 | 67 |

Get OPAL

68 |

69 | OPAL is available 70 | here. 71 |

72 | 73 | {% endblock %} 74 | 75 | -------------------------------------------------------------------------------- /doc/reference/source/.templates/indexsidebar.html: -------------------------------------------------------------------------------- 1 | 5 | 6 |

Download

7 | {% if version.endswith('(hg)') %} 8 |

This documentation is for version {{ version }}, which is 9 | not released yet.

10 |

You can use it from the 11 | Mercurial repo or look for 12 | released versions in the Python 13 | Package Index.

14 | {% else %} 15 |

Current version: {{ version }}

16 |

Get OPAL's stable version 17 | , or browse the repository

18 |

Latest development version docs 19 | are also available.

20 | {% endif %} 21 | 22 |

Questions? Suggestions?

23 |
24 | 25 | 26 | 27 |

28 | 29 |

30 | 31 | 32 |

33 | 34 |

35 | 36 | 37 |

38 | 39 |
40 | 41 | 42 |
43 |
44 |

or write to us. 45 | -------------------------------------------------------------------------------- /doc/reference/source/.templates/layout.html: -------------------------------------------------------------------------------- 1 | {% extends "!layout.html" %} 2 | 3 | {% block extrahead %} 4 | {{ super() }} 5 | {%- if not embedded %} 6 | 10 | {%- endif %} 11 | {% endblock %} 12 | 13 | {% block rootrellink %} 14 |

  • OPAL home | 
    • 15 |
  • Documentation 16 | »
    • 17 | {% endblock %} 18 | 19 | {% block header %} 20 |
    21 | 22 |

    OPAL—Optimization of Algorithms

    23 | 24 |
    25 | {% endblock %} 26 | 27 | -------------------------------------------------------------------------------- /doc/reference/source/api.rst: -------------------------------------------------------------------------------- 1 | .. API Documentation 2 | 3 | ================= 4 | API Documentation 5 | ================= 6 | 7 | 8 | .. automodule:: opal 9 | 10 | -------------------------------------------------------------------------------- /doc/reference/source/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # opal documentation build configuration file, created by 4 | # sphinx-quickstart on Thu May 21 22:41:01 2009. 5 | # 6 | # This file is execfile()d with the current directory set to its containing dir. 7 | # 8 | # The contents of this file are pickled, so don't put values in the namespace 9 | # that aren't pickleable (module imports are okay, they're removed automatically). 10 | # 11 | # Note that not all possible configuration values are present in this 12 | # autogenerated file. 13 | # 14 | # All configuration values have a default; values that are commented out 15 | # serve to show the default. 16 | 17 | import sys, os 18 | 19 | # If your extensions are in another directory, add it here. If the directory 20 | # is relative to the documentation root, use os.path.abspath to make it 21 | # absolute, like shown here. 22 | here=os.path.abspath('.') 23 | sys.path.append(here) 24 | sys.path.append('sphinxext') 25 | 26 | # Import support for ipython console session syntax highlighting (lives 27 | # in the sphinxext directory defined above) 28 | import ipython_console_highlighting 29 | import inheritance_diagram 30 | import mathjax 31 | 32 | # General configuration 33 | # --------------------- 34 | 35 | # Add any Sphinx extension module names here, as strings. They can be extensions 36 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 37 | extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest'] 38 | extensions += ['sphinx.ext.todo'] 39 | extensions += ['ipython_console_highlighting'] 40 | extensions += ['inheritance_diagram'] 41 | extensions += ['mathjax'] 42 | mathjax_path = 'http://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML' 43 | 44 | todo_include_todos = True 45 | 46 | # Add any paths that contain templates here, relative to this directory. 47 | templates_path = ['.templates'] 48 | 49 | # The suffix of source filenames. 50 | source_suffix = '.rst' 51 | 52 | # The encoding of source files. 53 | source_encoding = 'utf-8' 54 | 55 | # The master toctree document. 56 | master_doc = 'contents' 57 | html_index = 'index.html' 58 | 59 | # General information about the project. 60 | project = u'opal' 61 | copyright = u'2009-2011, C. Audet, K. C. Dang, D. Orban' 62 | 63 | # The version info for the project you're documenting, acts as replacement for 64 | # |version| and |release|, also used in various other places throughout the 65 | # built documents. 66 | # 67 | # The short X.Y version. 68 | version = '1.3' 69 | # The full version, including alpha/beta/rc tags. 70 | release = '1.3' 71 | 72 | # The language for content autogenerated by Sphinx. Refer to documentation 73 | # for a list of supported languages. 74 | #language = None 75 | 76 | # There are two options for replacing |today|: either, you set today to some 77 | # non-false value, then it is used: 78 | #today = '' 79 | # Else, today_fmt is used as the format for a strftime call. 80 | #today_fmt = '%B %d, %Y' 81 | 82 | # List of documents that shouldn't be included in the build. 83 | #unused_docs = [] 84 | 85 | # List of directories, relative to source directory, that shouldn't be searched 86 | # for source files. 87 | exclude_trees = [] 88 | 89 | # The reST default role (used for this markup: `text`) to use for all documents. 90 | #default_role = None 91 | 92 | # If true, '()' will be appended to :func: etc. cross-reference text. 93 | #add_function_parentheses = True 94 | 95 | # If true, the current module name will be prepended to all description 96 | # unit titles (such as .. function::). 97 | #add_module_names = True 98 | 99 | # If true, sectionauthor and moduleauthor directives will be shown in the 100 | # output. They are ignored by default. 101 | #show_authors = False 102 | 103 | # The name of the Pygments (syntax highlighting) style to use. 104 | pygments_style = 'sphinx' 105 | 106 | 107 | # Options for HTML output 108 | # ----------------------- 109 | 110 | html_theme = "sphinxdoc" 111 | 112 | # The style sheet to use for HTML and HTML Help pages. A file of that name 113 | # must exist either in Sphinx' static/ path, or in one of the custom paths 114 | # given in html_static_path. 115 | #html_style = 'default.css' 116 | #html_style = 'sphinxdoc.css' 117 | 118 | # The name for this set of Sphinx documents. If None, it defaults to 119 | # " v documentation". 120 | #html_title = None 121 | 122 | # A shorter title for the navigation bar. Default is the same as html_title. 123 | #html_short_title = None 124 | 125 | # The name of an image file (relative to this directory) to place at the top 126 | # of the sidebar. 127 | #html_logo = None 128 | 129 | # The name of an image file (within the static path) to use as favicon of the 130 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 131 | # pixels large. 132 | #html_favicon = None 133 | 134 | # Add any paths that contain custom static files (such as style sheets) here, 135 | # relative to this directory. They are copied after the builtin static files, 136 | # so a file named "default.css" will overwrite the builtin "default.css". 137 | #html_static_path = ['_static'] 138 | 139 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 140 | # using the given strftime format. 141 | #html_last_updated_fmt = '%b %d, %Y' 142 | 143 | # If true, SmartyPants will be used to convert quotes and dashes to 144 | # typographically correct entities. 145 | #html_use_smartypants = True 146 | 147 | # Custom sidebar templates, maps document names to template names. 148 | html_sidebars = { 149 | 'index' : ['indexsidebar.html', 150 | 'searchbox.html'] 151 | } 152 | 153 | # Additional templates that should be rendered to pages, maps page names to 154 | # template names. 155 | html_additional_pages = {'index': 'index.html'} 156 | 157 | # If false, no module index is generated. 158 | #html_use_modindex = True 159 | 160 | # If false, no index is generated. 161 | #html_use_index = True 162 | 163 | # If true, the index is split into individual pages for each letter. 164 | #html_split_index = False 165 | 166 | # If true, links to the reST sources are added to the pages. 167 | #html_show_sourcelink = True 168 | 169 | # If true, an OpenSearch description file will be output, and all pages will 170 | # contain a tag referring to it. The value of this option must be the 171 | # base URL from which the finished HTML is served. 172 | #html_use_opensearch = '' 173 | 174 | # If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). 175 | #html_file_suffix = '' 176 | 177 | # Output file base name for HTML help builder. 178 | htmlhelp_basename = 'opaldoc' 179 | 180 | 181 | # Options for LaTeX output 182 | # ------------------------ 183 | 184 | # The paper size ('letter' or 'a4'). 185 | latex_paper_size = 'letter' 186 | 187 | # The font size ('10pt', '11pt' or '12pt'). 188 | #latex_font_size = '10pt' 189 | 190 | # Grouping the document tree into LaTeX files. List of tuples 191 | # (source start file, target name, title, author, document class [howto/manual]). 192 | latex_documents = [ 193 | ('index', 'opal.tex', ur'OPAL Documentation', 194 | ur'C. Audet, K. C. Dang, D. Orban', 'manual'), 195 | ] 196 | 197 | # The name of an image file (relative to this directory) to place at the top of 198 | # the title page. 199 | #latex_logo = None 200 | 201 | # For "manual" documents, if this is true, then toplevel headings are parts, 202 | # not chapters. 203 | #latex_use_parts = False 204 | 205 | # Additional stuff for the LaTeX preamble. 206 | #latex_preamble = '' 207 | 208 | # Documents to append as an appendix to all manuals. 209 | #latex_appendices = [] 210 | 211 | # If false, no module index is generated. 212 | #latex_use_modindex = True 213 | -------------------------------------------------------------------------------- /doc/reference/source/contents.rst: -------------------------------------------------------------------------------- 1 | .. OPAL documentation master file, created by sphinx-quickstart on Thu May 21 22:41:01 2009. 2 | You can adapt this file completely to your liking, but it should at least 3 | contain the root `toctree` directive. 4 | 5 | .. _contents: 6 | 7 | ========================== 8 | OPAL General Documentation 9 | ========================== 10 | 11 | :Release: |version| 12 | :Date: |today| 13 | 14 | .. module:: opal 15 | 16 | This is the general documentation for OPAL, a pure Python package for 17 | the optimization of algorithmic parameters. 18 | 19 | Contents: 20 | 21 | .. toctree:: 22 | :numbered: 23 | :maxdepth: 2 24 | 25 | Installation 26 | Tutorial 27 | OPAL components description 28 | API 29 | 30 | .. TODO List 31 | ========= 32 | 33 | .. Add todo items in the sections of the documentation and they will appear 34 | .. here. 35 | 36 | .. 37 | .. todolist:: 38 | 39 | 40 | Indices and tables 41 | ================== 42 | 43 | * :ref:`genindex` 44 | * :ref:`modindex` 45 | * :ref:`search` 46 | 47 | -------------------------------------------------------------------------------- /doc/reference/source/installation.rst: -------------------------------------------------------------------------------- 1 | ============ 2 | Installation 3 | ============ 4 | 5 | OPAL is a Python package and can be easily installed with 6 | `distutils`. 7 | 8 | Requirements 9 | ============ 10 | 11 | Python version 2.6 or later is required. Previous versions may result in errors 12 | relating to the `import` statement. OPAL was not tested with Python 3k. 13 | 14 | OPAL uses NOMAD as default solver. A working NOMAD installation is not needed to 15 | install OPAL but is needed to run the simplest examples provided in the 16 | package. NOMAD can be downloaded from ``_. 17 | 18 | `Sphinx `_ if you intend to rebuild the documentation. 19 | 20 | Download and installation 21 | ========================= 22 | 23 | After downloading the OPAL source code, simply type:: 24 | 25 | shell$ python setup.py install 26 | 27 | If you do not have write permissions to the system-wide ``site-packages`` 28 | directory, a local ``site-packages`` should be provided when installing, e.g.:: 29 | 30 | shell$ python setup.py install --prefix=$LOCAL_PYTHON_DIR 31 | 32 | and the package will be installed in 33 | ``$LOCAL_PYTHON_DIR/lib/python-2.6/site-packages`` (assuming you are using 34 | Python 2.6) 35 | 36 | Documentation 37 | ============= 38 | 39 | The documentation can be rebuilt from source using Sphinx:: 40 | 41 | shell$ cd doc/reference 42 | shell$ make html 43 | 44 | The documentation is in `build/html`. 45 | -------------------------------------------------------------------------------- /doc/reference/source/opal.rst: -------------------------------------------------------------------------------- 1 | .. Description of objects. 2 | 3 | =========================== 4 | OPAL components description 5 | =========================== 6 | 7 | 8 | Core objects 9 | ============ 10 | 11 | .. automodule:: opal.core.data 12 | 13 | Data 14 | ---- 15 | 16 | .. autoclass:: Data 17 | :show-inheritance: 18 | :members: 19 | :inherited-members: 20 | :undoc-members: 21 | 22 | DataSet 23 | ------- 24 | 25 | .. autoclass:: DataSet 26 | :show-inheritance: 27 | :members: 28 | :inherited-members: 29 | :undoc-members: 30 | 31 | 32 | Basic objects 33 | ============= 34 | 35 | Parameter 36 | --------- 37 | 38 | .. inheritance-diagram:: opal.core.parameter 39 | 40 | .. automodule:: opal.core.parameter 41 | 42 | .. autoclass:: Parameter 43 | :show-inheritance: 44 | :members: 45 | :inherited-members: 46 | :undoc-members: 47 | 48 | .. autoclass:: ParameterSet 49 | :show-inheritance: 50 | :members: 51 | :inherited-members: 52 | :undoc-members: 53 | 54 | .. autoclass:: ParameterConstraint 55 | :show-inheritance: 56 | :members: 57 | :inherited-members: 58 | :undoc-members: 59 | 60 | 61 | 62 | Measure 63 | ------- 64 | 65 | .. inheritance-diagram:: opal.core.measure 66 | 67 | .. automodule:: opal.core.measure 68 | 69 | .. autoclass:: Measure 70 | :show-inheritance: 71 | :members: 72 | :inherited-members: 73 | :undoc-members: 74 | 75 | MeasureTable 76 | ------------ 77 | 78 | .. autoclass:: MeasureValueTable 79 | :show-inheritance: 80 | :members: 81 | :inherited-members: 82 | :undoc-members: 83 | 84 | Problem 85 | ------- 86 | 87 | .. inheritance-diagram:: opal.core.testproblem 88 | 89 | .. automodule:: opal.core.testproblem 90 | 91 | .. autoclass:: TestProblem 92 | :show-inheritance: 93 | :members: 94 | :inherited-members: 95 | :undoc-members: 96 | 97 | .. autoclass:: OptimizationTestProblem 98 | :show-inheritance: 99 | :members: 100 | :inherited-members: 101 | :undoc-members: 102 | 103 | .. Algorithm experiment description object 104 | .. ======================================= 105 | 106 | 107 | 108 | Parameter Optimization Problem Definition 109 | ========================================= 110 | 111 | In the OPAL framework, a parameter optimization model (shortly OPAL model) has 112 | the following components: 113 | 114 | #. A structure, which is a sort of optimization problem template, disconnected 115 | from any specifics such as the particular algorithm being tuned or the particular 116 | test set used to tune the parameters. An example of problem structure may 117 | be as simple as "minimize the total CPU time" or "maximize the throughput 118 | subject to network constraints." 119 | 120 | #. Data, which fills in the blanks and brings the model to life. Data, in a 121 | sense, instantiates the problem structure by populating it with the 122 | particular algorithm being tuned, the parameters being fine-tuned, the test 123 | set, etc. 124 | 125 | Instead of ``problem``, we will often use the term ``model`` and we will refer 126 | to its components as the ``model structure`` and ``model data``. 127 | 128 | Once the model structure and model data have been defined, we can put them 129 | together into a single composite object representing the parameter optimization 130 | problem. It is then referred to as a ``model``. 131 | 132 | 133 | Algorithm experiment 134 | -------------------- 135 | 136 | .. automodule:: opal.core.algorithm 137 | 138 | .. autoclass:: Algorithm 139 | :show-inheritance: 140 | :members: 141 | :inherited-members: 142 | :undoc-members: 143 | 144 | The most important ingredients of an algorithmic parameter optimization problem 145 | are: 146 | 147 | #. The algorithm to be fine tuned, 148 | #. The parameters to fine tune, 149 | #. The measures defining the performance criterion. 150 | 151 | In this section, we examine the basic ingredients in detail. 152 | 153 | ModelStructure 154 | -------------- 155 | 156 | .. automodule:: opal.core.modelstructure 157 | 158 | .. autoclass:: ModelStructure 159 | :show-inheritance: 160 | :members: 161 | :inherited-members: 162 | :undoc-members: 163 | 164 | ModelData 165 | --------- 166 | 167 | .. automodule:: opal.core.modeldata 168 | 169 | .. autoclass:: ModelData 170 | :show-inheritance: 171 | :members: 172 | :inherited-members: 173 | :undoc-members: 174 | 175 | Model 176 | ----- 177 | 178 | .. automodule:: opal.core.blackbox 179 | 180 | .. autoclass:: BlackBoxModel 181 | :show-inheritance: 182 | :members: 183 | :inherited-members: 184 | :undoc-members: 185 | 186 | 187 | 188 | -------------------------------------------------------------------------------- /doc/reference/source/sphinxext/ipython_console_highlighting.py: -------------------------------------------------------------------------------- 1 | from pygments.lexer import Lexer, do_insertions 2 | from pygments.lexers.agile import PythonConsoleLexer, PythonLexer, \ 3 | PythonTracebackLexer 4 | from pygments.token import Comment, Generic 5 | from sphinx import highlighting 6 | import re 7 | 8 | line_re = re.compile('.*?\n') 9 | 10 | class IPythonConsoleLexer(Lexer): 11 | """ 12 | For IPython console output or doctests, such as: 13 | 14 | Tracebacks are not currently supported. 15 | 16 | .. sourcecode:: ipython 17 | 18 | In [1]: a = 'foo' 19 | 20 | In [2]: a 21 | Out[2]: 'foo' 22 | 23 | In [3]: print a 24 | foo 25 | 26 | In [4]: 1 / 0 27 | """ 28 | name = 'IPython console session' 29 | aliases = ['ipython'] 30 | mimetypes = ['text/x-ipython-console'] 31 | input_prompt = re.compile("(In \[[0-9]+\]: )|( \.\.\.+:)") 32 | output_prompt = re.compile("(Out\[[0-9]+\]: )|( \.\.\.+:)") 33 | continue_prompt = re.compile(" \.\.\.+:") 34 | tb_start = re.compile("\-+") 35 | 36 | def get_tokens_unprocessed(self, text): 37 | pylexer = PythonLexer(**self.options) 38 | tblexer = PythonTracebackLexer(**self.options) 39 | 40 | curcode = '' 41 | insertions = [] 42 | for match in line_re.finditer(text): 43 | line = match.group() 44 | input_prompt = self.input_prompt.match(line) 45 | continue_prompt = self.continue_prompt.match(line.rstrip()) 46 | output_prompt = self.output_prompt.match(line) 47 | if line.startswith("#"): 48 | insertions.append((len(curcode), 49 | [(0, Comment, line)])) 50 | elif input_prompt is not None: 51 | insertions.append((len(curcode), 52 | [(0, Generic.Prompt, input_prompt.group())])) 53 | curcode += line[input_prompt.end():] 54 | elif continue_prompt is not None: 55 | insertions.append((len(curcode), 56 | [(0, Generic.Prompt, continue_prompt.group())])) 57 | curcode += line[continue_prompt.end():] 58 | elif output_prompt is not None: 59 | insertions.append((len(curcode), 60 | [(0, Generic.Output, output_prompt.group())])) 61 | curcode += line[output_prompt.end():] 62 | else: 63 | if curcode: 64 | for item in do_insertions(insertions, 65 | pylexer.get_tokens_unprocessed(curcode)): 66 | yield item 67 | curcode = '' 68 | insertions = [] 69 | yield match.start(), Generic.Output, line 70 | if curcode: 71 | for item in do_insertions(insertions, 72 | pylexer.get_tokens_unprocessed(curcode)): 73 | yield item 74 | 75 | highlighting.lexers['ipython'] = IPythonConsoleLexer() 76 | -------------------------------------------------------------------------------- /doc/reference/source/sphinxext/mathjax.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | sphinx.ext.mathjax 4 | ~~~~~~~~~~~~~~~~~~ 5 | 6 | Allow `MathJax `_ to be used to display math 7 | in Sphinx's HTML writer - requires the MathJax JavaScript library 8 | on your webserver/computer. 9 | 10 | Kevin Dunn, kgdunn@gmail.com, 3-clause BSD license. 11 | 12 | 13 | For background, installation details and support: 14 | 15 | https://bitbucket.org/kevindunn/sphinx-extension-mathjax 16 | 17 | """ 18 | from docutils import nodes 19 | from sphinx.application import ExtensionError 20 | from sphinx.ext.mathbase import setup_math as mathbase_setup 21 | 22 | def html_visit_math(self, node): 23 | self.body.append(self.starttag(node, 'span', '', CLASS='math')) 24 | self.body.append(self.builder.config.mathjax_inline[0] + \ 25 | self.encode(node['latex']) +\ 26 | self.builder.config.mathjax_inline[1] + '') 27 | raise nodes.SkipNode 28 | 29 | def html_visit_displaymath(self, node): 30 | self.body.append(self.starttag(node, 'div', CLASS='math')) 31 | if node['nowrap']: 32 | self.body.append(self.builder.config.mathjax_display[0] + \ 33 | node['latex'] +\ 34 | self.builder.config.mathjax_display[1]) 35 | self.body.append('') 36 | raise nodes.SkipNode 37 | 38 | parts = [prt for prt in node['latex'].split('\n\n') if prt.strip() != ''] 39 | for i, part in enumerate(parts): 40 | part = self.encode(part) 41 | if i == 0: 42 | # necessary to e.g. set the id property correctly 43 | if node['number']: 44 | self.body.append('(%s)' % 45 | node['number']) 46 | if '&' in part or '\\\\' in part: 47 | self.body.append(self.builder.config.mathjax_display[0] + \ 48 | '\\begin{split}' + part + '\\end{split}' + \ 49 | self.builder.config.mathjax_display[1]) 50 | else: 51 | self.body.append(self.builder.config.mathjax_display[0] + part + \ 52 | self.builder.config.mathjax_display[1]) 53 | self.body.append('\n') 54 | raise nodes.SkipNode 55 | 56 | def builder_inited(app): 57 | if not app.config.mathjax_path: 58 | raise ExtensionError('mathjax_path config value must be set for the ' 59 | 'mathjax extension to work') 60 | app.add_javascript(app.config.mathjax_path) 61 | 62 | def setup(app): 63 | mathbase_setup(app, (html_visit_math, None), (html_visit_displaymath, None)) 64 | app.add_config_value('mathjax_path', '', False) 65 | app.add_config_value('mathjax_inline', [r'\(', r'\)'], 'html') 66 | app.add_config_value('mathjax_display', [r'\[', r'\]'], 'html') 67 | app.connect('builder-inited', builder_inited) 68 | 69 | -------------------------------------------------------------------------------- /doc/technical-report/analysis-design.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PythonOptimizers/opal/b6e12bafe7210910a6df2a9b59829cde584153cb/doc/technical-report/analysis-design.pdf -------------------------------------------------------------------------------- /examples/coopsort/benchmark_algorithms.py: -------------------------------------------------------------------------------- 1 | from coopsort import CooperationTreeFactory, coopsort 2 | from coopsort import create_test_list, verify_sorted_list, benchmarkTime 3 | import sort 4 | import sys 5 | from inspect import isfunction 6 | 7 | coopTreeFactory = CooperationTreeFactory(name="test factory") 8 | 9 | coopTrees = {} 10 | 11 | coopTrees["quick sort"] = sort.quicksort 12 | coopTrees["quick sort mixed"] = sort.quicksort6 13 | coopTrees["quick sort non-recursive"] = sort.quicksort0 14 | coopTrees["merge sort"] = sort.mergesort 15 | coopTrees['heap sort'] = sort.heapsort 16 | #coopTrees["tim sort"] = sort.timsort 17 | 18 | benchmarkSpec = {'algorithms': {'coopsort 522': [5, 2, 2], 19 | 'coopsort 5522522': [5, 5, 2, 2, 5, 2, 2], 20 | 'coopsort 411': [4, 1, 1] 21 | }, 22 | 'lists': {'specs': [(8, 0.0125)], 23 | 'unit length': 20, 24 | 'number of list': 200 25 | }, 26 | 'testing': {'repeats': 100 27 | } 28 | } 29 | 30 | if len(sys.argv) > 1: 31 | f = open(sys.argv) 32 | benchmarkSpec = eval(f.read()) 33 | f.close() 34 | 35 | algorithms = benchmarkSpec['algorithms'] 36 | createAlgo = coopTreeFactory.createTree 37 | for algoName in algorithms: 38 | coopTrees[algoName] = createAlgo(name='coopsort', 39 | methodSequence=algorithms[algoName]) 40 | 41 | cpuTime = {} 42 | 43 | listSpecs = benchmarkSpec['lists']['specs'] 44 | listLenghUnit = benchmarkSpec['lists']['unit length'] 45 | numberOfList = benchmarkSpec['lists']['number of list'] 46 | numberOfRepeat = benchmarkSpec['testing']['repeats'] 47 | 48 | for listSpec in listSpecs: 49 | listType = str(listSpec) 50 | cpuTime[listType] = {} 51 | for name, coopTree in coopTrees.iteritems(): 52 | cpuTime[listType][name] = {} 53 | for k in range(numberOfList): 54 | n = listLengthUnit * (k + 1) 55 | l = create_test_list(listSpec, n, 0) 56 | for name, coopTree in coopTrees.iteritems(): 57 | meanTime = 0.0 58 | for r in range(numberOfRepeat): 59 | l1 = list(l) 60 | if isfunction(coopTree): 61 | tb = benchmarkTime() 62 | l1 = coopTree(l1) 63 | else: 64 | tb = benchmarkTime() 65 | l1 = coopsort(l1, coopTree.getRoot()) 66 | te = benchmarkTime() 67 | listIsSorted = verify_sorted_list(l1) 68 | if listIsSorted: 69 | meanTime = meanTime + (te - tb) 70 | else: 71 | raise Exception("There is a problem with " + name) 72 | del l1[:] 73 | del l1 74 | cpuTime[listType][name][n] = meanTime / numberOfRepeat 75 | print cpuTime 76 | -------------------------------------------------------------------------------- /examples/coopsort/cleanup: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | rm *.log 2> /dev/null 4 | rm *.dat* 2> /dev/null 5 | rm nomad*.txt 2> /dev/null 6 | rm blackbox.py 2> /dev/null 7 | rm neighbors.py 2> /dev/null 8 | rm surrogate.py 2> /dev/null 9 | 10 | if [[ $# > 0 && $1 == '-a' ]] 11 | then 12 | rm *.pyc 2> /dev/null 13 | rm iterations*.txt 2> /dev/null 14 | fi 15 | -------------------------------------------------------------------------------- /examples/coopsort/coopsort_declaration.py: -------------------------------------------------------------------------------- 1 | from opal.core.algorithm import Algorithm 2 | from opal.core.parameter import Parameter 3 | from opal.core.measure import Measure 4 | 5 | # Define new algorithm. 6 | coopsort = Algorithm(name='CoopSort', description='Sort Algorithm') 7 | 8 | # Register executable. 9 | coopsort.set_executable_command('python coopsort_run.py') 10 | 11 | # Define parameters. 12 | 13 | # The following coop tree amounts to 5522522 (in base 6.) 14 | coopsort.add_param(Parameter(name='coopTree', 15 | kind='categorical', 16 | default=275378, 17 | #default=284354431, 18 | description='Encoded cooperation tree')) 19 | 20 | # This dummy parameter is just there to circumvent a bug in NOMAD 21 | # that occurs when the problem has a single parameter and this parameter 22 | # is categorical. 23 | coopsort.add_param(Parameter(name='nothing', 24 | kind='integer', 25 | default=0, 26 | description='To avoid a bug in NOMAD')) 27 | 28 | coopsort.add_measure(Measure(name='TIME', 29 | kind='real', 30 | description='Computing time')) 31 | -------------------------------------------------------------------------------- /examples/coopsort/coopsort_optimize.py: -------------------------------------------------------------------------------- 1 | from coopsort_declaration import coopsort 2 | 3 | from opal import ModelStructure, ModelData, Model, MeasureFunction, TestProblem 4 | from opal.Solvers import NOMAD 5 | 6 | 7 | def sum_time(parameters, measures): 8 | val = sum(measures['TIME']) 9 | return val 10 | 11 | 12 | def get_neighbors(parameters): 13 | tmpStr = parameters[0] 14 | encodedNumber = tmpStr[0:tmpStr.find('.')] 15 | 16 | # Set of names for the leaf node 17 | s_s = ['0', '1', '2', '3'] 18 | # Set of names for the internal node 19 | s_d = ['4', '5'] 20 | # Neighborhood relation for a single node name 21 | n_s = {'0': ['2', '3'], 22 | '1': ['2', '3'], 23 | '2': ['0', '3'], 24 | '3': ['1', '2'], 25 | '4': ['5'], 26 | '5': ['4'] 27 | } 28 | # Initialize an empty list of neighbors 29 | neighbors = [] 30 | for i in range(len(encodedNumber)): 31 | c = encodedNumber[i] 32 | for n in n_s[c]: 33 | tmpStr = encodedNumber[0:i] + n + encodedNumber[i + 1:] 34 | 35 | # A neighbor is a parameter point 36 | neighbor = list(parameters) 37 | 38 | # Change value of the first coordinate of the parameter point 39 | neighbor[0] = int(float(tmpStr)) 40 | 41 | # Add the neighbor to the neighbor list 42 | neighbors.append(neighbor) 43 | 44 | if c in s_s: # Is is a leaf node 45 | for d in s_d: 46 | tmpStr = encodedNumber[0:i] + \ 47 | d + n_s[c][0] + n_s[c][1] + \ 48 | encodedNumber[i + 1:] 49 | neighbor = list(parameters) 50 | 51 | # Change value of the first coordinate of the parameter point 52 | neighbor[0] = int(float(tmpStr)) 53 | 54 | # Add the neighbor to the neighbor list 55 | neighbors.append(neighbor) 56 | 57 | elif i < len(encodedNumber) - 2: 58 | s_1 = encodedNumber[i + 1] 59 | s_2 = encodedNumber[i + 2] 60 | if (s_1 in s_s) and (s_2 in s_s): 61 | tmpStr = encodedNumber[0:i] + s_1 + encodedNumber[i + 3:] 62 | neighbor = list(parameters) 63 | neighbor[0] = int(float(tmpStr)) 64 | neighbors.append(neighbor) 65 | tmpStr = encodedNumber[0:i] + s_2 + encodedNumber[i + 3:] 66 | neighbor = list(parameters) 67 | neighbor[0] = int(float(tmpStr)) 68 | neighbors.append(neighbor) 69 | return neighbors 70 | 71 | problems = [] 72 | listTypes = [0, 5, 6, 7] 73 | numberOfList = 16 74 | listLengthStep = 25 75 | for listType in listTypes: 76 | for k in range(numberOfList): 77 | n = listLengthStep * (k + 1) 78 | probName = str(listType) + '-' + str(n) + '-50' 79 | problems.append(TestProblem(name=probName)) 80 | 81 | #SMP.set_parameter(name='MAX_PROC', value=5); 82 | 83 | # Define parameter optimization problem. 84 | data = ModelData(algorithm=coopsort, problems=problems) 85 | 86 | struct = ModelStructure(objective=MeasureFunction(sum_time), 87 | constraints=[], # Unconstrained 88 | neighborhood=get_neighbors) 89 | 90 | prob = Model(modelData=data, modelStructure=struct) 91 | 92 | # Solve parameter optimization problem. 93 | 94 | if __name__ == '__main__': 95 | from opal.Solvers import NOMAD 96 | NOMAD.set_parameter(name='MAX_BB_EVAL', value=100) 97 | NOMAD.set_parameter(name='DISPLAY_DEGREE', value=4) 98 | NOMAD.set_parameter(name='EXTENDED_POLL_TRIGGER', value='r0.1') 99 | NOMAD.set_parameter(name='MAX_MESH_INDEX', value='2') 100 | NOMAD.set_parameter(name='MODEL_SEARCH_OPTIMISTIC', value='no') 101 | NOMAD.solve(blackbox=prob) 102 | -------------------------------------------------------------------------------- /examples/coopsort/coopsort_optimize_surrogate.py: -------------------------------------------------------------------------------- 1 | from coopsort_declaration import coopsort 2 | 3 | from opal import ModelStructure, ModelData, Model, MeasureFunction, TestProblem 4 | from opal.Solvers import NOMAD 5 | 6 | 7 | def sum_time(parameters, measures): 8 | val = sum(measures['TIME']) 9 | return val 10 | 11 | 12 | def get_neighbors(parameters): 13 | def encode(number, base=6): 14 | numberStr = '' 15 | if number == 0: 16 | return '0' 17 | while number > 0: 18 | (number, r) = divmod(number, base) 19 | numberStr = str(r) + numberStr 20 | return numberStr 21 | 22 | def decode(numberStr, base=6): 23 | invertedStr = numberStr[::-1] 24 | number = 0 25 | quotient = 1 26 | for c in invertedStr: 27 | number = number + int(c) * quotient 28 | quotient = quotient * base 29 | return number 30 | 31 | import logging 32 | logger = logging.getLogger(__name__) 33 | fh = logging.FileHandler('neighborhood.log') 34 | logger.setLevel(logging.INFO) 35 | fh.setLevel(logging.INFO) 36 | logger.addHandler(fh) 37 | encodedNumber = encode(int(float(parameters[0])), 6) 38 | logMessage = parameters[0] + ', ' + \ 39 | str(int(float(parameters[0]))) + ', ' + \ 40 | encodedNumber + ': ' 41 | # Set of names for the leaf node 42 | s_s = ['0', '1', '2', '3'] 43 | # Set of names for the internal node 44 | s_d = ['4', '5'] 45 | # Neighborhood relation for a single node name 46 | n_s = {'0': ['2', '3'], 47 | '1': ['2', '3'], 48 | '2': ['1', '3'], 49 | '3': ['1', '2'], 50 | '4': ['5'], 51 | '5': ['4'] 52 | } 53 | # Initialize an empty list of neighbors 54 | neighbors = [] 55 | for i in range(len(encodedNumber)): 56 | c = encodedNumber[i] 57 | for n in n_s[c]: 58 | tmpStr = encodedNumber[0:i] + n + encodedNumber[i + 1:] 59 | 60 | # A neighbor is a parameter point 61 | neighbor = list(parameters) 62 | 63 | # Change value of the first coordinate of the parameter point 64 | neighbor[0] = decode(tmpStr) 65 | logMessage = logMessage + tmpStr + ' -> ' + \ 66 | str(neighbor[0]) + ', ' 67 | 68 | # Add the neighbor to the neighbor list 69 | neighbors.append(neighbor) 70 | 71 | if c in s_s: # It is a leaf node 72 | for d in s_d: 73 | tmpStr = encodedNumber[0:i] + \ 74 | d + n_s[c][0] + n_s[c][1] + \ 75 | encodedNumber[i + 1:] 76 | neighbor = list(parameters) 77 | 78 | # Change value of the first coordinate of the parameter point 79 | neighbor[0] = decode(tmpStr) 80 | logMessage = logMessage + tmpStr + ' -> ' + \ 81 | str(neighbor[0]) + ', ' 82 | 83 | # Add the neighbor to the neighbor list 84 | neighbors.append(neighbor) 85 | 86 | elif i < len(encodedNumber) - 2: 87 | s_1 = encodedNumber[i + 1] 88 | s_2 = encodedNumber[i + 2] 89 | if (s_1 in s_s) and (s_2 in s_s): 90 | tmpStr = encodedNumber[0:i] + s_1 + encodedNumber[i + 3:] 91 | neighbor = list(parameters) 92 | neighbor[0] = decode(tmpStr) 93 | logMessage = logMessage + tmpStr + ' -> ' + \ 94 | str(neighbor[0]) + ', ' 95 | neighbors.append(neighbor) 96 | tmpStr = encodedNumber[0:i] + s_2 + encodedNumber[i + 3:] 97 | neighbor = list(parameters) 98 | neighbor[0] = decode(tmpStr) 99 | logMessage = logMessage + tmpStr + ' -> ' + \ 100 | str(neighbor[0]) + ', ' 101 | neighbors.append(neighbor) 102 | logger.info(logMessage + '\n') 103 | return neighbors 104 | 105 | problems = [] 106 | listSpecIndices = [1] 107 | numberOfList = 1 108 | listLengthStep = 4000 109 | numberOfRepetition = 100 110 | for listSpecIndex in listSpecIndices: 111 | for k in range(numberOfList): 112 | n = listLengthStep * (k + 1) 113 | probName = str(listSpecIndex) + \ 114 | '-' + str(n) + '-' + str(numberOfRepetition) 115 | problems.append(TestProblem(name=probName)) 116 | 117 | surrogate_problems = [] 118 | for listSpecIndex in listSpecIndices: 119 | for k in range(numberOfList): 120 | n = listLengthStep * (k + 1) 121 | probName = str(listSpecIndex) + \ 122 | '-' + str(n) + '-10' 123 | surrogate_problems.append(TestProblem(name=probName)) 124 | 125 | #SMP.set_parameter(name='MAX_PROC', value=5); 126 | 127 | # Define parameter optimization problem. 128 | data = ModelData(algorithm=coopsort, problems=problems) 129 | 130 | surrogate_data = ModelData(algorithm=coopsort, problems=surrogate_problems) 131 | 132 | struct = ModelStructure(objective=MeasureFunction(sum_time), 133 | constraints=[], # Unconstrained 134 | neighborhood=get_neighbors) 135 | 136 | prob = Model(modelData=data, modelStructure=struct) 137 | surrogate_prob = Model(modelData=surrogate_data, modelStructure=struct) 138 | 139 | # Solve parameter optimization problem. 140 | 141 | if __name__ == '__main__': 142 | from opal.Solvers import NOMAD 143 | NOMAD.set_parameter(name='MAX_BB_EVAL', value=100) 144 | NOMAD.set_parameter(name='DISPLAY_DEGREE', value=1) 145 | NOMAD.set_parameter(name="DISPLAY_STATS", 146 | value="%3dEVAL %3dBBE %4dSGTE [ %15lSOL, ] %8.2eBBO %4.2fTIME") 147 | NOMAD.set_parameter(name="STATS_FILE", 148 | value="iterations.txt %3dEVAL %3dBBE %4dSGTE [ %15lSOL, ] %8.2eBBO %4.2fTIME") 149 | #NOMAD.set_parameter(name='EXTENDED_POLL_TRIGGER', value='r0.1') 150 | NOMAD.set_parameter(name='MAX_MESH_INDEX', value='2') 151 | NOMAD.solve(blackbox=prob, surrogate=surrogate_prob) 152 | -------------------------------------------------------------------------------- /examples/coopsort/coopsort_run.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | 4 | from opal.core.io import * 5 | from opal.core.tools import * 6 | 7 | from coopsort import CooperationTreeFactory, coopsort 8 | from coopsort import create_test_list, benchmarkTime 9 | import sort 10 | import random 11 | import time 12 | 13 | 14 | listSpecs = [(8, 1), (8, 0.0125)] 15 | 16 | 17 | def run(param_file, problem): 18 | probInfos = problem.split('-') 19 | listSpecIndex = int(probInfos[0]) 20 | if len(probInfos) > 1: 21 | listLength = int(probInfos[1]) 22 | else: 23 | listLength = 4000 24 | if len(probInfos) > 2: 25 | numberOfRepeat = int(probInfos[2]) 26 | else: 27 | numberOfRepeat = 1 28 | if len(probInfos) > 3: 29 | groupSize = int(probInfos[3]) 30 | else: 31 | groupSize = 1 32 | params = read_params_from_file(param_file) 33 | if int(params['nothing']) != 0: 34 | return {'TIME': float("inf")} 35 | 36 | coopTreeFactory = CooperationTreeFactory(name="test factory") 37 | try: 38 | treeEncodedNumber = int(params['coopTree']) 39 | except: 40 | treeEncodedNumber = int(float(params['coopTree'])) 41 | createTree = coopTreeFactory.createTreeFromEncodedNumber 42 | coopTree = createTree(name='coop tree', 43 | encodedNumber=treeEncodedNumber, 44 | radix=6) 45 | l = [] 46 | for g in range(groupSize): 47 | l.append(create_test_list(listSpecs[listSpecIndex], listLength, 0)) 48 | meanTime = 0.0 49 | for r in range(numberOfRepeat): 50 | sortingTime = 0.0 51 | for i in range(groupSize): 52 | l1 = list(l[i]) 53 | tb = benchmarkTime() 54 | try: 55 | l1 = coopsort(l1, coopTree.getRoot()) 56 | except: 57 | return {'TIME': float("inf")} 58 | te = benchmarkTime() 59 | sortingTime = sortingTime + (te - tb) 60 | del l1[:] 61 | del l1 62 | meanTime = meanTime + sortingTime 63 | 64 | measures = {'TIME': meanTime / numberOfRepeat} 65 | return measures 66 | 67 | 68 | if __name__ == '__main__': 69 | param_file = sys.argv[1] 70 | problem = sys.argv[2] 71 | output_file = sys.argv[3] 72 | 73 | # Solve, gather measures and write to file. 74 | param_file = os.path.abspath(param_file) 75 | measures = run(param_file, problem) 76 | write_measures_to_file(output_file, measures) 77 | -------------------------------------------------------------------------------- /examples/coopsort/generate_performance_profiles.py: -------------------------------------------------------------------------------- 1 | from coopsort import CooperationTreeFactory, coopsort 2 | from coopsort import create_test_list, benchmarkTime 3 | import sort 4 | import sys 5 | 6 | from inspect import isfunction 7 | 8 | coopTreeFactory = CooperationTreeFactory(name="test factory") 9 | 10 | coopTrees = {} 11 | coopTrees["quick sort"] = sort.quicksort6 12 | coopTrees["merge sort"] = sort.mergesort 13 | coopTrees["radix sort"] = sort.radixsort 14 | 15 | 16 | benchmarkSpec = {'algorithms': {'coopsort 522': [5, 2, 2], 17 | 'coopsort 5522522': [5, 5, 2, 2, 5, 2, 2], 18 | 'coopsort 411': [4, 1, 1] 19 | }, 20 | 'lists': {'specs': [(8, 0.0125)], 21 | 'unit length': 4000, 22 | 'number of list': 1000 23 | }, 24 | 'testing': {'repeats': 100 25 | } 26 | } 27 | 28 | if len(sys.argv) > 1: 29 | f = open(sys.argv) 30 | benchmarkSpec = eval(f.read()) 31 | f.close() 32 | 33 | algorithms = benchmarkSpec['algorithms'] 34 | createAlgo = coopTreeFactory.createTree 35 | for algoName in algorithms: 36 | coopTrees[algoName] = createAlgo(name='coopsort', 37 | methodSequence=algorithms[algoName]) 38 | 39 | listSpec = benchmarkSpec['lists']['specs'][0] 40 | listLengh = benchmarkSpec['lists']['unit length'] 41 | numberOfList = benchmarkSpec['lists']['number of list'] 42 | numberOfRepeat = benchmarkSpec['testing']['repeats'] 43 | 44 | cpuTime = {} 45 | profiles = {} 46 | for name, coopTree in coopTrees.iteritems(): 47 | profiles[name] = [] 48 | 49 | for i in range(numberOfList): 50 | l = create_test_list(listSpec, listLength, 0) 51 | minTime = float('inf') 52 | cpuTime = {} 53 | for name, coopTree in coopTrees.iteritems(): 54 | meanTime = 0.0 55 | for r in range(numberOfRepeat): 56 | l1 = list(l) 57 | if isfunction(coopTree): 58 | tb = benchmarkTime() 59 | l1 = coopTree(l1) 60 | else: 61 | tb = benchmarkTime() 62 | l1 = coopsort(l1, coopTree.getRoot()) 63 | te = benchmarkTime() 64 | del l1[:] 65 | del l1 66 | meanTime = meanTime + (te - tb) 67 | meanTime = meanTime / numberOfRepeat 68 | cpuTime[name] = meanTime 69 | if (meanTime < minTime): 70 | minTime = meanTime 71 | 72 | for name, t in cpuTime.iteritems(): 73 | profiles[name].append(t / minTime) 74 | 75 | percentage = [float(i + 1) / numberOfList for i in range(numberOfList)] 76 | 77 | for name, x in profiles.iteritems(): 78 | profiles[name] = (sorted(x), percentage) 79 | 80 | print profiles 81 | -------------------------------------------------------------------------------- /examples/coopsort/test.py: -------------------------------------------------------------------------------- 1 | from inspect import isfunction 2 | import sort 3 | import random 4 | import time 5 | 6 | N = 100000 7 | 8 | # Extract all functions from sort module. 9 | fcns = [val for key, val in sort.__dict__.iteritems() if isfunction(val)] 10 | 11 | # Time. 12 | for fcn in fcns: 13 | l = [int(N * random.random()) for i in xrange(N)] 14 | args = () 15 | if fcn.__name__ == 'bubblesort': 16 | continue 17 | if fcn.__name__ == 'countsort': 18 | args = (1+max(l),) 19 | t = time.clock() 20 | fcn(l, *args) 21 | t = time.clock() - t 22 | print '%15s %6.2f' % (fcn.__name__, t) 23 | -------------------------------------------------------------------------------- /examples/coopsort/test_coopsort.py: -------------------------------------------------------------------------------- 1 | from coopsort import CooperationTreeFactory, coopsort 2 | import sort 3 | import random 4 | import time 5 | 6 | N = 20 7 | 8 | # Extract all functions from sort module. 9 | #fcns = [val for key, val in sort.__dict__.iteritems() if isfunction(val)] 10 | 11 | # Time. 12 | #for fcn in fcns: 13 | 14 | coopTreeFactory = CooperationTreeFactory(name="test factory") 15 | 16 | coopTrees = {} 17 | coopTrees["quick sort"] = coopTreeFactory.createTree(name="quick sort", 18 | methodSequence=[2]) 19 | coopTrees["merge sort"] = coopTreeFactory.createTreeFromEncodedNumber(name="merge sort", 20 | encodedNumber=1) 21 | coopTrees["insertion sort"] = coopTreeFactory.createTree(name="insertion sort", 22 | methodSequence=[0]) 23 | coopTrees["radix sort"] = coopTreeFactory.createTree(name="radix sort", 24 | methodSequence=[3]) 25 | coopTrees['coop sort 1'] = coopTreeFactory.createTree(name='test tree 1', 26 | methodSequence=[4,2,3]) 27 | coopTrees['coop sort 2'] = coopTreeFactory.createTree(name='test tree 2', 28 | methodSequence=[4,4,2,3,3]) 29 | coopTrees['coop sort 3'] = coopTreeFactory.createTreeFromEncodedNumber(name='test tree 3', 30 | encodedNumber=45203) 31 | 32 | for name, coopTree in coopTrees.iteritems(): 33 | l = [int(N * random.random()) for i in xrange(N)] 34 | print "CoopSort with the tree:", coopTree._name 35 | t = time.clock() 36 | l = coopsort(l, coopTree.getRoot()) 37 | t = time.clock() - t 38 | #print "after sorting", l 39 | print 'Computing time: %6.2f second \n' % (t) 40 | -------------------------------------------------------------------------------- /examples/coopsort/test_generate_list.py: -------------------------------------------------------------------------------- 1 | from coopsort import create_test_list 2 | import sys 3 | 4 | if len(sys.argv) > 1: 5 | listSpecs = eval(sys.argv[1]) 6 | else: 7 | listSpecs = [[8, 0.0125]] 8 | if len(sys.argv) > 2: 9 | numberOfList = int(sys.argv[2]) 10 | else: 11 | numberOfList = 200 12 | if len(sys.argv) > 3: 13 | listLengthUnit = int(sys.argv[3]) 14 | else: 15 | listLengthUnit = 20 16 | 17 | generatedLists = {} 18 | for listSpec in listSpecs: 19 | listType = listSpec[0] 20 | for k in range(numberOfList): 21 | n = listLengthUnit * (k + 1) 22 | #for n in [250, 500, 750, 1000, 1250, 1500, 1750, 2000]: 23 | listName = str(listSpec) + 'l' + str(n) 24 | generatedLists[listName] = create_test_list(listSpec, n, 0) 25 | print generatedLists 26 | -------------------------------------------------------------------------------- /examples/coopsort/test_sort.py: -------------------------------------------------------------------------------- 1 | from inspect import isfunction 2 | import sort 3 | import random 4 | import time 5 | 6 | N = 10000 7 | 8 | # Extract all functions from sort module. 9 | fcns = [val for key, val in sort.__dict__.iteritems() if isfunction(val)] 10 | 11 | # Time. 12 | for fcn in fcns: 13 | #l = [int(N * random.random()) for i in xrange(N)] 14 | l = [i for i in xrange(N)] 15 | #l = [N - i for i in xrange(N)] 16 | args = () 17 | ignoredAlgorithms = ['bubblesort', 'insertionsort', 'selectionsort', 18 | 'quicksort2', 'quicksort3', 'quicksort4', 19 | 'quicksort5', 'quicksort'] 20 | if fcn.__name__ in ignoredAlgorithms: 21 | continue 22 | if fcn.__name__ == 'countsort': 23 | args = (1 + max(l),) 24 | t = time.time() 25 | fcn(l, *args) 26 | t = time.time() - t 27 | print '%15s %9.5f' % (fcn.__name__, t) 28 | -------------------------------------------------------------------------------- /examples/coopsort/transform_iteration_log.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import re 3 | 4 | import coopsort 5 | 6 | if __name__ == '__main__': 7 | regExp = r'(.+\[\s*)(\d+)\.?\d*( , 0\s+\].*)' 8 | f = open(sys.argv[1]) 9 | lines = f.readlines() 10 | f.close() 11 | for line in lines: 12 | beginPart = (re.sub(regExp, r'\1', line)).strip() 13 | midPart = (re.sub(regExp, r'\2', line)).strip() 14 | endPart = (re.sub(regExp, r'\3', line)).strip() 15 | #print midPart 16 | treeCode10 = int(re.sub(regExp, r'\2', line).strip()) 17 | treeCode6 = coopsort.encode(treeCode10, 6) 18 | print beginPart, treeCode6, endPart 19 | -------------------------------------------------------------------------------- /examples/dfo/dfo_declaration.py: -------------------------------------------------------------------------------- 1 | from opal.core.algorithm import Algorithm 2 | from opal.core.parameter import Parameter 3 | from opal.core.measure import Measure 4 | 5 | # Define new algorithm. 6 | DFO = Algorithm(name='DFO', description='Derivative-free Optimization') 7 | 8 | # Register executable for DFO. 9 | DFO.set_executable_command('python dfo_run.py') 10 | 11 | # Define parameters. 12 | nx = Parameter(kind='integer', default=1, name='NX') 13 | maxit = Parameter(kind='integer', default=5000, name='MAXIT') 14 | maxef = Parameter(kind='integer', default=9500, name='MAXNF') 15 | stpcrtr = Parameter(kind='integer', default=2, name='STPCRTR') 16 | delmin = Parameter(default=1.0e-4, name='DELMIN',bound=(1.0e-8,1.0e-3)) 17 | stpthr = Parameter(default=1.0e-3, name='STPTHR',bound=(0,None)) 18 | cnstol = Parameter(default=1.0e-5, name='CNSTOL',bound=(0,0.1)) 19 | delta = Parameter(default=1.0e0, name='DELTA',bound=(1.0e-8,None)) 20 | pp = Parameter(default=1.0e0, name='PP',bound=(1,None)) 21 | scale = Parameter(kind='integer', default=0, name='SCALE') 22 | iprint = Parameter(kind='integer', default=1, name='IPRINT') 23 | 24 | # Register parameters with algorithm. 25 | DFO.add_param(nx) 26 | DFO.add_param(maxit) 27 | DFO.add_param(maxef) 28 | DFO.add_param(stpcrtr) 29 | DFO.add_param(delmin) 30 | DFO.add_param(stpthr) 31 | DFO.add_param(cnstol) 32 | DFO.add_param(delta) 33 | DFO.add_param(pp) 34 | DFO.add_param(scale) 35 | DFO.add_param(iprint) 36 | 37 | # Define the feasible region. 38 | DFO.add_parameter_constraint('DELTA >= DELMIN') 39 | 40 | # Define and register measures. 41 | exitcode = Measure(kind='integer', name='EXITCODE', description='Exit code') 42 | fval = Measure(kind='real', name='FVAL', description='Function value') 43 | cpu = Measure(kind='real', name='CPU', description='CPU time usage') 44 | feval = Measure(kind='real', name='FEVAL', 45 | description='Number of function evaluations') 46 | 47 | DFO.add_measure(exitcode) 48 | DFO.add_measure(fval) 49 | DFO.add_measure(cpu) 50 | DFO.add_measure(feval) 51 | -------------------------------------------------------------------------------- /examples/dfo/dfo_optimize.py: -------------------------------------------------------------------------------- 1 | # Simple demo: tune DFO parameters for CPU time on simple HS problems. 2 | from opal import ModelStructure, ModelData, Model 3 | from opal.TestProblemCollections import CUTEr 4 | from dfo_declaration import DFO 5 | 6 | 7 | from opal.Solvers import NOMAD 8 | 9 | def avg_time(parameters,measures): 10 | n = len(measures['CPU']) 11 | if n == 0: 12 | return 0.0 13 | return (sum(measures["CPU"]) + 0.0)/(n + 0.0) 14 | 15 | # Select real parameters from DFO. 16 | params = [par for par in DFO.parameters if par.is_real] 17 | 18 | # Select tiny unconstrained HS problems. 19 | problems = [prb for prb in CUTEr.HS if prb.nvar <= 5 and prb.ncon == 0] 20 | 21 | print 'Working with parameters ', [par.name for par in params] 22 | print 'Testing on problems ', [prb.name for prb in problems] 23 | 24 | data = ModelData(DFO, problems, params) 25 | structure = ModelStructure(objective=avg_time, constraints=[]) # Unconstrained 26 | 27 | # Instantiate black-box solver. 28 | model = Model(modelData=data, modelStructure=structure) 29 | 30 | # Solve parameter optimization problem. 31 | NOMAD.solve(blackbox=model) 32 | -------------------------------------------------------------------------------- /examples/dfo/dfo_run.py: -------------------------------------------------------------------------------- 1 | # A sample gateway to DFO to be used by OPAL. 2 | 3 | import os 4 | import sys 5 | import pickle 6 | from string import atof 7 | from string import atoi 8 | from opal.core.io import * 9 | 10 | def write_specfile(parameter_file, loc='.', name='DFO.SPC'): 11 | "Write a valid DFO.SPC given a parameter file." 12 | # Read parameters into a dictionary 13 | parms = read_params_from_file(parameter_file) 14 | 15 | # Write out DFO.SPC. Parameters must be in order. 16 | f = open(os.path.join(loc, name), 'w') 17 | f.write('%10i\n' % parms['NX']) 18 | f.write('%10i\n' % parms['MAXIT']) 19 | f.write('%10i\n' % parms['MAXNF']) 20 | f.write('%10i\n' % parms['STPCRTR']) 21 | f.write('%10.3e\n' % float(parms['DELMIN'])) 22 | f.write('%10.3e\n' % float(parms['STPTHR'])) 23 | f.write('%10.3e\n' % float(parms['CNSTOL'])) 24 | f.write('%10.3e\n' % float(parms['DELTA'])) 25 | f.write('%10.3e\n' % float(parms['PP'])) 26 | f.write('%10i\n' % parms['SCALE']) 27 | f.write('%10i\n' % parms['IPRINT']) 28 | f.close() 29 | return 30 | 31 | 32 | def solve(problem_name): 33 | os.chdir(problem_name) 34 | os.system('./dfomin > run.log') 35 | 36 | ctime = 0.0 37 | f = open('cuter.log', 'r') 38 | for line in f.xreadlines(): 39 | line = line.strip('\n') 40 | if len(line) <= 0: 41 | continue 42 | stats = line.split(':') 43 | stat = stats[1] 44 | stat = stat.lstrip() 45 | stat = stat.rstrip() 46 | if stat == 'PTIME': 47 | ctime = ctime + atof(stats[-1]) 48 | if stat == 'STIME': 49 | ctime = ctime + atof(stats[-1]) 50 | if stat == 'FVAL': 51 | fval = atof(stats[-1]) 52 | if stat == 'NFEVAL': 53 | nfeval = atoi(stats[-1]) 54 | if stat == 'EXITCODE': 55 | exitcode = atoi(stats[-1]) 56 | if stat == 'FZERO': 57 | fzero = atof(stats[-1]) 58 | if stat == 'NCON': 59 | ncon = atof(stats[-1]) 60 | 61 | f.close() 62 | os.chdir('..') 63 | return {'EXITCODE' : exitcode, 64 | 'FVAL' : fval, 65 | 'CPU' : ctime, 66 | 'FEVAL' : nfeval*(ncon + 1), 67 | 'DESCENT' : fzero-fval} 68 | 69 | 70 | def compile_driver(problem_name, log_file='compile.log'): 71 | if not os.path.exists(problem_name): 72 | os.system('mkdir %s' % problem) 73 | os.chdir(problem_name) 74 | os.system('sifdecode %s > %s 2>&1' % (problem_name, log_file)) 75 | os.system('runcuter --package dfo --keep --blas none --lapack none ' +\ 76 | '> /dev/null') 77 | os.chdir('..') 78 | 79 | 80 | if __name__ == '__main__': 81 | param_file = sys.argv[1] 82 | problem = sys.argv[2] 83 | output_file = sys.argv[3] 84 | 85 | # Ensure executable is present for current problem. 86 | executable = os.path.join(problem, 'dfomin') 87 | if not os.path.exists(executable): 88 | compile_driver(problem) 89 | 90 | # Ensure spec file is in place and solve. 91 | write_specfile(param_file, loc=problem) 92 | measure_values = solve(problem) 93 | write_measures_to_file(output_file, measure_values) 94 | 95 | -------------------------------------------------------------------------------- /examples/fd/README.rst: -------------------------------------------------------------------------------- 1 | Example of algorithmic tuning 2 | ============================= 3 | 4 | This simple example demonstrates OPAL usage on the simple problem of adjusting 5 | the finite-difference parameter for the computation of an approximation to the 6 | first-order derivative of a noise-free function. 7 | 8 | The example uses the function f(x) = sin(x) and approximates the derivative 9 | using the formula:: 10 | 11 | f(x) ≠ (f(x+h) - f(x))/h 12 | 13 | where h > 0 is the finite-difference parameter. It is known that if f is 14 | continuously differentiable and noise free, an optimal value of h---i.e., 15 | minimizing the approximation and roundoff errors---is located around the square 16 | root of the machine epsilon. On platforms using IEEE double precision 17 | arithmetic, this value is around 1.0e-8. 18 | 19 | Below is the output of a run on a i386 Intel Core2 Duo running OSX 10.5.8:: 20 | 21 | MADS run{ 22 | 23 | EVAL BBE [ SOL, ] OBJ TIME \\ 24 | 25 | 1 1 [ 0.5 ] 0.2022210836 0 \\ 26 | 2 2 [ 0.4 ] 0.1582516709 0 \\ 27 | 3 3 [ 0.1 ] 0.03650380828 1 \\ 28 | 11 6 [ 0.075 ] 0.0271668032 2 \\ 29 | 15 8 [ 0.05 ] 0.01796857799 3 \\ 30 | 19 10 [ 0.025 ] 0.008912029073 3 \\ 31 | 27 12 [ 0.01875 ] 0.006670363173 4 \\ 32 | 31 14 [ 0.0125 ] 0.004437773933 5 \\ 33 | 35 16 [ 0.00625 ] 0.002214305049 6 \\ 34 | 43 18 [ 0.00380859375 ] 0.001348249081 7 \\ 35 | 47 20 [ 0.0013671875 ] 0.0004835939884 8 \\ 36 | 55 23 [ 0.0006286621094 ] 0.0002223121897 9 \\ 37 | 61 25 [ 0.0002380371094 ] 8.416550444e-05 9 \\ 38 | 67 27 [ 5.340576172e-05 ] 1.88821243e-05 10 \\ 39 | 75 30 [ 5.125999451e-06 ] 1.812326825e-06 11 \\ 40 | 87 35 [ 2.074893564e-06 ] 7.335931301e-07 13 \\ 41 | 93 37 [ 5.490146577e-07 ] 1.940868919e-07 14 \\ 42 | 101 40 [ 1.675449312e-07 ] 5.898094313e-08 15 \\ 43 | 109 43 [ 7.217749951e-08 ] 2.629330675e-08 16 \\ 44 | 115 45 [ 2.450397003e-08 ] 9.569121828e-09 17 \\ 45 | 123 49 [ 1.258558766e-08 ] 2.656297737e-09 19 \\ 46 | 134 57 [ 1.407570378e-08 ] 8.138155705e-10 22 \\ 47 | 157 78 [ 1.407861414e-08 ] 3.433888729e-10 30 \\ 48 | 170 87 [ 1.407861414e-08 ] 3.433888729e-10 33 \\ 49 | 50 | }end of run (mesh size reached NOMAD precision) 51 | 52 | The outcome of the run is that the final value of the stepsize found is 53 | h=1.407861414e-08 and the error value is 3.433888729e-10. The error is computed 54 | as the absolute value of the difference between the "exact" derivative and 55 | the approximation computed by finite differences. 56 | -------------------------------------------------------------------------------- /examples/fd/fd.py: -------------------------------------------------------------------------------- 1 | # Simple forward finite-difference approximation. 2 | 3 | def fd(f, x, h): 4 | # Return the forward finite-difference approximation to df(x). 5 | if h == 0.0: return float("infinity") 6 | return (f(x+h) - f(x))/h 7 | 8 | -------------------------------------------------------------------------------- /examples/fd/fd_declaration.py: -------------------------------------------------------------------------------- 1 | # Description of the forward finite-difference "algorithm". 2 | from opal.core.algorithm import Algorithm 3 | from opal.core.parameter import Parameter 4 | from opal.core.measure import Measure 5 | 6 | # Define Algorithm object. 7 | FD = Algorithm(name='FD', description='Forward Finite Differences') 8 | 9 | # Register executable for FD. 10 | FD.set_executable_command('python fd_run.py') 11 | 12 | # Register parameter file used by black-box solver to communicate with FD. 13 | #FD.set_parameter_file('fd.param') # Should be chosen automatically and hidden. 14 | 15 | # Define parameter and register it with algorithm. 16 | h = Parameter(kind='real', default=0.5, bound=(0, None), 17 | name='h', description='Step size') 18 | FD.add_param(h) 19 | 20 | # Define relevant measure and register with algorithm. 21 | error = Measure(kind='real', name='ERROR', description='Error in derivative') 22 | FD.add_measure(error) 23 | -------------------------------------------------------------------------------- /examples/fd/fd_optimize.py: -------------------------------------------------------------------------------- 1 | # Define a parameter optimization problem in relation to the 2 | # FD algorithm. 3 | from fd_declaration import FD 4 | 5 | from opal import ModelStructure, ModelData, Model 6 | 7 | from opal.Solvers import NOMAD 8 | 9 | # Return the error measure. 10 | def get_error(parameters, measures): 11 | return sum(measures["ERROR"]) 12 | 13 | # Define parameter optimization problem. 14 | data = ModelData(FD) 15 | struct = ModelStructure(objective=get_error) # Unconstrained 16 | model = Model(modelData=data, modelStructure=struct) 17 | 18 | # Solve parameter optimization problem. 19 | #NOMAD.set_parameter(name='DISPLAY_STATS', 20 | # value='%3dBBE %7.1eSOL %8.3eOBJ %5.2fTIME') 21 | NOMAD.solve(blackbox=model) 22 | 23 | # Inform user of expected optimal value for information. 24 | try: 25 | import numpy as np 26 | eps = np.finfo(np.double).eps 27 | except: 28 | # Approximate machine epsilon. 29 | eps = 1.0 30 | while 1+eps > 1: eps /= 2 31 | eps *= 2 32 | 33 | from math import sqrt 34 | print 'Expected optimal value is approximately %21.15e' % sqrt(eps) 35 | -------------------------------------------------------------------------------- /examples/fd/fd_run.py: -------------------------------------------------------------------------------- 1 | from opal.core.io import * 2 | from fd import fd 3 | from math import pi, sin, cos 4 | import sys 5 | 6 | f = sin ; df = cos # Target function and its derivative. 7 | x = pi/4 # This is where the derivative will be approximated. 8 | dfx = df(x) # "Exact" derivative at x. 9 | 10 | def run(param_file, problem): 11 | "Run FD with given parameters." 12 | 13 | params = read_params_from_file(param_file) 14 | h = params['h'] 15 | return {'ERROR': abs(dfx - fd(f,x,h))} 16 | 17 | 18 | if __name__ == '__main__': 19 | param_file = sys.argv[1] 20 | problem = sys.argv[2] 21 | output_file = sys.argv[3] 22 | 23 | # Solve, gather measures and write to file. 24 | measures = run(param_file, problem) 25 | write_measures_to_file(output_file, measures) 26 | 27 | -------------------------------------------------------------------------------- /examples/fd/test_fd.py: -------------------------------------------------------------------------------- 1 | # Define a parameter optimization problem in relation to the TRUNK solver. 2 | def test_fd_optimize(): 3 | import fd_optimize 4 | -------------------------------------------------------------------------------- /examples/ipopt/ipopt_composite_measures.py: -------------------------------------------------------------------------------- 1 | def avg_time(parameters, measures): 2 | n = len(measures['CPU']) 3 | if n == 0: 4 | return 0.0 5 | return (sum(measures["CPU"]) + 0.0)/(n + 0.0) 6 | 7 | def sum_eval(parameters, measures): 8 | return sum(measures['FEVAL']) + sum(measures['GEVAL']) +\ 9 | sum(measures['EQCVAL']) + sum(measures['EQJVAL']) +\ 10 | sum(measures['INCVAL']) + sum(measures['INJVAL']) 11 | 12 | def weighted_sum_eval(parameters, measures): 13 | result = 0.0 14 | for (feval, geval, eqcval, eqjval, incval, injval, weight) in \ 15 | zip(measures['FEVAL'], measures['GEVAL'], measures['EQCVAL'], \ 16 | measures['EQJVAL'], measures['INCVAL'], measures['INJVAL'], \ 17 | measures['WEIGHT']): 18 | result = result + weight*(feval + geval + eqcval + eqjval + incval + \ 19 | injval) 20 | return result 21 | 22 | def sum_ecode_square(parameters, measures): 23 | result = 0.0 24 | for val in measures['ECODE']: 25 | result = result + val*val 26 | return result 27 | 28 | 29 | def sum_ecode(parameters, measures): 30 | return sum(measures['ECODE']) 31 | 32 | def sum_unsolvability(parameters, measures): 33 | unsolvabilities = 0 34 | for eCode in measures['ECODE']: 35 | if eCode < 0: 36 | unsolvabilities = unsolvabilities + 1 37 | return unsolvabilities 38 | 39 | def sum_iteration(parameters, measures): 40 | return sum(measures['NITER']) 41 | 42 | -------------------------------------------------------------------------------- /examples/ipopt/ipopt_declaration.py: -------------------------------------------------------------------------------- 1 | from opal.core.algorithm import Algorithm 2 | from opal.core.parameter import Parameter 3 | from opal.core.measure import Measure 4 | 5 | # Define new algorithm. 6 | IPOPT = Algorithm(name='IPOPT', description='Interior Point for OPTimization') 7 | 8 | # Register executable for IPOPT. 9 | IPOPT.set_executable_command('python ipopt_run.py') 10 | 11 | # Define parameters. 12 | 13 | # 5. Line search parameters 14 | IPOPT.add_param(Parameter(name='tau_min', 15 | kind='real', 16 | bound=[0, 1], 17 | default=0.99, 18 | description='For fraction-to-boundary rule')) 19 | IPOPT.add_param(Parameter(name='s_theta', 20 | kind='real', 21 | bound=[0, None], 22 | default=1.1, 23 | description='Exponent for current constraint ' +\ 24 | 'violation')) 25 | IPOPT.add_param(Parameter(name='s_phi', 26 | kind='real', 27 | bound=[0, None], 28 | default=2.3, 29 | description='Exponent for linear barrier function ' +\ 30 | 'model')) 31 | IPOPT.add_param(Parameter(name='delta', 32 | kind='real', 33 | bound=[0, None], 34 | default=1.0, 35 | description='Multiplier for constraint violation')) 36 | IPOPT.add_param(Parameter(name='eta_phi', 37 | kind='real', 38 | bound=[0, None], 39 | default=1.0e-8, 40 | description='Multiplier for constraint violation')) 41 | IPOPT.add_param(Parameter(name='theta_min_fact', 42 | kind='real', 43 | bound=[0, None], 44 | default=1.0e-4, 45 | description='Factor for constraint violation ' +\ 46 | 'threshod')) 47 | IPOPT.add_param(Parameter(name='theta_max_fact', 48 | kind='real', 49 | bound=[0, None], 50 | default=1.0e4, 51 | description='Factor of upper bound for constraint '+\ 52 | 'violation')) 53 | IPOPT.add_param(Parameter(name='gamma_theta', 54 | kind='real', 55 | bound=[0, 1], 56 | default=1.0e-8, 57 | description='Relaxation factor in the filter ' +\ 58 | 'margin for the constraint violation')) 59 | IPOPT.add_param(Parameter(name='gamma_phi', 60 | kind='real', 61 | bound=[0, 1], 62 | default=1.0e-8, 63 | description='Relaxation factor in the filter ' +\ 64 | 'margin for barrier function')) 65 | IPOPT.add_param(Parameter(name='max_soc', 66 | kind='integer', 67 | bound=[0, None], 68 | default=4, 69 | description='Maximum number of iteration for ' +\ 70 | 'second order correction')) 71 | IPOPT.add_param(Parameter(name='kappa_soc', 72 | kind='real', 73 | bound=[0, 1], 74 | default=0.99, 75 | description='Maximum number of iteration for ' +\ 76 | 'second order correction')) 77 | 78 | # Define the feasible region. 79 | IPOPT.add_parameter_constraint('tau_min > 0') 80 | IPOPT.add_parameter_constraint('tau_min < 1') 81 | IPOPT.add_parameter_constraint('s_theta > 1') 82 | IPOPT.add_parameter_constraint('s_phi > 1') 83 | IPOPT.add_parameter_constraint('delta > 0') 84 | IPOPT.add_parameter_constraint('eta_phi > 0') 85 | IPOPT.add_parameter_constraint('eta_phi < 0.5') 86 | IPOPT.add_parameter_constraint('theta_min_fact > 0') 87 | IPOPT.add_parameter_constraint('theta_max_fact > 0') 88 | IPOPT.add_parameter_constraint('gamma_theta > 0') 89 | IPOPT.add_parameter_constraint('gamma_theta < 1') 90 | IPOPT.add_parameter_constraint('gamma_phi > 0') 91 | IPOPT.add_parameter_constraint('gamma_phi < 1') 92 | IPOPT.add_parameter_constraint('kappa_soc > 0') 93 | IPOPT.add_parameter_constraint('kappa_soc < 1') 94 | # Define and register measures. 95 | IPOPT.add_measure(Measure(name='CPU', 96 | kind='real', 97 | description='Computing time')) 98 | IPOPT.add_measure(Measure(name='FEVAL', 99 | kind='integer', 100 | description='Number of evaluation of objective function')) 101 | IPOPT.add_measure(Measure(name='EQCVAL', 102 | kind='integer', 103 | description='Number of evaluation of equality constraints')) 104 | IPOPT.add_measure(Measure(name='INCVAL', 105 | kind='integer', 106 | description='Number of evaluation of inequality constraints')) 107 | IPOPT.add_measure(Measure(name='GEVAL', 108 | kind='integer', 109 | description='Number of evaluation of function objective gradient')) 110 | IPOPT.add_measure(Measure(name='EQJVAL', 111 | kind='integer', 112 | description='Number of evaluation of equality constraint jacobian matrix')) 113 | IPOPT.add_measure(Measure(name='INJVAL', 114 | kind='integer', 115 | description='Number of evaluation of inequatily constraint jacobian matrix')) 116 | IPOPT.add_measure(Measure(name='ECODE', 117 | kind='integer', 118 | description='Exit code')) 119 | 120 | IPOPT.add_measure(Measure(name='NITER', 121 | kind='integer', 122 | description='Number of iteration')) 123 | 124 | IPOPT.add_measure(Measure(name='WEIGHT', 125 | kind='real', 126 | description='weight for a measure vector of a problem')) 127 | -------------------------------------------------------------------------------- /examples/ipopt/ipopt_optimize.py: -------------------------------------------------------------------------------- 1 | from opal import ModelStructure, ModelData, Model, MeasureFunction 2 | from ipopt_declaration import IPOPT 3 | 4 | import sys 5 | 6 | # Choose all parameters defined in declaration file 7 | params = [param for param in IPOPT.parameters 8 | if param.name in ['tau_min', 's_theta', 's_phi', 'delta', 9 | 'max_soc', 'kappa_soc']] 10 | 11 | # Choose all solvable problems 12 | #from ipopt_test_problems import ipopt_solvable_problems as problems 13 | #from ipopt_test_problems import test_problems as problems 14 | from ipopt_test_problems import CUTEr_constrained_problems, test_problems 15 | 16 | if len(sys.argv) > 1: 17 | f = open(sys.argv[1]) 18 | representativeProblems = eval(f.read()) 19 | f.close() 20 | 21 | problems = [ prob for prob in CUTEr_constrained_problems 22 | if prob.name in representativeProblems ] 23 | else: 24 | problems = CUTEr_constrained_problems 25 | problems = test_problems 26 | 27 | data = ModelData(IPOPT, problems, params) 28 | 29 | from ipopt_composite_measures import sum_eval, sum_unsolvability 30 | structure = ModelStructure(objective=MeasureFunction(sum_eval, addivity=1), 31 | constraints=[(None, sum_unsolvability, 0)]) 32 | 33 | # Instantiate black-box solver. 34 | #from opal.Platforms import LSF 35 | #LSF.set_parameter({'MAX_TASK':10}) 36 | 37 | from opal.Platforms import SunGrid 38 | SunGrid.set_config('-q','all.q') 39 | SunGrid.set_parameter({'MAX_TASK':10}) 40 | 41 | model = Model(modelData=data, modelStructure=structure, platform=SunGrid) 42 | 43 | if len(sys.argv) > 2: # The initial point is provided by external file 44 | f = open(sys.argv[2]) 45 | paramValues = eval(f.read()) 46 | f.close() 47 | model.initial_points = [] # Reset the initial point set 48 | for tag in paramValues: 49 | model.add_initial_point(paramValues[tag]) 50 | #print model.get_initial_points() 51 | 52 | # Solve parameter optimization problem. 53 | from opal.Solvers import NOMAD 54 | #NOMAD.set_parameter(name='MAX_BB_EVAL', value=100) 55 | #NOMAD.set_parameter(name='SCALING', value='scaling.txt') 56 | NOMAD.set_parameter(name='INITIAL_MESH_SIZE', 57 | value='(0.05 5 5 1 4 0.05)') 58 | #NOMAD.set_parameter(name='MIN_MESH_SIZE', value=1.0e-6) 59 | NOMAD.set_parameter(name='MAX_MESH_INDEX', value=6) 60 | NOMAD.set_parameter(name='DISPLAY_DEGREE', value=4) 61 | NOMAD.solve(blackbox=model) 62 | -------------------------------------------------------------------------------- /examples/ipopt/ipopt_run.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import pickle 4 | import re 5 | import tempfile 6 | import shutil 7 | from string import atof 8 | from string import atoi 9 | from opal.core.io import * 10 | 11 | 12 | 13 | def extract_measure(content, description, name, 14 | valueType = 'int',dem='\s+:\s+'): 15 | numberPattern = {'int':'-?\d+', 16 | 'real':'-?\d*\.\d+', 17 | 'float':'-?\d*\.\d+E(\+|-)\d+'} 18 | converters = {'int':int, 19 | 'real': float, 20 | 'float': float} 21 | 22 | matches = re.finditer(description + dem + '(?P<' + 23 | name + '>' + 24 | numberPattern[valueType] + ')', 25 | content) 26 | value = None 27 | for m in matches: 28 | if name not in m.groupdict().keys(): 29 | raise Exception('Could not to extract measure ' + name) 30 | continue 31 | value = converters[valueType](m.groupdict()[name]) 32 | return value 33 | 34 | def write_specfile(parameter_file, loc='.', name='ipopt.opt'): 35 | # Read parameters into a dictionary 36 | #parms = read_params_from_file(parameter_file) 37 | parms = {} 38 | 39 | # Write out ipopt.opt 40 | f = open(os.path.join(loc, name), 'w') 41 | for pName in parms: 42 | f.write( pName + ' ' + str(parms[pName]) + '\n') 43 | # Fix parameters 44 | f.write('linear_solver ma57\n') 45 | f.write('print_level 3\n') 46 | f.write('print_timing_statistics yes\n') 47 | f.close() 48 | return 49 | 50 | def get_measures(stats_file, weight=1.0): 51 | 52 | measures = {} 53 | f = open(stats_file) 54 | content = f.read() 55 | f.close() 56 | #print content 57 | measures['CPU'] = extract_measure( 58 | content=content, 59 | description='OverallAlgorithm', 60 | dem='\.*:\s+', 61 | name='CPU', 62 | valueType='real') 63 | measures['HTIME'] = extract_measure( 64 | content=content, 65 | description=' UpdateHessian', 66 | dem='\.*:\s+', 67 | name='HTIME', 68 | valueType='real') 69 | measures['DIRTIME'] = extract_measure( 70 | content=content, 71 | description=' ComputeSearchDirection', 72 | dem='\.*:\s+', 73 | name='DIRTIME', 74 | valueType='real') 75 | measures['PTRTIME'] = extract_measure( 76 | content=content, 77 | description=' ComputeAcceptableTrialPoint', 78 | dem='\.*:\s+', 79 | name='PTRTIME', 80 | valueType='real') 81 | measures['FTIME'] = extract_measure( 82 | content=content, 83 | description='Function Evaluations', 84 | dem='\.*:\s+', 85 | name='FTIME', 86 | valueType='real') 87 | measures['NITER'] = extract_measure( 88 | content=content, 89 | description='Number of Iterations', 90 | dem='\.+:\s+', 91 | name='NITER', 92 | valueType='int') 93 | 94 | measures['FEVAL'] = extract_measure( 95 | content=content, 96 | description='Number of objective function evaluations', 97 | dem='\s*=\s+', 98 | name='FEVAL', 99 | valueType='int') 100 | measures['GEVAL'] = extract_measure( 101 | content=content, 102 | description='Number of objective gradient evaluations', 103 | dem='\s*=\s+', 104 | name='GEVAL', 105 | valueType='int') 106 | measures['EQCVAL'] = extract_measure( 107 | content=content, 108 | description='Number of equality constraint evaluations', 109 | dem='\s*=\s+', 110 | name='EQCVAL', 111 | valueType='int') 112 | measures['INCVAL'] = extract_measure( 113 | content=content, 114 | description='Number of inequality constraint evaluations', 115 | dem='\s*=\s+', 116 | name='INCVAL', 117 | valueType='int') 118 | measures['EQJVAL'] = extract_measure( 119 | content=content, 120 | description='Number of equality constraint Jacobian evaluations', 121 | dem='\s*=\s+', 122 | name='EQJVAL', 123 | valueType='int') 124 | measures['INJVAL'] = extract_measure( 125 | content=content, 126 | description='Number of inequality constraint Jacobian evaluations', 127 | dem='\s*=\s+', 128 | name='INJVAL', 129 | valueType='int') 130 | measures['ECODE'] = extract_measure( 131 | content=content, 132 | description='Exit code', 133 | dem='\s*=\s+', 134 | name='ECODE', 135 | valueType='int') 136 | 137 | measures['WEIGHT'] = weight 138 | return measures 139 | 140 | 141 | def run(param_file, problem, keep=False): 142 | curDir = os.getcwd() 143 | if not keep: 144 | workDir = tempfile.mkdtemp() 145 | else: 146 | workDir = os.path.join(curDir,problem) 147 | if not os.path.exists(workDir): 148 | os.mkdir(workDir) 149 | 150 | os.chdir(workDir) 151 | write_specfile(param_file) 152 | os.system('sifdecode ' + problem + ' > /dev/null') 153 | os.system('runcuter -p ipopt > ' + problem + '.sol') 154 | measures = get_measures(problem + '.sol') 155 | os.chdir(curDir) 156 | if not keep: 157 | shutil.rmtree(workDir) 158 | return measures 159 | 160 | if __name__ == '__main__': 161 | param_file = sys.argv[1] 162 | problem = sys.argv[2] 163 | output_file = sys.argv[3] 164 | 165 | # Solve, gather measures and write to file. 166 | param_file = os.path.abspath(param_file) 167 | #measures = run(param_file, problem, keep=True) 168 | measures = run(param_file, problem) 169 | write_measures_to_file(output_file, measures) 170 | 171 | 172 | -------------------------------------------------------------------------------- /examples/test_examples.py: -------------------------------------------------------------------------------- 1 | 2 | def test_examples(): 3 | import os 4 | dirList = os.listdir('.') 5 | print 'We have ' + str(len(dirList)) + ' examples' 6 | 7 | -------------------------------------------------------------------------------- /examples/trunk/test_trunk.py: -------------------------------------------------------------------------------- 1 | # Define a parameter optimization problem in relation to the TRUNK solver. 2 | def test_trunk_optimize(): 3 | from trunk_optimize import prob 4 | from opal.Solvers import NOMAD 5 | 6 | NOMAD.set_parameter(name='MAX_BB_EVAL', value=10) 7 | NOMAD.solve(blackbox=prob) 8 | 9 | if __name__ == '__main__': 10 | test_trunk_optimize() 11 | 12 | -------------------------------------------------------------------------------- /examples/trunk/trunk_declaration.py: -------------------------------------------------------------------------------- 1 | # Description of TRUNK. 2 | from opal.core.algorithm import Algorithm 3 | from opal.core.parameter import Parameter 4 | from opal.core.parameter import ParameterConstraint 5 | from opal.core.measure import Measure 6 | 7 | # Define Algorithm object. 8 | trunk = Algorithm(name='TRUNK', 9 | description='Trust Region for UNConstrained problems') 10 | 11 | # Register executable command. 12 | trunk.set_executable_command('python trunk_run.py') 13 | 14 | # Register parameters. 15 | trunk.add_param(Parameter(name='eta1', 16 | kind='real', 17 | default=0.25, 18 | bound=[0, 1], 19 | description='Gradient scaling cut-off')) 20 | trunk.add_param(Parameter(name='eta2', 21 | kind='real', 22 | default=0.75, 23 | bound=[0,1], 24 | description='Trust-region increase threashold')) 25 | trunk.add_param(Parameter(name='gamma1', 26 | kind='real', 27 | default=0.5, 28 | bound=[0,1], 29 | description='Trust-region shrink factor')) 30 | trunk.add_param(Parameter(name='gamma2', 31 | kind='real', 32 | default=1.000, 33 | bound=[0,1], 34 | description='Trust-region moderate shrink factor')) 35 | trunk.add_param(Parameter(name='gamma3', 36 | kind='real', 37 | default=2.000, 38 | bound=[1,None], 39 | description='Trust-region increase factor')) 40 | trunk.add_param(Parameter(name='maxit', 41 | kind='integer', 42 | default=1000, 43 | description='Maximum number of iterations')) 44 | trunk.add_param(Parameter(name='stptol', 45 | kind='real', 46 | default=1.0e-05, 47 | description='Stop tolerance')) 48 | trunk.add_param(Parameter(name='rdgrad', 49 | kind='real', 50 | default=1.0e-05, 51 | description='Relative decrease in gradient')) 52 | trunk.add_param(Parameter(name='delta0', 53 | kind='real', 54 | default=1.000, 55 | description='Initial trust-region radius')) 56 | trunk.add_param(Parameter(name='sband', 57 | kind='integer', 58 | default=5, 59 | description='Semi-bandwidth of band preconditioner')) 60 | trunk.add_param(Parameter(name='nmmem', 61 | kind='integer', 62 | default=10, 63 | description='Non-monotone algorithm memory')) 64 | trunk.add_param(Parameter(name='level', 65 | kind='real', 66 | default=0.1000, 67 | description='Level, used for interpolation')) 68 | 69 | # Register constraints on the parameters. 70 | 71 | trunk.add_parameter_constraint(ParameterConstraint('eta1 < eta2')) 72 | trunk.add_parameter_constraint(ParameterConstraint('eta1 > 0')) 73 | trunk.add_parameter_constraint(ParameterConstraint('eta2 < 1')) 74 | trunk.add_parameter_constraint(ParameterConstraint('gamma1 > 0')) 75 | trunk.add_parameter_constraint(ParameterConstraint('gamma1 <= gamma2')) 76 | #trunk.add_parameter_constraint(ParameterConstraint('gamma2 < 1')) 77 | trunk.add_parameter_constraint(ParameterConstraint('gamma3 > 1')) 78 | 79 | # Register atomic measures. 80 | trunk.add_measure(Measure(name='CPU', 81 | kind='real', 82 | description='Computing time')) 83 | trunk.add_measure(Measure(name='FEVAL', 84 | kind='integer', 85 | description='Number of function evaluations')) 86 | trunk.add_measure(Measure(name='GEVAL', 87 | kind='integer', 88 | description='Number of gradient evaluations')) 89 | trunk.add_measure(Measure(name='NITER', 90 | kind='integer', 91 | description='Number of iterations')) 92 | trunk.add_measure(Measure(name='CGITER', 93 | kind='integer', 94 | description='Number of CG iterations')) 95 | trunk.add_measure(Measure(name='RDGRAD', 96 | kind='real', 97 | description='Relative decrease in gradient achieved')) 98 | trunk.add_measure(Measure(name='FVAL', 99 | kind='real', 100 | description='Final objective function value')) 101 | trunk.add_measure(Measure(name='HEVAL', 102 | kind='integer', 103 | description='Number of Hessian matrix evaluations')) 104 | trunk.add_measure(Measure(name='FVAL0', 105 | kind='real', 106 | description='Initial objective function value')) 107 | trunk.add_measure(Measure(name='GNORM', 108 | kind='real', 109 | description='Gradient 2-norm at final point')) 110 | trunk.add_measure(Measure(name='DELTA', 111 | kind='real', 112 | description='Final trust-region radius')) 113 | trunk.add_measure(Measure(name='ECODE', 114 | kind='integer', 115 | description='Exit code')) 116 | 117 | -------------------------------------------------------------------------------- /examples/trunk/trunk_optimize.py: -------------------------------------------------------------------------------- 1 | # Define a parameter optimization problem in relation to the TRUNK solver. 2 | from trunk_declaration import trunk 3 | 4 | from opal import ModelStructure, ModelData, Model, MeasureFunction 5 | from opal.Solvers import NOMAD 6 | from opal.Platforms import SMP 7 | from opal.TestProblemCollections import CUTEr 8 | 9 | def sum_heval(parameters, measures): 10 | val = sum(measures['HEVAL']) 11 | return val 12 | 13 | def get_error(parameters, measures): 14 | val = sum(measures["ECODE"]) 15 | return val 16 | 17 | # Parameters being tuned and problem list. 18 | par_names = ['eta1', 'eta2', 'gamma1', 'gamma2', 'gamma3'] 19 | #params = [param for param in trunk.parameters if param.name in par_names] 20 | 21 | params = [trunk.parameters['eta1'], 22 | trunk.parameters['eta2'], 23 | trunk.parameters['gamma1'], 24 | trunk.parameters['gamma2'], 25 | trunk.parameters['gamma3'] 26 | ] 27 | problems = [problem for problem in CUTEr if problem.name in ['BDQRTIC', 28 | 'BROYDN7D', 29 | 'BRYBND', 30 | 'CURLY10', 31 | # 'CURLY20', 32 | # 'CURLY30', 33 | 'CRAGGLVY', 34 | 'DIXON3DQ', 35 | 'EIGENALS', 36 | 'FMINSRF2', 37 | 'FMINSURF', 38 | 'GENROSE', 39 | 'HIELOW', 40 | 'MANCINO', 41 | 'NCB20', 42 | 'NCB20B', 43 | 'NONDQUAR', 44 | 'POWER', 45 | 'SENSORS', 46 | 'SINQUAD', 47 | 'TESTQUAD', 48 | 'TRIDIA', 49 | 'WOODS']] 50 | 51 | #SMP.set_parameter(name='MAX_PROC', value=5); 52 | 53 | # Define parameter optimization problem. 54 | data = ModelData(algorithm=trunk, 55 | problems=problems, 56 | parameters=params, 57 | measures=trunk.measures) 58 | 59 | struct = ModelStructure(objective=MeasureFunction(sum_heval, 60 | additivity=1), 61 | constraints=[(None, 62 | MeasureFunction(get_error, addivity=1), 63 | 0)]) # One constraint get_error(p) <= 0 64 | 65 | 66 | prob = Model(modelData=data, 67 | modelStructure=struct, 68 | platform='SMP', 69 | synchoronized=False, 70 | interruptible=True 71 | ) 72 | 73 | # Solve parameter optimization problem. 74 | 75 | if __name__ == '__main__': 76 | from opal.Solvers import NOMAD 77 | NOMAD.set_parameter(name='MAX_BB_EVAL', value=10) 78 | NOMAD.solve(model=prob) 79 | 80 | -------------------------------------------------------------------------------- /examples/trunk/trunk_optimize_lsf.py: -------------------------------------------------------------------------------- 1 | # Define a parameter optimization problem in relation to the TRUNK solver. 2 | # This is a parallel version in which the test problems are placed in a 3 | # LSF queue to be solved in parallel. This strategy effectively 4 | # parallelizes the black box. This is an alternative to the parallelization 5 | # implemented in trunk_optimize_opalmpi.py. 6 | from trunk_declaration import trunk 7 | from opal import ModelStructure, ModelData, Model 8 | from opal.Solvers import NOMAD 9 | 10 | from opal.TestProblemCollections import CUTEr 11 | from opal.Platforms import LSF 12 | 13 | def get_error(parameters, measures): 14 | val = sum(measures["FEVAL"]) 15 | return val 16 | 17 | # Parameters being tuned and problem list. 18 | par_names = ['eta1', 'eta2', 'gamma1', 'gamma2', 'gamma3'] 19 | params = [param for param in trunk.parameters if param.name in par_names] 20 | 21 | problems = [problem for problem in CUTEr if problem.name in ['BDQRTIC', 22 | 'BROYDN7D', 23 | 'BRYBND', 24 | 'CURLY10', 25 | 'CURLY20', 26 | 'CURLY30', 27 | 'CRAGGLVY', 28 | 'DIXON3DQ', 29 | 'EIGENALS', 30 | 'FMINSRF2', 31 | 'FMINSURF', 32 | 'GENROSE', 33 | 'HIELOW', 34 | 'MANCINO', 35 | 'NCB20', 36 | 'NCB20B', 37 | 'NONDQUAR', 38 | 'NONDQUAR', 39 | 'POWER', 40 | 'SENSORS', 41 | 'SINQUAD', 42 | 'TESTQUAD', 43 | 'TRIDIA', 44 | 'WOODS']] 45 | 46 | # Define parameter optimization problem. 47 | LSF.set_config(parameterName="-m", 48 | parameterValue='"lin01 lin02 lin03 lin04"') 49 | LSF.set_config(parameterName="-q", 50 | parameterValue="fedora") 51 | data = ModelData(algorithm=trunk, 52 | problems=problems, 53 | parameters=params, 54 | platform=LSF) 55 | struct = ModelStructure(objective=get_error, 56 | constraints=[]) # Unconstrained 57 | blackbox = Model(modelData=data, modelStructure=struct) 58 | 59 | # Solve parameter optimization problem. 60 | NOMAD.set_parameter(name='MAX_BB_EVAL', value=10) 61 | NOMAD.solve(model=blackbox) 62 | -------------------------------------------------------------------------------- /examples/trunk/trunk_optimize_mpi.py: -------------------------------------------------------------------------------- 1 | # Define a parameter optimization problem in relation to the TRUNK solver. 2 | # This is a parallel version in which the black box solver evaluates the 3 | # worth of neighbors in parallel. The black box evaluation itself is 4 | # sequential. 5 | from trunk_declaration import trunk 6 | from opal import ModelStructure, ModelData, Model 7 | from opal.Solvers import NOMADMPI 8 | 9 | from opal.TestProblemCollections import CUTEr 10 | 11 | 12 | def get_error(parameters, measures): 13 | val = sum(measures["FEVAL"]) 14 | return val 15 | 16 | # Parameters being tuned and problem list. 17 | par_names = ['eta1', 'eta2', 'gamma1', 'gamma2', 'gamma3'] 18 | params = [param for param in trunk.parameters if param.name in par_names] 19 | 20 | problems = [problem for problem in CUTEr if problem.name in ['BDQRTIC', 21 | 'BROYDN7D', 22 | 'BRYBND', 23 | 'CURLY10', 24 | 'CURLY20', 25 | 'CURLY30', 26 | 'CRAGGLVY', 27 | 'DIXON3DQ', 28 | 'EIGENALS', 29 | 'FMINSRF2', 30 | 'FMINSURF', 31 | 'GENROSE', 32 | 'HIELOW', 33 | 'MANCINO', 34 | 'NCB20', 35 | 'NCB20B', 36 | 'NONDQUAR', 37 | 'POWER', 38 | 'SENSORS', 39 | 'SINQUAD', 40 | 'TESTQUAD', 41 | 'TRIDIA', 42 | 'WOODS']] 43 | 44 | # Define parameter optimization problem. 45 | data = ModelData(algorithm=trunk, 46 | problems=problems, 47 | parameters=params) 48 | struct = ModelStructure(objective=get_error, 49 | constraints=[]) # Unconstrained 50 | blackbox = Model(modelData=data, modelStructure=struct) 51 | 52 | # Solve parameter optimization problem. 53 | NOMADMPI.set_mpi_config(name='np', value=8) 54 | NOMADMPI.set_mpi_config(name='-host', value='lin01,lin02,lin03,lin04') 55 | NOMADMPI.set_parameter(name='MAX_BB_EVAL', value=50) 56 | NOMADMPI.set_parameter(name='DISPLAY_DEGREE', value=2) 57 | NOMADMPI.solve(model=blackbox) 58 | -------------------------------------------------------------------------------- /examples/trunk/trunk_optimize_opalmpi.py: -------------------------------------------------------------------------------- 1 | # Define a parameter optimization problem in relation to the TRUNK solver. 2 | # This is a parallel version in which the test problems are solved in 3 | # parallel. This parallelism is managed via MPI. This is an alternative to 4 | # the parallelization implemented in trunk_optimize_lsf.py. 5 | from trunk_declaration import trunk 6 | from opal import ModelStructure, ModelData, Model 7 | from opal.Solvers import NOMAD 8 | 9 | from opal.TestProblemCollections import CUTEr 10 | from opal.Platforms import OPALMPI 11 | 12 | def get_error(parameters, measures): 13 | val = sum(measures["FEVAL"]) 14 | return val 15 | 16 | # Parameters being tuned and problem list. 17 | par_names = ['eta1', 'eta2', 'gamma1', 'gamma2', 'gamma3'] 18 | params = [param for param in trunk.parameters if param.name in par_names] 19 | 20 | problems = [problem for problem in CUTEr if problem.name in ['BDQRTIC', 21 | 'BROYDN7D', 22 | 'BRYBND', 23 | 'CURLY10', 24 | 'CURLY20', 25 | 'CURLY30', 26 | 'CRAGGLVY', 27 | 'DIXON3DQ', 28 | 'EIGENALS', 29 | 'FMINSRF2', 30 | 'FMINSURF', 31 | 'GENROSE', 32 | 'HIELOW', 33 | 'MANCINO', 34 | 'NCB20', 35 | 'NCB20B', 36 | 'NONDQUAR', 37 | 'POWER', 38 | 'SENSORS', 39 | 'SINQUAD', 40 | 'TESTQUAD', 41 | 'TRIDIA', 42 | 'WOODS']] 43 | 44 | 45 | # Define parameter optimization problem. 46 | 47 | data = ModelData(algorithm=trunk, 48 | problems=problems, 49 | parameters=params, 50 | platform=OPALMPI) 51 | struct = ModelStructure(objective=get_error, 52 | constraints=[]) # Unconstrained 53 | model = Model(modelData=data, modelStructure=struct) 54 | 55 | # Solve parameter optimization problem. 56 | NOMAD.set_parameter(name='MAX_BB_EVAL', value=500) 57 | NOMAD.solve(blackbox=model) 58 | -------------------------------------------------------------------------------- /examples/trunk/trunk_optimize_opalsmp_nomadmpi.py: -------------------------------------------------------------------------------- 1 | # Define a parameter optimization problem in relation to the TRUNK solver. 2 | # This is a parallel version in which the test problems are solved in 3 | # parallel in independent subprocesses. This strategy effectively 4 | # parallelizes the black box. This is an alternative to the parallelization 5 | # implemented in trunk_optimize_opalmpi.py. 6 | from trunk_declaration import trunk 7 | from opal import ModelStructure, ModelData, Model 8 | from opal.Solvers import NOMADMPI 9 | from opal.TestProblemCollections import CUTEr 10 | from opal.Platforms import SMP 11 | 12 | def get_error(parameters, measures): 13 | val = sum(measures["FEVAL"]) 14 | return val 15 | 16 | # Parameters being tuned and problem list. 17 | par_names = ['eta1', 'eta2', 'gamma1', 'gamma2', 'gamma3'] 18 | params = [param for param in trunk.parameters if param.name in par_names] 19 | 20 | problems = [problem for problem in CUTEr if problem.name in ['BDQRTIC', 21 | 'BROYDN7D', 22 | 'BRYBND', 23 | 'CURLY10', 24 | 'CURLY20', 25 | 'CURLY30', 26 | 'CRAGGLVY', 27 | 'DIXON3DQ', 28 | 'EIGENALS', 29 | 'FMINSRF2', 30 | 'FMINSURF', 31 | 'GENROSE', 32 | 'HIELOW', 33 | 'MANCINO', 34 | 'NCB20', 35 | 'NCB20B', 36 | 'NONDQUAR', 37 | 'POWER', 38 | 'SENSORS', 39 | 'SINQUAD', 40 | 'TESTQUAD', 41 | 'TRIDIA', 42 | 'WOODS']] 43 | 44 | 45 | # Define parameter optimization problem. 46 | 47 | data = ModelData(algorithm=trunk, 48 | problems=problems, 49 | parameters=params) 50 | struct = ModelStructure(objective=get_error, 51 | constraints=[]) # Unconstrained 52 | model = Model(modelData=data, modelStructure=struct, platform=SMP) 53 | 54 | # Solve parameter optimization problem. 55 | NOMADMPI.set_mpi_config(name='np', value=5) 56 | NOMADMPI.set_parameter(name='MAX_BB_EVAL', value=5) 57 | NOMADMPI.set_parameter(name='DISPLAY_DEGREE', value=2) 58 | NOMADMPI.solve(blackbox=model) 59 | -------------------------------------------------------------------------------- /examples/trunk/trunk_optimize_surrogate.py: -------------------------------------------------------------------------------- 1 | # Define a parameter optimization problem in relation to the TRUNK solver. 2 | # This is the sequential version. 3 | from trunk_declaration import trunk 4 | from opal import ModelStructure, ModelData, Model 5 | from opal.Solvers import NOMAD 6 | from opal.TestProblemCollections import CUTEr 7 | 8 | def sum_heval(parameters, measures): 9 | val = sum(measures["HEVAL"]) 10 | return val 11 | 12 | def get_error(parameters,measures): 13 | val = sum(measures['ECODE']) 14 | return val 15 | 16 | # Parameters being tuned and problem list. 17 | par_names = ['eta1', 'eta2', 'gamma1', 'gamma2', 'gamma3'] 18 | params = [param for param in trunk.parameters if param.name in par_names] 19 | 20 | problems = [problem for problem in CUTEr if problem.name in ['BDQRTIC', 21 | 'BROYDN7D', 22 | 'BRYBND', 23 | # 'CURLY10', 24 | # 'CURLY20', 25 | # 'CURLY30', 26 | # 'CRAGGLVY', 27 | # 'DIXON3DQ', 28 | # 'EIGENALS', 29 | # 'FMINSRF2', 30 | # 'FMINSURF', 31 | # 'GENROSE', 32 | 'HIELOW', 33 | # 'MANCINO', 34 | # 'NCB20', 35 | # 'NCB20B', 36 | # 'NONDQUAR', 37 | 'POWER', 38 | 'SENSORS', 39 | 'SINQUAD', 40 | 'TESTQUAD', 41 | 'TRIDIA', 42 | 'WOODS']] 43 | 44 | # Define parameter optimization problem. 45 | data = ModelData(algorithm=trunk, 46 | problems=problems, 47 | parameters=params) 48 | struct = ModelStructure(objective=sum_heval, 49 | constraints=[(None,get_error, 0)]) 50 | model = Model(modelData=data, modelStructure=struct) 51 | 52 | # Define a surrogate 53 | 54 | surr_data = ModelData(algorithm=trunk, 55 | problems= [problem for problem in CUTEr \ 56 | if problem.name in ['BDQRTIC', 57 | 'BROYDN7D', 58 | 'BRYBND']], 59 | parameters=params) 60 | surr_struct = ModelStructure(objective=sum_heval, 61 | constraints=[]) 62 | surr_model = Model(modelData=surr_data, modelStructure=surr_struct, 63 | dataFile='surrogate.dat') 64 | 65 | # Solve parameter optimization problem. 66 | NOMAD.set_parameter(name='MAX_BB_EVAL', value=10) 67 | NOMAD.solve(blackbox=model, surrogate=surr_model) 68 | -------------------------------------------------------------------------------- /examples/trunk/trunk_run.py: -------------------------------------------------------------------------------- 1 | from opal.core.io import * 2 | from opal.core.tools import * 3 | 4 | from distutils.dir_util import copy_tree 5 | import tempfile 6 | import os 7 | import sys 8 | import shutil 9 | import re 10 | 11 | src = 'trunk.src' 12 | 13 | def get_measures(stats_file): 14 | 15 | measures = {} 16 | f = open(stats_file) 17 | content = f.read() 18 | f.close() 19 | measures['CPU'] = extract_measure(content=content, 20 | description='user', 21 | name='CPU', 22 | valueType='real') 23 | measures['NITER'] = extract_measure(content=content, 24 | description='\#iterations\.*', 25 | name='NITER', 26 | valueType='int') 27 | measures['CGITER'] = extract_measure(content=content, 28 | description='CG iterations\.*', 29 | name='CGITER', 30 | valueType='int') 31 | measures['FEVAL'] = extract_measure(content=content, 32 | description='Function evals\.*', 33 | name='FEVAL', 34 | valueType='int') 35 | measures['GEVAL'] = extract_measure(content=content, 36 | description='Gradient evals\.*', 37 | name='GEVAL', 38 | valueType='int') 39 | measures['HEVAL'] = extract_measure( 40 | content=content, 41 | description='Hessian-vector products\.*', 42 | name='HEVAL', 43 | valueType='int') 44 | measures['GNORM'] = extract_measure(content=content, 45 | description='Gradient 2-norm\.*', 46 | name='GNORM', 47 | valueType='float') 48 | measures['RDGRAD'] = extract_measure( 49 | content=content, 50 | description='Relative decrease in gradient\.*', 51 | name='RDGRAD', 52 | valueType='float') 53 | measures['DELTA'] = extract_measure( 54 | content=content, 55 | description='Final Trust-Region radius\.*', 56 | name='DELTA', 57 | valueType='float') 58 | measures['ECODE'] = extract_measure(content=content, 59 | description='Exit code\.*', 60 | name='ECODE', 61 | valueType='int') 62 | match = re.search('objective value\.*\s:\s+' + \ 63 | '(?P-?\d*\.\d+E(\+|-)\d+)/\s+' + \ 64 | '(?P-?\d*\.\d+E(\+|-)\d+)', 65 | content) 66 | if match is not None: 67 | measures['FVAL0'] = float(match.groupdict()['FVAL0']) 68 | measures['FVAL'] = float(match.groupdict()['FVAL']) 69 | return measures 70 | 71 | 72 | def write_specfile(param_file): 73 | params = read_params_from_file(param_file) 74 | f = open('trunk.spc','w') 75 | f.write('{0:<10d} ## Max. number of iterations : (I4)\n'\ 76 | .format(params['maxit'])) 77 | f.write('{0:<10.2e} ## Stopping tolerance : (D7.2)\n'\ 78 | .format(params['stptol'])) 79 | f.write('{0:<10.2e} ## Relative decrease in gradient : (D7.2)\n'\ 80 | .format(params['rdgrad'])) 81 | f.write('{0:<10.4f} \n'\ 82 | .format(params['eta1'])) 83 | f.write('{0:<10.4f} \n'\ 84 | .format(params['eta2'])) 85 | f.write('{0:<10.4f} \n'\ 86 | .format(params['gamma1'])) 87 | f.write('{0:<10.4f} \n'\ 88 | .format(params['gamma2'])) 89 | f.write('{0:<10.4f} \n'\ 90 | .format(params['gamma3'])) 91 | f.write('{0:<10.4f} ## Initial trust-region radius Delta : (F6.4)\n'\ 92 | .format(params['delta0'])) 93 | f.write("{0:<10} ## Use Powell's rule for Delta : (L1)\n"\ 94 | .format('F')) 95 | f.write('{0:<10} ## Use banded preconditioner : (L1)\n'\ 96 | .format('F')) 97 | f.write('{0:<10d} ## Semi-bandwidth if preconditioner : (I4)\n'\ 98 | .format(params['sband'])) 99 | f.write('{0:<10} ## Use a non-monotone strategy : (L1)\n'\ 100 | .format('F')) 101 | f.write('{0:<10d} ## Non-monotone algorithm memory : (I4)\n'\ 102 | .format(params['nmmem'])) 103 | f.write('{0:<10} ## Update Delta using polynomials : (L1)\n'\ 104 | .format('F')) 105 | f.write('{0:<10} ## If so, interpolate rho? : (L1)\n'\ 106 | .format('F')) 107 | f.write('{0:<10.4f} ## Level, used for interpolation : (F6.4)\n'\ 108 | .format(params['level'])) 109 | f.write('{0:<10} ## Reentry of GLTR upon rejection : (L1)\n'\ 110 | .format('F')) 111 | f.write('{0:<10} ## Display results as Trunk proceeds : (L1)\n'\ 112 | .format('F')) 113 | f.close() 114 | return 115 | 116 | 117 | def run(param_file, problem): 118 | curDir = os.getcwd() 119 | workDir = tempfile.mkdtemp() 120 | copy_tree(src, workDir) 121 | os.chdir(workDir) 122 | write_specfile(param_file) 123 | os.system('sifdecode ' + problem + ' > /dev/null') 124 | os.system('make trunkd > /dev/null') 125 | os.system('./trunkd > ' + problem + '.sol') 126 | measures = get_measures(problem + '.sol') 127 | os.chdir(curDir) 128 | shutil.rmtree(workDir) 129 | return measures 130 | 131 | 132 | if __name__ == '__main__': 133 | param_file = sys.argv[1] 134 | problem = sys.argv[2] 135 | output_file = sys.argv[3] 136 | 137 | # Solve, gather measures and write to file. 138 | param_file = os.path.abspath(param_file) 139 | measures = run(param_file, problem) 140 | write_measures_to_file(output_file, measures) 141 | 142 | -------------------------------------------------------------------------------- /nosetests.py: -------------------------------------------------------------------------------- 1 | 2 | from nose import main 3 | 4 | if __name__ == '__main__': 5 | main() 6 | -------------------------------------------------------------------------------- /opal/Platforms/__init__.py: -------------------------------------------------------------------------------- 1 | from lsf import LSF 2 | from linux import LINUX 3 | from mpi import OPALMPI 4 | from smp import SMP 5 | from sungrid import SunGrid 6 | 7 | supported_platforms = {'LINUX': LINUX, 8 | 'LSF': LSF, 9 | 'SMP': SMP, 10 | 'SunGrid': SunGrid} 11 | -------------------------------------------------------------------------------- /opal/Platforms/linux.py: -------------------------------------------------------------------------------- 1 | import socket 2 | import os 3 | 4 | from ..core.platform import Platform 5 | from ..core.platform import Task 6 | 7 | class LINUXTask(Task): 8 | def __init__(self, 9 | name=None, 10 | command=None, 11 | sessionTag=None, 12 | output='/dev/null', 13 | logHandlers=[]): 14 | Task.__init__(self, 15 | name=name, 16 | command=command, 17 | sessionTag=sessionTag, 18 | output=output, 19 | logHandlers=logHandlers) 20 | return 21 | 22 | def run(self): 23 | # Execute the command 24 | os.system(self.command + ' > ' + self.output) 25 | # Inform the fininish 26 | Task.run(self) 27 | return 28 | 29 | class LINUXPlatform(Platform): 30 | def __init__(self, logHandlers=[]): 31 | Platform.__init__(self, name='LINUX', 32 | logHandlers=logHandlers) 33 | self.message_handlers['cfp-execute'] = self.create_task 34 | return 35 | 36 | def create_task(self, info): 37 | ''' 38 | 39 | Handle a call for proposal of executing a command 40 | ''' 41 | if 'proposition' not in info.keys(): 42 | self.logger.log('Proposal of executing a command has not ' + \ 43 | 'information to prcess') 44 | return 45 | 46 | proposition = info['proposition'] 47 | command = proposition['command'] 48 | 49 | name = proposition['tag'] 50 | if 'queue' in proposition.keys(): 51 | queueTag = proposition['queue'] 52 | else: 53 | queueTag = None 54 | queueTag = proposition['queue'] 55 | task = LINUXTask(name=name, 56 | command=command, 57 | sessionTag=proposition['tag']) 58 | self.submit(task, queue=queueTag) 59 | return 60 | 61 | def cancel_tasks(self, info): 62 | ''' 63 | 64 | Handle message terminate experiment 65 | ''' 66 | return 67 | 68 | 69 | def test_a(self): 70 | print 'Hello' 71 | 72 | 73 | 74 | LINUX = LINUXPlatform() 75 | -------------------------------------------------------------------------------- /opal/Platforms/lsf.py: -------------------------------------------------------------------------------- 1 | import socket 2 | import os 3 | import time 4 | 5 | from ..core.platform import Platform 6 | from ..core.platform import Task 7 | 8 | class LSFTask(Task): 9 | def __init__(self, name=None, 10 | taskId=None, 11 | command=None, 12 | lsfOptions=None, 13 | logHandlers=[]): 14 | self.output = "-N -oo /tmp/lsf-output.log" 15 | lsfCmd = "bsub " + \ 16 | " -J " + taskId + lsfOptions + command 17 | Task.__init__(self, 18 | name=name, 19 | taskId=taskId, 20 | command=lsfCmd, 21 | logHandlers=logHandlers) 22 | self.job_id = taskId 23 | return 24 | 25 | def run(self): 26 | os.system(self.command) 27 | self.wait('ended("' + self.job_id +')') 28 | Task.run(self) 29 | return 30 | 31 | def wait(self, condition): 32 | # This function playes in role of synchronyzers 33 | # 1 - Generate a synchronizing job including a segment code that 34 | # notifies to current process by socket (notifyToMaster) 35 | # 2 - Prepare a waiting socket: create, bind, .. 36 | # 3 - Submit the synchronizing with the condition specified in 37 | # the condition 38 | # 4 - Turn in waiting by listening the notify at created socket 39 | # Argument condition may be "ended(CUTEr-*)" 40 | #------------------- 41 | # Set default socket parameter 42 | #------------------- 43 | port = 19879 44 | hostname = socket.gethostname() 45 | ltime = time.localtime() 46 | keyStr = str(ltime.tm_year) + str(ltime.tm_mon) + str(ltime.tm_mday) +\ 47 | str(ltime.tm_hour) + str(ltime.tm_min) + str(ltime.tm_sec) 48 | #------------------------- 49 | # Prepare socket 50 | #----------- 51 | serversocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 52 | socketIsBound = 0 53 | #----------------- 54 | # Choose an available port to avoid conflit with the other routine 55 | # Particularly, the other parameter optimization 56 | while socketIsBound == 0: 57 | try: 58 | serversocket.bind((hostname, port)) 59 | socketIsBound = 1 60 | except: 61 | port = port + 1 62 | #print "waiting at",socket.gethostname(), port 63 | serversocket.listen(1) 64 | #----------------------- 65 | # Generating synchronizing job 66 | #---------------------- 67 | synchronizerFile = 'synchronizer_' + self.job_id + '.py' 68 | f = open('synchronizer.py','w') 69 | #synchronizerFile.write('#!/usr/bin/env python\n') 70 | f.write('import socket\n') 71 | f.write('port = ' + str(port) + '\n') 72 | f.write('s = socket.socket(socket.AF_INET, ' + \ 73 | 'socket.SOCK_STREAM)\n') 74 | f.write('s.connect(("' + hostname + '",' + \ 75 | str(port) + '))\n') 76 | f.write('s.send("'+ keyStr + '")\n') 77 | f.write('s.close()\n') 78 | f.close() 79 | 80 | #ltime = time.localtime() 81 | #timeStr = str(ltime.tm_year) + '-' + str(ltime.tm_mon) + '-' + \ 82 | # str(ltime.tm_mday) + ' ' + str(ltime.tm_hour) + ':' + \ 83 | # str(ltime.tm_min) + ':' + str(ltime.tm_sec) 84 | #os.system('echo ' + timeStr + ' Begin waiting >> lsf-sync.log') 85 | synchronizeCmd = 'bsub -w "' + condition + \ 86 | '" python ' + synchronizerFile + ' > /dev/null' 87 | os.system(synchronizeCmd) 88 | #----------------------- 89 | # Waiting for the notify from synchonizer 90 | # --------------------- 91 | recvKey = '' 92 | while recvKey != keyStr: 93 | (clientsocket, address) = serversocket.accept() 94 | recvKey = clientsocket.recv(len(keyStr)) 95 | clientsocket.close() 96 | # print address, "is connected" 97 | #-------------- 98 | # free the sockets if received a notification 99 | #----------------- 100 | 101 | #clientsocket.close() 102 | serversocket.close() 103 | #ltime = time.localtime() 104 | #timeStr = str(ltime.tm_year) + '-' + str(ltime.tm_mon) + '-' + \ 105 | # str(ltime.tm_mday) + ' ' + str(ltime.tm_hour) + ':' + \ 106 | # str(ltime.tm_min) + ':' + str(ltime.tm_sec) 107 | #os.system('echo ' + timeStr + ' End waiting >> lsf-sync.log') 108 | os.remove(synchronizerFile) 109 | return 110 | 111 | 112 | 113 | class LSFPlatform(Platform): 114 | def __init__(self, maxTask=3, synchronous=False, logHandlers=[]): 115 | Platform.__init__(self, 116 | name='LSF', 117 | maxTask=maxTask, 118 | logHandlers=logHandlers) 119 | self.configuration = {} 120 | self.message_handlers['cfp-execute'] = self.create_task 121 | pass 122 | 123 | def set_config(self, parameterName, parameterValue): 124 | self.configuration[parameterName] = parameterValue 125 | return 126 | 127 | def initialize(self, testId): 128 | return 129 | 130 | def create_task(self, info): 131 | ''' 132 | 133 | Handle a call for proposal of executing a command through LSF platform 134 | ''' 135 | 136 | if 'proposition' not in info.keys(): 137 | self.logger.log('Proposal of executing a command has not ' + \ 138 | 'information to process') 139 | return 140 | execCmd = info['proposition']['command'] 141 | tag = info['proposition']['tag'] 142 | if 'queue' in info['proposition'].keys(): 143 | queueTag = info['proposition']['queue'] 144 | else: 145 | queueTag = None 146 | 147 | # str(ltime.tm_year) + str(ltime.tm_mon) + str(ltime.tm_mday) + \ 148 | # str(ltime.tm_hour) + str(ltime.tm_min) + str(ltime.tm_sec) 149 | optionStr = " " 150 | for param in self.configuration.keys(): 151 | optionStr = optionStr + param + " " + \ 152 | self.configuration[param] + " " 153 | if queueTag is not None: 154 | optionStr = " -g " + queueTag + optionStr 155 | task = LSFTask(name=tag, 156 | taskId=tag, 157 | command=execCmd, 158 | lsfOptions=optionStr) 159 | self.submit(task, queue=queueTag) 160 | return 161 | 162 | LSF = LSFPlatform() 163 | -------------------------------------------------------------------------------- /opal/Platforms/mpi.py: -------------------------------------------------------------------------------- 1 | 2 | import os 3 | import time 4 | 5 | from ..core.platform import Platform 6 | from ..core import log 7 | 8 | class MPIPlatform(Platform): 9 | def __init__(self,logHandlers=[],**kwargs): 10 | Platform.__init__(self,'MPI',**kwargs) 11 | #self.communicator = MPI.COMM_WORLD 12 | self.children = [] 13 | self.wrapper_name = '' 14 | self.configuration = {} 15 | self.logger = log.OPALLogger(name='mpiPlatform', handlers=logHandlers) 16 | pass 17 | 18 | def set_config(self, parameterName, parameterValue): 19 | self.configuration[parameterName] = parameterValue 20 | return 21 | 22 | def initialize(self, testId): 23 | self.wrapper_name = 'run_wrapper_' + testId + '.py' 24 | if not os.path.exists(self.wrapper_name): 25 | self.create_wrapper() 26 | self.children = [] 27 | #self.logger.log('Platform is initialized') 28 | return 29 | 30 | def execute(self, command, output='/dev/null'): 31 | jobId = str(hash(command)) 32 | # str(ltime.tm_year) + str(ltime.tm_mon) + str(ltime.tm_mday) + \ 33 | # str(ltime.tm_hour) + str(ltime.tm_min) + str(ltime.tm_sec) 34 | optionStr = " " 35 | for param in self.configuration.keys(): 36 | optionStr = optionStr + param + " " + self.configuration[param] + " " 37 | #child = self.communicator.Spawn('python', 38 | # [self.wrapper_name, command]) 39 | from mpi4py import MPI 40 | #self.logger.log('Execute command: '+ command) 41 | child = MPI.COMM_WORLD.Spawn('python', 42 | [self.wrapper_name, command]) 43 | self.children.append(child) 44 | return 45 | 46 | def create_wrapper(self): 47 | tab = ' '*4 48 | f = open(self.wrapper_name,'w') 49 | f.write('import sys\n') 50 | f.write('import os\n') 51 | f.write('from mpi4py import MPI\n') 52 | f.write('parent = MPI.COMM_WORLD.Get_parent()\n') 53 | f.write('exeCmd = ""\n') 54 | f.write('for argv in sys.argv[1:]:\n') 55 | f.write(tab + 'exeCmd = exeCmd + argv + " "\n') 56 | #f.write('print exeCmd\n') 57 | f.write('os.system(exeCmd)\n') 58 | f.write('parent.Barrier()\n') 59 | f.write('parent.Disconnect()\n') 60 | f.close() 61 | 62 | 63 | def waitForCondition(self,condition): 64 | for child in self.children: 65 | child.Barrier() 66 | child.Disconnect() 67 | os.remove(self.wrapper_name) 68 | self.children = [] 69 | return 70 | 71 | 72 | OPALMPI = MPIPlatform() 73 | -------------------------------------------------------------------------------- /opal/Platforms/smp.py: -------------------------------------------------------------------------------- 1 | 2 | import os 3 | import time 4 | import subprocess 5 | import threading 6 | import shlex 7 | 8 | from ..core.platform import Platform 9 | from ..core.platform import Task 10 | 11 | from ..core import log 12 | 13 | 14 | class SMPTask(Task): 15 | """ 16 | 17 | Each task run on this platform need to 18 | some stubs to communicate with the platform 19 | 20 | """ 21 | def __init__(self, name=None, taskId=None, command=None, sessionTag=None): 22 | Task.__init__(self, 23 | name=name, 24 | taskId=taskId, 25 | command=command, 26 | sessionTag=sessionTag) 27 | self.proc = None 28 | self.pid = None 29 | return 30 | 31 | def run(self): 32 | #cmd = shlex.split(self.command) 33 | #self.proc = subprocess.Popen(args=cmd) 34 | cmd = self.command + '> /dev/null' 35 | self.proc = subprocess.Popen(args=cmd, shell=True) 36 | self.pid = self.proc.pid 37 | if self.proc.poll() is None: # check if child process is still running 38 | self.proc.wait() # wait until the child process finish 39 | # Inform the task is finished 40 | Task.run(self) 41 | return 42 | 43 | class SMPPlatform(Platform): 44 | def __init__(self, maxTask=2, logHandlers=[]): 45 | Platform.__init__(self, name='SMP', 46 | maxTask=maxTask, 47 | synchronous=False, 48 | logHandlers=logHandlers) 49 | self.configuration = {} 50 | #self.logger = log.OPALLogger(name='smpPlatform', handlers=logHandlers) 51 | self.message_handlers['cfp-execute'] = self.create_task 52 | pass 53 | 54 | def set_config(self, parameterName, parameterValue): 55 | self.configuration[parameterName] = parameterValue 56 | return 57 | 58 | 59 | def initialize(self, testId): 60 | #self.children = [] 61 | #self.logger.log('Platform is initialized for the test ' + testId) 62 | return 63 | 64 | # Message handlers 65 | 66 | def create_task(self, info): 67 | ''' 68 | 69 | Handle a call for proposal of executing a command 70 | ''' 71 | if 'proposition' not in info.keys(): 72 | self.logger.log('Proposal of executing a command has not ' + \ 73 | 'information to prcess') 74 | return 75 | 76 | proposition = info['proposition'] 77 | command = proposition['command'] 78 | name = proposition['tag'] 79 | if 'queue' in proposition.keys(): 80 | queueTag = proposition['queue'] 81 | else: 82 | queueTag = None 83 | task = SMPTask(name=name, 84 | command=command, 85 | sessionTag=proposition['tag']) 86 | self.submit(task, queue=queueTag) 87 | return 88 | 89 | 90 | 91 | 92 | SMP = SMPPlatform() 93 | -------------------------------------------------------------------------------- /opal/Platforms/sungrid.py: -------------------------------------------------------------------------------- 1 | import socket 2 | import os 3 | import time 4 | 5 | from ..core.platform import Platform 6 | from ..core.platform import Task 7 | 8 | class SunGridTask(Task): 9 | def __init__(self, name=None, 10 | taskId=None, 11 | command=None, 12 | options=None, 13 | logHandlers=[]): 14 | self.job_id = taskId 15 | sungridCmd = "qsub -cwd -V " + \ 16 | '-o ' + self.job_id + '-stdout.log ' +\ 17 | '-e ' + self.job_id + '-stderr.log ' +\ 18 | "-N " + self.job_id + ' ' +\ 19 | options + ' ' +\ 20 | '-b y ' + command + \ 21 | ' > /dev/null' 22 | 23 | Task.__init__(self, 24 | name=name, 25 | taskId=taskId, 26 | command=sungridCmd, 27 | logHandlers=logHandlers) 28 | #self.logger.log(sungridCmd) 29 | return 30 | 31 | def run(self): 32 | os.system(self.command) 33 | self.wait() 34 | os.remove(self.job_id + '-stdout.log') 35 | os.remove(self.job_id + '-stderr.log') 36 | Task.run(self) 37 | return 38 | 39 | def wait(self): 40 | # This function playes in role of synchronyzers 41 | # 1 - Generate a synchronizing job including a segment code that 42 | # notifies to current process by socket (notifyToMaster) 43 | # 2 - Prepare a waiting socket: create, bind, .. 44 | # 3 - Submit the synchronizing with the condition specified in 45 | # the condition 46 | # 4 - Turn in waiting by listening the notify at created socket 47 | # Argument condition may be "ended(CUTEr-*)" 48 | #------------------- 49 | # Set default socket parameter 50 | #------------------- 51 | port = 19879 52 | hostname = socket.gethostname() 53 | ltime = time.localtime() 54 | keyStr = str(ltime.tm_year) + str(ltime.tm_mon) + str(ltime.tm_mday) +\ 55 | str(ltime.tm_hour) + str(ltime.tm_min) + str(ltime.tm_sec) 56 | #------------------------- 57 | # Prepare socket 58 | #----------- 59 | serversocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 60 | socketIsBound = 0 61 | #----------------- 62 | # Choose an available port to avoid conflit with the other routine 63 | # Particularly, the other parameter optimization 64 | while socketIsBound == 0: 65 | try: 66 | serversocket.bind((hostname, port)) 67 | socketIsBound = 1 68 | except: 69 | port = port + 1 70 | #print "waiting at",socket.gethostname(), port 71 | serversocket.listen(1) 72 | #----------------------- 73 | # Generating synchronizing job 74 | #---------------------- 75 | synchronizerFile = 'synchronizer_' + self.job_id + '.py' 76 | f = open(synchronizerFile, 'w') 77 | #synchronizerFile.write('#!/usr/bin/env python\n') 78 | f.write('import socket\n') 79 | f.write('port = ' + str(port) + '\n') 80 | f.write('s = socket.socket(socket.AF_INET, ' + \ 81 | 'socket.SOCK_STREAM)\n') 82 | f.write('s.connect(("' + hostname + '",' + \ 83 | str(port) + '))\n') 84 | f.write('s.send("'+ keyStr + '")\n') 85 | f.write('s.close()\n') 86 | f.close() 87 | 88 | #ltime = time.localtime() 89 | #timeStr = str(ltime.tm_year) + '-' + str(ltime.tm_mon) + '-' + \ 90 | # str(ltime.tm_mday) + ' ' + str(ltime.tm_hour) + ':' + \ 91 | # str(ltime.tm_min) + ':' + str(ltime.tm_sec) 92 | #os.system('echo ' + timeStr + ' Begin waiting >> lsf-sync.log') 93 | synchronizeCmd = 'qsub -cwd -V ' +\ 94 | '-N sync_' + self.job_id + ' ' +\ 95 | '-o sync_' + self.job_id + '-stdout.log ' +\ 96 | '-e sync_' + self.job_id + '-stderr.log ' +\ 97 | '-hold_jid ' + self.job_id + ' ' +\ 98 | '-b y python ' + synchronizerFile + ' > /dev/null' 99 | os.system(synchronizeCmd) 100 | #----------------------- 101 | # Waiting for the notify from synchonizer 102 | # --------------------- 103 | recvKey = '' 104 | while recvKey != keyStr: 105 | (clientsocket, address) = serversocket.accept() 106 | recvKey = clientsocket.recv(len(keyStr)) 107 | clientsocket.close() 108 | # print address, "is connected" 109 | #-------------- 110 | # free the sockets if received a notification 111 | #----------------- 112 | 113 | #clientsocket.close() 114 | serversocket.close() 115 | #ltime = time.localtime() 116 | #timeStr = str(ltime.tm_year) + '-' + str(ltime.tm_mon) + '-' + \ 117 | # str(ltime.tm_mday) + ' ' + str(ltime.tm_hour) + ':' + \ 118 | # str(ltime.tm_min) + ':' + str(ltime.tm_sec) 119 | #os.system('echo ' + timeStr + ' End waiting >> lsf-sync.log') 120 | os.remove(synchronizerFile) 121 | os.remove('sync_' + self.job_id + '-stdout.log') 122 | os.remove('sync_' + self.job_id + '-stderr.log') 123 | return 124 | 125 | class SunGridPlatform(Platform): 126 | def __init__(self, maxTask=3, synchronous=False, logHandlers=[]): 127 | Platform.__init__(self, 128 | name='SunGrid', 129 | maxTask=maxTask, 130 | logHandlers=logHandlers) 131 | self.configuration = {} 132 | self.message_handlers['cfp-execute'] = self.create_task 133 | pass 134 | 135 | def set_config(self, parameterName, parameterValue=None): 136 | ''' 137 | Configuration can be express as option string and store in 138 | OPTIONS fields of setting attribute 139 | ''' 140 | if self.settings['OPTIONS'] is None: 141 | self.settings['OPTIONS'] = parameterName + ' ' + parameterValue 142 | else: 143 | self.settings['OPTIONS'] = self.settings['OPTIONS'] + ' ' +\ 144 | parameterName + ' ' + parameterValue 145 | self.configuration[parameterName] = parameterValue 146 | return 147 | 148 | def initialize(self, testId): 149 | return 150 | 151 | def create_task(self, info): 152 | ''' 153 | 154 | Handle a call for proposal of executing a command through SunGrid 155 | platform 156 | ''' 157 | 158 | if 'proposition' not in info.keys(): 159 | self.logger.log('Proposal of executing a command has not ' + \ 160 | 'information to process') 161 | return 162 | execCmd = info['proposition']['command'] 163 | tag = info['proposition']['tag'] 164 | if 'queue' in info['proposition'].keys(): 165 | queueTag = info['proposition']['queue'] 166 | else: 167 | queueTag = None 168 | 169 | # str(ltime.tm_year) + str(ltime.tm_mon) + str(ltime.tm_mday) + \ 170 | # str(ltime.tm_hour) + str(ltime.tm_min) + str(ltime.tm_sec) 171 | 172 | optionStr = self.settings['OPTIONS'] 173 | 174 | task = SunGridTask(name=tag, 175 | taskId='SGE_' + tag, 176 | command=execCmd, 177 | options=optionStr) 178 | self.submit(task, queue=queueTag) 179 | return 180 | 181 | SunGrid= SunGridPlatform() 182 | -------------------------------------------------------------------------------- /opal/Platforms/test_platforms.py: -------------------------------------------------------------------------------- 1 | def test_linux_platform(): 2 | #assert 'b' == 'b' 3 | from ..core.mafrw import Environment 4 | from linux import LINUX 5 | 6 | env = Environment(name='test platform environment') 7 | LINUX.register(env) 8 | LINUX.submit('ls') 9 | 10 | -------------------------------------------------------------------------------- /opal/Solvers/__init__.py: -------------------------------------------------------------------------------- 1 | #from NOMAD import NOMADBlackbox 2 | from nomad import NOMAD 3 | from nomad import NOMADMPI 4 | -------------------------------------------------------------------------------- /opal/TestProblemCollections/CUTEr.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | import os.path 3 | import pickle 4 | import sys 5 | #from paropt.components.testenv import CUTEr as testEnvironment 6 | 7 | from ..core.testproblem import * 8 | 9 | from cuterfactory import CUTErFactory 10 | from cuterfactory import CUTErQuery 11 | 12 | #data_file = os.path.join(os.path.dirname(os.path.abspath(sys.modules[__name__].__file__)),'CUTEr.data') 13 | data_file = os.path.join(os.path.expanduser('~'),'.opal/CUTEr-1.data') 14 | #print data_file 15 | # Object definition 16 | try: 17 | f = open(data_file,'r') 18 | CUTEr = pickle.load(f) 19 | f.close() 20 | except IOError: 21 | classfDir = os.environ['MASTSIF'] 22 | classfName='CLASSF.DB' # Standard name for CUTEr classify file 23 | classfFile = os.path.join(classfDir, classfName) 24 | CUTEr_factory = CUTErFactory(classifyFile=classfFile) 25 | CUTEr = CUTEr_factory.generate_collection() 26 | CUTEr.HS = CUTEr.select(CUTErQuery(name='HS\d+')) 27 | f = open(data_file,'w') 28 | pickle.dump(CUTEr,f) 29 | f.close() 30 | -------------------------------------------------------------------------------- /opal/TestProblemCollections/__init__.py: -------------------------------------------------------------------------------- 1 | try: 2 | from CUTEr import CUTEr 3 | from cuterfactory import CUTErQuery 4 | except: 5 | print 'Something went wrong while trying to import CUTEr.' 6 | print 'Skipping.' 7 | -------------------------------------------------------------------------------- /opal/TestProblemCollections/cuterfactory.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | import re 4 | from ..core.testproblem import * 5 | 6 | class CUTErTestProblem(OptimizationTestProblem): 7 | 8 | def __init__(self, 9 | name=None, 10 | description=None, 11 | classifyStr=None, 12 | nvar=0, 13 | ncon=0, 14 | paramString=None, 15 | **kwargs): 16 | OptimizationTestProblem.__init__(self, 17 | name=name, 18 | description=description, 19 | classifyStr=classifyStr, 20 | nvar=nvar, 21 | ncon=ncon, 22 | **kwargs) 23 | self.parameter_string = paramString 24 | 25 | class CUTErQuery: 26 | 27 | def __init__(self,qName=None, 28 | namePattern='', 29 | nMin=0,nMax=1000000, 30 | mMin=0,mMax=1000000, 31 | objectiveType='NCLQSO', 32 | constraintType='UXBNLQO', 33 | smoothType='IR', 34 | infoOrder=0, 35 | problemType="AMR", 36 | internalVar='YN', 37 | **kwargs): 38 | """ 39 | An example of a query 40 | query = Query(namePattern='HS', 41 | nMin=0, 42 | nMax=10, 43 | objectiveType="N", 44 | constraintType="U") 45 | """ 46 | self.name_pattern = namePattern 47 | self.nMin = nMin 48 | self.nMax = nMax 49 | self.mMin = mMin 50 | self.mMax = mMax 51 | self.objectiveType = objectiveType 52 | self.constraintType = constraintType 53 | self.smoothType = smoothType 54 | self.infoOrder = infoOrder 55 | self.problemType = problemType 56 | self.internalVar = internalVar 57 | 58 | def match(self, problem): 59 | #print self.nMax, self.mMax, probDescStr, 60 | descFields = problem.get_classify_string().split("-") 61 | for i in range(len(descFields)): 62 | descFields[i] = descFields[i].strip() 63 | #descFields[0:] = descFields[0:].strip() 64 | if re.compile(self.name_pattern).match(problem.get_name()) is None: 65 | return False 66 | if self.objectiveType.count(descFields[0][0]) <= 0: 67 | #print self.objectiveType, 'is not sastified' 68 | return False 69 | if self.constraintType.count(descFields[0][1]) <= 0: 70 | #print self.constraintType, 'is not sastified' 71 | return False 72 | if self.smoothType.count(descFields[0][2]) <= 0: 73 | #print self.smoothType, 'is not sastified' 74 | return False 75 | if self.infoOrder > int(descFields[0][3:]): 76 | #print 'info order is not sastified' 77 | return False 78 | if self.problemType.count(descFields[1][0]) <= 0: 79 | #print self.problemType, 'is not satisfied' 80 | return False 81 | if self.internalVar.count(descFields[1][1]) <= 0: 82 | return False 83 | if descFields[2].count("V") <= 0: 84 | if self.nMin > int(descFields[2]): 85 | #print descFields[2], 'variables is not satisifed' 86 | return False 87 | if self.nMax < int(descFields[2]): 88 | #print descFields[2], 'variables is not satisifed' 89 | return False 90 | if descFields[3].count("V") <= 0: 91 | if self.mMin > int(descFields[3]): 92 | #print descFields[3], 'constraints is not satisifed' 93 | return False 94 | if self.mMax < int(descFields[3]): 95 | #print descFields[3], 'constraints is not satisifed' 96 | return False 97 | #print 'Query is matched' 98 | return True 99 | 100 | #========== 101 | 102 | class CUTErFactory: 103 | def __init__(self,classifyFile=None,**kwargs): 104 | self.classifyFile = classifyFile 105 | self.decoder = 'sifdecode' 106 | self.dbDir = os.environ['MASTSIF'] 107 | pass 108 | 109 | def extract(self, **kwargs): 110 | queryResult = [] 111 | queryPharse = Query(**kwargs) 112 | if self.classifyFile == None: 113 | return queryResult 114 | f = open(self.classifyFile,'r') 115 | lines = f.readlines() 116 | f.close() 117 | for line in lines: 118 | line = line.strip() 119 | if len(line) <= 2: 120 | continue 121 | #print line 122 | fields = line.split(' ',1) 123 | if queryPharse.match(fields[0].strip(),fields[1].strip()): 124 | #print fields[0], 'is added' 125 | queryResult.append(fields[0].strip()) 126 | return queryResult 127 | 128 | def generate_collection(self): 129 | f = open(self.classifyFile, 'r') 130 | lines = f.readlines() 131 | f.close() 132 | CUTEr = ProblemCollection(name='CUTEr collection') 133 | for line in lines: 134 | line = line.strip() 135 | if len(line) <= 2: 136 | # Emtry line with new line symbol 137 | continue 138 | #print line 139 | fields = line.split(' ',1) 140 | prob = self.generate_problem(fields[0].strip(),fields[1].strip()) 141 | if prob is not None: 142 | CUTEr.add_problem(prob) 143 | return CUTEr 144 | 145 | def generate_problem(self,problem_name,classify_string=None,param=None): 146 | decode_cmd = self.decoder 147 | if param is None: 148 | decode_cmd = decode_cmd + ' ' + problem_name 149 | else: 150 | decode_cmd = decode_cmd + ' -param ' + param + ' ' + problem_name 151 | f = os.popen(decode_cmd) 152 | decode_log = f.read() 153 | f.close() 154 | variable_query = re.compile('[0-9]+[ a-z]+variable[s]?') 155 | constraint_query = re.compile('[0-9]+[ a-z]+constraint[s]?') 156 | number_query = re.compile('\d+') 157 | nvar = 0 158 | variable_declarations = variable_query.findall(decode_log) 159 | #print problem_name, 160 | for var_decl in variable_declarations: 161 | #print var_decl, 162 | nvar = nvar + int(number_query.findall(var_decl)[0]) 163 | #print '' 164 | constraint_declarations = constraint_query.findall(decode_log) 165 | ncon = 0 166 | for cons_decl in constraint_declarations: 167 | ncon = ncon + int(number_query.findall(cons_decl)[0]) 168 | if (nvar + ncon <= 0): 169 | return None 170 | # There is no problem nvar + ncon = 0 171 | problem = CUTErTestProblem(name=problem_name, 172 | classifyStr=classify_string, 173 | nvar=nvar, 174 | ncon=ncon) 175 | return problem 176 | -------------------------------------------------------------------------------- /opal/__init__.py: -------------------------------------------------------------------------------- 1 | from core.modelstructure import ModelStructure, MeasureFunction 2 | from core.modeldata import ModelData 3 | from core.model import Model 4 | from core.parameter import Parameter 5 | from core.measure import Measure 6 | from core.algorithm import Algorithm 7 | from core.testproblem import TestProblem 8 | 9 | 10 | -------------------------------------------------------------------------------- /opal/config.py: -------------------------------------------------------------------------------- 1 | import os 2 | import ConfigParser as cfg # In python 3, this module is changed to configparser 3 | from Platforms import LINUX as platform 4 | 5 | def create_default_configuration(): 6 | configuration = cfg.ConfigParser() 7 | configuration.add_section('SYSTEM') 8 | configuration.set('SYSTEM','Python','#!/usr/bin/env python') 9 | return configuration 10 | 11 | def read_config(): 12 | configFile = os.environ['HOME'] + '/.opal/opal.cfg' 13 | if not os.path.exists(configFile): 14 | return create_default_configuration() 15 | configuration = cfg.ConfigParser() 16 | configuration.read(configFile) 17 | return configuration 18 | 19 | configuration = read_config() 20 | python = configuration.get('SYSTEM','Python') 21 | #paroptRootPackage = 'dev.paropt' 22 | -------------------------------------------------------------------------------- /opal/core/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PythonOptimizers/opal/b6e12bafe7210910a6df2a9b59829cde584153cb/opal/core/__init__.py -------------------------------------------------------------------------------- /opal/core/datamanager.py: -------------------------------------------------------------------------------- 1 | import os 2 | import string 3 | import types 4 | import time 5 | import shutil 6 | import log 7 | import copy 8 | #import logging 9 | 10 | #import utility 11 | from testproblem import TestProblem 12 | from mafrw import Agent 13 | 14 | from .. import config 15 | 16 | 17 | class DataManager(Agent): 18 | """ 19 | 20 | An object of this class is responsable to collect data, 21 | support access to data of a test corresponding to one set 22 | of parameter 23 | 24 | During a session working of data generator, it is activated 25 | at first, wait for signal of a task to collect the data and 26 | store data following its rules. Normally, the data is stored 27 | in external memory (for example a file) but a part can be loaded 28 | into memory (a variable of DataController object). 29 | 30 | The other components needing data sends to this object a 31 | request and get the data. 32 | """ 33 | def __init__(self, 34 | name='data manager', 35 | rows=None, 36 | columns=None, 37 | storage=None, 38 | logHandlers=[]): 39 | self.file_name = 'data_storage.txt' 40 | Agent.__init__(self, name=name, logHandlers=logHandlers) 41 | self.message_handlers['inform-measure-values'] = self.add_data 42 | self.message_handlers['cfp-collect'] = self.collect_data 43 | return 44 | 45 | # Management functionality 46 | def update(self, problem, parameterTag, data): 47 | return 48 | 49 | def find(self, query): 50 | return 51 | 52 | # Message handlers 53 | def add_data(self, info): 54 | if info is None: 55 | return 56 | 57 | paramTag = info['proposition']['parameter-tag'] 58 | prob = info['proposition']['problem'] 59 | data = info['proposition']['values'] 60 | # Update the entry 61 | self.update(parameterTag=paramTag, 62 | problem=prob, 63 | data=data) 64 | # Get all data entry corresponding to the tag 65 | dataEntry = self.find(self, query={'tag':paramTag}) 66 | # Send a message informing availbility of data 67 | if dataEntry.is_complete(): 68 | msg = Message(sender=self.id, 69 | performative='inform', 70 | content={'proposition':{'what':'data-availability', 71 | 'how':'complete', 72 | 'data':dataEntry} 73 | } 74 | ) 75 | else: 76 | msg = Message(sender=self.id, 77 | performative='inform', 78 | content={'proposition':{'what':'data-availability', 79 | 'how':'partial', 80 | 'data':dataEntry} 81 | } 82 | ) 83 | self.send_message(msg) 84 | return 85 | 86 | def find_data(self, info): 87 | 88 | (paramValues, problem) = message.content['proposition'] 89 | measureValues = self.query_data(parameters=paramValues, 90 | problem=problem) 91 | # If getting data is successful 92 | if measureValues is not None: 93 | # Create a reply message whose content contains the 94 | # measure values 95 | msgCont = self.encrypt(measureValues) 96 | msg = Message(sender=self.id, 97 | content=msgCont, 98 | performative='inform', 99 | reference=message.id) 100 | return 101 | 102 | def collect_data(self, message): 103 | (paramValues, problem, measureFile) = message.content['proposition'] 104 | f = open(measureFile) 105 | f.close() 106 | return 107 | 108 | # Private methods 109 | def query_data(self, parameters=None, problem=None): 110 | return None 111 | -------------------------------------------------------------------------------- /opal/core/io.py: -------------------------------------------------------------------------------- 1 | # Simple helper functions for input and output. 2 | 3 | def read_params_from_file(filename): 4 | converters = {'categorical':str, 'integer':int, 'real':float} 5 | fp = open(filename, 'rb') 6 | params = {} 7 | for line in fp: 8 | words = line.strip('\n').split(':') 9 | params[words[0]] = converters[words[1]](words[2]) 10 | fp.close() 11 | return params 12 | 13 | 14 | def write_measures_to_file(filename, measures): 15 | fp = open(filename, 'w') 16 | for measure in measures: 17 | print >> fp, measure, measures[measure] 18 | fp.close() 19 | return 20 | 21 | -------------------------------------------------------------------------------- /opal/core/log.py: -------------------------------------------------------------------------------- 1 | import re 2 | import logging 3 | 4 | class HandlerDescription: 5 | def __init__(self, handler): 6 | self.file_name = handler.baseFilename 7 | self.level = handler.level 8 | 9 | def generate_handler(self): 10 | handler = logging.FileHandler(filename=self.file_name) 11 | handler.set_level(self.level) 12 | return handler 13 | 14 | class OPALLogger: 15 | ''' 16 | 17 | We specialize logging facility of Python by this class to support 18 | the ability of pickling an logger with handlers that are the streamming 19 | objects 20 | ''' 21 | 22 | def __init__(self, name=None, handlers=[]): 23 | self.name = name 24 | self.initialize() 25 | # Initialize an empty list of descriptions 26 | # of the user-required handlers 27 | self.handler_descriptions = [] 28 | # Get the description of the user-required handlers 29 | # and add it to logger 30 | for hdlr in handlers: 31 | self.handler_descriptions.append(HandlerDescription(hdlr)) 32 | self.logger.addHandler(hdlr) 33 | return 34 | 35 | def initialize(self): 36 | self.logger = logging.getLogger(self.name) 37 | self.logger.setLevel(logging.DEBUG) # Set level to highest level so 38 | # that actual level depends on the 39 | # handler level 40 | # A default handler is created for logging to file with INFO level 41 | handler = logging.FileHandler(filename='/var/tmp/opal.log') 42 | handler.setFormatter(logging.Formatter('%(asctime)s - %(name)s: %(message)s')) 43 | handler.setLevel(logging.INFO) 44 | self.logger.addHandler(handler) 45 | return 46 | 47 | def __getstate__(self): 48 | # To serialize a OPALLogger object, we save only 49 | # the name and the descriptions of the user-required handlers 50 | dict = {} 51 | dict['handler_descriptions'] = self.handler_descriptions 52 | dict['name'] = self.name 53 | return dict 54 | 55 | def __setstate__(self, dict): 56 | # The expected dict is two-element dictionary. 57 | # The first element of dict has key is 'handler_descriptions' 58 | # and has value is a list of description of handlers. The 59 | # second one is the name of logger. 60 | self.name = dict['name'] 61 | # Initialize the logger with the specified name 62 | self.initialize() 63 | # Create the handler descriptions for unpickled object 64 | # and create handlers for the logger 65 | self.handler_descriptions = dict['handler_descriptions'] 66 | for desc in self.handler_descriptions: 67 | handler = desc.generate_handler() 68 | self.logger.addHandler(handler) 69 | return 70 | 71 | def log(self, message, level=logging.INFO): 72 | self.logger.log(level, message + '\n') 73 | return 74 | 75 | 76 | class Debugger: 77 | def __init__(self, fileName='/var/tmp/opal-debug.log'): 78 | self.logger = logging.getLogger('DEBUG') 79 | self.logger.setLevel(logging.DEBUG) 80 | handler = logging.FileHandler(filename=fileName) 81 | handler.setLevel(logging.DEBUG) 82 | handler.setFormatter(logging.Formatter(\ 83 | '%(asctime)s - %(name)s: %(message)s')) 84 | self.logger.addHandler(handler) 85 | return 86 | 87 | def log(self, message, level=logging.DEBUG): 88 | self.logger.log(level, message) 89 | return 90 | 91 | debugger = Debugger() 92 | 93 | 94 | -------------------------------------------------------------------------------- /opal/core/measure.py: -------------------------------------------------------------------------------- 1 | import copy 2 | 3 | from data import Data 4 | from data import DataTable 5 | from tools import TableFormatter 6 | 7 | 8 | class Measure(Data): 9 | ''' 10 | 11 | A Measure object represent for an observation from output of 12 | algorithm running. It has two aspect 13 | - Data: its value is value of an observation 14 | - Functionality: It encapsulates the way to extract measure value 15 | from the output. 16 | ''' 17 | 18 | def __init__(self, name=None, description='', kind=None, **kwargs): 19 | Data.__init__(self, name=name, description=description, type=kind) 20 | return 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /opal/core/model.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import os.path 4 | import pickle 5 | import log 6 | 7 | from platform import Platform 8 | from ..Platforms import LINUX 9 | 10 | #from opal.core.modelstructure import ModelEvaluator 11 | 12 | __docformat__ = 'restructuredtext' 13 | 14 | 15 | class Model: 16 | def __init__(self, modelData=None, 17 | modelStructure=None, 18 | evaluatingOptions=None, 19 | dataFile='blackbox.dat', 20 | **kwargs): 21 | """ 22 | 23 | A `BlackBoxModel` encapsulates the 24 | information of a parameter optimization problem. 25 | From the parameter problem point of view, this class has 26 | two components: model data and model struture. 27 | 28 | Example:: 29 | 30 | blackbox = Model(modelStructure, modelData) 31 | 32 | An object of this class must contain a link to a solver. 33 | The link to a solver is created by the solver and added to the 34 | BlackBoxModel object upon solving the problem. 35 | 36 | """ 37 | 38 | self.data = modelData 39 | self.structure = modelStructure 40 | #self.runFileName = runFileName 41 | self.data_file = dataFile 42 | # The evaluating_options attribute accepts only 43 | # the options of simple type like boolean, integer 44 | # or string. In general, it accepts the options 45 | # of picklable data type. 46 | 47 | self.evaluating_options = {} 48 | # Update the running options from data 49 | self.evaluating_options.update(modelData.running_options) 50 | # If there is an option with the same name, it is overwritten by 51 | # the setting in model 52 | if evaluatingOptions is not None: 53 | self.evaluating_options.update(evaluatingOptions) 54 | self.evaluating_options.update(kwargs) 55 | 56 | # Get the information about used platform. Normally, platform object 57 | # is not be picklable, So we try to get the information to save 58 | # along with the model data, and use it to reconstruct the platform 59 | # object in run-time. 60 | # By default, LINUX is used 61 | self.platform_description = {'name':'LINUX', 62 | 'settings':{}} 63 | if 'platform' in self.evaluating_options.keys(): 64 | platform = self.evaluating_options['platform'] 65 | if type(platform) == type('a platform name'): 66 | self.platform_description['name'] = platform 67 | elif isinstance(platform, Platform): # A Platform object 68 | self.platform_description['name'] = platform.name 69 | self.platform_description['settings'] = platform.settings 70 | else: # Unable to recognize the specified platfom 71 | pass # Do nothing and use the default platform 72 | del self.evaluating_options['platform'] # Remove platform setting 73 | self.initialize() 74 | return 75 | 76 | def initialize(self): 77 | # Transformation to information of an optimization model 78 | # The variables are the parameters 79 | self.variables = self.data.parameters 80 | # Refomulate the constraints 81 | self.inequality_constraints = [] # c_i(x) <= 0 82 | self.equality_constraints = [] # c_e(x) = 0 83 | for constraint in self.structure.constraints: 84 | if constraint.lower_bound == constraint.upper_bound: 85 | self.equality_constraints.append(constraint) 86 | else: 87 | if constraint.lower_bound is not None: 88 | self.inequality_constraints.append(constraint) 89 | if constraint.upper_bound is not None: 90 | self.inequality_constraints.append(constraint) 91 | 92 | self.bounds = [var.bound for var in self.variables] 93 | 94 | # Initial points has at least one point that is the default values 95 | # of the parameters. 96 | self.initial_points = [[var.value for var in self.variables]] 97 | 98 | # The "simple constraints" that contain only the function of 99 | # parameters. This constraints will be verified before running 100 | # the test. 101 | # In the futre, the bound constraints will be considered as 102 | # simple_constraints too 103 | self.simple_constraints = [] 104 | pass 105 | 106 | # The methods to get information of a general model 107 | 108 | def get_n_variable(self): 109 | return len(self.variables) 110 | 111 | def get_n_constraints(self): 112 | return len(self.inequality_constraints) + len(self.equality_constraints) 113 | 114 | def get_initial_points(self): 115 | return self.initial_points 116 | 117 | def add_initial_point(self, point): 118 | converters = {'real':float, 119 | 'integer':int, 120 | 'categorical':str} 121 | initialPoint = [] 122 | for param, val in map(None, self.variables, point): 123 | if param is None: # The point is longer 124 | pass # do nothing 125 | elif val is None: # Set to default value 126 | initialPoint.append(param.get_default()) 127 | else: # Convert automatically to the corresponding type 128 | initialPoint.append(converters[param.kind](val)) 129 | self.initial_points.append(initialPoint) 130 | 131 | def get_bound_constraints(self): 132 | return self.bounds 133 | 134 | # The methods to get information of a OPAL model 135 | 136 | def get_algorithm(self): 137 | return self.data.get_algorithm() 138 | 139 | def get_parameters(self): 140 | return self.data.get_parameters() 141 | 142 | def get_problems(self): 143 | return self.data.get_problems() 144 | 145 | def get_measures(self): 146 | return self.data.get_measures() 147 | 148 | def get_structure(self): 149 | return self.structure 150 | 151 | 152 | 153 | 154 | -------------------------------------------------------------------------------- /opal/core/modeldata.py: -------------------------------------------------------------------------------- 1 | import os 2 | import string 3 | import types 4 | import time 5 | import shutil 6 | import log 7 | import copy 8 | #import logging 9 | 10 | #import utility 11 | from testproblem import TestProblem 12 | 13 | from .. import config 14 | 15 | 16 | 17 | # ============================= 18 | class ModelData: 19 | """ 20 | This class represents a data generator for a parameter optimization 21 | problem. The data is the values of the elementary measures that are needed 22 | to formulate the problem. To specify a data generator, we need provide: 23 | 24 | 1. The algorithm 25 | 2. the set of elementary measures concerned 26 | 3. the set of parameters to control 27 | 4. the test problems set. 28 | """ 29 | 30 | def __init__(self, 31 | algorithm, 32 | problems=[], 33 | parameters=None, 34 | measures=None, 35 | neighborhoods=[], 36 | **kwargs): 37 | # The core variables 38 | self.algorithm = algorithm 39 | if (problems is None) or (len(problems) == 0): 40 | self.problems = [TestProblem(name='TESTPROB')] 41 | else: 42 | self.problems = problems 43 | 44 | if parameters is None: 45 | self.parameters = algorithm.parameters 46 | else: 47 | self.parameters = parameters 48 | 49 | if measures is None: 50 | self.measures = algorithm.measures 51 | else: 52 | self.measures = measures 53 | # The information of parameter space can be provided by definitions 54 | # of neighborhood. 55 | self.neighborhoods = neighborhoods 56 | 57 | self.running_options = {} 58 | self.running_options.update(kwargs) 59 | return 60 | 61 | def get_problems(self): 62 | return self.problems 63 | 64 | def get_algorithm(self): 65 | return self.algorithm 66 | 67 | def get_parameters(self): 68 | return self.parameters 69 | 70 | def get_measures(self): 71 | return self.measures 72 | 73 | def save(self, fileName): 74 | return 75 | 76 | def load(self, fileName): 77 | return 78 | 79 | -------------------------------------------------------------------------------- /opal/core/modelevaluator.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import os.path 4 | import pickle 5 | import log 6 | import hashlib 7 | 8 | from .mafrw import Agent 9 | from .mafrw import Message 10 | from .datagenerator import DataGenerator 11 | from .structureevaluator import StructureEvaluator 12 | from ..Platforms import supported_platforms 13 | 14 | #from opal.core.modelstructure import ModelEvaluator 15 | 16 | __docformat__ = 'restructuredtext' 17 | 18 | 19 | class ModelEvaluator(Agent): 20 | def __init__(self, 21 | name='model evaluator', 22 | model=None, 23 | modelFile=None, 24 | options={}, 25 | logHandlers=[]): 26 | Agent.__init__(self, name=name, logHandlers=logHandlers) 27 | self.options = {'platform': 'LINUX', 28 | 'synchronized': False, 29 | 'interruptible': True} 30 | self.options.update(options) 31 | if model is None: 32 | if modelFile is not None: 33 | # The model is loaded by pickling 34 | # Be able to be serialized is a requirement for 35 | # a model object 36 | f = open(modelFile) 37 | model = pickle.load(f) 38 | f.close() 39 | 40 | self.model = model 41 | self.model_file = modelFile 42 | self.options.update(self.model.evaluating_options) 43 | 44 | # if platform option is provided by a string representing the name of 45 | # a OPAL-supported platform 46 | 47 | self.message_handlers['cfp-evaluate-point'] = \ 48 | self.activate_parameter_evaluation 49 | self.message_handlers['inform-experiment-failed'] = \ 50 | self.handle_experiment_failed 51 | self.message_handlers['inform-objective-partially-exceed'] = \ 52 | self.estimate_partially_model 53 | self.message_handlers['inform-constraint-partially-violated'] = \ 54 | self.estimate_partially_model 55 | return 56 | 57 | def register(self, environment): 58 | ''' 59 | 60 | After adding his name in the environment database, it will look for 61 | the agents that work as data manager, experiment manager and structure 62 | computer. 63 | If it could not find one of these agents within environment, it will 64 | create them and let them register to environment 65 | 66 | The find process is realized by sending a test message to environment 67 | and wait for replying. 68 | ''' 69 | Agent.register(self, environment) 70 | 71 | if self.model is None: 72 | return 73 | 74 | # Find the necessary collaborators. If could find it, create the 75 | # predefinded collaborators 76 | 77 | if self.find_collaborator('data generator', environment) is None: 78 | dataGenerator =\ 79 | DataGenerator(algorithm=self.model.get_algorithm(), 80 | parameters=self.model.get_parameters(), 81 | problems=self.model.get_problems(), 82 | platform=self.model.platform_description) 83 | dataGenerator.register(environment) 84 | 85 | if self.find_collaborator('structure evaluator', environment) is None: 86 | structureEvaluator = \ 87 | StructureEvaluator(structure=self.model.structure, 88 | problems=self.model.get_problems(), 89 | measures=self.model.get_measures()) 90 | structureEvaluator.register(environment) 91 | 92 | return 93 | 94 | def find_collaborator(self, name, environment): 95 | return None 96 | 97 | def create_tag(self, point): 98 | valuesStr = '_' 99 | for coordinate in point: 100 | valuesStr = valuesStr + str(coordinate) + '_' 101 | return hashlib.sha1(valuesStr).hexdigest() 102 | 103 | # Message handlers 104 | 105 | def activate_parameter_evaluation(self, info): 106 | parameterValues = info['proposition']['point'] 107 | if 'tag' in info['proposition'].keys(): 108 | parameterTag = info['proposition']['tag'] 109 | else: 110 | parameterTag = self.create_tag(parameterValues) 111 | 112 | message = Message(sender=self.id, 113 | performative='cfp', 114 | content={'action':'evaluate-parameter', 115 | 'proposition':{'parameter':parameterValues, 116 | 'tag':parameterTag} 117 | }) 118 | self.send_message(message) 119 | return 120 | 121 | def handle_experiment_failed(self, info): 122 | paramTag = info['proposition']['parameter-tag'] 123 | reason = info['proposition']['why'] 124 | # Send an inform message about the model-value with the value 125 | # is set to None. That indicates that there is something wrong in model 126 | # evaluation. 127 | message = Message(sender=self.id, 128 | performative='inform', 129 | content={'proposition':{'what':'model-value', 130 | 'values':None, 131 | 'why':reason, 132 | 'parameter-tag':paramTag 133 | } 134 | }) 135 | self.send_message(message) 136 | return 137 | 138 | def estimate_partially_model(self, info): 139 | 140 | paramTag = info['proposition']['parameter-tag'] 141 | message = Message(sender=self.id, 142 | performative='inform', 143 | content={'proposition':{'what':'model-value', 144 | 'values':None, 145 | 'why':'parameters invalid', 146 | 'parameter-tag':paramTag 147 | } 148 | }) 149 | self.send_message(message) 150 | return 151 | 152 | def stop(self, info=None): 153 | ''' 154 | 155 | Message handlers for STOP signal 156 | ''' 157 | # the model is saved back to file 158 | f = open(self.model_file, 'w') 159 | pickle.dump(self.model, f) 160 | f.close() 161 | Agent.stop(self, info) 162 | return 163 | 164 | 165 | -------------------------------------------------------------------------------- /opal/core/opalproblem.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import os.path 4 | import pickle 5 | import log 6 | 7 | #from opal.core.modelstructure import ModelEvaluator 8 | 9 | __docformat__ = 'restructuredtext' 10 | 11 | class BlackBox: 12 | """ 13 | This class manages the communication with the direct solver. 14 | The specific implementation depends on the solver used. For example, when 15 | the solver is NOMAD, an object of this class represents 16 | an executable file whose I/O methods depend on the specific solver. 17 | """ 18 | def __init__(self, solver=None, model=None, **kwargs): 19 | self.solver = solver 20 | self.model = model 21 | pass 22 | 23 | def run(self, *args, **kwargs): 24 | inputValues = [] 25 | paramValues = [] 26 | #print args 27 | (inputValues, paramValues) = self.read_input(*args, **kwargs) 28 | if self.model is None: 29 | return 30 | #print inputValues 31 | (objective,constraints) = self.model.evaluate(inputValues) 32 | self.write_output(objective,constraints) 33 | return 34 | 35 | def read_input(self,*args,**kwargs): 36 | return (None,None) 37 | 38 | def write_output(self,objectiveValue, constraintValues): 39 | return 40 | 41 | def set_parameter(self,*args,**kwargs): 42 | return 43 | 44 | class BlackBoxModel: 45 | def __init__(self, modelData=None, modelStructure=None, 46 | runFileName='blackbox.py', dataFile='blackbox.dat', 47 | logHandlers=[], **kwargs): 48 | """ 49 | 50 | A `BlackBoxModel` encapsulates the 51 | information of a parameter optimization problem. 52 | From the parameter problem point of view, this class has 53 | two components: model data and model struture. 54 | 55 | Example:: 56 | 57 | blackbox = BlackBoxModel(modelStructure, modelData) 58 | 59 | An object of this class must contain a link to a solver. 60 | The link to a solver is created by the solver and added to the 61 | BlackBoxModel object upon solving the problem. 62 | 63 | """ 64 | 65 | self.model_data = modelData 66 | self.model_structure = modelStructure 67 | #self.runFileName = runFileName 68 | self.data_file = dataFile 69 | self.logger = log.OPALLogger(name='opalBlackboxModel', 70 | handlers=logHandlers) 71 | 72 | 73 | activeParameters = self.model_data.get_parameters() 74 | 75 | self.n_var = len(activeParameters) 76 | self.m_con = len(self.model_structure.constraints) 77 | self.initial_points = [param.value for param in activeParameters] 78 | 79 | self.bounds = [param.bound for param in activeParameters] 80 | # The "simple constraints" that contain only the function of 81 | # parameters. This constraints will be verified before running 82 | # the test. 83 | # In the futre, the bound constraints will be considered as 84 | # simple_constraints too 85 | self.simple_constraints = [] 86 | 87 | 88 | #self.solver = None 89 | #self.surrogate = None 90 | # if no solver is specified, the blackbox is a general 91 | # executable file with two arguments: input file and parameter file 92 | # The output is standard screen 93 | self.save() 94 | pass 95 | 96 | #def set_options(self,**kwargs): 97 | # return 98 | 99 | #def has_surrogate(self): 100 | # return self.surrogate is not None 101 | 102 | #def get_surrogate(self): 103 | # return self.surrogate 104 | 105 | 106 | def evaluate(self, inputValues): 107 | """ 108 | Evaluate the model at given point 109 | Input: evaluated point coordinate 110 | Output: Value of objective function and constrains values list 111 | In the case of error, two None values are returned 112 | """ 113 | self.logger.log('Begin a blackbox evaluation') 114 | self.model_data.run(inputValues) 115 | testResult = self.model_data.get_test_result() 116 | #print 'ho ho after getTestResult', self.modelData.measures[0],\ 117 | # self.modelData.measures[0].valuetable 118 | # An evaluator object may be redudant, remove it in the future 119 | #modelEvaluator = ModelEvaluator(self.model_structure, 120 | # self.model_data.measures) 121 | (funcObj, constraints) = self.model_structure.evaluate(testResult) 122 | #print funcObj 123 | #print constraints 124 | self.logger.log('End of blackbox evaluation') 125 | return (funcObj, constraints) 126 | 127 | def save(self): 128 | #try: 129 | blackboxDataFile = open(self.data_file, "w") 130 | pickle.dump(self, blackboxDataFile) 131 | blackboxDataFile.close() 132 | #except TypeError: 133 | # print "Error in saving" 134 | return 135 | 136 | #def log(self): 137 | # if self.model_data.log != None: 138 | # self.model_data.log(self.logFileName) 139 | # if self.model_structure.log != None: 140 | # self.model_structure.log(self.logFileName) 141 | # return 142 | 143 | def get_iniitial_points(self): 144 | return self.initial_points 145 | 146 | def get_bound_constraints(self): 147 | return self.bounds 148 | 149 | #def generate_surrogate(self): 150 | # reducedModelData = self.model_data.reduce_problem_set() 151 | # data_fname = self.dataFileName.strip('.dat') + '_surrogate.dat' 152 | # log_fname = self.logFileName.strip('.log') + '_surrogate.log' 153 | # surrogate = BlackBoxModel(modelData=reducedModelData, 154 | # modelStructure=self.model_structure, 155 | # dataFileName=data_fname, 156 | # logFileName=log_fame) 157 | # return surrogate 158 | -------------------------------------------------------------------------------- /opal/core/parameter.py: -------------------------------------------------------------------------------- 1 | import string 2 | 3 | from opal.core.data import Data, DataSet 4 | 5 | class Parameter(Data): 6 | """ 7 | An abstract class to represent real, integer and categorical algorithmic 8 | parameters. Examples: 9 | 10 | >>> x = Parameter() # A real parameter with no default value 11 | >>> n = Parameter(kind='integer', default=3) 12 | >>> choice = Parameter(kind='categorical', default='left') 13 | 14 | Parameters can be given a name: 15 | 16 | >>> delmin = Parameter(default=1.0e-3, name='DELMIN') 17 | 18 | Their value can be changed from their default value: 19 | 20 | >>> delmin.set_value(0.1) 21 | 22 | Their default value is still available: 23 | 24 | >>> print delmin.get_default() 25 | 1.0e-3 26 | 27 | The bounds on a parameter may be a tuple indicating the lower and upper 28 | bounds if the parameter has an ordered kind, such as integer or real. 29 | Otherwise, `bound` is a set of feasible values. 30 | """ 31 | 32 | def __init__(self, kind='real', default=None, bound=None, name=None, 33 | description='', **kwargs): 34 | if kind not in ['real', 'integer', 'binary', 'categorical']: 35 | raise TypeError, 'kind must be real, integer or categorical' 36 | 37 | if kind == 'real' and type(default) != type(0.0): 38 | raise ValueError, 'default value must agree with type' 39 | 40 | if kind == 'integer' and type(default) != type(0): 41 | raise ValueError, 'default value must agree with type' 42 | 43 | self.kind = kind 44 | 45 | self.is_real = (kind == 'real') 46 | self.is_integer = (kind == 'integer') 47 | self.is_categorical = (kind == 'categorical') 48 | 49 | # The attribute value store the value of parameter in run time. 50 | # this is a run-time information 51 | # The _default is a description 52 | if default is not None: 53 | if kind == 'real' and type(default) != type(0.0): 54 | raise ValueError, 'default value must agree with type' 55 | if kind == 'integer' and type(default) != type(0): 56 | raise ValueError, 'default value must agree with type' 57 | self._default = default 58 | else: 59 | if self.kind == 'real': 60 | self._default = 0.0 61 | elif self.kind == 'integer': 62 | self._default = 0 63 | elif self.kind == 'binary': 64 | self._default = True 65 | else: 66 | self._default = 'something' 67 | 68 | Data.__init__(self, 69 | name=name, 70 | description=description, 71 | type=kind, 72 | value=self._default, 73 | dimension=1, 74 | domain=bound) 75 | self.bound = bound 76 | return 77 | 78 | def get_default(self): 79 | "Return default value" 80 | return self._default 81 | 82 | def set_default(self,value): 83 | "Set default value." 84 | if self.is_real: 85 | self._default = float(value) 86 | elif self.is_integer: 87 | self._default = int(value) 88 | else: 89 | self._default = value 90 | self.value = self._default 91 | return 92 | 93 | def set_value(self, value): 94 | "Set parameter value to a non-default value." 95 | if value is None: 96 | self.value = self._default 97 | return 98 | if self.is_real: 99 | self.value = float(value) 100 | elif self.is_integer: 101 | self.value = int(value) 102 | else: 103 | self.value = value 104 | return 105 | 106 | def set_as_const(self): 107 | "Treat parameter as a constant." 108 | self.kind = 'const' 109 | return 110 | 111 | def get_kind(self): 112 | "Return parameter kind." 113 | return self.kind 114 | 115 | def is_const(self): 116 | "Return True if parameter is being treated as a constant." 117 | return self.kind == 'const' 118 | 119 | def set_bound(self, bound): 120 | """ 121 | 122 | Set bounds on the parameter. For parameters of ordered kind, 123 | `set_bound((None, 0))` means that the lower bound is unchanged while 124 | the upper bound is reset to zero. For parameters of unordered kind, 125 | `set_bound` simply reassigns the list of feasible values. 126 | """ 127 | if self.is_categorical: 128 | self.bound = bound 129 | else: 130 | if self.bound is None: 131 | self.bound = (None, None) 132 | if bound[0] is not None: 133 | self.bound[0] = bound[0] 134 | if bound[1] is not None: 135 | self.bound[1] = bound[1] 136 | return 137 | 138 | def get_bound(self): 139 | """ 140 | 141 | Return bounds on the parameter. 142 | """ 143 | return self.bound 144 | 145 | def is_valid(self, value=None): 146 | """ 147 | 148 | Checks whether or not the specified value falls within the allowed 149 | range for this parameter. If `value` is None, the current value of 150 | the parameter is used. 151 | """ 152 | 153 | if value is not None: 154 | valueToVerify = value 155 | else: 156 | valueToVerify = self.value 157 | if self.bound is None: 158 | return True 159 | if self.is_categorical: 160 | return valueToVerify in self.bound 161 | # There is the error in transforming from string to int or float 162 | # For example, the value 0.0010000000 (in string in input file) is 163 | # 0.00100000000001 after forcing it as real number 164 | # Pay attention to verify the bound at bounded point 165 | if self.bound[0] is not None: 166 | if valueToVerify < self.bound[0]: 167 | return False 168 | if self.bound[1] is not None: 169 | if valueToVerify > self.bound[1]: 170 | return False 171 | return True 172 | 173 | def export_to_dict(self): 174 | """ 175 | 176 | Convert `Parameter` object to a dictionary. 177 | 178 | .. warning:: 179 | 180 | This does not include the bounds!!! 181 | """ 182 | return {'kind':self.kind, 183 | 'name':self.name, 184 | 'value':self.value, 185 | 'default':self._default} 186 | 187 | 188 | class ParameterConstraint: 189 | """ 190 | .. warning:: 191 | 192 | Document this class!!! 193 | """ 194 | 195 | violated = False 196 | def __init__(self,constraintStr='',*argv): 197 | self.constraintStr = constraintStr 198 | pass 199 | 200 | def __call__(self,parameters): 201 | constraintStr = self.constraintStr 202 | for param in parameters : 203 | constraintStr = constraintStr.replace(param.name,str(param.value)) 204 | return eval(constraintStr) 205 | 206 | 207 | def _test(): 208 | import doctest 209 | return doctest.testmod() 210 | 211 | 212 | def test_parameter_class(): 213 | p = Parameter(name='real_param') 214 | return 215 | 216 | if __name__ == "__main__": 217 | _test() 218 | 219 | -------------------------------------------------------------------------------- /opal/core/platform.py: -------------------------------------------------------------------------------- 1 | from mafrw import Agent 2 | from mafrw import Message 3 | 4 | class Task(Agent): 5 | def __init__(self, 6 | name=None, 7 | taskId=None, 8 | command=None, 9 | sessionTag=None, 10 | input=None, 11 | output='/dev/null', 12 | logHandlers=[]): 13 | ''' 14 | 15 | Each task object correspond to an application of the target 16 | algorithm on a test problem. 17 | 18 | ''' 19 | self.task_id = taskId # task_id is assigned by platform 20 | self.command = command 21 | self.output = output 22 | if name is None: 23 | Agent.__init__(self, name=command, logHandlers=logHandlers) 24 | else: 25 | Agent.__init__(self, name=name, logHandlers=logHandlers) 26 | if sessionTag is None: 27 | self.session_tag = self.name 28 | else: 29 | self.session_tag = sessionTag 30 | return 31 | 32 | def run(self): 33 | ''' 34 | 35 | The common activity of all task is send a message that informs 36 | termination of its work 37 | ''' 38 | message = Message(sender=self.id, 39 | performative='inform', 40 | receiver=None, 41 | content={'proposition':{'who':self.name, 42 | 'what':'task-finish', 43 | 'how':'no-error'} 44 | } 45 | ) 46 | self.send_message(message) 47 | message = Message(sender=self.id, 48 | performative='cfp', 49 | receiver=None, 50 | content={'action':'collect-result', 51 | 'proposition':{'session-tag':self.session_tag} 52 | } 53 | ) 54 | self.send_message(message) 55 | self.unregister() 56 | return 57 | 58 | 59 | 60 | class QueueSystem: 61 | # The default queue system managed the tasks in one or many queue 62 | # where each queue is identified by name 63 | def __init__(self): 64 | self.tasks = {'default':[]} 65 | return 66 | 67 | def get_length(self): 68 | length = 0 69 | for queueTag in self.tasks: 70 | length = length + len(self.tasks[queueTag]) 71 | return length 72 | 73 | def append(self, task, queue=None): 74 | if (queue is None) or (queue is 'default'): 75 | self.tasks['default'].append(task) 76 | else: 77 | if queue in self.tasks.keys(): 78 | self.tasks[queue].append(task) 79 | else: 80 | self.tasks[queue] = [task] 81 | return 82 | 83 | def pop(self, queue=None): 84 | if (queue is None) or (queue is 'default'): 85 | if len(self.tasks['default']) > 0 : 86 | return self.tasks['default'].pop() 87 | for queue in self.tasks.keys(): 88 | if queue is not 'default': 89 | return self.tasks[queue].pop() 90 | raise Exception('The task queue is empty') 91 | 92 | if queue in self.tasks.keys(): 93 | task = self.tasks[queue].pop() 94 | # Remove the queue if it is empty 95 | if len(self.tasks[queue]) == 0: 96 | del self.tasks[queue] 97 | return task 98 | raise Exception('The task queue does not exist') 99 | 100 | 101 | def remove_tasks(self, queue=None): 102 | if (queue is None) or (queue is 'default'): 103 | del self.tasks['default'][0:] 104 | return 105 | if queue in self.tasks.keys(): 106 | del self.tasks[queue] 107 | return 108 | return 109 | 110 | class Platform(Agent): 111 | def __init__(self, 112 | name='platform', 113 | maxTask=1, 114 | synchronous=False, 115 | queueSystem=None, 116 | settings=None, 117 | logHandlers=[], 118 | **kwargs): 119 | # A platform can contains many queues 120 | if queueSystem is None: 121 | self.queue_system = QueueSystem() 122 | else: 123 | self.queue_system = queueSystem 124 | self.settings = {'MAX_TASK':maxTask, 125 | 'SYNCHRONOUS':synchronous, 126 | 'OPTIONS':None} 127 | if settings is not None: 128 | self.settings.update(settings) 129 | self.settings.update(kwargs) 130 | 131 | # Running information 132 | self.running = {} 133 | self.task_id = 0 134 | Agent.__init__(self, name=name, logHandlers=logHandlers) 135 | self.message_handlers['inform-task-finish'] = self.finalize_task 136 | self.message_handlers['cfp-cancel-queue'] = self.cancel_queue 137 | return 138 | 139 | def set_parameter(self, settings=None, **kwargs): 140 | if settings is not None: 141 | self.settings.update(settings) 142 | self.settings.update(kwargs) 143 | return 144 | 145 | def submit(self, task, queue=None): 146 | # A task is created by each platform 147 | task.register(self.environment) 148 | self.queue_system.append(task, queue) 149 | #self.logger.log('Task ' + task.name + ' is added to queue') 150 | return task.id 151 | 152 | def finalize_task(self, info): 153 | taskName = info['proposition']['who'] 154 | del self.running[taskName] 155 | return 156 | 157 | def run(self): 158 | while self.working : 159 | if not self.settings['SYNCHRONOUS'] or len(self.running) == 0: 160 | # Submit task when there is no 161 | # running task 162 | #self.logger.log('Begin a launching session, we have ' + \ 163 | # str(len(self.queue)) + ' task') 164 | while (len(self.running) < self.settings['MAX_TASK']) and \ 165 | (self.queue_system.get_length() > 0): 166 | task = self.queue_system.pop() 167 | self.running[task.name] = task 168 | task.start() 169 | #self.logger.log('Task: ' + str(task.name) + ' is launched') 170 | # Work as an agent 171 | messages = self.fetch_messages() 172 | for msg in messages: 173 | self.handle_message(msg) 174 | del messages 175 | return 176 | 177 | # Message handlers 178 | 179 | def cancel_queue(self, info): 180 | if 'queue' in info['proposition'].keys(): 181 | queueTag = info['proposition']['queue'] 182 | else: 183 | queueTag = None 184 | self.queue_system.remove_tasks(queue=queueTag) 185 | return 186 | 187 | 188 | -------------------------------------------------------------------------------- /opal/core/savablefunction.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os.path 3 | import marshal 4 | import new 5 | import log 6 | 7 | class SavableFunction: 8 | """ 9 | This class contains the information of a measure function that 10 | built up from the parameter and elementary measure \varphi(p,\mu) 11 | """ 12 | def __init__(self, function=None, name=None,**kwargs): 13 | if function is None: 14 | raise Exception('The savable function definition is invalid') 15 | # The information of a function include all possible description 16 | # such as convexity, ... Any information is accepted 17 | # We concentrate on a property called possitively-additive. 18 | # A function objective is called possitively-additive if function value 19 | # of partial data is always less than or equal to the data-full one 20 | 21 | self.information = {} 22 | self.information.update(kwargs) 23 | # It's important to define a function with two arguments: 24 | #the parameter and the measure 25 | # We check if the given function sastifies this constraint: 26 | 27 | self.func = function 28 | if name is None: 29 | self.name = function.__code__.co_name 30 | else: 31 | self.name = name 32 | #self.name = function.__code__.co_name 33 | #self.code_string = None 34 | pass 35 | 36 | def evaluate(self, *args, **kwargs): 37 | if self.func is None: 38 | raise Exception('The measure function is not defined') 39 | return self.func(*args, **kwargs) 40 | #self.load() 41 | #value = self.func(*args, **kwargs) 42 | #del self.func 43 | #self.func = None # Keep self.func is None for the next pickling 44 | #return value 45 | 46 | def __getstate__(self): 47 | content = {} 48 | content['code'] = marshal.dumps(self.func.__code__) 49 | content['information'] = self.information 50 | content['name'] = self.name 51 | return content 52 | 53 | def __setstate__(self, content): 54 | self.func = new.function(marshal.loads(content['code']),globals()) 55 | self.information = content['information'] 56 | self.name = content['name'] 57 | return 58 | 59 | def __call__(self,*args,**kwargs): 60 | return self.evaluate(*args,**kwargs) 61 | 62 | 63 | -------------------------------------------------------------------------------- /opal/core/set.py: -------------------------------------------------------------------------------- 1 | class Set: 2 | """ 3 | 4 | Set is a group of elements distinguished by identity (given by identify() 5 | method). The Set class cover the apsects of two Python built-in data 6 | structures: dict and list. It keeps the order of the element like it added 7 | to Set. This means that we can access an element by its position. Each 8 | elements in the Set is assigned a key that is actually the identity of 9 | elements. A requirement applied to the elements of a Set object is each 10 | element has a method that show its identity. 11 | """ 12 | def __init__(self,name="", elements=[], *argv, **kwargv): 13 | self.name = name 14 | self.indices = {} 15 | self.db = [] 16 | if len(elements) > 0: 17 | index = 0 18 | for elem in elements: 19 | self.indices[elem.identify()] = index 20 | self.db.append(elem) 21 | index = index + 1 22 | return 23 | 24 | def __getitem__(self, id): 25 | ''' 26 | 27 | A data set object provide two ways to access an element: by order or by 28 | identity that provided by methode {\sf identify()} 29 | ''' 30 | if (type(id) == type(0)): 31 | return self.db[id] 32 | else: 33 | return self.db[self.indices[id]] 34 | 35 | def __len__(self): 36 | return len(self.db) 37 | 38 | def __contains__(self, elem): 39 | ''' 40 | 41 | There are two way to verify the existence of an element in a DataSet 42 | object. 43 | Either element or its identity can be provided for the verification. 44 | ''' 45 | # if this is an empty DataSet object the False signal is returned 46 | # immediately 47 | if len(self.indices) <= 0: 48 | return False 49 | idType = type(self.indices.keys()[0]) 50 | # Identity is provided to the verifcation 51 | if type(elem) == idType: 52 | return (elem in self.indices.keys()) 53 | # Element is provided 54 | else: 55 | try: 56 | return (elem.identify() in self.indices.keys()) 57 | except: 58 | return False 59 | 60 | def append(self, elem): 61 | ''' 62 | 63 | Add an element to the set 64 | ''' 65 | # An element with the same name is in the set. Nothing to add 66 | if elem.identify() in self.indices: 67 | return 68 | self.indices[elem.identify()] = len(self.db) 69 | self.db.append(elem) 70 | return 71 | 72 | def remove(self, elem): 73 | ''' 74 | 75 | Remove an element from the set. 76 | ''' 77 | if len(self.inidces) <= 0: 78 | return 79 | idType = type(self.indices.keys()[0]) 80 | if type(elem) == idType: 81 | try: 82 | index = self.indices[elem] 83 | except: 84 | raise IndexError 85 | else: 86 | try: 87 | index = self.indices[elem.identify()] 88 | except: 89 | raise IndexError 90 | i = index + 1 91 | # Update the indices 92 | while i < len(self.db): 93 | id = self.db[i].identify() 94 | self.indices[id] = self.indices[id] - 1 95 | i = i + 1 96 | # Remove the element from the database and remove its index 97 | id = self.db[index].identify() 98 | self.db.remove(self.db[index]) 99 | del self.indices[id] 100 | return 101 | 102 | def select(self, query): 103 | queryResult = Set(name='query-result') 104 | for elem in self.db: 105 | if query.match(elem): 106 | queryResult.append(prob) 107 | return queryResult 108 | -------------------------------------------------------------------------------- /opal/core/solver.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | class Solver: 4 | 5 | def __init__(self, name='', command=None, input=None, parameter='', 6 | output=None, **kwargs): 7 | self.name = name 8 | self.command = command 9 | self.args = [input, parameter, output] 10 | return 11 | 12 | #def run(self): 13 | # cmdStr = self.command 14 | # for argv in self.args: 15 | # if argv != None: 16 | # cmdStr = cmdStr + ' ' + str(argv) 17 | # os.system(cmdStr) 18 | # return 19 | 20 | def solve(self, problem, **kwargs): 21 | """ 22 | The solve method return a process describing the solving process 23 | according solver's algorithm. 24 | The solving process is either an iterative process or batch process. 25 | In the case of an iterative process, the user can interfere to 26 | solving process in each iteration. 27 | Otherwise, the user just runs the process and get final result 28 | Two form of invoking a solver 29 | 30 | >>> solvingProc = aSolver.solve(problem) 31 | >>> finalResult = solvingProc.run(problem.initial_point) 32 | 33 | >>> solvingProc = aSolver.solve(problem) 34 | >>> solvingProc.initialize(problem.initial_point) 35 | >>> while solvingProc.next() : 36 | >>> result = solvingProc.get_status() 37 | 38 | The advantage of this appoach is that, the solving process can 39 | be invoked with different initial information, for example the 40 | initial point 41 | """ 42 | 43 | return None 44 | -------------------------------------------------------------------------------- /opal/core/statsmeasure.py: -------------------------------------------------------------------------------- 1 | class StatisticalMeasure: 2 | def average(measureName): 3 | return lambda p,m : m[measureName].mean() 4 | average = staticmethod(average) 5 | 6 | 7 | -------------------------------------------------------------------------------- /opal/core/test_core.py: -------------------------------------------------------------------------------- 1 | def test_multi_agent_framework(): 2 | from mafrw import Agent 3 | from mafrw import Environment 4 | 5 | agent = Agent() 6 | env = Environment() 7 | 8 | agent.register(env) 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /opal/core/testproblem.py: -------------------------------------------------------------------------------- 1 | __docformat__ = 'restructuredText' 2 | 3 | from set import Set 4 | 5 | class TestProblem: 6 | """ 7 | Abstract class to describe a test problem in general form. 8 | Every test problem class is a sub-class of this class. 9 | The ``TestProblem`` class contains only basic information about the 10 | test problem: the problem name and a description (both strings). 11 | Example: 12 | 13 | >>> chem=TestProblem(name='Nuke', description='Chemical reactor') 14 | >>> print chem.name, chem.description 15 | """ 16 | 17 | def __init__(self, name='Test Problem', description=None, classifyStr=None, 18 | **kwargs): 19 | self.name = name 20 | self.description = description 21 | self.classify_string = classifyStr 22 | 23 | def get_name(self): 24 | return self.name 25 | 26 | def identify(self): 27 | return self.name 28 | 29 | def get_description(self): 30 | return self.description 31 | 32 | def get_classify_string(self): 33 | return self.classify_string 34 | 35 | 36 | class OptimizationTestProblem(TestProblem): 37 | """ 38 | An abstract class to represent a test problem in the field of optimization. 39 | Example: 40 | 41 | >>> hs26 = OptimizationTestProblem(name='HS26', nvar=3, ncon=1) 42 | >>> print hs26.name, hs26.nvar, hs26.ncon 43 | HS26 3 1 44 | """ 45 | def __init__(self, 46 | name='Optimization Problem', 47 | description=None, 48 | classifyStr=None, 49 | nvar=0, 50 | ncon=0, 51 | **kwargs): 52 | TestProblem.__init__(self, 53 | name=name, 54 | description=description, 55 | classifyStr=classifyStr, 56 | **kwargs) 57 | self.nvar = nvar 58 | self.ncon = ncon 59 | 60 | 61 | class ProblemSet(Set): 62 | """ 63 | An abstract class to represent a test set from which lists of 64 | specific problems may be extracted, and which may be passed to a solver. 65 | Example: 66 | 67 | >>> HS = ProblemSet(name='Hock and Schittkowski set') 68 | >>> hs13 = TestProblem(name='HS13', nvar=2, ncon=1) 69 | >>> hs26 = TestProblem(name='HS26', nvar=3, ncon=1) 70 | >>> HS.add_problem(hs13) 71 | >>> HS.add_problem(hs26) 72 | >>> print [prob.name for prob in HS] 73 | ['HS13', 'HS26'] 74 | 75 | In the above example, note that a `ProblemSet` is iterable. 76 | """ 77 | 78 | def __init__(self, name='Problem-Set', **kwargs): 79 | Set.__init__(self, name=name, **kwargs) 80 | return 81 | 82 | def add_problem(self, problem): 83 | "Add problem to collection." 84 | if isinstance(problem, TestProblem): 85 | self.append(problem) 86 | else: 87 | raise TypeError, 'Problem must be a TestProblem' 88 | 89 | def remove_problem(self, problem): 90 | self.remove(problem) 91 | 92 | 93 | def select(self, query): 94 | """ 95 | Return the list of problems matching the given query. 96 | The `query` object is any object that possesses a `match()` method. 97 | The result of `match(name, string)` must be True if `name` matches 98 | `string`. During the query, `name` is the problem name and `string` 99 | is its classification string. 100 | """ 101 | queryResult = ProblemSet(name='query result') 102 | for prob in self.db: 103 | if query.match(prob): 104 | queryResult.append(prob) 105 | return queryResult 106 | 107 | 108 | class ProblemCollection(ProblemSet): 109 | """ 110 | An abstract class to represet a collection of test problems. A collection 111 | is made up of subcollections, each of which is a `ProblemSet`. 112 | 113 | Example: 114 | 115 | >>> HS = ProblemSet(name='Hock and Schittkowski set') 116 | >>> hs13 = TestProblem(name='HS13', nvar=2, ncon=1) 117 | >>> hs26 = TestProblem(name='HS26', nvar=3, ncon=1) 118 | >>> HS.add_problem(hs13) 119 | >>> HS.add_problem(hs26) 120 | >>> NCVXQP = ProblemSet(name='Nonconvex Quadratic Programs') 121 | >>> ncvxqp2 = TestProblem(name='NCVXQP2', nvar=1000, ncon=2500) 122 | >>> NCVXQP.add_problem(ncvxqp2) 123 | >>> CUTEr = ProblemCollection(name='CUTEr collection') 124 | >>> CUTEr.add_subcollection(HS) 125 | >>> CUTEr.add_subcollection(NCVXQP) 126 | >>> print [coll.name for coll in CUTEr.subcollections] 127 | ['Hock and Schittkowski set', 'Nonconvex Quadratic Programs'] 128 | >>> print [prob.name for prob in CUTEr.all_problems()] 129 | ['HS13', 'HS26', 'NCVXQP2'] 130 | """ 131 | 132 | def __init__(self, name=None, description=None, **kwargs): 133 | self.name = name 134 | self.description = description 135 | ProblemSet.__init__(self, name='set of problems', **kwargs) 136 | self.subcollections = Set(name='sub-collections') # List of 137 | # subcollections of 138 | # this collection 139 | 140 | def __len__(self): 141 | size = ProblemSet.__len__(self) 142 | for collection in self.subcollections: 143 | size = size + collection.__len__() 144 | return size 145 | 146 | def __getitem__(self, key): 147 | try: 148 | return ProblemSet.__getitem__(self, key) 149 | except IndexError: 150 | for collection in self.subcollections: 151 | try: 152 | return collection.__getitem__(key) 153 | except IndexError: 154 | pass # Try to get from other collections 155 | # If all collections are searched wihtout result 156 | # an exception of index error is raised 157 | raise IndexError, 'Element can not be found in the set' 158 | 159 | def __contains__(self,prob): 160 | if ProblemSet.__contains__(self, prob): 161 | return True 162 | for collection in self.subcollections: 163 | # Search recursively in the subcollections 164 | if collection.__contains__(prob): 165 | return True 166 | return False 167 | 168 | def identify(self): 169 | return self.name 170 | 171 | def find_sub_collection(self, collectionId): 172 | for collection in self.subcollections: 173 | if collection.identify() is collectionId: 174 | return collection 175 | result = collection.find_sub_collection(collectionId) 176 | if result is not None: 177 | return result 178 | return None # No subcollection can be found 179 | 180 | def add_subcollection(self, collection): 181 | "Add a subcollection to this collection." 182 | if isinstance(collection, ProblemCollection): 183 | self.subcollections.append(collection) 184 | else: 185 | raise TypeError, 'Collection must be a ProblemCollection object' 186 | 187 | 188 | def _test(): 189 | import doctest 190 | return doctest.testmod() 191 | 192 | if __name__ == "__main__": 193 | _test() 194 | -------------------------------------------------------------------------------- /opal/core/tools.py: -------------------------------------------------------------------------------- 1 | # Miscellaneous tools. 2 | import re 3 | 4 | def extract_measure(content, description, name, valueType = 'int'): 5 | numberPattern = {'int':'\d+', 6 | 'real':'-?\d*\.\d+', 7 | 'float':'-?\d*\.\d+E(\+|-)\d+'} 8 | converters = {'int':int, 9 | 'real': float, 10 | 'float': float} 11 | 12 | matches = re.finditer(description + '\s+:\s+(?P<' + 13 | name + '>' + 14 | numberPattern[valueType] + ')', 15 | content) 16 | value = None 17 | for m in matches: 18 | if name not in m.groupdict().keys(): 19 | raise Exception('Could not to extract measure ' + name) 20 | continue 21 | value = converters[valueType](m.groupdict()[name]) 22 | return value 23 | 24 | 25 | class TableFormatter: 26 | def __init__(self, fieldDelimiter=' ', recordDelimiter='\n'): 27 | self.field_delimiter = fieldDelimiter 28 | self.record_delimiter = recordDelimiter 29 | self.row_template = None 30 | return 31 | 32 | def set_header(self, headers=None, *args): 33 | cols = [] 34 | if headers is not None: 35 | cols.extend(headers) 36 | cols.extend(args) 37 | if len(cols) == 0: 38 | return None 39 | self.row_template = '{0:<}' 40 | headerStr = 'PROB' 41 | for col in cols: 42 | self.row_template = self.row_template + self.field_delimiter + \ 43 | '{1['+ col + ']:>}' 44 | headerStr = headerStr + self.field_delimiter + col 45 | self.row_template = self.row_template + self.record_delimiter 46 | headerStr = headerStr + self.record_delimiter 47 | return headerStr 48 | 49 | def format(self, problem, record, **kwargs): 50 | record.update(kwargs) 51 | if len(record) == 0: 52 | return None 53 | try: 54 | return self.row_template.format(problem, record) 55 | except KeyError: 56 | return None 57 | -------------------------------------------------------------------------------- /opal/test_opal.py: -------------------------------------------------------------------------------- 1 | def test_opal(): 2 | return 3 | 4 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from distutils.core import setup 2 | 3 | setup(name='OPAL - OPtimization of ALgorithm', 4 | version='1.3', 5 | description='Algorithmic Parameter Optimization', 6 | author='Charles Audet, Cong-Kien Dang, Dominique Orban', 7 | author_email='cong-kien.dang@polymtl.ca', 8 | url='http://www.gerad.ca/~kiendc/build/hmtl', 9 | packages=['opal', 10 | 'opal.core', 11 | 'opal.Platforms', 12 | 'opal.Solvers', 13 | 'opal.TestProblemCollections'] 14 | ) 15 | 16 | -------------------------------------------------------------------------------- /test.py: -------------------------------------------------------------------------------- 1 | ''' 2 | 3 | This file for provoking a serie of tests relating all basic componnents 4 | in OPAL. It is used to provoke as well as the simple examples that are 5 | in the `examples` directory. 6 | 7 | The `nose` package is required to run the tests. 8 | 9 | For testing OPAL installation, just launch at shell prompt a command: 10 | 11 | shell$ python test.py 12 | 13 | For running a examples, for example the `finite-difference` examples: 14 | 15 | shell$ python test.py -w examples/fd 16 | ''' 17 | 18 | if __name__ == '__main__': 19 | try: 20 | from nose import main 21 | main() 22 | except NameError: 23 | print 'The NOSE package is required' 24 | --------------------------------------------------------------------------------