├── .gitignore ├── MANIFEST.in ├── README.rst ├── changelog.md ├── doc_src ├── Makefile ├── cite.rst ├── conf.py ├── fanova_starter.rst ├── includeme.rst ├── index.rst ├── install.rst └── manual.rst ├── docs ├── .buildinfo ├── .nojekyll ├── _images │ ├── Col1.png │ ├── Col2.png │ ├── figure1.png │ ├── figure2.png │ ├── pair.png │ └── pairwise.png ├── _sources │ ├── cite.txt │ ├── fanova_starter.txt │ ├── includeme.rst.txt │ ├── includeme.txt │ ├── index.rst.txt │ ├── index.txt │ ├── install.txt │ ├── manual.rst.txt │ └── manual.txt ├── _static │ ├── ajax-loader.gif │ ├── alabaster.css │ ├── basic.css │ ├── comment-bright.png │ ├── comment-close.png │ ├── comment.png │ ├── custom.css │ ├── default.css │ ├── doctools.js │ ├── down-pressed.png │ ├── down.png │ ├── file.png │ ├── jquery-3.1.0.js │ ├── jquery.js │ ├── minus.png │ ├── plus.png │ ├── pygments.css │ ├── searchtools.js │ ├── sidebar.js │ ├── underscore-1.3.1.js │ ├── underscore.js │ ├── up-pressed.png │ ├── up.png │ └── websupport.js ├── cite.html ├── doctrees │ ├── cite.doctree │ ├── environment.pickle │ ├── fanova_starter.doctree │ ├── includeme.doctree │ ├── index.doctree │ ├── install.doctree │ └── manual.doctree ├── fanova_starter.html ├── genindex.html ├── includeme.html ├── index.html ├── install.html ├── manual.html ├── objects.inv ├── search.html └── searchindex.js ├── examples ├── branin_pysmac_example.py ├── comparison │ ├── old_online_lda │ │ ├── Col1.png │ │ ├── instance-features.txt │ │ ├── instances.txt │ │ ├── pairwise.png │ │ ├── param-file-disc.txt │ │ ├── param-file.txt │ │ ├── paramstrings-it2.txt │ │ ├── runs_and_results-it2.csv │ │ ├── scenario.txt │ │ └── uniq_configurations-it2.csv │ ├── test_new.py │ ├── test_old.py │ └── test_old_new.py ├── example_data │ ├── csv-example │ │ └── test_data.csv │ ├── diabetes_features.csv │ ├── diabetes_responses.csv │ ├── online_lda │ │ ├── Col0.png │ │ ├── Col1.png │ │ ├── Col2.png │ │ ├── online_lda_features.csv │ │ ├── online_lda_responses.csv │ │ ├── pairwise.png │ │ └── param-file.txt │ └── test_plots │ │ └── ['0', '1'].png ├── fanova_example.py └── onlineLDA_example.py ├── fANOVA demo.ipynb ├── fanova ├── __init__.py ├── __version__.py ├── fanova.py └── visualizer.py ├── requirements.txt ├── setup.py └── tests ├── toy_data_set_features.csv ├── toy_data_set_responses.csv └── unit_tests.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | .project 3 | .pydevproject 4 | *.class 5 | .classpath 6 | *.prefs 7 | dist/ 8 | build/ 9 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include Readme.rst 2 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Fanova 2 | ====== 3 | 4 | Functional ANOVA: an implementation of the ICML 2014 paper "An Efficient Approach for Assessing Hyperparameter Importance" by Frank Hutter, Holger Hoos and Kevin Leyton-Brown. 5 | 6 | Documentation 7 | ------------- 8 | 9 | An 'ever growing' documentation for the Python bindings can be found at https://automl.github.io/fanova/ 10 | 11 | **Note** *To use fANOVA please make sure to use SWIG version 3.0.12. If you experience problems, try using anaconda or reinstalling pyrfr (#102).* 12 | -------------------------------------------------------------------------------- /changelog.md: -------------------------------------------------------------------------------- 1 | # 2.0.19 2 | 3 | ## Bug fixes 4 | 5 | * Fix #103, inverted labels in 3d-plot (using transpose) 6 | 7 | # 2.0.18 8 | 9 | ## Major Changes 10 | 11 | * Allow for X-input as `pandas.DataFrame`, add pandas requirement, based on #95 12 | * Also, log a warning, if input might be misinterpreted 13 | * Add support for `OrdinalHyperparameter` 14 | 15 | ## Minor Changes 16 | 17 | * Refactor some code for PEP8 conformity 18 | 19 | # 2.0.17 20 | 21 | ## Bug fixes 22 | 23 | * Fix #96, invert ticks and labels on categorical-categorical pairwise plot 24 | 25 | # 2.0.16 26 | 27 | ## Minor Changes 28 | 29 | * Prepare `visualizer.py` for `OrdinalHyperparameter`s 30 | * Generally simplify code for maintainablitiy 31 | * `generate_pairwise_marginal` does NOT swap values anymore 32 | 33 | ## Bug fixes 34 | 35 | * Fix inverted axes-bug (#96) 36 | 37 | # 2.0.15 38 | 39 | ## Bug fixes 40 | 41 | * Fix Constant-Handling and "swap"-axis logic (#93) 42 | 43 | # 2.0.14 44 | 45 | ## Minor Changes 46 | 47 | * Add try-except to pickling pairwise plots now fails silently (on info-level) 48 | * Add support for Constant hyperparameters 49 | * Beautify code in visualizer-class 50 | 51 | ## Bug fixes 52 | 53 | * Fix axis on pairwise mixed cat/non-cat plots 54 | * Fix visualizer's `create_all_plots` 55 | 56 | # 2.0.13 57 | 58 | ## Bug fixes 59 | 60 | * Unassigned variable for marginal plot 61 | 62 | # 2.0.12 63 | 64 | ## Major Changes 65 | 66 | * Enforce log-scale in plots if specified in argument of plot-function 67 | * Add option to plot incumbent(s) if it/they is/are passed 68 | 69 | ## Minor Changes 70 | 71 | * Add legend to plots 72 | * Add grid to plots 73 | 74 | # 2.0.11 75 | 76 | ## Bug fixes 77 | 78 | * Having a boolean in pairwise marginal plots does not lead to indice-crash anymore (#80) 79 | -------------------------------------------------------------------------------- /doc_src/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = ../docs 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " devhelp to make HTML files and a Devhelp project" 34 | @echo " epub to make an epub" 35 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 36 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 37 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 38 | @echo " text to make text files" 39 | @echo " man to make manual pages" 40 | @echo " texinfo to make Texinfo files" 41 | @echo " info to make Texinfo files and run them through makeinfo" 42 | @echo " gettext to make PO message catalogs" 43 | @echo " changes to make an overview of all changed/added/deprecated items" 44 | @echo " xml to make Docutils-native XML files" 45 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 46 | @echo " linkcheck to check all external links for integrity" 47 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 48 | 49 | clean: 50 | rm -rf $(BUILDDIR)/* 51 | 52 | html: 53 | @cd ..; python setup.py build_ext --inplace 54 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR) 55 | @echo 56 | @echo "Build finished. The HTML pages are in $(BUILDDIR)." 57 | 58 | dirhtml: 59 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 60 | @echo 61 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 62 | 63 | singlehtml: 64 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 65 | @echo 66 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 67 | 68 | pickle: 69 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 70 | @echo 71 | @echo "Build finished; now you can process the pickle files." 72 | 73 | json: 74 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 75 | @echo 76 | @echo "Build finished; now you can process the JSON files." 77 | 78 | htmlhelp: 79 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 80 | @echo 81 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 82 | ".hhp project file in $(BUILDDIR)/htmlhelp." 83 | 84 | qthelp: 85 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 86 | @echo 87 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 88 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 89 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/pyrfr.qhcp" 90 | @echo "To view the help file:" 91 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/pyrfr.qhc" 92 | 93 | devhelp: 94 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 95 | @echo 96 | @echo "Build finished." 97 | @echo "To view the help file:" 98 | @echo "# mkdir -p $$HOME/.local/share/devhelp/pyrfr" 99 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/pyrfr" 100 | @echo "# devhelp" 101 | 102 | epub: 103 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 104 | @echo 105 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 106 | 107 | latex: 108 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 109 | @echo 110 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 111 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 112 | "(use \`make latexpdf' here to do that automatically)." 113 | 114 | latexpdf: 115 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 116 | @echo "Running LaTeX files through pdflatex..." 117 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 118 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 119 | 120 | latexpdfja: 121 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 122 | @echo "Running LaTeX files through platex and dvipdfmx..." 123 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 124 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 125 | 126 | text: 127 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 128 | @echo 129 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 130 | 131 | man: 132 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 133 | @echo 134 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 135 | 136 | texinfo: 137 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 138 | @echo 139 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 140 | @echo "Run \`make' in that directory to run these through makeinfo" \ 141 | "(use \`make info' here to do that automatically)." 142 | 143 | info: 144 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 145 | @echo "Running Texinfo files through makeinfo..." 146 | make -C $(BUILDDIR)/texinfo info 147 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 148 | 149 | gettext: 150 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 151 | @echo 152 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 153 | 154 | changes: 155 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 156 | @echo 157 | @echo "The overview file is in $(BUILDDIR)/changes." 158 | 159 | linkcheck: 160 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 161 | @echo 162 | @echo "Link check complete; look for any errors in the above output " \ 163 | "or in $(BUILDDIR)/linkcheck/output.txt." 164 | 165 | doctest: 166 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 167 | @echo "Testing of doctests in the sources finished, look at the " \ 168 | "results in $(BUILDDIR)/doctest/output.txt." 169 | 170 | xml: 171 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 172 | @echo 173 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 174 | 175 | pseudoxml: 176 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 177 | @echo 178 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 179 | -------------------------------------------------------------------------------- /doc_src/cite.rst: -------------------------------------------------------------------------------- 1 | Citing fANOVA 2 | ================= 3 | 4 | If you use the fANOVA for your research, please cite the ICML 2014 paper "An Efficient Approach for Assessing Hyperparameter Importance" by Frank Hutter, Holger Hoos and Kevin Leyton-Brown. 5 | 6 | with the following Bibtex file: 7 | 8 | @inproceedings{HutHooLey14, 9 | lauthor = {Frank Hutter and Holger Hoos and Kevin Leyton-Brown}, 10 | author = {F. Hutter and H. Hoos and K. Leyton-Brown}, 11 | title = {An Efficient Approach for Assessing Hyperparameter Importance}, 12 | booktitle = {Proceedings of International Conference on Machine Learning 2014 (ICML 2014)}, 13 | year = {2014}, 14 | pages = {754--762}, 15 | month = jun, 16 | } 17 | -------------------------------------------------------------------------------- /doc_src/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # fanova documentation build configuration file, created by 4 | # sphinx-quickstart on Mon Mar 2 16:35:53 2015. 5 | # 6 | # This file is execfile()d with the current directory set to its 7 | # containing dir. 8 | # 9 | # Note that not all possible configuration values are present in this 10 | # autogenerated file. 11 | # 12 | # All configuration values have a default; values that are commented out 13 | # serve to show the default. 14 | 15 | import sys 16 | import os 17 | 18 | 19 | 20 | # If extensions (or modules to document with autodoc) are in another directory, 21 | # add these directories to sys.path here. If the directory is relative to the 22 | # documentation root, use os.path.abspath to make it absolute, like shown here. 23 | #sys.path.insert(0, os.path.abspath('.')) 24 | 25 | # -- General configuration ------------------------------------------------ 26 | 27 | # If your documentation needs a minimal Sphinx version, state it here. 28 | #needs_sphinx = '1.0' 29 | 30 | # Add any Sphinx extension module names here, as strings. They can be 31 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 32 | # ones. 33 | extensions = [ 34 | 'sphinx.ext.autodoc', 35 | ] 36 | 37 | # Add any paths that contain templates here, relative to this directory. 38 | templates_path = ['ntemplates'] 39 | 40 | # The suffix of source filenames. 41 | source_suffix = '.rst' 42 | 43 | # The encoding of source files. 44 | #source_encoding = 'utf-8-sig' 45 | 46 | # The master toctree document. 47 | master_doc = 'index' 48 | 49 | # General information about the project. 50 | project = u'fanova' 51 | copyright = u'2017, Frank Hutter, Stefan Falkner' 52 | 53 | # The version info for the project you're documenting, acts as replacement for 54 | # |version| and |release|, also used in various other places throughout the 55 | # built documents. 56 | # 57 | # The short X.Y version. 58 | version = '2.0' 59 | # The full version, including alpha/beta/rc tags. 60 | release = '2.0.5' 61 | 62 | # The language for content autogenerated by Sphinx. Refer to documentation 63 | # for a list of supported languages. 64 | #language = None 65 | 66 | # There are two options for replacing |today|: either, you set today to some 67 | # non-false value, then it is used: 68 | #today = '' 69 | # Else, today_fmt is used as the format for a strftime call. 70 | #today_fmt = '%B %d, %Y' 71 | 72 | # List of patterns, relative to source directory, that match files and 73 | # directories to ignore when looking for source files. 74 | exclude_patterns = [] 75 | 76 | # The reST default role (used for this markup: `text`) to use for all 77 | # documents. 78 | #default_role = None 79 | 80 | # If true, '()' will be appended to :func: etc. cross-reference text. 81 | #add_function_parentheses = True 82 | 83 | # If true, the current module name will be prepended to all description 84 | # unit titles (such as .. function::). 85 | #add_module_names = True 86 | 87 | # If true, sectionauthor and moduleauthor directives will be shown in the 88 | # output. They are ignored by default. 89 | #show_authors = False 90 | 91 | # The name of the Pygments (syntax highlighting) style to use. 92 | pygments_style = 'sphinx' 93 | 94 | # A list of ignored prefixes for module index sorting. 95 | #modindex_common_prefix = [] 96 | 97 | # If true, keep warnings as "system message" paragraphs in the built documents. 98 | #keep_warnings = False 99 | 100 | 101 | # -- Options for HTML output ---------------------------------------------- 102 | 103 | # The theme to use for HTML and HTML Help pages. See the documentation for 104 | # a list of builtin themes. 105 | #on_rtd = os.environ.get('READTHEDOCS', None) == 'True' 106 | 107 | #if not on_rtd: # only import and set the theme if we're building docs locally 108 | #import sphinx_rtd_theme 109 | #html_theme = 'sphinx_rtd_theme' 110 | #html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] 111 | 112 | # Theme options are theme-specific and customize the look and feel of a theme 113 | # further. For a list of options available for each theme, see the 114 | # documentation. 115 | #html_theme_options = {} 116 | 117 | # Add any paths that contain custom themes here, relative to this directory. 118 | #html_theme_path = [] 119 | 120 | # The name for this set of Sphinx documents. If None, it defaults to 121 | # " v documentation". 122 | #html_title = None 123 | 124 | # A shorter title for the navigation bar. Default is the same as html_title. 125 | #html_short_title = None 126 | 127 | # The name of an image file (relative to this directory) to place at the top 128 | # of the sidebar. 129 | #html_logo = None 130 | 131 | # The name of an image file (within the static path) to use as favicon of the 132 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 133 | # pixels large. 134 | #html_favicon = None 135 | 136 | # Add any paths that contain custom static files (such as style sheets) here, 137 | # relative to this directory. They are copied after the builtin static files, 138 | # so a file named "default.css" will overwrite the builtin "default.css". 139 | html_static_path = ['nstatic'] 140 | 141 | # Add any extra paths that contain custom files (such as robots.txt or 142 | # .htaccess) here, relative to this directory. These files are copied 143 | # directly to the root of the documentation. 144 | #html_extra_path = [] 145 | 146 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 147 | # using the given strftime format. 148 | #html_last_updated_fmt = '%b %d, %Y' 149 | 150 | # If true, SmartyPants will be used to convert quotes and dashes to 151 | # typographically correct entities. 152 | #html_use_smartypants = True 153 | 154 | # Custom sidebar templates, maps document names to template names. 155 | #html_sidebars = {} 156 | 157 | # Additional templates that should be rendered to pages, maps page names to 158 | # template names. 159 | #html_additional_pages = {} 160 | 161 | # If false, no module index is generated. 162 | #html_domain_indices = True 163 | 164 | # If false, no index is generated. 165 | #html_use_index = True 166 | 167 | # If true, the index is split into individual pages for each letter. 168 | #html_split_index = False 169 | 170 | # If true, links to the reST sources are added to the pages. 171 | #html_show_sourcelink = True 172 | 173 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 174 | #html_show_sphinx = True 175 | 176 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 177 | #html_show_copyright = True 178 | 179 | # If true, an OpenSearch description file will be output, and all pages will 180 | # contain a tag referring to it. The value of this option must be the 181 | # base URL from which the finished HTML is served. 182 | #html_use_opensearch = '' 183 | 184 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 185 | #html_file_suffix = None 186 | 187 | # Output file base name for HTML help builder. 188 | htmlhelp_basename = 'fanovadoc' 189 | 190 | 191 | # -- Options for LaTeX output --------------------------------------------- 192 | 193 | latex_elements = { 194 | # The paper size ('letterpaper' or 'a4paper'). 195 | #'papersize': 'letterpaper', 196 | 197 | # The font size ('10pt', '11pt' or '12pt'). 198 | #'pointsize': '10pt', 199 | 200 | # Additional stuff for the LaTeX preamble. 201 | #'preamble': '', 202 | } 203 | 204 | # Grouping the document tree into LaTeX files. List of tuples 205 | # (source start file, target name, title, 206 | # author, documentclass [howto, manual, or own class]). 207 | latex_documents = [ 208 | ('index', 'fanova.tex', u'fanova Documentation', 209 | u'Frank Hutter, Stefan Falkner', 'manual'), 210 | ] 211 | 212 | # The name of an image file (relative to this directory) to place at the top of 213 | # the title page. 214 | #latex_logo = None 215 | 216 | # For "manual" documents, if this is true, then toplevel headings are parts, 217 | # not chapters. 218 | #latex_use_parts = False 219 | 220 | # If true, show page references after internal links. 221 | #latex_show_pagerefs = False 222 | 223 | # If true, show URL addresses after external links. 224 | #latex_show_urls = False 225 | 226 | # Documents to append as an appendix to all manuals. 227 | #latex_appendices = [] 228 | 229 | # If false, no module index is generated. 230 | #latex_domain_indices = True 231 | 232 | 233 | # -- Options for manual page output --------------------------------------- 234 | 235 | # One entry per manual page. List of tuples 236 | # (source start file, name, description, authors, manual section). 237 | man_pages = [ 238 | ('index', 'fanova', u'fanova Documentation', 239 | [u'Frank Hutter, Stefan Falkner'], 1) 240 | ] 241 | 242 | # If true, show URL addresses after external links. 243 | #man_show_urls = False 244 | 245 | 246 | # -- Options for Texinfo output ------------------------------------------- 247 | 248 | # Grouping the document tree into Texinfo files. List of tuples 249 | # (source start file, target name, title, author, 250 | # dir menu entry, description, category) 251 | texinfo_documents = [ 252 | ('index', 'fanova', u'fanova Documentation', 253 | u'Frank Hutter, Stefan Falkner', 'fanova', 'One line description of project.', 254 | 'Miscellaneous'), 255 | ] 256 | 257 | # Documents to append as an appendix to all manuals. 258 | #texinfo_appendices = [] 259 | 260 | # If false, no module index is generated. 261 | #texinfo_domain_indices = True 262 | 263 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 264 | #texinfo_show_urls = 'footnote' 265 | 266 | # If true, do not generate a @detailmenu in the "Top" node's menu. 267 | #texinfo_no_detailmenu = False 268 | -------------------------------------------------------------------------------- /doc_src/fanova_starter.rst: -------------------------------------------------------------------------------- 1 | fANOVA for Starters 2 | ====================== 3 | 4 | .. role:: bash(code) 5 | :language: bash 6 | 7 | Introduction 8 | ------------ 9 | 10 | Whereas many algorithms presumably depend on a large hyperparameter space, it is known that in many cases only few parameter changes can be responsible for nearly all of the performance improvement. To this end, different techniques have been proposed 11 | that score parameters based on their importance. 12 | The functional analysis of variance (fANOVA) uses an empirical performance model (EPM) which is based on random forests in order to analyze how much of the performance 13 | variance in the configuration space is explained by single parameters or combinations of few parameters. 14 | 15 | Note that fANOVA's result is ideally used for minimizing the hyperparameter search space and should not be seen as a flawless parameter configurator. 16 | 17 | More detailed information in : `"An Efficient Approach for Assessing Hyperparameter Importance" 18 | `_ 19 | by Frank Hutter, Holger Hoos and Kevin Leyton-Brown 20 | 21 | How to interpret your results 22 | ----------------------------- 23 | In order to interpret your plots, you should have in mind what kind of evaluation measurement function you wanted to consider: 24 | Either you'd like to maximize or minimize your function. 25 | Therefore, depending on this, you'd have to consider low or high performance values. 26 | 27 | 28 | Let's start with the online LDA example from our examples folder: 29 | 30 | Here we have the perplexity as measurement. Thus we would like to reduce it. 31 | 32 | First, by looking at the importance of each parameter we can clearly see that parameter Col2 is marginally most important. So the result states that the Col2 parameter by itself is responsible for approx. 62% of the perplexity's variability across the entire space. 33 | On the contrary, Col1 is marginally less important with 3,5%. 34 | 35 | +------------+-------------------+ 36 | | Parameter | Importance | 37 | +============+===================+ 38 | | Col0 | 0.066 | 39 | +------------+-------------------+ 40 | | Col1 | 0.035 | 41 | +------------+-------------------+ 42 | | Col2 | 0.619 | 43 | +------------+-------------------+ 44 | 45 | And by looking at the pairwise marginals we have the combination of Col0 with Col2 as marginally most important. 46 | 47 | +---------------+-------------------+ 48 | | Parameter pair| Importance | 49 | +===============+===================+ 50 | | [Col0, Col1] | 0.011 | 51 | +---------------+-------------------+ 52 | | [Col0, Col2] | 0.146 | 53 | +---------------+-------------------+ 54 | | [Col1, Col2] | 0.067 | 55 | +---------------+-------------------+ 56 | 57 | Now let's take a look at the visualizations thereof: 58 | 59 | Since Col2 was chosen as marginally more important, we will take a closer look at its single marginal plot. 60 | It shows that large values for the parameter consistently yield lower performance (in our case perplexity). 61 | 62 | .. image:: ../examples/example_data/online_lda/Col2.png 63 | 64 | 65 | In order to capture interaction effects we take a closer look at the pairwise marginal plots of Col0 and Col2. 66 | 67 | Here we can see that Col0 is much more important for smaller values of Col2 than for larger ones. Such an interaction cannot be shown by single marginals. 68 | 69 | .. image:: ../examples/example_data/online_lda/figure1.png 70 | 71 | Note that nevertheless this example consists of a lower-dimensional dataset (3 parameters), fANOVA can still give interesting insights. But you should have in mind that it is important to have a large dataset consisting of enough examples so that fANOVA can deliver significant and interpretable results. 72 | 73 | -------------------------------------------------------------------------------- /doc_src/includeme.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../README.rst 2 | -------------------------------------------------------------------------------- /doc_src/index.rst: -------------------------------------------------------------------------------- 1 | Welcome to fANOVA's documentation! 2 | ================================== 3 | The functional analysis of variance (**fANOVA**) framework is used for quantifying the performance of functions. 4 | 5 | It provides insight into te relationships between parameter setting and performance. 6 | 7 | Contents: 8 | 9 | .. toctree:: 10 | :maxdepth: 2 11 | 12 | includeme 13 | install 14 | manual 15 | fanova_starter 16 | cite 17 | 18 | 19 | -------------------------------------------------------------------------------- /doc_src/install.rst: -------------------------------------------------------------------------------- 1 | Installation 2 | ============ 3 | 4 | .. role:: bash(code) 5 | :language: bash 6 | 7 | Requirements 8 | ------------ 9 | fANOVA requires: 10 | 11 | `Numpy `_ 12 | 13 | `matplotlib `_ (Version 1.4.2) 14 | 15 | `pyrfr `_ 16 | 17 | Manually 18 | ------------ 19 | 20 | To install fANOVA from command line type the following commands in your bash terminal: 21 | 22 | :bash:`git clone https://github.com/automl/fanova.git` 23 | 24 | :bash:`cd fanova/` 25 | 26 | :bash:`pip install -r requirements.txt` 27 | 28 | :bash:`python setup.py install` 29 | 30 | -------------------------------------------------------------------------------- /doc_src/manual.rst: -------------------------------------------------------------------------------- 1 | Manual 2 | ====== 3 | 4 | .. role:: bash(code) 5 | :language: bash 6 | 7 | Quick Start 8 | ----------- 9 | To run the examples, just download the `data `_ and start the python console. 10 | We can then import fANOVA and start it by typing 11 | 12 | >>> from fanova import fANOVA 13 | >>> import csv 14 | >>> path = os.path.dirname(os.path.realpath(__file__)) 15 | >>> X = np.loadtxt(path + '/example_data/online_lda/online_lda_features.csv', delimiter=",") 16 | >>> Y = np.loadtxt(path + '/example_data/online_lda/online_lda_responses.csv', delimiter=",") 17 | >>> f = fANOVA(X,Y) 18 | 19 | This creates a new fANOVA object and fits the Random Forest on the specified data set. 20 | 21 | To compute now the marginal of the first parameter type: 22 | 23 | >>> f.quantify_importance((0, )) 24 | 0.075414122571199116 25 | 26 | fANOVA also allows to specify parameters by their names. 27 | 28 | >>> f.quantify_importance(("Col0", )) 29 | 0.075414122571199116 30 | 31 | 32 | Advanced 33 | -------- 34 | 35 | If you want the fANOVA only a certain quantiles (let's say between 10% and 25%) of the data you can call it by: 36 | 37 | >>> f = fANOVA(X,Y) 38 | >>> f.set_cutoffs(quantile=(10, 25)) 39 | 40 | Furthermore fANOVA now supports cutoffs on the y values. These will exclude parts of the parameters space where the prediction is not within the provided cutoffs. 41 | 42 | >>> f.set_cutoffs(cutoffs=(-np.inf, np.inf)) 43 | 44 | You can also specify the number of trees in the random forest as well as the minimum number of points to make a new split in a tree or your already specified configuration space by: 45 | 46 | >>> f = fANOVA(X,Y, config_space=config_space, num_trees=30, min_samples_split=3) 47 | 48 | More functions 49 | -------------- 50 | 51 | * **f.get_most_important_pairwise_marginals(n)** 52 | 53 | Returns the **n** most important pairwise marginals 54 | 55 | * **f.get_most_important_pairwise_marginals(params)** 56 | 57 | Returns the pairwise marginals of all elements in the list. 58 | 59 | * **f.get_triple_marginals(params)** 60 | Returns the marginals of all possible triplets in the list. They are sorted by importance. 61 | 62 | 63 | * **fANOVA.marginal_mean_variance_for_values(p, v)** 64 | 65 | Computes the mean and standard deviation of the parameter (or parameterlist) **p** for a certain value **v** 66 | 67 | pysmac 68 | ------- 69 | In order to run fANOVA on pysmac output: 70 | 71 | >>> import pysmac.utils.pysmac_fanova as pysmac_fanova 72 | >>> fanova = pysmac_fanova.smac_to_fanova('path_to/pysmac_output_dir/out/scenario', 'path_to/merged_states') 73 | 74 | Visualization 75 | ------------- 76 | 77 | To visualize the single and pairwise marginals, we have to create a visualizer object first containing the fanova object, configspace and directory 78 | 79 | >>> import fanova.visualizer 80 | >>> vis = fanova.visualizer.Visualizer(f, config_space, "./plots/") 81 | 82 | We can then plot single marginals by 83 | 84 | >>> vis.plot_marginal(1) 85 | 86 | what should look like this 87 | 88 | .. image:: ../examples/example_data/online_lda/Col1.png 89 | 90 | The same can been done for pairwise marginals 91 | 92 | >>> vis.plot_pairwise_marginal([0,1]) 93 | 94 | .. image:: ../examples/example_data/online_lda/figure2.png 95 | 96 | 97 | If you are just interested in the N most important pairwise marginals you can plot them through: 98 | 99 | >>> create_most_important_pairwise_marginal_plots(dir, n) 100 | 101 | and fANOVA will save those plot in dir. However, be aware that to create the plots fANOVA needs to compute all pairwise marginal, which can take awhile! 102 | 103 | If you're not interested in the plot itself, but want to extract the values for your own plots, simply call 104 | 105 | >>> vis.generate_marginal(0) 106 | 107 | 108 | At last, all plots can be created together and stored in a directory with 109 | 110 | >>> vis.create_all_plots() 111 | 112 | How to load interactive plots 113 | ----------------------------- 114 | You will also find an extra directory in your specified plot directory called 'interactive_plots' in which you can find all interactive pairwise plots as pickle files. 115 | 116 | >>> import pickle 117 | >>> figx = pickle.load(open('/interactive_plots/param1_param2.fig.pickle', 'rb')) 118 | >>> figx.show() 119 | 120 | How to load a CSV-file 121 | -------------------------- 122 | 123 | import numpy as np 124 | 125 | data = np.loadtxt('your_file.csv', delimiter=",") 126 | 127 | -------------------------------------------------------------------------------- /docs/.buildinfo: -------------------------------------------------------------------------------- 1 | # Sphinx build info version 1 2 | # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done. 3 | config: 118bca52316b3df04d46fdb860f90d3f 4 | tags: 645f666f9bcd5a90fca523b33c5a78b7 5 | -------------------------------------------------------------------------------- /docs/.nojekyll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/fanova/46eb005320a0f67fc6b34ed74a91f702c2358c3f/docs/.nojekyll -------------------------------------------------------------------------------- /docs/_images/Col1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/fanova/46eb005320a0f67fc6b34ed74a91f702c2358c3f/docs/_images/Col1.png -------------------------------------------------------------------------------- /docs/_images/Col2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/fanova/46eb005320a0f67fc6b34ed74a91f702c2358c3f/docs/_images/Col2.png -------------------------------------------------------------------------------- /docs/_images/figure1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/fanova/46eb005320a0f67fc6b34ed74a91f702c2358c3f/docs/_images/figure1.png -------------------------------------------------------------------------------- /docs/_images/figure2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/fanova/46eb005320a0f67fc6b34ed74a91f702c2358c3f/docs/_images/figure2.png -------------------------------------------------------------------------------- /docs/_images/pair.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/fanova/46eb005320a0f67fc6b34ed74a91f702c2358c3f/docs/_images/pair.png -------------------------------------------------------------------------------- /docs/_images/pairwise.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/fanova/46eb005320a0f67fc6b34ed74a91f702c2358c3f/docs/_images/pairwise.png -------------------------------------------------------------------------------- /docs/_sources/cite.txt: -------------------------------------------------------------------------------- 1 | Citing fANOVA 2 | ================= 3 | 4 | If you use the fANOVA for your research, please cite the ICML 2014 paper "An Efficient Approach for Assessing Hyperparameter Importance" by Frank Hutter, Holger Hoos and Kevin Leyton-Brown. 5 | 6 | with the following Bibtex file: 7 | 8 | @inproceedings{HutHooLey14, 9 | lauthor = {Frank Hutter and Holger Hoos and Kevin Leyton-Brown}, 10 | author = {F. Hutter and H. Hoos and K. Leyton-Brown}, 11 | title = {An Efficient Approach for Assessing Hyperparameter Importance}, 12 | booktitle = {Proceedings of International Conference on Machine Learning 2014 (ICML 2014)}, 13 | year = {2014}, 14 | pages = {754--762}, 15 | month = jun, 16 | } 17 | -------------------------------------------------------------------------------- /docs/_sources/fanova_starter.txt: -------------------------------------------------------------------------------- 1 | fANOVA for Starters 2 | ====================== 3 | 4 | .. role:: bash(code) 5 | :language: bash 6 | 7 | Introduction 8 | ------------ 9 | 10 | Whereas many algorithms presumably depend on a large hyperparameter space, it is known that in many cases only few parameter changes can be responsible for nearly all of the performance improvement. To this end, different techniques have been proposed 11 | that score parameters based on their importance. 12 | The functional analysis of variance (fANOVA) uses an empirical performance model (EPM) which is based on random forests in order to analyze how much of the performance 13 | variance in the configuration space is explained by single parameters or combinations of few parameters. 14 | 15 | Note that fANOVA's result is ideally used for minimizing the hyperparameter search space and should not be seen as a flawless parameter configurator. 16 | 17 | More detailed information in : `"An Efficient Approach for Assessing Hyperparameter Importance" 18 | `_ 19 | by Frank Hutter, Holger Hoos and Kevin Leyton-Brown 20 | 21 | How to interpret your results 22 | ----------------------------- 23 | In order to interpret your plots, you should have in mind what kind of evaluation measurement function you wanted to consider: 24 | Either you'd like to maximize or minimize your function. 25 | Therefore, depending on this, you'd have to consider low or high performance values. 26 | 27 | 28 | Let's start with the online LDA example from our examples folder: 29 | 30 | Here we have the perplexity as measurement. Thus we would like to reduce it. 31 | 32 | First, by looking at the importance of each parameter we can clearly see that parameter Col2 is marginally most important. So the result states that the Col2 parameter by itself is responsible for approx. 62% of the perplexity's variability across the entire space. 33 | On the contrary, Col1 is marginally less important with 3,5%. 34 | 35 | +------------+-------------------+ 36 | | Parameter | Importance | 37 | +============+===================+ 38 | | Col0 | 0.066 | 39 | +------------+-------------------+ 40 | | Col1 | 0.035 | 41 | +------------+-------------------+ 42 | | Col2 | 0.619 | 43 | +------------+-------------------+ 44 | 45 | And by looking at the pairwise marginals we have the combination of Col0 with Col2 as marginally most important. 46 | 47 | +---------------+-------------------+ 48 | | Parameter pair| Importance | 49 | +===============+===================+ 50 | | [Col0, Col1] | 0.127 | 51 | +---------------+-------------------+ 52 | | [Col0, Col2] | 0.833 | 53 | +---------------+-------------------+ 54 | | [Col1, Col2] | 0.693 | 55 | +---------------+-------------------+ 56 | 57 | Now let's take a look at the visualizations thereof: 58 | 59 | Since Col2 was chosen as marginally more important, we will take a closer look at its single marginal plot. 60 | It shows that large values for the parameter consistently yield lower performance (in our case perplexity). 61 | 62 | .. image:: ../examples/example_data/online_lda/Col2.png 63 | 64 | 65 | In order to capture interaction effects we take a closer look at the pairwise marginal plots of Col0 and Col2. 66 | 67 | Here we can see that Col0 is much more important for smaller values of Col2 than for larger ones. Such an interaction cannot be shown by single marginals. 68 | 69 | .. image:: ../examples/example_data/online_lda/figure1.png 70 | 71 | Note that nevertheless this example consists of a lower-dimensional dataset (3 parameters), fANOVA can still give interesting insights. But you should have in mind that it is important to have a large dataset consisting of enough examples so that fANOVA can deliver significant and interpretable results. 72 | 73 | -------------------------------------------------------------------------------- /docs/_sources/includeme.rst.txt: -------------------------------------------------------------------------------- 1 | .. include:: ../README.rst 2 | -------------------------------------------------------------------------------- /docs/_sources/includeme.txt: -------------------------------------------------------------------------------- 1 | .. include:: ../README.rst 2 | -------------------------------------------------------------------------------- /docs/_sources/index.rst.txt: -------------------------------------------------------------------------------- 1 | Welcome to fanova's documentation! 2 | ================================== 3 | 4 | 5 | Contents: 6 | 7 | .. toctree:: 8 | :maxdepth: 2 9 | 10 | includeme 11 | install 12 | manual 13 | cite 14 | 15 | 16 | -------------------------------------------------------------------------------- /docs/_sources/index.txt: -------------------------------------------------------------------------------- 1 | Welcome to fANOVA's documentation! 2 | ================================== 3 | The functional analysis of variance (**fANOVA**) framework is used for quantifying the performance of functions. 4 | 5 | It provides insight into te relationships between parameter setting and performance. 6 | 7 | Contents: 8 | 9 | .. toctree:: 10 | :maxdepth: 2 11 | 12 | includeme 13 | install 14 | manual 15 | fanova_starter 16 | cite 17 | 18 | 19 | -------------------------------------------------------------------------------- /docs/_sources/install.txt: -------------------------------------------------------------------------------- 1 | Installation 2 | ============ 3 | 4 | .. role:: bash(code) 5 | :language: bash 6 | 7 | Requirements 8 | ------------ 9 | fANOVA requires: 10 | 11 | `Numpy `_ 12 | 13 | `matplotlib `_ (Version 1.4.2) 14 | 15 | `pyrfr `_ 16 | 17 | Manually 18 | ------------ 19 | 20 | To install fANOVA from command line type the following commands in your bash terminal: 21 | 22 | :bash:`git clone https://github.com/automl/fanova.git` 23 | 24 | :bash:`cd fanova/` 25 | 26 | :bash:`pip install -r requirements.txt` 27 | 28 | :bash:`python setup.py install` 29 | 30 | -------------------------------------------------------------------------------- /docs/_sources/manual.rst.txt: -------------------------------------------------------------------------------- 1 | python console. 2 | We can then import Fanova and start it by typing 3 | 4 | >>> from fanova import fANOVA 5 | >>> import csv 6 | >>> path = os.path.dirname(os.path.realpath(__file__)) 7 | >>> X = np.loadtxt(path + '/example_data/online_lda/online_lda_features.csv', delimiter=",") 8 | >>> Y = np.loadtxt(path + '/example_data/online_lda/online_lda_responses.csv', delimiter=",") 9 | >>> f = fANOVA(X,Y) 10 | 11 | This creates a new Fanova object and fits the Random Forest on the specified data set. 12 | 13 | To compute now the marginal of the first parameter type: 14 | 15 | >>> f.quantify_importance((0, )) 16 | 5.44551614362 17 | 18 | Fanova also allows to specify parameters by their names. 19 | 20 | >>> f.quantify_importance(("Col0", )) 21 | 5.44551614362 22 | 23 | 24 | Advanced 25 | -------- 26 | 27 | If you want the Fanova only a certain quantiles (let's say between 10% and 25%) of the data you can call it by: 28 | 29 | >>> f = fANOVA(X,Y) 30 | >>> f.set_cutoffs(quantile=(10, 25)) 31 | 32 | Furthermore fANOVA now supports cutoffs on the y values. These will exclude parts of the parameters space where the prediction is not within the provided cutoffs. 33 | 34 | >>> f.set_cutoffs(cutoffs=(-np.inf, np.inf)) 35 | 36 | You can also specify the number of trees in the random forest as well as the minimum number of points to make a new split in a tree or your already specified configuration space by: 37 | 38 | >>> f = fANOVA(X,Y, config_space=config_space, num_trees=30, min_samples_split=3) 39 | 40 | More functions 41 | -------------- 42 | 43 | * **f.get_most_important_pairwise_marginals(n)** 44 | 45 | Returns the **n** most important pairwise marginals 46 | 47 | 48 | * **fANOVA.marginal_mean_variance_for_values(p, v)** 49 | 50 | Computes the mean and standard deviation of the parameter (or parameterlist) **p** for a certain value **v** 51 | 52 | 53 | 54 | Visualization 55 | ------------- 56 | 57 | To visualize the single and pairwise marginals, we have to create a visualizer object first containing the fanova object and configspace 58 | 59 | >>> import fanova.visualizer 60 | >>> vis = visualizer.Visualizer(f, config_space) 61 | 62 | We can then plot single marginals by 63 | 64 | >>> vis.plot_marginal(1) 65 | 66 | what should look like this 67 | 68 | .. image:: ../examples/example_data/online_lda/Col1.png 69 | 70 | NOTE: For categorical values use the function plot_categorical_marginal(parameter) instead. 71 | 72 | The same can been done for pairwise marginals 73 | 74 | >>> vis.plot_pairwise_marginal([0,2]) 75 | 76 | .. image:: ../examples/example_data/online_lda/pairwise.png 77 | 78 | 79 | If you are just interested in the N most important pairwise marginals you can plot them through: 80 | 81 | >>> create_most_important_pairwise_marginal_plots(dir, n) 82 | 83 | and Fanova will save those plot in dir. However, be aware that to create the plots Fanova needs to compute all pairwise marginal, which can take awhile! 84 | 85 | If you're not interested in the plot itself, but want to extract the values for your own plots, simply call 86 | 87 | >>> vis.generate_marginal(0) 88 | 89 | The same for generate_pairwise_marginal([0,2]) and get_categorical_marginal(). 90 | 91 | 92 | At last, all plots can be created together and stored in a directory with 93 | 94 | >>> vis.create_all_plots("./plots/") 95 | 96 | 97 | How to load a CSV-file 98 | -------------------------- 99 | 100 | import numpy as np 101 | 102 | data = np.loadtxt('your_file.csv', delimiter=",") 103 | >>> vis = visualizer.Visualizer(f, config_space) 104 | 105 | We can then plot single marginals by 106 | 107 | >>> vis.plot_marginal(1) 108 | 109 | what should look like this 110 | 111 | .. image:: /../examples/example_data/online_lda/Col1.png 112 | 113 | NOTE: For categorical values use the function plot_categorical_marginal(parameter) instead. 114 | 115 | The same can been done for pairwise marginals 116 | 117 | >>> vis.plot_pairwise_marginal([0,2]) 118 | 119 | .. image:: /../examples/example_data/online_lda/pairwise.png 120 | 121 | 122 | If you are just interested in the N most important pairwise marginals you can plot them through: 123 | 124 | >>> create_most_important_pairwise_marginal_plots(dir, n) 125 | 126 | and Fanova will save those plot in dir. However, be aware that to create the plots Fanova needs to compute all pairwise marginal, which can take awhile! 127 | 128 | If you're not interested in the plot itself, but want to extract the values for your own plots, simply call 129 | 130 | >>> vis.generate_marginal(0) 131 | 132 | The same for generate_pairwise_marginal([0,2]) and get_categorical_marginal(). 133 | 134 | 135 | At last, all plots can be created together and stored in a directory with 136 | 137 | >>> vis.create_all_plots("./plots/") 138 | 139 | 140 | How to load a CSV-file 141 | -------------------------- 142 | 143 | import numpy as np 144 | 145 | data = np.loadtxt('your_file.csv', delimiter=",") 146 | 147 | -------------------------------------------------------------------------------- /docs/_sources/manual.txt: -------------------------------------------------------------------------------- 1 | Manual 2 | ====== 3 | 4 | .. role:: bash(code) 5 | :language: bash 6 | 7 | Quick Start 8 | ----------- 9 | To run the examples, just download the `data `_ and start the python console. 10 | We can then import fANOVA and start it by typing 11 | 12 | >>> from fanova import fANOVA 13 | >>> import csv 14 | >>> path = os.path.dirname(os.path.realpath(__file__)) 15 | >>> X = np.loadtxt(path + '/example_data/online_lda/online_lda_features.csv', delimiter=",") 16 | >>> Y = np.loadtxt(path + '/example_data/online_lda/online_lda_responses.csv', delimiter=",") 17 | >>> f = fANOVA(X,Y) 18 | 19 | This creates a new fANOVA object and fits the Random Forest on the specified data set. 20 | 21 | To compute now the marginal of the first parameter type: 22 | 23 | >>> f.quantify_importance((0, )) 24 | 0.075414122571199116 25 | 26 | fANOVA also allows to specify parameters by their names. 27 | 28 | >>> f.quantify_importance(("Col0", )) 29 | 0.075414122571199116 30 | 31 | 32 | Advanced 33 | -------- 34 | 35 | If you want the fANOVA only a certain quantiles (let's say between 10% and 25%) of the data you can call it by: 36 | 37 | >>> f = fANOVA(X,Y) 38 | >>> f.set_cutoffs(quantile=(10, 25)) 39 | 40 | Furthermore fANOVA now supports cutoffs on the y values. These will exclude parts of the parameters space where the prediction is not within the provided cutoffs. 41 | 42 | >>> f.set_cutoffs(cutoffs=(-np.inf, np.inf)) 43 | 44 | You can also specify the number of trees in the random forest as well as the minimum number of points to make a new split in a tree or your already specified configuration space by: 45 | 46 | >>> f = fANOVA(X,Y, config_space=config_space, num_trees=30, min_samples_split=3) 47 | 48 | More functions 49 | -------------- 50 | 51 | * **f.get_most_important_pairwise_marginals(n)** 52 | 53 | Returns the **n** most important pairwise marginals 54 | 55 | * **f.get_most_important_pairwise_marginals(params)** 56 | 57 | Returns the pairwise marginals of all elements in the list. 58 | 59 | 60 | * **fANOVA.marginal_mean_variance_for_values(p, v)** 61 | 62 | Computes the mean and standard deviation of the parameter (or parameterlist) **p** for a certain value **v** 63 | 64 | pysmac 65 | ------- 66 | In order to run fANOVA on pysmac output: 67 | 68 | >>> import pysmac.utils.pysmac_fanova as pysmac_fanova 69 | >>> fanova = pysmac_fanova.smac_to_fanova('path_to/pysmac_output_dir/out/scenario', 'path_to/merged_states') 70 | 71 | Visualization 72 | ------------- 73 | 74 | To visualize the single and pairwise marginals, we have to create a visualizer object first containing the fanova object, configspace and directory 75 | 76 | >>> import fanova.visualizer 77 | >>> vis = fanova.visualizer.Visualizer(f, config_space, "./plots/") 78 | 79 | We can then plot single marginals by 80 | 81 | >>> vis.plot_marginal(1) 82 | 83 | what should look like this 84 | 85 | .. image:: ../examples/example_data/online_lda/Col1.png 86 | 87 | The same can been done for pairwise marginals 88 | 89 | >>> vis.plot_pairwise_marginal([0,1]) 90 | 91 | .. image:: ../examples/example_data/online_lda/figure2.png 92 | 93 | 94 | If you are just interested in the N most important pairwise marginals you can plot them through: 95 | 96 | >>> create_most_important_pairwise_marginal_plots(dir, n) 97 | 98 | and fANOVA will save those plot in dir. However, be aware that to create the plots fANOVA needs to compute all pairwise marginal, which can take awhile! 99 | 100 | If you're not interested in the plot itself, but want to extract the values for your own plots, simply call 101 | 102 | >>> vis.generate_marginal(0) 103 | 104 | 105 | At last, all plots can be created together and stored in a directory with 106 | 107 | >>> vis.create_all_plots() 108 | 109 | How to load interactive plots 110 | ----------------------------- 111 | You will also find an extra directory in your specified plot directory called 'interactive_plots' in which you can find all interactive pairwise plots as pickle files. 112 | 113 | >>> import pickle 114 | >>> figx = pickle.load(open('/interactive_plots/param1_param2.fig.pickle', 'rb')) 115 | >>> figx.show() 116 | 117 | How to load a CSV-file 118 | -------------------------- 119 | 120 | import numpy as np 121 | 122 | data = np.loadtxt('your_file.csv', delimiter=",") 123 | 124 | -------------------------------------------------------------------------------- /docs/_static/ajax-loader.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/fanova/46eb005320a0f67fc6b34ed74a91f702c2358c3f/docs/_static/ajax-loader.gif -------------------------------------------------------------------------------- /docs/_static/basic.css: -------------------------------------------------------------------------------- 1 | /* 2 | * basic.css 3 | * ~~~~~~~~~ 4 | * 5 | * Sphinx stylesheet -- basic theme. 6 | * 7 | * :copyright: Copyright 2007-2014 by the Sphinx team, see AUTHORS. 8 | * :license: BSD, see LICENSE for details. 9 | * 10 | */ 11 | 12 | /* -- main layout ----------------------------------------------------------- */ 13 | 14 | div.clearer { 15 | clear: both; 16 | } 17 | 18 | /* -- relbar ---------------------------------------------------------------- */ 19 | 20 | div.related { 21 | width: 100%; 22 | font-size: 90%; 23 | } 24 | 25 | div.related h3 { 26 | display: none; 27 | } 28 | 29 | div.related ul { 30 | margin: 0; 31 | padding: 0 0 0 10px; 32 | list-style: none; 33 | } 34 | 35 | div.related li { 36 | display: inline; 37 | } 38 | 39 | div.related li.right { 40 | float: right; 41 | margin-right: 5px; 42 | } 43 | 44 | /* -- sidebar --------------------------------------------------------------- */ 45 | 46 | div.sphinxsidebarwrapper { 47 | padding: 10px 5px 0 10px; 48 | } 49 | 50 | div.sphinxsidebar { 51 | float: left; 52 | width: 230px; 53 | margin-left: -100%; 54 | font-size: 90%; 55 | } 56 | 57 | div.sphinxsidebar ul { 58 | list-style: none; 59 | } 60 | 61 | div.sphinxsidebar ul ul, 62 | div.sphinxsidebar ul.want-points { 63 | margin-left: 20px; 64 | list-style: square; 65 | } 66 | 67 | div.sphinxsidebar ul ul { 68 | margin-top: 0; 69 | margin-bottom: 0; 70 | } 71 | 72 | div.sphinxsidebar form { 73 | margin-top: 10px; 74 | } 75 | 76 | div.sphinxsidebar input { 77 | border: 1px solid #98dbcc; 78 | font-family: sans-serif; 79 | font-size: 1em; 80 | } 81 | 82 | div.sphinxsidebar #searchbox input[type="text"] { 83 | width: 170px; 84 | } 85 | 86 | div.sphinxsidebar #searchbox input[type="submit"] { 87 | width: 30px; 88 | } 89 | 90 | img { 91 | border: 0; 92 | max-width: 100%; 93 | } 94 | 95 | /* -- search page ----------------------------------------------------------- */ 96 | 97 | ul.search { 98 | margin: 10px 0 0 20px; 99 | padding: 0; 100 | } 101 | 102 | ul.search li { 103 | padding: 5px 0 5px 20px; 104 | background-image: url(file.png); 105 | background-repeat: no-repeat; 106 | background-position: 0 7px; 107 | } 108 | 109 | ul.search li a { 110 | font-weight: bold; 111 | } 112 | 113 | ul.search li div.context { 114 | color: #888; 115 | margin: 2px 0 0 30px; 116 | text-align: left; 117 | } 118 | 119 | ul.keywordmatches li.goodmatch a { 120 | font-weight: bold; 121 | } 122 | 123 | /* -- index page ------------------------------------------------------------ */ 124 | 125 | table.contentstable { 126 | width: 90%; 127 | } 128 | 129 | table.contentstable p.biglink { 130 | line-height: 150%; 131 | } 132 | 133 | a.biglink { 134 | font-size: 1.3em; 135 | } 136 | 137 | span.linkdescr { 138 | font-style: italic; 139 | padding-top: 5px; 140 | font-size: 90%; 141 | } 142 | 143 | /* -- general index --------------------------------------------------------- */ 144 | 145 | table.indextable { 146 | width: 100%; 147 | } 148 | 149 | table.indextable td { 150 | text-align: left; 151 | vertical-align: top; 152 | } 153 | 154 | table.indextable dl, table.indextable dd { 155 | margin-top: 0; 156 | margin-bottom: 0; 157 | } 158 | 159 | table.indextable tr.pcap { 160 | height: 10px; 161 | } 162 | 163 | table.indextable tr.cap { 164 | margin-top: 10px; 165 | background-color: #f2f2f2; 166 | } 167 | 168 | img.toggler { 169 | margin-right: 3px; 170 | margin-top: 3px; 171 | cursor: pointer; 172 | } 173 | 174 | div.modindex-jumpbox { 175 | border-top: 1px solid #ddd; 176 | border-bottom: 1px solid #ddd; 177 | margin: 1em 0 1em 0; 178 | padding: 0.4em; 179 | } 180 | 181 | div.genindex-jumpbox { 182 | border-top: 1px solid #ddd; 183 | border-bottom: 1px solid #ddd; 184 | margin: 1em 0 1em 0; 185 | padding: 0.4em; 186 | } 187 | 188 | /* -- general body styles --------------------------------------------------- */ 189 | 190 | a.headerlink { 191 | visibility: hidden; 192 | } 193 | 194 | h1:hover > a.headerlink, 195 | h2:hover > a.headerlink, 196 | h3:hover > a.headerlink, 197 | h4:hover > a.headerlink, 198 | h5:hover > a.headerlink, 199 | h6:hover > a.headerlink, 200 | dt:hover > a.headerlink { 201 | visibility: visible; 202 | } 203 | 204 | div.body p.caption { 205 | text-align: inherit; 206 | } 207 | 208 | div.body td { 209 | text-align: left; 210 | } 211 | 212 | .field-list ul { 213 | padding-left: 1em; 214 | } 215 | 216 | .first { 217 | margin-top: 0 !important; 218 | } 219 | 220 | p.rubric { 221 | margin-top: 30px; 222 | font-weight: bold; 223 | } 224 | 225 | img.align-left, .figure.align-left, object.align-left { 226 | clear: left; 227 | float: left; 228 | margin-right: 1em; 229 | } 230 | 231 | img.align-right, .figure.align-right, object.align-right { 232 | clear: right; 233 | float: right; 234 | margin-left: 1em; 235 | } 236 | 237 | img.align-center, .figure.align-center, object.align-center { 238 | display: block; 239 | margin-left: auto; 240 | margin-right: auto; 241 | } 242 | 243 | .align-left { 244 | text-align: left; 245 | } 246 | 247 | .align-center { 248 | text-align: center; 249 | } 250 | 251 | .align-right { 252 | text-align: right; 253 | } 254 | 255 | /* -- sidebars -------------------------------------------------------------- */ 256 | 257 | div.sidebar { 258 | margin: 0 0 0.5em 1em; 259 | border: 1px solid #ddb; 260 | padding: 7px 7px 0 7px; 261 | background-color: #ffe; 262 | width: 40%; 263 | float: right; 264 | } 265 | 266 | p.sidebar-title { 267 | font-weight: bold; 268 | } 269 | 270 | /* -- topics ---------------------------------------------------------------- */ 271 | 272 | div.topic { 273 | border: 1px solid #ccc; 274 | padding: 7px 7px 0 7px; 275 | margin: 10px 0 10px 0; 276 | } 277 | 278 | p.topic-title { 279 | font-size: 1.1em; 280 | font-weight: bold; 281 | margin-top: 10px; 282 | } 283 | 284 | /* -- admonitions ----------------------------------------------------------- */ 285 | 286 | div.admonition { 287 | margin-top: 10px; 288 | margin-bottom: 10px; 289 | padding: 7px; 290 | } 291 | 292 | div.admonition dt { 293 | font-weight: bold; 294 | } 295 | 296 | div.admonition dl { 297 | margin-bottom: 0; 298 | } 299 | 300 | p.admonition-title { 301 | margin: 0px 10px 5px 0px; 302 | font-weight: bold; 303 | } 304 | 305 | div.body p.centered { 306 | text-align: center; 307 | margin-top: 25px; 308 | } 309 | 310 | /* -- tables ---------------------------------------------------------------- */ 311 | 312 | table.docutils { 313 | border: 0; 314 | border-collapse: collapse; 315 | } 316 | 317 | table.docutils td, table.docutils th { 318 | padding: 1px 8px 1px 5px; 319 | border-top: 0; 320 | border-left: 0; 321 | border-right: 0; 322 | border-bottom: 1px solid #aaa; 323 | } 324 | 325 | table.field-list td, table.field-list th { 326 | border: 0 !important; 327 | } 328 | 329 | table.footnote td, table.footnote th { 330 | border: 0 !important; 331 | } 332 | 333 | th { 334 | text-align: left; 335 | padding-right: 5px; 336 | } 337 | 338 | table.citation { 339 | border-left: solid 1px gray; 340 | margin-left: 1px; 341 | } 342 | 343 | table.citation td { 344 | border-bottom: none; 345 | } 346 | 347 | /* -- other body styles ----------------------------------------------------- */ 348 | 349 | ol.arabic { 350 | list-style: decimal; 351 | } 352 | 353 | ol.loweralpha { 354 | list-style: lower-alpha; 355 | } 356 | 357 | ol.upperalpha { 358 | list-style: upper-alpha; 359 | } 360 | 361 | ol.lowerroman { 362 | list-style: lower-roman; 363 | } 364 | 365 | ol.upperroman { 366 | list-style: upper-roman; 367 | } 368 | 369 | dl { 370 | margin-bottom: 15px; 371 | } 372 | 373 | dd p { 374 | margin-top: 0px; 375 | } 376 | 377 | dd ul, dd table { 378 | margin-bottom: 10px; 379 | } 380 | 381 | dd { 382 | margin-top: 3px; 383 | margin-bottom: 10px; 384 | margin-left: 30px; 385 | } 386 | 387 | dt:target, .highlighted { 388 | background-color: #fbe54e; 389 | } 390 | 391 | dl.glossary dt { 392 | font-weight: bold; 393 | font-size: 1.1em; 394 | } 395 | 396 | .field-list ul { 397 | margin: 0; 398 | padding-left: 1em; 399 | } 400 | 401 | .field-list p { 402 | margin: 0; 403 | } 404 | 405 | .optional { 406 | font-size: 1.3em; 407 | } 408 | 409 | .versionmodified { 410 | font-style: italic; 411 | } 412 | 413 | .system-message { 414 | background-color: #fda; 415 | padding: 5px; 416 | border: 3px solid red; 417 | } 418 | 419 | .footnote:target { 420 | background-color: #ffa; 421 | } 422 | 423 | .line-block { 424 | display: block; 425 | margin-top: 1em; 426 | margin-bottom: 1em; 427 | } 428 | 429 | .line-block .line-block { 430 | margin-top: 0; 431 | margin-bottom: 0; 432 | margin-left: 1.5em; 433 | } 434 | 435 | .guilabel, .menuselection { 436 | font-family: sans-serif; 437 | } 438 | 439 | .accelerator { 440 | text-decoration: underline; 441 | } 442 | 443 | .classifier { 444 | font-style: oblique; 445 | } 446 | 447 | abbr, acronym { 448 | border-bottom: dotted 1px; 449 | cursor: help; 450 | } 451 | 452 | /* -- code displays --------------------------------------------------------- */ 453 | 454 | pre { 455 | overflow: auto; 456 | overflow-y: hidden; /* fixes display issues on Chrome browsers */ 457 | } 458 | 459 | td.linenos pre { 460 | padding: 5px 0px; 461 | border: 0; 462 | background-color: transparent; 463 | color: #aaa; 464 | } 465 | 466 | table.highlighttable { 467 | margin-left: 0.5em; 468 | } 469 | 470 | table.highlighttable td { 471 | padding: 0 0.5em 0 0.5em; 472 | } 473 | 474 | tt.descname { 475 | background-color: transparent; 476 | font-weight: bold; 477 | font-size: 1.2em; 478 | } 479 | 480 | tt.descclassname { 481 | background-color: transparent; 482 | } 483 | 484 | tt.xref, a tt { 485 | background-color: transparent; 486 | font-weight: bold; 487 | } 488 | 489 | h1 tt, h2 tt, h3 tt, h4 tt, h5 tt, h6 tt { 490 | background-color: transparent; 491 | } 492 | 493 | .viewcode-link { 494 | float: right; 495 | } 496 | 497 | .viewcode-back { 498 | float: right; 499 | font-family: sans-serif; 500 | } 501 | 502 | div.viewcode-block:target { 503 | margin: -1px -10px; 504 | padding: 0 10px; 505 | } 506 | 507 | /* -- math display ---------------------------------------------------------- */ 508 | 509 | img.math { 510 | vertical-align: middle; 511 | } 512 | 513 | div.body div.math p { 514 | text-align: center; 515 | } 516 | 517 | span.eqno { 518 | float: right; 519 | } 520 | 521 | /* -- printout stylesheet --------------------------------------------------- */ 522 | 523 | @media print { 524 | div.document, 525 | div.documentwrapper, 526 | div.bodywrapper { 527 | margin: 0 !important; 528 | width: 100%; 529 | } 530 | 531 | div.sphinxsidebar, 532 | div.related, 533 | div.footer, 534 | #top-link { 535 | display: none; 536 | } 537 | } -------------------------------------------------------------------------------- /docs/_static/comment-bright.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/fanova/46eb005320a0f67fc6b34ed74a91f702c2358c3f/docs/_static/comment-bright.png -------------------------------------------------------------------------------- /docs/_static/comment-close.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/fanova/46eb005320a0f67fc6b34ed74a91f702c2358c3f/docs/_static/comment-close.png -------------------------------------------------------------------------------- /docs/_static/comment.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/fanova/46eb005320a0f67fc6b34ed74a91f702c2358c3f/docs/_static/comment.png -------------------------------------------------------------------------------- /docs/_static/custom.css: -------------------------------------------------------------------------------- 1 | /* This file intentionally left blank. */ 2 | -------------------------------------------------------------------------------- /docs/_static/default.css: -------------------------------------------------------------------------------- 1 | /* 2 | * default.css_t 3 | * ~~~~~~~~~~~~~ 4 | * 5 | * Sphinx stylesheet -- default theme. 6 | * 7 | * :copyright: Copyright 2007-2014 by the Sphinx team, see AUTHORS. 8 | * :license: BSD, see LICENSE for details. 9 | * 10 | */ 11 | 12 | @import url("basic.css"); 13 | 14 | /* -- page layout ----------------------------------------------------------- */ 15 | 16 | body { 17 | font-family: sans-serif; 18 | font-size: 100%; 19 | background-color: #11303d; 20 | color: #000; 21 | margin: 0; 22 | padding: 0; 23 | } 24 | 25 | div.document { 26 | background-color: #1c4e63; 27 | } 28 | 29 | div.documentwrapper { 30 | float: left; 31 | width: 100%; 32 | } 33 | 34 | div.bodywrapper { 35 | margin: 0 0 0 230px; 36 | } 37 | 38 | div.body { 39 | background-color: #ffffff; 40 | color: #000000; 41 | padding: 0 20px 30px 20px; 42 | } 43 | 44 | div.footer { 45 | color: #ffffff; 46 | width: 100%; 47 | padding: 9px 0 9px 0; 48 | text-align: center; 49 | font-size: 75%; 50 | } 51 | 52 | div.footer a { 53 | color: #ffffff; 54 | text-decoration: underline; 55 | } 56 | 57 | div.related { 58 | background-color: #133f52; 59 | line-height: 30px; 60 | color: #ffffff; 61 | } 62 | 63 | div.related a { 64 | color: #ffffff; 65 | } 66 | 67 | div.sphinxsidebar { 68 | } 69 | 70 | div.sphinxsidebar h3 { 71 | font-family: 'Trebuchet MS', sans-serif; 72 | color: #ffffff; 73 | font-size: 1.4em; 74 | font-weight: normal; 75 | margin: 0; 76 | padding: 0; 77 | } 78 | 79 | div.sphinxsidebar h3 a { 80 | color: #ffffff; 81 | } 82 | 83 | div.sphinxsidebar h4 { 84 | font-family: 'Trebuchet MS', sans-serif; 85 | color: #ffffff; 86 | font-size: 1.3em; 87 | font-weight: normal; 88 | margin: 5px 0 0 0; 89 | padding: 0; 90 | } 91 | 92 | div.sphinxsidebar p { 93 | color: #ffffff; 94 | } 95 | 96 | div.sphinxsidebar p.topless { 97 | margin: 5px 10px 10px 10px; 98 | } 99 | 100 | div.sphinxsidebar ul { 101 | margin: 10px; 102 | padding: 0; 103 | color: #ffffff; 104 | } 105 | 106 | div.sphinxsidebar a { 107 | color: #98dbcc; 108 | } 109 | 110 | div.sphinxsidebar input { 111 | border: 1px solid #98dbcc; 112 | font-family: sans-serif; 113 | font-size: 1em; 114 | } 115 | 116 | 117 | 118 | /* -- hyperlink styles ------------------------------------------------------ */ 119 | 120 | a { 121 | color: #355f7c; 122 | text-decoration: none; 123 | } 124 | 125 | a:visited { 126 | color: #355f7c; 127 | text-decoration: none; 128 | } 129 | 130 | a:hover { 131 | text-decoration: underline; 132 | } 133 | 134 | 135 | 136 | /* -- body styles ----------------------------------------------------------- */ 137 | 138 | div.body h1, 139 | div.body h2, 140 | div.body h3, 141 | div.body h4, 142 | div.body h5, 143 | div.body h6 { 144 | font-family: 'Trebuchet MS', sans-serif; 145 | background-color: #f2f2f2; 146 | font-weight: normal; 147 | color: #20435c; 148 | border-bottom: 1px solid #ccc; 149 | margin: 20px -20px 10px -20px; 150 | padding: 3px 0 3px 10px; 151 | } 152 | 153 | div.body h1 { margin-top: 0; font-size: 200%; } 154 | div.body h2 { font-size: 160%; } 155 | div.body h3 { font-size: 140%; } 156 | div.body h4 { font-size: 120%; } 157 | div.body h5 { font-size: 110%; } 158 | div.body h6 { font-size: 100%; } 159 | 160 | a.headerlink { 161 | color: #c60f0f; 162 | font-size: 0.8em; 163 | padding: 0 4px 0 4px; 164 | text-decoration: none; 165 | } 166 | 167 | a.headerlink:hover { 168 | background-color: #c60f0f; 169 | color: white; 170 | } 171 | 172 | div.body p, div.body dd, div.body li { 173 | text-align: justify; 174 | line-height: 130%; 175 | } 176 | 177 | div.admonition p.admonition-title + p { 178 | display: inline; 179 | } 180 | 181 | div.admonition p { 182 | margin-bottom: 5px; 183 | } 184 | 185 | div.admonition pre { 186 | margin-bottom: 5px; 187 | } 188 | 189 | div.admonition ul, div.admonition ol { 190 | margin-bottom: 5px; 191 | } 192 | 193 | div.note { 194 | background-color: #eee; 195 | border: 1px solid #ccc; 196 | } 197 | 198 | div.seealso { 199 | background-color: #ffc; 200 | border: 1px solid #ff6; 201 | } 202 | 203 | div.topic { 204 | background-color: #eee; 205 | } 206 | 207 | div.warning { 208 | background-color: #ffe4e4; 209 | border: 1px solid #f66; 210 | } 211 | 212 | p.admonition-title { 213 | display: inline; 214 | } 215 | 216 | p.admonition-title:after { 217 | content: ":"; 218 | } 219 | 220 | pre { 221 | padding: 5px; 222 | background-color: #eeffcc; 223 | color: #333333; 224 | line-height: 120%; 225 | border: 1px solid #ac9; 226 | border-left: none; 227 | border-right: none; 228 | } 229 | 230 | tt { 231 | background-color: #ecf0f3; 232 | padding: 0 1px 0 1px; 233 | font-size: 0.95em; 234 | } 235 | 236 | th { 237 | background-color: #ede; 238 | } 239 | 240 | .warning tt { 241 | background: #efc2c2; 242 | } 243 | 244 | .note tt { 245 | background: #d6d6d6; 246 | } 247 | 248 | .viewcode-back { 249 | font-family: sans-serif; 250 | } 251 | 252 | div.viewcode-block:target { 253 | background-color: #f4debf; 254 | border-top: 1px solid #ac9; 255 | border-bottom: 1px solid #ac9; 256 | } -------------------------------------------------------------------------------- /docs/_static/doctools.js: -------------------------------------------------------------------------------- 1 | /* 2 | * doctools.js 3 | * ~~~~~~~~~~~ 4 | * 5 | * Sphinx JavaScript utilities for all documentation. 6 | * 7 | * :copyright: Copyright 2007-2014 by the Sphinx team, see AUTHORS. 8 | * :license: BSD, see LICENSE for details. 9 | * 10 | */ 11 | 12 | /** 13 | * select a different prefix for underscore 14 | */ 15 | $u = _.noConflict(); 16 | 17 | /** 18 | * make the code below compatible with browsers without 19 | * an installed firebug like debugger 20 | if (!window.console || !console.firebug) { 21 | var names = ["log", "debug", "info", "warn", "error", "assert", "dir", 22 | "dirxml", "group", "groupEnd", "time", "timeEnd", "count", "trace", 23 | "profile", "profileEnd"]; 24 | window.console = {}; 25 | for (var i = 0; i < names.length; ++i) 26 | window.console[names[i]] = function() {}; 27 | } 28 | */ 29 | 30 | /** 31 | * small helper function to urldecode strings 32 | */ 33 | jQuery.urldecode = function(x) { 34 | return decodeURIComponent(x).replace(/\+/g, ' '); 35 | }; 36 | 37 | /** 38 | * small helper function to urlencode strings 39 | */ 40 | jQuery.urlencode = encodeURIComponent; 41 | 42 | /** 43 | * This function returns the parsed url parameters of the 44 | * current request. Multiple values per key are supported, 45 | * it will always return arrays of strings for the value parts. 46 | */ 47 | jQuery.getQueryParameters = function(s) { 48 | if (typeof s == 'undefined') 49 | s = document.location.search; 50 | var parts = s.substr(s.indexOf('?') + 1).split('&'); 51 | var result = {}; 52 | for (var i = 0; i < parts.length; i++) { 53 | var tmp = parts[i].split('=', 2); 54 | var key = jQuery.urldecode(tmp[0]); 55 | var value = jQuery.urldecode(tmp[1]); 56 | if (key in result) 57 | result[key].push(value); 58 | else 59 | result[key] = [value]; 60 | } 61 | return result; 62 | }; 63 | 64 | /** 65 | * highlight a given string on a jquery object by wrapping it in 66 | * span elements with the given class name. 67 | */ 68 | jQuery.fn.highlightText = function(text, className) { 69 | function highlight(node) { 70 | if (node.nodeType == 3) { 71 | var val = node.nodeValue; 72 | var pos = val.toLowerCase().indexOf(text); 73 | if (pos >= 0 && !jQuery(node.parentNode).hasClass(className)) { 74 | var span = document.createElement("span"); 75 | span.className = className; 76 | span.appendChild(document.createTextNode(val.substr(pos, text.length))); 77 | node.parentNode.insertBefore(span, node.parentNode.insertBefore( 78 | document.createTextNode(val.substr(pos + text.length)), 79 | node.nextSibling)); 80 | node.nodeValue = val.substr(0, pos); 81 | } 82 | } 83 | else if (!jQuery(node).is("button, select, textarea")) { 84 | jQuery.each(node.childNodes, function() { 85 | highlight(this); 86 | }); 87 | } 88 | } 89 | return this.each(function() { 90 | highlight(this); 91 | }); 92 | }; 93 | 94 | /** 95 | * Small JavaScript module for the documentation. 96 | */ 97 | var Documentation = { 98 | 99 | init : function() { 100 | this.fixFirefoxAnchorBug(); 101 | this.highlightSearchWords(); 102 | this.initIndexTable(); 103 | }, 104 | 105 | /** 106 | * i18n support 107 | */ 108 | TRANSLATIONS : {}, 109 | PLURAL_EXPR : function(n) { return n == 1 ? 0 : 1; }, 110 | LOCALE : 'unknown', 111 | 112 | // gettext and ngettext don't access this so that the functions 113 | // can safely bound to a different name (_ = Documentation.gettext) 114 | gettext : function(string) { 115 | var translated = Documentation.TRANSLATIONS[string]; 116 | if (typeof translated == 'undefined') 117 | return string; 118 | return (typeof translated == 'string') ? translated : translated[0]; 119 | }, 120 | 121 | ngettext : function(singular, plural, n) { 122 | var translated = Documentation.TRANSLATIONS[singular]; 123 | if (typeof translated == 'undefined') 124 | return (n == 1) ? singular : plural; 125 | return translated[Documentation.PLURALEXPR(n)]; 126 | }, 127 | 128 | addTranslations : function(catalog) { 129 | for (var key in catalog.messages) 130 | this.TRANSLATIONS[key] = catalog.messages[key]; 131 | this.PLURAL_EXPR = new Function('n', 'return +(' + catalog.plural_expr + ')'); 132 | this.LOCALE = catalog.locale; 133 | }, 134 | 135 | /** 136 | * add context elements like header anchor links 137 | */ 138 | addContextElements : function() { 139 | $('div[id] > :header:first').each(function() { 140 | $('\u00B6'). 141 | attr('href', '#' + this.id). 142 | attr('title', _('Permalink to this headline')). 143 | appendTo(this); 144 | }); 145 | $('dt[id]').each(function() { 146 | $('\u00B6'). 147 | attr('href', '#' + this.id). 148 | attr('title', _('Permalink to this definition')). 149 | appendTo(this); 150 | }); 151 | }, 152 | 153 | /** 154 | * workaround a firefox stupidity 155 | */ 156 | fixFirefoxAnchorBug : function() { 157 | if (document.location.hash && $.browser.mozilla) 158 | window.setTimeout(function() { 159 | document.location.href += ''; 160 | }, 10); 161 | }, 162 | 163 | /** 164 | * highlight the search words provided in the url in the text 165 | */ 166 | highlightSearchWords : function() { 167 | var params = $.getQueryParameters(); 168 | var terms = (params.highlight) ? params.highlight[0].split(/\s+/) : []; 169 | if (terms.length) { 170 | var body = $('div.body'); 171 | if (!body.length) { 172 | body = $('body'); 173 | } 174 | window.setTimeout(function() { 175 | $.each(terms, function() { 176 | body.highlightText(this.toLowerCase(), 'highlighted'); 177 | }); 178 | }, 10); 179 | $('

' + _('Hide Search Matches') + '

') 181 | .appendTo($('#searchbox')); 182 | } 183 | }, 184 | 185 | /** 186 | * init the domain index toggle buttons 187 | */ 188 | initIndexTable : function() { 189 | var togglers = $('img.toggler').click(function() { 190 | var src = $(this).attr('src'); 191 | var idnum = $(this).attr('id').substr(7); 192 | $('tr.cg-' + idnum).toggle(); 193 | if (src.substr(-9) == 'minus.png') 194 | $(this).attr('src', src.substr(0, src.length-9) + 'plus.png'); 195 | else 196 | $(this).attr('src', src.substr(0, src.length-8) + 'minus.png'); 197 | }).css('display', ''); 198 | if (DOCUMENTATION_OPTIONS.COLLAPSE_INDEX) { 199 | togglers.click(); 200 | } 201 | }, 202 | 203 | /** 204 | * helper function to hide the search marks again 205 | */ 206 | hideSearchWords : function() { 207 | $('#searchbox .highlight-link').fadeOut(300); 208 | $('span.highlighted').removeClass('highlighted'); 209 | }, 210 | 211 | /** 212 | * make the url absolute 213 | */ 214 | makeURL : function(relativeURL) { 215 | return DOCUMENTATION_OPTIONS.URL_ROOT + '/' + relativeURL; 216 | }, 217 | 218 | /** 219 | * get the current relative url 220 | */ 221 | getCurrentURL : function() { 222 | var path = document.location.pathname; 223 | var parts = path.split(/\//); 224 | $.each(DOCUMENTATION_OPTIONS.URL_ROOT.split(/\//), function() { 225 | if (this == '..') 226 | parts.pop(); 227 | }); 228 | var url = parts.join('/'); 229 | return path.substring(url.lastIndexOf('/') + 1, path.length - 1); 230 | } 231 | }; 232 | 233 | // quick alias for translations 234 | _ = Documentation.gettext; 235 | 236 | $(document).ready(function() { 237 | Documentation.init(); 238 | }); 239 | -------------------------------------------------------------------------------- /docs/_static/down-pressed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/fanova/46eb005320a0f67fc6b34ed74a91f702c2358c3f/docs/_static/down-pressed.png -------------------------------------------------------------------------------- /docs/_static/down.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/fanova/46eb005320a0f67fc6b34ed74a91f702c2358c3f/docs/_static/down.png -------------------------------------------------------------------------------- /docs/_static/file.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/fanova/46eb005320a0f67fc6b34ed74a91f702c2358c3f/docs/_static/file.png -------------------------------------------------------------------------------- /docs/_static/minus.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/fanova/46eb005320a0f67fc6b34ed74a91f702c2358c3f/docs/_static/minus.png -------------------------------------------------------------------------------- /docs/_static/plus.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/fanova/46eb005320a0f67fc6b34ed74a91f702c2358c3f/docs/_static/plus.png -------------------------------------------------------------------------------- /docs/_static/pygments.css: -------------------------------------------------------------------------------- 1 | .highlight .hll { background-color: #ffffcc } 2 | .highlight { background: #eeffcc; } 3 | .highlight .c { color: #408090; font-style: italic } /* Comment */ 4 | .highlight .err { border: 1px solid #FF0000 } /* Error */ 5 | .highlight .k { color: #007020; font-weight: bold } /* Keyword */ 6 | .highlight .o { color: #666666 } /* Operator */ 7 | .highlight .cm { color: #408090; font-style: italic } /* Comment.Multiline */ 8 | .highlight .cp { color: #007020 } /* Comment.Preproc */ 9 | .highlight .c1 { color: #408090; font-style: italic } /* Comment.Single */ 10 | .highlight .cs { color: #408090; background-color: #fff0f0 } /* Comment.Special */ 11 | .highlight .gd { color: #A00000 } /* Generic.Deleted */ 12 | .highlight .ge { font-style: italic } /* Generic.Emph */ 13 | .highlight .gr { color: #FF0000 } /* Generic.Error */ 14 | .highlight .gh { color: #000080; font-weight: bold } /* Generic.Heading */ 15 | .highlight .gi { color: #00A000 } /* Generic.Inserted */ 16 | .highlight .go { color: #333333 } /* Generic.Output */ 17 | .highlight .gp { color: #c65d09; font-weight: bold } /* Generic.Prompt */ 18 | .highlight .gs { font-weight: bold } /* Generic.Strong */ 19 | .highlight .gu { color: #800080; font-weight: bold } /* Generic.Subheading */ 20 | .highlight .gt { color: #0044DD } /* Generic.Traceback */ 21 | .highlight .kc { color: #007020; font-weight: bold } /* Keyword.Constant */ 22 | .highlight .kd { color: #007020; font-weight: bold } /* Keyword.Declaration */ 23 | .highlight .kn { color: #007020; font-weight: bold } /* Keyword.Namespace */ 24 | .highlight .kp { color: #007020 } /* Keyword.Pseudo */ 25 | .highlight .kr { color: #007020; font-weight: bold } /* Keyword.Reserved */ 26 | .highlight .kt { color: #902000 } /* Keyword.Type */ 27 | .highlight .m { color: #208050 } /* Literal.Number */ 28 | .highlight .s { color: #4070a0 } /* Literal.String */ 29 | .highlight .na { color: #4070a0 } /* Name.Attribute */ 30 | .highlight .nb { color: #007020 } /* Name.Builtin */ 31 | .highlight .nc { color: #0e84b5; font-weight: bold } /* Name.Class */ 32 | .highlight .no { color: #60add5 } /* Name.Constant */ 33 | .highlight .nd { color: #555555; font-weight: bold } /* Name.Decorator */ 34 | .highlight .ni { color: #d55537; font-weight: bold } /* Name.Entity */ 35 | .highlight .ne { color: #007020 } /* Name.Exception */ 36 | .highlight .nf { color: #06287e } /* Name.Function */ 37 | .highlight .nl { color: #002070; font-weight: bold } /* Name.Label */ 38 | .highlight .nn { color: #0e84b5; font-weight: bold } /* Name.Namespace */ 39 | .highlight .nt { color: #062873; font-weight: bold } /* Name.Tag */ 40 | .highlight .nv { color: #bb60d5 } /* Name.Variable */ 41 | .highlight .ow { color: #007020; font-weight: bold } /* Operator.Word */ 42 | .highlight .w { color: #bbbbbb } /* Text.Whitespace */ 43 | .highlight .mf { color: #208050 } /* Literal.Number.Float */ 44 | .highlight .mh { color: #208050 } /* Literal.Number.Hex */ 45 | .highlight .mi { color: #208050 } /* Literal.Number.Integer */ 46 | .highlight .mo { color: #208050 } /* Literal.Number.Oct */ 47 | .highlight .sb { color: #4070a0 } /* Literal.String.Backtick */ 48 | .highlight .sc { color: #4070a0 } /* Literal.String.Char */ 49 | .highlight .sd { color: #4070a0; font-style: italic } /* Literal.String.Doc */ 50 | .highlight .s2 { color: #4070a0 } /* Literal.String.Double */ 51 | .highlight .se { color: #4070a0; font-weight: bold } /* Literal.String.Escape */ 52 | .highlight .sh { color: #4070a0 } /* Literal.String.Heredoc */ 53 | .highlight .si { color: #70a0d0; font-style: italic } /* Literal.String.Interpol */ 54 | .highlight .sx { color: #c65d09 } /* Literal.String.Other */ 55 | .highlight .sr { color: #235388 } /* Literal.String.Regex */ 56 | .highlight .s1 { color: #4070a0 } /* Literal.String.Single */ 57 | .highlight .ss { color: #517918 } /* Literal.String.Symbol */ 58 | .highlight .bp { color: #007020 } /* Name.Builtin.Pseudo */ 59 | .highlight .vc { color: #bb60d5 } /* Name.Variable.Class */ 60 | .highlight .vg { color: #bb60d5 } /* Name.Variable.Global */ 61 | .highlight .vi { color: #bb60d5 } /* Name.Variable.Instance */ 62 | .highlight .il { color: #208050 } /* Literal.Number.Integer.Long */ -------------------------------------------------------------------------------- /docs/_static/sidebar.js: -------------------------------------------------------------------------------- 1 | /* 2 | * sidebar.js 3 | * ~~~~~~~~~~ 4 | * 5 | * This script makes the Sphinx sidebar collapsible. 6 | * 7 | * .sphinxsidebar contains .sphinxsidebarwrapper. This script adds 8 | * in .sphixsidebar, after .sphinxsidebarwrapper, the #sidebarbutton 9 | * used to collapse and expand the sidebar. 10 | * 11 | * When the sidebar is collapsed the .sphinxsidebarwrapper is hidden 12 | * and the width of the sidebar and the margin-left of the document 13 | * are decreased. When the sidebar is expanded the opposite happens. 14 | * This script saves a per-browser/per-session cookie used to 15 | * remember the position of the sidebar among the pages. 16 | * Once the browser is closed the cookie is deleted and the position 17 | * reset to the default (expanded). 18 | * 19 | * :copyright: Copyright 2007-2014 by the Sphinx team, see AUTHORS. 20 | * :license: BSD, see LICENSE for details. 21 | * 22 | */ 23 | 24 | $(function() { 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | // global elements used by the functions. 34 | // the 'sidebarbutton' element is defined as global after its 35 | // creation, in the add_sidebar_button function 36 | var bodywrapper = $('.bodywrapper'); 37 | var sidebar = $('.sphinxsidebar'); 38 | var sidebarwrapper = $('.sphinxsidebarwrapper'); 39 | 40 | // for some reason, the document has no sidebar; do not run into errors 41 | if (!sidebar.length) return; 42 | 43 | // original margin-left of the bodywrapper and width of the sidebar 44 | // with the sidebar expanded 45 | var bw_margin_expanded = bodywrapper.css('margin-left'); 46 | var ssb_width_expanded = sidebar.width(); 47 | 48 | // margin-left of the bodywrapper and width of the sidebar 49 | // with the sidebar collapsed 50 | var bw_margin_collapsed = '.8em'; 51 | var ssb_width_collapsed = '.8em'; 52 | 53 | // colors used by the current theme 54 | var dark_color = $('.related').css('background-color'); 55 | var light_color = $('.document').css('background-color'); 56 | 57 | function sidebar_is_collapsed() { 58 | return sidebarwrapper.is(':not(:visible)'); 59 | } 60 | 61 | function toggle_sidebar() { 62 | if (sidebar_is_collapsed()) 63 | expand_sidebar(); 64 | else 65 | collapse_sidebar(); 66 | } 67 | 68 | function collapse_sidebar() { 69 | sidebarwrapper.hide(); 70 | sidebar.css('width', ssb_width_collapsed); 71 | bodywrapper.css('margin-left', bw_margin_collapsed); 72 | sidebarbutton.css({ 73 | 'margin-left': '0', 74 | 'height': bodywrapper.height() 75 | }); 76 | sidebarbutton.find('span').text('»'); 77 | sidebarbutton.attr('title', _('Expand sidebar')); 78 | document.cookie = 'sidebar=collapsed'; 79 | } 80 | 81 | function expand_sidebar() { 82 | bodywrapper.css('margin-left', bw_margin_expanded); 83 | sidebar.css('width', ssb_width_expanded); 84 | sidebarwrapper.show(); 85 | sidebarbutton.css({ 86 | 'margin-left': ssb_width_expanded-12, 87 | 'height': bodywrapper.height() 88 | }); 89 | sidebarbutton.find('span').text('«'); 90 | sidebarbutton.attr('title', _('Collapse sidebar')); 91 | document.cookie = 'sidebar=expanded'; 92 | } 93 | 94 | function add_sidebar_button() { 95 | sidebarwrapper.css({ 96 | 'float': 'left', 97 | 'margin-right': '0', 98 | 'width': ssb_width_expanded - 28 99 | }); 100 | // create the button 101 | sidebar.append( 102 | '
«
' 103 | ); 104 | var sidebarbutton = $('#sidebarbutton'); 105 | light_color = sidebarbutton.css('background-color'); 106 | // find the height of the viewport to center the '<<' in the page 107 | var viewport_height; 108 | if (window.innerHeight) 109 | viewport_height = window.innerHeight; 110 | else 111 | viewport_height = $(window).height(); 112 | sidebarbutton.find('span').css({ 113 | 'display': 'block', 114 | 'margin-top': (viewport_height - sidebar.position().top - 20) / 2 115 | }); 116 | 117 | sidebarbutton.click(toggle_sidebar); 118 | sidebarbutton.attr('title', _('Collapse sidebar')); 119 | sidebarbutton.css({ 120 | 'color': '#FFFFFF', 121 | 'border-left': '1px solid ' + dark_color, 122 | 'font-size': '1.2em', 123 | 'cursor': 'pointer', 124 | 'height': bodywrapper.height(), 125 | 'padding-top': '1px', 126 | 'margin-left': ssb_width_expanded - 12 127 | }); 128 | 129 | sidebarbutton.hover( 130 | function () { 131 | $(this).css('background-color', dark_color); 132 | }, 133 | function () { 134 | $(this).css('background-color', light_color); 135 | } 136 | ); 137 | } 138 | 139 | function set_position_from_cookie() { 140 | if (!document.cookie) 141 | return; 142 | var items = document.cookie.split(';'); 143 | for(var k=0; k 3 | 4 | 5 | 6 | 7 | 8 | 9 | Citing fANOVA — fanova 2.0.5 documentation 10 | 11 | 12 | 13 | 14 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 |
31 |

Navigation

32 | 41 |
42 | 43 |
44 |
45 |
46 |
47 | 48 |
49 |

Citing fANOVA

50 |

If you use the fANOVA for your research, please cite the ICML 2014 paper “An Efficient Approach for Assessing Hyperparameter Importance” by Frank Hutter, Holger Hoos and Kevin Leyton-Brown.

51 |

with the following Bibtex file:

52 |
53 |
@inproceedings{HutHooLey14, 54 | lauthor = {Frank Hutter and Holger Hoos and Kevin Leyton-Brown}, 55 | author = {F. Hutter and H. Hoos and K. Leyton-Brown}, 56 | title = {An Efficient Approach for Assessing Hyperparameter Importance}, 57 | booktitle = {Proceedings of International Conference on Machine Learning 2014 (ICML 2014)}, 58 | year = {2014}, 59 | pages = {754–762}, 60 | month = jun, 61 | }
62 |
63 | 64 | 65 |
66 |
67 |
68 |
69 |
70 |

Previous topic

71 |

fANOVA for Starters

73 |

This Page

74 | 78 | 90 | 91 |
92 |
93 |
94 |
95 |
96 |

Navigation

97 | 106 |
107 |
108 | © Copyright 2017, Frank Hutter, Stefan Falkner. 109 | Created using Sphinx 1.2.2. 110 |
111 | 112 | -------------------------------------------------------------------------------- /docs/doctrees/cite.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/fanova/46eb005320a0f67fc6b34ed74a91f702c2358c3f/docs/doctrees/cite.doctree -------------------------------------------------------------------------------- /docs/doctrees/environment.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/fanova/46eb005320a0f67fc6b34ed74a91f702c2358c3f/docs/doctrees/environment.pickle -------------------------------------------------------------------------------- /docs/doctrees/fanova_starter.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/fanova/46eb005320a0f67fc6b34ed74a91f702c2358c3f/docs/doctrees/fanova_starter.doctree -------------------------------------------------------------------------------- /docs/doctrees/includeme.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/fanova/46eb005320a0f67fc6b34ed74a91f702c2358c3f/docs/doctrees/includeme.doctree -------------------------------------------------------------------------------- /docs/doctrees/index.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/fanova/46eb005320a0f67fc6b34ed74a91f702c2358c3f/docs/doctrees/index.doctree -------------------------------------------------------------------------------- /docs/doctrees/install.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/fanova/46eb005320a0f67fc6b34ed74a91f702c2358c3f/docs/doctrees/install.doctree -------------------------------------------------------------------------------- /docs/doctrees/manual.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/fanova/46eb005320a0f67fc6b34ed74a91f702c2358c3f/docs/doctrees/manual.doctree -------------------------------------------------------------------------------- /docs/fanova_starter.html: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | fANOVA for Starters — fanova 2.0.5 documentation 10 | 11 | 12 | 13 | 14 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 |
32 |

Navigation

33 | 45 |
46 | 47 |
48 |
49 |
50 |
51 | 52 |
53 |

fANOVA for Starters

54 |
55 |

Introduction

56 |

Whereas many algorithms presumably depend on a large hyperparameter space, it is known that in many cases only few parameter changes can be responsible for nearly all of the performance improvement. To this end, different techniques have been proposed 57 | that score parameters based on their importance. 58 | The functional analysis of variance (fANOVA) uses an empirical performance model (EPM) which is based on random forests in order to analyze how much of the performance 59 | variance in the configuration space is explained by single parameters or combinations of few parameters.

60 |

Note that fANOVA’s result is ideally used for minimizing the hyperparameter search space and should not be seen as a flawless parameter configurator.

61 |

More detailed information in : “An Efficient Approach for Assessing Hyperparameter Importance” 62 | by Frank Hutter, Holger Hoos and Kevin Leyton-Brown

63 |
64 |
65 |

How to interpret your results

66 |

In order to interpret your plots, you should have in mind what kind of evaluation measurement function you wanted to consider: 67 | Either you’d like to maximize or minimize your function. 68 | Therefore, depending on this, you’d have to consider low or high performance values.

69 |

Let’s start with the online LDA example from our examples folder:

70 |

Here we have the perplexity as measurement. Thus we would like to reduce it.

71 |

First, by looking at the importance of each parameter we can clearly see that parameter Col2 is marginally most important. So the result states that the Col2 parameter by itself is responsible for approx. 62% of the perplexity’s variability across the entire space. 72 | On the contrary, Col1 is marginally less important with 3,5%.

73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 |
ParameterImportance
Col00.066
Col10.035
Col20.619
95 |

And by looking at the pairwise marginals we have the combination of Col0 with Col2 as marginally most important.

96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 |
Parameter pairImportance
[Col0, Col1]0.011
[Col0, Col2]0.146
[Col1, Col2]0.067
118 |

Now let’s take a look at the visualizations thereof:

119 |

Since Col2 was chosen as marginally more important, we will take a closer look at its single marginal plot. 120 | It shows that large values for the parameter consistently yield lower performance (in our case perplexity).

121 |
122 |
_images/Col2.png 123 |
124 |

In order to capture interaction effects we take a closer look at the pairwise marginal plots of Col0 and Col2.

125 |

Here we can see that Col0 is much more important for smaller values of Col2 than for larger ones. Such an interaction cannot be shown by single marginals.

126 |
127 |
_images/figure1.png 128 |
129 |

Note that nevertheless this example consists of a lower-dimensional dataset (3 parameters), fANOVA can still give interesting insights. But you should have in mind that it is important to have a large dataset consisting of enough examples so that fANOVA can deliver significant and interpretable results.

130 |
131 |
132 | 133 | 134 |
135 |
136 |
137 |
138 |
139 |

Table Of Contents

140 | 147 | 148 |

Previous topic

149 |

Manual

151 |

Next topic

152 |

Citing fANOVA

154 |

This Page

155 | 159 | 171 | 172 |
173 |
174 |
175 |
176 |
177 |

Navigation

178 | 190 |
191 |
192 | © Copyright 2017, Frank Hutter, Stefan Falkner. 193 | Created using Sphinx 1.2.2. 194 |
195 | 196 | -------------------------------------------------------------------------------- /docs/genindex.html: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | Index — fanova 2.0.5 documentation 11 | 12 | 13 | 14 | 15 | 24 | 25 | 26 | 27 | 28 | 29 | 30 |
31 |

Navigation

32 | 38 |
39 | 40 |
41 |
42 |
43 |
44 | 45 | 46 |

Index

47 | 48 |
49 | 50 |
51 | 52 | 53 |
54 |
55 |
56 |
57 |
58 | 59 | 60 | 61 | 73 | 74 |
75 |
76 |
77 |
78 |
79 |

Navigation

80 | 86 |
87 |
88 | © Copyright 2017, Frank Hutter, Stefan Falkner. 89 | Created using Sphinx 1.2.2. 90 |
91 | 92 | -------------------------------------------------------------------------------- /docs/includeme.html: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | fANOVA — fanova 2.0.5 documentation 10 | 11 | 12 | 13 | 14 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 |
32 |

Navigation

33 | 45 |
46 | 47 |
48 |
49 |
50 |
51 | 52 |
53 |

fANOVA

54 |

Functional ANOVA: an implementation of the ICML 2014 paper “An Efficient Approach for Assessing Hyperparameter Importance” by Frank Hutter, Holger Hoos and Kevin Leyton-Brown.

55 |
56 |

Documentation

57 |

An ‘ever growing’ documentation for the Python bindings can be found at https://automl.github.io/fanova/

58 |
59 |
60 | 61 | 62 |
63 |
64 |
65 |
66 |
67 |

Table Of Contents

68 | 74 | 75 |

Previous topic

76 |

Welcome to fANOVA’s documentation!

78 |

Next topic

79 |

Installation

81 |

This Page

82 | 86 | 98 | 99 |
100 |
101 |
102 |
103 |
104 |

Navigation

105 | 117 |
118 |
119 | © Copyright 2017, Frank Hutter, Stefan Falkner. 120 | Created using Sphinx 1.2.2. 121 |
122 | 123 | -------------------------------------------------------------------------------- /docs/index.html: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Welcome to fANOVA’s documentation! — fanova 2.0.5 documentation 10 | 11 | 12 | 13 | 14 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 |
31 |

Navigation

32 | 41 |
42 | 43 |
44 |
45 |
46 |
47 | 48 |
49 |

Welcome to fANOVA’s documentation!

50 |

The functional analysis of variance (fANOVA) framework is used for quantifying the performance of functions.

51 |

It provides insight into te relationships between parameter setting and performance.

52 |

Contents:

53 |
54 | 81 |
82 |
83 | 84 | 85 |
86 |
87 |
88 |
89 |
90 |

Next topic

91 |

fANOVA

93 |

This Page

94 | 98 | 110 | 111 |
112 |
113 |
114 |
115 |
116 |

Navigation

117 | 126 |
127 |
128 | © Copyright 2017, Frank Hutter, Stefan Falkner. 129 | Created using Sphinx 1.2.2. 130 |
131 | 132 | -------------------------------------------------------------------------------- /docs/install.html: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Installation — fanova 2.0.5 documentation 10 | 11 | 12 | 13 | 14 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 |
32 |

Navigation

33 | 45 |
46 | 47 |
48 |
49 |
50 |
51 | 52 |
53 |

Installation

54 |
55 |

Requirements

56 |

fANOVA requires:

57 |

Numpy

58 |

matplotlib (Version 1.4.2)

59 |

pyrfr

60 |
61 |
62 |

Manually

63 |

To install fANOVA from command line type the following commands in your bash terminal:

64 |
65 |

git clone https://github.com/automl/fanova.git

66 |

cd fanova/

67 |

pip install -r requirements.txt

68 |

python setup.py install

69 |
70 |
71 |
72 | 73 | 74 |
75 |
76 |
77 |
78 |
79 |

Table Of Contents

80 | 87 | 88 |

Previous topic

89 |

fANOVA

91 |

Next topic

92 |

Manual

94 |

This Page

95 | 99 | 111 | 112 |
113 |
114 |
115 |
116 |
117 |

Navigation

118 | 130 |
131 |
132 | © Copyright 2017, Frank Hutter, Stefan Falkner. 133 | Created using Sphinx 1.2.2. 134 |
135 | 136 | -------------------------------------------------------------------------------- /docs/manual.html: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Manual — fanova 2.0.5 documentation 10 | 11 | 12 | 13 | 14 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 |
32 |

Navigation

33 | 45 |
46 | 47 |
48 |
49 |
50 |
51 | 52 |
53 |

Manual

54 |
55 |

Quick Start

56 |

To run the examples, just download the data and start the python console. 57 | We can then import fANOVA and start it by typing

58 | 
>>> from fanova import fANOVA
 59 | >>> import csv
 60 | >>> path = os.path.dirname(os.path.realpath(__file__))
 61 | >>> X = np.loadtxt(path + '/example_data/online_lda/online_lda_features.csv', delimiter=",")
 62 | >>> Y = np.loadtxt(path + '/example_data/online_lda/online_lda_responses.csv', delimiter=",")
 63 | >>> f = fANOVA(X,Y)
 64 |
65 |
66 |

This creates a new fANOVA object and fits the Random Forest on the specified data set.

67 |

To compute now the marginal of the first parameter type:

68 | 
>>> f.quantify_importance((0, ))
 69 |     0.075414122571199116
 70 |
71 |
72 |

fANOVA also allows to specify parameters by their names.

73 | 
>>> f.quantify_importance(("Col0", ))
 74 |     0.075414122571199116
 75 |
76 |
77 |
78 |
79 |

Advanced

80 |

If you want the fANOVA only a certain quantiles (let’s say between 10% and 25%) of the data you can call it by:

81 | 
>>> f = fANOVA(X,Y)
 82 | >>> f.set_cutoffs(quantile=(10, 25))
 83 |
84 |
85 |

Furthermore fANOVA now supports cutoffs on the y values. These will exclude parts of the parameters space where the prediction is not within the provided cutoffs.

86 | 
>>> f.set_cutoffs(cutoffs=(-np.inf, np.inf))
 87 |
88 |
89 |

You can also specify the number of trees in the random forest as well as the minimum number of points to make a new split in a tree or your already specified configuration space by:

90 | 
>>> f = fANOVA(X,Y, config_space=config_space, num_trees=30, min_samples_split=3)
 91 |
92 |
93 |
94 |
95 |

More functions

96 |
97 |
    98 |
  • f.get_most_important_pairwise_marginals(n)
    • 99 |
100 |

Returns the n most important pairwise marginals

101 |
    102 |
  • f.get_most_important_pairwise_marginals(params)
    • 103 |
104 |

Returns the pairwise marginals of all elements in the list.

105 |
    106 |
  • f.get_triple_marginals(params)
    • 107 |
108 |

Returns the marginals of all possible triplets in the list. They are sorted by importance.

109 |
    110 |
  • fANOVA.marginal_mean_variance_for_values(p, v)
    • 111 |
112 |

Computes the mean and standard deviation of the parameter (or parameterlist) p for a certain value v

113 |
114 |
115 |
116 |

pysmac

117 |

In order to run fANOVA on pysmac output:

118 | 
>>> import pysmac.utils.pysmac_fanova as pysmac_fanova
119 | >>> fanova = pysmac_fanova.smac_to_fanova('path_to/pysmac_output_dir/out/scenario', 'path_to/merged_states')
120 |
121 |
122 |
123 |
124 |

Visualization

125 |

To visualize the single and pairwise marginals, we have to create a visualizer object first containing the fanova object, configspace and directory

126 | 
>>> import fanova.visualizer
127 | >>> vis = fanova.visualizer.Visualizer(f, config_space, "./plots/")
128 |
129 |
130 |

We can then plot single marginals by

131 | 
>>> vis.plot_marginal(1)
132 |
133 |
134 |

what should look like this

135 | _images/Col1.png 136 |

The same can been done for pairwise marginals

137 | 
>>> vis.plot_pairwise_marginal([0,1])
138 |
139 |
140 | _images/figure2.png 141 |

If you are just interested in the N most important pairwise marginals you can plot them through:

142 | 
>>> create_most_important_pairwise_marginal_plots(dir, n)
143 |
144 |
145 |

and fANOVA will save those plot in dir. However, be aware that to create the plots fANOVA needs to compute all pairwise marginal, which can take awhile!

146 |

If you’re not interested in the plot itself, but want to extract the values for your own plots, simply call

147 | 
>>> vis.generate_marginal(0)
148 |
149 |
150 |

At last, all plots can be created together and stored in a directory with

151 | 
>>> vis.create_all_plots()
152 |
153 |
154 |
155 |
156 |

How to load interactive plots

157 |

You will also find an extra directory in your specified plot directory called ‘interactive_plots’ in which you can find all interactive pairwise plots as pickle files.

158 | 
>>> import pickle
159 | >>> figx = pickle.load(open('/interactive_plots/param1_param2.fig.pickle', 'rb'))
160 | >>> figx.show()
161 |
162 |
163 |
164 |
165 |

How to load a CSV-file

166 |

import numpy as np

167 |

data = np.loadtxt(‘your_file.csv’, delimiter=”,”)

168 |
169 |
170 | 171 | 172 |
173 |
174 |
175 |
176 |
177 |

Table Of Contents

178 | 190 | 191 |

Previous topic

192 |

Installation

194 |

Next topic

195 |

fANOVA for Starters

197 |

This Page

198 | 202 | 214 | 215 |
216 |
217 |
218 |
219 |
220 |

Navigation

221 | 233 |
234 |
235 | © Copyright 2017, Frank Hutter, Stefan Falkner. 236 | Created using Sphinx 1.2.2. 237 |
238 | 239 | -------------------------------------------------------------------------------- /docs/objects.inv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/fanova/46eb005320a0f67fc6b34ed74a91f702c2358c3f/docs/objects.inv -------------------------------------------------------------------------------- /docs/search.html: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Search — fanova 2.0.5 documentation 10 | 11 | 12 | 13 | 14 | 23 | 24 | 25 | 26 | 27 | 28 | 31 | 32 | 33 | 34 | 35 | 36 | 37 |
38 |

Navigation

39 | 45 |
46 | 47 |
48 |
49 |
50 |
51 | 52 |

Search

53 |
54 | 55 |

56 | Please activate JavaScript to enable the search 57 | functionality. 58 |

59 |
60 |

61 | From here you can search these documents. Enter your search 62 | words into the box below and click "search". Note that the search 63 | function will automatically search for all of the words. Pages 64 | containing fewer words won't appear in the result list. 65 |

66 |
67 | 68 | 69 | 70 |
71 | 72 |
73 | 74 |
75 | 76 |
77 |
78 |
79 |
80 |
81 |
82 |
83 |
84 |
85 |
86 |

Navigation

87 | 93 |
94 |
95 | © Copyright 2017, Frank Hutter, Stefan Falkner. 96 | Created using Sphinx 1.2.2. 97 |
98 | 99 | -------------------------------------------------------------------------------- /docs/searchindex.js: -------------------------------------------------------------------------------- 1 | Search.setIndex({envversion:42,terms:{all:[1,3],month:5,signific:1,dirnam:3,follow:[5,4],evalu:1,middl:[],depend:1,deviat:3,brown:[1,5,2],quantil:3,util:3,figx:3,marginal_mean_variance_for_valu:3,list:3,small:[],dir:3,merged_st:3,pleas:5,smaller:1,rate:[],download:3,further:[],what:[1,3],version:4,"new":3,ever:2,here:1,let:[1,3],path:3,sinc:1,valu:[1,3],box:[],search:1,holger:[1,5,2],larger:1,observ:[],hyperparamet:[1,5,2],extra:3,appli:[],instal:[],total:[],plot:[],from:[1,3,4],describ:[],would:1,num_tre:3,two:[],few:1,call:3,type:[3,4],more:[],pyrfr:4,loadtxt:3,appropi:[],train:[],known:1,setup:4,can:[1,3,2],learn:5,give:1,predict:3,high:1,minimum:3,want:[1,3],onlin:1,end:1,hoo:[1,5,2],leyton:[1,5,2],path_to:3,how:[],forest:[1,3],csv:[],booktitl:5,clone:4,varianc:[0,1],data:3,grow:2,github:[4,2],bind:2,author:5,exclud:3,inform:1,combin:1,allow:3,order:[1,3],approx:1,sigmoid:[],insight:[0,1],orang:[],paper:[5,2],through:3,configspac:3,still:1,paramet:[0,1,3],fit:3,chosen:1,get_most_important_pairwise_margin:3,fig:3,therefor:1,them:3,good:[],"return":3,python:[3,4,2],col2:1,initi:[],col1:1,framework:0,now:[1,3],choic:[],name:3,softmax:[],each:1,found:2,mean:3,year:5,our:1,hyperband:[],extract:3,reduct:[],out:3,variabl:1,shown:1,network:[],space:[1,3],research:5,content:0,optim:[],quantifi:0,lauthor:5,linear:[],advanc:[],frank:[1,5,2],given:[],standard:3,quick:[],base:1,bash:4,flawless:1,first:[1,3],rang:[],number:3,thereof:1,alreadi:3,done:3,open:3,differ:1,interact:[],automl:[4,2],too:[],termin:4,store:3,plot_pairwise_margin:3,consol:3,relationship:0,specifi:3,part:3,than:1,png:[],kind:1,contrari:1,provid:[0,3],tree:3,matter:[],blabla:[],were:[],lowest:[],sai:3,comput:3,mind:1,clearli:1,have:[1,3],close:[],need:3,seen:1,seem:[],element:3,kevin:[1,5,2],techniqu:1,note:1,also:3,ideal:1,epm:1,take:[1,3],which:[1,3],boxplot:[],singl:[1,3],analysi:[0,1],realpath:3,distribut:[],object:3,marin:[],most:[1,3],pair:1,activation1:[],pysmac_output_dir:3,bibtex:5,determin:[],example_data:3,show:[1,3],random:[1,3],parameterlist:3,empir:1,find:3,onli:[1,3],explain:1,configur:[1,3],activ:[],state:1,should:[1,3],won:[],analyz:1,factor:[],folder:1,smac_to_fanova:3,nearli:1,get:[],"__file__":3,stop:[],cannot:1,requir:[],median:[],yield:1,contain:3,where:3,set:[0,3],see:1,result:[],respons:1,create_all_plot:3,best:[],closer:1,awar:3,enough:1,score:1,between:[0,3],"import":[1,3,5,2],approach:[1,5,2],across:1,interactive_plot:3,inproceed:5,entir:1,matplotlib:4,both:[],last:3,delimit:3,region:[],improv:1,uncertainti:[],com:4,generate_margin:3,load:[],simpli:3,point:3,anova:2,assum:[],param1_param2:3,numpi:[3,4],been:[1,3],much:1,interest:[1,3],pysmac_fanova:3,dropout:[],you:[1,3,5],togeth:3,those:3,online_lda:3,"case":1,online_lda_respons:3,subsequ:[],look:[1,3],tanh:[],align:[],pairwis:[1,3],error:[],margin:[1,3],howev:3,layer:[],almost:[],metric:[],deliv:1,itself:[1,3],minim:1,perform:[0,1],make:3,same:3,epoch:[],split:3,quantify_import:3,higher:[],http:[4,2],interquartil:[],wherea:1,effect:1,your_fil:3,col0:[1,3],mani:1,chang:1,squar:[],lower:1,scenario:3,neural:[],nevertheless:1,thu:1,well:3,pickl:3,exampl:[1,3],command:4,thi:[1,3],choos:[],model:1,dimension:1,propos:1,lda:1,just:3,less:1,huthooley14:5,concentr:[],param:3,save:3,loss:[],recurr:[],measur:1,like:[1,3],manual:[],either:1,output:3,page:5,"function":[],plot_margin:3,captur:1,maxim:1,intern:5,config_spac:3,txt:4,larg:1,machin:5,run:3,confer:5,proceed:5,cutoff:3,own:3,effici:[1,5,2],within:3,dataset:1,assess:[1,5,2],awhil:3,your:[],git:4,area:[],support:3,relu:[],start:[],low:1,create_most_important_pairwise_marginal_plot:3,perplex:1,icml:[5,2],jun:5,line:4,categor:[],furthermor:3,consist:1,set_cutoff:3,creat:3,certain:3,dure:[],repres:[],implement:2,file:[],pip:4,inf:3,titl:5,when:[],detail:1,hutter:[1,5,2],other:[],min_samples_split:3,online_lda_featur:3,presum:1,imag:[],consid:1,pysmac:[],reduc:1,algorithm:1,directori:3},objtypes:{},objnames:{},filenames:["index","fanova_starter","includeme","manual","install","cite"],titles:["Welcome to fANOVA’s documentation!","fANOVA for Starters","fANOVA","Manual","Installation","Citing fANOVA"],objects:{},titleterms:{load:3,interpret:1,result:1,file:3,instal:4,your:1,plot:3,welcom:0,how:[1,3],interact:3,start:3,starter:1,csv:3,cite:5,more:3,"function":3,advanc:3,visual:3,requir:4,pysmac:3,document:[0,2],manual:[3,4],quick:3,introduct:1,fanova:[0,1,5,2]}}) -------------------------------------------------------------------------------- /examples/branin_pysmac_example.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | branin example for testing pysmac-fanova 4 | """ 5 | 6 | import math 7 | import pysmac 8 | import pysmac.utils.pysmac_fanova as pysmac_fanova 9 | 10 | import os 11 | path = os.path.dirname(os.path.realpath(__file__)) 12 | 13 | ''' 14 | Branin as example 15 | ''' 16 | def modified_branin(x1, x2, x3): 17 | # enforce that x3 has an integer value 18 | if (int(x3) != x3): 19 | raise ValueError("parameter x3 has to be an integer!") 20 | a = 1 21 | b = 5.1 / (4*math.pi**2) 22 | c = 5 / math.pi 23 | r = 6 24 | s = 10 25 | t = 1 / (8*math.pi) 26 | ret = a*(x2-b*x1**2+c*x1-r)**2+s*(1-t)*math.cos(x1)+s + x3 27 | return ret 28 | 29 | # parameter definition 30 | parameter_definition=dict(\ 31 | x1=('real', [-5, 5], 1), # this line means x1 is a float between -5 and 5, with a default of 1 32 | x2=('real', [-5, 5], -1), # same as x1, but the default is -1 33 | x3=('integer', [0, 10], 1), # integer that ranges between 0 and 10, default is 1 34 | ) 35 | # optimizer object 36 | opt = pysmac.SMAC_optimizer(working_directory = path + '/pysmac_output', persistent_files=True) 37 | # call its minimize method 38 | value, parameters = opt.minimize(modified_branin, # the function to be minimized 39 | 100, # 1000 the maximum number of function evaluations 40 | parameter_definition, 41 | num_runs = 3) # the parameter dictionary 42 | 43 | # fanova object 44 | fanova = pysmac_fanova.smac_to_fanova(path + '/pysmac_output/out/scenario', path + "/merged_states") 45 | res = fanova.quantify_importance((2, )) 46 | print(res) 47 | best_margs = fanova.get_most_important_pairwise_marginals(n=3) 48 | print('Most important pairwise marginals: %s' %best_margs) 49 | -------------------------------------------------------------------------------- /examples/comparison/old_online_lda/Col1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/fanova/46eb005320a0f67fc6b34ed74a91f702c2358c3f/examples/comparison/old_online_lda/Col1.png -------------------------------------------------------------------------------- /examples/comparison/old_online_lda/instance-features.txt: -------------------------------------------------------------------------------- 1 | instance_name 2 | 0 3 | -------------------------------------------------------------------------------- /examples/comparison/old_online_lda/instances.txt: -------------------------------------------------------------------------------- 1 | 0 2 | -------------------------------------------------------------------------------- /examples/comparison/old_online_lda/pairwise.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/fanova/46eb005320a0f67fc6b34ed74a91f702c2358c3f/examples/comparison/old_online_lda/pairwise.png -------------------------------------------------------------------------------- /examples/comparison/old_online_lda/param-file-disc.txt: -------------------------------------------------------------------------------- 1 | Col0 {0.5,0.6,0.7,0.8,0.9,1.0}[0.5] 2 | Col1 {0.0,2.0,4.0,6.0,8.0,10.0}[0.0] 3 | Col2 {0.0,2.0,4.0,6.0,8.0,10.0,12.0,14.0}[0.0] 4 | -------------------------------------------------------------------------------- /examples/comparison/old_online_lda/param-file.txt: -------------------------------------------------------------------------------- 1 | Col0 [0.5, 1.0] [0.75] 2 | Col1 [0.0, 10.0] [5.0] 3 | Col2 [0.0, 14.0] [7.0] 4 | -------------------------------------------------------------------------------- /examples/comparison/old_online_lda/scenario.txt: -------------------------------------------------------------------------------- 1 | algo = . 2 | execdir = . 3 | deterministic = 0 4 | run_obj = qual 5 | overall_obj = mean 6 | cutoff_time = 10000000 7 | cutoff_length = max 8 | tunerTimeout = 10000000 9 | paramfile = nonlog_data_lda_all_performances//param-file.txt 10 | instance_file = nonlog_data_lda_all_performances//instances.txt 11 | feature_file = nonlog_data_lda_all_performances//instance-features.txt 12 | -------------------------------------------------------------------------------- /examples/comparison/old_online_lda/uniq_configurations-it2.csv: -------------------------------------------------------------------------------- 1 | 1,1.0,0.2,0.2857142857142857 2 | 2,0.8,1.0,0.8571428571428571 3 | 3,0.19999999999999996,1.0,0.8571428571428571 4 | 4,0.19999999999999996,0.4,0.14285714285714285 5 | 5,0.3999999999999999,0.0,0.8571428571428571 6 | 6,0.0,0.6,1.0 7 | 7,0.19999999999999996,0.6,0.14285714285714285 8 | 8,1.0,0.0,0.0 9 | 9,1.0,0.4,0.42857142857142855 10 | 10,0.0,0.8,1.0 11 | 11,0.6000000000000001,0.8,0.8571428571428571 12 | 12,0.19999999999999996,0.8,0.2857142857142857 13 | 13,0.0,0.2,0.42857142857142855 14 | 14,0.6000000000000001,0.6,0.5714285714285714 15 | 15,1.0,0.4,0.14285714285714285 16 | 16,0.0,0.0,0.14285714285714285 17 | 17,0.8,0.4,0.2857142857142857 18 | 18,0.6000000000000001,0.0,0.8571428571428571 19 | 19,1.0,0.6,0.7142857142857143 20 | 20,0.3999999999999999,0.0,0.42857142857142855 21 | 21,0.0,0.2,0.2857142857142857 22 | 22,0.3999999999999999,0.8,0.42857142857142855 23 | 23,0.8,0.4,0.7142857142857143 24 | 24,0.0,0.8,0.0 25 | 25,1.0,1.0,0.14285714285714285 26 | 26,0.6000000000000001,1.0,0.8571428571428571 27 | 27,0.8,0.4,0.14285714285714285 28 | 28,0.0,0.6,0.0 29 | 29,1.0,0.2,0.5714285714285714 30 | 30,1.0,0.6,0.2857142857142857 31 | 31,0.6000000000000001,1.0,0.14285714285714285 32 | 32,0.6000000000000001,0.6,0.2857142857142857 33 | 33,0.8,0.6,0.2857142857142857 34 | 34,0.8,1.0,0.5714285714285714 35 | 35,0.3999999999999999,0.2,0.0 36 | 36,0.8,0.0,0.42857142857142855 37 | 37,0.6000000000000001,0.2,0.2857142857142857 38 | 38,0.19999999999999996,0.0,0.2857142857142857 39 | 39,0.8,1.0,0.42857142857142855 40 | 40,0.3999999999999999,0.4,1.0 41 | 41,1.0,0.0,0.14285714285714285 42 | 42,1.0,1.0,1.0 43 | 43,0.6000000000000001,0.2,0.0 44 | 44,0.19999999999999996,0.6,0.42857142857142855 45 | 45,0.6000000000000001,0.4,0.2857142857142857 46 | 46,0.0,0.4,1.0 47 | 47,0.0,1.0,0.5714285714285714 48 | 48,0.19999999999999996,0.0,0.7142857142857143 49 | 49,0.19999999999999996,0.2,0.14285714285714285 50 | 50,0.19999999999999996,0.6,1.0 51 | 51,1.0,0.2,1.0 52 | 52,1.0,0.2,0.42857142857142855 53 | 53,0.19999999999999996,1.0,0.14285714285714285 54 | 54,0.0,0.0,0.5714285714285714 55 | 55,0.3999999999999999,1.0,0.7142857142857143 56 | 56,0.8,0.6,0.42857142857142855 57 | 57,0.19999999999999996,0.8,0.5714285714285714 58 | 58,0.8,0.2,0.14285714285714285 59 | 59,0.8,1.0,0.7142857142857143 60 | 60,0.8,0.4,0.42857142857142855 61 | 61,1.0,1.0,0.5714285714285714 62 | 62,0.19999999999999996,1.0,0.0 63 | 63,0.6000000000000001,1.0,0.42857142857142855 64 | 64,1.0,0.4,0.8571428571428571 65 | 65,0.8,0.4,0.8571428571428571 66 | 66,0.0,0.6,0.42857142857142855 67 | 67,0.6000000000000001,0.0,0.5714285714285714 68 | 68,0.19999999999999996,0.0,0.0 69 | 69,0.0,0.6,0.2857142857142857 70 | 70,0.3999999999999999,0.4,0.5714285714285714 71 | 71,1.0,0.6,0.42857142857142855 72 | 72,0.8,0.8,1.0 73 | 73,0.3999999999999999,0.8,0.7142857142857143 74 | 74,1.0,0.8,0.2857142857142857 75 | 75,0.19999999999999996,0.4,1.0 76 | 76,0.3999999999999999,1.0,0.14285714285714285 77 | 77,1.0,0.6,0.8571428571428571 78 | 78,0.6000000000000001,0.4,0.14285714285714285 79 | 79,0.8,0.2,0.42857142857142855 80 | 80,1.0,0.8,0.7142857142857143 81 | 81,1.0,1.0,0.2857142857142857 82 | 82,0.3999999999999999,1.0,0.42857142857142855 83 | 83,0.3999999999999999,0.2,1.0 84 | 84,0.19999999999999996,0.2,0.8571428571428571 85 | 85,0.19999999999999996,0.2,0.7142857142857143 86 | 86,0.8,0.0,0.0 87 | 87,0.8,0.6,1.0 88 | 88,0.6000000000000001,0.0,0.2857142857142857 89 | 89,0.3999999999999999,0.8,0.0 90 | 90,0.8,0.8,0.0 91 | 91,0.6000000000000001,0.6,0.8571428571428571 92 | 92,0.8,0.8,0.7142857142857143 93 | 93,0.3999999999999999,0.6,0.42857142857142855 94 | 94,0.8,0.0,1.0 95 | 95,0.19999999999999996,0.4,0.8571428571428571 96 | 96,0.0,0.0,0.8571428571428571 97 | 97,1.0,0.4,0.0 98 | 98,0.8,0.8,0.5714285714285714 99 | 99,0.6000000000000001,1.0,0.0 100 | 100,0.8,0.6,0.5714285714285714 101 | 101,0.6000000000000001,0.2,1.0 102 | 102,0.6000000000000001,0.4,1.0 103 | 103,0.19999999999999996,0.0,0.8571428571428571 104 | 104,0.6000000000000001,0.2,0.7142857142857143 105 | 105,0.0,0.8,0.8571428571428571 106 | 106,0.6000000000000001,1.0,0.7142857142857143 107 | 107,0.0,1.0,1.0 108 | 108,0.3999999999999999,1.0,0.0 109 | 109,0.3999999999999999,1.0,1.0 110 | 110,0.19999999999999996,1.0,0.7142857142857143 111 | 111,0.3999999999999999,0.6,0.5714285714285714 112 | 112,0.8,0.2,0.2857142857142857 113 | 113,0.6000000000000001,0.0,0.14285714285714285 114 | 114,0.0,1.0,0.42857142857142855 115 | 115,0.6000000000000001,0.8,0.0 116 | 116,0.19999999999999996,0.4,0.7142857142857143 117 | 117,1.0,0.8,1.0 118 | 118,0.6000000000000001,0.8,0.2857142857142857 119 | 119,0.0,0.2,0.7142857142857143 120 | 120,0.0,0.6,0.8571428571428571 121 | 121,0.0,0.0,0.42857142857142855 122 | 122,1.0,0.0,1.0 123 | 123,0.0,0.2,0.5714285714285714 124 | 124,0.6000000000000001,0.4,0.0 125 | 125,0.0,0.4,0.7142857142857143 126 | 126,0.6000000000000001,0.4,0.42857142857142855 127 | 127,1.0,1.0,0.7142857142857143 128 | 128,0.6000000000000001,1.0,0.2857142857142857 129 | 129,0.8,1.0,0.0 130 | 130,1.0,0.4,0.7142857142857143 131 | 131,0.8,0.8,0.42857142857142855 132 | 132,0.3999999999999999,0.4,0.42857142857142855 133 | 133,0.6000000000000001,0.4,0.7142857142857143 134 | 134,0.19999999999999996,0.8,0.0 135 | 135,1.0,1.0,0.8571428571428571 136 | 136,0.3999999999999999,0.6,0.2857142857142857 137 | 137,0.8,0.0,0.5714285714285714 138 | 138,0.3999999999999999,0.8,0.5714285714285714 139 | 139,0.0,0.6,0.5714285714285714 140 | 140,0.19999999999999996,1.0,0.42857142857142855 141 | 141,0.19999999999999996,0.8,0.14285714285714285 142 | 142,0.19999999999999996,0.2,0.42857142857142855 143 | 143,0.8,0.4,1.0 144 | 144,1.0,0.2,0.14285714285714285 145 | 145,0.0,0.8,0.2857142857142857 146 | 146,0.19999999999999996,0.2,0.2857142857142857 147 | 147,0.3999999999999999,0.6,0.0 148 | 148,0.3999999999999999,0.2,0.2857142857142857 149 | 149,0.8,0.8,0.14285714285714285 150 | 150,0.0,0.6,0.14285714285714285 151 | 151,0.6000000000000001,0.0,0.42857142857142855 152 | 152,0.19999999999999996,0.0,1.0 153 | 153,0.0,0.4,0.14285714285714285 154 | 154,0.8,1.0,0.14285714285714285 155 | 155,1.0,0.2,0.7142857142857143 156 | 156,1.0,0.0,0.2857142857142857 157 | 157,1.0,0.8,0.0 158 | 158,0.8,0.8,0.2857142857142857 159 | 159,1.0,0.4,1.0 160 | 160,0.0,0.2,0.0 161 | 161,1.0,0.6,0.0 162 | 162,0.19999999999999996,0.6,0.0 163 | 163,0.6000000000000001,0.8,0.14285714285714285 164 | 164,0.8,0.6,0.7142857142857143 165 | 165,0.3999999999999999,0.8,0.2857142857142857 166 | 166,0.19999999999999996,0.0,0.14285714285714285 167 | 167,0.19999999999999996,0.2,1.0 168 | 168,0.6000000000000001,0.4,0.5714285714285714 169 | 169,1.0,0.6,0.5714285714285714 170 | 170,0.6000000000000001,0.8,0.5714285714285714 171 | 171,0.19999999999999996,0.4,0.42857142857142855 172 | 172,0.8,0.6,0.8571428571428571 173 | 173,0.0,0.0,0.0 174 | 174,0.19999999999999996,0.0,0.5714285714285714 175 | 175,0.3999999999999999,0.6,0.7142857142857143 176 | 176,0.8,0.0,0.2857142857142857 177 | 177,0.0,1.0,0.8571428571428571 178 | 178,0.19999999999999996,0.2,0.0 179 | 179,0.0,0.4,0.0 180 | 180,0.0,1.0,0.2857142857142857 181 | 181,0.3999999999999999,0.8,0.8571428571428571 182 | 182,0.8,1.0,0.2857142857142857 183 | 183,1.0,0.4,0.5714285714285714 184 | 184,0.3999999999999999,0.0,0.5714285714285714 185 | 185,0.6000000000000001,0.8,0.42857142857142855 186 | 186,0.8,0.2,1.0 187 | 187,0.6000000000000001,0.8,0.7142857142857143 188 | 188,0.0,0.2,1.0 189 | 189,0.3999999999999999,0.4,0.8571428571428571 190 | 190,0.6000000000000001,0.6,1.0 191 | 191,0.8,0.0,0.7142857142857143 192 | 192,0.8,0.4,0.0 193 | 193,0.3999999999999999,0.6,1.0 194 | 194,0.3999999999999999,0.8,0.14285714285714285 195 | 195,1.0,0.0,0.5714285714285714 196 | 196,1.0,0.8,0.8571428571428571 197 | 197,0.8,0.0,0.14285714285714285 198 | 198,0.0,0.0,1.0 199 | 199,1.0,1.0,0.42857142857142855 200 | 200,0.3999999999999999,0.4,0.14285714285714285 201 | 201,0.0,0.2,0.8571428571428571 202 | 202,0.6000000000000001,0.2,0.8571428571428571 203 | 203,0.8,0.2,0.8571428571428571 204 | 204,0.0,1.0,0.7142857142857143 205 | 205,0.19999999999999996,1.0,0.5714285714285714 206 | 206,0.0,1.0,0.14285714285714285 207 | 207,0.6000000000000001,0.2,0.5714285714285714 208 | 208,1.0,0.2,0.8571428571428571 209 | 209,0.8,0.6,0.0 210 | 210,0.19999999999999996,0.6,0.2857142857142857 211 | 211,1.0,0.0,0.42857142857142855 212 | 212,0.0,0.8,0.7142857142857143 213 | 213,0.19999999999999996,1.0,1.0 214 | 214,1.0,0.4,0.2857142857142857 215 | 215,0.0,0.8,0.42857142857142855 216 | 216,0.8,0.8,0.8571428571428571 217 | 217,0.6000000000000001,0.0,0.0 218 | 218,0.3999999999999999,1.0,0.5714285714285714 219 | 219,0.8,0.2,0.5714285714285714 220 | 220,0.6000000000000001,0.0,1.0 221 | 221,0.19999999999999996,0.0,0.42857142857142855 222 | 222,1.0,0.0,0.8571428571428571 223 | 223,0.3999999999999999,0.2,0.14285714285714285 224 | 224,0.19999999999999996,0.8,0.7142857142857143 225 | 225,0.3999999999999999,0.0,1.0 226 | 226,0.19999999999999996,0.8,0.42857142857142855 227 | 227,0.3999999999999999,0.2,0.42857142857142855 228 | 228,0.0,0.8,0.5714285714285714 229 | 229,0.3999999999999999,0.0,0.14285714285714285 230 | 230,0.6000000000000001,0.2,0.14285714285714285 231 | 231,0.19999999999999996,0.8,1.0 232 | 232,0.19999999999999996,0.6,0.7142857142857143 233 | 233,0.19999999999999996,0.2,0.5714285714285714 234 | 234,0.3999999999999999,0.6,0.8571428571428571 235 | 235,0.3999999999999999,1.0,0.8571428571428571 236 | 236,0.6000000000000001,0.4,0.8571428571428571 237 | 237,0.6000000000000001,0.0,0.7142857142857143 238 | 238,0.19999999999999996,0.4,0.5714285714285714 239 | 239,0.6000000000000001,0.6,0.7142857142857143 240 | 240,0.3999999999999999,0.2,0.8571428571428571 241 | 241,0.3999999999999999,0.0,0.2857142857142857 242 | 242,0.6000000000000001,0.6,0.0 243 | 243,0.8,0.2,0.0 244 | 244,0.6000000000000001,1.0,0.5714285714285714 245 | 245,0.3999999999999999,0.8,1.0 246 | 246,0.0,1.0,0.0 247 | 247,0.8,0.4,0.5714285714285714 248 | 248,0.8,0.2,0.7142857142857143 249 | 249,0.19999999999999996,1.0,0.2857142857142857 250 | 250,0.8,0.6,0.14285714285714285 251 | 251,0.19999999999999996,0.8,0.8571428571428571 252 | 252,1.0,0.8,0.42857142857142855 253 | 253,1.0,0.2,0.0 254 | 254,1.0,0.0,0.7142857142857143 255 | 255,0.3999999999999999,0.0,0.0 256 | 256,0.0,0.4,0.5714285714285714 257 | 257,0.3999999999999999,0.4,0.0 258 | 258,0.8,1.0,1.0 259 | 259,1.0,1.0,0.0 260 | 260,0.8,0.0,0.8571428571428571 261 | 261,0.0,0.4,0.2857142857142857 262 | 262,0.0,0.2,0.14285714285714285 263 | 263,0.0,0.8,0.14285714285714285 264 | 264,0.3999999999999999,0.2,0.7142857142857143 265 | 265,0.0,0.0,0.7142857142857143 266 | 266,0.6000000000000001,1.0,1.0 267 | 267,0.0,0.4,0.42857142857142855 268 | 268,0.3999999999999999,1.0,0.2857142857142857 269 | 269,0.0,0.6,0.7142857142857143 270 | 270,0.6000000000000001,0.8,1.0 271 | 271,0.3999999999999999,0.4,0.2857142857142857 272 | 272,0.3999999999999999,0.4,0.7142857142857143 273 | 273,0.6000000000000001,0.6,0.14285714285714285 274 | 274,0.3999999999999999,0.6,0.14285714285714285 275 | 275,0.19999999999999996,0.6,0.8571428571428571 276 | 276,0.19999999999999996,0.4,0.0 277 | 277,1.0,0.8,0.5714285714285714 278 | 278,1.0,0.8,0.14285714285714285 279 | 279,0.19999999999999996,0.6,0.5714285714285714 280 | 280,0.3999999999999999,0.0,0.7142857142857143 281 | 281,1.0,0.6,0.14285714285714285 282 | 282,0.3999999999999999,0.2,0.5714285714285714 283 | 283,0.0,0.4,0.8571428571428571 284 | 284,0.0,0.0,0.2857142857142857 285 | 285,1.0,0.6,1.0 286 | 286,0.6000000000000001,0.6,0.42857142857142855 287 | 287,0.6000000000000001,0.2,0.42857142857142855 288 | 288,0.19999999999999996,0.4,0.2857142857142857 289 | -------------------------------------------------------------------------------- /examples/comparison/test_new.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Test for the newest fANOVA version 4 | """ 5 | import numpy as np 6 | import sys 7 | 8 | import fanova 9 | 10 | from collections import OrderedDict 11 | 12 | import ConfigSpace 13 | from ConfigSpace.hyperparameters import UniformFloatHyperparameter 14 | 15 | import os 16 | path = os.path.dirname(os.path.realpath(__file__)) 17 | 18 | my_dict = OrderedDict() 19 | 20 | ''' 21 | Online LDA example 22 | ''' 23 | def online_lda(): 24 | marginals =[] 25 | the_keys = [] 26 | X = np.loadtxt('../example_data/online_lda/online_lda_features.csv', delimiter=",") 27 | Y = np.loadtxt('../example_data/online_lda/online_lda_responses.csv', delimiter=",") 28 | f = fanova.fANOVA(X,Y, n_trees=32,bootstrapping=True) 29 | 30 | res = f.quantify_importance((0, 1, 2)) 31 | 32 | for key in res.keys(): 33 | if key != (0, 1, 2): 34 | marginals.append(res[key]['individual importance']) 35 | the_keys.append(key) 36 | return the_keys, marginals 37 | 38 | ''' 39 | Diabetes example 40 | ''' 41 | def csv_example(): 42 | marginals =[] 43 | the_keys = [] 44 | data = np.loadtxt("../example_data/csv-example/test_data.csv", delimiter=",") 45 | X = np.array(data[:, :2], dtype = np.float) 46 | Y = np.array(data[:,-1:], dtype = np.float).flatten() 47 | # config space 48 | pcs = list(zip(np.min(X,axis=0), np.max(X, axis=0))) 49 | cs = ConfigSpace.ConfigurationSpace() 50 | for i in range(len(pcs)): 51 | cs.add_hyperparameter(UniformFloatHyperparameter("%i" %i, pcs[i][0], pcs[i][1])) 52 | 53 | f2 = fanova.fANOVA(X, Y, cs) 54 | res = f2.quantify_importance((0, 1)) 55 | 56 | for key in res.keys(): 57 | marginals.append(res[key]['individual importance']) 58 | the_keys.append(key) 59 | 60 | return the_keys, marginals 61 | 62 | if __name__ == "__main__": 63 | example = sys.argv[1] 64 | if example == 'online_lda': 65 | res = online_lda() 66 | elif example == 'csv_example': 67 | res = csv_example() 68 | print(res) 69 | -------------------------------------------------------------------------------- /examples/comparison/test_old.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Test for the old fANOVA version 4 | """ 5 | import sys 6 | import numpy as np 7 | 8 | from pyfanova.fanova import Fanova 9 | from pyfanova.fanova_from_csv import FanovaFromCSV 10 | 11 | import os 12 | path = os.path.dirname(os.path.realpath(__file__)) 13 | 14 | ''' 15 | Online LDA example 16 | ''' 17 | def online_lda(): 18 | marginals = [] 19 | f = Fanova(path + '/old_online_lda') 20 | marginals.append(f.get_pairwise_marginal(0,1)) 21 | marginals.append(f.get_pairwise_marginal(1,2)) 22 | marginals.append(f.get_marginal(0)) 23 | marginals.append(f.get_marginal(1)) 24 | marginals.append(f.get_marginal(2)) 25 | marginals.append(f.get_pairwise_marginal(0,2)) 26 | 27 | return marginals 28 | 29 | def csv_example(): 30 | marginals = [] 31 | f = FanovaFromCSV("../example_data/csv-example/test_data.csv") 32 | marginals.append(f.get_pairwise_marginal(0,1)) 33 | marginals.append(f.get_marginal(0)) 34 | marginals.append(f.get_marginal(1)) 35 | 36 | return marginals 37 | 38 | if __name__ == "__main__": 39 | example = sys.argv[1] 40 | if example == 'online_lda': 41 | res = online_lda() 42 | elif example == 'csv_example': 43 | res = csv_example() 44 | print(res) -------------------------------------------------------------------------------- /examples/comparison/test_old_new.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import subprocess 3 | import numpy as np 4 | import ast 5 | 6 | 7 | 8 | def start_comparison(example): 9 | print('Starting the %s example comparison: \n' %example) 10 | p = subprocess.Popen("python3 test_new.py %s" %example, stdout=subprocess.PIPE, shell=True) 11 | new_res = p.communicate()[0] 12 | new_res = ast.literal_eval(new_res) 13 | marginals = np.array(new_res[1])*100 14 | params = np.array(new_res[0]) 15 | 16 | p2 = subprocess.Popen("python test_old.py %s" %example, stdout=subprocess.PIPE, shell=True) 17 | old_res = p2.communicate()[0] 18 | old_res = old_res.split('\n') 19 | old_marginals = np.asarray(old_res) 20 | old_vals = old_marginals[-2] 21 | marg_vals = [float(e) for e in old_vals.strip("[] \n").split(",")] 22 | for i, param in enumerate(params): 23 | diff_val = abs(float(marg_vals[i]) - float(marginals[i])) 24 | print('New fANOVA importance for parameter/s %s is: %f' %(param, marginals[i])) 25 | print('Old fANOVA importance for parameter/s %s is: %f' %(param, marg_vals[i])) 26 | print('The difference in parameter/s %s is: %f ' %(param, diff_val)) 27 | print('') 28 | 29 | start_comparison('online_lda') 30 | start_comparison('csv_example') -------------------------------------------------------------------------------- /examples/example_data/diabetes_responses.csv: -------------------------------------------------------------------------------- 1 | 1.510000000000000000e+02 2 | 7.500000000000000000e+01 3 | 1.410000000000000000e+02 4 | 2.060000000000000000e+02 5 | 1.350000000000000000e+02 6 | 9.700000000000000000e+01 7 | 1.380000000000000000e+02 8 | 6.300000000000000000e+01 9 | 1.100000000000000000e+02 10 | 3.100000000000000000e+02 11 | 1.010000000000000000e+02 12 | 6.900000000000000000e+01 13 | 1.790000000000000000e+02 14 | 1.850000000000000000e+02 15 | 1.180000000000000000e+02 16 | 1.710000000000000000e+02 17 | 1.660000000000000000e+02 18 | 1.440000000000000000e+02 19 | 9.700000000000000000e+01 20 | 1.680000000000000000e+02 21 | 6.800000000000000000e+01 22 | 4.900000000000000000e+01 23 | 6.800000000000000000e+01 24 | 2.450000000000000000e+02 25 | 1.840000000000000000e+02 26 | 2.020000000000000000e+02 27 | 1.370000000000000000e+02 28 | 8.500000000000000000e+01 29 | 1.310000000000000000e+02 30 | 2.830000000000000000e+02 31 | 1.290000000000000000e+02 32 | 5.900000000000000000e+01 33 | 3.410000000000000000e+02 34 | 8.700000000000000000e+01 35 | 6.500000000000000000e+01 36 | 1.020000000000000000e+02 37 | 2.650000000000000000e+02 38 | 2.760000000000000000e+02 39 | 2.520000000000000000e+02 40 | 9.000000000000000000e+01 41 | 1.000000000000000000e+02 42 | 5.500000000000000000e+01 43 | 6.100000000000000000e+01 44 | 9.200000000000000000e+01 45 | 2.590000000000000000e+02 46 | 5.300000000000000000e+01 47 | 1.900000000000000000e+02 48 | 1.420000000000000000e+02 49 | 7.500000000000000000e+01 50 | 1.420000000000000000e+02 51 | 1.550000000000000000e+02 52 | 2.250000000000000000e+02 53 | 5.900000000000000000e+01 54 | 1.040000000000000000e+02 55 | 1.820000000000000000e+02 56 | 1.280000000000000000e+02 57 | 5.200000000000000000e+01 58 | 3.700000000000000000e+01 59 | 1.700000000000000000e+02 60 | 1.700000000000000000e+02 61 | 6.100000000000000000e+01 62 | 1.440000000000000000e+02 63 | 5.200000000000000000e+01 64 | 1.280000000000000000e+02 65 | 7.100000000000000000e+01 66 | 1.630000000000000000e+02 67 | 1.500000000000000000e+02 68 | 9.700000000000000000e+01 69 | 1.600000000000000000e+02 70 | 1.780000000000000000e+02 71 | 4.800000000000000000e+01 72 | 2.700000000000000000e+02 73 | 2.020000000000000000e+02 74 | 1.110000000000000000e+02 75 | 8.500000000000000000e+01 76 | 4.200000000000000000e+01 77 | 1.700000000000000000e+02 78 | 2.000000000000000000e+02 79 | 2.520000000000000000e+02 80 | 1.130000000000000000e+02 81 | 1.430000000000000000e+02 82 | 5.100000000000000000e+01 83 | 5.200000000000000000e+01 84 | 2.100000000000000000e+02 85 | 6.500000000000000000e+01 86 | 1.410000000000000000e+02 87 | 5.500000000000000000e+01 88 | 1.340000000000000000e+02 89 | 4.200000000000000000e+01 90 | 1.110000000000000000e+02 91 | 9.800000000000000000e+01 92 | 1.640000000000000000e+02 93 | 4.800000000000000000e+01 94 | 9.600000000000000000e+01 95 | 9.000000000000000000e+01 96 | 1.620000000000000000e+02 97 | 1.500000000000000000e+02 98 | 2.790000000000000000e+02 99 | 9.200000000000000000e+01 100 | 8.300000000000000000e+01 101 | 1.280000000000000000e+02 102 | 1.020000000000000000e+02 103 | 3.020000000000000000e+02 104 | 1.980000000000000000e+02 105 | 9.500000000000000000e+01 106 | 5.300000000000000000e+01 107 | 1.340000000000000000e+02 108 | 1.440000000000000000e+02 109 | 2.320000000000000000e+02 110 | 8.100000000000000000e+01 111 | 1.040000000000000000e+02 112 | 5.900000000000000000e+01 113 | 2.460000000000000000e+02 114 | 2.970000000000000000e+02 115 | 2.580000000000000000e+02 116 | 2.290000000000000000e+02 117 | 2.750000000000000000e+02 118 | 2.810000000000000000e+02 119 | 1.790000000000000000e+02 120 | 2.000000000000000000e+02 121 | 2.000000000000000000e+02 122 | 1.730000000000000000e+02 123 | 1.800000000000000000e+02 124 | 8.400000000000000000e+01 125 | 1.210000000000000000e+02 126 | 1.610000000000000000e+02 127 | 9.900000000000000000e+01 128 | 1.090000000000000000e+02 129 | 1.150000000000000000e+02 130 | 2.680000000000000000e+02 131 | 2.740000000000000000e+02 132 | 1.580000000000000000e+02 133 | 1.070000000000000000e+02 134 | 8.300000000000000000e+01 135 | 1.030000000000000000e+02 136 | 2.720000000000000000e+02 137 | 8.500000000000000000e+01 138 | 2.800000000000000000e+02 139 | 3.360000000000000000e+02 140 | 2.810000000000000000e+02 141 | 1.180000000000000000e+02 142 | 3.170000000000000000e+02 143 | 2.350000000000000000e+02 144 | 6.000000000000000000e+01 145 | 1.740000000000000000e+02 146 | 2.590000000000000000e+02 147 | 1.780000000000000000e+02 148 | 1.280000000000000000e+02 149 | 9.600000000000000000e+01 150 | 1.260000000000000000e+02 151 | 2.880000000000000000e+02 152 | 8.800000000000000000e+01 153 | 2.920000000000000000e+02 154 | 7.100000000000000000e+01 155 | 1.970000000000000000e+02 156 | 1.860000000000000000e+02 157 | 2.500000000000000000e+01 158 | 8.400000000000000000e+01 159 | 9.600000000000000000e+01 160 | 1.950000000000000000e+02 161 | 5.300000000000000000e+01 162 | 2.170000000000000000e+02 163 | 1.720000000000000000e+02 164 | 1.310000000000000000e+02 165 | 2.140000000000000000e+02 166 | 5.900000000000000000e+01 167 | 7.000000000000000000e+01 168 | 2.200000000000000000e+02 169 | 2.680000000000000000e+02 170 | 1.520000000000000000e+02 171 | 4.700000000000000000e+01 172 | 7.400000000000000000e+01 173 | 2.950000000000000000e+02 174 | 1.010000000000000000e+02 175 | 1.510000000000000000e+02 176 | 1.270000000000000000e+02 177 | 2.370000000000000000e+02 178 | 2.250000000000000000e+02 179 | 8.100000000000000000e+01 180 | 1.510000000000000000e+02 181 | 1.070000000000000000e+02 182 | 6.400000000000000000e+01 183 | 1.380000000000000000e+02 184 | 1.850000000000000000e+02 185 | 2.650000000000000000e+02 186 | 1.010000000000000000e+02 187 | 1.370000000000000000e+02 188 | 1.430000000000000000e+02 189 | 1.410000000000000000e+02 190 | 7.900000000000000000e+01 191 | 2.920000000000000000e+02 192 | 1.780000000000000000e+02 193 | 9.100000000000000000e+01 194 | 1.160000000000000000e+02 195 | 8.600000000000000000e+01 196 | 1.220000000000000000e+02 197 | 7.200000000000000000e+01 198 | 1.290000000000000000e+02 199 | 1.420000000000000000e+02 200 | 9.000000000000000000e+01 201 | 1.580000000000000000e+02 202 | 3.900000000000000000e+01 203 | 1.960000000000000000e+02 204 | 2.220000000000000000e+02 205 | 2.770000000000000000e+02 206 | 9.900000000000000000e+01 207 | 1.960000000000000000e+02 208 | 2.020000000000000000e+02 209 | 1.550000000000000000e+02 210 | 7.700000000000000000e+01 211 | 1.910000000000000000e+02 212 | 7.000000000000000000e+01 213 | 7.300000000000000000e+01 214 | 4.900000000000000000e+01 215 | 6.500000000000000000e+01 216 | 2.630000000000000000e+02 217 | 2.480000000000000000e+02 218 | 2.960000000000000000e+02 219 | 2.140000000000000000e+02 220 | 1.850000000000000000e+02 221 | 7.800000000000000000e+01 222 | 9.300000000000000000e+01 223 | 2.520000000000000000e+02 224 | 1.500000000000000000e+02 225 | 7.700000000000000000e+01 226 | 2.080000000000000000e+02 227 | 7.700000000000000000e+01 228 | 1.080000000000000000e+02 229 | 1.600000000000000000e+02 230 | 5.300000000000000000e+01 231 | 2.200000000000000000e+02 232 | 1.540000000000000000e+02 233 | 2.590000000000000000e+02 234 | 9.000000000000000000e+01 235 | 2.460000000000000000e+02 236 | 1.240000000000000000e+02 237 | 6.700000000000000000e+01 238 | 7.200000000000000000e+01 239 | 2.570000000000000000e+02 240 | 2.620000000000000000e+02 241 | 2.750000000000000000e+02 242 | 1.770000000000000000e+02 243 | 7.100000000000000000e+01 244 | 4.700000000000000000e+01 245 | 1.870000000000000000e+02 246 | 1.250000000000000000e+02 247 | 7.800000000000000000e+01 248 | 5.100000000000000000e+01 249 | 2.580000000000000000e+02 250 | 2.150000000000000000e+02 251 | 3.030000000000000000e+02 252 | 2.430000000000000000e+02 253 | 9.100000000000000000e+01 254 | 1.500000000000000000e+02 255 | 3.100000000000000000e+02 256 | 1.530000000000000000e+02 257 | 3.460000000000000000e+02 258 | 6.300000000000000000e+01 259 | 8.900000000000000000e+01 260 | 5.000000000000000000e+01 261 | 3.900000000000000000e+01 262 | 1.030000000000000000e+02 263 | 3.080000000000000000e+02 264 | 1.160000000000000000e+02 265 | 1.450000000000000000e+02 266 | 7.400000000000000000e+01 267 | 4.500000000000000000e+01 268 | 1.150000000000000000e+02 269 | 2.640000000000000000e+02 270 | 8.700000000000000000e+01 271 | 2.020000000000000000e+02 272 | 1.270000000000000000e+02 273 | 1.820000000000000000e+02 274 | 2.410000000000000000e+02 275 | 6.600000000000000000e+01 276 | 9.400000000000000000e+01 277 | 2.830000000000000000e+02 278 | 6.400000000000000000e+01 279 | 1.020000000000000000e+02 280 | 2.000000000000000000e+02 281 | 2.650000000000000000e+02 282 | 9.400000000000000000e+01 283 | 2.300000000000000000e+02 284 | 1.810000000000000000e+02 285 | 1.560000000000000000e+02 286 | 2.330000000000000000e+02 287 | 6.000000000000000000e+01 288 | 2.190000000000000000e+02 289 | 8.000000000000000000e+01 290 | 6.800000000000000000e+01 291 | 3.320000000000000000e+02 292 | 2.480000000000000000e+02 293 | 8.400000000000000000e+01 294 | 2.000000000000000000e+02 295 | 5.500000000000000000e+01 296 | 8.500000000000000000e+01 297 | 8.900000000000000000e+01 298 | 3.100000000000000000e+01 299 | 1.290000000000000000e+02 300 | 8.300000000000000000e+01 301 | 2.750000000000000000e+02 302 | 6.500000000000000000e+01 303 | 1.980000000000000000e+02 304 | 2.360000000000000000e+02 305 | 2.530000000000000000e+02 306 | 1.240000000000000000e+02 307 | 4.400000000000000000e+01 308 | 1.720000000000000000e+02 309 | 1.140000000000000000e+02 310 | 1.420000000000000000e+02 311 | 1.090000000000000000e+02 312 | 1.800000000000000000e+02 313 | 1.440000000000000000e+02 314 | 1.630000000000000000e+02 315 | 1.470000000000000000e+02 316 | 9.700000000000000000e+01 317 | 2.200000000000000000e+02 318 | 1.900000000000000000e+02 319 | 1.090000000000000000e+02 320 | 1.910000000000000000e+02 321 | 1.220000000000000000e+02 322 | 2.300000000000000000e+02 323 | 2.420000000000000000e+02 324 | 2.480000000000000000e+02 325 | 2.490000000000000000e+02 326 | 1.920000000000000000e+02 327 | 1.310000000000000000e+02 328 | 2.370000000000000000e+02 329 | 7.800000000000000000e+01 330 | 1.350000000000000000e+02 331 | 2.440000000000000000e+02 332 | 1.990000000000000000e+02 333 | 2.700000000000000000e+02 334 | 1.640000000000000000e+02 335 | 7.200000000000000000e+01 336 | 9.600000000000000000e+01 337 | 3.060000000000000000e+02 338 | 9.100000000000000000e+01 339 | 2.140000000000000000e+02 340 | 9.500000000000000000e+01 341 | 2.160000000000000000e+02 342 | 2.630000000000000000e+02 343 | 1.780000000000000000e+02 344 | 1.130000000000000000e+02 345 | 2.000000000000000000e+02 346 | 1.390000000000000000e+02 347 | 1.390000000000000000e+02 348 | 8.800000000000000000e+01 349 | 1.480000000000000000e+02 350 | 8.800000000000000000e+01 351 | 2.430000000000000000e+02 352 | 7.100000000000000000e+01 353 | 7.700000000000000000e+01 354 | 1.090000000000000000e+02 355 | 2.720000000000000000e+02 356 | 6.000000000000000000e+01 357 | 5.400000000000000000e+01 358 | 2.210000000000000000e+02 359 | 9.000000000000000000e+01 360 | 3.110000000000000000e+02 361 | 2.810000000000000000e+02 362 | 1.820000000000000000e+02 363 | 3.210000000000000000e+02 364 | 5.800000000000000000e+01 365 | 2.620000000000000000e+02 366 | 2.060000000000000000e+02 367 | 2.330000000000000000e+02 368 | 2.420000000000000000e+02 369 | 1.230000000000000000e+02 370 | 1.670000000000000000e+02 371 | 6.300000000000000000e+01 372 | 1.970000000000000000e+02 373 | 7.100000000000000000e+01 374 | 1.680000000000000000e+02 375 | 1.400000000000000000e+02 376 | 2.170000000000000000e+02 377 | 1.210000000000000000e+02 378 | 2.350000000000000000e+02 379 | 2.450000000000000000e+02 380 | 4.000000000000000000e+01 381 | 5.200000000000000000e+01 382 | 1.040000000000000000e+02 383 | 1.320000000000000000e+02 384 | 8.800000000000000000e+01 385 | 6.900000000000000000e+01 386 | 2.190000000000000000e+02 387 | 7.200000000000000000e+01 388 | 2.010000000000000000e+02 389 | 1.100000000000000000e+02 390 | 5.100000000000000000e+01 391 | 2.770000000000000000e+02 392 | 6.300000000000000000e+01 393 | 1.180000000000000000e+02 394 | 6.900000000000000000e+01 395 | 2.730000000000000000e+02 396 | 2.580000000000000000e+02 397 | 4.300000000000000000e+01 398 | 1.980000000000000000e+02 399 | 2.420000000000000000e+02 400 | 2.320000000000000000e+02 401 | 1.750000000000000000e+02 402 | 9.300000000000000000e+01 403 | 1.680000000000000000e+02 404 | 2.750000000000000000e+02 405 | 2.930000000000000000e+02 406 | 2.810000000000000000e+02 407 | 7.200000000000000000e+01 408 | 1.400000000000000000e+02 409 | 1.890000000000000000e+02 410 | 1.810000000000000000e+02 411 | 2.090000000000000000e+02 412 | 1.360000000000000000e+02 413 | 2.610000000000000000e+02 414 | 1.130000000000000000e+02 415 | 1.310000000000000000e+02 416 | 1.740000000000000000e+02 417 | 2.570000000000000000e+02 418 | 5.500000000000000000e+01 419 | 8.400000000000000000e+01 420 | 4.200000000000000000e+01 421 | 1.460000000000000000e+02 422 | 2.120000000000000000e+02 423 | 2.330000000000000000e+02 424 | 9.100000000000000000e+01 425 | 1.110000000000000000e+02 426 | 1.520000000000000000e+02 427 | 1.200000000000000000e+02 428 | 6.700000000000000000e+01 429 | 3.100000000000000000e+02 430 | 9.400000000000000000e+01 431 | 1.830000000000000000e+02 432 | 6.600000000000000000e+01 433 | 1.730000000000000000e+02 434 | 7.200000000000000000e+01 435 | 4.900000000000000000e+01 436 | 6.400000000000000000e+01 437 | 4.800000000000000000e+01 438 | 1.780000000000000000e+02 439 | 1.040000000000000000e+02 440 | 1.320000000000000000e+02 441 | 2.200000000000000000e+02 442 | 5.700000000000000000e+01 443 | -------------------------------------------------------------------------------- /examples/example_data/online_lda/Col0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/fanova/46eb005320a0f67fc6b34ed74a91f702c2358c3f/examples/example_data/online_lda/Col0.png -------------------------------------------------------------------------------- /examples/example_data/online_lda/Col1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/fanova/46eb005320a0f67fc6b34ed74a91f702c2358c3f/examples/example_data/online_lda/Col1.png -------------------------------------------------------------------------------- /examples/example_data/online_lda/Col2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/fanova/46eb005320a0f67fc6b34ed74a91f702c2358c3f/examples/example_data/online_lda/Col2.png -------------------------------------------------------------------------------- /examples/example_data/online_lda/online_lda_responses.csv: -------------------------------------------------------------------------------- 1 | 2.014255351000000019e+03 2 | 1.680540179000000080e+03 3 | 1.328191297000000077e+03 4 | 2.859440419999999904e+03 5 | 1.313263742999999977e+03 6 | 1.271869566999999961e+03 7 | 2.630424468000000161e+03 8 | 2.547784743000000162e+03 9 | 1.558076377000000093e+03 10 | 1.278123798999999963e+03 11 | 1.361320422000000008e+03 12 | 1.800429159999999911e+03 13 | 2.084897677999999814e+03 14 | 1.349812010000000100e+03 15 | 2.079802408999999898e+03 16 | 3.624167804000000160e+03 17 | 1.749505296000000044e+03 18 | 1.310060860000000048e+03 19 | 1.343801911999999902e+03 20 | 1.983587762000000112e+03 21 | 2.804748619999999846e+03 22 | 1.427856381000000056e+03 23 | 1.317263697000000093e+03 24 | 5.000332115999999587e+03 25 | 1.637661865999999918e+03 26 | 1.537873628999999937e+03 27 | 2.040531678000000056e+03 28 | 5.022511472999999569e+03 29 | 1.472690781000000015e+03 30 | 1.572944520999999895e+03 31 | 1.528555961000000025e+03 32 | 1.631072353999999905e+03 33 | 1.541968526999999995e+03 34 | 1.468342464000000064e+03 35 | 2.968382912999999917e+03 36 | 1.886883403999999928e+03 37 | 2.101979941000000053e+03 38 | 2.601345619000000170e+03 39 | 1.506415674999999965e+03 40 | 1.271574589999999944e+03 41 | 2.524250164999999924e+03 42 | 2.250711024000000180e+03 43 | 2.566382536999999957e+03 44 | 1.630248430999999982e+03 45 | 1.836733097999999927e+03 46 | 1.266167382000000089e+03 47 | 1.364836274000000003e+03 48 | 1.435221761000000015e+03 49 | 3.053489798999999948e+03 50 | 1.268070557000000008e+03 51 | 1.293198126000000002e+03 52 | 1.718817530999999917e+03 53 | 2.045705842000000075e+03 54 | 1.730222584999999981e+03 55 | 1.369038389000000052e+03 56 | 1.454543818000000101e+03 57 | 1.353926400999999942e+03 58 | 2.240094435999999860e+03 59 | 1.521803360000000112e+03 60 | 1.537363364999999931e+03 61 | 1.612244480000000067e+03 62 | 2.918561303999999836e+03 63 | 1.447394203999999945e+03 64 | 1.295767129999999952e+03 65 | 1.284176858999999922e+03 66 | 1.786323249000000033e+03 67 | 1.590643011999999999e+03 68 | 3.722019016000000192e+03 69 | 2.346008412000000135e+03 70 | 1.419439444000000094e+03 71 | 1.495508178000000044e+03 72 | 1.568119022999999970e+03 73 | 1.309209493999999950e+03 74 | 1.593476294000000053e+03 75 | 1.267214011000000028e+03 76 | 1.711022236000000021e+03 77 | 1.356962017000000060e+03 78 | 2.175249552000000222e+03 79 | 1.706756992000000082e+03 80 | 1.450749101000000110e+03 81 | 1.696944578999999976e+03 82 | 1.413755920999999944e+03 83 | 1.273726730000000089e+03 84 | 1.299255182999999988e+03 85 | 1.382858639999999923e+03 86 | 2.364737106999999924e+03 87 | 1.383792566000000079e+03 88 | 2.271844458999999915e+03 89 | 2.566918453000000227e+03 90 | 1.865777546000000029e+03 91 | 1.296580969999999979e+03 92 | 1.392699949000000061e+03 93 | 1.505616635000000088e+03 94 | 1.287551165999999967e+03 95 | 1.285295192999999927e+03 96 | 1.329484556000000111e+03 97 | 2.369474345000000085e+03 98 | 1.391712438999999904e+03 99 | 1.787887829000000011e+03 100 | 1.358717853999999988e+03 101 | 1.273680399000000079e+03 102 | 1.277089752000000090e+03 103 | 1.317818663999999899e+03 104 | 1.343234780000000001e+03 105 | 1.274208685999999943e+03 106 | 1.420742784000000029e+03 107 | 1.321591239999999971e+03 108 | 2.204569257999999991e+03 109 | 1.524871728999999959e+03 110 | 1.317814554999999928e+03 111 | 1.359437760000000026e+03 112 | 2.006496131000000105e+03 113 | 2.477453344999999899e+03 114 | 1.523244259000000056e+03 115 | 2.077487352000000101e+03 116 | 1.334947038999999904e+03 117 | 1.697459511000000020e+03 118 | 1.504079040000000077e+03 119 | 1.413822885999999926e+03 120 | 1.278813004999999976e+03 121 | 2.227201820000000225e+03 122 | 1.301175617999999986e+03 123 | 1.643681520000000091e+03 124 | 2.560527039999999943e+03 125 | 1.364872251000000006e+03 126 | 1.572391231999999945e+03 127 | 1.738719119000000092e+03 128 | 1.509744746999999961e+03 129 | 1.654893493999999919e+03 130 | 1.335264417000000094e+03 131 | 1.453060437000000093e+03 132 | 1.668761813000000075e+03 133 | 1.309582372000000078e+03 134 | 3.294021456999999828e+03 135 | 1.847090224000000035e+03 136 | 1.784594763999999941e+03 137 | 1.582721013000000085e+03 138 | 1.343241802000000007e+03 139 | 1.457530338999999913e+03 140 | 1.438727853999999979e+03 141 | 2.360980122999999821e+03 142 | 1.943349721000000045e+03 143 | 1.282164334999999937e+03 144 | 2.306935465000000022e+03 145 | 2.088040327000000161e+03 146 | 2.482891469000000143e+03 147 | 2.810881942999999865e+03 148 | 2.259310535999999956e+03 149 | 1.601761692999999923e+03 150 | 3.198956319999999778e+03 151 | 1.917207977999999912e+03 152 | 1.272839987000000065e+03 153 | 3.346584734000000026e+03 154 | 1.513696030000000064e+03 155 | 1.358437431000000061e+03 156 | 2.244860584999999901e+03 157 | 1.855132826000000023e+03 158 | 1.509932841999999937e+03 159 | 1.302230812999999898e+03 160 | 5.149513804999999593e+03 161 | 2.109894998000000214e+03 162 | 3.619232212999999774e+03 163 | 1.713158517999999958e+03 164 | 1.318388431999999966e+03 165 | 1.594416105000000016e+03 166 | 3.126891154999999799e+03 167 | 1.270425850999999966e+03 168 | 1.385504627000000028e+03 169 | 1.388593292000000019e+03 170 | 1.347624790000000075e+03 171 | 1.786518679999999904e+03 172 | 1.318135676000000103e+03 173 | 5.258112825999999586e+03 174 | 1.660049798000000010e+03 175 | 1.303411360999999943e+03 176 | 2.210134257000000161e+03 177 | 1.296897774999999911e+03 178 | 3.747871322999999848e+03 179 | 5.229181160000000091e+03 180 | 1.880399470000000065e+03 181 | 1.314937940000000026e+03 182 | 1.586647560000000112e+03 183 | 1.404940141000000040e+03 184 | 1.619226173999999901e+03 185 | 1.408231244000000061e+03 186 | 1.280796293999999989e+03 187 | 1.330546636999999919e+03 188 | 1.270589627000000064e+03 189 | 1.281538285000000087e+03 190 | 1.320508949000000030e+03 191 | 1.405254245999999966e+03 192 | 2.341462081000000126e+03 193 | 1.282492457000000059e+03 194 | 1.951044824000000062e+03 195 | 1.593512913000000026e+03 196 | 1.588528831999999966e+03 197 | 2.429684276999999838e+03 198 | 1.271753365000000031e+03 199 | 1.579061709999999948e+03 200 | 2.487987431000000015e+03 201 | 1.309276413999999932e+03 202 | 1.289574824999999919e+03 203 | 1.294339615999999978e+03 204 | 1.302023693999999978e+03 205 | 1.351998671000000058e+03 206 | 2.542238813999999820e+03 207 | 1.478045732000000044e+03 208 | 1.304324411000000055e+03 209 | 2.118723891000000094e+03 210 | 2.033429988999999978e+03 211 | 1.913104368999999906e+03 212 | 1.307561279999999897e+03 213 | 1.393614636999999902e+03 214 | 1.750590736000000106e+03 215 | 1.626654117000000042e+03 216 | 1.439558927999999923e+03 217 | 2.548705257999999958e+03 218 | 1.363001015999999936e+03 219 | 1.463333298999999897e+03 220 | 1.281177353000000039e+03 221 | 2.086676124999999956e+03 222 | 1.323445870000000014e+03 223 | 2.711405960999999934e+03 224 | 1.302341384000000062e+03 225 | 1.276738560000000007e+03 226 | 1.501526137000000062e+03 227 | 1.843379711999999927e+03 228 | 1.401072001000000000e+03 229 | 2.729267232000000149e+03 230 | 2.384413805000000139e+03 231 | 1.310790772999999945e+03 232 | 1.309290893000000096e+03 233 | 1.568538127000000031e+03 234 | 1.276161920000000009e+03 235 | 1.397052265999999918e+03 236 | 1.280470260999999937e+03 237 | 1.406231487999999899e+03 238 | 1.473258878999999979e+03 239 | 1.304882655000000113e+03 240 | 1.291760502999999972e+03 241 | 2.422500695000000178e+03 242 | 2.347846308000000136e+03 243 | 2.452668607000000065e+03 244 | 1.389175686999999925e+03 245 | 1.377931876000000102e+03 246 | 4.432511510000000271e+03 247 | 1.381783824000000095e+03 248 | 1.345175995000000057e+03 249 | 1.626743742999999995e+03 250 | 1.789856352000000015e+03 251 | 1.290394504999999981e+03 252 | 1.503488972999999987e+03 253 | 2.562137775000000147e+03 254 | 1.416648457000000008e+03 255 | 2.975404067999999825e+03 256 | 1.549807416999999987e+03 257 | 2.961509739999999965e+03 258 | 1.929981674000000112e+03 259 | 1.693890538999999990e+03 260 | 1.314611327999999958e+03 261 | 2.612878740999999991e+03 262 | 3.481699285000000145e+03 263 | 2.939348265999999967e+03 264 | 1.360104102000000012e+03 265 | 1.460865657999999939e+03 266 | 1.670162786000000096e+03 267 | 1.969256740000000036e+03 268 | 1.505384479000000056e+03 269 | 1.329959505999999919e+03 270 | 1.442360685000000103e+03 271 | 2.035164062999999942e+03 272 | 1.318230943999999909e+03 273 | 1.943102595999999949e+03 274 | 2.250809494000000086e+03 275 | 1.271714717000000064e+03 276 | 3.770110325999999986e+03 277 | 1.417792032000000063e+03 278 | 1.620128844999999956e+03 279 | 1.395590971999999965e+03 280 | 1.414752907999999934e+03 281 | 1.812145882999999912e+03 282 | 1.521322055000000091e+03 283 | 1.292698577999999998e+03 284 | 2.836012122000000090e+03 285 | 1.433882411999999931e+03 286 | 1.454028739999999971e+03 287 | 1.752565517000000000e+03 288 | 2.277597145000000182e+03 289 | -------------------------------------------------------------------------------- /examples/example_data/online_lda/pairwise.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/fanova/46eb005320a0f67fc6b34ed74a91f702c2358c3f/examples/example_data/online_lda/pairwise.png -------------------------------------------------------------------------------- /examples/example_data/online_lda/param-file.txt: -------------------------------------------------------------------------------- 1 | Col0 [0.5, 1.0] [0.75] 2 | Col1 [0.0, 10.0] [5.0] 3 | Col2 [0.0, 14.0] [7.0] 4 | -------------------------------------------------------------------------------- /examples/example_data/test_plots/['0', '1'].png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/fanova/46eb005320a0f67fc6b34ed74a91f702c2358c3f/examples/example_data/test_plots/['0', '1'].png -------------------------------------------------------------------------------- /examples/fanova_example.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from fanova import fANOVA 4 | import fanova.visualizer 5 | 6 | import ConfigSpace 7 | from ConfigSpace.hyperparameters import UniformFloatHyperparameter 8 | 9 | import os 10 | path = os.path.dirname(os.path.realpath(__file__)) 11 | 12 | # directory in which you can find all plots 13 | plot_dir = path + '/example_data/test_plots' 14 | 15 | # artificial dataset (here: features) 16 | features = np.loadtxt(path + '/example_data/diabetes_features.csv', delimiter=",") 17 | responses = np.loadtxt(path + '/example_data/diabetes_responses.csv', delimiter=",") 18 | 19 | # config space 20 | pcs = list(zip(np.min(features,axis=0), np.max(features, axis=0))) 21 | cs = ConfigSpace.ConfigurationSpace() 22 | for i in range(len(pcs)): 23 | cs.add_hyperparameter(UniformFloatHyperparameter("%i" %i, pcs[i][0], pcs[i][1])) 24 | 25 | 26 | # create an instance of fanova with trained forest and ConfigSpace 27 | f = fANOVA(X = features, Y = responses, config_space=cs) 28 | 29 | # marginal of particular parameter: 30 | dims = (1, ) 31 | res = f.quantify_importance(dims) 32 | print(res) 33 | 34 | # getting the 10 most important pairwise marginals sorted by importance 35 | best_margs = f.get_most_important_pairwise_marginals(n=10) 36 | print(best_margs) 37 | 38 | # visualizations: 39 | # first create an instance of the visualizer with fanova object and configspace 40 | vis = fanova.visualizer.Visualizer(f, cs, 'example_output') 41 | # creating the plot of pairwise marginal: 42 | vis.plot_pairwise_marginal((0,2), resolution=20) 43 | # creating all plots in the directory 44 | vis.create_all_plots(plot_dir) 45 | -------------------------------------------------------------------------------- /examples/onlineLDA_example.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from smac.configspace import ConfigurationSpace 3 | from ConfigSpace.hyperparameters import UniformFloatHyperparameter 4 | from fanova import fANOVA 5 | import fanova.visualizer 6 | 7 | import os 8 | path = os.path.dirname(os.path.realpath(__file__)) 9 | 10 | # get sample data from online lda 11 | X = np.loadtxt(path + '/example_data/online_lda/online_lda_features.csv', delimiter=",") 12 | Y = np.loadtxt(path + '/example_data/online_lda/online_lda_responses.csv', delimiter=",") 13 | 14 | # setting up config space: 15 | param_file = path + '/example_data/online_lda/param-file.txt' 16 | f = open(param_file, 'rb') 17 | 18 | cs = ConfigurationSpace() 19 | for row in f: 20 | cs.add_hyperparameter(UniformFloatHyperparameter("%s" %row[0:4].decode('utf-8'), np.float(row[6:9]), np.float(row[10:13]),np.float(row[18:21]))) 21 | param = cs.get_hyperparameters() 22 | 23 | 24 | # create an instance of fanova with data for the random forest and the configSpace 25 | f = fANOVA(X = X, Y = Y, config_space = cs) 26 | 27 | # marginal for first parameter 28 | p_list = (0, ) 29 | res = f.quantify_importance(p_list) 30 | print(res) 31 | 32 | p2_list = ('Col1', 'Col2') 33 | res2 = f.quantify_importance(p2_list) 34 | print(res2) 35 | p2_list = ('Col0', 'Col2') 36 | res2 = f.quantify_importance(p2_list) 37 | print(res2) 38 | p2_list = ('Col1', 'Col0') 39 | res2 = f.quantify_importance(p2_list) 40 | print(res2) 41 | 42 | # getting the most important pairwise marginals sorted by importance 43 | best_p_margs = f.get_most_important_pairwise_marginals(n=3) 44 | print(best_p_margs) 45 | 46 | triplets = f.get_triple_marginals(['Col0','Col1', 'Col2']) 47 | print(triplets) 48 | # visualizations: 49 | 50 | # directory in which you can find all plots 51 | plot_dir = path + '/example_data/online_lda' 52 | # first create an instance of the visualizer with fanova object and configspace 53 | vis = fanova.visualizer.Visualizer(f, cs, plot_dir) 54 | # generating plot data for col0 55 | mean, std, grid = vis.generate_marginal(0) 56 | 57 | # creating all plots in the directory 58 | vis.create_all_plots() 59 | #vis.create_most_important_pairwise_marginal_plots(plot_dir, 3) 60 | -------------------------------------------------------------------------------- /fANOVA demo.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": { 7 | "collapsed": false 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "import sys\n", 12 | "sys.path.append('/home/sfalkner/repositories/github/random_forest_run/build')\n", 13 | "sys.path.append('/home/sfalkner/repositories/github/ConfigSpace')\n", 14 | "sys.path.append('/home/sfalkner/repositories/github/fanova')\n", 15 | "\n", 16 | "import numpy as np\n", 17 | "import matplotlib.pyplot as plt\n", 18 | "\n", 19 | "import ConfigSpace\n", 20 | "import fanova\n", 21 | "import fanova.visualizer as viz" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": null, 27 | "metadata": { 28 | "collapsed": false 29 | }, 30 | "outputs": [], 31 | "source": [ 32 | "# load data\n", 33 | "X_full = np.loadtxt('/home/sfalkner/repositories/github/random_forest_run/test_data_sets/online_lda_features.csv', delimiter=',')\n", 34 | "y_full = np.loadtxt('/home/sfalkner/repositories/github/random_forest_run/test_data_sets/online_lda_responses.csv', delimiter=',')\n", 35 | "\n", 36 | "\n", 37 | "#n_samples = X_full.shape[0]//2\n", 38 | "n_samples = 128\n", 39 | "\n", 40 | "indices = np.random.choice(X_full.shape[0], n_samples)\n", 41 | "\n", 42 | "if n_samples < X_full.shape[0]:\n", 43 | " X=X_full[indices]\n", 44 | " y=y_full[indices]\n", 45 | "else:\n", 46 | " X=X_full\n", 47 | " y=y_full\n", 48 | "\n", 49 | " \n", 50 | "# note that one can specify a ConfigSpace here, but if none is provided, all variables are\n", 51 | "# assumed to be continuous and the range is (min,max)\n", 52 | "f = fanova.fANOVA(X,y, n_trees=32,bootstrapping=True)" 53 | ] 54 | }, 55 | { 56 | "cell_type": "markdown", 57 | "metadata": { 58 | "collapsed": false 59 | }, 60 | "source": [ 61 | "## Reproducing the plots from Frank's fANOVA paper" 62 | ] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "execution_count": null, 67 | "metadata": { 68 | "collapsed": false, 69 | "scrolled": true 70 | }, 71 | "outputs": [], 72 | "source": [ 73 | "for i in range(3):\n", 74 | "\n", 75 | " #compute ground truth\n", 76 | " gt = []\n", 77 | "\n", 78 | " unique_values = list(set(X_full[:,i]))\n", 79 | " unique_values.sort()\n", 80 | "\n", 81 | " for v in unique_values:\n", 82 | " indices = np.where(X_full[:,i] == v)\n", 83 | " gt.append((v,np.mean(y_full[indices]), np.var(y_full[indices])))\n", 84 | "\n", 85 | " gt = np.array(gt)\n", 86 | "\n", 87 | " plt.figure()\n", 88 | " mew = np.linspace( np.min(X[:,i]), np.max(X[:,i]), 100)\n", 89 | " mew2 = np.array([f.marginal_mean_variance_for_values([i],[v]) for v in mew])\n", 90 | "\n", 91 | " m = mew2[:,0]\n", 92 | " s = np.sqrt(mew2[:,1])\n", 93 | "\n", 94 | " plt.plot(mew, m)\n", 95 | " plt.fill_between(mew, m-s, m+s, alpha=.3)\n", 96 | "\n", 97 | " #plt.errorbar(gt[:,0], gt[:,1], yerr=np.sqrt(gt[:,2]))\n", 98 | " plt.scatter(gt[:,0], gt[:,1])\n", 99 | " plt.ylabel('Perplexity')\n", 100 | " plt.xlabel('parameter {}'.format(i))\n", 101 | "\n", 102 | "plt.show()" 103 | ] 104 | }, 105 | { 106 | "cell_type": "code", 107 | "execution_count": null, 108 | "metadata": { 109 | "collapsed": false 110 | }, 111 | "outputs": [], 112 | "source": [ 113 | "v = viz.Visualizer(f, f.cs)" 114 | ] 115 | }, 116 | { 117 | "cell_type": "code", 118 | "execution_count": null, 119 | "metadata": { 120 | "collapsed": false 121 | }, 122 | "outputs": [], 123 | "source": [ 124 | "v.plot_marginal(0)\n", 125 | "v.plot_marginal(1)\n", 126 | "v.plot_marginal(2)\n", 127 | "v.plot_pairwise_marginal([0,2], resolution=50)" 128 | ] 129 | }, 130 | { 131 | "cell_type": "code", 132 | "execution_count": null, 133 | "metadata": { 134 | "collapsed": false 135 | }, 136 | "outputs": [], 137 | "source": [ 138 | "print(f.trees_total_variance)" 139 | ] 140 | }, 141 | { 142 | "cell_type": "code", 143 | "execution_count": null, 144 | "metadata": { 145 | "collapsed": false 146 | }, 147 | "outputs": [], 148 | "source": [ 149 | "v.plot_marginal(2)\n", 150 | "f.set_cutoffs((0,1500))\n", 151 | "print(f.trees_total_variance)\n", 152 | "\n", 153 | "# here is how one can change the labels and stuff\n", 154 | "v.plot_marginal(2, show=False)\n", 155 | "plt.title('cutoffs are now {}'.format(f.cutoffs))\n", 156 | "plt.ylabel('Perplexity')\n", 157 | "plt.show()" 158 | ] 159 | }, 160 | { 161 | "cell_type": "markdown", 162 | "metadata": {}, 163 | "source": [ 164 | "## Quantifying importance\n", 165 | "\n", 166 | "The method quantify_importance now returns a dictionary with all the individual and total importance of all the computed terms up to the specified combination of input dimensions. Below a short example:" 167 | ] 168 | }, 169 | { 170 | "cell_type": "code", 171 | "execution_count": null, 172 | "metadata": { 173 | "collapsed": false, 174 | "scrolled": true 175 | }, 176 | "outputs": [], 177 | "source": [ 178 | "f.set_cutoffs((-np.inf, np.inf)) #just reset the cutoffs\n", 179 | "print(f.trees_total_variance)\n", 180 | "#f.set_cutoffs((0,2000))\n", 181 | "print(f.trees_total_variance)\n", 182 | "\n", 183 | "importance_dict = f.quantify_importance([0,1,2]) " 184 | ] 185 | }, 186 | { 187 | "cell_type": "code", 188 | "execution_count": null, 189 | "metadata": { 190 | "collapsed": false 191 | }, 192 | "outputs": [], 193 | "source": [ 194 | "for k in sorted(list(importance_dict.keys()), key=lambda t: importance_dict[t]['individual importance'], reverse=True):\n", 195 | " print(k, importance_dict[k])" 196 | ] 197 | }, 198 | { 199 | "cell_type": "code", 200 | "execution_count": null, 201 | "metadata": { 202 | "collapsed": false 203 | }, 204 | "outputs": [], 205 | "source": [ 206 | "print(np.sum([importance_dict[v]['individual importance'] for v in importance_dict]))\n", 207 | "print(f.trees_total_variance)\n", 208 | "print(f.V_U_individual)\n", 209 | "for i in range(32):\n", 210 | " print(np.sum([f.V_U_individual[v_u][i] for v_u in f.V_U_individual]), f.trees_total_variance[i])" 211 | ] 212 | }, 213 | { 214 | "cell_type": "markdown", 215 | "metadata": {}, 216 | "source": [ 217 | "## What's still missing\n", 218 | "\n", 219 | "- options like 'improvement over default' or 'improvement over percentile', although the functionality is there by stetting the cutoffs manually\n", 220 | "- documentation for the new API\n", 221 | "- comparison against the Java implementation" 222 | ] 223 | }, 224 | { 225 | "cell_type": "code", 226 | "execution_count": null, 227 | "metadata": { 228 | "collapsed": false 229 | }, 230 | "outputs": [], 231 | "source": [ 232 | "sys.path.append('/home/sfalkner/repositories/github/fanova_old')\n", 233 | "from pyfanova.fanova_from_csv import FanovaFromCSV\n", 234 | "\n", 235 | "data=np.hstack([X_full,y_full[:,None]])\n", 236 | "np.savetxt('/tmp/tmp_data.csv', data, delimiter=',')\n", 237 | "f = FanovaFromCSV(\"/tmp/tmp_data.csv\")\n" 238 | ] 239 | }, 240 | { 241 | "cell_type": "code", 242 | "execution_count": null, 243 | "metadata": { 244 | "collapsed": true 245 | }, 246 | "outputs": [], 247 | "source": [] 248 | } 249 | ], 250 | "metadata": { 251 | "kernelspec": { 252 | "display_name": "Python 3", 253 | "language": "python", 254 | "name": "python3" 255 | }, 256 | "language_info": { 257 | "codemirror_mode": { 258 | "name": "ipython", 259 | "version": 3 260 | }, 261 | "file_extension": ".py", 262 | "mimetype": "text/x-python", 263 | "name": "python", 264 | "nbconvert_exporter": "python", 265 | "pygments_lexer": "ipython3", 266 | "version": "3.4.5" 267 | } 268 | }, 269 | "nbformat": 4, 270 | "nbformat_minor": 1 271 | } 272 | -------------------------------------------------------------------------------- /fanova/__init__.py: -------------------------------------------------------------------------------- 1 | from fanova.fanova import * 2 | -------------------------------------------------------------------------------- /fanova/__version__.py: -------------------------------------------------------------------------------- 1 | __version__ = "2.0.20.dev" 2 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | numpy 2 | pandas 3 | docutils>=0.3 4 | setuptools 5 | matplotlib>=1.4.2 6 | pyrfr>=0.5.0 7 | ConfigSpace 8 | 9 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | with open("fanova/__version__.py") as fh: 4 | version = fh.readlines()[-1].split()[-1].strip("\"'") 5 | 6 | setup( 7 | name = "fanova", 8 | version = version, 9 | packages = find_packages(), 10 | install_requires = [ 11 | 'numpy', 12 | 'pandas', 13 | 'docutils>=0.3', 14 | 'setuptools', 15 | 'matplotlib>=1.4.2', 16 | 'pyrfr>=0.5.0', 17 | 'ConfigSpace'], 18 | 19 | author = "Stefan Falkner and Christina Hernandez Wunsch. Legacy branch: Tobias Domhan, Aaron Klein and Frank Hutter (FANOVA)", 20 | author_email = "sfalkner@cs.uni-freiburg.de", 21 | description = "Functional ANOVA: an implementation of the ICML 2014 paper 'An Efficient Approach for Assessing Hyperparameter Importance' by Frank Hutter, Holger Hoos and Kevin Leyton-Brown.", 22 | include_package_data = True, 23 | keywords = "hyperparameter parameter optimization bayesian smac global variance analysis", 24 | license = "FANOVA is free for academic & non-commercial usage. Please contact Frank Hutter(fh@informatik.uni-freiburg.de) to discuss obtaining a license for commercial purposes.", 25 | url = "http://automl.org/fanova" 26 | ) 27 | -------------------------------------------------------------------------------- /tests/toy_data_set_features.csv: -------------------------------------------------------------------------------- 1 | 0.000000000000000000e+00,0.000000000000000000e+00 2 | 1.000000000000000000e+00,0.000000000000000000e+00 3 | 2.000000000000000000e+00,0.000000000000000000e+00 4 | 3.000000000000000000e+00,0.000000000000000000e+00 5 | 4.000000000000000000e+00,0.000000000000000000e+00 6 | 5.000000000000000000e+00,0.000000000000000000e+00 7 | 6.000000000000000000e+00,0.000000000000000000e+00 8 | 7.000000000000000000e+00,0.000000000000000000e+00 9 | 8.000000000000000000e+00,0.000000000000000000e+00 10 | 9.000000000000000000e+00,0.000000000000000000e+00 11 | 1.000000000000000000e+01,2.000000000000000000e+00 12 | 1.100000000000000000e+01,2.000000000000000000e+00 13 | 1.200000000000000000e+01,2.000000000000000000e+00 14 | 1.300000000000000000e+01,2.000000000000000000e+00 15 | 1.400000000000000000e+01,2.000000000000000000e+00 16 | 1.500000000000000000e+01,2.000000000000000000e+00 17 | 1.600000000000000000e+01,2.000000000000000000e+00 18 | 1.700000000000000000e+01,2.000000000000000000e+00 19 | 1.800000000000000000e+01,2.000000000000000000e+00 20 | 1.900000000000000000e+01,2.000000000000000000e+00 21 | 2.000000000000000000e+01,1.000000000000000000e+00 22 | 2.100000000000000000e+01,1.000000000000000000e+00 23 | 2.200000000000000000e+01,1.000000000000000000e+00 24 | 2.300000000000000000e+01,1.000000000000000000e+00 25 | 2.400000000000000000e+01,1.000000000000000000e+00 26 | 2.500000000000000000e+01,1.000000000000000000e+00 27 | 2.600000000000000000e+01,1.000000000000000000e+00 28 | 2.700000000000000000e+01,1.000000000000000000e+00 29 | 2.800000000000000000e+01,1.000000000000000000e+00 30 | 2.900000000000000000e+01,1.000000000000000000e+00 31 | 3.000000000000000000e+01,1.000000000000000000e+00 32 | 3.100000000000000000e+01,1.000000000000000000e+00 33 | 3.200000000000000000e+01,1.000000000000000000e+00 34 | 3.300000000000000000e+01,1.000000000000000000e+00 35 | 3.400000000000000000e+01,1.000000000000000000e+00 36 | 3.500000000000000000e+01,1.000000000000000000e+00 37 | 3.600000000000000000e+01,1.000000000000000000e+00 38 | 3.700000000000000000e+01,1.000000000000000000e+00 39 | 3.800000000000000000e+01,1.000000000000000000e+00 40 | 3.900000000000000000e+01,1.000000000000000000e+00 41 | 4.000000000000000000e+01,1.000000000000000000e+00 42 | 4.100000000000000000e+01,1.000000000000000000e+00 43 | 4.200000000000000000e+01,1.000000000000000000e+00 44 | 4.300000000000000000e+01,1.000000000000000000e+00 45 | 4.400000000000000000e+01,1.000000000000000000e+00 46 | 4.500000000000000000e+01,1.000000000000000000e+00 47 | 4.600000000000000000e+01,1.000000000000000000e+00 48 | 4.700000000000000000e+01,1.000000000000000000e+00 49 | 4.800000000000000000e+01,1.000000000000000000e+00 50 | 4.900000000000000000e+01,1.000000000000000000e+00 51 | 5.000000000000000000e+01,1.000000000000000000e+00 52 | 5.100000000000000000e+01,1.000000000000000000e+00 53 | 5.200000000000000000e+01,1.000000000000000000e+00 54 | 5.300000000000000000e+01,1.000000000000000000e+00 55 | 5.400000000000000000e+01,1.000000000000000000e+00 56 | 5.500000000000000000e+01,1.000000000000000000e+00 57 | 5.600000000000000000e+01,1.000000000000000000e+00 58 | 5.700000000000000000e+01,1.000000000000000000e+00 59 | 5.800000000000000000e+01,1.000000000000000000e+00 60 | 5.900000000000000000e+01,1.000000000000000000e+00 61 | 6.000000000000000000e+01,0.000000000000000000e+00 62 | 6.100000000000000000e+01,0.000000000000000000e+00 63 | 6.200000000000000000e+01,0.000000000000000000e+00 64 | 6.300000000000000000e+01,0.000000000000000000e+00 65 | 6.400000000000000000e+01,0.000000000000000000e+00 66 | 6.500000000000000000e+01,0.000000000000000000e+00 67 | 6.600000000000000000e+01,0.000000000000000000e+00 68 | 6.700000000000000000e+01,0.000000000000000000e+00 69 | 6.800000000000000000e+01,0.000000000000000000e+00 70 | 6.900000000000000000e+01,0.000000000000000000e+00 71 | 7.000000000000000000e+01,0.000000000000000000e+00 72 | 7.100000000000000000e+01,0.000000000000000000e+00 73 | 7.200000000000000000e+01,0.000000000000000000e+00 74 | 7.300000000000000000e+01,0.000000000000000000e+00 75 | 7.400000000000000000e+01,0.000000000000000000e+00 76 | 7.500000000000000000e+01,0.000000000000000000e+00 77 | 7.600000000000000000e+01,0.000000000000000000e+00 78 | 7.700000000000000000e+01,0.000000000000000000e+00 79 | 7.800000000000000000e+01,0.000000000000000000e+00 80 | 7.900000000000000000e+01,0.000000000000000000e+00 81 | 8.000000000000000000e+01,2.000000000000000000e+00 82 | 8.100000000000000000e+01,2.000000000000000000e+00 83 | 8.200000000000000000e+01,2.000000000000000000e+00 84 | 8.300000000000000000e+01,2.000000000000000000e+00 85 | 8.400000000000000000e+01,2.000000000000000000e+00 86 | 8.500000000000000000e+01,2.000000000000000000e+00 87 | 8.600000000000000000e+01,2.000000000000000000e+00 88 | 8.700000000000000000e+01,2.000000000000000000e+00 89 | 8.800000000000000000e+01,2.000000000000000000e+00 90 | 8.900000000000000000e+01,2.000000000000000000e+00 91 | 9.000000000000000000e+01,2.000000000000000000e+00 92 | 9.100000000000000000e+01,2.000000000000000000e+00 93 | 9.200000000000000000e+01,2.000000000000000000e+00 94 | 9.300000000000000000e+01,2.000000000000000000e+00 95 | 9.400000000000000000e+01,2.000000000000000000e+00 96 | 9.500000000000000000e+01,2.000000000000000000e+00 97 | 9.600000000000000000e+01,2.000000000000000000e+00 98 | 9.700000000000000000e+01,2.000000000000000000e+00 99 | 9.800000000000000000e+01,2.000000000000000000e+00 100 | 9.900000000000000000e+01,2.000000000000000000e+00 101 | -------------------------------------------------------------------------------- /tests/toy_data_set_responses.csv: -------------------------------------------------------------------------------- 1 | 1.000000000000000000e+00 2 | 1.000000000000000000e+00 3 | 1.000000000000000000e+00 4 | 1.000000000000000000e+00 5 | 1.000000000000000000e+00 6 | 1.000000000000000000e+00 7 | 1.000000000000000000e+00 8 | 1.000000000000000000e+00 9 | 1.000000000000000000e+00 10 | 1.000000000000000000e+00 11 | 1.000000000000000000e+00 12 | 1.000000000000000000e+00 13 | 1.000000000000000000e+00 14 | 1.000000000000000000e+00 15 | 1.000000000000000000e+00 16 | 1.000000000000000000e+00 17 | 1.000000000000000000e+00 18 | 1.000000000000000000e+00 19 | 1.000000000000000000e+00 20 | 1.000000000000000000e+00 21 | 2.000000000000000000e+00 22 | 2.000000000000000000e+00 23 | 2.000000000000000000e+00 24 | 2.000000000000000000e+00 25 | 2.000000000000000000e+00 26 | 2.000000000000000000e+00 27 | 2.000000000000000000e+00 28 | 2.000000000000000000e+00 29 | 2.000000000000000000e+00 30 | 2.000000000000000000e+00 31 | 2.000000000000000000e+00 32 | 2.000000000000000000e+00 33 | 2.000000000000000000e+00 34 | 2.000000000000000000e+00 35 | 2.000000000000000000e+00 36 | 2.000000000000000000e+00 37 | 2.000000000000000000e+00 38 | 2.000000000000000000e+00 39 | 2.000000000000000000e+00 40 | 2.000000000000000000e+00 41 | 2.000000000000000000e+00 42 | 2.000000000000000000e+00 43 | 2.000000000000000000e+00 44 | 2.000000000000000000e+00 45 | 2.000000000000000000e+00 46 | 2.000000000000000000e+00 47 | 2.000000000000000000e+00 48 | 2.000000000000000000e+00 49 | 2.000000000000000000e+00 50 | 2.000000000000000000e+00 51 | 2.000000000000000000e+00 52 | 2.000000000000000000e+00 53 | 2.000000000000000000e+00 54 | 2.000000000000000000e+00 55 | 2.000000000000000000e+00 56 | 2.000000000000000000e+00 57 | 2.000000000000000000e+00 58 | 2.000000000000000000e+00 59 | 2.000000000000000000e+00 60 | 2.000000000000000000e+00 61 | 3.000000000000000000e+00 62 | 3.000000000000000000e+00 63 | 3.000000000000000000e+00 64 | 3.000000000000000000e+00 65 | 3.000000000000000000e+00 66 | 3.000000000000000000e+00 67 | 3.000000000000000000e+00 68 | 3.000000000000000000e+00 69 | 3.000000000000000000e+00 70 | 3.000000000000000000e+00 71 | 3.000000000000000000e+00 72 | 3.000000000000000000e+00 73 | 3.000000000000000000e+00 74 | 3.000000000000000000e+00 75 | 3.000000000000000000e+00 76 | 3.000000000000000000e+00 77 | 3.000000000000000000e+00 78 | 3.000000000000000000e+00 79 | 3.000000000000000000e+00 80 | 3.000000000000000000e+00 81 | 4.000000000000000000e+00 82 | 4.000000000000000000e+00 83 | 4.000000000000000000e+00 84 | 4.000000000000000000e+00 85 | 4.000000000000000000e+00 86 | 4.000000000000000000e+00 87 | 4.000000000000000000e+00 88 | 4.000000000000000000e+00 89 | 4.000000000000000000e+00 90 | 4.000000000000000000e+00 91 | 4.000000000000000000e+00 92 | 4.000000000000000000e+00 93 | 4.000000000000000000e+00 94 | 4.000000000000000000e+00 95 | 4.000000000000000000e+00 96 | 4.000000000000000000e+00 97 | 4.000000000000000000e+00 98 | 4.000000000000000000e+00 99 | 4.000000000000000000e+00 100 | 4.000000000000000000e+00 101 | -------------------------------------------------------------------------------- /tests/unit_tests.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | sys.path.append("/home/sfalkner/repositories/github/random_forest_run/build") 4 | sys.path.append("/home/sfalkner/repositories/github/fanova") 5 | sys.path.append("/home/sfalkner/repositories/github/ConfigSpace") 6 | 7 | import os 8 | import pickle 9 | import tempfile 10 | import unittest 11 | 12 | 13 | import numpy as np 14 | import ConfigSpace as cfs 15 | import fanova 16 | 17 | 18 | class TestfANOVAtoyData(unittest.TestCase): 19 | 20 | def setUp(self): 21 | 22 | self.X = np.loadtxt('toy_data_set_features.csv', delimiter=',') 23 | self.y = np.loadtxt('toy_data_set_responses.csv', delimiter=',') 24 | 25 | self.cfs = cfs.ConfigurationSpace() 26 | 27 | 28 | f1 = cfs.UniformFloatHyperparameter('x1', 0,100 ) 29 | f2 = cfs.CategoricalHyperparameter('x2', [0,1,2]) 30 | 31 | self.cfs.add_hyperparameter(f1) 32 | self.cfs.add_hyperparameter(f2) 33 | 34 | 35 | def tearDown(self): 36 | self.X = None 37 | self.y = None 38 | 39 | def test_with_toy_data(self): 40 | 41 | f = fanova.fANOVA(self.X,self.y,self.cfs, bootstrapping=False, n_trees=1, seed=5, max_features=1) 42 | 43 | f.the_forest.save_latex_representation('/tmp/fanova_') 44 | print("="*80) 45 | print(f.the_forest.all_split_values()) 46 | print("total variances", f.the_forest.get_trees_total_variances()) 47 | print(f.quantify_importance([0,1])) 48 | print(f.trees_total_variance) 49 | 50 | print(f.V_U) 51 | 52 | 53 | if __name__ == '__main__': 54 | unittest.main() 55 | --------------------------------------------------------------------------------