├── .gitignore ├── .landscape.yml ├── .travis.yml ├── README.md ├── circle.yml ├── docs ├── .buildinfo ├── .doctrees │ ├── apis.doctree │ ├── environment.pickle │ ├── index.doctree │ ├── installation.doctree │ └── quickstart.doctree ├── .nojekyll ├── _sources │ ├── apis.rst.txt │ ├── apis.txt │ ├── index.rst.txt │ ├── index.txt │ ├── installation.rst.txt │ ├── installation.txt │ ├── quickstart.rst.txt │ └── quickstart.txt ├── _static │ ├── ajax-loader.gif │ ├── alabaster.css │ ├── basic.css │ ├── bootstrap-2.3.2 │ │ ├── css │ │ │ ├── bootstrap-responsive.css │ │ │ ├── bootstrap-responsive.min.css │ │ │ ├── bootstrap.css │ │ │ └── bootstrap.min.css │ │ ├── img │ │ │ ├── glyphicons-halflings-white.png │ │ │ └── glyphicons-halflings.png │ │ └── js │ │ │ ├── bootstrap.js │ │ │ └── bootstrap.min.js │ ├── bootstrap-3.3.6 │ │ ├── css │ │ │ ├── bootstrap-theme.css │ │ │ ├── bootstrap-theme.css.map │ │ │ ├── bootstrap-theme.min.css │ │ │ ├── bootstrap-theme.min.css.map │ │ │ ├── bootstrap.css │ │ │ ├── bootstrap.css.map │ │ │ ├── bootstrap.min.css │ │ │ └── bootstrap.min.css.map │ │ ├── fonts │ │ │ ├── glyphicons-halflings-regular.eot │ │ │ ├── glyphicons-halflings-regular.svg │ │ │ ├── glyphicons-halflings-regular.ttf │ │ │ ├── glyphicons-halflings-regular.woff │ │ │ └── glyphicons-halflings-regular.woff2 │ │ └── js │ │ │ ├── bootstrap.js │ │ │ ├── bootstrap.min.js │ │ │ └── npm.js │ ├── bootstrap-sphinx.css │ ├── bootstrap-sphinx.js │ ├── bootswatch-2.3.2 │ │ ├── amelia │ │ │ └── bootstrap.min.css │ │ ├── cerulean │ │ │ └── bootstrap.min.css │ │ ├── cosmo │ │ │ └── bootstrap.min.css │ │ ├── cyborg │ │ │ └── bootstrap.min.css │ │ ├── flatly │ │ │ └── bootstrap.min.css │ │ ├── img │ │ │ ├── glyphicons-halflings-white.png │ │ │ └── glyphicons-halflings.png │ │ ├── journal │ │ │ └── bootstrap.min.css │ │ ├── readable │ │ │ └── bootstrap.min.css │ │ ├── simplex │ │ │ └── bootstrap.min.css │ │ ├── slate │ │ │ └── bootstrap.min.css │ │ ├── spacelab │ │ │ └── bootstrap.min.css │ │ ├── spruce │ │ │ └── bootstrap.min.css │ │ ├── superhero │ │ │ └── bootstrap.min.css │ │ └── united │ │ │ └── bootstrap.min.css │ ├── bootswatch-3.3.6 │ │ ├── cerulean │ │ │ └── bootstrap.min.css │ │ ├── cosmo │ │ │ └── bootstrap.min.css │ │ ├── custom │ │ │ └── bootstrap.min.css │ │ ├── cyborg │ │ │ └── bootstrap.min.css │ │ ├── darkly │ │ │ └── bootstrap.min.css │ │ ├── flatly │ │ │ └── bootstrap.min.css │ │ ├── fonts │ │ │ ├── glyphicons-halflings-regular.eot │ │ │ ├── glyphicons-halflings-regular.svg │ │ │ ├── glyphicons-halflings-regular.ttf │ │ │ ├── glyphicons-halflings-regular.woff │ │ │ └── glyphicons-halflings-regular.woff2 │ │ ├── journal │ │ │ └── bootstrap.min.css │ │ ├── lumen │ │ │ └── bootstrap.min.css │ │ ├── paper │ │ │ └── bootstrap.min.css │ │ ├── readable │ │ │ └── bootstrap.min.css │ │ ├── sandstone │ │ │ └── bootstrap.min.css │ │ ├── simplex │ │ │ └── bootstrap.min.css │ │ ├── slate │ │ │ └── bootstrap.min.css │ │ ├── spacelab │ │ │ └── bootstrap.min.css │ │ ├── superhero │ │ │ └── bootstrap.min.css │ │ ├── united │ │ │ └── bootstrap.min.css │ │ └── yeti │ │ │ └── bootstrap.min.css │ ├── comment-bright.png │ ├── comment-close.png │ ├── comment.png │ ├── custom.css │ ├── doctools.js │ ├── down-pressed.png │ ├── down.png │ ├── file.png │ ├── jquery-3.1.0.js │ ├── jquery.js │ ├── js │ │ ├── jquery-1.11.0.min.js │ │ └── jquery-fix.js │ ├── minus.png │ ├── plus.png │ ├── pygments.css │ ├── searchtools.js │ ├── underscore-1.3.1.js │ ├── underscore.js │ ├── up-pressed.png │ ├── up.png │ └── websupport.js ├── apis.html ├── genindex.html ├── index.html ├── installation.html ├── objects.inv ├── py-modindex.html ├── quickstart.html ├── search.html ├── searchindex.js └── source │ ├── apis.rst │ ├── conf.py │ ├── index.rst │ ├── installation.rst │ └── quickstart.rst ├── examples ├── bo_on_branin.py ├── branin.py └── keras_mnist_mlp.py ├── labwatch ├── __about__.py ├── __init__.py ├── assistant.py ├── converters │ ├── __init__.py │ └── convert_to_configspace.py ├── hyperparameters.py ├── optimizers │ ├── __init__.py │ ├── base.py │ ├── bayesian_optimization.py │ ├── bohamiann.py │ ├── dngo.py │ ├── random_search.py │ └── smac_wrapper.py ├── searchspace.py └── utils │ ├── __init__.py │ ├── fixed_dict.py │ ├── hashing.py │ ├── types.py │ └── version_checks.py ├── setup.py └── tests ├── test_conversion.py └── test_searchspace.py /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Created by https://www.gitignore.io/api/pycharm,python,linux 3 | 4 | ### PyCharm ### 5 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm 6 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 7 | 8 | # User-specific stuff: 9 | .idea/ 10 | 11 | ## File-based project format: 12 | *.iws 13 | 14 | ## Plugin-specific files: 15 | 16 | # IntelliJ 17 | /out/ 18 | 19 | # mpeltonen/sbt-idea plugin 20 | .idea_modules/ 21 | 22 | # JIRA plugin 23 | atlassian-ide-plugin.xml 24 | 25 | # Crashlytics plugin (for Android Studio and IntelliJ) 26 | com_crashlytics_export_strings.xml 27 | crashlytics.properties 28 | crashlytics-build.properties 29 | fabric.properties 30 | 31 | 32 | ### Python ### 33 | # Byte-compiled / optimized / DLL files 34 | __pycache__/ 35 | *.py[cod] 36 | *$py.class 37 | 38 | # C extensions 39 | *.so 40 | 41 | # Distribution / packaging 42 | .Python 43 | env/ 44 | build/ 45 | develop-eggs/ 46 | dist/ 47 | downloads/ 48 | eggs/ 49 | .eggs/ 50 | lib/ 51 | lib64/ 52 | parts/ 53 | sdist/ 54 | var/ 55 | *.egg-info/ 56 | .installed.cfg 57 | *.egg 58 | 59 | # PyInstaller 60 | # Usually these files are written by a python script from a template 61 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 62 | *.manifest 63 | *.spec 64 | 65 | # Installer logs 66 | pip-log.txt 67 | pip-delete-this-directory.txt 68 | 69 | # Unit test / coverage reports 70 | htmlcov/ 71 | .tox/ 72 | .coverage 73 | .coverage.* 74 | .cache 75 | nosetests.xml 76 | coverage.xml 77 | *,cover 78 | .hypothesis/ 79 | 80 | # Translations 81 | *.mo 82 | *.pot 83 | 84 | # Django stuff: 85 | *.log 86 | local_settings.py 87 | 88 | # Flask instance folder 89 | instance/ 90 | 91 | # Sphinx documentation 92 | docs/_build/ 93 | 94 | # PyBuilder 95 | target/ 96 | 97 | # IPython Notebook 98 | .ipynb_checkpoints 99 | 100 | # pyenv 101 | .python-version 102 | 103 | # dotenv 104 | .env 105 | 106 | 107 | ### Linux ### 108 | *~ 109 | 110 | # temporary files which can be created if a process still has a handle open of a deleted file 111 | .fuse_hidden* 112 | 113 | # KDE directory preferences 114 | .directory 115 | 116 | # Linux trash folder which might appear on any partition or disk 117 | .Trash-* 118 | 119 | -------------------------------------------------------------------------------- /.landscape.yml: -------------------------------------------------------------------------------- 1 | pep8: 2 | full: true 3 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "3.4" 4 | - "3.5" 5 | - "3.6" 6 | os: 7 | - linux 8 | before_install: 9 | - sudo apt-get update 10 | - pip install --upgrade pip 11 | - pip install coverage 12 | - pip install -U pytest 13 | - sudo apt-get install python-dev gcc 14 | 15 | install: 16 | - python setup.py install 17 | script: coverage run --source=labwatch -m unittest discover tests -v 18 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # labwatch 2 | [![Build Status](https://travis-ci.org/automl/labwatch.svg?branch=master)](https://travis-ci.org/automl/labwatch) 3 | [![Code Health](https://landscape.io/github/automl/labwatch/master/landscape.svg?style=flat)](https://landscape.io/github/automl/labwatch/master) 4 |
5 | An extension to Sacred for automated hyperparameter optimization. 6 | 7 | # Documentation 8 | You can find the documentation for labwatch [here](https://automl.github.io/labwatch/) 9 | -------------------------------------------------------------------------------- /circle.yml: -------------------------------------------------------------------------------- 1 | machine: 2 | python: 3 | version: 3.5.3 4 | environment: 5 | # The github organization or username of the repository which hosts the 6 | # project and documentation. 7 | USERNAME: "numairmansur" 8 | 9 | # The repository where the documentation will be hosted 10 | DOC_REPO: "labwatch" 11 | 12 | # The base URL for the Github page where the documentation will be hosted 13 | DOC_URL: "" 14 | 15 | # The email is to be used for commits in the Github Page 16 | EMAIL: "numair.mansur@gmail.com" 17 | 18 | dependencies: 19 | 20 | # Various dependencies 21 | pre: 22 | - sudo apt-get update 23 | - sudo apt-get install python-dev 24 | - pip install Sphinx 25 | - pip install sphinx_bootstrap_theme 26 | - pip install . 27 | override: 28 | - echo "Sphinx installed" 29 | test: 30 | 31 | override: 32 | - echo "test" 33 | 34 | deployment: 35 | production: 36 | branch: master 37 | commands: 38 | # Move this to a sh script if it gets too big. 39 | - echo "Running SPHINX build" 40 | - sphinx-build -b html docs/source/ docs 41 | - git add * 42 | - git commit -am "Git push detected. Documentation on github pages automatically updated. [skip ci]" 43 | - git push 44 | -------------------------------------------------------------------------------- /docs/.buildinfo: -------------------------------------------------------------------------------- 1 | # Sphinx build info version 1 2 | # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done. 3 | config: 2bb26c18a04cd41390b394ff43a3dbeb 4 | tags: 645f666f9bcd5a90fca523b33c5a78b7 5 | -------------------------------------------------------------------------------- /docs/.doctrees/apis.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/labwatch/cdd7c55e5741bf4f1216f7b8da8de9fb1480c75b/docs/.doctrees/apis.doctree -------------------------------------------------------------------------------- /docs/.doctrees/environment.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/labwatch/cdd7c55e5741bf4f1216f7b8da8de9fb1480c75b/docs/.doctrees/environment.pickle -------------------------------------------------------------------------------- /docs/.doctrees/index.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/labwatch/cdd7c55e5741bf4f1216f7b8da8de9fb1480c75b/docs/.doctrees/index.doctree -------------------------------------------------------------------------------- /docs/.doctrees/installation.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/labwatch/cdd7c55e5741bf4f1216f7b8da8de9fb1480c75b/docs/.doctrees/installation.doctree -------------------------------------------------------------------------------- /docs/.doctrees/quickstart.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/labwatch/cdd7c55e5741bf4f1216f7b8da8de9fb1480c75b/docs/.doctrees/quickstart.doctree -------------------------------------------------------------------------------- /docs/.nojekyll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/labwatch/cdd7c55e5741bf4f1216f7b8da8de9fb1480c75b/docs/.nojekyll -------------------------------------------------------------------------------- /docs/_sources/apis.rst.txt: -------------------------------------------------------------------------------- 1 | API 2 | === 3 | 4 | 5 | ------------ 6 | Main Modules 7 | ------------ 8 | 9 | Assistant: 10 | ---------- 11 | .. autoclass:: labwatch.assistant.LabAssistant 12 | :members: 13 | 14 | Hyperparameters: 15 | ---------------- 16 | .. automodule:: labwatch.hyperparameters 17 | :members: 18 | 19 | .. autoclass:: labwatch.hyperparameters.Parameter 20 | :members: 21 | 22 | .. autoclass:: labwatch.hyperparameters.Constant 23 | :members: 24 | 25 | .. autoclass:: labwatch.hyperparameters.Categorical 26 | :members: 27 | 28 | .. autoclass:: labwatch.hyperparameters.UniformNumber 29 | :members: 30 | 31 | .. autoclass:: labwatch.hyperparameters.UniformFloat 32 | :members: 33 | 34 | .. autoclass:: labwatch.hyperparameters.UniformInt 35 | :members: 36 | 37 | .. autoclass:: labwatch.hyperparameters.Gaussian 38 | :members: 39 | 40 | .. autoclass:: labwatch.hyperparameters.ConditionResult 41 | :members: 42 | 43 | .. autoclass:: labwatch.hyperparameters.Condition 44 | :members: 45 | 46 | Converters: 47 | ----------- 48 | .. automodule:: labwatch.converters.convert_to_configspace 49 | :members: 50 | 51 | Optimizers: 52 | ----------- 53 | 54 | 55 | .. autoclass:: labwatch.optimizers.base.Optimizer 56 | :members: 57 | 58 | .. autoclass:: labwatch.optimizers.bayesian_optimization.BayesianOptimization 59 | :members: 60 | 61 | .. autoclass:: labwatch.optimizers.bohamiann.Bohamiann 62 | :members: 63 | 64 | .. autoclass:: labwatch.optimizers.dngo.DNGOWrapper 65 | :members: 66 | 67 | .. autoclass:: labwatch.optimizers.random_search.RandomSearch 68 | :members: 69 | -------------------------------------------------------------------------------- /docs/_sources/apis.txt: -------------------------------------------------------------------------------- 1 | API 2 | === 3 | -------------------------------------------------------------------------------- /docs/_sources/index.rst.txt: -------------------------------------------------------------------------------- 1 | .. labwatch documentation master file, created by 2 | sphinx-quickstart on Thu May 25 18:25:22 2017. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | ==================================== 6 | Welcome to labwatch's documentation! 7 | ==================================== 8 | 9 | Labwatch is an extension to `Sacred `_ for automated hyperparameter optimization 10 | of machine learning algorithm. It implements an interface to state-of-the-art hyperparameter optimization methods 11 | such as `RoBO `_ or `SMAC `_. 12 | 13 | 14 | 15 | Contents 16 | -------- 17 | 18 | .. toctree:: 19 | :maxdepth: 3 20 | 21 | installation 22 | quickstart 23 | apis 24 | -------------------------------------------------------------------------------- /docs/_sources/index.txt: -------------------------------------------------------------------------------- 1 | .. labwatch documentation master file, created by 2 | sphinx-quickstart on Thu May 25 18:25:22 2017. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | ==================================== 6 | Welcome to labwatch's documentation! 7 | ==================================== 8 | 9 | Labwatch is an extension to `Sacred `_ for automated hyperparameter optimization 10 | of machine learning algorithm. It implements an interface to state-of-the-art hyperparameter optimization methods 11 | such as `RoBO `_ or `SMAC `_. 12 | 13 | 14 | 15 | Contents 16 | -------- 17 | 18 | .. toctree:: 19 | :maxdepth: 2 20 | 21 | installation 22 | quickstart 23 | apis 24 | -------------------------------------------------------------------------------- /docs/_sources/installation.rst.txt: -------------------------------------------------------------------------------- 1 | Installation 2 | ============ 3 | 4 | 5 | To install the current version of labwatch from git execute the following commands: 6 | 7 | .. code:: bash 8 | 9 | git clone https://github.com/automl/labwatch.git 10 | cd labwatch 11 | pip install . 12 | 13 | 14 | Before you can use Bayesian optimization, you need to install `RoBO `_ first. 15 | Have a look `here `_ to see how you can install it. 16 | 17 | If you want to use `SMAC `_, you have to install it first from pypi: 18 | 19 | .. code:: bash 20 | 21 | pip install smac 22 | 23 | In case you want to install manually, look `here `_ 24 | 25 | -------------------------------------------------------------------------------- /docs/_sources/installation.txt: -------------------------------------------------------------------------------- 1 | Installation 2 | ============ 3 | 4 | 5 | To install the current version of labwatch from git execute the following commands: 6 | 7 | .. code:: bash 8 | 9 | git clone https://github.com/automl/labwatch.git 10 | cd labwatch 11 | pip install . 12 | 13 | 14 | Before you can use Bayesian optimization, you need to install `RoBO `_ first. 15 | Have a look `here `_ to see how you can install it. 16 | 17 | If you want to use `SMAC `_, you have to install it first from pypi: 18 | 19 | .. code:: bash 20 | 21 | pip install smac 22 | 23 | In case you want to install manually, look `here `_ 24 | 25 | -------------------------------------------------------------------------------- /docs/_sources/quickstart.rst.txt: -------------------------------------------------------------------------------- 1 | Quickstart 2 | ********** 3 | 4 | The following tutorial assumes that you have a basic understanding of how Sacred works (Experiments, Observers). 5 | In case you are unsure, have a look on the `Sacred Quickstart Guide `_ 6 | 7 | In this tutorial we will see how we can optimize the hyperparameters of a feed forward network trained on MNIST 8 | together with Labwatch and Sacred. 9 | We will use the MNIST example of `keras `_ to implement the neural network but note that 10 | both Labwatch and Sacred are completely independent of which framework you use. 11 | You can find the whole source code as well as more examples 12 | `here `_ 13 | 14 | Automatically Tuning of Hyperparameters 15 | ======================================= 16 | 17 | 18 | Sacred is a useful tool to keep track of all relevant information of your experiments such as hyperparameters, 19 | results, dependencies and so on. 20 | 21 | The following python code adapts the 22 | `keras mnist example `_ to work with Sacred: 23 | 24 | 25 | .. code:: python 26 | 27 | import keras 28 | from keras.datasets import mnist 29 | from keras.models import Sequential 30 | from keras.layers import Dense, Dropout 31 | from keras.optimizers import RMSprop 32 | 33 | from sacred import Experiment 34 | 35 | ex = Experiment() 36 | 37 | @ex.config 38 | def cfg(): 39 | batch_size = 128 40 | num_units_first_layer = 512 41 | num_units_second_layer = 512 42 | dropout_first_layer = 0.2 43 | dropout_second_layer = 0.2 44 | learning_rate = 0.001 45 | 46 | 47 | @ex.automain 48 | def run(batch_size, 49 | num_units_first_layer, 50 | num_units_second_layer, 51 | dropout_first_layer, 52 | dropout_second_layer, 53 | learning_rate): 54 | 55 | num_classes = 10 56 | epochs = 20 57 | 58 | # the data, shuffled and split between train and test sets 59 | (x_train, y_train), (x_test, y_test) = mnist.load_data() 60 | 61 | x_train = x_train.reshape(60000, 784) 62 | x_test = x_test.reshape(10000, 784) 63 | x_train = x_train.astype('float32') 64 | x_test = x_test.astype('float32') 65 | x_train /= 255 66 | x_test /= 255 67 | print(x_train.shape[0], 'train samples') 68 | print(x_test.shape[0], 'test samples') 69 | 70 | # convert class vectors to binary class matrices 71 | y_train = keras.utils.to_categorical(y_train, num_classes) 72 | y_test = keras.utils.to_categorical(y_test, num_classes) 73 | 74 | model = Sequential() 75 | model.add(Dense(num_units_first_layer, activation='relu', input_shape=(784,))) 76 | model.add(Dropout(dropout_first_layer)) 77 | model.add(Dense(num_units_second_layer, activation='relu')) 78 | model.add(Dropout(dropout_second_layer)) 79 | model.add(Dense(10, activation='softmax')) 80 | 81 | model.summary() 82 | 83 | model.compile(loss='categorical_crossentropy', 84 | optimizer=RMSprop(lr=learning_rate), 85 | metrics=['accuracy']) 86 | 87 | history = model.fit(x_train, y_train, 88 | batch_size=batch_size, 89 | epochs=epochs, 90 | verbose=1, 91 | validation_data=(x_test, y_test)) 92 | score = model.evaluate(x_test, y_test, verbose=0) 93 | print('Test loss:', score[0]) 94 | print('Test accuracy:', score[1]) 95 | 96 | results = dict() 97 | results["optimization_target"] = 1 - score[1] 98 | 99 | return results 100 | 101 | In the dark old days you would probably now spend a lot of time to find the right setting for you hyperparameters by 102 | iteratively trying out different setting. 103 | 104 | Well, you're are not alone with this problem and it is actually a common problem in machine learning. 105 | Recently a new subfield in machine learning (`AutoML `_) 106 | has emerged that tries to automated this procedure by casting it as an optimization problem. By now there exist several 107 | optimization methods that tackle the hyperparameter optimization problem. 108 | 109 | To make use of these methods in Labwatch, we fist have to instantiate a LabAssistant which will 110 | connect our Sacred experiment with the hyperparameter optimizer through a MongoDB: 111 | 112 | .. code:: python 113 | 114 | from labwatch.assistant import LabAssistant 115 | from labwatch.optimizers.random_search import RandomSearch 116 | 117 | a = LabAssistant(ex, "labwatch_demo_keras", optimizer=RandomSearch) 118 | 119 | After that we have to define our configuration search space with the hyperparameters that we want to optimize. 120 | 121 | .. code:: python 122 | 123 | @a.searchspace 124 | def search_space(): 125 | batch_size = UniformNumber(lower=8, 126 | upper=64, 127 | default=32, 128 | type=int, 129 | log_scale=True) 130 | num_units_first_layer = UniformNumber(lower=16, 131 | upper=1024, 132 | default=32, 133 | type=int, 134 | log_scale=True) 135 | num_units_second_layer = UniformNumber(lower=16, 136 | upper=1024, 137 | default=32, 138 | type=int, 139 | log_scale=True) 140 | dropout_first_layer = UniformFloat(lower=0, 141 | upper=.99, 142 | default=.2) 143 | dropout_second_layer = UniformFloat(lower=0, 144 | upper=.99, 145 | default=.2) 146 | learning_rate = UniformFloat(lower=10e-6, 147 | upper=10e-1, 148 | default=10e-2, 149 | log_scale=True) 150 | 151 | 152 | Here, for each hyperparameter we define a prior distribution, default value, its type and if we want 153 | to adapt it on a log scale or not. 154 | 155 | We can use now Sacred's command line interface to get a new configuration 156 | 157 | .. code:: bash 158 | 159 | python experiment.py with search_space 160 | 161 | Labwatch will now pass all already completed configurations that are stored in the database 162 | to the hyperparameter optimizer, let it suggest a new configuration and then run the experiment with this configuration. 163 | 164 | 165 | Configuration Search Spaces 166 | =========================== 167 | 168 | 169 | At this point is probably a good idea to talk a little bit more about configuration spaces. 170 | In general we distinguish between: 171 | 172 | - *categorical* hyperparameters that can take only discrete choices (e.g. {'a', 'b', 'c'}) 173 | - *numerical* hyperparameters that can have either integer or continuous values. 174 | 175 | Furthermore, Labwatch also allows you to define prior distributions (Gaussian, Uniform) for your hyperparameters. 176 | Some hyperparameter optimizers such as for instance random search can exploit this prior knowledge to 177 | suggest better configurations from early on. 178 | In the case that you do not have a prior about you hyperparameter, just use a uniform distribution which is 179 | simply defined by an upper and lower bound. 180 | 181 | 182 | Hyperparameter Optimizers 183 | ========================= 184 | 185 | 186 | Labwatch offers a simple interface to a variety of state-of-the-art hyperparameter optimization methods. 187 | Note that every optimizer has its own properties and might not work for all use cases. 188 | The following list will give you a brief overview of the optimizer that can be used with labwatch and in which 189 | setting they would work. For more details we refer to the corresponding papers: 190 | 191 | - **Random search** is probably the simplest hyperparameter optimization method. It just samples hyperparameter 192 | configurations from the prior. The nice thing with random search is that it works in all search 193 | spaces and is easy to parallelize. 194 | 195 | - **Bayesian optimization** fits a probabilistic model to capture the current believe of the objective function. 196 | To select a new configuration, it use an utility function that only depend on the 197 | probabilistic model to trade off exploration and exploitation. Here we use Gaussian process to model our objective 198 | function, which work well in low (<10) dimensional continuous input spaces but do not work with categorical 199 | hyperparameters. 200 | 201 | - **SMAC** is also a Bayesian optimization method but uses random forest instead of Gaussian processes to model 202 | the objective function. It works in high dimensional mixed continuous and discret input space but will be 203 | be probably outperformed by GP-based Bayesian optimization in the low dimensional continuous space. 204 | 205 | 206 | Multiple Search Spaces 207 | ====================== 208 | 209 | 210 | Sometimes it is quite convenient to have multiple different search space, for instance if you want to optimize 211 | first only a subset of your hyperparameters and keep the others fixed. 212 | 213 | Labwatch allows to have different search space as long as they have different names. For instance in our running 214 | example if we want to optimize only the learning rate and batch size we can define a second search space: 215 | 216 | 217 | .. code:: python 218 | 219 | @a.searchspace 220 | def small_search_space(): 221 | batch_size = UniformNumber(lower=32, upper=64, default=32, type=int, log_scale=True) 222 | learning_rate = UniformFloat(lower=10e-3, upper=10e-2, default=10e-2, log_scale=True) 223 | 224 | 225 | If we now call our experiment via: 226 | 227 | 228 | .. code:: bash 229 | 230 | python experiment.py with small_search_space 231 | 232 | we get a new configuration for the learning rate and the batch where as all other hyperparameter are set to the values 233 | defined in the config cfg(). 234 | 235 | Note: To prevent inconsistencies and to not fool the optimizer, Labwatch passes only completed configurations that were 236 | drawn from this search space to the optimizer. This means that our optimizer will not use the information from previous 237 | experiment with the other search space. 238 | -------------------------------------------------------------------------------- /docs/_sources/quickstart.txt: -------------------------------------------------------------------------------- 1 | Quickstart 2 | ********** 3 | 4 | The following tutorial assumes that you have a basic understanding of how Sacred works (Experiments, Observers). 5 | In case you are unsure, have a look on the `Sacred Quickstart Guide `_ 6 | 7 | In this tutorial we will see how we can optimize the hyperparameters of a feed forward network trained on MNIST 8 | together with Labwatch and Sacred. 9 | We will use the MNIST example of `keras `_ to implement the neural network but note that 10 | both Labwatch and Sacred are completely independent of which framework you use. 11 | You can find the whole source code as well as more examples 12 | `here `_ 13 | 14 | Automatically Tuning of Hyperparameters 15 | ======================================= 16 | 17 | 18 | Sacred is a useful tool to keep track of all relevant information of your experiments such as hyperparameters, 19 | results, dependencies and so on. 20 | 21 | The following python code adapts the 22 | `keras mnist example `_ to work with Sacred: 23 | 24 | 25 | .. code:: python 26 | 27 | import keras 28 | from keras.datasets import mnist 29 | from keras.models import Sequential 30 | from keras.layers import Dense, Dropout 31 | from keras.optimizers import RMSprop 32 | 33 | from sacred import Experiment 34 | 35 | ex = Experiment() 36 | 37 | @ex.config 38 | def cfg(): 39 | batch_size = 128 40 | num_units_first_layer = 512 41 | num_units_second_layer = 512 42 | dropout_first_layer = 0.2 43 | dropout_second_layer = 0.2 44 | learning_rate = 0.001 45 | 46 | 47 | @ex.automain 48 | def run(batch_size, 49 | num_units_first_layer, 50 | num_units_second_layer, 51 | dropout_first_layer, 52 | dropout_second_layer, 53 | learning_rate): 54 | 55 | num_classes = 10 56 | epochs = 20 57 | 58 | # the data, shuffled and split between train and test sets 59 | (x_train, y_train), (x_test, y_test) = mnist.load_data() 60 | 61 | x_train = x_train.reshape(60000, 784) 62 | x_test = x_test.reshape(10000, 784) 63 | x_train = x_train.astype('float32') 64 | x_test = x_test.astype('float32') 65 | x_train /= 255 66 | x_test /= 255 67 | print(x_train.shape[0], 'train samples') 68 | print(x_test.shape[0], 'test samples') 69 | 70 | # convert class vectors to binary class matrices 71 | y_train = keras.utils.to_categorical(y_train, num_classes) 72 | y_test = keras.utils.to_categorical(y_test, num_classes) 73 | 74 | model = Sequential() 75 | model.add(Dense(num_units_first_layer, activation='relu', input_shape=(784,))) 76 | model.add(Dropout(dropout_first_layer)) 77 | model.add(Dense(num_units_second_layer, activation='relu')) 78 | model.add(Dropout(dropout_second_layer)) 79 | model.add(Dense(10, activation='softmax')) 80 | 81 | model.summary() 82 | 83 | model.compile(loss='categorical_crossentropy', 84 | optimizer=RMSprop(lr=learning_rate), 85 | metrics=['accuracy']) 86 | 87 | history = model.fit(x_train, y_train, 88 | batch_size=batch_size, 89 | epochs=epochs, 90 | verbose=1, 91 | validation_data=(x_test, y_test)) 92 | score = model.evaluate(x_test, y_test, verbose=0) 93 | print('Test loss:', score[0]) 94 | print('Test accuracy:', score[1]) 95 | 96 | results = dict() 97 | results["optimization_target"] = 1 - score[1] 98 | 99 | return results 100 | 101 | In the dark old days you would probably now spend a lot of time to find the right setting for you hyperparameters by 102 | iteratively trying out different setting. 103 | 104 | Well, you're are not alone with this problem and it is actually a common problem in machine learning. 105 | Recently a new subfield in machine learning (`AutoML `_) 106 | has emerged that tries to automated this procedure by casting it as an optimization problem. By now there exist several 107 | optimization methods that tackle the hyperparameter optimization problem. 108 | 109 | To make use of these methods in Labwatch, we fist have to instantiate a LabAssistant which will 110 | connect our Sacred experiment with the hyperparameter optimizer through a MongoDB: 111 | 112 | .. code:: python 113 | 114 | from labwatch.assistant import LabAssistant 115 | from labwatch.optimizers.random_search import RandomSearch 116 | 117 | a = LabAssistant(ex, "labwatch_demo_keras", optimizer=RandomSearch) 118 | 119 | After that we have to define our configuration search space with the hyperparameters that we want to optimize. 120 | 121 | .. code:: python 122 | 123 | @a.searchspace 124 | def search_space(): 125 | batch_size = UniformNumber(lower=8, 126 | upper=64, 127 | default=32, 128 | type=int, 129 | log_scale=True) 130 | num_units_first_layer = UniformNumber(lower=16, 131 | upper=1024, 132 | default=32, 133 | type=int, 134 | log_scale=True) 135 | num_units_second_layer = UniformNumber(lower=16, 136 | upper=1024, 137 | default=32, 138 | type=int, 139 | log_scale=True) 140 | dropout_first_layer = UniformFloat(lower=0, 141 | upper=.99, 142 | default=.2) 143 | dropout_second_layer = UniformFloat(lower=0, 144 | upper=.99, 145 | default=.2) 146 | learning_rate = UniformFloat(lower=10e-6, 147 | upper=10e-1, 148 | default=10e-2, 149 | log_scale=True) 150 | 151 | 152 | Here, for each hyperparameter we define a prior distribution, default value, its type and if we want 153 | to adapt it on a log scale or not. 154 | 155 | We can use now Sacred's command line interface to get a new configuration 156 | 157 | .. code:: bash 158 | 159 | python experiment.py with search_space 160 | 161 | Labwatch will now pass all already completed configurations that are stored in the database 162 | to the hyperparameter optimizer, let it suggest a new configuration and then run the experiment with this configuration. 163 | 164 | 165 | Configuration Search Spaces 166 | =========================== 167 | 168 | 169 | At this point is probably a good idea to talk a little bit more about configuration spaces. 170 | In general we distinguish between: 171 | 172 | - *categorical* hyperparameters that can take only discrete choices (e.g. {'a', 'b', 'c'}) 173 | - *numerical* hyperparameters that can have either integer or continuous values. 174 | 175 | Furthermore, Labwatch also allows you to define prior distributions (Gaussian, Uniform) for your hyperparameters. 176 | Some hyperparameter optimizers such as for instance random search can exploit this prior knowledge to 177 | suggest better configurations from early on. 178 | In the case that you do not have a prior about you hyperparameter, just use a uniform distribution which is 179 | simply defined by an upper and lower bound. 180 | 181 | 182 | Hyperparameter Optimizers 183 | ========================= 184 | 185 | 186 | Labwatch offers a simple interface to a variety of state-of-the-art hyperparameter optimization methods. 187 | Note that every optimizer has its own properties and might not work for all use cases. 188 | The following list will give you a brief overview of the optimizer that can be used with labwatch and in which 189 | setting they would work. For more details we refer to the corresponding papers: 190 | 191 | - **Random search** is probably the simplest hyperparameter optimization method. It just samples hyperparameter 192 | configurations from the prior. The nice thing with random search is that it works in all search 193 | spaces and is easy to parallelize. 194 | 195 | - **Bayesian optimization** fits a probabilistic model to capture the current believe of the objective function. 196 | To select a new configuration, it use an utility function that only depend on the 197 | probabilistic model to trade off exploration and exploitation. Here we use Gaussian process to model our objective 198 | function, which work well in low (<10) dimensional continuous input spaces but do not work with categorical 199 | hyperparameters. 200 | 201 | - **SMAC** is also a Bayesian optimization method but uses random forest instead of Gaussian processes to model 202 | the objective function. It works in high dimensional mixed continuous and discret input space but will be 203 | be probably outperformed by GP-based Bayesian optimization in the low dimensional continuous space. 204 | 205 | 206 | Multiple Search Spaces 207 | ====================== 208 | 209 | 210 | Sometimes it is quite convenient to have multiple different search space, for instance if you want to optimize 211 | first only a subset of your hyperparameters and keep the others fixed. 212 | 213 | Labwatch allows to have different search space as long as they have different names. For instance in our running 214 | example if we want to optimize only the learning rate and batch size we can define a second search space: 215 | 216 | 217 | .. code:: python 218 | 219 | @a.searchspace 220 | def small_search_space(): 221 | batch_size = UniformNumber(lower=32, upper=64, default=32, type=int, log_scale=True) 222 | learning_rate = UniformFloat(lower=10e-3, upper=10e-2, default=10e-2, log_scale=True) 223 | 224 | 225 | If we now call our experiment via: 226 | 227 | 228 | .. code:: bash 229 | 230 | python experiment.py with small_search_space 231 | 232 | we get a new configuration for the learning rate and the batch where as all other hyperparameter are set to the values 233 | defined in the config cfg(). 234 | 235 | Note: To prevent inconsistencies and to not fool the optimizer, Labwatch passes only completed configurations that were 236 | drawn from this search space to the optimizer. This means that our optimizer will not use the information from previous 237 | experiment with the other search space. 238 | -------------------------------------------------------------------------------- /docs/_static/ajax-loader.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/labwatch/cdd7c55e5741bf4f1216f7b8da8de9fb1480c75b/docs/_static/ajax-loader.gif -------------------------------------------------------------------------------- /docs/_static/basic.css: -------------------------------------------------------------------------------- 1 | /* 2 | * basic.css 3 | * ~~~~~~~~~ 4 | * 5 | * Sphinx stylesheet -- basic theme. 6 | * 7 | * :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS. 8 | * :license: BSD, see LICENSE for details. 9 | * 10 | */ 11 | 12 | /* -- main layout ----------------------------------------------------------- */ 13 | 14 | div.clearer { 15 | clear: both; 16 | } 17 | 18 | /* -- relbar ---------------------------------------------------------------- */ 19 | 20 | div.related { 21 | width: 100%; 22 | font-size: 90%; 23 | } 24 | 25 | div.related h3 { 26 | display: none; 27 | } 28 | 29 | div.related ul { 30 | margin: 0; 31 | padding: 0 0 0 10px; 32 | list-style: none; 33 | } 34 | 35 | div.related li { 36 | display: inline; 37 | } 38 | 39 | div.related li.right { 40 | float: right; 41 | margin-right: 5px; 42 | } 43 | 44 | /* -- sidebar --------------------------------------------------------------- */ 45 | 46 | div.sphinxsidebarwrapper { 47 | padding: 10px 5px 0 10px; 48 | } 49 | 50 | div.sphinxsidebar { 51 | float: left; 52 | width: 230px; 53 | margin-left: -100%; 54 | font-size: 90%; 55 | word-wrap: break-word; 56 | overflow-wrap : break-word; 57 | } 58 | 59 | div.sphinxsidebar ul { 60 | list-style: none; 61 | } 62 | 63 | div.sphinxsidebar ul ul, 64 | div.sphinxsidebar ul.want-points { 65 | margin-left: 20px; 66 | list-style: square; 67 | } 68 | 69 | div.sphinxsidebar ul ul { 70 | margin-top: 0; 71 | margin-bottom: 0; 72 | } 73 | 74 | div.sphinxsidebar form { 75 | margin-top: 10px; 76 | } 77 | 78 | div.sphinxsidebar input { 79 | border: 1px solid #98dbcc; 80 | font-family: sans-serif; 81 | font-size: 1em; 82 | } 83 | 84 | div.sphinxsidebar #searchbox input[type="text"] { 85 | width: 170px; 86 | } 87 | 88 | img { 89 | border: 0; 90 | max-width: 100%; 91 | } 92 | 93 | /* -- search page ----------------------------------------------------------- */ 94 | 95 | ul.search { 96 | margin: 10px 0 0 20px; 97 | padding: 0; 98 | } 99 | 100 | ul.search li { 101 | padding: 5px 0 5px 20px; 102 | background-image: url(file.png); 103 | background-repeat: no-repeat; 104 | background-position: 0 7px; 105 | } 106 | 107 | ul.search li a { 108 | font-weight: bold; 109 | } 110 | 111 | ul.search li div.context { 112 | color: #888; 113 | margin: 2px 0 0 30px; 114 | text-align: left; 115 | } 116 | 117 | ul.keywordmatches li.goodmatch a { 118 | font-weight: bold; 119 | } 120 | 121 | /* -- index page ------------------------------------------------------------ */ 122 | 123 | table.contentstable { 124 | width: 90%; 125 | margin-left: auto; 126 | margin-right: auto; 127 | } 128 | 129 | table.contentstable p.biglink { 130 | line-height: 150%; 131 | } 132 | 133 | a.biglink { 134 | font-size: 1.3em; 135 | } 136 | 137 | span.linkdescr { 138 | font-style: italic; 139 | padding-top: 5px; 140 | font-size: 90%; 141 | } 142 | 143 | /* -- general index --------------------------------------------------------- */ 144 | 145 | table.indextable { 146 | width: 100%; 147 | } 148 | 149 | table.indextable td { 150 | text-align: left; 151 | vertical-align: top; 152 | } 153 | 154 | table.indextable ul { 155 | margin-top: 0; 156 | margin-bottom: 0; 157 | list-style-type: none; 158 | } 159 | 160 | table.indextable > tbody > tr > td > ul { 161 | padding-left: 0em; 162 | } 163 | 164 | table.indextable tr.pcap { 165 | height: 10px; 166 | } 167 | 168 | table.indextable tr.cap { 169 | margin-top: 10px; 170 | background-color: #f2f2f2; 171 | } 172 | 173 | img.toggler { 174 | margin-right: 3px; 175 | margin-top: 3px; 176 | cursor: pointer; 177 | } 178 | 179 | div.modindex-jumpbox { 180 | border-top: 1px solid #ddd; 181 | border-bottom: 1px solid #ddd; 182 | margin: 1em 0 1em 0; 183 | padding: 0.4em; 184 | } 185 | 186 | div.genindex-jumpbox { 187 | border-top: 1px solid #ddd; 188 | border-bottom: 1px solid #ddd; 189 | margin: 1em 0 1em 0; 190 | padding: 0.4em; 191 | } 192 | 193 | /* -- domain module index --------------------------------------------------- */ 194 | 195 | table.modindextable td { 196 | padding: 2px; 197 | border-collapse: collapse; 198 | } 199 | 200 | /* -- general body styles --------------------------------------------------- */ 201 | 202 | div.body p, div.body dd, div.body li, div.body blockquote { 203 | -moz-hyphens: auto; 204 | -ms-hyphens: auto; 205 | -webkit-hyphens: auto; 206 | hyphens: auto; 207 | } 208 | 209 | a.headerlink { 210 | visibility: hidden; 211 | } 212 | 213 | h1:hover > a.headerlink, 214 | h2:hover > a.headerlink, 215 | h3:hover > a.headerlink, 216 | h4:hover > a.headerlink, 217 | h5:hover > a.headerlink, 218 | h6:hover > a.headerlink, 219 | dt:hover > a.headerlink, 220 | caption:hover > a.headerlink, 221 | p.caption:hover > a.headerlink, 222 | div.code-block-caption:hover > a.headerlink { 223 | visibility: visible; 224 | } 225 | 226 | div.body p.caption { 227 | text-align: inherit; 228 | } 229 | 230 | div.body td { 231 | text-align: left; 232 | } 233 | 234 | .first { 235 | margin-top: 0 !important; 236 | } 237 | 238 | p.rubric { 239 | margin-top: 30px; 240 | font-weight: bold; 241 | } 242 | 243 | img.align-left, .figure.align-left, object.align-left { 244 | clear: left; 245 | float: left; 246 | margin-right: 1em; 247 | } 248 | 249 | img.align-right, .figure.align-right, object.align-right { 250 | clear: right; 251 | float: right; 252 | margin-left: 1em; 253 | } 254 | 255 | img.align-center, .figure.align-center, object.align-center { 256 | display: block; 257 | margin-left: auto; 258 | margin-right: auto; 259 | } 260 | 261 | .align-left { 262 | text-align: left; 263 | } 264 | 265 | .align-center { 266 | text-align: center; 267 | } 268 | 269 | .align-right { 270 | text-align: right; 271 | } 272 | 273 | /* -- sidebars -------------------------------------------------------------- */ 274 | 275 | div.sidebar { 276 | margin: 0 0 0.5em 1em; 277 | border: 1px solid #ddb; 278 | padding: 7px 7px 0 7px; 279 | background-color: #ffe; 280 | width: 40%; 281 | float: right; 282 | } 283 | 284 | p.sidebar-title { 285 | font-weight: bold; 286 | } 287 | 288 | /* -- topics ---------------------------------------------------------------- */ 289 | 290 | div.topic { 291 | border: 1px solid #ccc; 292 | padding: 7px 7px 0 7px; 293 | margin: 10px 0 10px 0; 294 | } 295 | 296 | p.topic-title { 297 | font-size: 1.1em; 298 | font-weight: bold; 299 | margin-top: 10px; 300 | } 301 | 302 | /* -- admonitions ----------------------------------------------------------- */ 303 | 304 | div.admonition { 305 | margin-top: 10px; 306 | margin-bottom: 10px; 307 | padding: 7px; 308 | } 309 | 310 | div.admonition dt { 311 | font-weight: bold; 312 | } 313 | 314 | div.admonition dl { 315 | margin-bottom: 0; 316 | } 317 | 318 | p.admonition-title { 319 | margin: 0px 10px 5px 0px; 320 | font-weight: bold; 321 | } 322 | 323 | div.body p.centered { 324 | text-align: center; 325 | margin-top: 25px; 326 | } 327 | 328 | /* -- tables ---------------------------------------------------------------- */ 329 | 330 | table.docutils { 331 | border: 0; 332 | border-collapse: collapse; 333 | } 334 | 335 | table caption span.caption-number { 336 | font-style: italic; 337 | } 338 | 339 | table caption span.caption-text { 340 | } 341 | 342 | table.docutils td, table.docutils th { 343 | padding: 1px 8px 1px 5px; 344 | border-top: 0; 345 | border-left: 0; 346 | border-right: 0; 347 | border-bottom: 1px solid #aaa; 348 | } 349 | 350 | table.footnote td, table.footnote th { 351 | border: 0 !important; 352 | } 353 | 354 | th { 355 | text-align: left; 356 | padding-right: 5px; 357 | } 358 | 359 | table.citation { 360 | border-left: solid 1px gray; 361 | margin-left: 1px; 362 | } 363 | 364 | table.citation td { 365 | border-bottom: none; 366 | } 367 | 368 | /* -- figures --------------------------------------------------------------- */ 369 | 370 | div.figure { 371 | margin: 0.5em; 372 | padding: 0.5em; 373 | } 374 | 375 | div.figure p.caption { 376 | padding: 0.3em; 377 | } 378 | 379 | div.figure p.caption span.caption-number { 380 | font-style: italic; 381 | } 382 | 383 | div.figure p.caption span.caption-text { 384 | } 385 | 386 | /* -- field list styles ----------------------------------------------------- */ 387 | 388 | table.field-list td, table.field-list th { 389 | border: 0 !important; 390 | } 391 | 392 | .field-list ul { 393 | margin: 0; 394 | padding-left: 1em; 395 | } 396 | 397 | .field-list p { 398 | margin: 0; 399 | } 400 | 401 | .field-name { 402 | -moz-hyphens: manual; 403 | -ms-hyphens: manual; 404 | -webkit-hyphens: manual; 405 | hyphens: manual; 406 | } 407 | 408 | /* -- other body styles ----------------------------------------------------- */ 409 | 410 | ol.arabic { 411 | list-style: decimal; 412 | } 413 | 414 | ol.loweralpha { 415 | list-style: lower-alpha; 416 | } 417 | 418 | ol.upperalpha { 419 | list-style: upper-alpha; 420 | } 421 | 422 | ol.lowerroman { 423 | list-style: lower-roman; 424 | } 425 | 426 | ol.upperroman { 427 | list-style: upper-roman; 428 | } 429 | 430 | dl { 431 | margin-bottom: 15px; 432 | } 433 | 434 | dd p { 435 | margin-top: 0px; 436 | } 437 | 438 | dd ul, dd table { 439 | margin-bottom: 10px; 440 | } 441 | 442 | dd { 443 | margin-top: 3px; 444 | margin-bottom: 10px; 445 | margin-left: 30px; 446 | } 447 | 448 | dt:target, .highlighted { 449 | background-color: #fbe54e; 450 | } 451 | 452 | dl.glossary dt { 453 | font-weight: bold; 454 | font-size: 1.1em; 455 | } 456 | 457 | .optional { 458 | font-size: 1.3em; 459 | } 460 | 461 | .sig-paren { 462 | font-size: larger; 463 | } 464 | 465 | .versionmodified { 466 | font-style: italic; 467 | } 468 | 469 | .system-message { 470 | background-color: #fda; 471 | padding: 5px; 472 | border: 3px solid red; 473 | } 474 | 475 | .footnote:target { 476 | background-color: #ffa; 477 | } 478 | 479 | .line-block { 480 | display: block; 481 | margin-top: 1em; 482 | margin-bottom: 1em; 483 | } 484 | 485 | .line-block .line-block { 486 | margin-top: 0; 487 | margin-bottom: 0; 488 | margin-left: 1.5em; 489 | } 490 | 491 | .guilabel, .menuselection { 492 | font-family: sans-serif; 493 | } 494 | 495 | .accelerator { 496 | text-decoration: underline; 497 | } 498 | 499 | .classifier { 500 | font-style: oblique; 501 | } 502 | 503 | abbr, acronym { 504 | border-bottom: dotted 1px; 505 | cursor: help; 506 | } 507 | 508 | /* -- code displays --------------------------------------------------------- */ 509 | 510 | pre { 511 | overflow: auto; 512 | overflow-y: hidden; /* fixes display issues on Chrome browsers */ 513 | } 514 | 515 | span.pre { 516 | -moz-hyphens: none; 517 | -ms-hyphens: none; 518 | -webkit-hyphens: none; 519 | hyphens: none; 520 | } 521 | 522 | td.linenos pre { 523 | padding: 5px 0px; 524 | border: 0; 525 | background-color: transparent; 526 | color: #aaa; 527 | } 528 | 529 | table.highlighttable { 530 | margin-left: 0.5em; 531 | } 532 | 533 | table.highlighttable td { 534 | padding: 0 0.5em 0 0.5em; 535 | } 536 | 537 | div.code-block-caption { 538 | padding: 2px 5px; 539 | font-size: small; 540 | } 541 | 542 | div.code-block-caption code { 543 | background-color: transparent; 544 | } 545 | 546 | div.code-block-caption + div > div.highlight > pre { 547 | margin-top: 0; 548 | } 549 | 550 | div.code-block-caption span.caption-number { 551 | padding: 0.1em 0.3em; 552 | font-style: italic; 553 | } 554 | 555 | div.code-block-caption span.caption-text { 556 | } 557 | 558 | div.literal-block-wrapper { 559 | padding: 1em 1em 0; 560 | } 561 | 562 | div.literal-block-wrapper div.highlight { 563 | margin: 0; 564 | } 565 | 566 | code.descname { 567 | background-color: transparent; 568 | font-weight: bold; 569 | font-size: 1.2em; 570 | } 571 | 572 | code.descclassname { 573 | background-color: transparent; 574 | } 575 | 576 | code.xref, a code { 577 | background-color: transparent; 578 | font-weight: bold; 579 | } 580 | 581 | h1 code, h2 code, h3 code, h4 code, h5 code, h6 code { 582 | background-color: transparent; 583 | } 584 | 585 | .viewcode-link { 586 | float: right; 587 | } 588 | 589 | .viewcode-back { 590 | float: right; 591 | font-family: sans-serif; 592 | } 593 | 594 | div.viewcode-block:target { 595 | margin: -1px -10px; 596 | padding: 0 10px; 597 | } 598 | 599 | /* -- math display ---------------------------------------------------------- */ 600 | 601 | img.math { 602 | vertical-align: middle; 603 | } 604 | 605 | div.body div.math p { 606 | text-align: center; 607 | } 608 | 609 | span.eqno { 610 | float: right; 611 | } 612 | 613 | span.eqno a.headerlink { 614 | position: relative; 615 | left: 0px; 616 | z-index: 1; 617 | } 618 | 619 | div.math:hover a.headerlink { 620 | visibility: visible; 621 | } 622 | 623 | /* -- printout stylesheet --------------------------------------------------- */ 624 | 625 | @media print { 626 | div.document, 627 | div.documentwrapper, 628 | div.bodywrapper { 629 | margin: 0 !important; 630 | width: 100%; 631 | } 632 | 633 | div.sphinxsidebar, 634 | div.related, 635 | div.footer, 636 | #top-link { 637 | display: none; 638 | } 639 | } -------------------------------------------------------------------------------- /docs/_static/bootstrap-2.3.2/img/glyphicons-halflings-white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/labwatch/cdd7c55e5741bf4f1216f7b8da8de9fb1480c75b/docs/_static/bootstrap-2.3.2/img/glyphicons-halflings-white.png -------------------------------------------------------------------------------- /docs/_static/bootstrap-2.3.2/img/glyphicons-halflings.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/labwatch/cdd7c55e5741bf4f1216f7b8da8de9fb1480c75b/docs/_static/bootstrap-2.3.2/img/glyphicons-halflings.png -------------------------------------------------------------------------------- /docs/_static/bootstrap-3.3.6/css/bootstrap-theme.min.css.map: -------------------------------------------------------------------------------- 1 | {"version":3,"sources":["less/theme.less","less/mixins/vendor-prefixes.less","less/mixins/gradients.less","less/mixins/reset-filter.less"],"names":[],"mappings":";;;;AAmBA,YAAA,aAAA,UAAA,aAAA,aAAA,aAME,YAAA,EAAA,KAAA,EAAA,eC2CA,mBAAA,MAAA,EAAA,IAAA,EAAA,sBAAA,EAAA,IAAA,IAAA,iBACQ,WAAA,MAAA,EAAA,IAAA,EAAA,sBAAA,EAAA,IAAA,IAAA,iBDvCR,mBAAA,mBAAA,oBAAA,oBAAA,iBAAA,iBAAA,oBAAA,oBAAA,oBAAA,oBAAA,oBAAA,oBCsCA,mBAAA,MAAA,EAAA,IAAA,IAAA,iBACQ,WAAA,MAAA,EAAA,IAAA,IAAA,iBDlCR,qBAAA,sBAAA,sBAAA,uBAAA,mBAAA,oBAAA,sBAAA,uBAAA,sBAAA,uBAAA,sBAAA,uBAAA,+BAAA,gCAAA,6BAAA,gCAAA,gCAAA,gCCiCA,mBAAA,KACQ,WAAA,KDlDV,mBAAA,oBAAA,iBAAA,oBAAA,oBAAA,oBAuBI,YAAA,KAyCF,YAAA,YAEE,iBAAA,KAKJ,aErEI,YAAA,EAAA,IAAA,EAAA,KACA,iBAAA,iDACA,iBAAA,4CAAA,iBAAA,qEAEA,iBAAA,+CCnBF,OAAA,+GH4CA,OAAA,0DACA,kBAAA,SAuC2C,aAAA,QAA2B,aAAA,KArCtE,mBAAA,mBAEE,iBAAA,QACA,oBAAA,EAAA,MAGF,oBAAA,oBAEE,iBAAA,QACA,aAAA,QAMA,sBAAA,6BAAA,4BAAA,6BAAA,4BAAA,4BAAA,uBAAA,8BAAA,6BAAA,8BAAA,6BAAA,6BAAA,gCAAA,uCAAA,sCAAA,uCAAA,sCAAA,sCAME,iBAAA,QACA,iBAAA,KAgBN,aEtEI,iBAAA,oDACA,iBAAA,+CACA,iBAAA,wEAAA,iBAAA,kDAEA,OAAA,+GCnBF,OAAA,0DH4CA,kBAAA,SACA,aAAA,QAEA,mBAAA,mBAEE,iBAAA,QACA,oBAAA,EAAA,MAGF,oBAAA,oBAEE,iBAAA,QACA,aAAA,QAMA,sBAAA,6BAAA,4BAAA,6BAAA,4BAAA,4BAAA,uBAAA,8BAAA,6BAAA,8BAAA,6BAAA,6BAAA,gCAAA,uCAAA,sCAAA,uCAAA,sCAAA,sCAME,iBAAA,QACA,iBAAA,KAiBN,aEvEI,iBAAA,oDACA,iBAAA,+CACA,iBAAA,wEAAA,iBAAA,kDAEA,OAAA,+GCnBF,OAAA,0DH4CA,kBAAA,SACA,aAAA,QAEA,mBAAA,mBAEE,iBAAA,QACA,oBAAA,EAAA,MAGF,oBAAA,oBAEE,iBAAA,QACA,aAAA,QAMA,sBAAA,6BAAA,4BAAA,6BAAA,4BAAA,4BAAA,uBAAA,8BAAA,6BAAA,8BAAA,6BAAA,6BAAA,gCAAA,uCAAA,sCAAA,uCAAA,sCAAA,sCAME,iBAAA,QACA,iBAAA,KAkBN,UExEI,iBAAA,oDACA,iBAAA,+CACA,iBAAA,wEAAA,iBAAA,kDAEA,OAAA,+GCnBF,OAAA,0DH4CA,kBAAA,SACA,aAAA,QAEA,gBAAA,gBAEE,iBAAA,QACA,oBAAA,EAAA,MAGF,iBAAA,iBAEE,iBAAA,QACA,aAAA,QAMA,mBAAA,0BAAA,yBAAA,0BAAA,yBAAA,yBAAA,oBAAA,2BAAA,0BAAA,2BAAA,0BAAA,0BAAA,6BAAA,oCAAA,mCAAA,oCAAA,mCAAA,mCAME,iBAAA,QACA,iBAAA,KAmBN,aEzEI,iBAAA,oDACA,iBAAA,+CACA,iBAAA,wEAAA,iBAAA,kDAEA,OAAA,+GCnBF,OAAA,0DH4CA,kBAAA,SACA,aAAA,QAEA,mBAAA,mBAEE,iBAAA,QACA,oBAAA,EAAA,MAGF,oBAAA,oBAEE,iBAAA,QACA,aAAA,QAMA,sBAAA,6BAAA,4BAAA,6BAAA,4BAAA,4BAAA,uBAAA,8BAAA,6BAAA,8BAAA,6BAAA,6BAAA,gCAAA,uCAAA,sCAAA,uCAAA,sCAAA,sCAME,iBAAA,QACA,iBAAA,KAoBN,YE1EI,iBAAA,oDACA,iBAAA,+CACA,iBAAA,wEAAA,iBAAA,kDAEA,OAAA,+GCnBF,OAAA,0DH4CA,kBAAA,SACA,aAAA,QAEA,kBAAA,kBAEE,iBAAA,QACA,oBAAA,EAAA,MAGF,mBAAA,mBAEE,iBAAA,QACA,aAAA,QAMA,qBAAA,4BAAA,2BAAA,4BAAA,2BAAA,2BAAA,sBAAA,6BAAA,4BAAA,6BAAA,4BAAA,4BAAA,+BAAA,sCAAA,qCAAA,sCAAA,qCAAA,qCAME,iBAAA,QACA,iBAAA,KA2BN,eAAA,WClCE,mBAAA,EAAA,IAAA,IAAA,iBACQ,WAAA,EAAA,IAAA,IAAA,iBD2CV,0BAAA,0BE3FI,iBAAA,QACA,iBAAA,oDACA,iBAAA,+CAAA,iBAAA,wEACA,iBAAA,kDACA,OAAA,+GF0FF,kBAAA,SAEF,yBAAA,+BAAA,+BEhGI,iBAAA,QACA,iBAAA,oDACA,iBAAA,+CAAA,iBAAA,wEACA,iBAAA,kDACA,OAAA,+GFgGF,kBAAA,SASF,gBE7GI,iBAAA,iDACA,iBAAA,4CACA,iBAAA,qEAAA,iBAAA,+CACA,OAAA,+GACA,OAAA,0DCnBF,kBAAA,SH+HA,cAAA,ICjEA,mBAAA,MAAA,EAAA,IAAA,EAAA,sBAAA,EAAA,IAAA,IAAA,iBACQ,WAAA,MAAA,EAAA,IAAA,EAAA,sBAAA,EAAA,IAAA,IAAA,iBD6DV,sCAAA,oCE7GI,iBAAA,oDACA,iBAAA,+CACA,iBAAA,wEAAA,iBAAA,kDACA,OAAA,+GACA,kBAAA,SD2CF,mBAAA,MAAA,EAAA,IAAA,IAAA,iBACQ,WAAA,MAAA,EAAA,IAAA,IAAA,iBD0EV,cAAA,iBAEE,YAAA,EAAA,IAAA,EAAA,sBAIF,gBEhII,iBAAA,iDACA,iBAAA,4CACA,iBAAA,qEAAA,iBAAA,+CACA,OAAA,+GACA,OAAA,0DCnBF,kBAAA,SHkJA,cAAA,IAHF,sCAAA,oCEhII,iBAAA,oDACA,iBAAA,+CACA,iBAAA,wEAAA,iBAAA,kDACA,OAAA,+GACA,kBAAA,SD2CF,mBAAA,MAAA,EAAA,IAAA,IAAA,gBACQ,WAAA,MAAA,EAAA,IAAA,IAAA,gBDgFV,8BAAA,iCAYI,YAAA,EAAA,KAAA,EAAA,gBAKJ,qBAAA,kBAAA,mBAGE,cAAA,EAqBF,yBAfI,mDAAA,yDAAA,yDAGE,MAAA,KE7JF,iBAAA,oDACA,iBAAA,+CACA,iBAAA,wEAAA,iBAAA,kDACA,OAAA,+GACA,kBAAA,UFqKJ,OACE,YAAA,EAAA,IAAA,EAAA,qBC3HA,mBAAA,MAAA,EAAA,IAAA,EAAA,sBAAA,EAAA,IAAA,IAAA,gBACQ,WAAA,MAAA,EAAA,IAAA,EAAA,sBAAA,EAAA,IAAA,IAAA,gBDsIV,eEtLI,iBAAA,oDACA,iBAAA,+CACA,iBAAA,wEAAA,iBAAA,kDACA,OAAA,+GACA,kBAAA,SF8KF,aAAA,QAKF,YEvLI,iBAAA,oDACA,iBAAA,+CACA,iBAAA,wEAAA,iBAAA,kDACA,OAAA,+GACA,kBAAA,SF8KF,aAAA,QAMF,eExLI,iBAAA,oDACA,iBAAA,+CACA,iBAAA,wEAAA,iBAAA,kDACA,OAAA,+GACA,kBAAA,SF8KF,aAAA,QAOF,cEzLI,iBAAA,oDACA,iBAAA,+CACA,iBAAA,wEAAA,iBAAA,kDACA,OAAA,+GACA,kBAAA,SF8KF,aAAA,QAeF,UEjMI,iBAAA,oDACA,iBAAA,+CACA,iBAAA,wEAAA,iBAAA,kDACA,OAAA,+GACA,kBAAA,SFuMJ,cE3MI,iBAAA,oDACA,iBAAA,+CACA,iBAAA,wEAAA,iBAAA,kDACA,OAAA,+GACA,kBAAA,SFwMJ,sBE5MI,iBAAA,oDACA,iBAAA,+CACA,iBAAA,wEAAA,iBAAA,kDACA,OAAA,+GACA,kBAAA,SFyMJ,mBE7MI,iBAAA,oDACA,iBAAA,+CACA,iBAAA,wEAAA,iBAAA,kDACA,OAAA,+GACA,kBAAA,SF0MJ,sBE9MI,iBAAA,oDACA,iBAAA,+CACA,iBAAA,wEAAA,iBAAA,kDACA,OAAA,+GACA,kBAAA,SF2MJ,qBE/MI,iBAAA,oDACA,iBAAA,+CACA,iBAAA,wEAAA,iBAAA,kDACA,OAAA,+GACA,kBAAA,SF+MJ,sBElLI,iBAAA,yKACA,iBAAA,oKACA,iBAAA,iKFyLJ,YACE,cAAA,IC9KA,mBAAA,EAAA,IAAA,IAAA,iBACQ,WAAA,EAAA,IAAA,IAAA,iBDgLV,wBAAA,8BAAA,8BAGE,YAAA,EAAA,KAAA,EAAA,QEnOE,iBAAA,oDACA,iBAAA,+CACA,iBAAA,wEAAA,iBAAA,kDACA,OAAA,+GACA,kBAAA,SFiOF,aAAA,QALF,+BAAA,qCAAA,qCAQI,YAAA,KAUJ,OCnME,mBAAA,EAAA,IAAA,IAAA,gBACQ,WAAA,EAAA,IAAA,IAAA,gBD4MV,8BE5PI,iBAAA,oDACA,iBAAA,+CACA,iBAAA,wEAAA,iBAAA,kDACA,OAAA,+GACA,kBAAA,SFyPJ,8BE7PI,iBAAA,oDACA,iBAAA,+CACA,iBAAA,wEAAA,iBAAA,kDACA,OAAA,+GACA,kBAAA,SF0PJ,8BE9PI,iBAAA,oDACA,iBAAA,+CACA,iBAAA,wEAAA,iBAAA,kDACA,OAAA,+GACA,kBAAA,SF2PJ,2BE/PI,iBAAA,oDACA,iBAAA,+CACA,iBAAA,wEAAA,iBAAA,kDACA,OAAA,+GACA,kBAAA,SF4PJ,8BEhQI,iBAAA,oDACA,iBAAA,+CACA,iBAAA,wEAAA,iBAAA,kDACA,OAAA,+GACA,kBAAA,SF6PJ,6BEjQI,iBAAA,oDACA,iBAAA,+CACA,iBAAA,wEAAA,iBAAA,kDACA,OAAA,+GACA,kBAAA,SFoQJ,MExQI,iBAAA,oDACA,iBAAA,+CACA,iBAAA,wEAAA,iBAAA,kDACA,OAAA,+GACA,kBAAA,SFsQF,aAAA,QC3NA,mBAAA,MAAA,EAAA,IAAA,IAAA,gBAAA,EAAA,IAAA,EAAA,qBACQ,WAAA,MAAA,EAAA,IAAA,IAAA,gBAAA,EAAA,IAAA,EAAA"} -------------------------------------------------------------------------------- /docs/_static/bootstrap-3.3.6/fonts/glyphicons-halflings-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/labwatch/cdd7c55e5741bf4f1216f7b8da8de9fb1480c75b/docs/_static/bootstrap-3.3.6/fonts/glyphicons-halflings-regular.eot -------------------------------------------------------------------------------- /docs/_static/bootstrap-3.3.6/fonts/glyphicons-halflings-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/labwatch/cdd7c55e5741bf4f1216f7b8da8de9fb1480c75b/docs/_static/bootstrap-3.3.6/fonts/glyphicons-halflings-regular.ttf -------------------------------------------------------------------------------- /docs/_static/bootstrap-3.3.6/fonts/glyphicons-halflings-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/labwatch/cdd7c55e5741bf4f1216f7b8da8de9fb1480c75b/docs/_static/bootstrap-3.3.6/fonts/glyphicons-halflings-regular.woff -------------------------------------------------------------------------------- /docs/_static/bootstrap-3.3.6/fonts/glyphicons-halflings-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/labwatch/cdd7c55e5741bf4f1216f7b8da8de9fb1480c75b/docs/_static/bootstrap-3.3.6/fonts/glyphicons-halflings-regular.woff2 -------------------------------------------------------------------------------- /docs/_static/bootstrap-3.3.6/js/npm.js: -------------------------------------------------------------------------------- 1 | // This file is autogenerated via the `commonjs` Grunt task. You can require() this file in a CommonJS environment. 2 | require('../../js/transition.js') 3 | require('../../js/alert.js') 4 | require('../../js/button.js') 5 | require('../../js/carousel.js') 6 | require('../../js/collapse.js') 7 | require('../../js/dropdown.js') 8 | require('../../js/modal.js') 9 | require('../../js/tooltip.js') 10 | require('../../js/popover.js') 11 | require('../../js/scrollspy.js') 12 | require('../../js/tab.js') 13 | require('../../js/affix.js') -------------------------------------------------------------------------------- /docs/_static/bootstrap-sphinx.css: -------------------------------------------------------------------------------- 1 | /* 2 | * bootstrap-sphinx.css 3 | * ~~~~~~~~~~~~~~~~~~~~ 4 | * 5 | * Sphinx stylesheet -- Bootstrap theme. 6 | */ 7 | 8 | /* 9 | * Imports to aggregate everything together. 10 | */ 11 | 12 | @import url("./basic.css"); 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | @import url("./bootswatch-3.3.6/cosmo/bootstrap.min.css"); 22 | 23 | 24 | 25 | /* 26 | * Styles 27 | */ 28 | 29 | .navbar-inverse .brand { 30 | color: #FFF; 31 | } 32 | 33 | /* 34 | * Reset navbar styles from overrides in: 35 | * https://bitbucket.org/birkenfeld/sphinx/commits/78d8ebf76b630ab4073a7328af9d91e8123b8d96 36 | */ 37 | .navbar .container { 38 | padding-top: 0; 39 | } 40 | 41 | /* 42 | * Reset the logo image dimensions. Sites like RTD can override with bad 43 | * results on mobile (mega-huge logo...) 44 | * 45 | * https://github.com/ryan-roemer/sphinx-bootstrap-theme/issues/142 46 | */ 47 | .navbar-brand img { 48 | width: auto; 49 | height: 100%; 50 | } 51 | 52 | .page-top { 53 | top: 0px; 54 | } 55 | 56 | 57 | 58 | 59 | 60 | body { 61 | 62 | padding-top: 60px; 63 | 64 | } 65 | .page-top { 66 | 67 | top: 60px; 68 | 69 | } 70 | 71 | 72 | 73 | .navbar-inner { 74 | padding-left: 12px !important; 75 | padding-right: 12px !important; 76 | } 77 | 78 | 79 | table { 80 | border: 0; 81 | } 82 | 83 | .highlighttable .code pre { 84 | font-size: 12px; 85 | } 86 | 87 | .highlighttable .linenos pre { 88 | word-break: normal; 89 | font-size: 12px; 90 | } 91 | 92 | div.highlight { 93 | background: none; 94 | } 95 | 96 | a.footnote-reference { 97 | vertical-align: super; 98 | font-size: 75%; 99 | } 100 | 101 | table.footnote td.label { 102 | color: inherit; 103 | font-size: 100%; 104 | display: block; 105 | line-height: normal; 106 | background: inherit; 107 | } 108 | 109 | table.footnote { 110 | width: auto; 111 | margin-bottom: 0px; 112 | } 113 | 114 | table.field-list { 115 | width: auto; 116 | } 117 | 118 | .footer { 119 | width: 100%; 120 | border-top: 1px solid #ccc; 121 | padding-top: 10px; 122 | } 123 | 124 | .bs-sidenav form, .bs-sidenav #sourcelink { 125 | padding: 5px 20px; 126 | } 127 | 128 | 129 | 130 | /* The code below is based on the bootstrap website sidebar */ 131 | 132 | .bs-sidenav.affix { 133 | position: static; 134 | } 135 | 136 | /* First level of nav */ 137 | .bs-sidenav { 138 | margin-top: 30px; 139 | margin-bottom: 30px; 140 | padding-top: 10px; 141 | padding-bottom: 10px; 142 | text-shadow: 0 1px 0 #fff; 143 | background-color: #f7f5fa; 144 | border-radius: 5px; 145 | } 146 | 147 | /* All levels of nav */ 148 | .bs-sidenav .nav > li > a { 149 | display: block; 150 | color: #716b7a; 151 | padding: 5px 20px; 152 | } 153 | .bs-sidenav .nav > li > a:hover, 154 | .bs-sidenav .nav > li > a:focus { 155 | text-decoration: none; 156 | background-color: #e5e3e9; 157 | border-right: 1px solid #dbd8e0; 158 | } 159 | .bs-sidenav .nav > .active > a, 160 | .bs-sidenav .nav > .active:hover > a, 161 | .bs-sidenav .nav > .active:focus > a { 162 | font-weight: bold; 163 | color: #563d7c; 164 | background-color: transparent; 165 | border-right: 1px solid #563d7c; 166 | } 167 | 168 | .bs-sidenav .nav .nav > li > a { 169 | padding-top: 3px; 170 | padding-bottom: 3px; 171 | padding-left: 30px; 172 | font-size: 90%; 173 | } 174 | 175 | .bs-sidenav .nav .nav .nav > li > a { 176 | padding-top: 3px; 177 | padding-bottom: 3px; 178 | padding-left: 40px; 179 | font-size: 90%; 180 | } 181 | 182 | .bs-sidenav .nav .nav .nav .nav > li > a { 183 | padding-top: 3px; 184 | padding-bottom: 3px; 185 | padding-left: 50px; 186 | font-size: 90%; 187 | } 188 | 189 | /* Show and affix the side nav when space allows it */ 190 | @media screen and (min-width: 992px) { 191 | .bs-sidenav .nav > .active > ul { 192 | display: block; 193 | } 194 | /* Widen the fixed sidenav */ 195 | .bs-sidenav.affix, 196 | .bs-sidenav.affix-bottom { 197 | width: 213px; 198 | } 199 | .bs-sidenav.affix { 200 | position: fixed; /* Undo the static from mobile first approach */ 201 | } 202 | .bs-sidenav.affix-bottom { 203 | position: absolute; /* Undo the static from mobile first approach */ 204 | } 205 | .bs-sidenav.affix-bottom .bs-sidenav, 206 | .bs-sidenav.affix .bs-sidenav { 207 | margin-top: 0; 208 | margin-bottom: 0; 209 | } 210 | } 211 | @media screen and (min-width: 1200px) { 212 | /* Widen the fixed sidenav again */ 213 | .bs-sidenav.affix-bottom, 214 | .bs-sidenav.affix { 215 | width: 263px; 216 | } 217 | } 218 | 219 | 220 | -------------------------------------------------------------------------------- /docs/_static/bootstrap-sphinx.js: -------------------------------------------------------------------------------- 1 | (function ($) { 2 | /** 3 | * Patch TOC list. 4 | * 5 | * Will mutate the underlying span to have a correct ul for nav. 6 | * 7 | * @param $span: Span containing nested UL"s to mutate. 8 | * @param minLevel: Starting level for nested lists. (1: global, 2: local). 9 | */ 10 | var patchToc = function ($ul, minLevel) { 11 | var findA, 12 | patchTables, 13 | $localLi; 14 | 15 | // Find all a "internal" tags, traversing recursively. 16 | findA = function ($elem, level) { 17 | level = level || 0; 18 | var $items = $elem.find("> li > a.internal, > ul, > li > ul"); 19 | 20 | // Iterate everything in order. 21 | $items.each(function (index, item) { 22 | var $item = $(item), 23 | tag = item.tagName.toLowerCase(), 24 | $childrenLi = $item.children("li"), 25 | $parentLi = $($item.parent("li"), $item.parent().parent("li")); 26 | 27 | // Add dropdowns if more children and above minimum level. 28 | if (tag === "ul" && level >= minLevel && $childrenLi.length > 0) { 29 | $parentLi 30 | .addClass("dropdown-submenu") 31 | .children("a").first().attr("tabindex", -1); 32 | 33 | $item.addClass("dropdown-menu"); 34 | } 35 | 36 | findA($item, level + 1); 37 | }); 38 | }; 39 | 40 | findA($ul); 41 | }; 42 | 43 | /** 44 | * Patch all tables to remove ``docutils`` class and add Bootstrap base 45 | * ``table`` class. 46 | */ 47 | patchTables = function () { 48 | $("table.docutils") 49 | .removeClass("docutils") 50 | .addClass("table") 51 | .attr("border", 0); 52 | }; 53 | 54 | $(window).load(function () { 55 | /* 56 | * Scroll the window to avoid the topnav bar 57 | * https://github.com/twbs/bootstrap/issues/1768 58 | */ 59 | if ($("#navbar.navbar-fixed-top").length > 0) { 60 | var navHeight = $("#navbar").height(), 61 | shiftWindow = function() { scrollBy(0, -navHeight - 10); }; 62 | 63 | if (location.hash) { 64 | setTimeout(shiftWindow, 1); 65 | } 66 | 67 | window.addEventListener("hashchange", shiftWindow); 68 | } 69 | }); 70 | 71 | $(document).ready(function () { 72 | // Add styling, structure to TOC"s. 73 | $(".dropdown-menu").each(function () { 74 | $(this).find("ul").each(function (index, item){ 75 | var $item = $(item); 76 | $item.addClass("unstyled"); 77 | }); 78 | }); 79 | 80 | // Global TOC. 81 | if ($("ul.globaltoc li").length) { 82 | patchToc($("ul.globaltoc"), 1); 83 | } else { 84 | // Remove Global TOC. 85 | $(".globaltoc-container").remove(); 86 | } 87 | 88 | // Local TOC. 89 | $(".bs-sidenav ul").addClass("nav nav-list"); 90 | $(".bs-sidenav > ul > li > a").addClass("nav-header"); 91 | 92 | 93 | // back to top 94 | setTimeout(function () { 95 | var $sideBar = $(".bs-sidenav"); 96 | var $content = $(".content"); 97 | 98 | // Enlarge content if sidebar is larger. 99 | if ($sideBar.outerHeight(true) > $content.outerHeight(true)) { 100 | $content.css("min-height", $sideBar.outerHeight(true)); 101 | } 102 | 103 | $sideBar 104 | // Add affix. 105 | .affix({ 106 | offset: { 107 | top: function () { 108 | var offsetTop = $sideBar.offset().top; 109 | var sideBarMargin = parseInt($sideBar.css("margin-top"), 10); 110 | var navOuterHeight = $("#navbar").outerHeight(true); 111 | 112 | return (this.top = offsetTop - navOuterHeight); 113 | }, 114 | bottom: function () { 115 | return (this.bottom = $(".footer").outerHeight(true)); 116 | } 117 | } 118 | }) 119 | // Trigger to reset if page content is scrolled to bottom. 120 | .trigger("scroll.bs.affix.data-api"); 121 | }, 0); 122 | 123 | 124 | // Local TOC. 125 | patchToc($("ul.localtoc"), 2); 126 | 127 | // Mutate sub-lists (for bs-2.3.0). 128 | $(".dropdown-menu ul").not(".dropdown-menu").each(function () { 129 | var $ul = $(this), 130 | $parent = $ul.parent(), 131 | tag = $parent[0].tagName.toLowerCase(), 132 | $kids = $ul.children().detach(); 133 | 134 | // Replace list with items if submenu header. 135 | if (tag === "ul") { 136 | $ul.replaceWith($kids); 137 | } else if (tag === "li") { 138 | // Insert into previous list. 139 | $parent.after($kids); 140 | $ul.remove(); 141 | } 142 | }); 143 | 144 | // Add divider in page TOC. 145 | $localLi = $("ul.localtoc li"); 146 | if ($localLi.length > 2) { 147 | $localLi.first().after("
  • "); 148 | } 149 | 150 | // Patch tables. 151 | patchTables(); 152 | 153 | // Add Note, Warning styles. (BS v2,3 compatible). 154 | $(".admonition").addClass("alert alert-info") 155 | .filter(".warning, .caution") 156 | .removeClass("alert-info") 157 | .addClass("alert-warning").end() 158 | .filter(".error, .danger") 159 | .removeClass("alert-info") 160 | .addClass("alert-danger alert-error").end(); 161 | 162 | // Inline code styles to Bootstrap style. 163 | $("tt.docutils.literal").not(".xref").each(function (i, e) { 164 | // ignore references 165 | if (!$(e).parent().hasClass("reference")) { 166 | $(e).replaceWith(function () { 167 | return $("").html($(this).html()); 168 | }); 169 | }}); 170 | 171 | // Update sourcelink to remove outerdiv (fixes appearance in navbar). 172 | var $srcLink = $(".nav #sourcelink"); 173 | $srcLink.parent().html($srcLink.html()); 174 | }); 175 | }(window.$jqTheme || window.jQuery)); -------------------------------------------------------------------------------- /docs/_static/bootswatch-2.3.2/img/glyphicons-halflings-white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/labwatch/cdd7c55e5741bf4f1216f7b8da8de9fb1480c75b/docs/_static/bootswatch-2.3.2/img/glyphicons-halflings-white.png -------------------------------------------------------------------------------- /docs/_static/bootswatch-2.3.2/img/glyphicons-halflings.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/labwatch/cdd7c55e5741bf4f1216f7b8da8de9fb1480c75b/docs/_static/bootswatch-2.3.2/img/glyphicons-halflings.png -------------------------------------------------------------------------------- /docs/_static/bootswatch-3.3.6/fonts/glyphicons-halflings-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/labwatch/cdd7c55e5741bf4f1216f7b8da8de9fb1480c75b/docs/_static/bootswatch-3.3.6/fonts/glyphicons-halflings-regular.eot -------------------------------------------------------------------------------- /docs/_static/bootswatch-3.3.6/fonts/glyphicons-halflings-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/labwatch/cdd7c55e5741bf4f1216f7b8da8de9fb1480c75b/docs/_static/bootswatch-3.3.6/fonts/glyphicons-halflings-regular.ttf -------------------------------------------------------------------------------- /docs/_static/bootswatch-3.3.6/fonts/glyphicons-halflings-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/labwatch/cdd7c55e5741bf4f1216f7b8da8de9fb1480c75b/docs/_static/bootswatch-3.3.6/fonts/glyphicons-halflings-regular.woff -------------------------------------------------------------------------------- /docs/_static/bootswatch-3.3.6/fonts/glyphicons-halflings-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/labwatch/cdd7c55e5741bf4f1216f7b8da8de9fb1480c75b/docs/_static/bootswatch-3.3.6/fonts/glyphicons-halflings-regular.woff2 -------------------------------------------------------------------------------- /docs/_static/comment-bright.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/labwatch/cdd7c55e5741bf4f1216f7b8da8de9fb1480c75b/docs/_static/comment-bright.png -------------------------------------------------------------------------------- /docs/_static/comment-close.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/labwatch/cdd7c55e5741bf4f1216f7b8da8de9fb1480c75b/docs/_static/comment-close.png -------------------------------------------------------------------------------- /docs/_static/comment.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/labwatch/cdd7c55e5741bf4f1216f7b8da8de9fb1480c75b/docs/_static/comment.png -------------------------------------------------------------------------------- /docs/_static/custom.css: -------------------------------------------------------------------------------- 1 | /* This file intentionally left blank. */ 2 | -------------------------------------------------------------------------------- /docs/_static/doctools.js: -------------------------------------------------------------------------------- 1 | /* 2 | * doctools.js 3 | * ~~~~~~~~~~~ 4 | * 5 | * Sphinx JavaScript utilities for all documentation. 6 | * 7 | * :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS. 8 | * :license: BSD, see LICENSE for details. 9 | * 10 | */ 11 | 12 | /** 13 | * select a different prefix for underscore 14 | */ 15 | $u = _.noConflict(); 16 | 17 | /** 18 | * make the code below compatible with browsers without 19 | * an installed firebug like debugger 20 | if (!window.console || !console.firebug) { 21 | var names = ["log", "debug", "info", "warn", "error", "assert", "dir", 22 | "dirxml", "group", "groupEnd", "time", "timeEnd", "count", "trace", 23 | "profile", "profileEnd"]; 24 | window.console = {}; 25 | for (var i = 0; i < names.length; ++i) 26 | window.console[names[i]] = function() {}; 27 | } 28 | */ 29 | 30 | /** 31 | * small helper function to urldecode strings 32 | */ 33 | jQuery.urldecode = function(x) { 34 | return decodeURIComponent(x).replace(/\+/g, ' '); 35 | }; 36 | 37 | /** 38 | * small helper function to urlencode strings 39 | */ 40 | jQuery.urlencode = encodeURIComponent; 41 | 42 | /** 43 | * This function returns the parsed url parameters of the 44 | * current request. Multiple values per key are supported, 45 | * it will always return arrays of strings for the value parts. 46 | */ 47 | jQuery.getQueryParameters = function(s) { 48 | if (typeof s == 'undefined') 49 | s = document.location.search; 50 | var parts = s.substr(s.indexOf('?') + 1).split('&'); 51 | var result = {}; 52 | for (var i = 0; i < parts.length; i++) { 53 | var tmp = parts[i].split('=', 2); 54 | var key = jQuery.urldecode(tmp[0]); 55 | var value = jQuery.urldecode(tmp[1]); 56 | if (key in result) 57 | result[key].push(value); 58 | else 59 | result[key] = [value]; 60 | } 61 | return result; 62 | }; 63 | 64 | /** 65 | * highlight a given string on a jquery object by wrapping it in 66 | * span elements with the given class name. 67 | */ 68 | jQuery.fn.highlightText = function(text, className) { 69 | function highlight(node) { 70 | if (node.nodeType == 3) { 71 | var val = node.nodeValue; 72 | var pos = val.toLowerCase().indexOf(text); 73 | if (pos >= 0 && !jQuery(node.parentNode).hasClass(className)) { 74 | var span = document.createElement("span"); 75 | span.className = className; 76 | span.appendChild(document.createTextNode(val.substr(pos, text.length))); 77 | node.parentNode.insertBefore(span, node.parentNode.insertBefore( 78 | document.createTextNode(val.substr(pos + text.length)), 79 | node.nextSibling)); 80 | node.nodeValue = val.substr(0, pos); 81 | } 82 | } 83 | else if (!jQuery(node).is("button, select, textarea")) { 84 | jQuery.each(node.childNodes, function() { 85 | highlight(this); 86 | }); 87 | } 88 | } 89 | return this.each(function() { 90 | highlight(this); 91 | }); 92 | }; 93 | 94 | /* 95 | * backward compatibility for jQuery.browser 96 | * This will be supported until firefox bug is fixed. 97 | */ 98 | if (!jQuery.browser) { 99 | jQuery.uaMatch = function(ua) { 100 | ua = ua.toLowerCase(); 101 | 102 | var match = /(chrome)[ \/]([\w.]+)/.exec(ua) || 103 | /(webkit)[ \/]([\w.]+)/.exec(ua) || 104 | /(opera)(?:.*version|)[ \/]([\w.]+)/.exec(ua) || 105 | /(msie) ([\w.]+)/.exec(ua) || 106 | ua.indexOf("compatible") < 0 && /(mozilla)(?:.*? rv:([\w.]+)|)/.exec(ua) || 107 | []; 108 | 109 | return { 110 | browser: match[ 1 ] || "", 111 | version: match[ 2 ] || "0" 112 | }; 113 | }; 114 | jQuery.browser = {}; 115 | jQuery.browser[jQuery.uaMatch(navigator.userAgent).browser] = true; 116 | } 117 | 118 | /** 119 | * Small JavaScript module for the documentation. 120 | */ 121 | var Documentation = { 122 | 123 | init : function() { 124 | this.fixFirefoxAnchorBug(); 125 | this.highlightSearchWords(); 126 | this.initIndexTable(); 127 | 128 | }, 129 | 130 | /** 131 | * i18n support 132 | */ 133 | TRANSLATIONS : {}, 134 | PLURAL_EXPR : function(n) { return n == 1 ? 0 : 1; }, 135 | LOCALE : 'unknown', 136 | 137 | // gettext and ngettext don't access this so that the functions 138 | // can safely bound to a different name (_ = Documentation.gettext) 139 | gettext : function(string) { 140 | var translated = Documentation.TRANSLATIONS[string]; 141 | if (typeof translated == 'undefined') 142 | return string; 143 | return (typeof translated == 'string') ? translated : translated[0]; 144 | }, 145 | 146 | ngettext : function(singular, plural, n) { 147 | var translated = Documentation.TRANSLATIONS[singular]; 148 | if (typeof translated == 'undefined') 149 | return (n == 1) ? singular : plural; 150 | return translated[Documentation.PLURALEXPR(n)]; 151 | }, 152 | 153 | addTranslations : function(catalog) { 154 | for (var key in catalog.messages) 155 | this.TRANSLATIONS[key] = catalog.messages[key]; 156 | this.PLURAL_EXPR = new Function('n', 'return +(' + catalog.plural_expr + ')'); 157 | this.LOCALE = catalog.locale; 158 | }, 159 | 160 | /** 161 | * add context elements like header anchor links 162 | */ 163 | addContextElements : function() { 164 | $('div[id] > :header:first').each(function() { 165 | $('\u00B6'). 166 | attr('href', '#' + this.id). 167 | attr('title', _('Permalink to this headline')). 168 | appendTo(this); 169 | }); 170 | $('dt[id]').each(function() { 171 | $('\u00B6'). 172 | attr('href', '#' + this.id). 173 | attr('title', _('Permalink to this definition')). 174 | appendTo(this); 175 | }); 176 | }, 177 | 178 | /** 179 | * workaround a firefox stupidity 180 | * see: https://bugzilla.mozilla.org/show_bug.cgi?id=645075 181 | */ 182 | fixFirefoxAnchorBug : function() { 183 | if (document.location.hash) 184 | window.setTimeout(function() { 185 | document.location.href += ''; 186 | }, 10); 187 | }, 188 | 189 | /** 190 | * highlight the search words provided in the url in the text 191 | */ 192 | highlightSearchWords : function() { 193 | var params = $.getQueryParameters(); 194 | var terms = (params.highlight) ? params.highlight[0].split(/\s+/) : []; 195 | if (terms.length) { 196 | var body = $('div.body'); 197 | if (!body.length) { 198 | body = $('body'); 199 | } 200 | window.setTimeout(function() { 201 | $.each(terms, function() { 202 | body.highlightText(this.toLowerCase(), 'highlighted'); 203 | }); 204 | }, 10); 205 | $('

    ' + _('Hide Search Matches') + '

    ') 207 | .appendTo($('#searchbox')); 208 | } 209 | }, 210 | 211 | /** 212 | * init the domain index toggle buttons 213 | */ 214 | initIndexTable : function() { 215 | var togglers = $('img.toggler').click(function() { 216 | var src = $(this).attr('src'); 217 | var idnum = $(this).attr('id').substr(7); 218 | $('tr.cg-' + idnum).toggle(); 219 | if (src.substr(-9) == 'minus.png') 220 | $(this).attr('src', src.substr(0, src.length-9) + 'plus.png'); 221 | else 222 | $(this).attr('src', src.substr(0, src.length-8) + 'minus.png'); 223 | }).css('display', ''); 224 | if (DOCUMENTATION_OPTIONS.COLLAPSE_INDEX) { 225 | togglers.click(); 226 | } 227 | }, 228 | 229 | /** 230 | * helper function to hide the search marks again 231 | */ 232 | hideSearchWords : function() { 233 | $('#searchbox .highlight-link').fadeOut(300); 234 | $('span.highlighted').removeClass('highlighted'); 235 | }, 236 | 237 | /** 238 | * make the url absolute 239 | */ 240 | makeURL : function(relativeURL) { 241 | return DOCUMENTATION_OPTIONS.URL_ROOT + '/' + relativeURL; 242 | }, 243 | 244 | /** 245 | * get the current relative url 246 | */ 247 | getCurrentURL : function() { 248 | var path = document.location.pathname; 249 | var parts = path.split(/\//); 250 | $.each(DOCUMENTATION_OPTIONS.URL_ROOT.split(/\//), function() { 251 | if (this == '..') 252 | parts.pop(); 253 | }); 254 | var url = parts.join('/'); 255 | return path.substring(url.lastIndexOf('/') + 1, path.length - 1); 256 | }, 257 | 258 | initOnKeyListeners: function() { 259 | $(document).keyup(function(event) { 260 | var activeElementType = document.activeElement.tagName; 261 | // don't navigate when in search box or textarea 262 | if (activeElementType !== 'TEXTAREA' && activeElementType !== 'INPUT' && activeElementType !== 'SELECT') { 263 | switch (event.keyCode) { 264 | case 37: // left 265 | var prevHref = $('link[rel="prev"]').prop('href'); 266 | if (prevHref) { 267 | window.location.href = prevHref; 268 | return false; 269 | } 270 | case 39: // right 271 | var nextHref = $('link[rel="next"]').prop('href'); 272 | if (nextHref) { 273 | window.location.href = nextHref; 274 | return false; 275 | } 276 | } 277 | } 278 | }); 279 | } 280 | }; 281 | 282 | // quick alias for translations 283 | _ = Documentation.gettext; 284 | 285 | $(document).ready(function() { 286 | Documentation.init(); 287 | }); -------------------------------------------------------------------------------- /docs/_static/down-pressed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/labwatch/cdd7c55e5741bf4f1216f7b8da8de9fb1480c75b/docs/_static/down-pressed.png -------------------------------------------------------------------------------- /docs/_static/down.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/labwatch/cdd7c55e5741bf4f1216f7b8da8de9fb1480c75b/docs/_static/down.png -------------------------------------------------------------------------------- /docs/_static/file.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/labwatch/cdd7c55e5741bf4f1216f7b8da8de9fb1480c75b/docs/_static/file.png -------------------------------------------------------------------------------- /docs/_static/js/jquery-fix.js: -------------------------------------------------------------------------------- 1 | // No Conflict in later (our) version of jQuery 2 | window.$jqTheme = jQuery.noConflict(true); -------------------------------------------------------------------------------- /docs/_static/minus.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/labwatch/cdd7c55e5741bf4f1216f7b8da8de9fb1480c75b/docs/_static/minus.png -------------------------------------------------------------------------------- /docs/_static/plus.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/labwatch/cdd7c55e5741bf4f1216f7b8da8de9fb1480c75b/docs/_static/plus.png -------------------------------------------------------------------------------- /docs/_static/pygments.css: -------------------------------------------------------------------------------- 1 | .highlight .hll { background-color: #ffffcc } 2 | .highlight { background: #eeffcc; } 3 | .highlight .c { color: #408090; font-style: italic } /* Comment */ 4 | .highlight .err { border: 1px solid #FF0000 } /* Error */ 5 | .highlight .k { color: #007020; font-weight: bold } /* Keyword */ 6 | .highlight .o { color: #666666 } /* Operator */ 7 | .highlight .ch { color: #408090; font-style: italic } /* Comment.Hashbang */ 8 | .highlight .cm { color: #408090; font-style: italic } /* Comment.Multiline */ 9 | .highlight .cp { color: #007020 } /* Comment.Preproc */ 10 | .highlight .cpf { color: #408090; font-style: italic } /* Comment.PreprocFile */ 11 | .highlight .c1 { color: #408090; font-style: italic } /* Comment.Single */ 12 | .highlight .cs { color: #408090; background-color: #fff0f0 } /* Comment.Special */ 13 | .highlight .gd { color: #A00000 } /* Generic.Deleted */ 14 | .highlight .ge { font-style: italic } /* Generic.Emph */ 15 | .highlight .gr { color: #FF0000 } /* Generic.Error */ 16 | .highlight .gh { color: #000080; font-weight: bold } /* Generic.Heading */ 17 | .highlight .gi { color: #00A000 } /* Generic.Inserted */ 18 | .highlight .go { color: #333333 } /* Generic.Output */ 19 | .highlight .gp { color: #c65d09; font-weight: bold } /* Generic.Prompt */ 20 | .highlight .gs { font-weight: bold } /* Generic.Strong */ 21 | .highlight .gu { color: #800080; font-weight: bold } /* Generic.Subheading */ 22 | .highlight .gt { color: #0044DD } /* Generic.Traceback */ 23 | .highlight .kc { color: #007020; font-weight: bold } /* Keyword.Constant */ 24 | .highlight .kd { color: #007020; font-weight: bold } /* Keyword.Declaration */ 25 | .highlight .kn { color: #007020; font-weight: bold } /* Keyword.Namespace */ 26 | .highlight .kp { color: #007020 } /* Keyword.Pseudo */ 27 | .highlight .kr { color: #007020; font-weight: bold } /* Keyword.Reserved */ 28 | .highlight .kt { color: #902000 } /* Keyword.Type */ 29 | .highlight .m { color: #208050 } /* Literal.Number */ 30 | .highlight .s { color: #4070a0 } /* Literal.String */ 31 | .highlight .na { color: #4070a0 } /* Name.Attribute */ 32 | .highlight .nb { color: #007020 } /* Name.Builtin */ 33 | .highlight .nc { color: #0e84b5; font-weight: bold } /* Name.Class */ 34 | .highlight .no { color: #60add5 } /* Name.Constant */ 35 | .highlight .nd { color: #555555; font-weight: bold } /* Name.Decorator */ 36 | .highlight .ni { color: #d55537; font-weight: bold } /* Name.Entity */ 37 | .highlight .ne { color: #007020 } /* Name.Exception */ 38 | .highlight .nf { color: #06287e } /* Name.Function */ 39 | .highlight .nl { color: #002070; font-weight: bold } /* Name.Label */ 40 | .highlight .nn { color: #0e84b5; font-weight: bold } /* Name.Namespace */ 41 | .highlight .nt { color: #062873; font-weight: bold } /* Name.Tag */ 42 | .highlight .nv { color: #bb60d5 } /* Name.Variable */ 43 | .highlight .ow { color: #007020; font-weight: bold } /* Operator.Word */ 44 | .highlight .w { color: #bbbbbb } /* Text.Whitespace */ 45 | .highlight .mb { color: #208050 } /* Literal.Number.Bin */ 46 | .highlight .mf { color: #208050 } /* Literal.Number.Float */ 47 | .highlight .mh { color: #208050 } /* Literal.Number.Hex */ 48 | .highlight .mi { color: #208050 } /* Literal.Number.Integer */ 49 | .highlight .mo { color: #208050 } /* Literal.Number.Oct */ 50 | .highlight .sa { color: #4070a0 } /* Literal.String.Affix */ 51 | .highlight .sb { color: #4070a0 } /* Literal.String.Backtick */ 52 | .highlight .sc { color: #4070a0 } /* Literal.String.Char */ 53 | .highlight .dl { color: #4070a0 } /* Literal.String.Delimiter */ 54 | .highlight .sd { color: #4070a0; font-style: italic } /* Literal.String.Doc */ 55 | .highlight .s2 { color: #4070a0 } /* Literal.String.Double */ 56 | .highlight .se { color: #4070a0; font-weight: bold } /* Literal.String.Escape */ 57 | .highlight .sh { color: #4070a0 } /* Literal.String.Heredoc */ 58 | .highlight .si { color: #70a0d0; font-style: italic } /* Literal.String.Interpol */ 59 | .highlight .sx { color: #c65d09 } /* Literal.String.Other */ 60 | .highlight .sr { color: #235388 } /* Literal.String.Regex */ 61 | .highlight .s1 { color: #4070a0 } /* Literal.String.Single */ 62 | .highlight .ss { color: #517918 } /* Literal.String.Symbol */ 63 | .highlight .bp { color: #007020 } /* Name.Builtin.Pseudo */ 64 | .highlight .fm { color: #06287e } /* Name.Function.Magic */ 65 | .highlight .vc { color: #bb60d5 } /* Name.Variable.Class */ 66 | .highlight .vg { color: #bb60d5 } /* Name.Variable.Global */ 67 | .highlight .vi { color: #bb60d5 } /* Name.Variable.Instance */ 68 | .highlight .vm { color: #bb60d5 } /* Name.Variable.Magic */ 69 | .highlight .il { color: #208050 } /* Literal.Number.Integer.Long */ -------------------------------------------------------------------------------- /docs/_static/underscore.js: -------------------------------------------------------------------------------- 1 | // Underscore.js 1.3.1 2 | // (c) 2009-2012 Jeremy Ashkenas, DocumentCloud Inc. 3 | // Underscore is freely distributable under the MIT license. 4 | // Portions of Underscore are inspired or borrowed from Prototype, 5 | // Oliver Steele's Functional, and John Resig's Micro-Templating. 6 | // For all details and documentation: 7 | // http://documentcloud.github.com/underscore 8 | (function(){function q(a,c,d){if(a===c)return a!==0||1/a==1/c;if(a==null||c==null)return a===c;if(a._chain)a=a._wrapped;if(c._chain)c=c._wrapped;if(a.isEqual&&b.isFunction(a.isEqual))return a.isEqual(c);if(c.isEqual&&b.isFunction(c.isEqual))return c.isEqual(a);var e=l.call(a);if(e!=l.call(c))return false;switch(e){case "[object String]":return a==String(c);case "[object Number]":return a!=+a?c!=+c:a==0?1/a==1/c:a==+c;case "[object Date]":case "[object Boolean]":return+a==+c;case "[object RegExp]":return a.source== 9 | c.source&&a.global==c.global&&a.multiline==c.multiline&&a.ignoreCase==c.ignoreCase}if(typeof a!="object"||typeof c!="object")return false;for(var f=d.length;f--;)if(d[f]==a)return true;d.push(a);var f=0,g=true;if(e=="[object Array]"){if(f=a.length,g=f==c.length)for(;f--;)if(!(g=f in a==f in c&&q(a[f],c[f],d)))break}else{if("constructor"in a!="constructor"in c||a.constructor!=c.constructor)return false;for(var h in a)if(b.has(a,h)&&(f++,!(g=b.has(c,h)&&q(a[h],c[h],d))))break;if(g){for(h in c)if(b.has(c, 10 | h)&&!f--)break;g=!f}}d.pop();return g}var r=this,G=r._,n={},k=Array.prototype,o=Object.prototype,i=k.slice,H=k.unshift,l=o.toString,I=o.hasOwnProperty,w=k.forEach,x=k.map,y=k.reduce,z=k.reduceRight,A=k.filter,B=k.every,C=k.some,p=k.indexOf,D=k.lastIndexOf,o=Array.isArray,J=Object.keys,s=Function.prototype.bind,b=function(a){return new m(a)};if(typeof exports!=="undefined"){if(typeof module!=="undefined"&&module.exports)exports=module.exports=b;exports._=b}else r._=b;b.VERSION="1.3.1";var j=b.each= 11 | b.forEach=function(a,c,d){if(a!=null)if(w&&a.forEach===w)a.forEach(c,d);else if(a.length===+a.length)for(var e=0,f=a.length;e2;a== 12 | null&&(a=[]);if(y&&a.reduce===y)return e&&(c=b.bind(c,e)),f?a.reduce(c,d):a.reduce(c);j(a,function(a,b,i){f?d=c.call(e,d,a,b,i):(d=a,f=true)});if(!f)throw new TypeError("Reduce of empty array with no initial value");return d};b.reduceRight=b.foldr=function(a,c,d,e){var f=arguments.length>2;a==null&&(a=[]);if(z&&a.reduceRight===z)return e&&(c=b.bind(c,e)),f?a.reduceRight(c,d):a.reduceRight(c);var g=b.toArray(a).reverse();e&&!f&&(c=b.bind(c,e));return f?b.reduce(g,c,d,e):b.reduce(g,c)};b.find=b.detect= 13 | function(a,c,b){var e;E(a,function(a,g,h){if(c.call(b,a,g,h))return e=a,true});return e};b.filter=b.select=function(a,c,b){var e=[];if(a==null)return e;if(A&&a.filter===A)return a.filter(c,b);j(a,function(a,g,h){c.call(b,a,g,h)&&(e[e.length]=a)});return e};b.reject=function(a,c,b){var e=[];if(a==null)return e;j(a,function(a,g,h){c.call(b,a,g,h)||(e[e.length]=a)});return e};b.every=b.all=function(a,c,b){var e=true;if(a==null)return e;if(B&&a.every===B)return a.every(c,b);j(a,function(a,g,h){if(!(e= 14 | e&&c.call(b,a,g,h)))return n});return e};var E=b.some=b.any=function(a,c,d){c||(c=b.identity);var e=false;if(a==null)return e;if(C&&a.some===C)return a.some(c,d);j(a,function(a,b,h){if(e||(e=c.call(d,a,b,h)))return n});return!!e};b.include=b.contains=function(a,c){var b=false;if(a==null)return b;return p&&a.indexOf===p?a.indexOf(c)!=-1:b=E(a,function(a){return a===c})};b.invoke=function(a,c){var d=i.call(arguments,2);return b.map(a,function(a){return(b.isFunction(c)?c||a:a[c]).apply(a,d)})};b.pluck= 15 | function(a,c){return b.map(a,function(a){return a[c]})};b.max=function(a,c,d){if(!c&&b.isArray(a))return Math.max.apply(Math,a);if(!c&&b.isEmpty(a))return-Infinity;var e={computed:-Infinity};j(a,function(a,b,h){b=c?c.call(d,a,b,h):a;b>=e.computed&&(e={value:a,computed:b})});return e.value};b.min=function(a,c,d){if(!c&&b.isArray(a))return Math.min.apply(Math,a);if(!c&&b.isEmpty(a))return Infinity;var e={computed:Infinity};j(a,function(a,b,h){b=c?c.call(d,a,b,h):a;bd?1:0}),"value")};b.groupBy=function(a,c){var d={},e=b.isFunction(c)?c:function(a){return a[c]};j(a,function(a,b){var c=e(a,b);(d[c]||(d[c]=[])).push(a)});return d};b.sortedIndex=function(a, 17 | c,d){d||(d=b.identity);for(var e=0,f=a.length;e>1;d(a[g])=0})})};b.difference=function(a){var c=b.flatten(i.call(arguments,1));return b.filter(a,function(a){return!b.include(c,a)})};b.zip=function(){for(var a=i.call(arguments),c=b.max(b.pluck(a,"length")),d=Array(c),e=0;e=0;d--)b=[a[d].apply(this,b)];return b[0]}}; 24 | b.after=function(a,b){return a<=0?b():function(){if(--a<1)return b.apply(this,arguments)}};b.keys=J||function(a){if(a!==Object(a))throw new TypeError("Invalid object");var c=[],d;for(d in a)b.has(a,d)&&(c[c.length]=d);return c};b.values=function(a){return b.map(a,b.identity)};b.functions=b.methods=function(a){var c=[],d;for(d in a)b.isFunction(a[d])&&c.push(d);return c.sort()};b.extend=function(a){j(i.call(arguments,1),function(b){for(var d in b)a[d]=b[d]});return a};b.defaults=function(a){j(i.call(arguments, 25 | 1),function(b){for(var d in b)a[d]==null&&(a[d]=b[d])});return a};b.clone=function(a){return!b.isObject(a)?a:b.isArray(a)?a.slice():b.extend({},a)};b.tap=function(a,b){b(a);return a};b.isEqual=function(a,b){return q(a,b,[])};b.isEmpty=function(a){if(b.isArray(a)||b.isString(a))return a.length===0;for(var c in a)if(b.has(a,c))return false;return true};b.isElement=function(a){return!!(a&&a.nodeType==1)};b.isArray=o||function(a){return l.call(a)=="[object Array]"};b.isObject=function(a){return a===Object(a)}; 26 | b.isArguments=function(a){return l.call(a)=="[object Arguments]"};if(!b.isArguments(arguments))b.isArguments=function(a){return!(!a||!b.has(a,"callee"))};b.isFunction=function(a){return l.call(a)=="[object Function]"};b.isString=function(a){return l.call(a)=="[object String]"};b.isNumber=function(a){return l.call(a)=="[object Number]"};b.isNaN=function(a){return a!==a};b.isBoolean=function(a){return a===true||a===false||l.call(a)=="[object Boolean]"};b.isDate=function(a){return l.call(a)=="[object Date]"}; 27 | b.isRegExp=function(a){return l.call(a)=="[object RegExp]"};b.isNull=function(a){return a===null};b.isUndefined=function(a){return a===void 0};b.has=function(a,b){return I.call(a,b)};b.noConflict=function(){r._=G;return this};b.identity=function(a){return a};b.times=function(a,b,d){for(var e=0;e/g,">").replace(/"/g,""").replace(/'/g,"'").replace(/\//g,"/")};b.mixin=function(a){j(b.functions(a), 28 | function(c){K(c,b[c]=a[c])})};var L=0;b.uniqueId=function(a){var b=L++;return a?a+b:b};b.templateSettings={evaluate:/<%([\s\S]+?)%>/g,interpolate:/<%=([\s\S]+?)%>/g,escape:/<%-([\s\S]+?)%>/g};var t=/.^/,u=function(a){return a.replace(/\\\\/g,"\\").replace(/\\'/g,"'")};b.template=function(a,c){var d=b.templateSettings,d="var __p=[],print=function(){__p.push.apply(__p,arguments);};with(obj||{}){__p.push('"+a.replace(/\\/g,"\\\\").replace(/'/g,"\\'").replace(d.escape||t,function(a,b){return"',_.escape("+ 29 | u(b)+"),'"}).replace(d.interpolate||t,function(a,b){return"',"+u(b)+",'"}).replace(d.evaluate||t,function(a,b){return"');"+u(b).replace(/[\r\n\t]/g," ")+";__p.push('"}).replace(/\r/g,"\\r").replace(/\n/g,"\\n").replace(/\t/g,"\\t")+"');}return __p.join('');",e=new Function("obj","_",d);return c?e(c,b):function(a){return e.call(this,a,b)}};b.chain=function(a){return b(a).chain()};var m=function(a){this._wrapped=a};b.prototype=m.prototype;var v=function(a,c){return c?b(a).chain():a},K=function(a,c){m.prototype[a]= 30 | function(){var a=i.call(arguments);H.call(a,this._wrapped);return v(c.apply(b,a),this._chain)}};b.mixin(b);j("pop,push,reverse,shift,sort,splice,unshift".split(","),function(a){var b=k[a];m.prototype[a]=function(){var d=this._wrapped;b.apply(d,arguments);var e=d.length;(a=="shift"||a=="splice")&&e===0&&delete d[0];return v(d,this._chain)}});j(["concat","join","slice"],function(a){var b=k[a];m.prototype[a]=function(){return v(b.apply(this._wrapped,arguments),this._chain)}});m.prototype.chain=function(){this._chain= 31 | true;return this};m.prototype.value=function(){return this._wrapped}}).call(this); 32 | -------------------------------------------------------------------------------- /docs/_static/up-pressed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/labwatch/cdd7c55e5741bf4f1216f7b8da8de9fb1480c75b/docs/_static/up-pressed.png -------------------------------------------------------------------------------- /docs/_static/up.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/labwatch/cdd7c55e5741bf4f1216f7b8da8de9fb1480c75b/docs/_static/up.png -------------------------------------------------------------------------------- /docs/genindex.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | Index — labwatch documentation 9 | 10 | 11 | 12 | 13 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 114 | 115 |
    116 |
    117 |
    118 | 120 |
    121 |
    122 | 123 | 124 |

    Index

    125 | 126 |
    127 | B 128 | | C 129 | | D 130 | | G 131 | | L 132 | | N 133 | | O 134 | | P 135 | | R 136 | | S 137 | | U 138 | 139 |
    140 |

    B

    141 | 142 | 146 | 150 |
    151 | 152 |

    C

    153 | 154 | 162 | 170 |
    171 | 172 |

    D

    173 | 174 | 178 | 182 |
    183 | 184 |

    G

    185 | 186 | 190 |
    191 | 192 |

    L

    193 | 194 | 198 | 204 |
    205 | 206 |

    N

    207 | 208 | 212 |
    213 | 214 |

    O

    215 | 216 | 220 |
    221 | 222 |

    P

    223 | 224 | 228 |
    229 | 230 |

    R

    231 | 232 | 236 |
    237 | 238 |

    S

    239 | 240 | 246 | 252 |
    253 | 254 |

    U

    255 | 256 | 262 | 268 |
    269 | 270 | 271 | 272 |
    273 | 274 |
    275 |
    276 |
    277 |
    278 |

    279 | Back to top 280 | 281 |
    282 | 283 | 284 |

    285 |

    286 | © Copyright 2017, Aaron Klein, Klaus Greff.
    287 | Created using Sphinx 1.6.2.
    288 |

    289 |
    290 |
    291 | 292 | -------------------------------------------------------------------------------- /docs/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | Welcome to labwatch’s documentation! — labwatch documentation 8 | 9 | 10 | 11 | 12 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 124 | 125 |
    126 |
    127 |
    128 | 136 |
    137 |
    138 | 139 |
    140 |

    Welcome to labwatch’s documentation!

    141 |

    Labwatch is an extension to Sacred for automated hyperparameter optimization 142 | of machine learning algorithm. It implements an interface to state-of-the-art hyperparameter optimization methods 143 | such as RoBO or SMAC.

    144 |
    145 |

    Contents

    146 |
    147 | 167 |
    168 |
    169 |
    170 | 171 | 172 |
    173 | 174 |
    175 |
    176 |
    177 |
    178 |

    179 | Back to top 180 | 181 |
    182 | 183 |

    187 | 188 |

    189 |

    190 | © Copyright 2017, Aaron Klein, Klaus Greff.
    191 | Created using Sphinx 1.6.2.
    192 |

    193 |
    194 |
    195 | 196 | -------------------------------------------------------------------------------- /docs/installation.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | Installation — labwatch documentation 8 | 9 | 10 | 11 | 12 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 126 | 127 |
    128 |
    129 |
    130 | 135 |
    136 |
    137 | 138 |
    139 |

    Installation

    140 |

    To install the current version of labwatch from git execute the following commands:

    141 | 
    git clone https://github.com/automl/labwatch.git
142 | cd labwatch
143 | pip install .
144 |
    145 |
    146 |

    Before you can use Bayesian optimization, you need to install RoBO first. 147 | Have a look here to see how you can install it.

    148 |

    If you want to use SMAC, you have to install it first from pypi:

    149 | 
    pip install smac
150 |
    151 |
    152 |

    In case you want to install manually, look here

    153 |
    154 | 155 | 156 |
    157 | 158 |
    159 |
    160 |
    161 |
    162 |

    163 | Back to top 164 | 165 |
    166 | 167 |

    171 | 172 |

    173 |

    174 | © Copyright 2017, Aaron Klein, Klaus Greff.
    175 | Created using Sphinx 1.6.2.
    176 |

    177 |
    178 |
    179 | 180 | -------------------------------------------------------------------------------- /docs/objects.inv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/labwatch/cdd7c55e5741bf4f1216f7b8da8de9fb1480c75b/docs/objects.inv -------------------------------------------------------------------------------- /docs/py-modindex.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | Python Module Index — labwatch documentation 8 | 9 | 10 | 11 | 12 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 116 | 117 |
    118 |
    119 |
    120 | 122 |
    123 |
    124 | 125 | 126 |

    Python Module Index

    127 | 128 |
    129 | l 130 |
    131 | 132 | 133 | 134 | 136 | 137 | 139 | 142 | 143 | 144 | 147 | 148 | 149 | 152 |
     
    135 | l
    140 | labwatch 141 |
        145 | labwatch.converters.convert_to_configspace 146 |
        150 | labwatch.hyperparameters 151 |
    153 | 154 | 155 |
    156 | 157 |
    158 |
    159 |
    160 |
    161 |

    162 | Back to top 163 | 164 |
    165 | 166 | 167 |

    168 |

    169 | © Copyright 2017, Aaron Klein, Klaus Greff.
    170 | Created using Sphinx 1.6.2.
    171 |

    172 |
    173 |
    174 | 175 | -------------------------------------------------------------------------------- /docs/search.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | Search — labwatch documentation 8 | 9 | 10 | 11 | 12 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 114 | 115 |
    116 |
    117 |
    118 | 120 |
    121 |
    122 | 123 |

    Search

    124 |
    125 | 126 |

    127 | Please activate JavaScript to enable the search 128 | functionality. 129 |

    130 |
    131 |

    132 | From here you can search these documents. Enter your search 133 | words into the box below and click "search". Note that the search 134 | function will automatically search for all of the words. Pages 135 | containing fewer words won't appear in the result list. 136 |

    137 | 138 | 139 |
    140 |
    141 | 142 |
    143 | 144 | 145 |
    146 | 147 | 148 | 149 |
    150 | 151 |
    152 | 153 |
    154 | 155 |
    156 |
    157 |
    158 |
    159 |

    160 | Back to top 161 | 162 |
    163 | 164 | 165 |

    166 |

    167 | © Copyright 2017, Aaron Klein, Klaus Greff.
    168 | Created using Sphinx 1.6.2.
    169 |

    170 |
    171 |
    172 | 173 | -------------------------------------------------------------------------------- /docs/searchindex.js: -------------------------------------------------------------------------------- 1 | Search.setIndex({docnames:["apis","index","installation","quickstart"],envversion:53,filenames:["apis.rst","index.rst","installation.rst","quickstart.rst"],objects:{"labwatch.assistant":{LabAssistant:[0,0,1,""]},"labwatch.assistant.LabAssistant":{search_space:[0,1,1,""]},"labwatch.converters":{convert_to_configspace:[0,2,0,"-"]},"labwatch.converters.convert_to_configspace":{configspace_config_to_sacred:[0,3,1,""],convert_simple_param:[0,3,1,""],sacred_config_to_configspace:[0,3,1,""],sacred_space_to_configspace:[0,3,1,""]},"labwatch.hyperparameters":{Categorical:[0,0,1,""],Condition:[0,0,1,""],ConditionResult:[0,0,1,""],Constant:[0,0,1,""],Gaussian:[0,0,1,""],Parameter:[0,0,1,""],UniformFloat:[0,0,1,""],UniformInt:[0,0,1,""],UniformNumber:[0,0,1,""],decode_param_or_op:[0,3,1,""]},"labwatch.optimizers.base":{Optimizer:[0,0,1,""]},"labwatch.optimizers.base.Optimizer":{needs_updates:[0,1,1,""],suggest_configuration:[0,1,1,""],update:[0,1,1,""]},"labwatch.optimizers.bayesian_optimization":{BayesianOptimization:[0,0,1,""]},"labwatch.optimizers.bohamiann":{Bohamiann:[0,0,1,""]},"labwatch.optimizers.dngo":{DNGOWrapper:[0,0,1,""]},"labwatch.optimizers.random_search":{RandomSearch:[0,0,1,""]},labwatch:{hyperparameters:[0,2,0,"-"]}},objnames:{"0":["py","class","Python class"],"1":["py","method","Python method"],"2":["py","module","Python module"],"3":["py","function","Python function"]},objtypes:{"0":"py:class","1":"py:method","2":"py:module","3":"py:function"},terms:{"10e":3,"case":[2,3],"class":[0,3],"default":[0,3],"float":0,"function":[0,3],"import":3,"int":3,"long":3,"new":[0,3],"return":[0,3],"true":[0,3],"try":[0,3],For:3,The:[0,3],about:[0,3],accuraci:3,activ:3,actual:3,adapt:3,add:3,addit:0,after:3,algorithm:1,all:[0,3],allow:3,alon:3,alreadi:3,also:3,always_inject_observ:0,api:1,art:[1,3],assist:[1,3],associ:0,assum:3,astyp:3,autom:[1,3],automain:3,automat:1,automl:[2,3],base:[0,3],basic:3,batch:3,batch_siz:3,bayesian:[2,3],bayesian_optim:0,bayesianoptim:0,befor:2,believ:3,better:3,between:3,binari:3,bit:3,bohamiann:0,bool:0,both:3,bound:3,brief:3,bson:0,burnin:0,call:3,can:[0,2,3],captur:3,cast:3,categor:[0,3],categorical_crossentropi:3,cfg:3,chain_length:0,check:0,choic:[0,3],choices_in:0,clone:2,code:3,com:2,command:[2,3],common:3,compil:3,complet:3,condit:0,conditionresult:0,config:[0,3],config_spac:0,configspac:0,configspace_config_to_sacr:0,configur:[0,1],configurationspac:0,connect:3,constant:0,contain:0,continu:3,conveni:3,convert:[1,3],convert_simple_param:0,convert_to_configspac:0,correspond:3,cost:0,creat:0,cspace:0,current:[2,3],dai:3,dark:3,data:3,databas:[0,3],database_nam:0,dataset:3,decod:0,decode_param_or_op:0,decor:0,def:3,defin:[0,3],definit:0,dens:3,depend:3,describ:0,detail:3,dict:[0,3],dictionari:0,differ:[0,3],dimension:3,discret:3,distinguish:3,distribut:[0,3],dngo:0,dngowrapp:0,done:0,drawn:3,dropout:3,dropout_first_lay:3,dropout_second_lay:3,each:[0,3],earli:3,easi:3,either:3,els:0,emerg:3,epoch:3,equival:0,evalu:3,everi:3,exampl:3,execut:2,exist:3,experi:[0,3],exploit:3,explor:3,extens:1,fals:0,feed:3,fill:0,find:3,first:[2,3],fist:3,fit:3,fix:[0,3],float32:3,follow:[2,3],fool:3,forest:3,forward:3,framework:3,from:[0,2,3],furthermor:3,gaussian:[0,3],gener:3,get:3,get_random_config:0,git:2,github:2,give:3,given:0,good:3,guid:3,has:[0,3],have:[2,3],here:[2,3],high:3,histori:3,how:[2,3],http:2,hyperparamet:1,idea:3,implement:[1,3],inconsist:3,independ:3,inform:[0,3],input:3,input_shap:3,instal:1,instanc:[0,3],instanti:3,instead:[0,3],integ:3,interfac:[0,1,3],intern:0,iter:3,its:3,just:[0,3],keep:3,kera:3,knowledg:3,labassist:[0,3],labwatch:[0,2,3],labwatch_demo_kera:3,layer:3,learn:[1,3],learning_r:3,let:3,like:0,line:3,list:[0,3],littl:3,load_data:3,localhost:0,log:3,log_scal:[0,3],look:[2,3],loss:3,lot:3,low:3,lower:[0,3],machin:[1,3],main:1,make:3,manual:2,map:0,matric:3,mean:3,method:[0,1,3],metric:3,might:3,mix:3,mnist:3,model:3,modul:1,mongodb:3,more:3,multipl:1,n_hyper:0,n_iter:0,name:[0,3],need:[0,2],needs_upd:0,network:3,neural:3,nice:3,none:0,note:3,noth:0,now:3,num_class:3,num_units_first_lay:3,num_units_second_lay:3,numer:3,object:3,observ:3,off:3,offer:3,old:3,onli:3,optim:[1,2],optimization_target:3,other:3,otherwis:0,our:3,out:3,outperform:3,overview:3,own:3,paper:3,parallel:3,param:0,paramet:0,pass:3,pip:2,point:3,popul:0,prefix:0,prevent:3,previou:3,print:3,prior:3,probabilist:3,probabl:3,problem:3,procedur:3,process:3,properti:3,pypi:2,python:3,queue:0,quickstart:1,quit:3,random:3,random_search:[0,3],randomsearch:[0,3],rate:3,recent:3,refer:3,relev:3,relu:3,reshap:3,result:[0,3],right:3,rmsprop:3,robo:[1,2],run:[0,3],sacr:[1,3],sacred_config_to_configspac:0,sacred_space_to_configspac:0,sampl:3,scale:3,score:3,seachspac:0,search:1,search_spac:[0,3],searchspac:[0,3],second:3,see:[2,3],select:3,sequenti:3,set:3,sever:3,shape:3,shuffl:3,sigma:0,simpl:[0,3],simplest:3,simpli:3,size:3,smac:[1,2,3],small_search_spac:3,softmax:3,some:3,someth:0,sometim:3,sourc:3,space:[0,1],spend:3,split:3,state:[0,1,3],storag:0,store:[0,3],str:0,subfield:3,subset:3,suggest:[0,3],suggest_configur:0,summari:3,tackl:3,take:3,talk:3,test:3,thei:3,thi:[0,3],thing:3,through:3,time:3,to_categor:3,togeth:3,tool:3,track:3,trade:3,train:3,tri:3,tune:1,tutori:3,type:[0,3],uid:0,understand:3,uniform:3,uniformfloat:[0,3],uniformint:0,uniformnumb:[0,3],unsur:3,updat:0,upper:[0,3],url:0,use:[2,3],used:3,useful:3,user:0,uses:[0,3],util:3,validation_data:3,valu:[0,3],varieti:3,vector:3,verbos:3,version:2,via:3,want:[0,2,3],watcher:0,well:3,were:3,where:3,which:3,whole:3,work:3,would:3,x_test:3,x_train:3,y_test:3,y_train:3,you:[0,2,3],your:3},titles:["API","Welcome to labwatch\u2019s documentation!","Installation","Quickstart"],titleterms:{api:0,assist:0,automat:3,configur:3,content:1,convert:0,document:1,hyperparamet:[0,3],instal:2,labwatch:1,main:0,modul:0,multipl:3,optim:[0,3],quickstart:3,search:3,space:3,tune:3,welcom:1}}) -------------------------------------------------------------------------------- /docs/source/apis.rst: -------------------------------------------------------------------------------- 1 | API 2 | === 3 | 4 | 5 | ------------ 6 | Main Modules 7 | ------------ 8 | 9 | Assistant: 10 | ---------- 11 | .. autoclass:: labwatch.assistant.LabAssistant 12 | :members: 13 | 14 | Hyperparameters: 15 | ---------------- 16 | .. automodule:: labwatch.hyperparameters 17 | :members: 18 | 19 | .. autoclass:: labwatch.hyperparameters.Parameter 20 | :members: 21 | 22 | .. autoclass:: labwatch.hyperparameters.Constant 23 | :members: 24 | 25 | .. autoclass:: labwatch.hyperparameters.Categorical 26 | :members: 27 | 28 | .. autoclass:: labwatch.hyperparameters.UniformNumber 29 | :members: 30 | 31 | .. autoclass:: labwatch.hyperparameters.UniformFloat 32 | :members: 33 | 34 | .. autoclass:: labwatch.hyperparameters.UniformInt 35 | :members: 36 | 37 | .. autoclass:: labwatch.hyperparameters.Gaussian 38 | :members: 39 | 40 | .. autoclass:: labwatch.hyperparameters.ConditionResult 41 | :members: 42 | 43 | .. autoclass:: labwatch.hyperparameters.Condition 44 | :members: 45 | 46 | Converters: 47 | ----------- 48 | .. automodule:: labwatch.converters.convert_to_configspace 49 | :members: 50 | 51 | Optimizers: 52 | ----------- 53 | 54 | 55 | .. autoclass:: labwatch.optimizers.base.Optimizer 56 | :members: 57 | 58 | .. autoclass:: labwatch.optimizers.bayesian_optimization.BayesianOptimization 59 | :members: 60 | 61 | .. autoclass:: labwatch.optimizers.bohamiann.Bohamiann 62 | :members: 63 | 64 | .. autoclass:: labwatch.optimizers.dngo.DNGOWrapper 65 | :members: 66 | 67 | .. autoclass:: labwatch.optimizers.random_search.RandomSearch 68 | :members: 69 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # labwatch documentation build configuration file, created by 4 | # sphinx-quickstart on Thu May 25 18:25:22 2017. 5 | # 6 | # This file is execfile()d with the current directory set to its 7 | # containing dir. 8 | # 9 | # Note that not all possible configuration values are present in this 10 | # autogenerated file. 11 | # 12 | # All configuration values have a default; values that are commented out 13 | # serve to show the default. 14 | 15 | # If extensions (or modules to document with autodoc) are in another directory, 16 | # add these directories to sys.path here. If the directory is relative to the 17 | # documentation root, use os.path.abspath to make it absolute, like shown here. 18 | # 19 | # import os 20 | # import sys 21 | import sphinx_bootstrap_theme 22 | # sys.path.insert(0, os.path.abspath('.')) 23 | 24 | 25 | # -- General configuration ------------------------------------------------ 26 | 27 | # If your documentation needs a minimal Sphinx version, state it here. 28 | # 29 | # needs_sphinx = '1.0' 30 | 31 | # Add any Sphinx extension module names here, as strings. They can be 32 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 33 | # ones. 34 | extensions = ['sphinx.ext.autodoc', 35 | 'sphinx.ext.coverage'] 36 | 37 | # Add any paths that contain templates here, relative to this directory. 38 | templates_path = ['.templates'] 39 | 40 | # The suffix(es) of source filenames. 41 | # You can specify multiple suffix as a list of string: 42 | # 43 | # source_suffix = ['.rst', '.md'] 44 | source_suffix = '.rst' 45 | 46 | # The master toctree document. 47 | master_doc = 'index' 48 | 49 | # General information about the project. 50 | project = u'labwatch' 51 | copyright = u'2017, Aaron Klein, Klaus Greff' 52 | author = u'Aaron Klein, Klaus Greff' 53 | 54 | # The version info for the project you're documenting, acts as replacement for 55 | # |version| and |release|, also used in various other places throughout the 56 | # built documents. 57 | # 58 | # The short X.Y version. 59 | version = u'' 60 | # The full version, including alpha/beta/rc tags. 61 | release = u'' 62 | 63 | # The language for content autogenerated by Sphinx. Refer to documentation 64 | # for a list of supported languages. 65 | # 66 | # This is also used if you do content translation via gettext catalogs. 67 | # Usually you set "language" from the command line for these cases. 68 | language = None 69 | 70 | # List of patterns, relative to source directory, that match files and 71 | # directories to ignore when looking for source files. 72 | # This patterns also effect to html_static_path and html_extra_path 73 | exclude_patterns = ['.build', 'Thumbs.db', '.DS_Store'] 74 | 75 | # The name of the Pygments (syntax highlighting) style to use. 76 | pygments_style = 'sphinx' 77 | 78 | # If true, `todo` and `todoList` produce output, else they produce nothing. 79 | todo_include_todos = False 80 | 81 | 82 | # -- Options for HTML output ---------------------------------------------- 83 | 84 | # The theme to use for HTML and HTML Help pages. See the documentation for 85 | # a list of builtin themes. 86 | # 87 | #html_theme = 'alabaster' 88 | html_theme = 'bootstrap' 89 | html_theme_path = sphinx_bootstrap_theme.get_html_theme_path() 90 | 91 | 92 | # Theme options are theme-specific and customize the look and feel of a theme 93 | # further. For a list of options available for each theme, see the 94 | # documentation. 95 | # 96 | html_theme_options = { 97 | # Navigation bar title. (Default: ``project`` value) 98 | 'navbar_title': "Labwatch", 99 | 100 | # Tab name for entire site. (Default: "Site") 101 | # 'navbar_site_name': "Site", 102 | 103 | # A list of tuples containting pages to link to. The value should 104 | # be in the form [(name, page), ..] 105 | 'navbar_links': [ 106 | ('Start', 'index'), 107 | ('Installation', 'installation'), 108 | ('Quickstart', 'quickstart'), 109 | ('APIs', 'apis') 110 | ], 111 | 112 | # Render the next and previous page links in navbar. (Default: true) 113 | 'navbar_sidebarrel': True, 114 | 115 | # Render the current pages TOC in the navbar. (Default: true) 116 | 'navbar_pagenav': True, 117 | 118 | # Tab name for the current pages TOC. (Default: "Page") 119 | 'navbar_pagenav_name': "On this page", 120 | 121 | # Global TOC depth for "site" navbar tab. (Default: 1) 122 | # Switching to -1 shows all levels. 123 | 'globaltoc_depth': 1, 124 | 125 | # Include hidden TOCs in Site navbar? 126 | # 127 | # Note: If this is "false", you cannot have mixed ``:hidden:`` and 128 | # non-hidden ``toctree`` directives in the same page, or else the build 129 | # will break. 130 | # 131 | # Values: "true" (default) or "false" 132 | 'globaltoc_includehidden': "false", 133 | 134 | # HTML navbar class (Default: "navbar") to attach to
    element. 135 | # For black navbar, do "navbar navbar-inverse" 136 | 'navbar_class': "navbar", 137 | 138 | # Fix navigation bar to top of page? 139 | # Values: "true" (default) or "false" 140 | 'navbar_fixed_top': "true", 141 | 142 | # Location of link to source. 143 | # Options are "nav" (default), "footer" or anything else to exclude. 144 | 'source_link_position': "footer", 145 | 146 | # Bootswatch (http://bootswatch.com/) theme. 147 | # 148 | # Options are nothing with "" (default) or the name of a valid theme 149 | # such as "amelia" or "cosmo". 150 | 'bootswatch_theme': "cosmo", 151 | 152 | # Choose Bootstrap version. 153 | # Values: "3" (default) or "2" (in quotes) 154 | 'bootstrap_version': "3", 155 | } 156 | 157 | html_sidebars = {'**': ['localtoc.html']} 158 | 159 | # Add any paths that contain custom static files (such as style sheets) here, 160 | # relative to this directory. They are copied after the builtin static files, 161 | # so a file named "default.css" will overwrite the builtin "default.css". 162 | html_static_path = ['.static'] 163 | 164 | 165 | # -- Options for HTMLHelp output ------------------------------------------ 166 | 167 | # Output file base name for HTML help builder. 168 | htmlhelp_basename = 'labwatchdoc' 169 | 170 | 171 | # -- Options for LaTeX output --------------------------------------------- 172 | 173 | latex_elements = { 174 | # The paper size ('letterpaper' or 'a4paper'). 175 | # 176 | # 'papersize': 'letterpaper', 177 | 178 | # The font size ('10pt', '11pt' or '12pt'). 179 | # 180 | # 'pointsize': '10pt', 181 | 182 | # Additional stuff for the LaTeX preamble. 183 | # 184 | # 'preamble': '', 185 | 186 | # Latex figure (float) alignment 187 | # 188 | # 'figure_align': 'htbp', 189 | } 190 | 191 | # Grouping the document tree into LaTeX files. List of tuples 192 | # (source start file, target name, title, 193 | # author, documentclass [howto, manual, or own class]). 194 | latex_documents = [ 195 | (master_doc, 'labwatch.tex', u'labwatch Documentation', 196 | u'Aaron Klein, Klaus Greff', 'manual'), 197 | ] 198 | 199 | 200 | # -- Options for manual page output --------------------------------------- 201 | 202 | # One entry per manual page. List of tuples 203 | # (source start file, name, description, authors, manual section). 204 | man_pages = [ 205 | (master_doc, 'labwatch', u'labwatch Documentation', 206 | [author], 1) 207 | ] 208 | 209 | 210 | # -- Options for Texinfo output ------------------------------------------- 211 | 212 | # Grouping the document tree into Texinfo files. List of tuples 213 | # (source start file, target name, title, author, 214 | # dir menu entry, description, category) 215 | texinfo_documents = [ 216 | (master_doc, 'labwatch', u'labwatch Documentation', 217 | author, 'labwatch', 'One line description of project.', 218 | 'Miscellaneous'), 219 | ] 220 | 221 | 222 | 223 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | .. labwatch documentation master file, created by 2 | sphinx-quickstart on Thu May 25 18:25:22 2017. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | ==================================== 6 | Welcome to labwatch's documentation! 7 | ==================================== 8 | 9 | Labwatch is an extension to `Sacred `_ for automated hyperparameter optimization 10 | of machine learning algorithm. It implements an interface to state-of-the-art hyperparameter optimization methods 11 | such as `RoBO `_ or `SMAC `_. 12 | 13 | 14 | 15 | Contents 16 | -------- 17 | 18 | .. toctree:: 19 | :maxdepth: 3 20 | 21 | installation 22 | quickstart 23 | apis 24 | -------------------------------------------------------------------------------- /docs/source/installation.rst: -------------------------------------------------------------------------------- 1 | Installation 2 | ============ 3 | 4 | 5 | To install the current version of labwatch from git execute the following commands: 6 | 7 | .. code:: bash 8 | 9 | git clone https://github.com/automl/labwatch.git 10 | cd labwatch 11 | pip install . 12 | 13 | 14 | Before you can use Bayesian optimization, you need to install `RoBO `_ first. 15 | Have a look `here `_ to see how you can install it. 16 | 17 | If you want to use `SMAC `_, you have to install it first from pypi: 18 | 19 | .. code:: bash 20 | 21 | pip install smac 22 | 23 | In case you want to install manually, look `here `_ 24 | 25 | -------------------------------------------------------------------------------- /docs/source/quickstart.rst: -------------------------------------------------------------------------------- 1 | Quickstart 2 | ********** 3 | 4 | The following tutorial assumes that you have a basic understanding of how Sacred works (Experiments, Observers). 5 | In case you are unsure, have a look on the `Sacred Quickstart Guide `_ 6 | 7 | In this tutorial we will see how we can optimize the hyperparameters of a feed forward network trained on MNIST 8 | together with Labwatch and Sacred. 9 | We will use the MNIST example of `keras `_ to implement the neural network but note that 10 | both Labwatch and Sacred are completely independent of which framework you use. 11 | You can find the whole source code as well as more examples 12 | `here `_ 13 | 14 | Automatically Tuning of Hyperparameters 15 | ======================================= 16 | 17 | 18 | Sacred is a useful tool to keep track of all relevant information of your experiments such as hyperparameters, 19 | results, dependencies and so on. 20 | 21 | The following python code adapts the 22 | `keras mnist example `_ to work with Sacred: 23 | 24 | 25 | .. code:: python 26 | 27 | import keras 28 | from keras.datasets import mnist 29 | from keras.models import Sequential 30 | from keras.layers import Dense, Dropout 31 | from keras.optimizers import RMSprop 32 | 33 | from sacred import Experiment 34 | 35 | ex = Experiment() 36 | 37 | @ex.config 38 | def cfg(): 39 | batch_size = 128 40 | num_units_first_layer = 512 41 | num_units_second_layer = 512 42 | dropout_first_layer = 0.2 43 | dropout_second_layer = 0.2 44 | learning_rate = 0.001 45 | 46 | 47 | @ex.automain 48 | def run(batch_size, 49 | num_units_first_layer, 50 | num_units_second_layer, 51 | dropout_first_layer, 52 | dropout_second_layer, 53 | learning_rate): 54 | 55 | num_classes = 10 56 | epochs = 20 57 | 58 | # the data, shuffled and split between train and test sets 59 | (x_train, y_train), (x_test, y_test) = mnist.load_data() 60 | 61 | x_train = x_train.reshape(60000, 784) 62 | x_test = x_test.reshape(10000, 784) 63 | x_train = x_train.astype('float32') 64 | x_test = x_test.astype('float32') 65 | x_train /= 255 66 | x_test /= 255 67 | print(x_train.shape[0], 'train samples') 68 | print(x_test.shape[0], 'test samples') 69 | 70 | # convert class vectors to binary class matrices 71 | y_train = keras.utils.to_categorical(y_train, num_classes) 72 | y_test = keras.utils.to_categorical(y_test, num_classes) 73 | 74 | model = Sequential() 75 | model.add(Dense(num_units_first_layer, activation='relu', input_shape=(784,))) 76 | model.add(Dropout(dropout_first_layer)) 77 | model.add(Dense(num_units_second_layer, activation='relu')) 78 | model.add(Dropout(dropout_second_layer)) 79 | model.add(Dense(10, activation='softmax')) 80 | 81 | model.summary() 82 | 83 | model.compile(loss='categorical_crossentropy', 84 | optimizer=RMSprop(lr=learning_rate), 85 | metrics=['accuracy']) 86 | 87 | history = model.fit(x_train, y_train, 88 | batch_size=batch_size, 89 | epochs=epochs, 90 | verbose=1, 91 | validation_data=(x_test, y_test)) 92 | score = model.evaluate(x_test, y_test, verbose=0) 93 | print('Test loss:', score[0]) 94 | print('Test accuracy:', score[1]) 95 | 96 | results = dict() 97 | results["optimization_target"] = 1 - score[1] 98 | 99 | return results 100 | 101 | In the dark old days you would probably now spend a lot of time to find the right setting for you hyperparameters by 102 | iteratively trying out different setting. 103 | 104 | Well, you're are not alone with this problem and it is actually a common problem in machine learning. 105 | Recently a new subfield in machine learning (`AutoML `_) 106 | has emerged that tries to automated this procedure by casting it as an optimization problem. By now there exist several 107 | optimization methods that tackle the hyperparameter optimization problem. 108 | 109 | To make use of these methods in Labwatch, we fist have to instantiate a LabAssistant which will 110 | connect our Sacred experiment with the hyperparameter optimizer through a MongoDB: 111 | 112 | .. code:: python 113 | 114 | from labwatch.assistant import LabAssistant 115 | from labwatch.optimizers.random_search import RandomSearch 116 | 117 | a = LabAssistant(ex, "labwatch_demo_keras", optimizer=RandomSearch) 118 | 119 | After that we have to define our configuration search space with the hyperparameters that we want to optimize. 120 | 121 | .. code:: python 122 | 123 | @a.searchspace 124 | def search_space(): 125 | batch_size = UniformNumber(lower=8, 126 | upper=64, 127 | default=32, 128 | type=int, 129 | log_scale=True) 130 | num_units_first_layer = UniformNumber(lower=16, 131 | upper=1024, 132 | default=32, 133 | type=int, 134 | log_scale=True) 135 | num_units_second_layer = UniformNumber(lower=16, 136 | upper=1024, 137 | default=32, 138 | type=int, 139 | log_scale=True) 140 | dropout_first_layer = UniformFloat(lower=0, 141 | upper=.99, 142 | default=.2) 143 | dropout_second_layer = UniformFloat(lower=0, 144 | upper=.99, 145 | default=.2) 146 | learning_rate = UniformFloat(lower=10e-6, 147 | upper=10e-1, 148 | default=10e-2, 149 | log_scale=True) 150 | 151 | 152 | Here, for each hyperparameter we define a prior distribution, default value, its type and if we want 153 | to adapt it on a log scale or not. 154 | 155 | We can use now Sacred's command line interface to get a new configuration 156 | 157 | .. code:: bash 158 | 159 | python experiment.py with search_space 160 | 161 | Labwatch will now pass all already completed configurations that are stored in the database 162 | to the hyperparameter optimizer, let it suggest a new configuration and then run the experiment with this configuration. 163 | 164 | 165 | Configuration Search Spaces 166 | =========================== 167 | 168 | 169 | At this point is probably a good idea to talk a little bit more about configuration spaces. 170 | In general we distinguish between: 171 | 172 | - *categorical* hyperparameters that can take only discrete choices (e.g. {'a', 'b', 'c'}) 173 | - *numerical* hyperparameters that can have either integer or continuous values. 174 | 175 | Furthermore, Labwatch also allows you to define prior distributions (Gaussian, Uniform) for your hyperparameters. 176 | Some hyperparameter optimizers such as for instance random search can exploit this prior knowledge to 177 | suggest better configurations from early on. 178 | In the case that you do not have a prior about you hyperparameter, just use a uniform distribution which is 179 | simply defined by an upper and lower bound. 180 | 181 | 182 | Hyperparameter Optimizers 183 | ========================= 184 | 185 | 186 | Labwatch offers a simple interface to a variety of state-of-the-art hyperparameter optimization methods. 187 | Note that every optimizer has its own properties and might not work for all use cases. 188 | The following list will give you a brief overview of the optimizer that can be used with labwatch and in which 189 | setting they would work. For more details we refer to the corresponding papers: 190 | 191 | - **Random search** is probably the simplest hyperparameter optimization method. It just samples hyperparameter 192 | configurations from the prior. The nice thing with random search is that it works in all search 193 | spaces and is easy to parallelize. 194 | 195 | - **Bayesian optimization** fits a probabilistic model to capture the current believe of the objective function. 196 | To select a new configuration, it use an utility function that only depend on the 197 | probabilistic model to trade off exploration and exploitation. Here we use Gaussian process to model our objective 198 | function, which work well in low (<10) dimensional continuous input spaces but do not work with categorical 199 | hyperparameters. 200 | 201 | - **SMAC** is also a Bayesian optimization method but uses random forest instead of Gaussian processes to model 202 | the objective function. It works in high dimensional mixed continuous and discret input space but will be 203 | be probably outperformed by GP-based Bayesian optimization in the low dimensional continuous space. 204 | 205 | 206 | Multiple Search Spaces 207 | ====================== 208 | 209 | 210 | Sometimes it is quite convenient to have multiple different search space, for instance if you want to optimize 211 | first only a subset of your hyperparameters and keep the others fixed. 212 | 213 | Labwatch allows to have different search space as long as they have different names. For instance in our running 214 | example if we want to optimize only the learning rate and batch size we can define a second search space: 215 | 216 | 217 | .. code:: python 218 | 219 | @a.searchspace 220 | def small_search_space(): 221 | batch_size = UniformNumber(lower=32, upper=64, default=32, type=int, log_scale=True) 222 | learning_rate = UniformFloat(lower=10e-3, upper=10e-2, default=10e-2, log_scale=True) 223 | 224 | 225 | If we now call our experiment via: 226 | 227 | 228 | .. code:: bash 229 | 230 | python experiment.py with small_search_space 231 | 232 | we get a new configuration for the learning rate and the batch where as all other hyperparameter are set to the values 233 | defined in the config cfg(). 234 | 235 | Note: To prevent inconsistencies and to not fool the optimizer, Labwatch passes only completed configurations that were 236 | drawn from this search space to the optimizer. This means that our optimizer will not use the information from previous 237 | experiment with the other search space. 238 | -------------------------------------------------------------------------------- /examples/bo_on_branin.py: -------------------------------------------------------------------------------- 1 | from branin import ex 2 | 3 | print("RUNNING sampled configs") 4 | num_configs = 100 5 | for i in range(num_configs): 6 | ex.run(named_configs=['search_space']) 7 | -------------------------------------------------------------------------------- /examples/branin.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | from labwatch.optimizers.random_search import RandomSearch 6 | from sacred import Experiment 7 | from labwatch.assistant import LabAssistant 8 | from labwatch.hyperparameters import UniformFloat 9 | import numpy as np 10 | 11 | 12 | ex = Experiment() 13 | a = LabAssistant(ex, database_name='branin', optimizer=RandomSearch) 14 | 15 | 16 | @ex.config 17 | def cfg(): 18 | x = (0., 5.) 19 | 20 | 21 | @a.search_space 22 | def search_space(): 23 | x = (UniformFloat(-5, 10), UniformFloat(0, 15)) 24 | 25 | 26 | @ex.automain 27 | def branin(x): 28 | x1, x2 = x 29 | print("{:.2f}, {:.2f}".format(x1, x2)) 30 | y = (x2 - (5.1 / (4 * np.pi ** 2)) * x1 ** 2 + 5 * x1 / np.pi - 6) ** 2 31 | y += 10 * (1 - 1 / (8 * np.pi)) * np.cos(x1) + 10 32 | 33 | return y 34 | -------------------------------------------------------------------------------- /examples/keras_mnist_mlp.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import keras 4 | from keras.datasets import mnist 5 | from keras.models import Sequential 6 | from keras.layers import Dense, Dropout 7 | from keras.optimizers import RMSprop 8 | 9 | from sacred import Experiment 10 | from labwatch.assistant import LabAssistant 11 | from labwatch.hyperparameters import UniformInt, UniformFloat 12 | from labwatch.optimizers.random_search import RandomSearch 13 | 14 | ex = Experiment() 15 | a = LabAssistant(ex, "labwatch_demo_keras", optimizer=RandomSearch) 16 | 17 | 18 | @ex.config 19 | def cfg(): 20 | batch_size = 128 21 | num_units_first_layer = 512 22 | num_units_second_layer = 512 23 | dropout_first_layer = 0.2 24 | dropout_second_layer = 0.2 25 | learning_rate = 0.001 26 | 27 | 28 | @a.search_space 29 | def small_search_space(): 30 | batch_size = UniformInt(lower=32, upper=64, default=32, log_scale=True) 31 | learning_rate = UniformFloat(lower=10e-3, upper=10e-2, default=10e-2, log_scale=True) 32 | 33 | 34 | @a.search_space 35 | def search_space(): 36 | batch_size = UniformInt(lower=8, upper=64, default=32, log_scale=True) 37 | num_units_first_layer = UniformInt(lower=16, upper=1024, default=32, log_scale=True) 38 | num_units_second_layer = UniformInt(lower=16, upper=1024, default=32, log_scale=True) 39 | dropout_first_layer = UniformFloat(lower=0, upper=.99, default=.2) 40 | dropout_second_layer = UniformFloat(lower=0, upper=.99, default=.2) 41 | learning_rate = UniformFloat(lower=10e-6, upper=10e-1, default=10e-2, log_scale=True) 42 | 43 | 44 | @ex.automain 45 | def run(batch_size, 46 | num_units_first_layer, 47 | num_units_second_layer, 48 | dropout_first_layer, 49 | dropout_second_layer, 50 | learning_rate): 51 | 52 | num_classes = 10 53 | epochs = 20 54 | 55 | # the data, shuffled and split between train and test sets 56 | (x_train, y_train), (x_test, y_test) = mnist.load_data() 57 | 58 | x_train = x_train.reshape(60000, 784) 59 | x_test = x_test.reshape(10000, 784) 60 | x_train = x_train.astype('float32') 61 | x_test = x_test.astype('float32') 62 | x_train /= 255 63 | x_test /= 255 64 | print(x_train.shape[0], 'train samples') 65 | print(x_test.shape[0], 'test samples') 66 | 67 | # convert class vectors to binary class matrices 68 | y_train = keras.utils.to_categorical(y_train, num_classes) 69 | y_test = keras.utils.to_categorical(y_test, num_classes) 70 | 71 | model = Sequential() 72 | model.add(Dense(num_units_first_layer, activation='relu', input_shape=(784,))) 73 | model.add(Dropout(dropout_first_layer)) 74 | model.add(Dense(num_units_second_layer, activation='relu')) 75 | model.add(Dropout(dropout_second_layer)) 76 | model.add(Dense(10, activation='softmax')) 77 | 78 | model.summary() 79 | 80 | model.compile(loss='categorical_crossentropy', 81 | optimizer=RMSprop(lr=learning_rate), 82 | metrics=['accuracy']) 83 | 84 | history = model.fit(x_train, y_train, 85 | batch_size=batch_size, 86 | epochs=epochs, 87 | verbose=1, 88 | validation_data=(x_test, y_test)) 89 | score = model.evaluate(x_test, y_test, verbose=0) 90 | print('Test loss:', score[0]) 91 | print('Test accuracy:', score[1]) 92 | 93 | results = dict() 94 | results["optimization_target"] = 1 - score[1] 95 | 96 | return results 97 | -------------------------------------------------------------------------------- /labwatch/__about__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | """ 4 | This module contains meta-information about the labwatch package. 5 | It is kept simple and separate from the main module, because this information 6 | is also read by the setup.py. And during installation the labwatch module cannot 7 | yet be imported. 8 | """ 9 | 10 | from __future__ import division, print_function, unicode_literals 11 | 12 | __all__ = ["__version__", "__authors__", "__url__"] 13 | 14 | __version__ = "0.1.0" 15 | 16 | __authors__ = 'Aaron Klein, Klaus Greff' 17 | 18 | __url__ = "https://github.com/automl/labwatch" 19 | -------------------------------------------------------------------------------- /labwatch/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | from labwatch.assistant import LabAssistant 5 | 6 | __version__ = "0.1dev" 7 | __authors__ = ["Jost Tobias Springenberg", "Aaron Klein", "Klaus Greff"] 8 | __url__ = "https://github.com/automl/labwatch" 9 | __all__ = ['LabAssistant', '__version__', '__authors__', '__url__'] 10 | -------------------------------------------------------------------------------- /labwatch/converters/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/labwatch/cdd7c55e5741bf4f1216f7b8da8de9fb1480c75b/labwatch/converters/__init__.py -------------------------------------------------------------------------------- /labwatch/converters/convert_to_configspace.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | from labwatch.searchspace import SearchSpace 6 | from labwatch.utils.types import basic_types, str_to_types 7 | from labwatch.utils.types import ParamValueExcept 8 | import numpy as np 9 | 10 | # TODO: guard ConfigSpace import 11 | from ConfigSpace import ConfigurationSpace, Configuration 12 | import ConfigSpace.hyperparameters as csh 13 | from ConfigSpace.conditions import InCondition 14 | 15 | 16 | def convert_simple_param(name, param): 17 | """ 18 | Convert a simple labwatch parameter to a ConfigSpace parameter. 19 | 20 | Parameters 21 | ---------- 22 | name: str 23 | The name of the parameter. 24 | 25 | param: dict 26 | Dictionary describing the parameter. 27 | 28 | Returns 29 | ------- 30 | ConfigSpace.hyperparameters.Hyperparameter: 31 | The converted hyperparameter. 32 | """ 33 | if param["_class"] == 'Constant': 34 | return csh.Constant(name, param["value"]) 35 | elif param["_class"] == 'Categorical': 36 | # convert the choices to only contain 37 | # basic types (they might contain Constant parameters 38 | basic_choices = [] 39 | for choice in param["choices"]: 40 | if isinstance(choice, dict): 41 | basic_choices.append(choice["default"]) 42 | elif not isinstance(choice, basic_types): 43 | err = "Choice parameter {} is not " \ 44 | "a base type or Constant!" 45 | raise ParamValueExcept(err.format(choice)) 46 | else: 47 | basic_choices.append(choice) 48 | return csh.CategoricalHyperparameter(name=name, 49 | choices=basic_choices, 50 | default_value=basic_choices[0]) 51 | elif param["_class"] == 'UniformFloat': 52 | return csh.UniformFloatHyperparameter(name=name, 53 | lower=param["lower"], 54 | upper=param["upper"], 55 | default_value=param["default"], 56 | log=param["log_scale"]) 57 | elif param["_class"] == 'UniformInt': 58 | return csh.UniformIntegerHyperparameter(name=name, 59 | lower=param["lower"], 60 | upper=param["upper"], 61 | default_value=param["default"], 62 | log=param["log_scale"]) 63 | elif param["_class"] == 'UniformNumber': 64 | ptype = str_to_types[param["type"]] 65 | if ptype == float: 66 | return csh.UniformFloatHyperparameter(name=name, 67 | lower=param["lower"], 68 | upper=param["upper"], 69 | default_value=param["default"], 70 | log=param["log_scale"]) 71 | elif ptype == int: 72 | return csh.UniformIntegerHyperparameter(name=name, 73 | lower=param["lower"], 74 | upper=param["upper"], 75 | default_value=param["default"], 76 | log=param["log_scale"]) 77 | else: 78 | raise ValueError("Don't know how to represent UniformNumber with " 79 | "type: {} in ConfigSpace".format(param["type"])) 80 | elif param["_class"] == 'Gaussian': 81 | return csh.NormalFloatHyperparameter(name=name, 82 | mu=param["mu"], 83 | sigma=param["sigma"], 84 | log=param["log_scale"]) 85 | 86 | else: 87 | raise ValueError("Don't know how to represent {} in ConfigSpace " 88 | "notation.".format(param)) 89 | 90 | 91 | def sacred_space_to_configspace(space): 92 | """ 93 | Convert a Labwatch searchspace to a ConfigSpace. 94 | 95 | Parameters 96 | ---------- 97 | space: labwatch.searchspace.SearchSpace 98 | A labwatch searchspace to be converted. 99 | 100 | Returns 101 | ------- 102 | ConfigSpace.ConfigurationSpace: 103 | A ConfigurationSpace equivalent to the given SeachSpace. 104 | """ 105 | # first convert all non conditionals 106 | non_conditions = {} 107 | conditions = [] 108 | for name in space.non_conditions: 109 | param = space.parameters[name] 110 | converted_param = convert_simple_param(name, param) 111 | non_conditions[name] = converted_param 112 | for name in space.conditions: 113 | param = space.parameters[name] 114 | converted_result = convert_simple_param(name, param["result"]) 115 | # next build the condition as required by the ConfigSpace 116 | condition = param["condition"] 117 | condition_name = space.uids_to_names[condition["uid"]] 118 | if condition_name not in non_conditions: 119 | raise ValueError("Unknown parameter in Condition") 120 | converted_condition = non_conditions[condition_name] 121 | converted_choices = [] 122 | for choice in condition["choices"]: 123 | if isinstance(choice, dict): 124 | if choice["_class"] != "Constant": 125 | raise ValueError("Invalid choice encountered in Condition") 126 | converted_choices.append(choice["value"]) 127 | else: 128 | converted_choices.append(choice) 129 | cond = InCondition(converted_result, 130 | converted_condition, 131 | values=converted_choices) 132 | non_conditions[name] = converted_result 133 | conditions.append(cond) 134 | # finally build the ConfigSpace 135 | cs = ConfigurationSpace(seed=np.random.seed()) 136 | for _name, param in non_conditions.items(): 137 | cs.add_hyperparameter(param) 138 | for cond in conditions: 139 | cs.add_condition(cond) 140 | return cs 141 | 142 | 143 | def sacred_config_to_configspace(cspace, config): 144 | """ 145 | Fill a ConfigurationSpace with the given values and return the resulting 146 | Configuration. 147 | 148 | Parameters 149 | ---------- 150 | cspace: ConfigSpace.ConfigurationSpace 151 | The configuration space to be populated. 152 | 153 | config: dict 154 | The configuration values as a dictionary mapping names to values. 155 | 156 | Returns 157 | ------- 158 | ConfigSpace.Configuration: 159 | The resulting Configuration. 160 | """ 161 | if isinstance(cspace, SearchSpace): 162 | raise ValueError("You called sacred_config_to_configspace " 163 | "with an instance of labwatch.SearchSpace " 164 | "but an instance of ConfigSpace.ConfigurationSpace " 165 | "is required. Use sacred_space_to_configspace().") 166 | return Configuration(cspace, values=config) 167 | 168 | 169 | def configspace_config_to_sacred(config): 170 | """ 171 | Convert a Configuration into a dict mapping parameter names to values. 172 | 173 | Parameters 174 | ---------- 175 | config: ConfigSpace.Configuration 176 | 177 | Returns 178 | ------- 179 | dict: 180 | Dictionary mapping parameter names to values. 181 | """ 182 | values = config.get_dictionary() 183 | # we remove all None entries as our (sacred searchspace) convention is 184 | # to not include them 185 | return {name: value for name, value in values.items() if value is not None} 186 | -------------------------------------------------------------------------------- /labwatch/optimizers/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | from .base import Optimizer 6 | from .random_search import RandomSearch 7 | 8 | # try: 9 | # from .smac import SMAC 10 | # except ImportError: 11 | # print('WARNING: SMAC not found') 12 | 13 | try: 14 | from .bayesian_optimization import BayesianOptimization 15 | except ImportError: 16 | print('WARNING: BayesianOptimization not found') 17 | -------------------------------------------------------------------------------- /labwatch/optimizers/base.py: -------------------------------------------------------------------------------- 1 | 2 | import numpy as np 3 | 4 | from labwatch.converters.convert_to_configspace import sacred_config_to_configspace 5 | 6 | 7 | class Optimizer(object): 8 | """Defines the interface for all optimizers.""" 9 | 10 | def __init__(self, config_space): 11 | self.config_space = config_space 12 | self.X = None 13 | self.y = None 14 | 15 | def get_random_config(self): 16 | return self.config_space.sample() 17 | 18 | def get_default_config(self): 19 | return self.config_space.default() 20 | 21 | def suggest_configuration(self): 22 | """Suggests a configuration of hyperparameters to be run. 23 | 24 | Returns 25 | ------- 26 | dict: 27 | Dictionary mapping parameter names to suggested values. 28 | 29 | The default is to return nothing this is done such that the 30 | user / the watcher can then check the queue or try something else. 31 | If you want get_random_config as default create an instance of 32 | RandomSearch instead. 33 | """ 34 | return None 35 | 36 | def update(self, configs, costs, runs): 37 | """ 38 | Update the internal state of the optimizer with a list of new results. 39 | 40 | Parameters 41 | ---------- 42 | configs: list[dict] 43 | List of configurations mapping parameter names to values. 44 | costs: list[float] 45 | List of costs associated to each config. 46 | runs: list[dict] 47 | List of dictionaries containing additional run information. 48 | """ 49 | 50 | converted_configs = [ 51 | sacred_config_to_configspace(self.config_space, config) 52 | for config in configs] 53 | 54 | for (config, cost) in zip(converted_configs, costs): 55 | # Maps configuration to [0, 1]^D space 56 | x = config.get_array() 57 | 58 | if self.X is None and self.y is None: 59 | self.X = np.array([x]) 60 | self.y = np.array([cost]) 61 | elif x not in self.X: 62 | self.X = np.append(self.X, x[np.newaxis, :], axis=0) 63 | self.y = np.append(self.y, np.array([cost]), axis=0) 64 | 65 | def needs_updates(self): 66 | """ 67 | Returns 68 | ------- 69 | bool: 70 | True if this optimizer needs updates, False otherwise. 71 | """ 72 | return False 73 | -------------------------------------------------------------------------------- /labwatch/optimizers/bayesian_optimization.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | import numpy as np 6 | from ConfigSpace import Configuration 7 | try: 8 | import george 9 | from robo.priors.default_priors import DefaultPrior 10 | from robo.models.gaussian_process_mcmc import GaussianProcessMCMC 11 | from robo.maximizers.direct import Direct 12 | from robo.acquisition_functions.log_ei import LogEI 13 | from robo.acquisition_functions.marginalization import MarginalizationGPMCMC 14 | from robo.initial_design.init_random_uniform import init_random_uniform 15 | except ImportError as e: 16 | print("If you want to use BayesianOptimization you have to install the following dependencies:\n" 17 | "https://github.com/automl/RoBO\n" 18 | "george") 19 | from labwatch.optimizers.base import Optimizer 20 | from labwatch.converters.convert_to_configspace import sacred_space_to_configspace, configspace_config_to_sacred 21 | from labwatch.utils.types import SearchSpaceNotSupported 22 | 23 | 24 | class BayesianOptimization(Optimizer): 25 | 26 | def __init__(self, config_space, burnin=100, chain_length=200, 27 | n_hypers=20): 28 | 29 | if config_space.has_categorical: 30 | raise SearchSpaceNotSupported("GP-based Bayesian optimization only supports numerical hyperparameters.") 31 | 32 | super(BayesianOptimization, self).__init__(config_space) 33 | self.rng = np.random.RandomState(np.random.seed()) 34 | 35 | self.burnin = burnin 36 | self.chain_length = chain_length 37 | self.n_hypers = n_hypers 38 | 39 | self.config_space = sacred_space_to_configspace(config_space) 40 | 41 | n_inputs = len(self.config_space.get_hyperparameters()) 42 | 43 | self.lower = np.zeros([n_inputs]) 44 | self.upper = np.ones([n_inputs]) 45 | 46 | self.X = None 47 | self.y = None 48 | 49 | def suggest_configuration(self): 50 | if self.X is None and self.y is None: 51 | new_x = init_random_uniform(self.lower, self.upper, 52 | n_points=1, rng=self.rng)[0, :] 53 | 54 | elif self.X.shape[0] == 1: 55 | # We need at least 2 data points to train a GP 56 | new_x = init_random_uniform(self.lower, self.upper, 57 | n_points=1, rng=self.rng)[0, :] 58 | 59 | else: 60 | cov_amp = 1 61 | n_dims = self.lower.shape[0] 62 | 63 | initial_ls = np.ones([n_dims]) 64 | exp_kernel = george.kernels.Matern52Kernel(initial_ls, 65 | ndim=n_dims) 66 | kernel = cov_amp * exp_kernel 67 | 68 | prior = DefaultPrior(len(kernel) + 1) 69 | 70 | model = GaussianProcessMCMC(kernel, prior=prior, 71 | n_hypers=self.n_hypers, 72 | chain_length=self.chain_length, 73 | burnin_steps=self.burnin, 74 | normalize_input=False, 75 | normalize_output=True, 76 | rng=self.rng, 77 | lower=self.lower, 78 | upper=self.upper) 79 | 80 | a = LogEI(model) 81 | 82 | acquisition_func = MarginalizationGPMCMC(a) 83 | 84 | max_func = Direct(acquisition_func, self.lower, self.upper, verbose=False) 85 | 86 | model.train(self.X, self.y) 87 | 88 | acquisition_func.update(model) 89 | 90 | new_x = max_func.maximize() 91 | 92 | next_config = Configuration(self.config_space, vector=new_x) 93 | 94 | # Transform to sacred configuration 95 | result = configspace_config_to_sacred(next_config) 96 | 97 | return result 98 | -------------------------------------------------------------------------------- /labwatch/optimizers/bohamiann.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | import numpy as np 6 | 7 | 8 | from ConfigSpace import Configuration 9 | try: 10 | from robo.initial_design.init_random_uniform import init_random_uniform 11 | from robo.models.bnn import BayesianNeuralNetwork 12 | from robo.maximizers.direct import Direct 13 | from robo.acquisition_functions.log_ei import LogEI 14 | except: 15 | print("If you want to use Bohamiann you have to install the following dependencies:\n" 16 | "RoBO (https://github.com/automl/RoBO)") 17 | from labwatch.optimizers.base import Optimizer 18 | from labwatch.converters.convert_to_configspace import ( 19 | sacred_space_to_configspace, configspace_config_to_sacred) 20 | 21 | 22 | class Bohamiann(Optimizer): 23 | 24 | def __init__(self, config_space, burnin=3000, n_iters=10000): 25 | 26 | super(Bohamiann, self).__init__(sacred_space_to_configspace(config_space)) 27 | self.rng = np.random.RandomState(np.random.seed()) 28 | self.n_dims = len(self.config_space.get_hyperparameters()) 29 | 30 | # All inputs are mapped to be in [0, 1]^D 31 | self.lower = np.zeros([self.n_dims]) 32 | self.upper = np.ones([self.n_dims]) 33 | self.incumbents = [] 34 | self.X = None 35 | self.y = None 36 | 37 | self.model = BayesianNeuralNetwork(sampling_method="sghmc", 38 | l_rate=np.sqrt(1e-4), 39 | mdecay=0.05, 40 | burn_in=burnin, 41 | n_iters=n_iters, 42 | precondition=True, 43 | normalize_input=True, 44 | normalize_output=True) 45 | 46 | self.acquisition_func = LogEI(self.model) 47 | 48 | self.maximizer = Direct(self.acquisition_func, self.lower, self.upper, verbose=False) 49 | 50 | def suggest_configuration(self): 51 | 52 | if self.X is None and self.y is None: 53 | # No data points yet to train a model, just return a random configuration instead 54 | new_x = init_random_uniform(self.lower, self.upper, 55 | n_points=1, rng=self.rng)[0, :] 56 | 57 | else: 58 | # Train the model on all finished runs 59 | self.model.train(self.X, self.y) 60 | self.acquisition_func.update(self.model) 61 | 62 | # Maximize the acquisition function 63 | new_x = self.maximizer.maximize() 64 | 65 | # Maps from [0, 1]^D space back to original space 66 | next_config = Configuration(self.config_space, vector=new_x) 67 | 68 | # Transform to sacred configuration 69 | result = configspace_config_to_sacred(next_config) 70 | 71 | return result 72 | -------------------------------------------------------------------------------- /labwatch/optimizers/dngo.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | import numpy as np 6 | 7 | from ConfigSpace import Configuration 8 | try: 9 | from robo.initial_design.init_random_uniform import init_random_uniform 10 | from robo.acquisition.log_ei import LogEI 11 | from robo.acquisition.integrated_acquisition import IntegratedAcquisition 12 | from robo.maximizers.direct import Direct 13 | from robo.priors.dngo_priors import DNGOPrior 14 | from robo.models.dngo import DNGO 15 | except: 16 | print("If you want to use DNGOWrapper you have to install the following dependencies:\n" 17 | "RoBO (https://github.com/automl/RoBO)") 18 | from labwatch.optimizers.base import Optimizer 19 | from labwatch.converters.convert_to_configspace import ( 20 | sacred_space_to_configspace, sacred_config_to_configspace, 21 | configspace_config_to_sacred) 22 | 23 | 24 | class DNGOWrapper(Optimizer): 25 | 26 | def __init__(self, config_space, burnin=1000, chain_length=200, 27 | n_hypers=20): 28 | 29 | super(DNGOWrapper, self).__init__(config_space) 30 | self.rng = np.random.RandomState(np.random.seed()) 31 | self.config_space = sacred_space_to_configspace(config_space) 32 | self.n_dims = len(self.config_space.get_hyperparameters()) 33 | 34 | # All inputs are mapped to be in [0, 1]^D 35 | self.X_lower = np.zeros([self.n_dims]) 36 | self.X_upper = np.ones([self.n_dims]) 37 | self.incumbents = [] 38 | self.X = None 39 | self.Y = None 40 | 41 | 42 | def suggest_configuration(self): 43 | if self.X is None and self.Y is None: 44 | new_x = init_random_uniform(self.X_lower, self.X_upper, 45 | N=1, rng=self.rng) 46 | 47 | elif self.X.shape[0] == 1: 48 | # We need at least 2 data points to train a GP 49 | Xopt = init_random_uniform(self.X_lower, self.X_upper, 50 | N=1, rng=self.rng) 51 | 52 | else: 53 | prior = DNGOPrior() 54 | model = DNGO(batch_size=100, num_epochs=20000, 55 | learning_rate=0.1, momentum=0.9, 56 | l2=1e-16, adapt_epoch=5000, 57 | n_hypers=20, prior=prior, 58 | do_optimize=True, do_mcmc=True) 59 | 60 | 61 | #acquisition_func = EI(model, task.X_lower, task.X_upper) 62 | lo = np.ones([model.n_units_3]) * -1 63 | up = np.ones([model.n_units_3]) 64 | ei = LogEI(model, lo, up) 65 | 66 | acquisition_func = IntegratedAcquisition( 67 | model, ei, self.X_lower, self.X_upper) 68 | 69 | maximizer = Direct(acquisition_func, self.X_lower, self.X_upper) 70 | 71 | model.train(self.X, self.Y) 72 | 73 | acquisition_func.update(model) 74 | 75 | new_x = maximizer.maximize() 76 | 77 | 78 | # Map from [0, 1]^D space back to original space 79 | next_config = Configuration(self.config_space, vector=new_x[0, :]) 80 | 81 | # Transform to sacred configuration 82 | result = configspace_config_to_sacred(next_config) 83 | 84 | return result 85 | 86 | 87 | def update(self, configs, costs, runs): 88 | converted_configs = [ 89 | sacred_config_to_configspace(self.config_space, config) 90 | for config in configs] 91 | 92 | for (config, cost) in zip(converted_configs, costs): 93 | # Maps configuration to [0, 1]^D space 94 | x = config.get_array() 95 | 96 | if self.X is None and self.Y is None: 97 | self.X = np.array([x]) 98 | self.Y = np.array([[cost]]) 99 | else: 100 | self.X = np.append(self.X, x[np.newaxis, :], axis=0) 101 | self.Y = np.append(self.Y, np.array([[cost]]), axis=0) 102 | 103 | def needs_updates(self): 104 | return True 105 | -------------------------------------------------------------------------------- /labwatch/optimizers/random_search.py: -------------------------------------------------------------------------------- 1 | 2 | from labwatch.optimizers import Optimizer 3 | 4 | 5 | class RandomSearch(Optimizer): 6 | 7 | def __init__(self, config_space): 8 | super(RandomSearch, self).__init__(config_space) 9 | 10 | def suggest_configuration(self): 11 | return self.get_random_config() 12 | 13 | def update(self, configs, costs, run_info): 14 | pass 15 | 16 | def needs_updates(self): 17 | return False 18 | -------------------------------------------------------------------------------- /labwatch/optimizers/smac_wrapper.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | import numpy as np 6 | try: 7 | from smac.scenario.scenario import Scenario 8 | from smac.tae.execute_ta_run import StatusType 9 | from smac.facade import smac_facade 10 | except: 11 | print("If you want to use LabwatchScenario you have to install the following dependencies:\n" 12 | "SMAC (https://github.com/automl/SMAC3)") 13 | from labwatch.optimizers.base import Optimizer 14 | from labwatch.converters.convert_to_configspace import ( 15 | sacred_space_to_configspace, sacred_config_to_configspace, 16 | configspace_config_to_sacred) 17 | 18 | 19 | class LabwatchScenario(Scenario): 20 | """ 21 | Specialize the smac3 scenario here since we want to create 22 | everything within code without reading a smac scenario file. 23 | """ 24 | 25 | def __init__(self, config_space, logger): 26 | self.logger = logger 27 | # we don't actually have a target algorithm here 28 | # we will implement algorithm calling and the SMBO loop ourselves 29 | self.ta = None 30 | self.execdir = None 31 | self.pcs_fn = None 32 | self.run_obj = 'quality' 33 | self.overall_obj = self.run_obj 34 | 35 | # Time limits for smac 36 | # these will never be used since we call 37 | # smac.choose_next() manually 38 | self.cutoff = None 39 | self.algo_runs_timelimit = None 40 | self.wallclock_limit = None 41 | 42 | # no instances 43 | self.train_inst_fn = None 44 | self.test_inst_fn = None 45 | self.feature_fn = None 46 | self.train_insts = [] 47 | self.test_inst = [] 48 | self.feature_dict = {} 49 | self.feature_array = None 50 | self.instance_specific = None 51 | self.n_features = 0 52 | 53 | # save reference to config_space 54 | self.cs = config_space 55 | 56 | # We do not need a TAE Runner as this is done by the Sacred Experiment 57 | self.tae_runner = None 58 | self.deterministic = False 59 | 60 | 61 | class SMAC(Optimizer): 62 | def __init__(self, config_space, seed=None): 63 | 64 | if seed is None: 65 | self.seed = np.random.randint(0, 10000) 66 | else: 67 | self.seed = seed 68 | 69 | self.rng = np.random.RandomState(self.seed) 70 | 71 | super(SMAC, self).__init__(sacred_space_to_configspace(config_space)) 72 | 73 | self.scenario = Scenario({"run_obj": "quality", 74 | "cs": self.config_space, 75 | "deterministic": "true"}) 76 | self.solver = smac_facade.SMAC(scenario=self.scenario, 77 | rng=self.rng) 78 | 79 | def suggest_configuration(self): 80 | if self.X is None and self.y is None: 81 | next_config = self.config_space.sample_configuration() 82 | 83 | else: 84 | l = list(self.solver.solver.choose_next(self.X, self.y[:, None], incumbent_value=np.min(self.y))) 85 | next_config = l[0] 86 | 87 | result = configspace_config_to_sacred(next_config) 88 | 89 | return result 90 | 91 | def needs_updates(self): 92 | return True 93 | -------------------------------------------------------------------------------- /labwatch/searchspace.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | import re 6 | 7 | from sacred.config import ConfigScope 8 | from sacred.utils import join_paths 9 | from labwatch.hyperparameters import Parameter, ConditionResult, Categorical 10 | from labwatch.hyperparameters import decode_param_or_op 11 | from labwatch.utils.types import InconsistentSpace, ParamValueExcept 12 | 13 | 14 | class SearchSpace(object): 15 | 16 | def __init__(self, search_space): 17 | super(SearchSpace, self).__init__() 18 | # extract the _id from the searchspace definition 19 | self._id = search_space.get('_id', None) 20 | if '_id' in search_space: 21 | del search_space['_id'] 22 | 23 | self.search_space = search_space 24 | parameters = collect_hyperparameters(search_space) 25 | params = sorted(parameters.values(), key=lambda x: x['uid']) 26 | self.uids_to_names = {param["uid"]: param['name'] for param in params} 27 | self.conditions = [] 28 | self.non_conditions = [] 29 | self.parameters = {} 30 | self.has_categorical = False 31 | # first simply insert all 32 | for param in params: 33 | if isinstance(param, Categorical): 34 | self.has_categorical = True 35 | 36 | assert(isinstance(param, Parameter)) 37 | self.parameters[param['name']] = param 38 | if isinstance(param, ConditionResult): 39 | self.conditions.append(param["name"]) 40 | else: 41 | self.non_conditions.append(param["name"]) 42 | 43 | self.contains_conditions = len(self.conditions) > 0 44 | self.validate_conditions() 45 | 46 | def to_json(self): 47 | son = dict(self.search_space) 48 | son['_class'] = 'SearchSpace' 49 | if self._id is not None: 50 | son['_id'] = self._id 51 | return son 52 | 53 | @classmethod 54 | def from_json(cls, son): 55 | assert son['_class'] == 'SearchSpace' 56 | del son['_class'] 57 | return SearchSpace(son) 58 | 59 | def validate_conditions(self): 60 | for pname in self.conditions: 61 | cparam = self.parameters[pname] 62 | conditioned_on = cparam["condition"]["uid"] 63 | if not (conditioned_on in self.uids_to_names): 64 | err = "Conditional parameter: {} depends on: {} " \ 65 | "which does not exist in the defined SearchSpace!" 66 | raise InconsistentSpace(err.format(pname, conditioned_on)) 67 | 68 | def is_valid_name(self, name): 69 | return name in self.uids_to_names.values() 70 | 71 | def valid(self, config): 72 | # TODO check this again for consistency 73 | valid = True 74 | for pname in self.non_conditions: 75 | valid &= self.parameters[pname].valid(config[pname]) 76 | if not valid: 77 | return False 78 | for pname in self.conditions: 79 | cparam = self.parameters[pname] 80 | conditioned_on = cparam["condition"]["uid"] 81 | if cparam in config.keys(): 82 | if conditioned_on in self.uids_to_names.keys(): 83 | valid &= conditioned_on.sample(config[self.uids_to_names[conditioned_on]]) 84 | valid &= cparam.valid(config[pname]) 85 | else: 86 | valid = False 87 | return valid 88 | 89 | def sample(self, max_iters_till_cycle=50, strategy="random"): 90 | if strategy not in ["random", "default"]: 91 | raise ParamValueExcept("Unknown sampling strategy {}".format(strategy)) 92 | # allocate result dict 93 | res = {} 94 | # first add all fixed parameters 95 | # for pname in self.fixed: 96 | # res[pname] = self[pname] 97 | # second sample all non conditions 98 | considered_params = set() 99 | for pname in self.non_conditions: 100 | if strategy == "random": 101 | res[pname] = self.parameters[pname].sample() 102 | else: 103 | res[pname] = self.parameters[pname].default() 104 | considered_params.add(pname) 105 | # then the conditional parameters 106 | remaining_params = set(self.conditions) 107 | i = 0 108 | while remaining_params: 109 | for pname in self.conditions: 110 | if pname in remaining_params: 111 | cparam = self.parameters[pname] 112 | conditioned_on = self.uids_to_names[cparam["condition"]["uid"]] 113 | if conditioned_on in res.keys(): 114 | if strategy == "random": 115 | cres = self.parameters[pname].sample(res[conditioned_on]) 116 | else: 117 | cres = self.parameters[pname].default(res[conditioned_on]) 118 | if cres: 119 | res[pname] = cres 120 | considered_params.add(pname) 121 | remaining_params.remove(pname) 122 | elif conditioned_on in considered_params: 123 | remaining_params.remove(pname) 124 | else: 125 | continue 126 | i += 1 127 | if i > max_iters_till_cycle: 128 | err = "Cannot satisfy conditionals involving " \ 129 | "parameters {} probably a loop! If you are sure " \ 130 | "no loop exists increase max_iters_till_cycle" 131 | raise InconsistentSpace(err.format(remaining_params)) 132 | return res 133 | 134 | def default(self, max_iters_till_cycle=50): 135 | return self.sample(max_iters_till_cycle, strategy="default") 136 | 137 | def __eq__(self, other): 138 | if not isinstance(other, SearchSpace): 139 | return False 140 | else: 141 | return self.search_space == other.search_space 142 | 143 | 144 | # decorator 145 | def build_search_space(function): 146 | # abuse configscope to parse search space definitions 147 | scope = ConfigScope(function) 148 | space_dict = dict(scope()) 149 | 150 | # parse generic dict to a search space 151 | space = SearchSpace(space_dict) 152 | return space 153 | 154 | 155 | def set_name(hparam, name): 156 | if ('name' not in hparam or 157 | len(hparam['name']) > len(name) or 158 | hparam['name'] > name): 159 | hparam['name'] = name 160 | 161 | 162 | def merge_parameters(params, new_params): 163 | for k, v in new_params.items(): 164 | if k not in params: 165 | params[k] = v 166 | else: 167 | set_name(params[k], v['name']) 168 | return params 169 | 170 | 171 | def collect_hyperparameters(search_space, path=''): 172 | """ 173 | Recursively collect all the hyperparameters from a search space. 174 | 175 | Parameters 176 | ---------- 177 | search_space : dict 178 | A JSON-like structure that describes the search space. 179 | path : str 180 | The path to the current entry. Used to determine the name of the 181 | detected hyperparameters. Optional: Only used for the recursion. 182 | 183 | Returns 184 | ------- 185 | parameters : dict 186 | A dictionary that to all the collected hyperparameters from their uids. 187 | """ 188 | # First try to decode to hyperparameter 189 | if isinstance(search_space, dict): 190 | try: 191 | hparam = decode_param_or_op(search_space) 192 | set_name(hparam, path) 193 | return {hparam['uid']: hparam} 194 | except ValueError: 195 | pass 196 | 197 | parameters = {} 198 | # if the space is a dict (but not a hyperparameter) we parse it recursively 199 | if isinstance(search_space, dict): 200 | for k, v in search_space.items(): 201 | # add the current key and a '.' as prefix when recursing 202 | sub_params = collect_hyperparameters(v, join_paths(path, k)) 203 | parameters = merge_parameters(parameters, sub_params) 204 | return parameters 205 | 206 | # if the space is a list we iterate it recursively 207 | elif isinstance(search_space, (tuple, list)): 208 | for i, v in enumerate(search_space): 209 | # add '[N]' to the name when recursing 210 | sub_params = collect_hyperparameters(v, path + '[{}]'.format(i)) 211 | parameters = merge_parameters(parameters, sub_params) 212 | return parameters 213 | else: 214 | # if the space is anything else do nothing 215 | return parameters 216 | 217 | 218 | def fill_in_values(search_space, values, fill_by='uid'): 219 | """ 220 | Recursively insert given values into a search space to receive a config. 221 | 222 | Parameters 223 | ---------- 224 | search_space : dict 225 | A JSON-like structure that describes the search space. 226 | values : dict 227 | A dictionary mapping uids to values. 228 | 229 | Returns 230 | ------- 231 | dict 232 | A configuration that results from replacing hyperparameters by the 233 | corresponding values. 234 | 235 | """ 236 | if isinstance(search_space, dict): 237 | if '_class' in search_space and fill_by in search_space: 238 | return values[search_space[fill_by]] 239 | else: 240 | return {k: fill_in_values(v, values, fill_by) 241 | for k, v in search_space.items()} 242 | elif isinstance(search_space, (list, tuple)): 243 | config = [fill_in_values(v, values, fill_by) for v in search_space] 244 | return type(search_space)(config) 245 | else: 246 | return search_space 247 | 248 | 249 | def get_by_path(config, path): 250 | """ 251 | Get a config-entry by its dotted and indexed name. 252 | 253 | Parameters 254 | ---------- 255 | config : dict 256 | The configuration dictionary to get the values from. 257 | path : str 258 | The config entry corresponding to the given path. 259 | Returns 260 | ------- 261 | object 262 | The configuration entry that corresponds to the given path. 263 | 264 | """ 265 | current = config 266 | for p in filter(None, re.split('[.\[\]]', path)): 267 | try: 268 | p = int(p) 269 | except ValueError: 270 | pass 271 | current = current[p] 272 | return current 273 | 274 | 275 | def get_values_from_config(config, hyperparams): 276 | """ 277 | Infer the values of hyperparameters from a given configuration. 278 | Parameters 279 | ---------- 280 | config : dict 281 | A JSON configuration that has to correspond to the search space. 282 | hyperparams : dict 283 | A dictionary that maps uids to hyperparameters. 284 | 285 | Returns 286 | ------- 287 | dict 288 | A dictionary mapping names to values. 289 | """ 290 | return {hparam['name']: get_by_path(config, hparam['name']) 291 | for uid, hparam in hyperparams.items()} 292 | -------------------------------------------------------------------------------- /labwatch/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from .fixed_dict import FixedDict 2 | from .hashing import hash_dict 3 | -------------------------------------------------------------------------------- /labwatch/utils/fixed_dict.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | from labwatch.utils.types import fullname 6 | 7 | 8 | def warn_not_allowed(self, key): 9 | warning = "WARNING: you tried to set key {}" \ 10 | " for class {} which is among the fixed keys!" 11 | print(warning.format(key, fullname(self))) 12 | 13 | 14 | class FixedDict(dict): 15 | def __init__(self, fixed=None): 16 | if fixed is None: 17 | fixed = {} 18 | super(FixedDict, self).__init__() 19 | for key in fixed.keys(): 20 | dict.__setitem__(self, key, fixed[key]) 21 | dict.__setitem__(self, "_class", fullname(self)) 22 | self.fixed = set(fixed.keys()).union({"_class"}) 23 | 24 | def __setitem__(self, key, value): 25 | """ __setitem__ for parameters only works 26 | for non fixed values! 27 | """ 28 | if key not in self.fixed: 29 | return dict.__setitem__(self, key, value) 30 | else: 31 | warn_not_allowed(self, key) 32 | 33 | def __delitem__(self, key): 34 | if key not in self.fixed: 35 | dict.__delitem__(self, key) 36 | -------------------------------------------------------------------------------- /labwatch/utils/hashing.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | def hash_dict(storage): 4 | return hash(json.dumps(storage, sort_keys=True)) 5 | 6 | -------------------------------------------------------------------------------- /labwatch/utils/types.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | from six import integer_types, string_types 3 | 4 | 5 | def str_to_class(cls_str): 6 | # module_name, class_name = cls_str.rsplit('.', 1) 7 | # somemod = importlib.import_module(module_name) 8 | somemod = importlib.import_module('labwatch.hyperparameters') 9 | class_name = cls_str 10 | return getattr(somemod, class_name) 11 | 12 | 13 | def fullname(o): 14 | # return o.__module__ + "." + o.__class__.__name__ 15 | return o.__class__.__name__ 16 | 17 | types_to_str = { 18 | float: "float", 19 | int: "int", 20 | str: "str" 21 | } 22 | 23 | for t in integer_types: 24 | types_to_str[t] = 'int' 25 | for t in string_types: 26 | types_to_str[t] = 'str' 27 | 28 | 29 | str_to_types = { 30 | "float": float, 31 | "int": int, 32 | "str": str 33 | } 34 | 35 | basic_types = tuple(types_to_str.keys()) 36 | 37 | 38 | class ParamValueExcept(Exception): 39 | pass 40 | 41 | 42 | class InconsistentSpace(Exception): 43 | pass 44 | 45 | 46 | class ParamInconsistent(Exception): 47 | pass 48 | 49 | 50 | class SearchSpaceNotSupported(Exception): 51 | pass 52 | -------------------------------------------------------------------------------- /labwatch/utils/version_checks.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | from pkg_resources import parse_version 6 | 7 | 8 | def parse_name_ver(name_version): 9 | name, _, ver = name_version.partition('==') 10 | return name, parse_version(ver) 11 | 12 | 13 | def check_dependencies(ex_dep, run_dep, version_policy): 14 | ex_dep = dict([parse_name_ver(name_version) for name_version in ex_dep]) 15 | check_version = { 16 | 'newer': lambda ex, name, b: name in ex and ex[name] >= b, 17 | 'equal': lambda ex, name, b: name in ex and ex[name] == b, 18 | 'exists': lambda ex, name, b: name in ex 19 | }[version_policy] 20 | for name_version in run_dep: 21 | name, ver = parse_name_ver(name_version) 22 | assert check_version(ex_dep, name, parse_version(ver)), \ 23 | "{} mismatch: ex={}, run={}".format(name, ex_dep[name], ver) 24 | 25 | 26 | def check_sources(ex_sources, run_sources): 27 | for ex_source, run_source in zip(ex_sources, run_sources): 28 | if not ex_source == tuple(run_source): 29 | raise KeyError('Source files did not match: experiment:' 30 | ' {} [{}] != {} [{}] (run)'.format( 31 | ex_source[0], ex_source[1], 32 | run_source[0], run_source[1])) 33 | 34 | 35 | def check_names(ex_name, run_name): 36 | if not ex_name == run_name: 37 | raise KeyError('experiment names did not match: experiment name ' 38 | '{} != {} (run name)'.format(ex_name, run_name)) 39 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | classifiers = """ 4 | Intended Audience :: Science/Research 5 | Natural Language :: English 6 | Programming Language :: Python 7 | Topic :: Utilities 8 | Topic :: Scientific/Engineering 9 | Topic :: Scientific/Engineering :: Artificial Intelligence 10 | Topic :: Software Development :: Libraries :: Python Modules 11 | License :: OSI Approved :: MIT License 12 | """ 13 | 14 | requires = [ 15 | 'numpy >= 1.7', 16 | 'sacred', 17 | 'pymongo', 18 | 'ConfigSpace' 19 | ] 20 | 21 | try: 22 | from labwatch import __about__ 23 | about = __about__.__dict__ 24 | except ImportError: 25 | about = dict() 26 | exec(open("labwatch/__about__.py").read(), about) 27 | 28 | 29 | 30 | setup(name='labwatch', 31 | version=about['__version__'], 32 | description='Hyperparameter optimization extension to Sacred', 33 | long_description=open('README.md').read(), 34 | classifiers=list(filter(None, classifiers.split('\n'))), 35 | author=about['__authors__'], 36 | author_email='kleinaa@cs.infomatik.uni-freiburg.de, springj@cs.uni-freiburg.de', 37 | url=about['__url__'], 38 | packages=['labwatch', 'labwatch.utils', 'labwatch.optimizers', 'labwatch.converters'], 39 | include_package_data=True, 40 | tests_require=['mock', 'mongomock', 'pytest'], 41 | install_requires=requires 42 | ) 43 | -------------------------------------------------------------------------------- /tests/test_conversion.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | import json 5 | import pytest 6 | 7 | from labwatch.hyperparameters import * 8 | from labwatch.searchspace import build_search_space, collect_hyperparameters, \ 9 | fill_in_values 10 | from labwatch.converters.convert_to_configspace import ( 11 | sacred_space_to_configspace, sacred_config_to_configspace, 12 | configspace_config_to_sacred) 13 | 14 | 15 | def simple_sp(): 16 | batch_size = UniformNumber(lower=32, upper=64, default=32, type=int) 17 | num_units_first_conv = UniformNumber(lower=32, upper=64, default=32, 18 | type=int) 19 | num_units_second_conv = UniformNumber(lower=32, upper=64, default=32, 20 | type=int) 21 | dropout_rate = UniformNumber(lower=0.2, upper=0.9, default=0.5, type=float) 22 | 23 | 24 | def space_with_condition(): 25 | batch_size = UniformNumber(lower=32, upper=64, default=32, type=int) 26 | n_layers = Categorical([1, 2]) 27 | units_first = UniformNumber(lower=32, 28 | upper=64, default=32, type=int) 29 | two = Constant(2) 30 | units_second = UniformNumber(lower=32, 31 | upper=64, default=32, type=int) | Condition( 32 | n_layers, [two]) 33 | dropout_second = UniformNumber(lower=0.2, upper=0.8, 34 | default=0.5, type=float) | Condition( 35 | n_layers, [2]) 36 | 37 | 38 | def test_convert_small_config_space(): 39 | space = build_searchspace(simple_sp) 40 | cspace = sacred_space_to_configspace(space) 41 | 42 | cs_non_conditions = cspace.get_all_unconditional_hyperparameters() 43 | for name in space.non_conditions: 44 | assert name in cs_non_conditions 45 | 46 | 47 | def test_convert_larger_config_space(): 48 | space = build_searchspace(space_with_condition) 49 | cspace = sacred_space_to_configspace(space) 50 | 51 | cs_non_conditions = cspace.get_all_unconditional_hyperparameters() 52 | for name in space.non_conditions: 53 | assert name in cs_non_conditions 54 | cs_conditions = cspace.get_all_conditional_hyperparameters() 55 | for name in space.conditions: assert name in cs_conditions 56 | 57 | 58 | def test_convert_config(): 59 | space = build_searchspace(space_with_condition) 60 | cspace = sacred_space_to_configspace(space) 61 | 62 | config = space.sample() 63 | cs_config = sacred_config_to_configspace(cspace, config) 64 | assert config == cs_config.get_dictionary() 65 | config_convert_back = configspace_config_to_sacred(cs_config) 66 | assert config == config_convert_back 67 | 68 | 69 | def test_config_config_wrong_space_raises(): 70 | space = build_searchspace(space_with_condition) 71 | cspace = sacred_space_to_configspace(space) 72 | 73 | config = space.sample() 74 | # passing the wrong type of space to config_to_configspace raises 75 | with pytest.raises(ValueError): 76 | cs_config = sacred_config_to_configspace(space, config) 77 | 78 | 79 | @pytest.mark.parametrize("hparam, name", [ 80 | (Constant(23), 'Constant'), 81 | (UniformFloat(0., 1.), 'UniformFloat'), 82 | (UniformInt(1, 10), 'UniformInt'), 83 | (UniformNumber(0, 1, float), 'UniformNumber'), 84 | (Categorical([2, 4, 6]), 'Categorical'), 85 | (Gaussian(0, 1.0), 'Gaussian') 86 | ]) 87 | def test_automatic_hyperparameters_to_dict_conversion(hparam, name): 88 | # use JSON serialization to force dict conversion 89 | d = json.loads(json.dumps(hparam)) 90 | assert isinstance(d, dict) 91 | assert "uid" in d 92 | assert "_class" in d 93 | assert d['_class'] == name 94 | 95 | 96 | @pytest.mark.parametrize("hparam", [ 97 | Constant(23), 98 | UniformFloat(0., 1.), 99 | UniformInt(1, 10), 100 | UniformNumber(0, 1, float), 101 | Categorical([2, 4, 6]), 102 | Gaussian(0, 1.0), 103 | ]) 104 | def test_decoding_hyperparameters_from_dict(hparam): 105 | # use JSON serialization to force dict conversion 106 | d = json.loads(json.dumps(hparam)) 107 | # decode 108 | h = decode_param_or_op(d) 109 | assert isinstance(h, Parameter) 110 | assert isinstance(h, type(hparam)) 111 | assert h == hparam 112 | 113 | 114 | def test_simple_searchspace_conversion(): 115 | a = Constant(7) 116 | b = UniformFloat(0, 1) 117 | space = { 118 | 'a': a, 119 | 'b': b 120 | } 121 | sp_dict = json.loads(json.dumps(space)) 122 | 123 | params = collect_hyperparameters(sp_dict, {}) 124 | assert params == { 125 | a['uid']: a, 126 | b['uid']: b 127 | } 128 | 129 | 130 | def test_searchspace_conversion_with_repetition(): 131 | a = UniformFloat(0, 1) 132 | space = { 133 | 'a': a, 134 | 'b': a 135 | } 136 | sp_dict = json.loads(json.dumps(space)) 137 | 138 | params = collect_hyperparameters(sp_dict, {}) 139 | assert params == { 140 | a['uid']: a 141 | } 142 | 143 | 144 | def test_searchspace_conversion_with_substructure(): 145 | a = UniformFloat(0, 1) 146 | b = UniformInt(2, 12) 147 | c = Gaussian(0, 1) 148 | space = { 149 | 'a': a, 150 | 'foo': { 151 | 'bar': b, 152 | 'nested': { 153 | 'a': a 154 | } 155 | }, 156 | 'using_list': [a, b, c] 157 | } 158 | sp_dict = json.loads(json.dumps(space)) 159 | 160 | params = collect_hyperparameters(sp_dict, {}) 161 | assert params == { 162 | a['uid']: a, 163 | b['uid']: b, 164 | c['uid']: c 165 | } 166 | 167 | assert params[a['uid']]['name'] == 'a' 168 | assert params[b['uid']]['name'] == 'foo.bar' 169 | assert params[c['uid']]['name'] == 'using_list[2]' 170 | 171 | 172 | def test_fill_in_values(): 173 | a = UniformFloat(0, 1) 174 | b = UniformInt(2, 12) 175 | c = Gaussian(0, 1) 176 | search_space = json.loads(json.dumps({ 177 | 'a': a, 178 | 'foo': { 179 | 'bar': b, 180 | 'nested': { 181 | 'a': a 182 | } 183 | }, 184 | 'using_list': [a, b, c] 185 | })) 186 | values = { 187 | a['uid']: 11, 188 | b['uid']: 2.2, 189 | c['uid']: 'c' 190 | } 191 | cfg = fill_in_values(search_space, values) 192 | assert cfg == { 193 | 'a': 11, 194 | 'foo': { 195 | 'bar': 2.2, 196 | 'nested': { 197 | 'a': 11 198 | } 199 | }, 200 | 'using_list': [11, 2.2, 'c'] 201 | } 202 | -------------------------------------------------------------------------------- /tests/test_searchspace.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | from labwatch.hyperparameters import * 6 | from labwatch.searchspace import build_search_space 7 | 8 | import pprint 9 | 10 | pp = pprint.PrettyPrinter(indent=4) 11 | 12 | 13 | def test_small_config_space(): 14 | def simple_sp(): 15 | batch_size = UniformNumber(lower=32, upper=64, default=32, type=int) 16 | num_units_first_conv = UniformNumber(lower=32, upper=64, default=32, 17 | type=int) 18 | num_units_second_conv = UniformNumber(lower=32, upper=64, default=32, 19 | type=int) 20 | dropout_rate = UniformNumber(lower=0.2, upper=0.9, default=0.5, 21 | type=float) 22 | 23 | space = build_searchspace(simple_sp) 24 | cfg = space.sample() 25 | assert space.valid(cfg) == True 26 | 27 | 28 | def test_config_space_with_condition(): 29 | def space_with_condition(): 30 | batch_size = UniformNumber(lower=32, upper=64, default=32, type=int) 31 | n_layers = Categorical([1, 2]) 32 | units_first = UniformNumber(lower=32, 33 | upper=64, default=32, type=int) 34 | two = Constant(2) 35 | units_second = UniformNumber(lower=32, 36 | upper=64, default=32, 37 | type=int) | Condition(n_layers, [two]) 38 | dropout_second = UniformNumber(lower=0.2, upper=0.8, 39 | default=0.5, type=float) | Condition( 40 | n_layers, [2]) 41 | 42 | space = build_searchspace(space_with_condition) 43 | cfg = space.sample() 44 | pp.pprint(space) 45 | pp.pprint(cfg) 46 | assert space.valid(cfg) == True 47 | --------------------------------------------------------------------------------