├── .github └── workflows │ ├── run_singularity_versions.yml │ └── run_tests.yml ├── .gitignore ├── LICENSE ├── README.md ├── changelog.md ├── ci_scripts ├── codestyle.sh ├── container_examples.sh ├── install.sh ├── install_singularity.sh ├── local_examples.sh └── script.sh ├── codecov.yml ├── examples ├── container │ ├── create_connection_to_benchmark.py │ ├── tabular_benchmark_example.py │ └── xgboost_with_container.py ├── local │ └── xgboost_local.py └── w_optimizer │ ├── SVMSurrogate_minicomparison.py │ ├── cartpole_bohb.py │ └── cartpole_smachb.py ├── extra_requirements ├── cartpole.json ├── examples.json ├── ml_mfbb.json ├── nasbench_101.json ├── nasbench_1shot1.json ├── outlier_detection.json ├── paramnet.json ├── pybnn.json ├── svm.json ├── tabular_benchmarks.json ├── tests.json └── xgboost.json ├── hpobench ├── __init__.py ├── __version__.py ├── abstract_benchmark.py ├── benchmarks │ ├── __init__.py │ ├── ml │ │ ├── README.md │ │ ├── __init__.py │ │ ├── histgb_benchmark.py │ │ ├── lr_benchmark.py │ │ ├── nn_benchmark.py │ │ ├── pybnn.py │ │ ├── rf_benchmark.py │ │ ├── svm_benchmark.py │ │ ├── svm_benchmark_old.py │ │ ├── tabular_benchmark.py │ │ ├── xgboost_benchmark.py │ │ └── xgboost_benchmark_old.py │ ├── nas │ │ ├── __init__.py │ │ ├── nasbench_101.py │ │ ├── nasbench_1shot1.py │ │ ├── nasbench_201.py │ │ └── tabular_benchmarks.py │ ├── od │ │ ├── __init__.py │ │ ├── od_ae.py │ │ ├── od_benchmarks.py │ │ ├── od_kde.py │ │ └── od_ocsvm.py │ ├── rl │ │ ├── __init__.py │ │ ├── cartpole.py │ │ └── learna_benchmark.py │ └── surrogates │ │ ├── __init__.py │ │ ├── paramnet_benchmark.py │ │ └── svm_benchmark.py ├── config.py ├── container │ ├── __init__.py │ ├── benchmarks │ │ ├── __init__.py │ │ ├── ml │ │ │ ├── __init__.py │ │ │ ├── histgb_benchmark.py │ │ │ ├── lr_benchmark.py │ │ │ ├── nn_benchmark.py │ │ │ ├── pybnn.py │ │ │ ├── rf_benchmark.py │ │ │ ├── svm_benchmark.py │ │ │ ├── svm_benchmark_old.py │ │ │ ├── tabular_benchmark.py │ │ │ ├── xgboost_benchmark.py │ │ │ └── xgboost_benchmark_old.py │ │ ├── nas │ │ │ ├── __init__.py │ │ │ ├── nasbench_101.py │ │ │ ├── nasbench_1shot1.py │ │ │ ├── nasbench_201.py │ │ │ └── tabular_benchmarks.py │ │ ├── od │ │ │ └── od_benchmarks.py │ │ ├── rl │ │ │ ├── __init__.py │ │ │ ├── cartpole.py │ │ │ └── learna_benchmark.py │ │ └── surrogates │ │ │ ├── __init__.py │ │ │ ├── paramnet_benchmark.py │ │ │ └── svm_benchmark.py │ ├── client_abstract_benchmark.py │ ├── recipes │ │ ├── Singularity.template │ │ ├── ml │ │ │ ├── Singularity.PyBNN │ │ │ ├── Singularity.SupportVectorMachine │ │ │ ├── Singularity.XGBoostBenchmark │ │ │ ├── Singularity.ml_mmfb │ │ │ └── Singularity.ml_tabular_benchmark │ │ ├── nas │ │ │ ├── Singularity.TabularBenchmarks │ │ │ ├── Singularity.nasbench_101 │ │ │ ├── Singularity.nasbench_1shot1 │ │ │ └── Singularity.nasbench_201 │ │ ├── od │ │ │ ├── Singularity.ODBenchmarks │ │ │ └── Singularity.ODKernelDensityEstimation │ │ ├── rl │ │ │ ├── Singularity.Cartpole │ │ │ └── Singularity.learnaBenchmark │ │ └── surrogates │ │ │ ├── Singularity.ParamnetBenchmark │ │ │ └── Singularity.SupportVectorMachine │ └── server_abstract_benchmark.py ├── dependencies │ ├── __init__.py │ ├── ml │ │ ├── __init__.py │ │ ├── data_manager.py │ │ └── ml_benchmark_template.py │ └── od │ │ ├── __init__.py │ │ ├── backbones │ │ ├── __init__.py │ │ └── mlp.py │ │ ├── callbacks │ │ ├── __init__.py │ │ ├── checkpoint_saver.py │ │ └── earlystopping.py │ │ ├── data_manager.py │ │ ├── models │ │ ├── __init__.py │ │ └── autoencoder.py │ │ ├── traditional_benchmark.py │ │ └── utils │ │ ├── __init__.py │ │ ├── activations.py │ │ └── scaler.py └── util │ ├── __init__.py │ ├── clean_up_script.py │ ├── container_utils.py │ ├── data_manager.py │ ├── dependencies.py │ ├── example_utils.py │ ├── openml_data_manager.py │ └── rng_helper.py ├── requirements.txt ├── setup.py └── tests ├── __init__.py ├── test_abstract_benchmark.py ├── test_check_configuration.py ├── test_container_availbable.py ├── test_data_manager.py ├── test_hpobenchconfig.py ├── test_nasbench_201.py ├── test_od.py ├── test_openml_datamanager.py ├── test_paramnet.py ├── test_pybnn.py ├── test_server.py ├── test_svm.py ├── test_tabular_benchmarks.py ├── test_utils.py └── test_whitebox.py /.github/workflows/run_singularity_versions.yml: -------------------------------------------------------------------------------- 1 | name: Test Support for different Singularity Versions 2 | 3 | on: [push] 4 | 5 | jobs: 6 | Tests: 7 | runs-on: ubuntu-latest 8 | 9 | strategy: 10 | matrix: 11 | include: 12 | - python-version: 3.7 13 | DISPLAY_NAME: "Singularity Container Examples with S3.5" 14 | RUN_CONTAINER_EXAMPLES: true 15 | USE_SINGULARITY: false 16 | SINGULARITY_VERSION: "3.5" 17 | - python-version: 3.7 18 | DISPLAY_NAME: "Singularity Container Examples with S3.6" 19 | RUN_CONTAINER_EXAMPLES: true 20 | USE_SINGULARITY: false 21 | SINGULARITY_VERSION: "3.6" 22 | - python-version: 3.7 23 | DISPLAY_NAME: "Singularity Container Examples with S3.7" 24 | RUN_CONTAINER_EXAMPLES: true 25 | USE_SINGULARITY: false 26 | SINGULARITY_VERSION: "3.7" 27 | - python-version: 3.7 28 | DISPLAY_NAME: "Singularity Container Examples with S3.8" 29 | RUN_CONTAINER_EXAMPLES: true 30 | USE_SINGULARITY: false 31 | SINGULARITY_VERSION: "3.8" 32 | 33 | fail-fast: false 34 | 35 | name: Tests ${{ matrix.python-version }} ${{ matrix.DISPLAY_NAME }} 36 | 37 | env: 38 | RUN_TESTS: ${{ matrix.RUN_TESTS }} 39 | USE_SINGULARITY: ${{ matrix.USE_SINGULARITY }} 40 | RUN_CODECOV: ${{ matrix.RUN_CODECOV }} 41 | RUN_CODESTYLE: ${{ matrix.RUN_CODESTYLE }} 42 | RUN_CONTAINER_EXAMPLES: ${{ matrix.RUN_CONTAINER_EXAMPLES }} 43 | RUN_LOCAL_EXAMPLES: ${{ matrix.RUN_LOCAL_EXAMPLES }} 44 | SINGULARITY_VERSION: ${{ matrix.SINGULARITY_VERSION }} 45 | 46 | steps: 47 | - uses: actions/checkout@v2 48 | - name: Set up Python ${{ matrix.python-version }} 49 | uses: actions/setup-python@v2 50 | with: 51 | python-version: ${{ matrix.python-version }} 52 | - name: Set up Go for Singularity 53 | if: matrix.USE_SINGULARITY == true 54 | uses: actions/setup-go@v2 55 | with: 56 | go-version: '1.14.15' # The Go version to download (if necessary) and use. 57 | - name: Install dependencies 58 | run: | 59 | python -m pip install --upgrade pip 60 | chmod +x ci_scripts/install_singularity.sh && source ./ci_scripts/install_singularity.sh 61 | - name: Run Tests 62 | run: chmod +x ci_scripts/script.sh && source ./ci_scripts/script.sh -------------------------------------------------------------------------------- /.github/workflows/run_tests.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests, code coverage, ... 2 | 3 | name: Test Pull Requests 4 | 5 | on: [push, pull_request] 6 | 7 | jobs: 8 | Tests: 9 | runs-on: ubuntu-latest 10 | 11 | strategy: 12 | matrix: 13 | include: 14 | - python-version: 3.6 15 | DISPLAY_NAME: "Singularity Tests" 16 | RUN_TESTS: true 17 | USE_SINGULARITY: true 18 | - python-version: 3.7 19 | DISPLAY_NAME: "Singularity Tests + CODECOV" 20 | RUN_TESTS: true 21 | USE_SINGULARITY: true 22 | RUN_CODECOV: true 23 | - python-version: 3.7 24 | DISPLAY_NAME: "Codestyle" 25 | RUN_CODESTYLE: true 26 | - python-version: 3.7 27 | DISPLAY_NAME: "Singularity Container Examples" 28 | RUN_CONTAINER_EXAMPLES: true 29 | USE_SINGULARITY: true 30 | - python-version: 3.7 31 | DISPLAY_NAME: "Local Examples" 32 | RUN_LOCAL_EXAMPLES: true 33 | USE_SINGULARITY: false 34 | - python-version: 3.8 35 | DISPLAY_NAME: "Singularity Tests" 36 | RUN_TESTS: true 37 | USE_SINGULARITY: true 38 | - python-version: 3.9 39 | DISPLAY_NAME: "Singularity Tests" 40 | RUN_TESTS: true 41 | USE_SINGULARITY: true 42 | fail-fast: false 43 | 44 | name: Tests ${{ matrix.python-version }} ${{ matrix.DISPLAY_NAME }} 45 | 46 | env: 47 | RUN_TESTS: ${{ matrix.RUN_TESTS }} 48 | USE_SINGULARITY: ${{ matrix.USE_SINGULARITY }} 49 | RUN_CODECOV: ${{ matrix.RUN_CODECOV }} 50 | RUN_CODESTYLE: ${{ matrix.RUN_CODESTYLE }} 51 | RUN_CONTAINER_EXAMPLES: ${{ matrix.RUN_CONTAINER_EXAMPLES }} 52 | RUN_LOCAL_EXAMPLES: ${{ matrix.RUN_LOCAL_EXAMPLES }} 53 | 54 | steps: 55 | - uses: actions/checkout@v2 56 | - name: Set up Python ${{ matrix.python-version }} 57 | uses: actions/setup-python@v2 58 | with: 59 | python-version: ${{ matrix.python-version }} 60 | - name: Set up Go for Singularity 61 | if: matrix.USE_SINGULARITY == true 62 | uses: actions/setup-go@v2 63 | with: 64 | go-version: '1.14.15' # The Go version to download (if necessary) and use. 65 | - name: Install dependencies 66 | run: | 67 | python -m pip install --upgrade pip 68 | chmod +x ci_scripts/install.sh && source ./ci_scripts/install.sh 69 | - name: Run Tests 70 | run: chmod +x ci_scripts/script.sh && source ./ci_scripts/script.sh -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | # Misc 132 | .idea/ 133 | experiments/ 134 | .DS_Store 135 | 136 | # Vagrant 137 | .vagrant 138 | Vagrantfile 139 | /hpobench/container/recipes_local/ 140 | -------------------------------------------------------------------------------- /changelog.md: -------------------------------------------------------------------------------- 1 | # 0.0.10 2 | * Cartpole Benchmark Version 0.0.4: 3 | Fix: Pass the hp `entropy_regularization` to the PPO Agent. 4 | Increase the lower limit of `likelihood_ratio_clipping` from 0 to 10e-7 (0 is invalid.) 5 | * Tabular Benchmark (NAS) Version 0.0.5 + NAS201 Version 0.0.5: 6 | We add for each benchmark in nas/tabular_benchmarks and nas/nasbench_201 a new benchmark class with a modified fidelity space. The new benchmark are called _Original_, e.g. _SliceLocalizationBenchmarkOriginal_ compared to _SliceLocalizationBenchmark_ 7 | These new benchmarks have the same fidelity space as used in previous experiments by [DEHB](https://ml.informatik.uni-freiburg.de/wp-content/uploads/papers/21-IJCAI-DEHB.pdf) and [BOHB](http://proceedings.mlr.press/v80/falkner18a/falkner18a.pdf). 8 | Specifically, we increase the lowest fidelity from 1 to 3 for nas/tabular_benchmarks and from 1 to 12 for nas/nasbench_201. The upper fidelity and the old benchmarks remain unchanged. 9 | 10 | # 0.0.9 11 | * Add new Benchmarks: Tabular Benchmarks. 12 | Provided by @Neeratyoy. 13 | * New Benchmark: ML Benchmark Class 14 | This new benchmark class offers a unified interface for XGB, SVM, MLP, HISTGB, RF, LR benchmarks operating on OpenML 15 | tasks. 16 | Provided by @Neeratyoy. 17 | * This version is the used for the paper: 18 | "HPOBench: A Collection of Reproducible Multi-Fidelity Benchmark Problems for HPO" (Eggensperger et al.) 19 | https://openreview.net/forum?id=1k4rJYEwda- 20 | 21 | # 0.0.8 22 | * Improve container integration 23 | The containers had some problems when the file system was read-only. In this case, the home directory, which contains the 24 | hpobenchrc file, was not mounted, so the init of the containers failed. To circumvent this problem, we make multiple 25 | changes: 26 | * We introduce an option to mount additional folder. This might be helpful when working on a cluster where the home 27 | directory is not available in the computation process. 28 | * We also change the configuration file. The container does not read the yaml file anymore. Instead we bind the 29 | cache dir, data dir and socket dir into the container and let the container use them directly. We also remove the 30 | global data directory and use only the data dir from now onwards. 31 | * Add the surrogate SVM on MNIST benchmark from the BOHB paper. 32 | * ParamNetBenchmark: 33 | Suppress some unuseful warnings and introduce a new param net benchmark that has a reduced search space. 34 | * Add Clean Up script: 35 | See the documentation for more information. You can clean all caches and container files by calling: 36 | `python util/clean_up_scripts.py --clear_all` 37 | * Improve Information on how to add a new benchmark 38 | * Fix an error in PyBnn Benchmark: 39 | We introduce the benchmark version 0.0.4. 40 | In this new version, we prevent infinity values in the nll loss when the predicted variance 41 | is 0. We set the predicted variance before computing the log to max(var, 1e-10) 42 | # 0.0.7 43 | * Fix an error in the NASBench1shot1 Benchmark (SearchSpace3). 44 | * Improve the behavior when a benchmark container is shut down. 45 | * Fix an error in PyBnn Benchmark: 46 | Set the minimum number of steps to burn to 1. 47 | * Fix an error in ParamNetOnTime Benchmarks: 48 | The maximum budget was not properly determined. 49 | * Move Container to Gitlab: 50 | Add support for communicating with the gitlab registry. We host the container now on \ 51 | https://gitlab.tf.uni-freiburg.de/muelleph/hpobench-registry/container_registry 52 | * Update the version check: Instead of requesting the same version, we check if the configuration file version and the 53 | hpobench version are in the same partition. Each hpobench version that has a compatible configuration file definition 54 | is in the same distribution. 55 | * Introduce a new version of the XGBoostBenchmark: A benchmark with an additional parameter, `booster`. 56 | * New Parameter Container Tag: 57 | The container-benchmark interface takes as input an optional container tag. by specifying this parameter, 58 | different container are downloaded from the registry. 59 | * Improve the procedure to find a container on the local filesystem: 60 | If a container_source is given, we check if it is either the direct address of a container on the filesystem 61 | or a directory. In case, it is a directory, we try to find the correct container in this directory by appending the 62 | container_tag to the container_name. (/) 63 | 64 | # 0.0.6 65 | * Add NasBench1shot1 Benchmark 66 | * Add info about incumbents for nasbench201 to its docstrings. 67 | * XGB and SVM's `get_meta_information()`-function returns now information about the used data set split. 68 | * Simplify the wrapper. Remove the support for configurations as lists and arrays. 69 | * Enforce the correct class-interfaces with the pylint package. To deviate from the standard interface, 70 | you have to explicitly deactivate the pylint error. 71 | * Nas1shot1 and Nas101 take as as input parameter now a seed. 72 | * The config file is now based on yaml. Also, it automatically raises a warning if the configuration file-version 73 | does not match the HPOBench-version. 74 | 75 | # 0.0.5 76 | * Rename package to HPOBench 77 | * Add BNN (pybnn) benchmark 78 | * Update returned loss values for nasbench101 and tabular benchmarks 79 | * Updat returned loss values for nasbench201 as well its data 80 | * Nasbench201 is now 1 indexed instead of 0. 81 | * Add MinMaxScaler to SVM Benchmark's preprocessing 82 | * Add further tests 83 | 84 | # 0.0.4 85 | * improve test coverage 86 | * update HowToAddANewBenchmark.md 87 | * Add SVM benchmark working on OpenML data 88 | * Add nasbench201 benchmark 89 | 90 | # 0.0.3 91 | * improve forwarding exceptions in containerized benchmarks 92 | * allow to set debug level with env variable 93 | * rename everything to HPOlib2 94 | 95 | # 0.0.2 96 | * add first set of benchmarks 97 | 98 | # 0.0.1 99 | * initial release 100 | -------------------------------------------------------------------------------- /ci_scripts/codestyle.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | if [[ "$RUN_CODESTYLE" == "true" ]]; then 4 | echo "Performing codestyle checking" 5 | 6 | test_codestyle=$(pycodestyle --max-line-length=120 ./hpobench) 7 | if [[ $test_codestyle ]]; then 8 | echo $test_codestyle 9 | exit 1 10 | else 11 | echo "Codesytle: No errors found" 12 | fi 13 | 14 | test_flake=$(flake8 --max-line-length=120 ./hpobench) 15 | if [[ $test_flake ]]; then 16 | echo $test_flake 17 | exit 1 18 | else 19 | echo "Flake8: No errors found" 20 | fi 21 | 22 | # Enable the error W0221: Parameters differ from overridden method (arguments-differ) 23 | pylint --disable=all --enable=W0221 ./hpobench 24 | exit_code=$? 25 | if [[ "$exit_code" -eq 0 ]]; then 26 | echo "Pylint: No signature errors found" 27 | else 28 | echo "Pylint: Signature Failure!" 29 | exit 1 30 | fi 31 | else 32 | echo "Skip code style checking" 33 | fi 34 | -------------------------------------------------------------------------------- /ci_scripts/container_examples.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | cd examples/container 4 | 5 | for script in *.py 6 | do 7 | python $script --on_travis 8 | rval=$? 9 | if [ "$rval" != 0 ]; then 10 | echo "Error running example $script" 11 | exit $rval 12 | fi 13 | done -------------------------------------------------------------------------------- /ci_scripts/install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | install_packages="" 4 | 5 | if [[ "$RUN_TESTS" == "true" ]]; then 6 | echo "Install tools for testing" 7 | install_packages="${install_packages}xgboost,pytest,test_paramnet,test_tabular_datamanager," 8 | pip install codecov 9 | 10 | # The param net benchmark does not work with a scikit-learn version != 0.23.2. (See notes in the benchmark) 11 | # To make sure that no newer version is installed, we install it before the other requirements. 12 | # Since we are not using a "--upgrade" option later on, pip skips to install another scikit-learn version. 13 | echo "Install the right scikit-learn function for the param net tests." 14 | pip install --upgrade scikit-learn==0.23.2 15 | else 16 | echo "Skip installing tools for testing" 17 | fi 18 | 19 | if [[ "$RUN_CODESTYLE" == "true" ]]; then 20 | echo "Install tools for codestyle checking" 21 | install_packages="${install_packages}codestyle," 22 | else 23 | echo "Skip installing tools for codestyle checking" 24 | fi 25 | 26 | if [[ "$RUN_CONTAINER_EXAMPLES" == "true" ]]; then 27 | echo "Install packages for container examples" 28 | echo "Install swig" 29 | sudo apt-get update && sudo apt-get install -y build-essential swig 30 | else 31 | echo "Skip installing packages for container examples" 32 | fi 33 | 34 | if [[ "$RUN_LOCAL_EXAMPLES" == "true" ]]; then 35 | echo "Install packages for local examples" 36 | echo "Install swig" 37 | sudo apt-get update && sudo apt-get install -y build-essential swig 38 | install_packages="${install_packages}xgboost," 39 | else 40 | echo "Skip installing packages for local examples" 41 | fi 42 | 43 | if [[ "$USE_SINGULARITY" == "true" ]]; then 44 | echo "Install Singularity" 45 | 46 | sudo apt-get update && sudo apt-get install -y \ 47 | build-essential \ 48 | libssl-dev \ 49 | uuid-dev \ 50 | libgpgme11-dev \ 51 | squashfs-tools \ 52 | libseccomp-dev \ 53 | wget \ 54 | pkg-config \ 55 | git \ 56 | cryptsetup 57 | 58 | export VERSION=3.5.3 && # adjust this as necessary \ 59 | wget https://github.com/sylabs/singularity/archive/refs/tags/v${VERSION}.tar.gz && \ 60 | tar -xzf v${VERSION}.tar.gz && \ 61 | cd singularity-${VERSION} 62 | 63 | ./mconfig && \ 64 | make -C builddir && \ 65 | sudo make -C builddir install 66 | 67 | cd .. 68 | install_packages="${install_packages}placeholder," 69 | else 70 | echo "Skip installing Singularity" 71 | fi 72 | 73 | # remove the trailing comma 74 | install_packages="$(echo ${install_packages} | sed 's/,*\r*$//')" 75 | echo "Install HPOBench with options: ${install_packages}" 76 | pip install .["${install_packages}"] 77 | -------------------------------------------------------------------------------- /ci_scripts/install_singularity.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | echo "Install Singularity" 4 | 5 | sudo apt-get update && sudo apt-get install -y \ 6 | build-essential \ 7 | libssl-dev \ 8 | uuid-dev \ 9 | libgpgme11-dev \ 10 | squashfs-tools \ 11 | libseccomp-dev \ 12 | wget \ 13 | pkg-config \ 14 | git \ 15 | cryptsetup 16 | 17 | if [[ "$SINGULARITY_VERSION" == "3.5" ]]; then 18 | export VERSION=3.5.3 19 | elif [[ "$SINGULARITY_VERSION" == "3.6" ]]; then 20 | export VERSION=3.6.4 21 | elif [[ "$SINGULARITY_VERSION" == "3.7" ]]; then 22 | export VERSION=3.7.3 23 | elif [[ "$SINGULARITY_VERSION" == "3.8" ]]; then 24 | export VERSION=3.8.0 25 | else 26 | echo "Skip installing Singularity" 27 | fi 28 | 29 | wget https://github.com/sylabs/singularity/archive/refs/tags/v${VERSION}.tar.gz && \ 30 | tar -xzf v${VERSION}.tar.gz && \ 31 | cd singularity-${VERSION} && \ 32 | ./mconfig && \ 33 | make -C builddir && \ 34 | sudo make -C builddir install 35 | 36 | cd .. 37 | pip install . 38 | -------------------------------------------------------------------------------- /ci_scripts/local_examples.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | cd examples/local 4 | 5 | for script in *.py 6 | do 7 | python $script --on_travis 8 | rval=$? 9 | if [ "$rval" != 0 ]; then 10 | echo "Error running example $script" 11 | exit $rval 12 | fi 13 | done -------------------------------------------------------------------------------- /ci_scripts/script.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | if [[ "$RUN_TESTS" == "true" ]]; then 4 | if [[ "$RUN_CODECOV" == "true" ]]; then 5 | echo "Run tests with code coverage" 6 | pytest -sv --cov=hpobench tests/ 7 | exit_code=$? 8 | 9 | echo "Run code coverage" 10 | codecov 11 | else 12 | echo "Run tests without code coverage" 13 | pytest -sv tests/ 14 | exit_code=$? 15 | fi 16 | 17 | if [[ "$exit_code" -eq 0 ]]; then 18 | echo "All test have passed." 19 | else 20 | echo "Some Tests have failed." 21 | exit 1 22 | fi 23 | fi 24 | 25 | if [[ "$RUN_CODESTYLE" == "true" ]]; then 26 | echo "Run codestyle" 27 | chmod +x ci_scripts/codestyle.sh && source ./ci_scripts/codestyle.sh 28 | fi 29 | 30 | if [[ "$RUN_CONTAINER_EXAMPLES" == "true" ]]; then 31 | echo "Run containerized examples" 32 | chmod +x ci_scripts/container_examples.sh && source ./ci_scripts/container_examples.sh 33 | fi 34 | 35 | if [[ "$RUN_LOCAL_EXAMPLES" == "true" ]]; then 36 | echo "Run containerized examples" 37 | chmod +x ci_scripts/local_examples.sh && source ./ci_scripts/local_examples.sh 38 | fi -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | codecov: 2 | require_ci_to_pass: yes 3 | 4 | coverage: 5 | precision: 2 6 | round: down 7 | range: "70...100" 8 | 9 | parsers: 10 | gcov: 11 | branch_detection: 12 | conditional: yes 13 | loop: yes 14 | method: no 15 | macro: no 16 | 17 | comment: 18 | layout: "reach,diff,flags,tree" 19 | behavior: default 20 | require_changes: no 21 | 22 | ignore: 23 | - "hpobench/benchmarks" 24 | - "hpobench/util/dependencies.py" 25 | - "hpobench/util/example_utils.py" -------------------------------------------------------------------------------- /examples/container/create_connection_to_benchmark.py: -------------------------------------------------------------------------------- 1 | """ 2 | ADVANCED: Create multiple connections to a Benchmark 3 | ==================================================== 4 | 5 | Sometimes it is useful to have only a single container instance running for multiple experiments. 6 | For example if starting a container is very expensive. 7 | 8 | Since version 0.0.8, we can now start a benchmark container in a background process and connect to the benchmark from 9 | various other processes. 10 | This example shows how to create a benchmark and create a separate proxy connection to it. 11 | 12 | The benchmark is based on Pyro4 and should handle a certain amount of simultaneous calls. 13 | 14 | Also make sure not have to many open Pyro4.Proxy connections. 15 | (See https://pyro4.readthedocs.io/en/stable/tipstricks.html#after-x-simultaneous-proxy-connections-pyro-seems-to-freeze-fix-release-your-proxies-when-you-can) 16 | 17 | Please install the necessary dependencies via ``pip install .`` and singularity (v3.5). 18 | https://sylabs.io/guides/3.5/user-guide/quick_start.html#quick-installation-steps 19 | """ 20 | 21 | import argparse 22 | 23 | from hpobench.container.benchmarks.nas.tabular_benchmarks import SliceLocalizationBenchmark as TabBenchmarkContainer 24 | 25 | 26 | def run_experiment(on_travis=False): 27 | 28 | # First, we start the benchmark. This generates the unix-socket (address) where the benchmark is reachable. 29 | benchmark = TabBenchmarkContainer(container_name='tabular_benchmarks', 30 | container_source='library://phmueller/automl', 31 | rng=1) 32 | 33 | print(benchmark.socket_id) 34 | 35 | # Now, we could use this `socket_id` to connect to the benchmark from another process. For simplicity, we just 36 | # create a new proxy connection to it. 37 | # Note that you don't have to specify the container name or other things, since we only connect to it. 38 | 39 | proxy_to_benchmark = TabBenchmarkContainer(socket_id=benchmark.socket_id) 40 | 41 | cs = proxy_to_benchmark.get_configuration_space(seed=1) 42 | config = cs.sample_configuration() 43 | print(config) 44 | 45 | # You can pass the configuration either as a dictionary or a ConfigSpace.configuration 46 | result_dict_1 = proxy_to_benchmark.objective_function(configuration=config.get_dictionary()) 47 | result_dict_2 = proxy_to_benchmark.objective_function(configuration=config) 48 | print(result_dict_1, result_dict_2) 49 | 50 | 51 | if __name__ == "__main__": 52 | parser = argparse.ArgumentParser(prog='TabularNad') 53 | 54 | parser.add_argument('--on_travis', action='store_true', 55 | help='Flag to speed up the run on the continuous integration tool \"travis\". This flag can be' 56 | 'ignored by the user') 57 | 58 | args = parser.parse_args() 59 | run_experiment(on_travis=args.on_travis) 60 | -------------------------------------------------------------------------------- /examples/container/tabular_benchmark_example.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tabular benchmark 3 | ================= 4 | 5 | This examples shows the usage of the containerized tabular benchmark. 6 | To note: You don't have to pass the container name to the Benchmark-Constructor. It is automatically set, but for 7 | demonstration purpose, we show how to set it. 8 | 9 | container_source can be either a path to a registry (e.g. sylabs.io, singularity_hub.org) or a local path on your local 10 | file system. If it is a link to a registry, the container will be downloaded to the default data dir, set in the 11 | hpobenchrc. A second call, will first look into the data directory, if the container is already available, so it will not 12 | be downloaded twice. 13 | 14 | Please install the necessary dependencies via ``pip install .`` and singularity (v3.5). 15 | https://sylabs.io/guides/3.5/user-guide/quick_start.html#quick-installation-steps 16 | """ 17 | 18 | import argparse 19 | 20 | from hpobench.container.benchmarks.nas.tabular_benchmarks import SliceLocalizationBenchmark as TabBenchmarkContainer 21 | 22 | 23 | def run_experiment(on_travis=False): 24 | 25 | benchmark = TabBenchmarkContainer(container_name='tabular_benchmarks', 26 | container_source='library://phmueller/automl', 27 | rng=1) 28 | 29 | cs = benchmark.get_configuration_space(seed=1) 30 | config = cs.sample_configuration() 31 | print(config) 32 | 33 | # You can pass the configuration either as a dictionary or a ConfigSpace.configuration 34 | result_dict_1 = benchmark.objective_function(configuration=config.get_dictionary()) 35 | result_dict_2 = benchmark.objective_function(configuration=config) 36 | print(result_dict_1, result_dict_2) 37 | 38 | 39 | if __name__ == "__main__": 40 | parser = argparse.ArgumentParser(prog='TabularNad') 41 | 42 | parser.add_argument('--on_travis', action='store_true', 43 | help='Flag to speed up the run on the continuous integration tool \"travis\". This flag can be' 44 | 'ignored by the user') 45 | 46 | args = parser.parse_args() 47 | run_experiment(on_travis=args.on_travis) 48 | -------------------------------------------------------------------------------- /examples/container/xgboost_with_container.py: -------------------------------------------------------------------------------- 1 | """ 2 | Example with XGBoost (container) 3 | ================================ 4 | 5 | In this example, we show how to use a benchmark with a container. We provide container for some benchmarks. 6 | They are hosted on https://cloud.sylabs.io/library/muelleph/automl. 7 | 8 | Furthermore, we use different fidelities to train the xgboost model - the number of estimators as well as the fraction 9 | of training data points. 10 | 11 | To use the container-example, you have to have singulartiy (>3.5) installed. Follow the official installation guide on 12 | https://sylabs.io/guides/3.1/user-guide/quick_start.html#quick-installation-steps 13 | 14 | Furthermore, make sure to install the right dependencies for the hpobench via: 15 | ``pip3 install .``. 16 | """ 17 | 18 | import argparse 19 | import logging 20 | from time import time 21 | 22 | from hpobench.container.benchmarks.ml.xgboost_benchmark import XGBoostBenchmark as Benchmark 23 | from hpobench.util.openml_data_manager import get_openmlcc18_taskids 24 | 25 | logging.basicConfig(level=logging.INFO) 26 | 27 | 28 | def run_experiment(on_travis: bool = False): 29 | task_ids = get_openmlcc18_taskids() 30 | for task_no, task_id in enumerate(task_ids): 31 | 32 | if on_travis and task_no == 5: 33 | break 34 | 35 | print(f'# ################### TASK {task_no + 1} of {len(task_ids)}: Task-Id: {task_id} ################### #') 36 | if task_id == 167204: 37 | continue # due to memory limits 38 | 39 | b = Benchmark(task_id=task_id, 40 | container_name='xgboost_benchmark', 41 | container_source='library://phmueller/automl') 42 | 43 | cs = b.get_configuration_space() 44 | start = time() 45 | num_configs = 1 46 | for i in range(num_configs): 47 | configuration = cs.sample_configuration() 48 | print(configuration) 49 | for n_estimator in [8, 64]: 50 | for subsample in [0.4, 1]: 51 | fidelity = {'n_estimators': n_estimator, 'dataset_fraction': subsample} 52 | result_dict = b.objective_function(configuration.get_dictionary(), 53 | fidelity=fidelity) 54 | valid_loss = result_dict['function_value'] 55 | train_loss = result_dict['info']['train_loss'] 56 | assert result_dict['info']['fidelity'] == fidelity 57 | 58 | result_dict = b.objective_function_test(configuration, fidelity={'n_estimators': n_estimator}) 59 | test_loss = result_dict['function_value'] 60 | 61 | print(f'[{i+1}|{num_configs}] No Estimator: {n_estimator:3d} - ' 62 | f'Subsample Rate: {subsample:.1f} - Test {test_loss:.4f} ' 63 | f'- Valid {valid_loss:.4f} - Train {train_loss:.4f}') 64 | b.__del__() 65 | print(f'Done, took totally {time()-start:.2f}') 66 | 67 | 68 | if __name__ == '__main__': 69 | parser = argparse.ArgumentParser(prog='HPOBench CC Datasets', description='HPOBench on the CC18 data sets.', 70 | usage='%(prog)s --array_id ') 71 | 72 | parser.add_argument('--on_travis', action='store_true', 73 | help='Flag to speed up the run on the continuous integration tool \'travis\'. This flag can be' 74 | 'ignored by the user') 75 | 76 | args = parser.parse_args() 77 | run_experiment(on_travis=args.on_travis) 78 | -------------------------------------------------------------------------------- /examples/local/xgboost_local.py: -------------------------------------------------------------------------------- 1 | """ 2 | Example with XGBoost (local) 3 | ============================ 4 | This example executes the xgboost benchmark locally with random configurations on the CC18 openml tasks. 5 | 6 | To run this example please install the necessary dependencies via: 7 | ``pip3 install .[xgboost_example]`` 8 | """ 9 | 10 | import argparse 11 | from time import time 12 | 13 | from hpobench.benchmarks.ml.xgboost_benchmark_old import XGBoostBenchmark as Benchmark 14 | from hpobench.util.openml_data_manager import get_openmlcc18_taskids 15 | 16 | 17 | def run_experiment(on_travis: bool = False): 18 | task_ids = get_openmlcc18_taskids() 19 | for task_no, task_id in enumerate(task_ids): 20 | 21 | if on_travis and task_no == 5: 22 | break 23 | 24 | print(f'# ################### TASK {task_no + 1} of {len(task_ids)}: Task-Id: {task_id} ################### #') 25 | if task_id == 167204: 26 | continue # due to memory limits 27 | 28 | b = Benchmark(task_id=task_id) 29 | cs = b.get_configuration_space() 30 | start = time() 31 | num_configs = 1 32 | for i in range(num_configs): 33 | configuration = cs.sample_configuration() 34 | print(configuration) 35 | for n_estimator in [8, 64]: 36 | for subsample in [0.4, 1]: 37 | fidelity = {'n_estimators': n_estimator, 'dataset_fraction': subsample} 38 | result_dict = b.objective_function(configuration.get_dictionary(), 39 | fidelity=fidelity) 40 | valid_loss = result_dict['function_value'] 41 | train_loss = result_dict['info']['train_loss'] 42 | assert result_dict['info']['fidelity'] == fidelity 43 | 44 | result_dict = b.objective_function_test(configuration) 45 | test_loss = result_dict['function_value'] 46 | 47 | print(f'[{i+1}|{num_configs}] No Estimator: {n_estimator:3d} - ' 48 | f'Subsample Rate: {subsample:.1f} - Test {test_loss:.4f} ' 49 | f'- Valid {valid_loss:.4f} - Train {train_loss:.4f}') 50 | print(f'Done, took totally {time()-start:.2f}') 51 | 52 | 53 | if __name__ == '__main__': 54 | parser = argparse.ArgumentParser(prog='HPOBench CC Datasets', description='HPOBench on the CC18 data sets.', 55 | usage='%(prog)s --array_id ') 56 | 57 | parser.add_argument('--on_travis', action='store_true', 58 | help='Flag to speed up the run on the continuous integration tool \'travis\'. This flag can be' 59 | 'ignored by the user') 60 | 61 | args = parser.parse_args() 62 | run_experiment(on_travis=args.on_travis) 63 | -------------------------------------------------------------------------------- /examples/w_optimizer/SVMSurrogate_minicomparison.py: -------------------------------------------------------------------------------- 1 | """ 2 | Multiple Optimizers on SVMSurrogate 3 | ======================================= 4 | 5 | This example shows how to run SMAC-HB and SMAC-random-search on SVMSurrogate 6 | 7 | Please install the necessary dependencies via ``pip install .`` and singularity (v3.5). 8 | https://sylabs.io/guides/3.5/user-guide/quick_start.html#quick-installation-steps 9 | """ 10 | import logging 11 | from pathlib import Path 12 | from time import time 13 | 14 | import numpy as np 15 | from smac.facade.smac_mf_facade import SMAC4MF 16 | from smac.facade.roar_facade import ROAR 17 | from smac.intensification.hyperband import Hyperband 18 | from smac.scenario.scenario import Scenario 19 | from smac.callbacks import IncorporateRunResultCallback 20 | 21 | from hpobench.container.benchmarks.surrogates.svm_benchmark import SurrogateSVMBenchmark 22 | from hpobench.abstract_benchmark import AbstractBenchmark 23 | from hpobench.util.example_utils import set_env_variables_to_use_only_one_core 24 | 25 | logger = logging.getLogger("minicomp") 26 | logging.basicConfig(level=logging.INFO) 27 | set_env_variables_to_use_only_one_core() 28 | 29 | 30 | class Callback(IncorporateRunResultCallback): 31 | def __init__(self): 32 | self.budget = 10 33 | 34 | def __call__(self, smbo, run_info, result, time_left) -> None: 35 | self.budget -= run_info.budget 36 | if self.budget < 0: 37 | # No budget left 38 | raise ValueError 39 | 40 | def create_smac_rs(benchmark, output_dir: Path, seed: int): 41 | # Set up SMAC-HB 42 | cs = benchmark.get_configuration_space(seed=seed) 43 | 44 | scenario_dict = {"run_obj": "quality", # we optimize quality (alternative to runtime) 45 | "wallclock-limit": 60, 46 | "cs": cs, 47 | "deterministic": "true", 48 | "runcount-limit": 200, 49 | "limit_resources": True, # Uses pynisher to limit memory and runtime 50 | "cutoff": 1800, # runtime limit for target algorithm 51 | "memory_limit": 10000, # adapt this to reasonable value for your hardware 52 | "output_dir": output_dir, 53 | "abort_on_first_run_crash": True, 54 | } 55 | 56 | scenario = Scenario(scenario_dict) 57 | def optimization_function_wrapper(cfg, seed, **kwargs): 58 | """ Helper-function: simple wrapper to use the benchmark with smac """ 59 | result_dict = benchmark.objective_function(cfg, rng=seed) 60 | cs.sample_configuration() 61 | return result_dict['function_value'] 62 | 63 | smac = ROAR(scenario=scenario, 64 | rng=np.random.RandomState(seed), 65 | tae_runner=optimization_function_wrapper, 66 | ) 67 | return smac 68 | 69 | def create_smac_hb(benchmark, output_dir: Path, seed: int): 70 | # Set up SMAC-HB 71 | cs = benchmark.get_configuration_space(seed=seed) 72 | 73 | scenario_dict = {"run_obj": "quality", # we optimize quality (alternative to runtime) 74 | "wallclock-limit": 60, 75 | "cs": cs, 76 | "deterministic": "true", 77 | "runcount-limit": 200, 78 | "limit_resources": True, # Uses pynisher to limit memory and runtime 79 | "cutoff": 1800, # runtime limit for target algorithm 80 | "memory_limit": 10000, # adapt this to reasonable value for your hardware 81 | "output_dir": output_dir, 82 | "abort_on_first_run_crash": True, 83 | } 84 | 85 | scenario = Scenario(scenario_dict) 86 | def optimization_function_wrapper(cfg, seed, instance, budget): 87 | """ Helper-function: simple wrapper to use the benchmark with smac """ 88 | result_dict = benchmark.objective_function(cfg, rng=seed, 89 | fidelity={"dataset_fraction": budget}) 90 | cs.sample_configuration() 91 | return result_dict['function_value'] 92 | 93 | smac = SMAC4MF(scenario=scenario, 94 | rng=np.random.RandomState(seed), 95 | tae_runner=optimization_function_wrapper, 96 | intensifier=Hyperband, 97 | intensifier_kwargs={'initial_budget': 0.1, 'max_budget': 1, 'eta': 3} 98 | ) 99 | return smac 100 | 101 | 102 | def run_experiment(out_path: str, on_travis: bool = False): 103 | 104 | out_path = Path(out_path) 105 | out_path.mkdir(exist_ok=True) 106 | 107 | hb_res = [] 108 | rs_res = [] 109 | for i in range(4): 110 | benchmark = SurrogateSVMBenchmark(rng=i) 111 | smac = create_smac_hb(benchmark=benchmark, seed=i, output_dir=out_path) 112 | callback = Callback() 113 | smac.register_callback(callback) 114 | try: 115 | smac.optimize() 116 | except ValueError: 117 | print("Done") 118 | incumbent = smac.solver.incumbent 119 | inc_res = benchmark.objective_function(configuration=incumbent) 120 | hb_res.append(inc_res["function_value"]) 121 | 122 | benchmark = SurrogateSVMBenchmark(rng=i) 123 | smac = create_smac_rs(benchmark=benchmark, seed=i, output_dir=out_path) 124 | callback = Callback() 125 | smac.register_callback(callback) 126 | try: 127 | smac.optimize() 128 | except ValueError: 129 | print("Done") 130 | incumbent = smac.solver.incumbent 131 | inc_res = benchmark.objective_function(configuration=incumbent) 132 | rs_res.append(inc_res["function_value"]) 133 | 134 | print("SMAC-HB", hb_res, np.median(hb_res)) 135 | print("SMAC-RS", rs_res, np.median(rs_res)) 136 | 137 | 138 | if __name__ == "__main__": 139 | import argparse 140 | parser = argparse.ArgumentParser(prog='HPOBench - SVM comp', 141 | description='Run different opts on SVM Surrogate', 142 | usage='%(prog)s --out_path ') 143 | parser.add_argument('--out_path', default='./svm_comp', type=str) 144 | parser.add_argument('--on_travis', action='store_true', 145 | help='Flag to speed up the run on the continuous integration tool \"travis\". This flag can be' 146 | 'ignored by the user') 147 | args = parser.parse_args() 148 | 149 | run_experiment(out_path=args.out_path, on_travis=args.on_travis) 150 | -------------------------------------------------------------------------------- /examples/w_optimizer/cartpole_bohb.py: -------------------------------------------------------------------------------- 1 | """ 2 | BOHB on Cartpole 3 | ========================== 4 | 5 | This example shows the usage of an Hyperparameter Tuner, such as BOHB on the cartpole benchmark. 6 | BOHB is a combination of Bayesian optimization and Hyperband. 7 | 8 | **Note**: This is a raw benchmark, i.e. it actually runs an algorithms, and will take some time 9 | 10 | Please install the necessary dependencies via ``pip install .[examples]`` and singularity (v3.5). 11 | https://sylabs.io/guides/3.5/user-guide/quick_start.html#quick-installation-steps 12 | 13 | """ 14 | import logging 15 | import pickle 16 | from pathlib import Path 17 | 18 | import hpbandster.core.nameserver as hpns 19 | import hpbandster.core.result as hpres 20 | from hpbandster.core.worker import Worker 21 | from hpbandster.optimizers import BOHB 22 | 23 | from hpobench.container.benchmarks.rl.cartpole import CartpoleReduced as Benchmark 24 | from hpobench.util.example_utils import get_travis_settings, set_env_variables_to_use_only_one_core 25 | from hpobench.util.rng_helper import get_rng 26 | 27 | logger = logging.getLogger('BOHB on cartpole') 28 | set_env_variables_to_use_only_one_core() 29 | 30 | 31 | class CustomWorker(Worker): 32 | def __init__(self, seed, max_budget, *args, **kwargs): 33 | super().__init__(*args, **kwargs) 34 | self.seed = seed 35 | self.max_budget = max_budget 36 | 37 | # pylint: disable=arguments-differ 38 | def compute(self, config, budget, **kwargs): 39 | b = Benchmark(rng=self.seed) 40 | # Old API ---- NO LONGER SUPPORTED ---- This will simply ignore the fidelities 41 | # result_dict = b.objective_function(config, budget=int(budget)) 42 | 43 | # New API ---- Use this 44 | result_dict = b.objective_function(config, fidelity={"budget": int(budget)}) 45 | return {'loss': result_dict['function_value'], 46 | 'info': {'cost': result_dict['cost'], 47 | 'budget': result_dict['budget']}} 48 | 49 | 50 | def run_experiment(out_path, on_travis): 51 | 52 | settings = {'min_budget': 1, 53 | 'max_budget': 9, # number of repetitions; this is the fidelity for this bench 54 | 'num_iterations': 10, # Set this to a low number for demonstration 55 | 'eta': 3, 56 | 'output_dir': Path(out_path) 57 | } 58 | if on_travis: 59 | settings.update(get_travis_settings('bohb')) 60 | 61 | b = Benchmark(rng=1) 62 | 63 | b.get_configuration_space(seed=1) 64 | settings.get('output_dir').mkdir(exist_ok=True) 65 | 66 | cs = b.get_configuration_space() 67 | seed = get_rng(rng=0) 68 | run_id = 'BOHB_on_cartpole' 69 | 70 | result_logger = hpres.json_result_logger(directory=str(settings.get('output_dir')), overwrite=True) 71 | 72 | ns = hpns.NameServer(run_id=run_id, host='localhost', working_directory=str(settings.get('output_dir'))) 73 | ns_host, ns_port = ns.start() 74 | 75 | worker = CustomWorker(seed=seed, 76 | nameserver=ns_host, 77 | nameserver_port=ns_port, 78 | run_id=run_id, 79 | max_budget=settings.get('max_budget')) 80 | worker.run(background=True) 81 | 82 | master = BOHB(configspace=cs, 83 | run_id=run_id, 84 | host=ns_host, 85 | nameserver=ns_host, 86 | nameserver_port=ns_port, 87 | eta=settings.get('eta'), 88 | min_budget=settings.get('min_budget'), 89 | max_budget=settings.get('max_budget'), 90 | result_logger=result_logger) 91 | 92 | result = master.run(n_iterations=settings.get('num_iterations')) 93 | master.shutdown(shutdown_workers=True) 94 | ns.shutdown() 95 | 96 | with open(settings.get('output_dir') / 'results.pkl', 'wb') as f: 97 | pickle.dump(result, f) 98 | 99 | id2config = result.get_id2config_mapping() 100 | incumbent = result.get_incumbent_id() 101 | inc_value = result.get_runs_by_id(incumbent)[-1]['loss'] 102 | inc_cfg = id2config[incumbent]['config'] 103 | 104 | logger.info(f'Inc Config:\n{inc_cfg}\n' 105 | f'with Performance: {inc_value:.2f}') 106 | 107 | if not on_travis: 108 | benchmark = Benchmark(container_source='library://phmueller/automl') 109 | incumbent_result = benchmark.objective_function_test(configuration=inc_cfg, 110 | fidelity={"budget": settings['max_budget']}) 111 | print(incumbent_result) 112 | 113 | 114 | if __name__ == '__main__': 115 | import argparse 116 | parser = argparse.ArgumentParser(prog='HPOBench - BOHB', 117 | description='HPOBench with BOHB on Cartpole', 118 | usage='%(prog)s --out_path ') 119 | parser.add_argument('--out_path', default='./cartpole_smac_hb', type=str) 120 | parser.add_argument('--on_travis', action='store_true', 121 | help='Flag to speed up the run on the continuous integration tool \"travis\". This flag can be' 122 | 'ignored by the user') 123 | args = parser.parse_args() 124 | 125 | run_experiment(out_path=args.out_path, on_travis=args.on_travis) 126 | -------------------------------------------------------------------------------- /examples/w_optimizer/cartpole_smachb.py: -------------------------------------------------------------------------------- 1 | """ 2 | SMAC-HB on Cartpole with Hyperband 3 | =============================== 4 | 5 | This example shows the usage of an Hyperparameter Tuner, such as SMAC on the cartpole benchmark. 6 | We use SMAC with Hyperband. 7 | 8 | **Note**: This is a raw benchmark, i.e. it actually runs an algorithms, and will take some time 9 | 10 | Please install the necessary dependencies via ``pip install .[examples]`` and singularity (v3.5). 11 | https://sylabs.io/guides/3.5/user-guide/quick_start.html#quick-installation-steps 12 | """ 13 | import logging 14 | from pathlib import Path 15 | from time import time 16 | 17 | import numpy as np 18 | from smac.facade.smac_mf_facade import SMAC4MF 19 | from smac.intensification.hyperband import Hyperband 20 | from smac.scenario.scenario import Scenario 21 | 22 | from hpobench.container.benchmarks.rl.cartpole import CartpoleReduced as Benchmark 23 | from hpobench.util.example_utils import get_travis_settings, set_env_variables_to_use_only_one_core 24 | 25 | logger = logging.getLogger("SMAC-HB on cartpole") 26 | logging.basicConfig(level=logging.INFO) 27 | set_env_variables_to_use_only_one_core() 28 | 29 | 30 | def run_experiment(out_path: str, on_travis: bool = False): 31 | 32 | out_path = Path(out_path) 33 | out_path.mkdir(exist_ok=True) 34 | 35 | benchmark = Benchmark(rng=1) 36 | 37 | scenario_dict = {"run_obj": "quality", 38 | "wallclock-limit": 5 * 60 * 60, # max duration to run the optimization (in seconds) 39 | "cs": benchmark.get_configuration_space(seed=1), 40 | "deterministic": "true", 41 | "runcount-limit": 200, 42 | "limit_resources": True, # Uses pynisher to limit memory and runtime 43 | "cutoff": 1800, # runtime limit for target algorithm 44 | "memory_limit": 10000, # adapt this to reasonable value for your hardware 45 | "output_dir": str(out_path), 46 | } 47 | 48 | if on_travis: 49 | scenario_dict.update(get_travis_settings('smac')) 50 | 51 | scenario = Scenario(scenario_dict) 52 | 53 | # Number of Agents, which are trained to solve the cartpole experiment 54 | max_budget = 9 if not on_travis else 2 55 | 56 | def optimization_function_wrapper(cfg, seed, instance, budget): 57 | """ Helper-function: simple wrapper to use the benchmark with smac""" 58 | 59 | # Now that we have already downloaded the container, 60 | # we only have to start a new instance. This is a fast operation. 61 | b = Benchmark(rng=seed) 62 | 63 | # Old API ---- NO LONGER SUPPORTED ---- This will simply ignore the fidelities 64 | # result_dict = b.objective_function(cfg, budget=int(budget)) 65 | 66 | # New API ---- Use this 67 | result_dict = b.objective_function(cfg, fidelity={"budget": int(budget)}) 68 | return result_dict['function_value'] 69 | 70 | smac = SMAC4MF(scenario=scenario, 71 | rng=np.random.RandomState(42), 72 | tae_runner=optimization_function_wrapper, 73 | intensifier=Hyperband, 74 | intensifier_kwargs={'initial_budget': 1, 'max_budget': max_budget, 'eta': 3} 75 | ) 76 | 77 | start_time = time() 78 | try: 79 | smac.optimize() 80 | finally: 81 | incumbent = smac.solver.incumbent 82 | end_time = time() 83 | 84 | if not on_travis: 85 | inc_value = smac.get_tae_runner().run(config=incumbent, instance='1', budget=max_budget, seed=0)[1] 86 | print(f"Value for optimized configuration: {inc_value:.4f}.\n" 87 | f"Optimization took {end_time-start_time:.0f}s") 88 | 89 | 90 | if __name__ == "__main__": 91 | import argparse 92 | parser = argparse.ArgumentParser(prog='HPOBench - Hyperband', 93 | description='HPOBench with HB on Cartpole', 94 | usage='%(prog)s --out_path ') 95 | parser.add_argument('--out_path', default='./cartpole_smac_hb', type=str) 96 | parser.add_argument('--on_travis', action='store_true', 97 | help='Flag to speed up the run on the continuous integration tool \"travis\". This flag can be' 98 | 'ignored by the user') 99 | args = parser.parse_args() 100 | 101 | run_experiment(out_path=args.out_path, on_travis=args.on_travis) 102 | -------------------------------------------------------------------------------- /extra_requirements/cartpole.json: -------------------------------------------------------------------------------- 1 | { 2 | "cartpole": ["tensorflow==1.13.2","gym==0.10.9","tensorforce==0.4.3"] 3 | } -------------------------------------------------------------------------------- /extra_requirements/examples.json: -------------------------------------------------------------------------------- 1 | { 2 | "xgboost_example": ["xgboost==0.90","json_tricks==3.14.0","openml==0.10.2"], 3 | "cartpole_example": ["tensorflow==1.13.2","gym==0.10.9","tensorforce==0.4.3","scikit-learn==0.22.0", 4 | "smac==0.12.2","hpbandster==0.7.4"], 5 | "nasbench_101_example": ["torch>=1.2.0,<=1.5.1","torchvision>=0.4.0", 6 | "nasbench@git+https://github.com/google-research/nasbench#egg=nasbench-0", 7 | "nas_benchmarks@git+https://github.com/automl/nas_benchmarks#egg=nas_benchmarks-0.0.1"] 8 | } 9 | -------------------------------------------------------------------------------- /extra_requirements/ml_mfbb.json: -------------------------------------------------------------------------------- 1 | { 2 | "ml_tabular_benchmarks": ["tqdm","pandas==1.2.4","scikit-learn==0.24.2","openml==0.12.2","xgboost==1.3.1"], 3 | "ml_mfbb": ["tqdm","pandas==1.2.4","scikit-learn==0.24.2","openml==0.12.2","xgboost==1.3.1"] 4 | } -------------------------------------------------------------------------------- /extra_requirements/nasbench_101.json: -------------------------------------------------------------------------------- 1 | { 2 | "nasbench_101": ["tensorflow==1.15.0","tqdm"] 3 | } -------------------------------------------------------------------------------- /extra_requirements/nasbench_1shot1.json: -------------------------------------------------------------------------------- 1 | { 2 | "nasbench_1shot1": ["tensorflow==1.15.0","matplotlib","seaborn", "networkx", "tqdm"] 3 | } -------------------------------------------------------------------------------- /extra_requirements/outlier_detection.json: -------------------------------------------------------------------------------- 1 | { 2 | "outlier_detection": ["torch==1.9.0", "pytorch_lightning==1.3.8", "scikit-learn==0.24.2"] 3 | } -------------------------------------------------------------------------------- /extra_requirements/paramnet.json: -------------------------------------------------------------------------------- 1 | { 2 | "paramnet": ["tqdm","scikit-learn==0.23.2"] 3 | } -------------------------------------------------------------------------------- /extra_requirements/pybnn.json: -------------------------------------------------------------------------------- 1 | { 2 | "pybnn": ["Theano==1.0.5","sgmcmc@git+https://github.com/automl/sgmcmc#egg=sgmcmc-0.0.1", 3 | "Lasagne@git+https://github.com/Lasagne/Lasagne#egg=Lasagne-0.0.1"] 4 | } -------------------------------------------------------------------------------- /extra_requirements/svm.json: -------------------------------------------------------------------------------- 1 | { 2 | "svm": ["pandas>=0.22.2,<0.24.2","openml==0.10.2","scikit-learn>=0.18.1"] 3 | } -------------------------------------------------------------------------------- /extra_requirements/tabular_benchmarks.json: -------------------------------------------------------------------------------- 1 | { 2 | "tabular_benchmarks": ["tensorflow==1.15.0"] 3 | } -------------------------------------------------------------------------------- /extra_requirements/tests.json: -------------------------------------------------------------------------------- 1 | { 2 | "codestyle": ["pycodestyle","flake8","pylint"], 3 | "pytest": ["pytest>=4.6","pytest-cov"], 4 | "test_paramnet": ["tqdm", "scikit-learn==0.23.2"], 5 | "test_tabular_datamanager": ["pyarrow", "fastparquet"] 6 | } -------------------------------------------------------------------------------- /extra_requirements/xgboost.json: -------------------------------------------------------------------------------- 1 | { 2 | "xgboost": ["xgboost==0.90","pandas>=1.0.0,<1.1.5","openml==0.10.2","scikit-learn>=0.18.1"] 3 | } -------------------------------------------------------------------------------- /hpobench/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | _default_log_format = '[%(levelname)s] %(name)s at %(asctime)s --- %(message)s' 4 | logging.basicConfig(format=_default_log_format, level=logging.WARNING) 5 | root_logger = logging.getLogger("hpobench") 6 | 7 | from hpobench.__version__ import __version__ # noqa: F401, E402 8 | from hpobench.config import config_file # noqa: F401, E402 9 | 10 | __contact__ = "automl.org" 11 | -------------------------------------------------------------------------------- /hpobench/__version__.py: -------------------------------------------------------------------------------- 1 | __version__ = '0.0.10' 2 | -------------------------------------------------------------------------------- /hpobench/benchmarks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/HPOBench/47bf141f79e6bdfb26d1f1218b5d5aac09d7d2ce/hpobench/benchmarks/__init__.py -------------------------------------------------------------------------------- /hpobench/benchmarks/ml/README.md: -------------------------------------------------------------------------------- 1 | Each function evalution returns a dictionary with the following information: 2 | 3 | ``` 4 | └───function_value: 1 - accuracy (acc.) on validation set 5 | └───cost: time to fit model + time to evaluate acc. training set + time to evaluate acc. validation set 6 | └───info: dictionary (dict) with miscellaneous information 7 | | └───train_loss: 1 - accuracy (acc.) on training set 8 | | └───val_loss: 1 - accuracy (acc.) on validation set 9 | | └───model_cost: time taken to fit the model 10 | | └───train_scores: performance on all metrics over the training set (dict) 11 | | | └───f1: F1-score 12 | | | └───acc: Accuracy 13 | | | └───bal_acc: Balanced accuracy 14 | | └───train_costs: time taken to compute performance on all metrics over the training set (dict) 15 | | | └───f1: F1-score 16 | | | └───acc: Accuracy 17 | | | └───bal_acc: Balanced accuracy 18 | | └───valid_scores: performance on all metrics over the validation set (dict) 19 | | | └───... 20 | | └───valid_costs: time taken to compute performance on all metrics over the validation set (dict) 21 | | | └───... 22 | | └───test_scores: performance on all metrics over the test set 23 | | | └───... 24 | | └───test_costs: time taken to compute performance on all metrics over the test set (dict) 25 | | | └───... 26 | ``` 27 | 28 | *NOTE*: the keys `function_value`, `cost`, `info` need to exist when creating a new objective 29 | function, while `info` can house any kind of auxilliary information required. -------------------------------------------------------------------------------- /hpobench/benchmarks/ml/__init__.py: -------------------------------------------------------------------------------- 1 | from hpobench.benchmarks.ml.histgb_benchmark import HistGBBenchmark, HistGBBenchmarkBB, HistGBBenchmarkMF 2 | from hpobench.benchmarks.ml.lr_benchmark import LRBenchmark, LRBenchmarkBB, LRBenchmarkMF 3 | from hpobench.benchmarks.ml.nn_benchmark import NNBenchmark, NNBenchmarkBB, NNBenchmarkMF 4 | from hpobench.benchmarks.ml.rf_benchmark import RandomForestBenchmark, RandomForestBenchmarkBB, \ 5 | RandomForestBenchmarkMF 6 | from hpobench.benchmarks.ml.svm_benchmark import SVMBenchmark, SVMBenchmarkBB, SVMBenchmarkMF 7 | from hpobench.benchmarks.ml.tabular_benchmark import TabularBenchmark 8 | 9 | try: 10 | from hpobench.benchmarks.ml.xgboost_benchmark import XGBoostBenchmark, XGBoostBenchmarkBB, XGBoostBenchmarkMF 11 | except ImportError: 12 | pass 13 | 14 | 15 | __all__ = ['HistGBBenchmark', 'HistGBBenchmarkBB', 'HistGBBenchmarkMF', 16 | 'LRBenchmark', 'LRBenchmarkBB', 'LRBenchmarkMF', 17 | 'NNBenchmark', 'NNBenchmarkBB', 'NNBenchmarkMF', 18 | 'RandomForestBenchmark', 'RandomForestBenchmarkBB', 'RandomForestBenchmarkMF', 19 | 'SVMBenchmark', 'SVMBenchmarkBB', 'SVMBenchmarkMF', 20 | 'TabularBenchmark', 21 | 'XGBoostBenchmark', 'XGBoostBenchmarkBB', 'XGBoostBenchmarkMF', 22 | ] 23 | -------------------------------------------------------------------------------- /hpobench/benchmarks/ml/histgb_benchmark.py: -------------------------------------------------------------------------------- 1 | """ 2 | Changelog: 3 | ========== 4 | 5 | 0.0.1: 6 | * First implementation of the HistGB Benchmarks. 7 | """ 8 | 9 | from typing import Union, Tuple, Dict 10 | 11 | import ConfigSpace as CS 12 | import numpy as np 13 | from ConfigSpace.hyperparameters import Hyperparameter 14 | # https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.HistGradientBoostingClassifier.html 15 | from sklearn.experimental import enable_hist_gradient_boosting # noqa 16 | from sklearn.ensemble import HistGradientBoostingClassifier 17 | 18 | from hpobench.dependencies.ml.ml_benchmark_template import MLBenchmark 19 | 20 | __version__ = '0.0.1' 21 | 22 | 23 | class HistGBBenchmark(MLBenchmark): 24 | def __init__(self, 25 | task_id: int, 26 | rng: Union[np.random.RandomState, int, None] = None, 27 | valid_size: float = 0.33, 28 | data_path: Union[str, None] = None): 29 | super(HistGBBenchmark, self).__init__(task_id, rng, valid_size, data_path) 30 | 31 | @staticmethod 32 | def get_configuration_space(seed: Union[int, None] = None) -> CS.ConfigurationSpace: 33 | """Parameter space to be optimized --- contains the hyperparameters""" 34 | cs = CS.ConfigurationSpace(seed=seed) 35 | 36 | cs.add_hyperparameters([ 37 | CS.UniformIntegerHyperparameter( 38 | 'max_depth', lower=6, upper=30, default_value=6, log=True 39 | ), 40 | CS.UniformIntegerHyperparameter( 41 | 'max_leaf_nodes', lower=2, upper=64, default_value=32, log=True 42 | ), 43 | CS.UniformFloatHyperparameter( 44 | 'learning_rate', lower=2**-10, upper=1, default_value=0.1, log=True 45 | ), 46 | CS.UniformFloatHyperparameter( 47 | 'l2_regularization', lower=2**-10, upper=2**10, default_value=0.1, log=True 48 | ) 49 | ]) 50 | return cs 51 | 52 | @staticmethod 53 | def get_fidelity_space(seed: Union[int, None] = None) -> CS.ConfigurationSpace: 54 | fidelity_space = CS.ConfigurationSpace(seed=seed) 55 | fidelity_space.add_hyperparameters( 56 | # gray-box setting (multi-multi-fidelity) - ntrees + data subsample 57 | HistGBBenchmark._get_fidelity_choices(ntrees_choice='variable', subsample_choice='variable') 58 | ) 59 | return fidelity_space 60 | 61 | @staticmethod 62 | def _get_fidelity_choices(ntrees_choice: str, subsample_choice: str) -> Tuple[Hyperparameter, Hyperparameter]: 63 | 64 | assert ntrees_choice in ['fixed', 'variable'] 65 | assert subsample_choice in ['fixed', 'variable'] 66 | 67 | fidelity1 = dict( 68 | # TODO: this value was 100 in the original code. Please check if 100 or 1000. 69 | fixed=CS.Constant('n_estimators', value=1000), 70 | variable=CS.UniformIntegerHyperparameter( 71 | 'n_estimators', lower=100, upper=1000, default_value=1000, log=False 72 | ) 73 | ) 74 | fidelity2 = dict( 75 | fixed=CS.Constant('subsample', value=1), 76 | variable=CS.UniformFloatHyperparameter( 77 | 'subsample', lower=0.1, upper=1, default_value=1, log=False 78 | ) 79 | ) 80 | ntrees = fidelity1[ntrees_choice] 81 | subsample = fidelity2[subsample_choice] 82 | return ntrees, subsample 83 | 84 | def init_model(self, config: Union[CS.Configuration, Dict], 85 | fidelity: Union[CS.Configuration, Dict, None] = None, 86 | rng: Union[int, np.random.RandomState, None] = None): 87 | """ Function that returns the model initialized based on the configuration and fidelity 88 | """ 89 | rng = self.rng if rng is None else rng 90 | 91 | if isinstance(config, CS.Configuration): 92 | config = config.get_dictionary() 93 | if isinstance(fidelity, CS.Configuration): 94 | fidelity = fidelity.get_dictionary() 95 | 96 | model = HistGradientBoostingClassifier( 97 | **config, 98 | max_iter=fidelity['n_estimators'], # a fidelity being used during initialization 99 | early_stopping=False, 100 | random_state=rng 101 | ) 102 | return model 103 | 104 | 105 | class HistGBBenchmarkBB(HistGBBenchmark): 106 | def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationSpace: 107 | fidelity_space = CS.ConfigurationSpace(seed=seed) 108 | fidelity_space.add_hyperparameters( 109 | # black-box setting (full fidelity) 110 | HistGBBenchmark._get_fidelity_choices(ntrees_choice='fixed', subsample_choice='fixed') 111 | ) 112 | return fidelity_space 113 | 114 | 115 | class HistGBBenchmarkMF(HistGBBenchmark): 116 | def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationSpace: 117 | fidelity_space = CS.ConfigurationSpace(seed=seed) 118 | fidelity_space.add_hyperparameters( 119 | # gray-box setting (multi-fidelity) - ntrees 120 | HistGBBenchmark._get_fidelity_choices(ntrees_choice='variable', subsample_choice='fixed') 121 | ) 122 | return fidelity_space 123 | 124 | 125 | __all__ = ['HistGBBenchmark', 'HistGBBenchmarkBB', 'HistGBBenchmarkMF'] 126 | -------------------------------------------------------------------------------- /hpobench/benchmarks/ml/lr_benchmark.py: -------------------------------------------------------------------------------- 1 | """ 2 | Changelog: 3 | ========== 4 | 5 | 0.0.1: 6 | * First implementation of the LR Benchmarks. 7 | """ 8 | 9 | 10 | from typing import Union, Tuple, Dict 11 | 12 | import ConfigSpace as CS 13 | import numpy as np 14 | from ConfigSpace.hyperparameters import Hyperparameter 15 | from sklearn.linear_model import SGDClassifier 16 | 17 | from hpobench.dependencies.ml.ml_benchmark_template import MLBenchmark 18 | 19 | __version__ = '0.0.1' 20 | 21 | 22 | class LRBenchmark(MLBenchmark): 23 | def __init__(self, 24 | task_id: int, 25 | rng: Union[np.random.RandomState, int, None] = None, 26 | valid_size: float = 0.33, 27 | data_path: Union[str, None] = None): 28 | 29 | super(LRBenchmark, self).__init__(task_id, rng, valid_size, data_path) 30 | self.cache_size = 500 31 | 32 | @staticmethod 33 | def get_configuration_space(seed: Union[int, None] = None) -> CS.ConfigurationSpace: 34 | """Parameter space to be optimized --- contains the hyperparameters 35 | """ 36 | cs = CS.ConfigurationSpace(seed=seed) 37 | cs.add_hyperparameters([ 38 | CS.UniformFloatHyperparameter( 39 | "alpha", 1e-5, 1, log=True, default_value=1e-3 40 | ), 41 | CS.UniformFloatHyperparameter( 42 | "eta0", 1e-5, 1, log=True, default_value=1e-2 43 | ) 44 | ]) 45 | return cs 46 | 47 | def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationSpace: 48 | fidelity_space = CS.ConfigurationSpace(seed=seed) 49 | fidelity_space.add_hyperparameters( 50 | # gray-box setting (multi-multi-fidelity) - iterations + data subsample 51 | LRBenchmark._get_fidelity_choices(iter_choice='variable', subsample_choice='variable') 52 | ) 53 | return fidelity_space 54 | 55 | @staticmethod 56 | def _get_fidelity_choices(iter_choice: str, subsample_choice: str) -> Tuple[Hyperparameter, Hyperparameter]: 57 | """Fidelity space available --- specifies the fidelity dimensions 58 | 59 | For SVM, only a single fidelity exists, i.e., subsample fraction. 60 | if fidelity_choice == 0 61 | uses the entire data (subsample=1), reflecting the black-box setup 62 | else 63 | parameterizes the fraction of data to subsample 64 | 65 | """ 66 | 67 | assert iter_choice in ['fixed', 'variable'] 68 | assert subsample_choice in ['fixed', 'variable'] 69 | 70 | fidelity1 = dict( 71 | fixed=CS.Constant('iter', value=1000), 72 | variable=CS.UniformIntegerHyperparameter( 73 | 'iter', lower=10, upper=1000, default_value=1000, log=False 74 | ) 75 | ) 76 | fidelity2 = dict( 77 | fixed=CS.Constant('subsample', value=1.0), 78 | variable=CS.UniformFloatHyperparameter( 79 | 'subsample', lower=0.1, upper=1.0, default_value=1.0, log=False 80 | ) 81 | ) 82 | 83 | iter = fidelity1[iter_choice] 84 | subsample = fidelity2[subsample_choice] 85 | return iter, subsample 86 | 87 | def init_model(self, config: Union[CS.Configuration, Dict], 88 | fidelity: Union[CS.Configuration, Dict, None] = None, 89 | rng: Union[int, np.random.RandomState, None] = None): 90 | # initializing model 91 | rng = self.rng if rng is None else rng 92 | 93 | if isinstance(config, CS.Configuration): 94 | config = config.get_dictionary() 95 | if isinstance(fidelity, CS.Configuration): 96 | fidelity = fidelity.get_dictionary() 97 | 98 | # https://scikit-learn.org/stable/modules/sgd.html 99 | model = SGDClassifier( 100 | **config, 101 | loss="log", # performs Logistic Regression 102 | max_iter=fidelity["iter"], 103 | learning_rate="adaptive", 104 | tol=None, 105 | random_state=rng, 106 | 107 | ) 108 | return model 109 | 110 | 111 | class LRBenchmarkBB(LRBenchmark): 112 | def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationSpace: 113 | fidelity_space = CS.ConfigurationSpace(seed=seed) 114 | fidelity_space.add_hyperparameters( 115 | # black-box setting (full fidelity) 116 | LRBenchmark._get_fidelity_choices(iter_choice='fixed', subsample_choice='fixed') 117 | ) 118 | return fidelity_space 119 | 120 | 121 | class LRBenchmarkMF(LRBenchmark): 122 | def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationSpace: 123 | fidelity_space = CS.ConfigurationSpace(seed=seed) 124 | fidelity_space.add_hyperparameters( 125 | # gray-box setting (multi-fidelity) - iterations 126 | LRBenchmark._get_fidelity_choices(iter_choice='variable', subsample_choice='fixed') 127 | ) 128 | return fidelity_space 129 | 130 | 131 | __all__ = ['LRBenchmark', 'LRBenchmarkBB', 'LRBenchmarkMF'] 132 | -------------------------------------------------------------------------------- /hpobench/benchmarks/ml/nn_benchmark.py: -------------------------------------------------------------------------------- 1 | """ 2 | Changelog: 3 | ========== 4 | 5 | 0.0.1: 6 | * First implementation of the NN Benchmarks. 7 | """ 8 | 9 | from copy import deepcopy 10 | from typing import Union, Tuple, Dict 11 | 12 | import ConfigSpace as CS 13 | import numpy as np 14 | from ConfigSpace.hyperparameters import Hyperparameter 15 | from sklearn.neural_network import MLPClassifier 16 | 17 | from hpobench.dependencies.ml.ml_benchmark_template import MLBenchmark 18 | 19 | __version__ = '0.0.1' 20 | 21 | 22 | class NNBenchmark(MLBenchmark): 23 | def __init__(self, 24 | task_id: int, 25 | rng: Union[np.random.RandomState, int, None] = None, 26 | valid_size: float = 0.33, 27 | data_path: Union[str, None] = None): 28 | super(NNBenchmark, self).__init__(task_id, rng, valid_size, data_path) 29 | 30 | @staticmethod 31 | def get_configuration_space(seed: Union[int, None] = None) -> CS.ConfigurationSpace: 32 | """Parameter space to be optimized --- contains the hyperparameters 33 | """ 34 | cs = CS.ConfigurationSpace(seed=seed) 35 | 36 | cs.add_hyperparameters([ 37 | CS.UniformIntegerHyperparameter( 38 | 'depth', default_value=3, lower=1, upper=3, log=False 39 | ), 40 | CS.UniformIntegerHyperparameter( 41 | 'width', default_value=64, lower=16, upper=1024, log=True 42 | ), 43 | CS.UniformIntegerHyperparameter( 44 | 'batch_size', lower=4, upper=256, default_value=32, log=True 45 | ), 46 | CS.UniformFloatHyperparameter( 47 | 'alpha', lower=10**-8, upper=1, default_value=10**-3, log=True 48 | ), 49 | CS.UniformFloatHyperparameter( 50 | 'learning_rate_init', lower=10**-5, upper=1, default_value=10**-3, log=True 51 | ) 52 | ]) 53 | return cs 54 | 55 | @staticmethod 56 | def get_fidelity_space(seed: Union[int, None] = None) -> CS.ConfigurationSpace: 57 | 58 | fidelity_space = CS.ConfigurationSpace(seed=seed) 59 | fidelity_space.add_hyperparameters( 60 | # gray-box setting (multi-multi-fidelity) - iterations + data subsample 61 | NNBenchmark._get_fidelity_choices(iter_choice='variable', subsample_choice='variable') 62 | ) 63 | return fidelity_space 64 | 65 | @staticmethod 66 | def _get_fidelity_choices(iter_choice: str, subsample_choice: str) -> Tuple[Hyperparameter, Hyperparameter]: 67 | 68 | fidelity1 = dict( 69 | fixed=CS.Constant('iter', value=243), 70 | variable=CS.UniformIntegerHyperparameter( 71 | 'iter', lower=3, upper=243, default_value=243, log=False 72 | ) 73 | ) 74 | fidelity2 = dict( 75 | fixed=CS.Constant('subsample', value=1), 76 | variable=CS.UniformFloatHyperparameter( 77 | 'subsample', lower=0.1, upper=1, default_value=1, log=False 78 | ) 79 | ) 80 | iter = fidelity1[iter_choice] 81 | subsample = fidelity2[subsample_choice] 82 | return iter, subsample 83 | 84 | def init_model(self, config: Union[CS.Configuration, Dict], 85 | fidelity: Union[CS.Configuration, Dict, None] = None, 86 | rng: Union[int, np.random.RandomState, None] = None): 87 | """ Function that returns the model initialized based on the configuration and fidelity 88 | """ 89 | rng = self.rng if rng is None else rng 90 | 91 | if isinstance(config, CS.Configuration): 92 | config = config.get_dictionary() 93 | if isinstance(fidelity, CS.Configuration): 94 | fidelity = fidelity.get_dictionary() 95 | 96 | config = deepcopy(config) 97 | depth = config["depth"] 98 | width = config["width"] 99 | config.pop("depth") 100 | config.pop("width") 101 | hidden_layers = [width] * depth 102 | model = MLPClassifier( 103 | **config, 104 | hidden_layer_sizes=hidden_layers, 105 | activation="relu", 106 | solver="adam", 107 | max_iter=fidelity['iter'], # a fidelity being used during initialization 108 | random_state=rng 109 | ) 110 | return model 111 | 112 | 113 | class NNBenchmarkBB(NNBenchmark): 114 | def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationSpace: 115 | fidelity_space = CS.ConfigurationSpace(seed=seed) 116 | fidelity_space.add_hyperparameters( 117 | # black-box setting (full fidelity) 118 | NNBenchmarkBB._get_fidelity_choices(iter_choice='fixed', subsample_choice='fixed') 119 | ) 120 | return fidelity_space 121 | 122 | 123 | class NNBenchmarkMF(NNBenchmark): 124 | def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationSpace: 125 | fidelity_space = CS.ConfigurationSpace(seed=seed) 126 | fidelity_space.add_hyperparameters( 127 | # gray-box setting (multi-fidelity) - iterations 128 | NNBenchmarkMF._get_fidelity_choices(iter_choice='variable', subsample_choice='fixed') 129 | ) 130 | return fidelity_space 131 | 132 | 133 | __all__ = ['NNBenchmark', 'NNBenchmarkBB', 'NNBenchmarkMF'] 134 | -------------------------------------------------------------------------------- /hpobench/benchmarks/ml/rf_benchmark.py: -------------------------------------------------------------------------------- 1 | """ 2 | Changelog: 3 | ========== 4 | 5 | 0.0.1: 6 | * First implementation of the RF Benchmarks. 7 | """ 8 | 9 | from copy import deepcopy 10 | from typing import Union, Tuple, Dict 11 | 12 | import ConfigSpace as CS 13 | import numpy as np 14 | from ConfigSpace.hyperparameters import Hyperparameter 15 | from sklearn.ensemble import RandomForestClassifier 16 | 17 | from hpobench.dependencies.ml.ml_benchmark_template import MLBenchmark 18 | 19 | __version__ = '0.0.1' 20 | 21 | 22 | class RandomForestBenchmark(MLBenchmark): 23 | def __init__(self, 24 | task_id: int, 25 | rng: Union[np.random.RandomState, int, None] = None, 26 | valid_size: float = 0.33, 27 | data_path: Union[str, None] = None): 28 | super(RandomForestBenchmark, self).__init__(task_id, rng, valid_size, data_path) 29 | 30 | @staticmethod 31 | def get_configuration_space(seed: Union[int, None] = None) -> CS.ConfigurationSpace: 32 | """Parameter space to be optimized --- contains the hyperparameters 33 | """ 34 | cs = CS.ConfigurationSpace(seed=seed) 35 | cs.add_hyperparameters([ 36 | CS.UniformIntegerHyperparameter( 37 | 'max_depth', lower=1, upper=50, default_value=10, log=True 38 | ), 39 | CS.UniformIntegerHyperparameter( 40 | 'min_samples_split', lower=2, upper=128, default_value=32, log=True 41 | ), 42 | # the use of a float max_features is different than the sklearn usage 43 | CS.UniformFloatHyperparameter( 44 | 'max_features', lower=0, upper=1.0, default_value=0.5, log=False 45 | ), 46 | CS.UniformIntegerHyperparameter( 47 | 'min_samples_leaf', lower=1, upper=20, default_value=1, log=False 48 | ), 49 | ]) 50 | return cs 51 | 52 | @staticmethod 53 | def get_fidelity_space(seed: Union[int, None] = None) -> CS.ConfigurationSpace: 54 | fidelity_space = CS.ConfigurationSpace(seed=seed) 55 | fidelity_space.add_hyperparameters( 56 | # gray-box setting (multi-multi-fidelity) - ntrees + data subsample 57 | RandomForestBenchmark._get_fidelity_choices(n_estimators_choice='variable', subsample_choice='variable') 58 | ) 59 | return fidelity_space 60 | 61 | @staticmethod 62 | def _get_fidelity_choices(n_estimators_choice: str, subsample_choice: str) -> Tuple[Hyperparameter, Hyperparameter]: 63 | 64 | assert n_estimators_choice in ['fixed', 'variable'] 65 | assert subsample_choice in ['fixed', 'variable'] 66 | 67 | fidelity1 = dict( 68 | fixed=CS.Constant('n_estimators', value=512), 69 | variable=CS.UniformIntegerHyperparameter( 70 | 'n_estimators', lower=16, upper=512, default_value=512, log=False 71 | ) 72 | ) 73 | 74 | fidelity2 = dict( 75 | fixed=CS.Constant('subsample', value=1), 76 | variable=CS.UniformFloatHyperparameter( 77 | 'subsample', lower=0.1, upper=1, default_value=1, log=False 78 | ) 79 | ) 80 | n_estimators = fidelity1[n_estimators_choice] 81 | subsample = fidelity2[subsample_choice] 82 | return n_estimators, subsample 83 | 84 | def init_model(self, config: Union[CS.Configuration, Dict], 85 | fidelity: Union[CS.Configuration, Dict, None] = None, 86 | rng: Union[int, np.random.RandomState, None] = None): 87 | """ Function that returns the model initialized based on the configuration and fidelity 88 | """ 89 | rng = self.rng if rng is None else rng 90 | if isinstance(config, CS.Configuration): 91 | config = config.get_dictionary() 92 | if isinstance(fidelity, CS.Configuration): 93 | fidelity = fidelity.get_dictionary() 94 | 95 | config = deepcopy(config) 96 | n_features = self.train_X.shape[1] 97 | config["max_features"] = int(np.rint(np.power(n_features, config["max_features"]))) 98 | model = RandomForestClassifier( 99 | **config, 100 | n_estimators=fidelity['n_estimators'], # a fidelity being used during initialization 101 | bootstrap=True, 102 | random_state=rng 103 | ) 104 | return model 105 | 106 | 107 | class RandomForestBenchmarkBB(RandomForestBenchmark): 108 | def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationSpace: 109 | fidelity_space = CS.ConfigurationSpace(seed=seed) 110 | fidelity_space.add_hyperparameters( 111 | # black-box setting (full fidelity) 112 | RandomForestBenchmark._get_fidelity_choices(n_estimators_choice='fixed', subsample_choice='fixed') 113 | ) 114 | return fidelity_space 115 | 116 | 117 | class RandomForestBenchmarkMF(RandomForestBenchmark): 118 | def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationSpace: 119 | fidelity_space = CS.ConfigurationSpace(seed=seed) 120 | fidelity_space.add_hyperparameters( 121 | # gray-box setting (multi-fidelity) - ntrees 122 | RandomForestBenchmark._get_fidelity_choices(n_estimators_choice='variable', subsample_choice='fixed') 123 | ) 124 | return fidelity_space 125 | 126 | 127 | __all__ = ['RandomForestBenchmark', 'RandomForestBenchmarkBB', 'RandomForestBenchmarkMF'] 128 | -------------------------------------------------------------------------------- /hpobench/benchmarks/ml/svm_benchmark.py: -------------------------------------------------------------------------------- 1 | """ 2 | Changelog: 3 | ========== 4 | 5 | 0.0.1: 6 | * First implementation of the new SVM Benchmarks. 7 | """ 8 | 9 | from typing import Union, Dict 10 | 11 | import ConfigSpace as CS 12 | import numpy as np 13 | from ConfigSpace.hyperparameters import Hyperparameter 14 | from sklearn.svm import SVC 15 | 16 | from hpobench.dependencies.ml.ml_benchmark_template import MLBenchmark 17 | 18 | __version__ = '0.0.1' 19 | 20 | 21 | class SVMBenchmark(MLBenchmark): 22 | def __init__(self, 23 | task_id: int, 24 | rng: Union[np.random.RandomState, int, None] = None, 25 | valid_size: float = 0.33, 26 | data_path: Union[str, None] = None): 27 | super(SVMBenchmark, self).__init__(task_id, rng, valid_size, data_path) 28 | 29 | self.cache_size = 200 30 | 31 | @staticmethod 32 | def get_configuration_space(seed: Union[int, None] = None) -> CS.ConfigurationSpace: 33 | """Parameter space to be optimized --- contains the hyperparameters 34 | """ 35 | cs = CS.ConfigurationSpace(seed=seed) 36 | # https://jmlr.org/papers/volume20/18-444/18-444.pdf (Table 1) 37 | cs.add_hyperparameters([ 38 | CS.UniformFloatHyperparameter( 39 | "C", 2**-10, 2**10, log=True, default_value=1.0 40 | ), 41 | CS.UniformFloatHyperparameter( 42 | "gamma", 2**-10, 2**10, log=True, default_value=0.1 43 | ) 44 | ]) 45 | return cs 46 | 47 | @staticmethod 48 | def get_fidelity_space(seed: Union[int, None] = None) -> CS.ConfigurationSpace: 49 | fidelity_space = CS.ConfigurationSpace(seed=seed) 50 | fidelity_space.add_hyperparameter( 51 | SVMBenchmark._get_fidelity_choices(subsample_choice='variable') 52 | ) 53 | return fidelity_space 54 | 55 | @staticmethod 56 | def _get_fidelity_choices(subsample_choice: str) -> Hyperparameter: 57 | 58 | assert subsample_choice in ['fixed', 'variable'] 59 | 60 | fidelity = dict( 61 | fixed=CS.Constant('subsample', value=1), 62 | variable=CS.UniformFloatHyperparameter( 63 | 'subsample', lower=0.1, upper=1.0, default_value=1.0, log=False 64 | ) 65 | ) 66 | subsample = fidelity[subsample_choice] 67 | 68 | return subsample 69 | 70 | def init_model(self, config: Union[CS.Configuration, Dict], 71 | fidelity: Union[CS.Configuration, Dict, None] = None, 72 | rng: Union[int, np.random.RandomState, None] = None): 73 | # initializing model 74 | rng = self.rng if rng is None else rng 75 | if isinstance(config, CS.Configuration): 76 | config = config.get_dictionary() 77 | model = SVC( 78 | **config, 79 | random_state=rng, 80 | cache_size=self.cache_size 81 | ) 82 | return model 83 | 84 | 85 | class SVMBenchmarkBB(SVMBenchmark): 86 | def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationSpace: 87 | fidelity_space = CS.ConfigurationSpace(seed=seed) 88 | fidelity_space.add_hyperparameter( 89 | # uses the entire data (subsample=1), reflecting the black-box setup 90 | SVMBenchmark._get_fidelity_choices(subsample_choice='fixed') 91 | ) 92 | return fidelity_space 93 | 94 | 95 | # To keep the parity of the the overall design 96 | SVMBenchmarkMF = SVMBenchmark 97 | 98 | __all__ = ['SVMBenchmark', 'SVMBenchmarkMF', 'SVMBenchmarkBB'] 99 | -------------------------------------------------------------------------------- /hpobench/benchmarks/ml/xgboost_benchmark.py: -------------------------------------------------------------------------------- 1 | """ 2 | Changelog: 3 | ========== 4 | 5 | 0.0.1: 6 | * First implementation of the new XGB Benchmarks. 7 | """ 8 | from typing import Union, Tuple, Dict 9 | 10 | import ConfigSpace as CS 11 | import numpy as np 12 | import xgboost as xgb 13 | from ConfigSpace.hyperparameters import Hyperparameter 14 | 15 | from hpobench.dependencies.ml.ml_benchmark_template import MLBenchmark 16 | 17 | __version__ = '0.0.1' 18 | 19 | 20 | class XGBoostBenchmark(MLBenchmark): 21 | def __init__(self, 22 | task_id: int, 23 | rng: Union[np.random.RandomState, int, None] = None, 24 | valid_size: float = 0.33, 25 | data_path: Union[str, None] = None): 26 | super(XGBoostBenchmark, self).__init__(task_id, rng, valid_size, data_path) 27 | 28 | @staticmethod 29 | def get_configuration_space(seed: Union[int, None] = None) -> CS.ConfigurationSpace: 30 | """Parameter space to be optimized --- contains the hyperparameters 31 | """ 32 | cs = CS.ConfigurationSpace(seed=seed) 33 | 34 | cs.add_hyperparameters([ 35 | CS.UniformFloatHyperparameter( 36 | 'eta', lower=2**-10, upper=1., default_value=0.3, log=True 37 | ), # learning rate 38 | CS.UniformIntegerHyperparameter( 39 | 'max_depth', lower=1, upper=50, default_value=10, log=True 40 | ), 41 | CS.UniformFloatHyperparameter( 42 | 'colsample_bytree', lower=0.1, upper=1., default_value=1., log=False 43 | ), 44 | CS.UniformFloatHyperparameter( 45 | 'reg_lambda', lower=2**-10, upper=2**10, default_value=1, log=True 46 | ) 47 | ]) 48 | return cs 49 | 50 | @staticmethod 51 | def get_fidelity_space(seed: Union[int, None] = None) -> CS.ConfigurationSpace: 52 | fidelity_space = CS.ConfigurationSpace(seed=seed) 53 | fidelity_space.add_hyperparameters( 54 | # gray-box setting (multi-multi-fidelity) - ntrees + data subsample 55 | XGBoostBenchmark._get_fidelity_choices(n_estimators_choice='variable', subsample_choice='variable') 56 | ) 57 | return fidelity_space 58 | 59 | @staticmethod 60 | def _get_fidelity_choices(n_estimators_choice: str, subsample_choice: str) -> Tuple[Hyperparameter, Hyperparameter]: 61 | 62 | assert n_estimators_choice in ['fixed', 'variable'] 63 | assert subsample_choice in ['fixed', 'variable'] 64 | 65 | fidelity1 = dict( 66 | fixed=CS.Constant('n_estimators', value=2000), 67 | variable=CS.UniformIntegerHyperparameter( 68 | 'n_estimators', lower=50, upper=2000, default_value=2000, log=False 69 | ) 70 | ) 71 | fidelity2 = dict( 72 | fixed=CS.Constant('subsample', value=1), 73 | variable=CS.UniformFloatHyperparameter( 74 | 'subsample', lower=0.1, upper=1, default_value=1, log=False 75 | ) 76 | ) 77 | 78 | n_estimators = fidelity1[n_estimators_choice] 79 | subsample = fidelity2[subsample_choice] 80 | return n_estimators, subsample 81 | 82 | def init_model(self, 83 | config: Union[CS.Configuration, Dict], 84 | fidelity: Union[CS.Configuration, Dict, None] = None, 85 | rng: Union[int, np.random.RandomState, None] = None): 86 | """ Function that returns the model initialized based on the configuration and fidelity 87 | """ 88 | if isinstance(config, CS.Configuration): 89 | config = config.get_dictionary() 90 | if isinstance(fidelity, CS.Configuration): 91 | fidelity = fidelity.get_dictionary() 92 | 93 | rng = rng if (rng is None or isinstance(rng, int)) else self.seed 94 | extra_args = dict( 95 | booster="gbtree", 96 | n_estimators=fidelity['n_estimators'], 97 | objective="binary:logistic", 98 | random_state=rng, 99 | subsample=1 100 | ) 101 | if self.n_classes > 2: 102 | extra_args["objective"] = "multi:softmax" 103 | extra_args.update({"num_class": self.n_classes}) 104 | 105 | model = xgb.XGBClassifier( 106 | **config, 107 | **extra_args 108 | ) 109 | return model 110 | 111 | 112 | class XGBoostBenchmarkBB(XGBoostBenchmark): 113 | def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationSpace: 114 | fidelity_space = CS.ConfigurationSpace(seed=seed) 115 | fidelity_space.add_hyperparameters( 116 | # black-box setting (full fidelity) 117 | XGBoostBenchmark._get_fidelity_choices(n_estimators_choice='fixed', subsample_choice='fixed') 118 | ) 119 | return fidelity_space 120 | 121 | 122 | class XGBoostBenchmarkMF(XGBoostBenchmark): 123 | def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationSpace: 124 | fidelity_space = CS.ConfigurationSpace(seed=seed) 125 | fidelity_space.add_hyperparameters( 126 | # gray-box setting (multi-fidelity) - ntrees 127 | XGBoostBenchmark._get_fidelity_choices(n_estimators_choice='variable', subsample_choice='fixed') 128 | ) 129 | return fidelity_space 130 | 131 | 132 | __all__ = ['XGBoostBenchmarkBB', 'XGBoostBenchmarkMF', 'XGBoostBenchmark'] 133 | -------------------------------------------------------------------------------- /hpobench/benchmarks/nas/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/HPOBench/47bf141f79e6bdfb26d1f1218b5d5aac09d7d2ce/hpobench/benchmarks/nas/__init__.py -------------------------------------------------------------------------------- /hpobench/benchmarks/od/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/HPOBench/47bf141f79e6bdfb26d1f1218b5d5aac09d7d2ce/hpobench/benchmarks/od/__init__.py -------------------------------------------------------------------------------- /hpobench/benchmarks/od/od_benchmarks.py: -------------------------------------------------------------------------------- 1 | """ 2 | This is just an entry point for starting the benchmarks. 3 | """ 4 | 5 | 6 | from hpobench.benchmarks.od.od_ae import ODAutoencoder 7 | from hpobench.benchmarks.od.od_kde import ODKernelDensityEstimation 8 | from hpobench.benchmarks.od.od_ocsvm import ODOneClassSupportVectorMachine 9 | 10 | __all__ = [ODAutoencoder, ODKernelDensityEstimation, ODOneClassSupportVectorMachine] 11 | -------------------------------------------------------------------------------- /hpobench/benchmarks/od/od_kde.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from typing import Union 3 | 4 | import ConfigSpace as CS 5 | import ConfigSpace.hyperparameters as CSH 6 | import numpy as np 7 | from sklearn.neighbors import KernelDensity 8 | 9 | from hpobench.dependencies.od.traditional_benchmark import ODTraditional 10 | 11 | __version__ = '0.0.1' 12 | 13 | logger = logging.getLogger('ODKernelDensityEstimation') 14 | 15 | 16 | class ODKernelDensityEstimation(ODTraditional): 17 | """ 18 | Benchmark to train a Kernel Density Estimation (KDE) model for outlier detection. Overall, 19 | this benchmark can be used with one of 15 datasets (using a contamination ratio of 10%) provided by the 20 | ODDS Library (Rayana, 2016). Internally, a 4-fold cross-validation is used to prevent overfitting. 21 | Area under the precission-recall curve (AUPR) is used as metric. 22 | """ 23 | 24 | def __init__(self, 25 | dataset_name: str, 26 | rng: Union[np.random.RandomState, int, None] = None): 27 | """ 28 | Parameters 29 | ---------- 30 | dataset_name : str 31 | Must be one of [ 32 | "annthyroid", "arrhythmia", "breastw", "cardio", "ionosphere", 33 | "mammography", "musk", "optdigits", "pendigits", "pima", 34 | "satellite", "satimage-2", "thyroid", "vowels", "wbc"] 35 | rng : np.random.RandomState, int, None 36 | """ 37 | super(ODKernelDensityEstimation, self).__init__( 38 | dataset_name=dataset_name, 39 | rng=rng 40 | ) 41 | 42 | def get_name(self): 43 | """Returns the name of the model for the meta information.""" 44 | return "Kernel Density Estimation" 45 | 46 | def get_model(self, configuration): 47 | """Returns the unfitted model given a configuration.""" 48 | hp_bandwidth = float(configuration['bandwidth']) 49 | 50 | return KernelDensity(kernel=configuration["kernel"], bandwidth=hp_bandwidth) 51 | 52 | def calculate_scores(self, model, X): 53 | """Calculates the scores based on the model and X.""" 54 | return (-1.) * model.score_samples(X) 55 | 56 | @staticmethod 57 | def get_configuration_space(seed: Union[int, None] = None) -> CS.ConfigurationSpace: 58 | """ 59 | Creates a ConfigSpace.ConfigurationSpace containing all parameters for 60 | the OCSVM Model. 61 | 62 | Parameters 63 | ---------- 64 | seed : int, None 65 | Fixing the seed for the ConfigSpace.ConfigurationSpace 66 | 67 | Returns 68 | ------- 69 | ConfigSpace.ConfigurationSpace 70 | """ 71 | 72 | seed = seed if seed is not None else np.random.randint(1, 100000) 73 | cs = CS.ConfigurationSpace(seed=seed) 74 | 75 | bandwidth = CSH.UniformFloatHyperparameter('bandwidth', lower=pow(2, -5), upper=pow(2, 5), log=True) 76 | cs.add_hyperparameter(bandwidth) 77 | 78 | # Kernel 79 | kernels = ['gaussian', 'tophat', 'epanechnikov', 'exponential', 'linear', 'cosine'] 80 | choice = CSH.CategoricalHyperparameter( 81 | 'kernel', 82 | kernels, 83 | default_value=kernels[0] 84 | ) 85 | cs.add_hyperparameter(choice) 86 | 87 | # Scaler 88 | scalers = ["None", "MinMax", "Standard"] 89 | choice = CSH.CategoricalHyperparameter( 90 | 'scaler', 91 | scalers, 92 | default_value=scalers[0] 93 | ) 94 | cs.add_hyperparameter(choice) 95 | 96 | return cs 97 | -------------------------------------------------------------------------------- /hpobench/benchmarks/od/od_ocsvm.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from typing import Union 3 | 4 | import ConfigSpace as CS 5 | import ConfigSpace.hyperparameters as CSH 6 | import numpy as np 7 | from sklearn.svm import OneClassSVM 8 | 9 | from hpobench.dependencies.od.traditional_benchmark import ODTraditional 10 | 11 | __version__ = '0.0.1' 12 | 13 | logger = logging.getLogger('ODOneClassSupportVectorMachine') 14 | 15 | 16 | class ODOneClassSupportVectorMachine(ODTraditional): 17 | """ 18 | Benchmark to train a One-Class Support Vector Machine (OC-SVM) model for outlier detection. Overall, 19 | this benchmark can be used with one of 15 datasets (using a contamination ratio of 10%) provided by the 20 | ODDS Library (Rayana, 2016). Internally, a 4-fold cross-validation is used to prevent overfitting. 21 | Area under the precission-recall curve (AUPR) is used as metric. 22 | """ 23 | 24 | def __init__(self, 25 | dataset_name: str, 26 | rng: Union[np.random.RandomState, int, None] = None): 27 | """ 28 | Parameters 29 | ---------- 30 | dataset_name : str 31 | Must be one of [ 32 | "annthyroid", "arrhythmia", "breastw", "cardio", "ionosphere", 33 | "mammography", "musk", "optdigits", "pendigits", "pima", 34 | "satellite", "satimage-2", "thyroid", "vowels", "wbc"] 35 | rng : np.random.RandomState, int, None 36 | """ 37 | super(ODOneClassSupportVectorMachine, self).__init__( 38 | dataset_name=dataset_name, 39 | rng=rng 40 | ) 41 | 42 | def get_name(self): 43 | """Returns the name of the model for the meta information.""" 44 | return "One Class Support Vector Machine" 45 | 46 | def get_model(self, configuration): 47 | """Returns the unfitted model given a configuration.""" 48 | hp_gamma = float(configuration['gamma']) 49 | hp_nu = float(configuration['nu']) 50 | 51 | return OneClassSVM(kernel="rbf", gamma=hp_gamma, nu=hp_nu) 52 | 53 | def calculate_scores(self, model, X): 54 | """Calculates the scores based on the model and X.""" 55 | return (-1) * model.decision_function(X) 56 | 57 | @staticmethod 58 | def get_configuration_space(seed: Union[int, None] = None) -> CS.ConfigurationSpace: 59 | """ 60 | Creates a ConfigSpace.ConfigurationSpace containing all parameters for 61 | the OCSVM Model. 62 | 63 | Parameters 64 | ---------- 65 | seed : int, None 66 | Fixing the seed for the ConfigSpace.ConfigurationSpace 67 | 68 | Returns 69 | ------- 70 | ConfigSpace.ConfigurationSpace 71 | """ 72 | 73 | seed = seed if seed is not None else np.random.randint(1, 100000) 74 | cs = CS.ConfigurationSpace(seed=seed) 75 | 76 | cs.add_hyperparameters([ 77 | CS.UniformFloatHyperparameter('gamma', lower=pow(2, -20), upper=pow(2, -2), log=True), 78 | CS.UniformFloatHyperparameter('nu', lower=0.0, upper=1.0, default_value=0.5), 79 | ]) 80 | 81 | # Scaler 82 | scalers = ["None", "MinMax", "Standard"] 83 | choice = CSH.CategoricalHyperparameter( 84 | 'scaler', 85 | scalers, 86 | default_value=scalers[0] 87 | ) 88 | cs.add_hyperparameter(choice) 89 | 90 | return cs 91 | -------------------------------------------------------------------------------- /hpobench/benchmarks/rl/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/HPOBench/47bf141f79e6bdfb26d1f1218b5d5aac09d7d2ce/hpobench/benchmarks/rl/__init__.py -------------------------------------------------------------------------------- /hpobench/benchmarks/surrogates/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/HPOBench/47bf141f79e6bdfb26d1f1218b5d5aac09d7d2ce/hpobench/benchmarks/surrogates/__init__.py -------------------------------------------------------------------------------- /hpobench/container/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/HPOBench/47bf141f79e6bdfb26d1f1218b5d5aac09d7d2ce/hpobench/container/__init__.py -------------------------------------------------------------------------------- /hpobench/container/benchmarks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/HPOBench/47bf141f79e6bdfb26d1f1218b5d5aac09d7d2ce/hpobench/container/benchmarks/__init__.py -------------------------------------------------------------------------------- /hpobench/container/benchmarks/ml/__init__.py: -------------------------------------------------------------------------------- 1 | from hpobench.container.benchmarks.ml.histgb_benchmark import HistGBBenchmark, HistGBBenchmarkBB, HistGBBenchmarkMF 2 | from hpobench.container.benchmarks.ml.lr_benchmark import LRBenchmark, LRBenchmarkBB, LRBenchmarkMF 3 | from hpobench.container.benchmarks.ml.nn_benchmark import NNBenchmark, NNBenchmarkBB, NNBenchmarkMF 4 | from hpobench.container.benchmarks.ml.rf_benchmark import RandomForestBenchmark, RandomForestBenchmarkBB, \ 5 | RandomForestBenchmarkMF 6 | from hpobench.container.benchmarks.ml.svm_benchmark import SVMBenchmark, SVMBenchmarkBB, SVMBenchmarkMF 7 | from hpobench.container.benchmarks.ml.tabular_benchmark import TabularBenchmark 8 | from hpobench.container.benchmarks.ml.xgboost_benchmark import XGBoostBenchmark, XGBoostBenchmarkBB, XGBoostBenchmarkMF 9 | 10 | 11 | __all__ = ['HistGBBenchmark', 'HistGBBenchmarkBB', 'HistGBBenchmarkMF', 12 | 'LRBenchmark', 'LRBenchmarkBB', 'LRBenchmarkMF', 13 | 'NNBenchmark', 'NNBenchmarkBB', 'NNBenchmarkMF', 14 | 'RandomForestBenchmark', 'RandomForestBenchmarkBB', 'RandomForestBenchmarkMF', 15 | 'SVMBenchmark', 'SVMBenchmarkBB', 'SVMBenchmarkMF', 16 | 'TabularBenchmark', 17 | 'XGBoostBenchmark', 'XGBoostBenchmarkBB', 'XGBoostBenchmarkMF'] 18 | -------------------------------------------------------------------------------- /hpobench/container/benchmarks/ml/histgb_benchmark.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | 4 | """ Benchmark for the HistGB Benchmarks from hpobench/benchmarks/ml_mmfb/histgb_benchmark.py """ 5 | 6 | from hpobench.container.client_abstract_benchmark import AbstractBenchmarkClient 7 | 8 | 9 | class HistGBBenchmark(AbstractBenchmarkClient): 10 | def __init__(self, **kwargs): 11 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'HistGBBenchmark') 12 | kwargs['container_name'] = kwargs.get('container_name', 'ml_mmfb') 13 | kwargs['latest'] = kwargs.get('container_tag', '0.0.1') 14 | super(HistGBBenchmark, self).__init__(**kwargs) 15 | 16 | 17 | class HistGBBenchmarkBB(AbstractBenchmarkClient): 18 | def __init__(self, **kwargs): 19 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'HistGBBenchmarkBB') 20 | kwargs['container_name'] = kwargs.get('container_name', 'ml_mmfb') 21 | kwargs['latest'] = kwargs.get('container_tag', '0.0.1') 22 | super(HistGBBenchmarkBB, self).__init__(**kwargs) 23 | 24 | 25 | class HistGBBenchmarkMF(AbstractBenchmarkClient): 26 | def __init__(self, **kwargs): 27 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'HistGBBenchmarkMF') 28 | kwargs['container_name'] = kwargs.get('container_name', 'ml_mmfb') 29 | kwargs['latest'] = kwargs.get('container_tag', '0.0.1') 30 | super(HistGBBenchmarkMF, self).__init__(**kwargs) 31 | 32 | 33 | __all__ = ['HistGBBenchmark', 'HistGBBenchmarkBB', 'HistGBBenchmarkMF'] 34 | -------------------------------------------------------------------------------- /hpobench/container/benchmarks/ml/lr_benchmark.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | 4 | """ Benchmark for the learning rate Benchmarks from hpobench/benchmarks/ml_mmfb/lr_benchmarks.py """ 5 | 6 | from hpobench.container.client_abstract_benchmark import AbstractBenchmarkClient 7 | 8 | 9 | class LRBenchmark(AbstractBenchmarkClient): 10 | def __init__(self, **kwargs): 11 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'LRBenchmark') 12 | kwargs['container_name'] = kwargs.get('container_name', 'ml_mmfb') 13 | kwargs['latest'] = kwargs.get('container_tag', '0.0.1') 14 | super(LRBenchmark, self).__init__(**kwargs) 15 | 16 | 17 | class LRBenchmarkBB(AbstractBenchmarkClient): 18 | def __init__(self, **kwargs): 19 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'LRBenchmarkBB') 20 | kwargs['container_name'] = kwargs.get('container_name', 'ml_mmfb') 21 | kwargs['latest'] = kwargs.get('container_tag', '0.0.1') 22 | super(LRBenchmarkBB, self).__init__(**kwargs) 23 | 24 | 25 | class LRBenchmarkMF(AbstractBenchmarkClient): 26 | def __init__(self, **kwargs): 27 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'LRBenchmarkMF') 28 | kwargs['container_name'] = kwargs.get('container_name', 'ml_mmfb') 29 | kwargs['latest'] = kwargs.get('container_tag', '0.0.1') 30 | super(LRBenchmarkMF, self).__init__(**kwargs) 31 | 32 | 33 | __all__ = ['LRBenchmark', 'LRBenchmarkBB', 'LRBenchmarkMF'] 34 | -------------------------------------------------------------------------------- /hpobench/container/benchmarks/ml/nn_benchmark.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | 4 | """ Benchmark for the Neural Network Benchmarks from hpobench/benchmarks/ml_mmfb/nn_benchmark.py """ 5 | 6 | from hpobench.container.client_abstract_benchmark import AbstractBenchmarkClient 7 | 8 | 9 | class NNBenchmark(AbstractBenchmarkClient): 10 | def __init__(self, **kwargs): 11 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'NNBenchmark') 12 | kwargs['container_name'] = kwargs.get('container_name', 'ml_mmfb') 13 | kwargs['latest'] = kwargs.get('container_tag', '0.0.1') 14 | super(NNBenchmark, self).__init__(**kwargs) 15 | 16 | 17 | class NNBenchmarkBB(AbstractBenchmarkClient): 18 | def __init__(self, **kwargs): 19 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'NNBenchmarkBB') 20 | kwargs['container_name'] = kwargs.get('container_name', 'ml_mmfb') 21 | kwargs['latest'] = kwargs.get('container_tag', '0.0.1') 22 | super(NNBenchmarkBB, self).__init__(**kwargs) 23 | 24 | 25 | class NNBenchmarkMF(AbstractBenchmarkClient): 26 | def __init__(self, **kwargs): 27 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'NNBenchmarkMF') 28 | kwargs['container_name'] = kwargs.get('container_name', 'ml_mmfb') 29 | kwargs['latest'] = kwargs.get('container_tag', '0.0.1') 30 | super(NNBenchmarkMF, self).__init__(**kwargs) 31 | 32 | 33 | __all__ = ['NNBenchmark', 'NNBenchmarkBB', 'NNBenchmarkMF'] 34 | -------------------------------------------------------------------------------- /hpobench/container/benchmarks/ml/pybnn.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | 4 | """ Benchmark for the pybnn Benchmark from hpobench/benchmarks/ml/pybnn.py """ 5 | 6 | from hpobench.container.client_abstract_benchmark import AbstractBenchmarkClient 7 | 8 | 9 | class BNNOnToyFunction(AbstractBenchmarkClient): 10 | def __init__(self, **kwargs): 11 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'BNNOnToyFunction') 12 | kwargs['container_name'] = kwargs.get('container_name', 'pybnn') 13 | kwargs['latest'] = kwargs.get('container_tag', '0.0.4') 14 | super(BNNOnToyFunction, self).__init__(**kwargs) 15 | 16 | 17 | class BNNOnBostonHousing(AbstractBenchmarkClient): 18 | def __init__(self, **kwargs): 19 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'BNNOnBostonHousing') 20 | kwargs['container_name'] = kwargs.get('container_name', 'pybnn') 21 | kwargs['latest'] = kwargs.get('container_tag', '0.0.4') 22 | super(BNNOnBostonHousing, self).__init__(**kwargs) 23 | 24 | 25 | class BNNOnProteinStructure(AbstractBenchmarkClient): 26 | def __init__(self, **kwargs): 27 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'BNNOnProteinStructure') 28 | kwargs['container_name'] = kwargs.get('container_name', 'pybnn') 29 | kwargs['latest'] = kwargs.get('container_tag', '0.0.4') 30 | super(BNNOnProteinStructure, self).__init__(**kwargs) 31 | 32 | 33 | class BNNOnYearPrediction(AbstractBenchmarkClient): 34 | def __init__(self, **kwargs): 35 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'BNNOnYearPrediction') 36 | kwargs['container_name'] = kwargs.get('container_name', 'pybnn') 37 | kwargs['latest'] = kwargs.get('container_tag', '0.0.4') 38 | super(BNNOnYearPrediction, self).__init__(**kwargs) 39 | -------------------------------------------------------------------------------- /hpobench/container/benchmarks/ml/rf_benchmark.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | 4 | """ Benchmark for the Random Forest Benchmarks from hpobench/benchmarks/ml_mmfb/rf_benchmark.py """ 5 | 6 | from hpobench.container.client_abstract_benchmark import AbstractBenchmarkClient 7 | 8 | 9 | class RandomForestBenchmark(AbstractBenchmarkClient): 10 | def __init__(self, **kwargs): 11 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'RandomForestBenchmark') 12 | kwargs['container_name'] = kwargs.get('container_name', 'ml_mmfb') 13 | kwargs['latest'] = kwargs.get('container_tag', '0.0.1') 14 | super(RandomForestBenchmark, self).__init__(**kwargs) 15 | 16 | 17 | class RandomForestBenchmarkBB(AbstractBenchmarkClient): 18 | def __init__(self, **kwargs): 19 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'RandomForestBenchmarkBB') 20 | kwargs['container_name'] = kwargs.get('container_name', 'ml_mmfb') 21 | kwargs['latest'] = kwargs.get('container_tag', '0.0.1') 22 | super(RandomForestBenchmarkBB, self).__init__(**kwargs) 23 | 24 | 25 | class RandomForestBenchmarkMF(AbstractBenchmarkClient): 26 | def __init__(self, **kwargs): 27 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'RandomForestBenchmarkMF') 28 | kwargs['container_name'] = kwargs.get('container_name', 'ml_mmfb') 29 | kwargs['latest'] = kwargs.get('container_tag', '0.0.1') 30 | super(RandomForestBenchmarkMF, self).__init__(**kwargs) 31 | 32 | 33 | __all__ = ['RandomForestBenchmark', 'RandomForestBenchmarkBB', 'RandomForestBenchmarkMF'] 34 | -------------------------------------------------------------------------------- /hpobench/container/benchmarks/ml/svm_benchmark.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | 4 | """ Benchmark for the SVM Benchmarks from hpobench/benchmarks/ml_mmfb/svm_benchmark.py """ 5 | 6 | from hpobench.container.client_abstract_benchmark import AbstractBenchmarkClient 7 | 8 | 9 | class SVMBenchmark(AbstractBenchmarkClient): 10 | def __init__(self, **kwargs): 11 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'SVMBenchmark') 12 | kwargs['container_name'] = kwargs.get('container_name', 'ml_mmfb') 13 | kwargs['latest'] = kwargs.get('container_tag', '0.0.1') 14 | super(SVMBenchmark, self).__init__(**kwargs) 15 | 16 | 17 | class SVMBenchmarkMF(AbstractBenchmarkClient): 18 | def __init__(self, **kwargs): 19 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'SVMBenchmarkMF') 20 | kwargs['container_name'] = kwargs.get('container_name', 'ml_mmfb') 21 | kwargs['latest'] = kwargs.get('container_tag', '0.0.1') 22 | super(SVMBenchmarkMF, self).__init__(**kwargs) 23 | 24 | 25 | class SVMBenchmarkBB(AbstractBenchmarkClient): 26 | def __init__(self, **kwargs): 27 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'SVMBenchmarkBB') 28 | kwargs['container_name'] = kwargs.get('container_name', 'ml_mmfb') 29 | kwargs['latest'] = kwargs.get('container_tag', '0.0.1') 30 | super(SVMBenchmarkBB, self).__init__(**kwargs) 31 | 32 | 33 | __all__ = ['SVMBenchmark', 'SVMBenchmarkMF', 'SVMBenchmarkBB'] 34 | -------------------------------------------------------------------------------- /hpobench/container/benchmarks/ml/svm_benchmark_old.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | 4 | """ Benchmark for the XGBoost Benchmark from hpobench/benchmarks/ml/xgboost_benchmark """ 5 | 6 | from hpobench.container.client_abstract_benchmark import AbstractBenchmarkClient 7 | 8 | 9 | class SupportVectorMachine(AbstractBenchmarkClient): 10 | def __init__(self, task_id: int, **kwargs): 11 | kwargs['task_id'] = task_id 12 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'SupportVectorMachine') 13 | kwargs['container_name'] = kwargs.get('container_name', 'svm_benchmark') 14 | kwargs['latest'] = kwargs.get('container_tag', '0.0.3') 15 | super(SupportVectorMachine, self).__init__(**kwargs) 16 | -------------------------------------------------------------------------------- /hpobench/container/benchmarks/ml/tabular_benchmark.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | 4 | """ Benchmark for the Tabular Benchmarks from hpobench/benchmarks/ml_mmfb/tabular_benchmark.py """ 5 | 6 | from hpobench.container.client_abstract_benchmark import AbstractBenchmarkClient 7 | 8 | 9 | class TabularBenchmark(AbstractBenchmarkClient): 10 | def __init__(self, **kwargs): 11 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'TabularBenchmark') 12 | kwargs['container_name'] = kwargs.get('container_name', 'ml_tabular_benchmarks') 13 | kwargs['latest'] = kwargs.get('container_tag', '0.0.1') 14 | super(TabularBenchmark, self).__init__(**kwargs) 15 | 16 | 17 | __all__ = ['TabularBenchmark'] 18 | -------------------------------------------------------------------------------- /hpobench/container/benchmarks/ml/xgboost_benchmark.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | 4 | """ Benchmark for the XGB Benchmarks from hpobench/benchmarks/ml_mmfb/xgboost_benchmark.py """ 5 | 6 | from hpobench.container.client_abstract_benchmark import AbstractBenchmarkClient 7 | 8 | 9 | class XGBoostBenchmark(AbstractBenchmarkClient): 10 | def __init__(self, **kwargs): 11 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'XGBoostBenchmark') 12 | kwargs['container_name'] = kwargs.get('container_name', 'ml_mmfb') 13 | kwargs['latest'] = kwargs.get('container_tag', '0.0.1') 14 | super(XGBoostBenchmark, self).__init__(**kwargs) 15 | 16 | 17 | class XGBoostBenchmarkBB(AbstractBenchmarkClient): 18 | def __init__(self, **kwargs): 19 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'XGBoostBenchmarkBB') 20 | kwargs['container_name'] = kwargs.get('container_name', 'ml_mmfb') 21 | kwargs['latest'] = kwargs.get('container_tag', '0.0.1') 22 | super(XGBoostBenchmarkBB, self).__init__(**kwargs) 23 | 24 | 25 | class XGBoostBenchmarkMF(AbstractBenchmarkClient): 26 | def __init__(self, **kwargs): 27 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'XGBoostBenchmarkMF') 28 | kwargs['container_name'] = kwargs.get('container_name', 'ml_mmfb') 29 | kwargs['latest'] = kwargs.get('container_tag', '0.0.1') 30 | super(XGBoostBenchmarkMF, self).__init__(**kwargs) 31 | 32 | 33 | class XGBoostSearchSpace3Benchmark(AbstractBenchmarkClient): 34 | def __init__(self, **kwargs): 35 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'XGBoostSearchSpace3Benchmark') 36 | kwargs['container_name'] = kwargs.get('container_name', 'ml_mmfb') 37 | kwargs['latest'] = kwargs.get('container_tag', '0.0.1') 38 | super(XGBoostSearchSpace3Benchmark, self).__init__(**kwargs) 39 | 40 | 41 | __all__ = ['XGBoostBenchmark', 'XGBoostBenchmarkBB', 'XGBoostBenchmarkMF'] 42 | -------------------------------------------------------------------------------- /hpobench/container/benchmarks/ml/xgboost_benchmark_old.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | 4 | """ Benchmark for the XGBoost Benchmark from hpobench/benchmarks/ml/xgboost_benchmark """ 5 | 6 | from hpobench.container.client_abstract_benchmark import AbstractBenchmarkClient 7 | 8 | 9 | class XGBoostBenchmark(AbstractBenchmarkClient): 10 | def __init__(self, task_id: int, **kwargs): 11 | kwargs['task_id'] = task_id 12 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'XGBoostBenchmark') 13 | kwargs['container_name'] = kwargs.get('container_name', 'xgboost_benchmark') 14 | kwargs['latest'] = kwargs.get('container_tag', '0.0.3') 15 | super(XGBoostBenchmark, self).__init__(**kwargs) 16 | 17 | 18 | class XGBoostExtendedBenchmark(AbstractBenchmarkClient): 19 | def __init__(self, task_id: int, **kwargs): 20 | kwargs['task_id'] = task_id 21 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'XGBoostExtendedBenchmark') 22 | kwargs['container_name'] = kwargs.get('container_name', 'xgboost_benchmark') 23 | kwargs['latest'] = kwargs.get('container_tag', '0.0.3') 24 | super(XGBoostExtendedBenchmark, self).__init__(**kwargs) 25 | -------------------------------------------------------------------------------- /hpobench/container/benchmarks/nas/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/HPOBench/47bf141f79e6bdfb26d1f1218b5d5aac09d7d2ce/hpobench/container/benchmarks/nas/__init__.py -------------------------------------------------------------------------------- /hpobench/container/benchmarks/nas/nasbench_101.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | 4 | """ Benchmark for the Tabular Benchmark from hpobench/benchmarks/nas/nasbench_101.py """ 5 | 6 | from hpobench.container.client_abstract_benchmark import AbstractBenchmarkClient 7 | 8 | 9 | class NASCifar10ABenchmark(AbstractBenchmarkClient): 10 | def __init__(self, **kwargs): 11 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'NASCifar10ABenchmark') 12 | kwargs['container_name'] = kwargs.get('container_name', 'nasbench_101') 13 | kwargs['latest'] = kwargs.get('container_tag', '0.0.4') 14 | super(NASCifar10ABenchmark, self).__init__(**kwargs) 15 | 16 | 17 | class NASCifar10BBenchmark(AbstractBenchmarkClient): 18 | def __init__(self, **kwargs): 19 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'NASCifar10BBenchmark') 20 | kwargs['container_name'] = kwargs.get('container_name', 'nasbench_101') 21 | kwargs['latest'] = kwargs.get('container_tag', '0.0.4') 22 | super(NASCifar10BBenchmark, self).__init__(**kwargs) 23 | 24 | 25 | class NASCifar10CBenchmark(AbstractBenchmarkClient): 26 | def __init__(self, **kwargs): 27 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'NASCifar10CBenchmark') 28 | kwargs['container_name'] = kwargs.get('container_name', 'nasbench_101') 29 | kwargs['latest'] = kwargs.get('container_tag', '0.0.4') 30 | super(NASCifar10CBenchmark, self).__init__(**kwargs) 31 | -------------------------------------------------------------------------------- /hpobench/container/benchmarks/nas/nasbench_1shot1.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | 4 | """ Benchmark for the nasbench 1shot1 benchmarks from hpobench/benchmarks/nas/nasbench_1shot1.py """ 5 | 6 | from hpobench.container.client_abstract_benchmark import AbstractBenchmarkClient 7 | 8 | 9 | class NASBench1shot1SearchSpace1Benchmark(AbstractBenchmarkClient): 10 | def __init__(self, **kwargs): 11 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'NASBench1shot1SearchSpace1Benchmark') 12 | kwargs['container_name'] = kwargs.get('container_name', 'nasbench_1shot1') 13 | kwargs['latest'] = kwargs.get('container_tag', '0.0.4') 14 | super(NASBench1shot1SearchSpace1Benchmark, self).__init__(**kwargs) 15 | 16 | 17 | class NASBench1shot1SearchSpace2Benchmark(AbstractBenchmarkClient): 18 | def __init__(self, **kwargs): 19 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'NASBench1shot1SearchSpace2Benchmark') 20 | kwargs['container_name'] = kwargs.get('container_name', 'nasbench_1shot1') 21 | kwargs['latest'] = kwargs.get('container_tag', '0.0.4') 22 | super(NASBench1shot1SearchSpace2Benchmark, self).__init__(**kwargs) 23 | 24 | 25 | class NASBench1shot1SearchSpace3Benchmark(AbstractBenchmarkClient): 26 | def __init__(self, **kwargs): 27 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'NASBench1shot1SearchSpace3Benchmark') 28 | kwargs['container_name'] = kwargs.get('container_name', 'nasbench_1shot1') 29 | kwargs['latest'] = kwargs.get('container_tag', '0.0.4') 30 | super(NASBench1shot1SearchSpace3Benchmark, self).__init__(**kwargs) 31 | -------------------------------------------------------------------------------- /hpobench/container/benchmarks/nas/nasbench_201.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | 4 | """ Benchmark for the NasBench201 Benchmark from hpobench/benchmarks/nas/nasbench_201.py """ 5 | 6 | from hpobench.container.client_abstract_benchmark import AbstractBenchmarkClient 7 | 8 | 9 | class Cifar10ValidNasBench201Benchmark(AbstractBenchmarkClient): 10 | def __init__(self, **kwargs): 11 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'Cifar10ValidNasBench201Benchmark') 12 | kwargs['container_name'] = kwargs.get('container_name', 'nasbench_201') 13 | kwargs['latest'] = kwargs.get('container_tag', '0.0.5') 14 | super(Cifar10ValidNasBench201Benchmark, self).__init__(**kwargs) 15 | 16 | 17 | class Cifar100NasBench201Benchmark(AbstractBenchmarkClient): 18 | def __init__(self, **kwargs): 19 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'Cifar100NasBench201Benchmark') 20 | kwargs['container_name'] = kwargs.get('container_name', 'nasbench_201') 21 | kwargs['latest'] = kwargs.get('container_tag', '0.0.5') 22 | super(Cifar100NasBench201Benchmark, self).__init__(**kwargs) 23 | 24 | 25 | class ImageNetNasBench201Benchmark(AbstractBenchmarkClient): 26 | def __init__(self, **kwargs): 27 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'ImageNetNasBench201Benchmark') 28 | kwargs['container_name'] = kwargs.get('container_name', 'nasbench_201') 29 | kwargs['latest'] = kwargs.get('container_tag', '0.0.5') 30 | super(ImageNetNasBench201Benchmark, self).__init__(**kwargs) 31 | 32 | 33 | class Cifar10ValidNasBench201BenchmarkOriginal(AbstractBenchmarkClient): 34 | def __init__(self, **kwargs): 35 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'Cifar10ValidNasBench201BenchmarkOriginal') 36 | kwargs['container_name'] = kwargs.get('container_name', 'nasbench_201') 37 | kwargs['latest'] = kwargs.get('container_tag', '0.0.5') 38 | super(Cifar10ValidNasBench201BenchmarkOriginal, self).__init__(**kwargs) 39 | 40 | 41 | class Cifar100NasBench201BenchmarkOriginal(AbstractBenchmarkClient): 42 | def __init__(self, **kwargs): 43 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'Cifar100NasBench201BenchmarkOriginal') 44 | kwargs['container_name'] = kwargs.get('container_name', 'nasbench_201') 45 | kwargs['latest'] = kwargs.get('container_tag', '0.0.5') 46 | super(Cifar100NasBench201BenchmarkOriginal, self).__init__(**kwargs) 47 | 48 | 49 | class ImageNetNasBench201BenchmarkOriginal(AbstractBenchmarkClient): 50 | def __init__(self, **kwargs): 51 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'ImageNetNasBench201BenchmarkOriginal') 52 | kwargs['container_name'] = kwargs.get('container_name', 'nasbench_201') 53 | kwargs['latest'] = kwargs.get('container_tag', '0.0.5') 54 | super(ImageNetNasBench201BenchmarkOriginal, self).__init__(**kwargs) 55 | 56 | 57 | __all__ = ["Cifar10ValidNasBench201Benchmark", 58 | "Cifar100NasBench201Benchmark", 59 | "ImageNetNasBench201Benchmark", 60 | "Cifar10ValidNasBench201BenchmarkOriginal", 61 | "Cifar100NasBench201BenchmarkOriginal", 62 | "ImageNetNasBench201BenchmarkOriginal"] 63 | -------------------------------------------------------------------------------- /hpobench/container/benchmarks/nas/tabular_benchmarks.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | 4 | """ Benchmark for the Tabular Benchmark from hpobench/benchmarks/nas/tabular_benchmarks.py """ 5 | 6 | from hpobench.container.client_abstract_benchmark import AbstractBenchmarkClient 7 | 8 | 9 | class SliceLocalizationBenchmark(AbstractBenchmarkClient): 10 | def __init__(self, **kwargs): 11 | kwargs['data_path'] = '/home/fcnet_tabular_benchmarks' 12 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'SliceLocalizationBenchmark') 13 | kwargs['container_name'] = kwargs.get('container_name', 'tabular_benchmarks') 14 | kwargs['latest'] = kwargs.get('container_tag', '0.0.5') 15 | super(SliceLocalizationBenchmark, self).__init__(**kwargs) 16 | 17 | 18 | class ProteinStructureBenchmark(AbstractBenchmarkClient): 19 | def __init__(self, **kwargs): 20 | kwargs['data_path'] = '/home/fcnet_tabular_benchmarks' 21 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'ProteinStructureBenchmark') 22 | kwargs['container_name'] = kwargs.get('container_name', 'tabular_benchmarks') 23 | kwargs['latest'] = kwargs.get('container_tag', '0.0.5') 24 | super(ProteinStructureBenchmark, self).__init__(**kwargs) 25 | 26 | 27 | class NavalPropulsionBenchmark(AbstractBenchmarkClient): 28 | def __init__(self, **kwargs): 29 | kwargs['data_path'] = '/home/fcnet_tabular_benchmarks' 30 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'NavalPropulsionBenchmark') 31 | kwargs['container_name'] = kwargs.get('container_name', 'tabular_benchmarks') 32 | kwargs['latest'] = kwargs.get('container_tag', '0.0.5') 33 | super(NavalPropulsionBenchmark, self).__init__(**kwargs) 34 | 35 | 36 | class ParkinsonsTelemonitoringBenchmark(AbstractBenchmarkClient): 37 | def __init__(self, **kwargs): 38 | kwargs['data_path'] = '/home/fcnet_tabular_benchmarks' 39 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'ParkinsonsTelemonitoringBenchmark') 40 | kwargs['container_name'] = kwargs.get('container_name', 'tabular_benchmarks') 41 | kwargs['latest'] = kwargs.get('container_tag', '0.0.5') 42 | super(ParkinsonsTelemonitoringBenchmark, self).__init__(**kwargs) 43 | 44 | 45 | class SliceLocalizationBenchmarkOriginal(AbstractBenchmarkClient): 46 | def __init__(self, **kwargs): 47 | kwargs['data_path'] = '/home/fcnet_tabular_benchmarks' 48 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'SliceLocalizationBenchmarkOriginal') 49 | kwargs['container_name'] = kwargs.get('container_name', 'tabular_benchmarks') 50 | kwargs['latest'] = kwargs.get('container_tag', '0.0.5') 51 | super(SliceLocalizationBenchmarkOriginal, self).__init__(**kwargs) 52 | 53 | 54 | class ProteinStructureBenchmarkOriginal(AbstractBenchmarkClient): 55 | def __init__(self, **kwargs): 56 | kwargs['data_path'] = '/home/fcnet_tabular_benchmarks' 57 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'ProteinStructureBenchmarkOriginal') 58 | kwargs['container_name'] = kwargs.get('container_name', 'tabular_benchmarks') 59 | kwargs['latest'] = kwargs.get('container_tag', '0.0.5') 60 | super(ProteinStructureBenchmarkOriginal, self).__init__(**kwargs) 61 | 62 | 63 | class NavalPropulsionBenchmarkOriginal(AbstractBenchmarkClient): 64 | def __init__(self, **kwargs): 65 | kwargs['data_path'] = '/home/fcnet_tabular_benchmarks' 66 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'NavalPropulsionBenchmarkOriginal') 67 | kwargs['container_name'] = kwargs.get('container_name', 'tabular_benchmarks') 68 | kwargs['latest'] = kwargs.get('container_tag', '0.0.5') 69 | super(NavalPropulsionBenchmarkOriginal, self).__init__(**kwargs) 70 | 71 | 72 | class ParkinsonsTelemonitoringBenchmarkOriginal(AbstractBenchmarkClient): 73 | def __init__(self, **kwargs): 74 | kwargs['data_path'] = '/home/fcnet_tabular_benchmarks' 75 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'ParkinsonsTelemonitoringBenchmarkOriginal') 76 | kwargs['container_name'] = kwargs.get('container_name', 'tabular_benchmarks') 77 | kwargs['latest'] = kwargs.get('container_tag', '0.0.5') 78 | super(ParkinsonsTelemonitoringBenchmarkOriginal, self).__init__(**kwargs) 79 | 80 | 81 | __all__ = ["SliceLocalizationBenchmark", "SliceLocalizationBenchmarkOriginal", 82 | "ProteinStructureBenchmark", "ProteinStructureBenchmarkOriginal", 83 | "NavalPropulsionBenchmark", "NavalPropulsionBenchmarkOriginal", 84 | "ParkinsonsTelemonitoringBenchmark", "ParkinsonsTelemonitoringBenchmarkOriginal"] 85 | -------------------------------------------------------------------------------- /hpobench/container/benchmarks/od/od_benchmarks.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | 4 | """ Benchmark for OCSVM and outlier detection """ 5 | 6 | from hpobench.container.client_abstract_benchmark import AbstractBenchmarkClient 7 | 8 | 9 | class ODAutoencoder(AbstractBenchmarkClient): 10 | def __init__(self, dataset_name: str, **kwargs): 11 | kwargs['dataset_name'] = dataset_name 12 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'ODAutoencoder') 13 | kwargs['container_name'] = kwargs.get('container_name', 'outlier_detection') 14 | kwargs['latest'] = kwargs.get('container_tag', '0.0.1') 15 | super(ODAutoencoder, self).__init__(**kwargs) 16 | 17 | 18 | class ODKernelDensityEstimation(AbstractBenchmarkClient): 19 | def __init__(self, dataset_name: str, **kwargs): 20 | kwargs['dataset_name'] = dataset_name 21 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'ODKernelDensityEstimation') 22 | kwargs['container_name'] = kwargs.get('container_name', 'outlier_detection') 23 | kwargs['latest'] = kwargs.get('container_tag', '0.0.1') 24 | super(ODKernelDensityEstimation, self).__init__(**kwargs) 25 | 26 | 27 | class ODOneClassSupportVectorMachine(AbstractBenchmarkClient): 28 | def __init__(self, dataset_name: str, **kwargs): 29 | kwargs['dataset_name'] = dataset_name 30 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'ODOneClassSupportVectorMachine') 31 | kwargs['container_name'] = kwargs.get('container_name', 'outlier_detection') 32 | kwargs['latest'] = kwargs.get('container_tag', '0.0.1') 33 | super(ODOneClassSupportVectorMachine, self).__init__(**kwargs) 34 | -------------------------------------------------------------------------------- /hpobench/container/benchmarks/rl/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/HPOBench/47bf141f79e6bdfb26d1f1218b5d5aac09d7d2ce/hpobench/container/benchmarks/rl/__init__.py -------------------------------------------------------------------------------- /hpobench/container/benchmarks/rl/cartpole.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | 4 | """ Benchmark for the Cartpole Benchmark from hpobench/benchmarks/rl/cartpole.py """ 5 | 6 | from hpobench.container.client_abstract_benchmark import AbstractBenchmarkClient 7 | 8 | 9 | class CartpoleReduced(AbstractBenchmarkClient): 10 | def __init__(self, **kwargs): 11 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'CartpoleReduced') 12 | kwargs['container_name'] = kwargs.get('container_name', 'cartpole') 13 | kwargs['latest'] = kwargs.get('container_tag', '0.0.4') 14 | super(CartpoleReduced, self).__init__(**kwargs) 15 | 16 | 17 | class CartpoleFull(AbstractBenchmarkClient): 18 | def __init__(self, **kwargs): 19 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'CartpoleFull') 20 | kwargs['container_name'] = kwargs.get('container_name', 'cartpole') 21 | kwargs['latest'] = kwargs.get('container_tag', '0.0.4') 22 | super(CartpoleFull, self).__init__(**kwargs) 23 | -------------------------------------------------------------------------------- /hpobench/container/benchmarks/rl/learna_benchmark.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | 4 | """ Benchmark for the learna benchmark from hpobench/benchmarks/rl/learna_benchmarks.py """ 5 | 6 | from hpobench.container.client_abstract_benchmark import AbstractBenchmarkClient 7 | 8 | 9 | class Learna(AbstractBenchmarkClient): 10 | def __init__(self, **kwargs): 11 | kwargs['data_path'] = '/home/learna/data' 12 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'Learna') 13 | kwargs['container_name'] = kwargs.get('container_name', 'learna_benchmark') 14 | kwargs['latest'] = kwargs.get('container_tag', '0.0.4') 15 | super(Learna, self).__init__(**kwargs) 16 | 17 | 18 | class MetaLearna(AbstractBenchmarkClient): 19 | def __init__(self, **kwargs): 20 | kwargs['data_path'] = '/home/learna/data' 21 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'MetaLearna') 22 | kwargs['container_name'] = kwargs.get('container_name', 'learna_benchmark') 23 | kwargs['latest'] = kwargs.get('container_tag', '0.0.4') 24 | super(MetaLearna, self).__init__(**kwargs) 25 | -------------------------------------------------------------------------------- /hpobench/container/benchmarks/surrogates/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/HPOBench/47bf141f79e6bdfb26d1f1218b5d5aac09d7d2ce/hpobench/container/benchmarks/surrogates/__init__.py -------------------------------------------------------------------------------- /hpobench/container/benchmarks/surrogates/svm_benchmark.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | 4 | """ Benchmark for the svm surrogates Benchmark from hpobench/benchmarks/surrogates/svm_benchmark.py 5 | """ 6 | 7 | from hpobench.container.client_abstract_benchmark import AbstractBenchmarkClient 8 | 9 | 10 | class SurrogateSVMBenchmark(AbstractBenchmarkClient): 11 | def __init__(self, **kwargs): 12 | kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'SurrogateSVMBenchmark') 13 | kwargs['container_name'] = kwargs.get('container_name', 'surrogate_svm') 14 | kwargs['latest'] = kwargs.get('container_tag', '0.0.2') 15 | super(SurrogateSVMBenchmark, self).__init__(**kwargs) 16 | -------------------------------------------------------------------------------- /hpobench/container/recipes/Singularity.template: -------------------------------------------------------------------------------- 1 | Bootstrap: docker 2 | From: python:3.7-slim 3 | 4 | %labels 5 | MAINTAINER 6 | VERSION v0.0.1 7 | 8 | %help 9 | This is a template for a Singularity recipe 10 | 11 | %post 12 | apt update -y 13 | apt install build-essential git wget -y 14 | 15 | cd /home \ 16 | && mkdir data && cd data \ 17 | && echo "Here you could download data e.g. using wget" \ 18 | 19 | cd /home \ 20 | && echo "Here you can install everything you need, e.g. dependencies not available on pypi" \ 21 | && echo "Next, we clone and install HPOBench" \ 22 | && git clone https://github.com/automl/HPOBench.git \ 23 | && cd HPOBench \ 24 | && echo "Please never push a recipe that checks out any other branch than development or master" \ 25 | && git checkout development \ 26 | && echo "Here you can install extra requirements additional to singularity" \ 27 | && pip install .[] \ 28 | && echo "Please don't touch the following lines" 29 | && cd / \ 30 | && mkdir /var/lib/hpobench/ \ 31 | && chmod -R 777 /var/lib/hpobench/ \ 32 | && rm -rf /var/lib/apt/lists/* \ 33 | && pip cache purge 34 | 35 | echo "Finally, please change the benchmark in the runscript to point to your benchmark" 36 | 37 | %runscript 38 | python -s /home/HPOBench/hpobench/container/server_abstract_benchmark.py . $@ -------------------------------------------------------------------------------- /hpobench/container/recipes/ml/Singularity.PyBNN: -------------------------------------------------------------------------------- 1 | Bootstrap: docker 2 | From: python:3.7-slim 3 | 4 | %labels 5 | MAINTAINER muelleph@cs.uni-freiburg.de 6 | VERSION v0.0.1 7 | 8 | %post 9 | apt update -y \ 10 | && apt install build-essential git -y \ 11 | && cd /home \ 12 | && git clone https://github.com/automl/HPOBench.git \ 13 | && cd HPOBench \ 14 | && git checkout master \ 15 | && pip install .[pybnn] \ 16 | && cd / \ 17 | && mkdir /var/lib/hpobench/ \ 18 | && chmod -R 777 /var/lib/hpobench/ \ 19 | && rm -rf /var/lib/apt/lists/* \ 20 | && pip cache purge 21 | 22 | %runscript 23 | python -s /home/HPOBench/hpobench/container/server_abstract_benchmark.py ml.pybnn $@ 24 | -------------------------------------------------------------------------------- /hpobench/container/recipes/ml/Singularity.SupportVectorMachine: -------------------------------------------------------------------------------- 1 | Bootstrap: docker 2 | From: python:3.7-slim 3 | 4 | %labels 5 | MAINTAINER muelleph@cs.uni-freiburg.de 6 | VERSION v0.0.1 7 | 8 | %post 9 | apt update -y \ 10 | && apt install build-essential git -y \ 11 | && cd /home \ 12 | && git clone https://github.com/automl/HPOBench.git \ 13 | && cd HPOBench \ 14 | && git checkout master \ 15 | && pip install .[svm] \ 16 | && cd / \ 17 | && mkdir /var/lib/hpobench/ \ 18 | && chmod -R 777 /var/lib/hpobench/ \ 19 | && rm -rf /var/lib/apt/lists/* \ 20 | && pip cache purge 21 | 22 | %runscript 23 | python -s /home/HPOBench/hpobench/container/server_abstract_benchmark.py ml.svm_benchmark $@ 24 | -------------------------------------------------------------------------------- /hpobench/container/recipes/ml/Singularity.XGBoostBenchmark: -------------------------------------------------------------------------------- 1 | Bootstrap: docker 2 | From: python:3.7-slim 3 | 4 | %labels 5 | MAINTAINER muelleph@cs.uni-freiburg.de 6 | VERSION v0.0.1 7 | 8 | %post 9 | apt update -y \ 10 | && apt install build-essential git -y \ 11 | && cd /home \ 12 | && git clone https://github.com/automl/HPOBench.git \ 13 | && cd HPOBench \ 14 | && git checkout master \ 15 | && pip install .[xgboost] \ 16 | && cd / \ 17 | && mkdir /var/lib/hpobench/ \ 18 | && chmod -R 777 /var/lib/hpobench/ \ 19 | && pip cache purge \ 20 | && rm -rf /var/lib/apt/lists/* 21 | 22 | 23 | %runscript 24 | python -s /home/HPOBench/hpobench/container/server_abstract_benchmark.py ml.xgboost_benchmark $@ 25 | -------------------------------------------------------------------------------- /hpobench/container/recipes/ml/Singularity.ml_mmfb: -------------------------------------------------------------------------------- 1 | Bootstrap: docker 2 | From: python:3.8-slim 3 | 4 | %labels 5 | MAINTAINER muelleph@cs.uni-freiburg.de 6 | VERSION v0.0.1 7 | 8 | %post 9 | apt update -y \ 10 | && apt install build-essential git -y \ 11 | 12 | cd /home \ 13 | && git clone https://github.com/automl/HPOBench.git \ 14 | && cd HPOBench \ 15 | && git checkout development \ 16 | && pip install ".[ml_mfbb]" \ 17 | && cd / \ 18 | && mkdir /var/lib/hpobench/ \ 19 | && chmod -R 777 /var/lib/hpobench/ \ 20 | && pip cache purge \ 21 | && rm -rf /var/lib/apt/lists/* 22 | 23 | 24 | %runscript 25 | python -s /home/HPOBench/hpobench/container/server_abstract_benchmark.py ml $@ 26 | -------------------------------------------------------------------------------- /hpobench/container/recipes/ml/Singularity.ml_tabular_benchmark: -------------------------------------------------------------------------------- 1 | Bootstrap: docker 2 | From: python:3.8-slim 3 | 4 | %labels 5 | MAINTAINER muelleph@cs.uni-freiburg.de 6 | VERSION v0.0.1 7 | 8 | %post 9 | apt update -y \ 10 | && apt install build-essential git -y \ 11 | 12 | cd /home \ 13 | && git clone https://github.com/automl/HPOBench.git \ 14 | && cd HPOBench \ 15 | && git checkout development \ 16 | && pip install ".[ml_tabular_benchmarks]" \ 17 | && cd / \ 18 | && mkdir /var/lib/hpobench/ \ 19 | && chmod -R 777 /var/lib/hpobench/ \ 20 | && pip cache purge \ 21 | && rm -rf /var/lib/apt/lists/* 22 | 23 | 24 | %runscript 25 | python -s /home/HPOBench/hpobench/container/server_abstract_benchmark.py ml $@ 26 | -------------------------------------------------------------------------------- /hpobench/container/recipes/nas/Singularity.TabularBenchmarks: -------------------------------------------------------------------------------- 1 | Bootstrap: docker 2 | From: python:3.7-slim 3 | 4 | %labels 5 | MAINTAINER muelleph@cs.uni-freiburg.de 6 | VERSION v0.0.1 7 | 8 | %post 9 | apt update -y 10 | apt install build-essential git wget -y 11 | 12 | cd /home \ 13 | && wget http://ml4aad.org/wp-content/uploads/2019/01/fcnet_tabular_benchmarks.tar.gz \ 14 | && tar xf fcnet_tabular_benchmarks.tar.gz 15 | 16 | cd /home \ 17 | && pip install git+https://github.com/google-research/nasbench.git@master \ 18 | && pip install git+https://github.com/automl/nas_benchmarks.git \ 19 | && git clone https://github.com/automl/HPOBench.git \ 20 | && cd HPOBench \ 21 | && git checkout master \ 22 | && pip install .[tabular_benchmarks] \ 23 | && cd / \ 24 | && mkdir /var/lib/hpobench/ \ 25 | && chmod -R 777 /var/lib/hpobench/ \ 26 | && rm -rf /var/lib/apt/lists/* \ 27 | && pip cache purge 28 | 29 | %runscript 30 | python -s /home/HPOBench/hpobench/container/server_abstract_benchmark.py nas.tabular_benchmarks $@ 31 | -------------------------------------------------------------------------------- /hpobench/container/recipes/nas/Singularity.nasbench_101: -------------------------------------------------------------------------------- 1 | Bootstrap: docker 2 | From: python:3.7-slim 3 | 4 | %labels 5 | MAINTAINER muelleph@cs.uni-freiburg.de 6 | VERSION v0.0.1 7 | 8 | %post 9 | apt update -y 10 | apt install build-essential git wget -y 11 | 12 | cd /home \ 13 | && pip install git+https://github.com/google-research/nasbench.git@master \ 14 | && pip install git+https://github.com/automl/nas_benchmarks.git \ 15 | && git clone https://github.com/automl/HPOBench.git \ 16 | && cd HPOBench \ 17 | && git checkout master \ 18 | && pip install .[nasbench_101] \ 19 | && cd / \ 20 | && mkdir /var/lib/hpobench/ \ 21 | && chmod -R 777 /var/lib/hpobench/ \ 22 | && rm -rf /var/lib/apt/lists/* \ 23 | && pip cache purge 24 | 25 | %runscript 26 | python -s /home/HPOBench/hpobench/container/server_abstract_benchmark.py nas.nasbench_101 $@ 27 | -------------------------------------------------------------------------------- /hpobench/container/recipes/nas/Singularity.nasbench_1shot1: -------------------------------------------------------------------------------- 1 | Bootstrap: docker 2 | From: python:3.7-slim 3 | 4 | %labels 5 | MAINTAINER muelleph@cs.uni-freiburg.de 6 | VERSION v0.0.1 7 | 8 | %environment 9 | export PYTHONPATH=/home/nasbench-1shot1:$PYTHONPATH 10 | 11 | %post 12 | apt update -y \ 13 | && apt install build-essential git wget -y 14 | 15 | cd /home \ 16 | && pip install tensorflow==1.15.0 \ 17 | && pip install git+https://github.com/google-research/nasbench.git@master \ 18 | && git clone https://github.com/automl/nasbench-1shot1.git \ 19 | && git clone https://github.com/automl/HPOBench.git \ 20 | && cd HPOBench \ 21 | && git checkout master \ 22 | && pip install .[nasbench_1shot1] \ 23 | && cd / \ 24 | && mkdir /var/lib/hpobench/ \ 25 | && chmod -R 777 /var/lib/hpobench/ \ 26 | && rm -rf /var/lib/apt/lists/* \ 27 | && pip cache purge 28 | 29 | %runscript 30 | python -s /home/HPOBench/hpobench/container/server_abstract_benchmark.py nas.nasbench_1shot1 $@ -------------------------------------------------------------------------------- /hpobench/container/recipes/nas/Singularity.nasbench_201: -------------------------------------------------------------------------------- 1 | Bootstrap: docker 2 | From: python:3.7-slim 3 | 4 | %labels 5 | MAINTAINER muelleph@cs.uni-freiburg.de 6 | VERSION v0.0.1 7 | 8 | %post 9 | apt update -y 10 | apt install build-essential git -y 11 | 12 | cd /home \ 13 | && git clone https://github.com/automl/HPOBench.git \ 14 | && cd HPOBench \ 15 | && git checkout master \ 16 | && pip install . \ 17 | && cd / \ 18 | && mkdir /var/lib/hpobench/ \ 19 | && chmod -R 777 /var/lib/hpobench/ \ 20 | && rm -rf /var/lib/apt/lists/* \ 21 | && pip cache purge 22 | 23 | %runscript 24 | python -s /home/HPOBench/hpobench/container/server_abstract_benchmark.py nas.nasbench_201 $@ 25 | -------------------------------------------------------------------------------- /hpobench/container/recipes/od/Singularity.ODBenchmarks: -------------------------------------------------------------------------------- 1 | Bootstrap: docker 2 | From: python:3.7-slim 3 | 4 | %labels 5 | MAINTAINER sass@tnt.uni-hannover.de 6 | VERSION v0.0.1 7 | 8 | %post 9 | apt update -y \ 10 | && apt install build-essential git -y \ 11 | && cd /home \ 12 | && git clone https://github.com/automl/HPOBench.git \ 13 | && cd HPOBench \ 14 | && git checkout development \ 15 | && pip install .[outlier_detection] \ 16 | && cd / \ 17 | && mkdir /var/lib/hpobench/ \ 18 | && chmod -R 777 /var/lib/hpobench/ \ 19 | && rm -rf /var/lib/apt/lists/* \ 20 | && pip cache purge 21 | 22 | %runscript 23 | python -s /home/HPOBench/hpobench/container/server_abstract_benchmark.py od.od_benchmarks $@ 24 | -------------------------------------------------------------------------------- /hpobench/container/recipes/od/Singularity.ODKernelDensityEstimation: -------------------------------------------------------------------------------- 1 | Bootstrap: docker 2 | From: python:3.7-slim 3 | 4 | %labels 5 | MAINTAINER sass@tnt.uni-hannover.de 6 | VERSION v0.0.1 7 | 8 | %post 9 | apt update -y \ 10 | && apt install build-essential git -y \ 11 | && cd /home \ 12 | && git clone https://github.com/automl/HPOBench.git \ 13 | && cd HPOBench \ 14 | && git checkout development \ 15 | && pip install .[outlier_detection] \ 16 | && cd / \ 17 | && mkdir /var/lib/hpobench/ \ 18 | && chmod -R 777 /var/lib/hpobench/ \ 19 | && rm -rf /var/lib/apt/lists/* \ 20 | && pip cache purge 21 | 22 | %runscript 23 | python -s /home/HPOBench/hpobench/container/server_abstract_benchmark.py od.od_kde $@ 24 | -------------------------------------------------------------------------------- /hpobench/container/recipes/rl/Singularity.Cartpole: -------------------------------------------------------------------------------- 1 | Bootstrap: docker 2 | From: python:3.7-slim 3 | 4 | %labels 5 | MAINTAINER muelleph@cs.uni-freiburg.de 6 | VERSION v0.0.1 7 | 8 | %post 9 | apt update -y \ 10 | && apt install build-essential git -y \ 11 | && pip install numpy==1.18.1 cython==0.29.14 12 | 13 | cd /home \ 14 | && git clone https://github.com/automl/HPOBench.git \ 15 | && cd HPOBench \ 16 | && git checkout master \ 17 | && pip install .[cartpole] \ 18 | && cd / \ 19 | && mkdir /var/lib/hpobench/ \ 20 | && chmod -R 777 /var/lib/hpobench/ \ 21 | && rm -rf /var/lib/apt/lists/* \ 22 | && pip cache purge 23 | 24 | %runscript 25 | python -s /home/HPOBench/hpobench/container/server_abstract_benchmark.py rl.cartpole $@ 26 | -------------------------------------------------------------------------------- /hpobench/container/recipes/rl/Singularity.learnaBenchmark: -------------------------------------------------------------------------------- 1 | Bootstrap: docker 2 | From: python:3.7-slim 3 | 4 | %labels 5 | MAINTAINER muelleph@cs.uni-freiburg.de 6 | VERSION v0.0.1 7 | 8 | %post 9 | apt update -y \ 10 | && apt install build-essential git wget -y 11 | 12 | cd /home \ 13 | && git clone --single-branch --branch development https://github.com/PhMueller/learna.git \ 14 | && cd learna \ 15 | && ./thirdparty/miniconda/make_miniconda.sh \ 16 | && ./thirdparty/miniconda/miniconda/bin/conda env create -f environment.yml \ 17 | && ./thirdparty/miniconda/miniconda/envs/learna/bin/python -m pip install docutils==0.16 \ 18 | && ./thirdparty/miniconda/miniconda/envs/learna/bin/python -m pip install tensorforce==0.3.3 \ 19 | && ./thirdparty/miniconda/miniconda/envs/learna/bin/python -m pip install . \ 20 | && ./thirdparty/miniconda/miniconda/envs/learna/bin/python -m learna.data.download_and_build_eterna ./learna/data/secondaries_to_single_files.sh data/eterna data/eterna/interim/eterna.txt \ 21 | && ./learna/data/download_and_build_rfam_taneda.sh \ 22 | && ./learna/data/download_and_build_rfam_learn.sh \ 23 | && mv data/rfam_learn/test data/rfam_learn_test \ 24 | && mv data/rfam_learn/validation data/rfam_learn_validation \ 25 | && mv data/rfam_learn/train data/rfam_learn_train \ 26 | && rm -rf data/rfam_learn \ 27 | && chmod -R 755 data/ \ 28 | && cd /home \ 29 | && git clone https://github.com/automl/HPOBench.git \ 30 | && cd HPOBench \ 31 | && git checkout master \ 32 | && ../learna/thirdparty/miniconda/miniconda/envs/learna/bin/python -m pip install . \ 33 | && cd / \ 34 | && mkdir /var/lib/hpobench/ \ 35 | && chmod -R 777 /var/lib/hpobench/ \ 36 | && rm -rf /var/lib/apt/lists/* \ 37 | && pip cache purge 38 | 39 | %runscript 40 | /home/learna/thirdparty/miniconda/miniconda/envs/learna/bin/python -s \ 41 | /home/HPOBench/hpobench/container/server_abstract_benchmark.py rl.learna_benchmark $@ 42 | -------------------------------------------------------------------------------- /hpobench/container/recipes/surrogates/Singularity.ParamnetBenchmark: -------------------------------------------------------------------------------- 1 | Bootstrap: docker 2 | From: python:3.7-slim 3 | 4 | %labels 5 | MAINTAINER muelleph@cs.uni-freiburg.de 6 | VERSION v0.0.1 7 | 8 | %post 9 | apt update -y \ 10 | && apt install build-essential git wget -y 11 | 12 | cd /home \ 13 | && git clone https://github.com/automl/HPOBench.git \ 14 | && cd HPOBench \ 15 | && git checkout master \ 16 | && pip install --upgrade .[paramnet] \ 17 | && cd / \ 18 | && mkdir /var/lib/hpobench/ \ 19 | && chmod -R 777 /var/lib/hpobench/ \ 20 | && rm -rf /var/lib/apt/lists/* \ 21 | && pip cache purge 22 | 23 | %runscript 24 | python -s /home/HPOBench/hpobench/container/server_abstract_benchmark.py surrogates.paramnet_benchmark $@ -------------------------------------------------------------------------------- /hpobench/container/recipes/surrogates/Singularity.SupportVectorMachine: -------------------------------------------------------------------------------- 1 | Bootstrap: docker 2 | From: python:3.7-slim 3 | 4 | %labels 5 | MAINTAINER muelleph@cs.uni-freiburg.de 6 | VERSION v0.0.1 7 | 8 | %post 9 | 10 | %post 11 | apt update -y \ 12 | && apt install build-essential git wget -y 13 | 14 | cd /home \ 15 | && git clone https://github.com/automl/HPOBench.git \ 16 | && cd HPOBench \ 17 | && git checkout master \ 18 | && pip install --upgrade .[paramnet] \ 19 | && cd / \ 20 | && mkdir /var/lib/hpobench/ \ 21 | && chmod -R 777 /var/lib/hpobench/ \ 22 | && rm -rf /var/lib/apt/lists/* \ 23 | && pip cache purge 24 | 25 | %runscript 26 | python -s /home/HPOBench/hpobench/container/server_abstract_benchmark.py surrogates.svm_benchmark $@ -------------------------------------------------------------------------------- /hpobench/container/server_abstract_benchmark.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | 4 | """ Defines the server-side for using benchmarks with containers 5 | 6 | BenchmarkServer defines the server side for the communication between the 7 | container (benchmark) and the client. 8 | It starts the Pyro4 server and awaits commands from the client. Make sure 9 | that all payloads are json-serializable. 10 | """ 11 | 12 | import argparse 13 | import json 14 | import logging 15 | import os 16 | 17 | import Pyro4 18 | from ConfigSpace.read_and_write import json as csjson 19 | 20 | from hpobench.config import HPOBenchConfig 21 | from hpobench.util.container_utils import BenchmarkEncoder, BenchmarkDecoder 22 | 23 | # Read in the verbosity level from the environment variable HPOBENCH_DEBUG 24 | log_level_str = os.environ.get('HPOBENCH_DEBUG', 'false') 25 | LOG_LEVEL = logging.DEBUG if log_level_str == 'true' else logging.INFO 26 | 27 | root = logging.getLogger("hpobench") 28 | root.setLevel(LOG_LEVEL) 29 | 30 | logger = logging.getLogger('BenchmarkServer') 31 | 32 | 33 | @Pyro4.expose 34 | @Pyro4.behavior(instance_mode="single") 35 | class BenchmarkServer: 36 | def __init__(self, socket_id): 37 | self.pyro_running = True 38 | config = HPOBenchConfig() 39 | self.benchmark = None 40 | 41 | self.socket_id = socket_id 42 | socket_path = config.socket_dir / (self.socket_id + "_unix.sock") 43 | logger.debug(f'Socket Path: {socket_path}') 44 | logger.info(f'Logging level: {logger.level}') 45 | 46 | if socket_path.exists(): 47 | os.remove(socket_path) 48 | self.daemon = Pyro4.Daemon(unixsocket=str(socket_path)) 49 | 50 | _ = self.daemon.register(self, self.socket_id + ".unixsock") 51 | 52 | # start the event loop of the server to wait for calls 53 | self.daemon.requestLoop(loopCondition=lambda: self.pyro_running) 54 | 55 | def init_benchmark(self, kwargs_str): 56 | try: 57 | kwargs = json.loads(kwargs_str, cls=BenchmarkDecoder) 58 | self.benchmark = Benchmark(**kwargs) # noqa: F821 59 | logger.info('Server: Connected Successfully') 60 | except Exception as e: 61 | logger.exception(e) 62 | 63 | def get_configuration_space(self, kwargs_str: str) -> str: 64 | logger.debug(f'Server: get_config_space: kwargs_str: {kwargs_str}') 65 | 66 | kwargs = json.loads(kwargs_str, cls=BenchmarkDecoder) 67 | seed = kwargs.get('seed', None) 68 | 69 | result = self.benchmark.get_configuration_space(seed=seed) 70 | logger.debug(f'Server: Configspace: {result}') 71 | return csjson.write(result, indent=None) 72 | 73 | def get_fidelity_space(self, kwargs_str: str) -> str: 74 | logger.debug(f'Server: get_fidelity_space: kwargs_str: {kwargs_str}') 75 | 76 | kwargs = json.loads(kwargs_str, cls=BenchmarkDecoder) 77 | seed = kwargs.get('seed', None) 78 | 79 | result = self.benchmark.get_fidelity_space(seed=seed) 80 | logger.debug(f'Server: Fidelity Space: {result}') 81 | return csjson.write(result, indent=None) 82 | 83 | def objective_function(self, c_str: str, f_str: str, kwargs_str: str) -> str: 84 | logger.debug(f'Server: objective_function: c_str: {c_str} f_str: {f_str} kwargs_str: {kwargs_str}') 85 | 86 | configuration = json.loads(c_str, cls=BenchmarkDecoder) 87 | fidelity = json.loads(f_str, cls=BenchmarkDecoder) 88 | kwargs = json.loads(kwargs_str, cls=BenchmarkDecoder) 89 | 90 | result = self.benchmark.objective_function(configuration=configuration, fidelity=fidelity, **kwargs) 91 | return json.dumps(result, indent=None, cls=BenchmarkEncoder) 92 | 93 | def objective_function_test(self, c_str: str, f_str: str, kwargs_str: str) -> str: 94 | logger.debug(f'Server: objective_function: c_str: {c_str} f_str: {f_str} kwargs_str: {kwargs_str}') 95 | 96 | configuration = json.loads(c_str, cls=BenchmarkDecoder) 97 | fidelity = json.loads(f_str, cls=BenchmarkDecoder) 98 | kwargs = json.loads(kwargs_str, cls=BenchmarkDecoder) 99 | 100 | result = self.benchmark.objective_function_test(configuration=configuration, fidelity=fidelity, **kwargs) 101 | return json.dumps(result, indent=None, cls=BenchmarkEncoder) 102 | 103 | def get_meta_information(self): 104 | logger.debug('Server: get_meta_info called') 105 | return json.dumps(self.benchmark.get_meta_information(), indent=None, cls=BenchmarkEncoder) 106 | 107 | @Pyro4.oneway # in case call returns much later than daemon.shutdown 108 | def shutdown(self): 109 | logger.debug('Server: Shutting down...') 110 | Pyro4.config.COMMTIMEOUT = 0.5 111 | self.pyro_running = False 112 | self.daemon.shutdown() 113 | 114 | 115 | if __name__ == "__main__": 116 | Pyro4.config.REQUIRE_EXPOSE = False 117 | 118 | parser = argparse.ArgumentParser(prog='server_abstract_benchmark.py', 119 | description='HPOBench Container Server', 120 | usage='%(prog)s ') 121 | parser.add_argument('importBase', type=str, 122 | help='Relative path to benchmark file in hpobench/benchmarks, e.g. ml.xgboost_benchmark') 123 | parser.add_argument('benchmark', type=str, 124 | help='Classname of the benchmark, e.g. XGBoostBenchmark') 125 | parser.add_argument('socket_id', type=str, 126 | help='socket_id for pyro-server') 127 | 128 | args = parser.parse_args() 129 | 130 | # pylint: disable=logging-fstring-interpolation 131 | exec(f"from hpobench.benchmarks.{args.importBase} import {args.benchmark} as Benchmark") 132 | bp = BenchmarkServer(args.socket_id) 133 | -------------------------------------------------------------------------------- /hpobench/dependencies/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/HPOBench/47bf141f79e6bdfb26d1f1218b5d5aac09d7d2ce/hpobench/dependencies/__init__.py -------------------------------------------------------------------------------- /hpobench/dependencies/ml/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/HPOBench/47bf141f79e6bdfb26d1f1218b5d5aac09d7d2ce/hpobench/dependencies/ml/__init__.py -------------------------------------------------------------------------------- /hpobench/dependencies/od/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/HPOBench/47bf141f79e6bdfb26d1f1218b5d5aac09d7d2ce/hpobench/dependencies/od/__init__.py -------------------------------------------------------------------------------- /hpobench/dependencies/od/backbones/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/HPOBench/47bf141f79e6bdfb26d1f1218b5d5aac09d7d2ce/hpobench/dependencies/od/backbones/__init__.py -------------------------------------------------------------------------------- /hpobench/dependencies/od/backbones/mlp.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | 4 | from hpobench.dependencies.od.utils.activations import ACTIVATIONS 5 | 6 | 7 | class Block(nn.Module): 8 | def __init__(self, in_channels, out_channels, activation, batch_normalization=False, dropout_rate=0.0): 9 | super().__init__() 10 | 11 | self.batch_normalization = batch_normalization 12 | self.linear_layer = nn.Linear(in_features=in_channels, out_features=out_channels) 13 | self.batch_normalization_layer = nn.BatchNorm1d(out_channels) 14 | self.activation_layer = activation 15 | self.dropout_layer = nn.Dropout(dropout_rate) 16 | 17 | def forward(self, x1, x2=None): 18 | if x2 is not None: 19 | x = torch.cat([x1, x2], dim=1) 20 | else: 21 | x = x1 22 | 23 | z = self.linear_layer(x) 24 | 25 | if self.batch_normalization: 26 | # Batch normalization causes some troubles if it comes 27 | # to validation sanity run since mean/variance 28 | # are not initialized to that moment 29 | try: 30 | z = self.batch_normalization_layer(z) 31 | except: # noqa E722 32 | pass 33 | 34 | z = self.activation_layer(z) 35 | z = self.dropout_layer(z) 36 | 37 | return z 38 | 39 | 40 | class MLP(nn.Module): 41 | def __init__(self, num_features, config): 42 | super(MLP, self).__init__() 43 | self.config = config 44 | self.num_features = num_features 45 | self._build_backbone() 46 | 47 | def _get_activation(self): 48 | if self.config["activation"] == "swish" or self.config["activation"] == "swish-1": 49 | train_beta = False 50 | if self.config["activation"] == "swish": 51 | train_beta = True 52 | 53 | return ACTIVATIONS["swish"](train_beta=train_beta) 54 | else: 55 | return ACTIVATIONS[self.config["activation"]]() 56 | 57 | def _build_backbone(self): 58 | features = self.num_features 59 | activation = self._get_activation() 60 | latent_dim = self.config["num_latent_units"] 61 | 62 | encoder_features = [features] 63 | for i in range(1, self.config["num_layers"]+1): 64 | encoder_features += [self.config[f"num_units_layer_{i}"]] 65 | 66 | decoder_features = [latent_dim] + encoder_features[::-1] 67 | 68 | features = encoder_features + decoder_features 69 | in_features = features.copy() # We need different in_features if we use skip connections 70 | 71 | if self.config["skip_connection"]: 72 | # If skip connection 73 | # Usually we'd have the following: 74 | # 768 -> 128 -> 64 -> 8 -> 64 -> 128 -> 768 75 | # But since we merge the layers we get 76 | # 768 -> 128 -> 64 -> 8+64 -> 64+128 -> 128+768 -> 768 77 | decoder_index = int(len(features) / 2) 78 | encoder_index = decoder_index - 1 79 | for i in range(decoder_index, len(features)-1): 80 | in_features[i] = features[decoder_index] + features[encoder_index] 81 | encoder_index -= 1 82 | decoder_index += 1 83 | 84 | decoder_in_features = in_features[int(len(features) / 2):] 85 | else: 86 | decoder_in_features = in_features[int(len(features) / 2):] 87 | 88 | # Build encoder 89 | self.encoder_blocks = [] 90 | for i in range(len(encoder_features)-1): 91 | self.encoder_blocks += [ 92 | Block( 93 | encoder_features[i], 94 | encoder_features[i+1], 95 | activation, 96 | batch_normalization=self.config["batch_normalization"], 97 | dropout_rate=0.0 if not self.config["dropout"] else self.config["dropout_rate"] 98 | ) 99 | ] 100 | 101 | # Build decoder 102 | self.decoder_blocks = [] 103 | for i in range(len(decoder_features)-2): 104 | self.decoder_blocks += [ 105 | Block( 106 | decoder_in_features[i], 107 | decoder_features[i+1], 108 | activation, 109 | batch_normalization=self.config["batch_normalization"], 110 | dropout_rate=0.0 if not self.config["dropout"] else self.config["dropout_rate"] 111 | ) 112 | ] 113 | 114 | self.latent_dim = latent_dim 115 | self.encoder_features = encoder_features 116 | self.decoder_features = decoder_features 117 | self.decoder_in_features = decoder_in_features 118 | 119 | # Make sure the parameters are within the model 120 | self.encoder_blocks = nn.Sequential(*self.encoder_blocks) 121 | self.decoder_blocks = nn.Sequential(*self.decoder_blocks) 122 | 123 | def encode(self, x): 124 | encoder_outputs = [] 125 | 126 | output = x 127 | encoder_outputs += [output] 128 | 129 | # Processing encoder 130 | for block in self.encoder_blocks: 131 | output = block(output) 132 | encoder_outputs += [output] 133 | 134 | return encoder_outputs 135 | 136 | def decode(self, z, encoder_outputs=None): 137 | # Use encoder outputs only if skip connection is used 138 | if not self.config["skip_connection"]: 139 | encoder_outputs = None 140 | 141 | if encoder_outputs is not None: 142 | encoder_outputs = encoder_outputs[::-1] 143 | 144 | # Processing decoder 145 | output = z 146 | for i, block in enumerate(self.decoder_blocks): 147 | if encoder_outputs is not None: 148 | output = block(output, encoder_outputs[i]) 149 | else: 150 | output = block(output) 151 | 152 | # concat if skip connection available 153 | if encoder_outputs is not None: 154 | output = torch.cat([output, encoder_outputs[-1]], dim=1) 155 | 156 | return output 157 | -------------------------------------------------------------------------------- /hpobench/dependencies/od/callbacks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/HPOBench/47bf141f79e6bdfb26d1f1218b5d5aac09d7d2ce/hpobench/dependencies/od/callbacks/__init__.py -------------------------------------------------------------------------------- /hpobench/dependencies/od/callbacks/checkpoint_saver.py: -------------------------------------------------------------------------------- 1 | import pytorch_lightning as pl 2 | 3 | 4 | class CheckpointSaver(pl.Callback): 5 | def __init__(self): 6 | self.best_checkpoint = None 7 | 8 | def save(self, trainer, model): 9 | """ 10 | Saves the best weights locally. 11 | """ 12 | 13 | checkpoint = { 14 | 'should_stop': trainer.should_stop, 15 | 'current_epoch': trainer.current_epoch, 16 | 'weights': model.state_dict(), 17 | } 18 | 19 | self.best_checkpoint = checkpoint 20 | 21 | def load(self, trainer, model): 22 | checkpoint = None 23 | if hasattr(self, "best_checkpoint"): 24 | checkpoint = self.best_checkpoint 25 | 26 | if checkpoint is not None: 27 | trainer.should_stop = checkpoint["should_stop"] 28 | trainer.current_epoch = checkpoint["current_epoch"] + 1 29 | model.load_state_dict(checkpoint["weights"]) 30 | 31 | def on_validation_end(self, trainer, model): 32 | # We already saved if the trainer has stopped 33 | if not trainer.should_stop: 34 | # Save if it's the best epoch 35 | if model.val_auprs[-1] == max(model.val_auprs): 36 | self.save(trainer, model) 37 | 38 | def on_test_start(self, trainer, model): 39 | # Load best weights here 40 | self.load(trainer, model) 41 | -------------------------------------------------------------------------------- /hpobench/dependencies/od/callbacks/earlystopping.py: -------------------------------------------------------------------------------- 1 | import pytorch_lightning as pl 2 | 3 | 4 | class EarlyStopping(pl.Callback): 5 | def __init__(self, activated: bool, patience: int, worst_loss: float): 6 | self.patience = patience 7 | self.lowest_loss = worst_loss 8 | self.counter = 0 9 | self.activated = activated 10 | 11 | def setup(self, trainer, model, stage): 12 | if not self.activated: 13 | trainer.should_stop = False 14 | 15 | def on_validation_end(self, trainer, model): 16 | if not self.activated: 17 | return 18 | 19 | last_loss = model.val_auprs[-1] 20 | 21 | if last_loss > self.lowest_loss: 22 | self.counter += 1 23 | if self.counter >= self.patience: 24 | trainer.should_stop = True 25 | else: 26 | self.lowest_loss = last_loss 27 | self.counter = 0 28 | -------------------------------------------------------------------------------- /hpobench/dependencies/od/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/HPOBench/47bf141f79e6bdfb26d1f1218b5d5aac09d7d2ce/hpobench/dependencies/od/models/__init__.py -------------------------------------------------------------------------------- /hpobench/dependencies/od/models/autoencoder.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pytorch_lightning as pl 3 | import torch 4 | import torch.nn.functional as F 5 | from sklearn.metrics import precision_recall_curve, auc 6 | from torch import nn 7 | 8 | 9 | class Autoencoder(pl.LightningModule): 10 | def __init__(self, backbone, config): 11 | super().__init__() 12 | 13 | self.config = config 14 | self.backbone = backbone 15 | 16 | self.train_losses = [] 17 | self.train_auprs = [] 18 | self.val_auprs = [] 19 | self.test_aupr = None 20 | 21 | # Setup latent layer 22 | self.latent_layer = nn.Linear( 23 | in_features=self.backbone.encoder_features[-1], 24 | out_features=self.backbone.latent_dim 25 | ) 26 | 27 | # Setup output layer 28 | self.output_layer = nn.Linear( 29 | in_features=self.backbone.decoder_in_features[-2], 30 | out_features=self.backbone.decoder_features[-1] 31 | ) 32 | 33 | def configure_optimizers(self): 34 | optimizer = torch.optim.AdamW( 35 | self.parameters(), 36 | lr=self.config["lr"], 37 | betas=(self.config["beta1"], self.config["beta2"]), 38 | weight_decay=self.config["weight_decay"] 39 | ) 40 | 41 | return optimizer 42 | 43 | @staticmethod 44 | def calculate_aupr(labels, scores): 45 | precision, recall, _ = precision_recall_curve(labels, scores) 46 | aupr = auc(recall, precision) 47 | 48 | return aupr 49 | 50 | @staticmethod 51 | def calculate_loss(x, x_hat): 52 | return F.mse_loss(x_hat, x) 53 | 54 | def forward(self, x): 55 | # Encode first 56 | encoder_outputs = self.backbone.encode(x) 57 | z = self.latent_layer(encoder_outputs[-1]) 58 | 59 | # Decode 60 | x_hat = self.output_layer(self.backbone.decode(z, encoder_outputs)) 61 | 62 | return x_hat 63 | 64 | def training_step(self, batch, _): 65 | x, _ = batch 66 | x_hat = self(x) 67 | loss = self.calculate_loss(x, x_hat) 68 | 69 | return loss 70 | 71 | def training_epoch_end(self, outputs): 72 | losses = torch.stack([o['loss'] for o in outputs]).cpu().numpy().flatten() 73 | self.train_losses.append(np.mean(losses)) 74 | 75 | def validation_step(self, batch, _): 76 | x, y = batch 77 | x_hat = self(x) 78 | loss = self.calculate_loss(x, x_hat) 79 | 80 | return { 81 | 'labels': y.flatten(), 82 | 'loss': loss 83 | } 84 | 85 | def validation_epoch_end(self, outputs): 86 | labels = torch.stack([o['labels'] for o in outputs]).cpu().numpy().flatten() 87 | losses = torch.stack([o['loss'] for o in outputs]).cpu().numpy().flatten() 88 | 89 | aupr = self.calculate_aupr(labels, losses) 90 | self.val_auprs.append(aupr) 91 | 92 | def test_step(self, batch, _): 93 | x, y = batch 94 | x_hat = self(x) 95 | loss = self.calculate_loss(x, x_hat) 96 | 97 | return { 98 | 'labels': y.flatten(), 99 | 'loss': loss 100 | } 101 | 102 | def test_epoch_end(self, outputs): 103 | labels = np.array([o['labels'].item() for o in outputs]).flatten() 104 | losses = np.array([o['loss'].item() for o in outputs]).flatten() 105 | 106 | self.test_aupr = self.calculate_aupr(labels, losses) 107 | -------------------------------------------------------------------------------- /hpobench/dependencies/od/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/HPOBench/47bf141f79e6bdfb26d1f1218b5d5aac09d7d2ce/hpobench/dependencies/od/utils/__init__.py -------------------------------------------------------------------------------- /hpobench/dependencies/od/utils/activations.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from torch.nn.parameter import Parameter 4 | 5 | 6 | class Swish(nn.Module): 7 | def __init__(self, train_beta=False): 8 | super(Swish, self).__init__() 9 | if train_beta: 10 | self.weight = Parameter(torch.Tensor([1.])) 11 | else: 12 | self.weight = 1.0 13 | 14 | def forward(self, input): 15 | return input * torch.sigmoid(self.weight * input) 16 | 17 | 18 | ACTIVATIONS = { 19 | "relu": torch.nn.ReLU, 20 | "tanh": torch.nn.Tanh, 21 | "sigmoid": torch.nn.Sigmoid, 22 | "swish": Swish 23 | } 24 | -------------------------------------------------------------------------------- /hpobench/dependencies/od/utils/scaler.py: -------------------------------------------------------------------------------- 1 | from sklearn.preprocessing import MinMaxScaler, StandardScaler 2 | 3 | 4 | def get_fitted_scaler(X_train, name=None): 5 | """ 6 | Instantiates a scaler by a given name and fits the scaler 7 | with X_train. 8 | """ 9 | 10 | if name == "MinMax": 11 | scaler = MinMaxScaler(feature_range=(0, 1), copy=True) 12 | elif name == "Standard": 13 | scaler = StandardScaler(copy=True) 14 | elif name is None or name == "None": 15 | return None 16 | else: 17 | raise NotImplementedError() 18 | 19 | scaler.fit(X_train) 20 | return lambda x: scaler.transform(x) 21 | -------------------------------------------------------------------------------- /hpobench/util/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/HPOBench/47bf141f79e6bdfb26d1f1218b5d5aac09d7d2ce/hpobench/util/__init__.py -------------------------------------------------------------------------------- /hpobench/util/clean_up_script.py: -------------------------------------------------------------------------------- 1 | from hpobench import config_file 2 | 3 | import shutil 4 | import logging 5 | logger = logging.getLogger('Clean-up') 6 | logger.setLevel(logging.INFO) 7 | 8 | 9 | def _ask_for_del(directory, name): 10 | logger.info(f'Going to remove the {name} directory {directory}') 11 | inp = input('Do you want to proceed? [N|y] ') 12 | if inp in ['y', 'j', 'Y']: 13 | shutil.rmtree(directory) 14 | logger.info(f'Successfully removed the {name} directory.') 15 | 16 | 17 | def delete_container(): 18 | _ask_for_del(config_file.container_dir, 'container') 19 | 20 | 21 | def clear_socket_dir(): 22 | _ask_for_del(config_file.socket_dir, 'socket') 23 | 24 | 25 | def clear_cache(): 26 | _ask_for_del(config_file.cache_dir, 'cache') 27 | 28 | 29 | def clear_data_dir(): 30 | _ask_for_del(config_file.data_dir, 'data') 31 | 32 | 33 | if __name__ == '__main__': 34 | import argparse 35 | parser = argparse.ArgumentParser() 36 | parser.add_argument("--clear_all", help="Remove containers, clear socket, data, and cache directory", 37 | action="store_true") 38 | parser.add_argument("--clear_container", help="Delete the HPOBench container", action="store_true") 39 | parser.add_argument("--clear_cache", help="Delete the HPOBench cache", action="store_true") 40 | parser.add_argument("--clear_data", help="Delete the HPOBench data", action="store_true") 41 | parser.add_argument("--clear_socket", help="Delete the HPOBench socket", action="store_true") 42 | args = parser.parse_args() 43 | 44 | if args.clear_all or args.clear_container: 45 | delete_container() 46 | if args.clear_all or args.clear_cache: 47 | clear_cache() 48 | if args.clear_all or args.clear_data: 49 | clear_data_dir() 50 | if args.clear_all or args.clear_socket: 51 | clear_socket_dir() 52 | -------------------------------------------------------------------------------- /hpobench/util/container_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import importlib 3 | import json 4 | import numpy as np 5 | import enum 6 | 7 | from typing import Any, Union 8 | 9 | from hpobench.util.rng_helper import serialize_random_state, deserialize_random_state 10 | 11 | 12 | class BenchmarkEncoder(json.JSONEncoder): 13 | """ Json Encoder to save tuple and or numpy arrays | numpy floats / integer. 14 | from: https://stackoverflow.com/questions/15721363/preserve-python-tuples-with-json 15 | 16 | Serializing tuple/numpy array may not work. We need to annotate those types, to reconstruct them correctly. 17 | """ 18 | # pylint: disable=arguments-differ 19 | def encode(self, obj): 20 | def hint(item): 21 | # Annotate the different item types 22 | if isinstance(item, tuple): 23 | return {'__type__': 'tuple', '__items__': [hint(e) for e in item]} 24 | if isinstance(item, np.ndarray): 25 | return {'__type__': 'np.ndarray', '__items__': item.tolist()} 26 | if isinstance(item, np.floating): 27 | return {'__type__': 'np.float', '__items__': float(item)} 28 | if isinstance(item, np.integer): 29 | return {'__type__': 'np.int', '__items__': item.tolist()} 30 | if isinstance(item, enum.Enum): 31 | return str(item) 32 | if isinstance(item, np.random.RandomState): 33 | rs = serialize_random_state(item) 34 | return {'__type__': 'random_state', '__items__': rs} 35 | 36 | # If it is a container data structure, go also through the items. 37 | if isinstance(item, list): 38 | return [hint(e) for e in item] 39 | if isinstance(item, dict): 40 | return {key: hint(value) for key, value in item.items()} 41 | return item 42 | 43 | return super(BenchmarkEncoder, self).encode(hint(obj)) 44 | 45 | 46 | class BenchmarkDecoder(json.JSONDecoder): 47 | def __init__(self, *args, **kwargs): 48 | json.JSONDecoder.__init__(self, object_hook=self.object_hook, *args, **kwargs) 49 | 50 | def object_hook(self, obj: Any) -> Union[Union[tuple, np.ndarray, float, float, int], Any]: 51 | if '__type__' in obj: 52 | __type = obj['__type__'] 53 | 54 | if __type == 'tuple': 55 | return tuple(obj['__items__']) 56 | if __type == 'np.ndarray': 57 | return np.array(obj['__items__']) 58 | if __type == 'np.float': 59 | return np.float(obj['__items__']) 60 | if __type == 'np.int': 61 | return np.int(obj['__items__']) 62 | if __type == 'random_state': 63 | return deserialize_random_state(obj['__items__']) 64 | return obj 65 | 66 | 67 | def __reload_module(): 68 | """ 69 | The env variable which enables the debug level is read in during the import of the client module. 70 | Reloading the module, re-reads the env variable and therefore changes the level. 71 | """ 72 | import hpobench.container.client_abstract_benchmark as client 73 | importlib.reload(client) 74 | 75 | 76 | def enable_container_debug(): 77 | """ Sets the environment variable "HPOBENCH_DEBUG" to true. The container checks this variable and if set to true, 78 | enables debugging on the container side. """ 79 | os.environ['HPOBENCH_DEBUG'] = 'true' 80 | __reload_module() 81 | 82 | 83 | def disable_container_debug(): 84 | os.environ['HPOBENCH_DEBUG'] = 'false' 85 | __reload_module() 86 | -------------------------------------------------------------------------------- /hpobench/util/dependencies.py: -------------------------------------------------------------------------------- 1 | import re 2 | from distutils.version import LooseVersion 3 | 4 | import pkg_resources 5 | 6 | RE_PATTERN = re.compile( 7 | r'^(?P[\w\-]+)((?P==|>=|>)' 8 | r'(?P(\d+)?(\.[a-zA-Z0-9]+)?(\.\d+)?))?$') 9 | 10 | """Generic code to verify package versions (a.k.a. dependencies). 11 | 12 | Written by Anatolii Domashnev (@ayaro) for auto-sklearn. Licensed under a BSD 13 | 3-clause license. 14 | 15 | See the following link for the original PR: 16 | https://github.com/Ayaro/auto-sklearn/commit/ 17 | f59c6e9751061ec0e68a402507c83f6c10ae5bbd 18 | """ 19 | 20 | 21 | def verify_packages(packages): 22 | if not packages: 23 | return 24 | if isinstance(packages, str): 25 | packages = packages.splitlines() 26 | 27 | for package in packages: 28 | if not package: 29 | continue 30 | 31 | match = RE_PATTERN.match(package) 32 | if match: 33 | name = match.group('name') 34 | operation = match.group('operation') 35 | version = match.group('version') 36 | _verify_package(name, operation, version) 37 | else: 38 | raise ValueError('Unable to read requirement: %s' % package) 39 | 40 | 41 | def _verify_package(name, operation, version): 42 | try: 43 | module = pkg_resources.get_distribution(name) 44 | except pkg_resources.DistributionNotFound: 45 | raise MissingPackageError(name) 46 | 47 | if not operation: 48 | return 49 | 50 | required_version = LooseVersion(version) 51 | installed_version = LooseVersion(module.version) 52 | 53 | if operation == '==': 54 | check = required_version == installed_version 55 | elif operation == '>': 56 | check = installed_version > required_version 57 | elif operation == '>=': 58 | check = installed_version > required_version or \ 59 | installed_version == required_version 60 | else: 61 | raise NotImplementedError( 62 | 'operation \'%s\' is not supported' % operation) 63 | if not check: 64 | raise IncorrectPackageVersionError(name, installed_version, operation, 65 | required_version) 66 | 67 | 68 | class MissingPackageError(Exception): 69 | error_message = 'mandatory package \'{name}\' not found' 70 | 71 | def __init__(self, package_name): 72 | self.package_name = package_name 73 | super(MissingPackageError, self).__init__( 74 | self.error_message.format(name=package_name)) 75 | 76 | 77 | class IncorrectPackageVersionError(Exception): 78 | error_message = '\'{name} {installed_version}\' version mismatch ' \ 79 | '({operation}{required_version})' 80 | 81 | def __init__(self, package_name, installed_version, operation, 82 | required_version): 83 | self.package_name = package_name 84 | self.installed_version = installed_version 85 | self.operation = operation 86 | self.required_version = required_version 87 | message = self.error_message.format( 88 | name=package_name, 89 | installed_version=installed_version, 90 | operation=operation, 91 | required_version=required_version) 92 | super(IncorrectPackageVersionError, self).__init__(message) 93 | -------------------------------------------------------------------------------- /hpobench/util/example_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | from pathlib import Path 3 | from typing import Dict 4 | 5 | 6 | def get_travis_settings(optimizer_type: str) -> Dict: 7 | """ Helper function to reduce time consumption for test runs on travis.ci""" 8 | if optimizer_type == 'smac': 9 | return {"runcount-limit": 5, 'wallclock-limit': 50, 'cutoff': 50, 'memory_limit': 10000, 'output_dir': '.'} 10 | if optimizer_type == 'bohb': 11 | return {'max_budget': 2, 'num_iterations': 1, 'output_dir': Path('./')} 12 | 13 | raise ValueError(f'Unknown type {optimizer_type}. Must be one of [smac, bohb]') 14 | 15 | 16 | def set_env_variables_to_use_only_one_core(): 17 | """ Helper function: Sets all variables which are responsible for using multiple threads to 1. 18 | This is necessary/useful, if you are computing on a cluster.""" 19 | os.environ['OMP_NUM_THREADS'] = '1' 20 | os.environ['OPENBLAS_NUM_THREADS'] = '1' 21 | os.environ['MKL_NUM_THREADS'] = '1' 22 | os.environ['VECLIB_MAXIMUM_THREADS'] = '1' 23 | os.environ['NUMEXPR_NUM_THREADS'] = '1' 24 | os.environ['NUMEXPR_MAX_THREADS'] = '1' 25 | -------------------------------------------------------------------------------- /hpobench/util/rng_helper.py: -------------------------------------------------------------------------------- 1 | """ Helper functions to easily obtain randomState """ 2 | from typing import Union, Tuple, List 3 | 4 | import numpy as np 5 | 6 | 7 | def get_rng(rng: Union[int, np.random.RandomState, None] = None, 8 | self_rng: Union[int, np.random.RandomState, None] = None) -> np.random.RandomState: 9 | """ 10 | Helper function to obtain RandomState from int or create a new one. 11 | 12 | Sometimes a default random state (self_rng) is already available, but a 13 | new random state is desired. In this case ``rng`` is not None and not already 14 | a random state (int or None) -> a new random state is created. 15 | If ``rng`` is already a randomState, it is just returned. 16 | Same if ``rng`` is None, but the default rng is given. 17 | 18 | Parameters 19 | ---------- 20 | rng : int, np.random.RandomState, None 21 | self_rng : np.random.RandomState, None 22 | 23 | Returns 24 | ------- 25 | np.random.RandomState 26 | """ 27 | 28 | if rng is not None: 29 | return _cast_int_to_random_state(rng) 30 | if rng is None and self_rng is not None: 31 | return _cast_int_to_random_state(self_rng) 32 | return np.random.RandomState() 33 | 34 | 35 | def _cast_int_to_random_state(rng: Union[int, np.random.RandomState]) -> np.random.RandomState: 36 | """ 37 | Helper function to cast ``rng`` from int to np.random.RandomState if necessary. 38 | 39 | Parameters 40 | ---------- 41 | rng : int, np.random.RandomState 42 | 43 | Returns 44 | ------- 45 | np.random.RandomState 46 | """ 47 | if isinstance(rng, np.random.RandomState): 48 | return rng 49 | if int(rng) == rng: 50 | # As seed is sometimes -1 (e.g. if SMAC optimizes a deterministic function) -> use abs() 51 | return np.random.RandomState(np.abs(rng)) 52 | raise ValueError(f"{rng} is neither a number nor a RandomState. Initializing RandomState failed") 53 | 54 | 55 | def serialize_random_state(random_state: np.random.RandomState) -> Tuple[int, List, int, int, int]: 56 | (rnd0, rnd1, rnd2, rnd3, rnd4) = random_state.get_state() 57 | rnd1 = rnd1.tolist() 58 | return rnd0, rnd1, rnd2, rnd3, rnd4 59 | 60 | 61 | def deserialize_random_state(random_state: Tuple[int, List, int, int, int]) -> np.random.RandomState: 62 | (rnd0, rnd1, rnd2, rnd3, rnd4) = random_state 63 | rnd1 = [np.uint32(number) for number in rnd1] 64 | random_state = np.random.RandomState() 65 | random_state.set_state((rnd0, rnd1, rnd2, rnd3, rnd4)) 66 | return random_state 67 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | scipy>=1.4.1 2 | numpy>=1.18.1 3 | ConfigSpace>=0.4.12 4 | Pyro4==4.80 5 | oslo.concurrency>=4.2.0 -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | import setuptools 3 | import json 4 | import os 5 | 6 | 7 | def get_extra_requirements(): 8 | """ Helper function to read in all extra requirement files in the extra 9 | requirement folder. """ 10 | extra_requirements = {} 11 | for file in os.listdir('./extra_requirements'): 12 | with open(f'./extra_requirements/{file}', encoding='utf-8') as fh: 13 | requirements = json.load(fh) 14 | extra_requirements.update(requirements) 15 | return extra_requirements 16 | 17 | 18 | def read_file(file_name): 19 | with open(file_name, encoding='utf-8') as fh: 20 | text = fh.read() 21 | return text 22 | 23 | 24 | setuptools.setup( 25 | name='hpobench', 26 | author_email='eggenspk@informatik.uni-freiburg.de', 27 | description='Benchmark-Suite for Hyperparameter Optimization', 28 | long_description=read_file('README.md'), 29 | long_description_content_type='text/markdown', 30 | license='Apache-2.0', 31 | url='https://www.automl.org/automl/hpobench/', 32 | project_urls={ 33 | 'Documentation': 'https://automl.github.io/HPOBench/', 34 | 'Source Code': 'https://github.com/automl/HPOBench' 35 | }, 36 | version=read_file('hpobench/__version__.py').split()[-1].strip('\''), 37 | packages=setuptools.find_packages(exclude=['*.tests', '*.tests.*', 38 | 'tests.*', 'tests'],), 39 | python_requires='>=3.6, <=3.10', 40 | install_requires=read_file('./requirements.txt').split('\n'), 41 | extras_require=get_extra_requirements(), 42 | test_suite='pytest', 43 | platforms=['Linux'], 44 | classifiers=[ 45 | 'Programming Language :: Python :: 3.5', 46 | 'Programming Language :: Python :: 3.6', 47 | 'Programming Language :: Python :: 3.7', 48 | 'Programming Language :: Python :: 3.8', 49 | 'Programming Language :: Python :: 3.9', 50 | 'Programming Language :: Python :: 3.10', 51 | 'Development Status :: 3 - Alpha', 52 | 'Natural Language :: English', 53 | 'Environment :: Console', 54 | 'Intended Audience :: Developers', 55 | 'Intended Audience :: Education', 56 | 'Intended Audience :: Science/Research', 57 | 'License :: OSI Approved :: Apache Software License', 58 | 'Operating System :: POSIX :: Linux', 59 | 'Topic :: Scientific/Engineering :: Artificial Intelligence', 60 | 'Topic :: Scientific/Engineering', 61 | 'Topic :: Software Development', 62 | ] 63 | ) 64 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/automl/HPOBench/47bf141f79e6bdfb26d1f1218b5d5aac09d7d2ce/tests/__init__.py -------------------------------------------------------------------------------- /tests/test_abstract_benchmark.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from hpobench.abstract_benchmark import AbstractBenchmark 4 | 5 | with pytest.raises(NotImplementedError): 6 | AbstractBenchmark.get_configuration_space() 7 | 8 | with pytest.raises(NotImplementedError): 9 | AbstractBenchmark.get_fidelity_space() 10 | 11 | with pytest.raises(NotImplementedError): 12 | AbstractBenchmark.get_meta_information() 13 | -------------------------------------------------------------------------------- /tests/test_container_availbable.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import logging 3 | # from hpobench import config_file 4 | 5 | # Currently, the gitlab registry does not easily support the search functionality. 6 | # The container are still available on sylabs (old registry), in case the gitlab registry is somehow not reachable. 7 | # TODO: Write a search functionality for the gitlab registry. 8 | library = 'library://phmueller/automl' # config_file.container_source 9 | 10 | 11 | def search_container(container_name): 12 | out = subprocess.getoutput(f'singularity search {container_name}') 13 | logging.debug(out) 14 | 15 | out = out.split('\n\n') 16 | container_available = any((f'{library}/{container_name}' in line for line in out)) 17 | return container_available 18 | 19 | 20 | def test_availability(): 21 | container_names = ['pybnn', 22 | 'paramnet', 23 | 'svm_benchmark', 24 | 'xgboost_benchmark', 25 | 'nasbench_101', 26 | 'nasbench_201', 27 | 'nasbench_1shot1', 28 | 'tabular_benchmarks', 29 | 'cartpole', 30 | 'learna_benchmark' 31 | ] 32 | 33 | all_available = True 34 | not_available = [] 35 | for container in container_names: 36 | container_available = search_container(container) 37 | 38 | if not container_available: 39 | logging.warning(f'Container for {container} is not found in {library}') 40 | all_available = False 41 | not_available.append(container) 42 | 43 | assert all_available, f'Some containers are not online available. {not_available}' -------------------------------------------------------------------------------- /tests/test_data_manager.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | from multiprocessing import Pool 3 | 4 | import pytest 5 | 6 | import hpobench 7 | from hpobench.util.data_manager import NASBench_201Data, YearPredictionMSDData, ProteinStructureData, BostonHousingData 8 | skip_message = 'We currently skip this test because it takes too much time.' 9 | 10 | 11 | @pytest.mark.skip(reason=skip_message) 12 | def test_nasbench_201_load_thread_safe(): 13 | shutil.rmtree(hpobench.config_file.data_dir / "nasbench_201", ignore_errors=True) 14 | function = lambda: NASBench_201Data(dataset='cifar100').load() 15 | with Pool(3) as pool: 16 | pool.map(function, []) 17 | 18 | 19 | @pytest.mark.skip(reason=skip_message) 20 | def test_nasbench_201_init(): 21 | 22 | data_manager = NASBench_201Data(dataset='cifar100') 23 | assert len(data_manager.files) == 3 24 | assert all([file.startswith('NAS-Bench') for file in data_manager.files]) 25 | 26 | with pytest.raises(AssertionError): 27 | NASBench_201Data(dataset='Non_existing_dataset') 28 | 29 | assert data_manager._save_dir == hpobench.config_file.data_dir / "nasbench_201" 30 | assert data_manager._save_dir.exists() 31 | 32 | 33 | @pytest.mark.skip(reason=skip_message) 34 | def test_nasbench_201_load(): 35 | 36 | shutil.rmtree(hpobench.config_file.data_dir / "nasbench_201", ignore_errors=True) 37 | 38 | data_manager = NASBench_201Data(dataset='cifar100') 39 | data = data_manager.load() 40 | 41 | assert len(data) == 3 42 | assert (hpobench.config_file.data_dir / "nasbench_201").exists() 43 | assert len(list((hpobench.config_file.data_dir / "nasbench_201").glob('*.json'))) == 3 44 | assert not (hpobench.config_file.data_dir / "nasbench_201_data_v1.3.zip").exists() 45 | 46 | data_manager.data = None 47 | 48 | data_manager = NASBench_201Data(dataset='cifar100') 49 | data = data_manager.load() 50 | assert len(data) == 3 51 | 52 | 53 | def test_year_prediction_msd_data(): 54 | dm = YearPredictionMSDData() 55 | assert dm.url_source == 'https://archive.ics.uci.edu/ml/machine-learning-databases/00203/YearPredictionMSD.txt.zip' 56 | assert dm._save_dir.exists() 57 | 58 | # First one downloads the data 59 | x_train, y_train, x_valid, y_valid, x_test, y_test = dm.load() 60 | 61 | # second call should check the 'if exists' branch 62 | _ = dm.load() 63 | 64 | # train = 70%, valid = 20%, test = 10% 65 | assert 0 < len(x_test) == len(y_test) 66 | assert len(y_test) < len(x_valid) == len(y_valid) 67 | assert len(y_valid) < len(x_train) == len(y_train) 68 | 69 | 70 | def test_protein_structure_data(): 71 | dm = ProteinStructureData() 72 | assert dm.url_source == 'https://archive.ics.uci.edu/ml/machine-learning-databases/00265/CASP.csv' 73 | assert dm._save_dir.exists() 74 | 75 | # First one downloads the data 76 | x_train, y_train, x_valid, y_valid, x_test, y_test = dm.load() 77 | 78 | # second call should check the 'if exists' branch 79 | _ = dm.load() 80 | 81 | # train = 60%, valid = 20%, test = 20% 82 | assert 0 < len(x_test) == len(y_test) 83 | assert 0 < len(x_valid) == len(y_valid) 84 | assert len(y_valid) < len(x_train) == len(y_train) 85 | 86 | 87 | def test_boston_data(): 88 | dm = BostonHousingData() 89 | assert dm.url_source == 'https://archive.ics.uci.edu/ml/machine-learning-databases/housing/housing.data' 90 | assert dm._save_dir.exists() 91 | 92 | # First one downloads the data 93 | x_train, y_train, x_valid, y_valid, x_test, y_test = dm.load() 94 | 95 | # second call should check the 'if exists' branch 96 | _ = dm.load() 97 | 98 | # train = 60%, valid = 20%, test = 20% 99 | assert 0 < len(x_test) == len(y_test) 100 | assert 0 < len(x_valid) == len(y_valid) 101 | assert len(y_valid) < len(x_train) == len(y_train) 102 | 103 | 104 | def test_tabular_datamanager(): 105 | from hpobench.util.data_manager import TabularDataManager 106 | dm = TabularDataManager(model='lr', 107 | task_id='3') 108 | 109 | table, meta_data = dm.load() 110 | 111 | assert (hpobench.config_file.data_dir / "TabularData" / 'lr' / str(3) / f'lr_3_data.parquet.gzip').exists() 112 | assert (hpobench.config_file.data_dir / "TabularData" / 'lr' / str(3) / f'lr_3_metadata.json').exists() 113 | 114 | table_2, meta_data_2 = dm.load() 115 | -------------------------------------------------------------------------------- /tests/test_hpobenchconfig.py: -------------------------------------------------------------------------------- 1 | def test_version_check(): 2 | from hpobench.config import HPOBenchConfig 3 | 4 | # Same partition 5 | assert HPOBenchConfig._check_version('0.0.0', '0.0.1') 6 | assert HPOBenchConfig._check_version('0.0.0', '0.0.5') 7 | 8 | assert HPOBenchConfig._check_version('0.0.6', '0.0.7') 9 | assert HPOBenchConfig._check_version('0.0.8', '0.0.1234') 10 | assert HPOBenchConfig._check_version('1.0.0', '0.0.8') 11 | 12 | assert not HPOBenchConfig._check_version(None, '0.0.1') 13 | assert not HPOBenchConfig._check_version(None, '0.0.6') 14 | assert not HPOBenchConfig._check_version('0.0.5', '0.0.6') 15 | 16 | 17 | def test_is_container(): 18 | import os 19 | os.environ['SINGULARITY_NAME'] = 'test_name' 20 | from hpobench.config import HPOBenchConfig 21 | 22 | try: 23 | config = HPOBenchConfig() 24 | except PermissionError as err: 25 | # We now if the link is set to /var/lib/hpobench that it is looking for the socket dir for the container 26 | # thus, it has set the _is_container - flag. 27 | assert str(err) == '[Errno 13] Permission denied: \'/var/lib/hpobench\'' 28 | 29 | del os.environ['SINGULARITY_NAME'] 30 | config = HPOBenchConfig() 31 | assert not config._is_container 32 | -------------------------------------------------------------------------------- /tests/test_nasbench_201.py: -------------------------------------------------------------------------------- 1 | import logging 2 | logging.basicConfig(level=logging.DEBUG) 3 | 4 | import pytest 5 | 6 | from hpobench.benchmarks.nas.nasbench_201 import ImageNetNasBench201Benchmark, Cifar100NasBench201Benchmark, \ 7 | Cifar10ValidNasBench201Benchmark 8 | 9 | from hpobench.util.container_utils import disable_container_debug, enable_container_debug 10 | 11 | skip_message = 'We currently skip this test because it takes too much time.' 12 | 13 | 14 | @pytest.fixture(scope='module') 15 | def enable_debug(): 16 | enable_container_debug() 17 | yield 18 | disable_container_debug() 19 | 20 | 21 | @pytest.mark.skip(reason=skip_message) 22 | def test_nasbench201_cifar10valid(enable_debug): 23 | 24 | b = Cifar10ValidNasBench201Benchmark(rng=0) 25 | 26 | cs = b.get_configuration_space(seed=0) 27 | config = cs.sample_configuration() 28 | fidelity = {'epoch': 199} 29 | 30 | result = b.objective_function(configuration=config, fidelity=fidelity, data_seed=(777, 888, 999)) 31 | 32 | assert result['function_value'] == pytest.approx(0.411, abs=0.1) 33 | assert result['cost'] == pytest.approx(6650.88, abs=0.1) 34 | assert result['info']['train_precision'] == result['function_value'] 35 | assert result['info']['train_cost'] == result['cost'] 36 | 37 | result = b.objective_function_test(configuration=config, fidelity=fidelity, data_seed=(777, 888, 999)) 38 | 39 | with pytest.raises(AssertionError): 40 | result = b.objective_function_test(configuration=config, fidelity={'epoch': 10}) 41 | 42 | @pytest.mark.skip(reason=skip_message) 43 | def test_nasbench201_cifar100(enable_debug): 44 | b = Cifar100NasBench201Benchmark(rng=0) 45 | 46 | cs = b.get_configuration_space(seed=0) 47 | config = cs.sample_configuration() 48 | fidelity = {'epoch': 199} 49 | 50 | result = b.objective_function(configuration=config, fidelity=fidelity, data_seed=(777, 888, 999)) 51 | 52 | assert result is not None 53 | assert result['function_value'] == pytest.approx(7.8259, abs=0.1) 54 | assert result['cost'] == pytest.approx(13301.76, abs=0.1) 55 | assert result['info']['train_precision'] == result['function_value'] 56 | assert result['info']['train_cost'] == result['cost'] 57 | 58 | 59 | @pytest.mark.skip(reason=skip_message) 60 | def test_nasbench201_Image(enable_debug): 61 | b = ImageNetNasBench201Benchmark(rng=0) 62 | 63 | cs = b.get_configuration_space(seed=0) 64 | config = cs.sample_configuration() 65 | fidelity = {'epoch': 199} 66 | 67 | result = b.objective_function(configuration=config, fidelity=fidelity, data_seed=(777, 888, 999)) 68 | 69 | assert result is not None 70 | assert result['function_value'] == pytest.approx(62.858, abs=0.1) 71 | assert result['cost'] == pytest.approx(40357.56, abs=0.1) 72 | assert result['info']['train_precision'] == result['function_value'] 73 | assert result['info']['train_cost'] == result['cost'] 74 | 75 | 76 | def test_nasbench201_fidelity_space(): 77 | fs = Cifar10ValidNasBench201Benchmark.get_fidelity_space() 78 | assert len(fs.get_hyperparameters()) == 1 79 | 80 | 81 | def test_nasbench201_config(): 82 | cs = Cifar10ValidNasBench201Benchmark.get_configuration_space(seed=0) 83 | c = cs.sample_configuration() 84 | func = Cifar10ValidNasBench201Benchmark.config_to_structure_func(4) 85 | struct = func(c) 86 | 87 | assert struct.__repr__() == '_Structure(4 nodes with |avg_pool_3x3~0|+|none~0|nor_conv_3x3~1|+' \ 88 | '|nor_conv_3x3~0|nor_conv_3x3~1|skip_connect~2|)' 89 | assert len(struct) == 4 90 | assert struct[0] == (('avg_pool_3x3', 0),) 91 | 92 | struct_str = struct.tostr() 93 | assert struct_str == '|avg_pool_3x3~0|+|none~0|nor_conv_3x3~1|+|nor_conv_3x3~0|nor_conv_3x3~1|skip_connect~2|' 94 | -------------------------------------------------------------------------------- /tests/test_od.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | def test_ocsvm(): 5 | from hpobench.container.benchmarks.od.od_benchmarks import ODOneClassSupportVectorMachine 6 | seed = 6 7 | benchmark = ODOneClassSupportVectorMachine("cardio", rng=seed) 8 | 9 | config = benchmark.get_configuration_space(seed=seed).sample_configuration() 10 | result = benchmark.objective_function_test(configuration=config, rng=seed) 11 | print(config['gamma'], result['function_value']) 12 | assert config['gamma'] == pytest.approx(0.065674, abs=0.0001) 13 | assert result['function_value'] == pytest.approx(0.08180, abs=0.001) 14 | 15 | 16 | def test_kde(): 17 | from hpobench.container.benchmarks.od.od_benchmarks import ODKernelDensityEstimation 18 | seed = 6 19 | benchmark = ODKernelDensityEstimation("cardio", rng=seed) 20 | 21 | config = benchmark.get_configuration_space(seed=seed).sample_configuration() 22 | result = benchmark.objective_function_test(configuration=config, rng=seed) 23 | print(config['kernel'], config['bandwidth'], result['function_value']) 24 | 25 | assert config['kernel'] == "exponential" 26 | assert config['bandwidth'] == pytest.approx(15.2274, abs=0.001) 27 | assert result['function_value'] == pytest.approx(0.14409, abs=0.0001) 28 | 29 | 30 | def test_ae(): 31 | from hpobench.container.benchmarks.od.od_benchmarks import ODAutoencoder 32 | seed = 6 33 | benchmark = ODAutoencoder("cardio", rng=seed) 34 | 35 | config = benchmark.get_configuration_space(seed=seed).sample_configuration() 36 | result = benchmark.objective_function(configuration=config, rng=seed) 37 | print(config['dropout_rate'], result['function_value']) 38 | 39 | assert config['dropout_rate'] == pytest.approx(0.69512, abs=0.00001) 40 | assert result['function_value'] == pytest.approx(0.2833, abs=0.0001) 41 | -------------------------------------------------------------------------------- /tests/test_openml_datamanager.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from hpobench.util.openml_data_manager import OpenMLHoldoutDataManager 4 | 5 | 6 | def test_convert_nan_values_in_cat_columns(): 7 | x = np.array([[1, np.nan, 3, 4], 8 | [5, 6, 7, 8], 9 | [np.nan, 10, 11, np.nan]]) 10 | 11 | is_cat = [True, True, False, False] 12 | 13 | x, _, _, categories = OpenMLHoldoutDataManager.replace_nans_in_cat_columns(x, x, x, is_cat) 14 | 15 | solution = np.array([[1., 5., 3., 4.], 16 | [5., 6., 7., 8.], 17 | [0., 10., 11., np.nan]]) 18 | 19 | solution_cat = np.array([[1., 5., 0.], 20 | [5., 6., 10.]]) 21 | 22 | assert np.array_equiv(x[:, :3], solution[:, :3]) # unfortunately np.nan != np.nan :) 23 | assert np.isnan(x[2, 3]) 24 | 25 | cats = np.array(categories).flatten() 26 | cats.sort() 27 | solution_cat = solution_cat.flatten() 28 | solution_cat.sort() 29 | assert np.array_equal(cats, solution_cat) 30 | -------------------------------------------------------------------------------- /tests/test_paramnet.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | # import logging 4 | # logging.basicConfig(level=logging.DEBUG) 5 | # from hpobench.util.container_utils import enable_container_debug 6 | # enable_container_debug() 7 | 8 | 9 | def test_load_data(): 10 | from hpobench.util.data_manager import ParamNetDataManager 11 | 12 | with pytest.raises(AssertionError): 13 | dm = ParamNetDataManager(dataset='unknown_dataset') 14 | 15 | dm = ParamNetDataManager(dataset='higgs') 16 | obj_v_fn, costs_fn = dm.load() 17 | 18 | assert obj_v_fn is not None 19 | assert costs_fn is not None 20 | 21 | 22 | def test_obj_func(): 23 | 24 | from hpobench.container.benchmarks.surrogates.paramnet_benchmark import ParamNetHiggsOnStepsBenchmark 25 | 26 | benchmark = ParamNetHiggsOnStepsBenchmark() 27 | cs = benchmark.get_configuration_space(0) 28 | fs = benchmark.get_fidelity_space(0) 29 | cfg = cs.sample_configuration() 30 | 31 | result = benchmark.objective_function(cfg) 32 | assert result['function_value'] == pytest.approx(0.3244, 0.01) 33 | 34 | full_budget = fs.get_default_configuration() 35 | assert full_budget['step'] == 50 36 | 37 | result_on_full_budget = benchmark.objective_function(configuration=cfg, fidelity=full_budget) 38 | assert result['function_value'] == pytest.approx(result_on_full_budget['function_value'], abs=0.000001) 39 | 40 | result_on_small_budget = benchmark.objective_function(configuration=cfg, fidelity={'step': 1}) 41 | assert result['cost'] == pytest.approx(result_on_small_budget['cost'] * 50, abs=0.0001) 42 | 43 | 44 | def test_param_net_time(): 45 | from hpobench.container.benchmarks.surrogates.paramnet_benchmark import ParamNetHiggsOnTimeBenchmark 46 | 47 | benchmark = ParamNetHiggsOnTimeBenchmark() 48 | cs = benchmark.get_configuration_space(0) 49 | cfg = cs.sample_configuration() 50 | 51 | result = benchmark.objective_function(cfg) 52 | assert result['function_value'] == pytest.approx(0.3244, 0.01) 53 | assert isinstance(result['info']['learning_curve'], list) 54 | 55 | assert result['info']['observed_epochs'] == 193 56 | assert result['info']['learning_curve'][49] == result['info']['learning_curve'][192] 57 | 58 | # Test the case when the budget is less than the costs for 50 epochs 59 | fidelity = {'budget': 50} 60 | result = benchmark.objective_function(configuration=cfg, fidelity=fidelity) 61 | assert result['info']['observed_epochs'] == 38 62 | assert len(result['info']['learning_curve']) == 38 63 | -------------------------------------------------------------------------------- /tests/test_pybnn.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from hpobench.container.benchmarks.ml.pybnn import BNNOnToyFunction, BNNOnBostonHousing, BNNOnProteinStructure, \ 4 | BNNOnYearPrediction 5 | 6 | import logging 7 | logging.basicConfig(level=logging.DEBUG) 8 | from hpobench.util.container_utils import enable_container_debug 9 | enable_container_debug() 10 | 11 | 12 | def test_bnn_init(): 13 | benchmark = BNNOnToyFunction(rng=1) 14 | 15 | fs = benchmark.get_fidelity_space(seed=0) 16 | fidelity = fs.sample_configuration().get_dictionary() 17 | assert fidelity['budget'] == 5714 18 | 19 | meta = benchmark.get_meta_information() 20 | assert meta is not None 21 | 22 | cs = benchmark.get_configuration_space(seed=0) 23 | config = cs.sample_configuration().get_dictionary() 24 | 25 | assert config['l_rate'] == pytest.approx(0.0037, abs=0.001) 26 | assert config['burn_in'] == pytest.approx(0.43905, abs=0.001) 27 | assert config['n_units_1'] == 104 28 | assert config['n_units_2'] == 68 29 | assert config['mdecay'] == pytest.approx(0.6027, abs=0.001) 30 | 31 | result = benchmark.objective_function(configuration=config, fidelity=fidelity, rng=1) 32 | assert result['function_value'] == pytest.approx(380.08, abs=0.1) 33 | assert result['cost'] > 1 34 | assert result['info']['fidelity']['budget'] == 5714 35 | 36 | result = benchmark.objective_function_test(configuration=config) 37 | assert result['function_value'] == pytest.approx(183.6146, abs=0.1) 38 | assert result['cost'] is not None 39 | # test if budget is maximal: 40 | assert result['info']['fidelity']['budget'] == 10000 41 | 42 | 43 | def simple_call(benchmark): 44 | cs = benchmark.get_configuration_space(seed=0) 45 | config = cs.sample_configuration().get_dictionary() 46 | 47 | fidelity = {'budget': 1000} 48 | 49 | result = benchmark.objective_function(configuration=config, fidelity=fidelity, rng=1) 50 | return result 51 | 52 | 53 | def test_bnn_boston_housing(): 54 | benchmark = BNNOnBostonHousing(rng=1) 55 | test_result = simple_call(benchmark) 56 | assert test_result['function_value'] == pytest.approx(1262.0869, abs=0.1) 57 | assert test_result['cost'] > 0 58 | assert test_result['info']['fidelity']['budget'] == 1000 59 | 60 | 61 | def test_bnn_protein(): 62 | benchmark = BNNOnProteinStructure(rng=1) 63 | test_result = simple_call(benchmark) 64 | assert test_result['function_value'] == pytest.approx(1050.5733, abs=0.1) 65 | assert test_result['cost'] > 0 66 | assert test_result['info']['fidelity']['budget'] == 1000 67 | 68 | 69 | def test_year_pred(): 70 | benchmark = BNNOnYearPrediction(rng=1) 71 | test_result = simple_call(benchmark) 72 | assert test_result['function_value'] == pytest.approx(2105.2726, abs=0.1) 73 | assert test_result['cost'] > 0 74 | assert test_result['info']['fidelity']['budget'] == 1000 75 | -------------------------------------------------------------------------------- /tests/test_server.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | import logging 3 | import os 4 | 5 | 6 | def set_log_level(debug): 7 | os.environ['HPOBENCH_DEBUG'] = 'true' if debug else 'false' 8 | import hpobench.container.client_abstract_benchmark as client 9 | importlib.reload(client) 10 | 11 | 12 | def test_debug_env_variable_1(): 13 | set_log_level(False) 14 | from hpobench.container.client_abstract_benchmark import LOG_LEVEL 15 | assert LOG_LEVEL == logging.INFO 16 | 17 | set_log_level(True) 18 | from hpobench.container.client_abstract_benchmark import LOG_LEVEL 19 | assert LOG_LEVEL == logging.DEBUG 20 | 21 | 22 | def test_debug_container(): 23 | # Test if the debug option works. Check if some debug output from the server is visible. 24 | 25 | set_log_level(True) 26 | 27 | from hpobench.container.benchmarks.ml.xgboost_benchmark_old import XGBoostBenchmark as Benchmark 28 | from hpobench.util.openml_data_manager import get_openmlcc18_taskids 29 | 30 | task_id = get_openmlcc18_taskids()[0] 31 | 32 | b = Benchmark(task_id=task_id, 33 | container_name='xgboost_benchmark', 34 | container_source='library://phmueller/automl') 35 | cs = b.get_configuration_space() 36 | assert cs is not None 37 | 38 | set_log_level(False) 39 | -------------------------------------------------------------------------------- /tests/test_svm.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from hpobench.container.benchmarks.ml.svm_benchmark_old import SupportVectorMachine 4 | from hpobench.util.openml_data_manager import get_openmlcc18_taskids 5 | 6 | task_ids = get_openmlcc18_taskids() 7 | 8 | import logging 9 | logging.basicConfig(level=logging.DEBUG) 10 | 11 | 12 | def test_svm_init(): 13 | benchmark = SupportVectorMachine(task_id=task_ids[0]) 14 | 15 | fs = benchmark.get_fidelity_space(seed=0) 16 | fidelity = fs.sample_configuration().get_dictionary() 17 | assert fidelity['dataset_fraction'] == pytest.approx(0.54881, abs=0.001) 18 | 19 | meta = benchmark.get_meta_information() 20 | assert meta is not None 21 | 22 | cs = benchmark.get_configuration_space(seed=0) 23 | config = cs.sample_configuration().get_dictionary() 24 | assert config['C'] == pytest.approx(0.9762, abs=0.001) 25 | assert config['gamma'] == pytest.approx(4.3037, abs=0.001) 26 | 27 | result = benchmark.objective_function(configuration=config, fidelity=fidelity) 28 | assert result['function_value'] == pytest.approx(0.4837, abs=0.1) 29 | assert result['cost'] is not None 30 | 31 | with pytest.raises(AssertionError): 32 | result = benchmark.objective_function_test(configuration=config, fidelity=fidelity) 33 | 34 | result = benchmark.objective_function_test(configuration=config) 35 | assert result['function_value'] == pytest.approx(0.4648, abs=0.1) 36 | assert result['cost'] is not None 37 | -------------------------------------------------------------------------------- /tests/test_tabular_benchmarks.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import unittest 3 | import pytest 4 | 5 | logging.basicConfig(level=logging.DEBUG) 6 | 7 | import os 8 | 9 | os.environ['HPOBENCH_DEBUG'] = 'true' 10 | 11 | from hpobench.container.benchmarks.nas.tabular_benchmarks import SliceLocalizationBenchmark, \ 12 | NavalPropulsionBenchmark, ParkinsonsTelemonitoringBenchmark, ProteinStructureBenchmark 13 | 14 | 15 | class TestTabularBenchmark(unittest.TestCase): 16 | def setUp(self) -> None: 17 | self.benchmark = SliceLocalizationBenchmark( 18 | rng=1, 19 | ) 20 | self.default_config = self.benchmark.get_configuration_space(seed=1).get_default_configuration() 21 | self.socket_id = self.benchmark.socket_id 22 | 23 | def test_tabular_benchmark_wrong_input(self): 24 | default_config = self.default_config 25 | 26 | benchmark = SliceLocalizationBenchmark( 27 | socket_id=self.socket_id 28 | ) 29 | 30 | with pytest.raises(ValueError): 31 | benchmark.objective_function(configuration=default_config, fidelity=dict(budget=0)) 32 | 33 | with pytest.raises(ValueError): 34 | benchmark.objective_function(configuration=default_config, fidelity=dict(budget=1), run_index=0.1) 35 | 36 | with pytest.raises(AssertionError): 37 | benchmark.objective_function(configuration=default_config, fidelity=dict(budget=1), run_index=[4]) 38 | 39 | with pytest.raises(AssertionError): 40 | benchmark.objective_function(configuration=default_config, fidelity=dict(budget=1), run_index=[]) 41 | 42 | with pytest.raises(AssertionError): 43 | benchmark.objective_function(configuration=default_config, fidelity=dict(budget=1), run_index=-1) 44 | 45 | with pytest.raises(AssertionError): 46 | benchmark.objective_function(configuration=default_config, fidelity=dict(budget=1), run_index=4) 47 | 48 | with pytest.raises(ValueError): 49 | benchmark.objective_function(configuration=default_config, fidelity=dict(budget=101), run_index=3) 50 | 51 | with pytest.raises((AssertionError, ValueError)): 52 | benchmark.objective_function_test(configuration=default_config, fidelity=dict(budget=107)) 53 | 54 | def test_slice_benchmark(self): 55 | default_config = self.default_config 56 | 57 | benchmark = SliceLocalizationBenchmark( 58 | rng=1, 59 | ) 60 | result = benchmark.objective_function(configuration=default_config, fidelity=dict(budget=1), 61 | run_index=[0, 1, 2, 3]) 62 | 63 | mean = 0.01828 64 | assert result['function_value'] == pytest.approx(mean, abs=0.0001) 65 | 66 | runs = result['info']['valid_rmse_per_run'] 67 | calculated_mean = sum(runs) / len(runs) 68 | assert calculated_mean == pytest.approx(mean, abs=0.0001) 69 | 70 | runtime = 23.1000 71 | assert result['cost'] == pytest.approx(runtime, abs=0.0001) 72 | 73 | runtimes = sum(result['info']['runtime_per_run']) 74 | assert runtimes == pytest.approx(runtime, abs=0.0001) 75 | 76 | def test_naval_benchmark(self): 77 | default_config = self.default_config 78 | 79 | benchmark = NavalPropulsionBenchmark( 80 | rng=1, 81 | ) 82 | result = benchmark.objective_function(configuration=default_config, fidelity=dict(budget=1), 83 | run_index=[0, 1, 2, 3]) 84 | 85 | mean = 0.8928 86 | assert result['function_value'] == pytest.approx(mean, abs=0.0001) 87 | 88 | runs = result['info']['valid_rmse_per_run'] 89 | calculated_mean = sum(runs) / len(runs) 90 | assert calculated_mean == pytest.approx(mean, abs=0.0001) 91 | 92 | runtime = 5.2477 93 | assert result['cost'] == pytest.approx(runtime, abs=0.0001) 94 | 95 | runtimes = sum(result['info']['runtime_per_run']) 96 | assert runtimes == pytest.approx(runtime, abs=0.0001) 97 | 98 | def test_protein_benchmark(self): 99 | default_config = self.default_config 100 | 101 | benchmark = ProteinStructureBenchmark( 102 | rng=1, 103 | ) 104 | result = benchmark.objective_function(configuration=default_config, fidelity=dict(budget=1), 105 | run_index=[0, 1, 2, 3]) 106 | 107 | mean = 0.4474 108 | assert result['function_value'] == pytest.approx(mean, abs=0.0001) 109 | 110 | runs = result['info']['valid_rmse_per_run'] 111 | calculated_mean = sum(runs) / len(runs) 112 | assert calculated_mean == pytest.approx(mean, abs=0.0001) 113 | 114 | runtime = 19.24213 115 | assert result['cost'] == pytest.approx(runtime, abs=0.0001) 116 | 117 | runtimes = result['info']['runtime_per_run'] 118 | calculated_runtime = sum(runtimes) 119 | assert calculated_runtime == pytest.approx(runtime, abs=0.0001) 120 | 121 | def test_parkinson_benchmark(self): 122 | default_config = self.default_config 123 | 124 | benchmark = ParkinsonsTelemonitoringBenchmark( 125 | rng=1, 126 | ) 127 | result = benchmark.objective_function(configuration=default_config, fidelity=dict(budget=1), 128 | run_index=[0, 1, 2, 3]) 129 | 130 | mean = 0.7425 131 | assert result['function_value'] == pytest.approx(mean, abs=0.0001) 132 | 133 | with pytest.raises(AssertionError): 134 | benchmark.objective_function_test(default_config, fidelity=dict(budget=1, )) 135 | 136 | result = benchmark.objective_function_test(configuration=default_config, fidelity=dict(budget=100)) 137 | assert pytest.approx(0.15010187, result['function_value'], abs=0.001) 138 | 139 | runtime = 62.7268 140 | assert result['cost'] == pytest.approx(runtime, abs=0.0001) 141 | -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pytest 3 | 4 | 5 | def test_example_utils(): 6 | from hpobench.util.example_utils import get_travis_settings 7 | 8 | res = get_travis_settings('smac') 9 | assert res['runcount-limit'] == 5 10 | 11 | res = get_travis_settings('bohb') 12 | assert res['max_budget'] == 2 13 | 14 | with pytest.raises(ValueError): 15 | res = get_travis_settings('unknown') 16 | 17 | 18 | def test_example_utils_2(): 19 | from hpobench.util.example_utils import set_env_variables_to_use_only_one_core 20 | import os 21 | set_env_variables_to_use_only_one_core() 22 | assert os.environ['OMP_NUM_THREADS'] == '1' 23 | assert os.environ['OPENBLAS_NUM_THREADS'] == '1' 24 | assert os.environ['MKL_NUM_THREADS'] == '1' 25 | assert os.environ['VECLIB_MAXIMUM_THREADS'] == '1' 26 | assert os.environ['NUMEXPR_NUM_THREADS'] == '1' 27 | assert os.environ['NUMEXPR_MAX_THREADS'] == '1' 28 | 29 | 30 | def test_rng_helper(): 31 | from hpobench.util.rng_helper import _cast_int_to_random_state 32 | 33 | rng = np.random.RandomState(123) 34 | 35 | with pytest.raises(ValueError): 36 | _cast_int_to_random_state('not_an_int') 37 | 38 | assert rng == _cast_int_to_random_state(rng) 39 | 40 | rng = np.random.RandomState(123) 41 | assert rng.random() == _cast_int_to_random_state(123).random() 42 | 43 | 44 | def test_rng_helper_2(): 45 | from hpobench.util.rng_helper import get_rng 46 | 47 | rng = get_rng(None, None) 48 | assert isinstance(rng, np.random.RandomState) 49 | 50 | old_rng = np.random.RandomState(123) 51 | rng = get_rng(None, old_rng) 52 | assert rng == old_rng 53 | 54 | 55 | def test_rng_serialization(): 56 | from hpobench.util.rng_helper import deserialize_random_state, serialize_random_state 57 | rs_old = np.random.RandomState(1) 58 | r_str = serialize_random_state(rs_old) 59 | rs_new = deserialize_random_state(r_str) 60 | 61 | assert np.array_equiv(rs_old.random_sample(10), rs_new.random_sample(10)) 62 | 63 | 64 | def test_rng_serialization_xgb(): 65 | import json 66 | from hpobench.util.container_utils import BenchmarkEncoder, BenchmarkDecoder 67 | from hpobench.benchmarks.ml.xgboost_benchmark_old import XGBoostBenchmark 68 | 69 | b = XGBoostBenchmark(task_id=167149, rng=0) 70 | meta = b.get_meta_information() 71 | 72 | meta_str = json.dumps(meta, indent=None, cls=BenchmarkEncoder) 73 | meta_new = json.loads(meta_str, cls=BenchmarkDecoder) 74 | assert isinstance(meta_new['initial random seed'], np.random.RandomState) 75 | assert np.array_equiv(meta['initial random seed'].random(10), meta_new['initial random seed'].random(10)) 76 | 77 | 78 | def test_benchmark_encoder(): 79 | from enum import Enum 80 | class test_enum(Enum): 81 | obj = 'name' 82 | 83 | def __str__(self): 84 | return str(self.value) 85 | 86 | from hpobench.util.container_utils import BenchmarkEncoder, BenchmarkDecoder 87 | import json 88 | import numpy as np 89 | 90 | enum_obj = test_enum.obj 91 | enum_obj_str = json.dumps(enum_obj, cls=BenchmarkEncoder) 92 | assert enum_obj_str == '"name"' 93 | 94 | array = np.array([1, 2, 3, 4]) 95 | array_str = json.dumps(array, cls=BenchmarkEncoder) 96 | array_ = json.loads(array_str, cls=BenchmarkDecoder) 97 | assert np.array_equiv(array, array_) 98 | 99 | 100 | def test_debug_level(): 101 | from hpobench.util.container_utils import enable_container_debug, disable_container_debug 102 | import os 103 | enable_container_debug() 104 | assert os.environ['HPOBENCH_DEBUG'] == 'true' 105 | 106 | disable_container_debug() 107 | assert os.environ['HPOBENCH_DEBUG'] == 'false' 108 | -------------------------------------------------------------------------------- /tests/test_whitebox.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import numpy as np 4 | import pytest 5 | 6 | logging.basicConfig(level=logging.DEBUG) 7 | 8 | try: 9 | import Pyro4 10 | 11 | skip_container_test = False 12 | except ImportError: 13 | skip_container_test = True 14 | 15 | 16 | def test_whitebox_without_container_xgb(): 17 | from hpobench.benchmarks.ml.xgboost_benchmark_old import XGBoostBenchmark as Benchmark 18 | b = Benchmark(task_id=167199, rng=0) 19 | cs = b.get_configuration_space(seed=0) 20 | 21 | configuration = cs.get_default_configuration() 22 | assert configuration['colsample_bylevel'] == 1.0 23 | assert len(configuration.keys()) == 8 24 | 25 | n_estimator = 32 26 | subsample = 1 27 | result_dict = b.objective_function(configuration, fidelity=dict(n_estimators=n_estimator, dataset_fraction=subsample), 28 | rng=0) 29 | valid_loss = result_dict['function_value'] 30 | train_loss = result_dict['info']['train_loss'] 31 | 32 | result_dict = b.objective_function_test(configuration, fidelity=dict(n_estimators=n_estimator), rng=0) 33 | test_loss = result_dict['function_value'] 34 | 35 | assert np.isclose(train_loss, 0.02678, atol=0.001) 36 | assert np.isclose(valid_loss, 0.49549, atol=0.001) 37 | assert np.isclose(test_loss, 0.43636, atol=0.001) 38 | 39 | 40 | @pytest.mark.skipif(skip_container_test, reason="Requires singularity and flask") 41 | def test_whitebox_with_container(): 42 | from hpobench.container.benchmarks.ml.xgboost_benchmark_old import XGBoostBenchmark as Benchmark 43 | b = Benchmark(container_name='xgboost_benchmark', 44 | task_id=167199, 45 | rng=0) 46 | 47 | cs = b.get_configuration_space() 48 | configuration = cs.get_default_configuration() 49 | assert configuration['colsample_bylevel'] == 1.0 50 | assert len(configuration.keys()) == 8 51 | 52 | n_estimator = 32 53 | subsample = 1 54 | result_dict = b.objective_function(configuration, fidelity=dict(n_estimators=n_estimator, 55 | dataset_fraction=subsample)) 56 | valid_loss = result_dict['function_value'] 57 | train_loss = result_dict['info']['train_loss'] 58 | result_dict = b.objective_function_test(configuration, fidelity=dict(n_estimators=n_estimator)) 59 | test_loss = result_dict['function_value'] 60 | 61 | assert np.isclose(train_loss, 0.02232, atol=0.001) 62 | assert np.isclose(valid_loss, 0.4234, atol=0.001) 63 | assert np.isclose(test_loss, 0.43636, atol=0.001) 64 | 65 | 66 | def test_cartpole(): 67 | from hpobench.container.benchmarks.rl.cartpole import CartpoleReduced as Benchmark 68 | b = Benchmark(container_name='cartpole', 69 | rng=1) 70 | cs = b.get_configuration_space(seed=1) 71 | print(cs.get_default_configuration()) 72 | 73 | from hpobench.container.benchmarks.rl.cartpole import CartpoleFull as Benchmark 74 | b = Benchmark(container_name='cartpole', 75 | rng=1) 76 | cs = b.get_configuration_space(seed=1) 77 | print(cs.get_default_configuration()) 78 | --------------------------------------------------------------------------------