├── .editorconfig
├── .github
├── dependabot.yml
└── workflows
│ ├── pr-conventional-commit.yml
│ ├── pre-commit.yaml
│ └── tests.yaml
├── .gitignore
├── .pre-commit-config.yaml
├── CITATION.cff
├── CONTRIBUTING.md
├── LICENSE
├── README.md
├── docs
├── _code
│ ├── api_generator.py
│ └── example_generator.py
├── api
│ └── index.md
├── citations.md
├── dev_docs
│ └── contributing.md
├── doc_images
│ ├── examples
│ │ └── val_loss_image_segmentation.jpg
│ ├── optimizers
│ │ ├── bo_acqu.jpg
│ │ ├── bo_surrogate.jpg
│ │ ├── freeze_thaw_fantasizing.jpg
│ │ ├── freeze_thawing.jpg
│ │ ├── pibo_acqus.jpg
│ │ ├── priorband_sampler.jpg
│ │ └── sh_wikipedia.jpg
│ └── tensorboard
│ │ ├── tblogger_hparam1.jpg
│ │ ├── tblogger_hparam2.jpg
│ │ ├── tblogger_hparam3.jpg
│ │ ├── tblogger_image.jpg
│ │ └── tblogger_scalar.jpg
├── getting_started.md
├── index.md
├── reference
│ ├── analyse.md
│ ├── evaluate_pipeline.md
│ ├── neps_run.md
│ ├── optimizers.md
│ ├── pipeline_space.md
│ ├── search_algorithms
│ │ ├── bayesian_optimization.md
│ │ ├── landing_page_algo.md
│ │ ├── multifidelity.md
│ │ ├── multifidelity_prior.md
│ │ └── prior.md
│ └── seeding.md
└── stylesheets
│ └── custom.css
├── mkdocs.yml
├── neps
├── NOTICE
├── __init__.py
├── api.py
├── env.py
├── exceptions.py
├── optimizers
│ ├── __init__.py
│ ├── acquisition
│ │ ├── __init__.py
│ │ ├── cost_cooling.py
│ │ ├── pibo.py
│ │ └── weighted_acquisition.py
│ ├── algorithms.py
│ ├── ask_and_tell.py
│ ├── bayesian_optimization.py
│ ├── bracket_optimizer.py
│ ├── grid_search.py
│ ├── ifbo.py
│ ├── models
│ │ ├── __init__.py
│ │ ├── ftpfn.py
│ │ └── gp.py
│ ├── optimizer.py
│ ├── priorband.py
│ ├── random_search.py
│ └── utils
│ │ ├── __init__.py
│ │ ├── brackets.py
│ │ ├── grid.py
│ │ ├── initial_design.py
│ │ └── multiobjective
│ │ ├── __init__.py
│ │ └── epsnet.py
├── plot
│ ├── __init__.py
│ ├── __main__.py
│ ├── plot.py
│ ├── plot3D.py
│ ├── plotting.py
│ ├── read_results.py
│ └── tensorboard_eval.py
├── py.typed
├── runtime.py
├── sampling
│ ├── __init__.py
│ ├── distributions.py
│ ├── priors.py
│ └── samplers.py
├── space
│ ├── __init__.py
│ ├── domain.py
│ ├── encoding.py
│ ├── parameters.py
│ ├── parsing.py
│ └── search_space.py
├── state
│ ├── __init__.py
│ ├── err_dump.py
│ ├── filebased.py
│ ├── neps_state.py
│ ├── optimizer.py
│ ├── pipeline_eval.py
│ ├── seed_snapshot.py
│ ├── settings.py
│ └── trial.py
├── status
│ ├── __init__.py
│ ├── __main__.py
│ └── status.py
└── utils
│ ├── __init__.py
│ ├── common.py
│ └── files.py
├── neps_examples
├── README.md
├── __init__.py
├── __main__.py
├── basic_usage
│ ├── analyse.py
│ └── hyperparameters.py
├── convenience
│ ├── logging_additional_info.py
│ ├── neps_tblogger_tutorial.py
│ ├── running_on_slurm_scripts.py
│ └── working_directory_per_pipeline.py
├── efficiency
│ ├── README.md
│ ├── expert_priors_for_hyperparameters.py
│ ├── multi_fidelity.py
│ ├── multi_fidelity_and_expert_priors.py
│ ├── pytorch_lightning_ddp.py
│ ├── pytorch_lightning_fsdp.py
│ ├── pytorch_native_ddp.py
│ └── pytorch_native_fsdp.py
├── experimental
│ └── freeze_thaw.py
└── real_world
│ ├── README.md
│ └── image_segmentation_hpo.py
├── pyproject.toml
└── tests
├── __init__.py
├── test_config_encoder.py
├── test_domain.py
├── test_examples.py
├── test_runtime
├── __init__.py
├── test_default_report_values.py
├── test_error_handling_strategies.py
└── test_stopping_criterion.py
├── test_samplers.py
├── test_search_space.py
├── test_search_space_parsing.py
├── test_state
├── __init__.py
├── test_filebased_neps_state.py
├── test_neps_state.py
├── test_rng.py
└── test_trial.py
└── test_user_parse_result.py
/.editorconfig:
--------------------------------------------------------------------------------
1 | # https://editorconfig.org
2 |
3 | root = true
4 |
5 | [*]
6 | charset = utf-8
7 | end_of_line = lf
8 | insert_final_newline = true
9 | trim_trailing_whitespace = true
10 |
11 | [*.py]
12 | max_line_length = 90
13 | indent_style = space
14 | indent_size = 4
15 |
--------------------------------------------------------------------------------
/.github/dependabot.yml:
--------------------------------------------------------------------------------
1 | version: 2
2 |
3 | updates:
4 | # This will check for updates to github actions every week
5 | # https://docs.github.com/en/enterprise-server@3.4/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot
6 | - package-ecosystem: "github-actions"
7 | directory: "/"
8 | schedule:
9 | interval: "weekly"
10 |
--------------------------------------------------------------------------------
/.github/workflows/pr-conventional-commit.yml:
--------------------------------------------------------------------------------
1 | name: PR Conventional Commit Validation
2 |
3 | on:
4 | pull_request:
5 | types: [opened, synchronize, reopened, edited]
6 |
7 | jobs:
8 | validate-pr-title:
9 | runs-on: ubuntu-latest
10 | steps:
11 | - name: PR Conventional Commit Validation
12 | uses: ytanikin/PRConventionalCommits@1.3.0
13 | with:
14 | task_types: '["feat","fix","docs","test","ci","refactor","perf","chore","revert"]'
15 | add_label: 'false'
--------------------------------------------------------------------------------
/.github/workflows/pre-commit.yaml:
--------------------------------------------------------------------------------
1 | name: pre-commit
2 |
3 | # This will prevent multiple runs of the same workflow from running concurrently
4 | concurrency:
5 | group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
6 | cancel-in-progress: true
7 |
8 | on:
9 | # Trigger manually
10 | workflow_dispatch:
11 |
12 | # Trigger on any push to the master
13 | push:
14 | branches:
15 | - master
16 |
17 | # Trigger on any push to a PR that targets master
18 | pull_request:
19 | branches:
20 | - master
21 |
22 | jobs:
23 |
24 | run-all-files:
25 | runs-on: ubuntu-latest
26 | steps:
27 | - uses: actions/checkout@v4
28 | - name: install the latest version uv
29 | uses: astral-sh/setup-uv@v5
30 | with:
31 | python-version: '3.10'
32 | version: latest
33 | enable-cache: true
34 | prune-cache: false
35 | cache-dependency-glob: "**/pyproject.toml"
36 | - name: install pre-commit hooks
37 | run: uv run --all-extras pre-commit install
38 | - name: Run pre-commit hooks
39 | run: uv run --all-extras pre-commit run --all-files
--------------------------------------------------------------------------------
/.github/workflows/tests.yaml:
--------------------------------------------------------------------------------
1 | name: tests
2 | concurrency:
3 | group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
4 | cancel-in-progress: true
5 | on:
6 | workflow_dispatch:
7 | push:
8 | branches:
9 | - master
10 | pull_request:
11 | branches:
12 | - master
13 |
14 | jobs:
15 | tests:
16 | strategy:
17 | fail-fast: false
18 | matrix:
19 | python-version: ['3.10', '3.11', '3.12', '3.13']
20 | os: [ubuntu-latest, macos-latest, windows-latest]
21 | defaults:
22 | run:
23 | shell: bash
24 |
25 | runs-on: ${{ matrix.os }}
26 | steps:
27 | - uses: actions/checkout@v4
28 | - name: install the latest version uv
29 | uses: astral-sh/setup-uv@v5
30 | with:
31 | python-version: ${{ matrix.python-version }}
32 | version: latest
33 | enable-cache: true
34 | prune-cache: false
35 | cache-dependency-glob: "**/pyproject.toml"
36 | - name: run tests
37 | run: uv run --all-extras pytest -m "" # Run all markers
38 |
39 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # False Python
2 | __pycache__
3 | dist
4 | **/*.egg-info
5 | uv.lock
6 |
7 | # Log files
8 | *.out
9 | *.err
10 | *.log
11 | *.json
12 | *.speedscope
13 |
14 | # slurm scripts
15 | slurm_scripts/*
16 |
17 | # Documentation
18 | site/*
19 |
20 | # IDE related
21 | .vscode/
22 | .idea/
23 |
24 | # Misc
25 | *.sh
26 | *.model
27 | *.pth
28 | *.png
29 |
30 | # Results
31 | results
32 | neps_examples/results
33 | tests_tmpdir
34 | usage_example
35 | lightning_logs
36 |
37 | # Regression tests
38 | !losses.json
39 | jahs_bench_data/
40 | *_runs/
41 |
42 | # Jupyter
43 | .ipynb_checkpoints/
44 |
45 | # MacOS
46 | *.DS_Store
47 |
48 | # Yaml tests
49 | path
50 |
51 | # From example that uses MNIST
52 | .data
53 |
--------------------------------------------------------------------------------
/.pre-commit-config.yaml:
--------------------------------------------------------------------------------
1 | default_language_version:
2 | python: python3
3 | files: |
4 | (?x)^(
5 | neps|
6 | tests
7 | )/.*\.py$
8 | repos:
9 | - repo: https://github.com/pre-commit/pre-commit-hooks
10 | rev: v5.0.0
11 | hooks:
12 | - id: check-added-large-files
13 | files: ".*"
14 | - id: check-case-conflict
15 | files: ".*"
16 | - id: check-merge-conflict
17 | files: ".*"
18 | - id: check-yaml
19 | files: ".*"
20 | - id: end-of-file-fixer
21 | files: ".*"
22 | types: ["yaml"]
23 | - id: check-toml
24 | files: ".*"
25 | types: ["toml"]
26 | - id: debug-statements
27 | files: '^src/.*\.py$'
28 |
29 | - repo: https://github.com/pre-commit/mirrors-mypy
30 | rev: v1.14.1
31 | hooks:
32 | - id: mypy
33 | files: |
34 | (?x)^(
35 | neps
36 | )/.*\.py$
37 | additional_dependencies:
38 | - "types-pyyaml"
39 | - "types-requests"
40 | args:
41 | - "--no-warn-return-any" # Disable this because it doesn't know about 3rd party imports
42 | - "--ignore-missing-imports"
43 | - "--show-traceback"
44 |
45 | - repo: https://github.com/python-jsonschema/check-jsonschema
46 | rev: 0.31.0
47 | hooks:
48 | - id: check-github-workflows
49 | files: '^github/workflows/.*\.ya?ml$'
50 | types: ["yaml"]
51 | - id: check-dependabot
52 | files: '^\.github/dependabot\.ya?ml$'
53 |
54 | - repo: https://github.com/charliermarsh/ruff-pre-commit
55 | rev: v0.9.1
56 | hooks:
57 | - id: ruff
58 | args: [--fix, --exit-non-zero-on-fix, --no-cache]
59 | - id: ruff-format
60 |
--------------------------------------------------------------------------------
/CITATION.cff:
--------------------------------------------------------------------------------
1 | cff-version: 1.2.0
2 | message: "If you use this software, please cite it as below."
3 | authors:
4 | - family-names: Stoll
5 | given-names: Danny
6 | - family-names: Mallik
7 | given-names: Neeratyoy
8 | - family-names: Bergman
9 | given-names: Eddie
10 | - family-names: Schrodi
11 | given-names: Simon
12 | - family-names: Garibov
13 | given-names: Samir
14 | - family-names: Abou Chakra
15 | given-names: Tarek
16 | - family-names: Carstensen
17 | given-names: Timur
18 | - family-names: Janowski
19 | given-names: Maciej
20 | - family-names: Gaur
21 | given-names: Gopalji
22 | - family-names: Geburek
23 | given-names: Anton Merlin
24 | - family-names: Rogalla
25 | given-names: Daniel
26 | - family-names: Hvarfner
27 | given-names: Carl
28 | - family-names: Binxin
29 | given-names: Ru
30 | - family-names: Hutter
31 | given-names: Frank
32 | title: "Neural Pipeline Search (NePS)"
33 | version: 0.13.0
34 | date-released: 2025-04-11
35 | url: "https://github.com/automl/neps"
36 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Neural Pipeline Search (NePS)
2 |
3 | [](https://pypi.org/project/neural-pipeline-search/)
4 | [](https://pypi.org/project/neural-pipeline-search/)
5 | [](LICENSE)
6 | [](https://github.com/automl/neps/actions)
7 |
8 | Welcome to NePS, a powerful and flexible Python library for hyperparameter optimization (HPO) and neural architecture search (NAS) that **makes HPO and NAS practical for deep learners**.
9 |
10 | NePS houses recently published and also well-established algorithms that can all be run massively parallel on distributed setups and, in general, NePS is tailored to the needs of deep learning experts.
11 |
12 | To learn about NePS, check-out [the documentation](https://automl.github.io/neps/latest/), [our examples](neps_examples/), or a [colab tutorial](https://colab.research.google.com/drive/11IOhkmMKsIUhWbHyMYzT0v786O9TPWlH?usp=sharing).
13 |
14 | ## Key Features
15 |
16 | In addition to the features offered by traditional HPO and NAS libraries, NePS stands out with:
17 |
18 | 1. **Hyperparameter Optimization (HPO) Efficient Enough for Deep Learning:**
19 | NePS excels in efficiently tuning hyperparameters using algorithms that enable users to make use of their prior knowledge, while also using many other efficiency boosters.
20 | - [PriorBand: Practical Hyperparameter Optimization in the Age of Deep Learning (NeurIPS 2023)](https://arxiv.org/abs/2306.12370)
21 | - [πBO: Augmenting Acquisition Functions with User Beliefs for Bayesian Optimization (ICLR 2022)](https://arxiv.org/abs/2204.11051)
22 | 1. **Neural Architecture Search (NAS) with Expressive Search Spaces:**
23 | NePS provides capabilities for optimizing DL architectures in an expressive and natural fashion.
24 | - [Construction of Hierarchical Neural Architecture Search Spaces based on Context-free Grammars (NeurIPS 2023)](https://arxiv.org/abs/2211.01842)
25 | 1. **Zero-effort Parallelization and an Experience Tailored to DL:**
26 | NePS simplifies the process of parallelizing optimization tasks both on individual computers and in distributed
27 | computing environments. As NePS is made for deep learners, all technical choices are made with DL in mind and common
28 | DL tools such as Tensorboard are [embraced](https://automl.github.io/neps/latest/reference/analyse/#visualizing-results).
29 |
30 | ## Installation
31 |
32 | To install the latest release from PyPI run
33 |
34 | ```bash
35 | pip install neural-pipeline-search
36 | ```
37 |
38 | ## Basic Usage
39 |
40 | Using `neps` always follows the same pattern:
41 |
42 | 1. Define a `evaluate_pipeline` function capable of evaluating different architectural and/or hyperparameter configurations
43 | for your problem.
44 | 1. Define a search space named `pipeline_space` of those Parameters e.g. via a dictionary
45 | 1. Call `neps.run(evaluate_pipeline, pipeline_space)`
46 |
47 | In code, the usage pattern can look like this:
48 |
49 | ```python
50 | import neps
51 | import logging
52 |
53 | logging.basicConfig(level=logging.INFO)
54 |
55 | # 1. Define a function that accepts hyperparameters and computes the validation error
56 | def evaluate_pipeline(lr: float, alpha: int, optimizer: str) -> float:
57 | # Create your model
58 | model = MyModel(lr=lr, alpha=alpha, optimizer=optimizer)
59 |
60 | # Train and evaluate the model with your training pipeline
61 | validation_error = train_and_eval(model)
62 | return validation_error
63 |
64 |
65 | # 2. Define a search space of parameters; use the same parameter names as in evaluate_pipeline
66 | pipeline_space = dict(
67 | lr=neps.Float(
68 | lower=1e-5,
69 | upper=1e-1,
70 | log=True, # Log spaces
71 | prior=1e-3, # Incorporate you knowledge to help optimization
72 | ),
73 | alpha=neps.Integer(lower=1, upper=42),
74 | optimizer=neps.Categorical(choices=["sgd", "adam"])
75 | )
76 |
77 | # 3. Run the NePS optimization
78 | neps.run(
79 | evaluate_pipeline=evaluate_pipeline,
80 | pipeline_space=pipeline_space,
81 | root_directory="path/to/save/results", # Replace with the actual path.
82 | max_evaluations_total=100,
83 | )
84 | ```
85 |
86 | ## Examples
87 |
88 | Discover how NePS works through these examples:
89 |
90 | - **[Hyperparameter Optimization](neps_examples/basic_usage/hyperparameters.py)**: Learn the essentials of hyperparameter optimization with NePS.
91 |
92 | - **[Multi-Fidelity Optimization](neps_examples/efficiency/multi_fidelity.py)**: Understand how to leverage multi-fidelity optimization for efficient model tuning.
93 |
94 | - **[Utilizing Expert Priors for Hyperparameters](neps_examples/efficiency/expert_priors_for_hyperparameters.py)**: Learn how to incorporate expert priors for more efficient hyperparameter selection.
95 |
96 | - **[Additional NePS Examples](neps_examples/)**: Explore more examples, including various use cases and advanced configurations in NePS.
97 |
98 | ## Contributing
99 |
100 | Please see the [documentation for contributors](https://automl.github.io/neps/latest/dev_docs/contributing/).
101 |
102 | ## Citations
103 |
104 | For pointers on citing the NePS package and papers refer to our [documentation on citations](https://automl.github.io/neps/latest/citations/).
105 |
--------------------------------------------------------------------------------
/docs/_code/api_generator.py:
--------------------------------------------------------------------------------
1 | """Generate the code reference pages and navigation.
2 |
3 | # https://mkdocstrings.github.io/recipes/
4 | """
5 |
6 | import logging
7 | from pathlib import Path
8 |
9 | import mkdocs_gen_files
10 |
11 | logger = logging.getLogger(__name__)
12 |
13 | # Modules whose members should not include inherited attributes or methods
14 | NO_INHERITS = tuple()
15 |
16 | SRCDIR = Path("neps").absolute().resolve()
17 | ROOT = SRCDIR.parent
18 | TAB = " "
19 |
20 |
21 | if not SRCDIR.exists():
22 | raise FileNotFoundError(
23 | f"{SRCDIR} does not exist, make sure you are running this from the root of the repository."
24 | )
25 |
26 | for path in sorted(SRCDIR.rglob("*.py")):
27 | module_path = path.relative_to(ROOT).with_suffix("")
28 | doc_path = path.relative_to(ROOT).with_suffix(".md")
29 | full_doc_path = Path("api", doc_path)
30 |
31 | parts = tuple(module_path.parts)
32 |
33 | if parts[-1] in ("__main__", "__version__", "__init__"):
34 | continue
35 |
36 | if any(part.startswith("_") for part in parts):
37 | continue
38 |
39 | with mkdocs_gen_files.open(full_doc_path, "w") as fd:
40 | ident = ".".join(parts)
41 | fd.write(f"::: {ident}")
42 |
43 | if ident.endswith(NO_INHERITS):
44 | fd.write(f"\n{TAB}options:")
45 | fd.write(f"\n{TAB}{TAB}inherited_members: false")
46 |
47 | mkdocs_gen_files.set_edit_path(full_doc_path, path)
48 |
--------------------------------------------------------------------------------
/docs/_code/example_generator.py:
--------------------------------------------------------------------------------
1 | """Generate the code reference pages and navigation.
2 |
3 | # https://mkdocstrings.github.io/recipes/
4 | """
5 |
6 | import logging
7 | from pathlib import Path
8 |
9 | import mkdocs_gen_files
10 |
11 | logger = logging.getLogger(__name__)
12 |
13 | SRCDIR = Path("neps").absolute().resolve()
14 | ROOT = SRCDIR.parent
15 | EXAMPLE_FOLDER = ROOT / "neps_examples"
16 | TAB = " "
17 |
18 | if not SRCDIR.exists():
19 | raise FileNotFoundError(
20 | f"{SRCDIR} does not exist, make sure you are running this from the root of the repository."
21 | )
22 |
23 | # Write the index page of the examples
24 | EXAMPLE_INDEX = EXAMPLE_FOLDER / "README.md"
25 | if not EXAMPLE_INDEX.exists():
26 | raise FileNotFoundError(
27 | f"{EXAMPLE_INDEX} does not exist, make sure you are running this from the root of the repository."
28 | )
29 |
30 | with EXAMPLE_INDEX.open() as fd:
31 | example_index_contents = fd.read()
32 |
33 | DOCS_EXAMPLE_INDEX = Path("examples", "index.md")
34 |
35 | with mkdocs_gen_files.open(DOCS_EXAMPLE_INDEX, "w") as fd:
36 | fd.write(example_index_contents)
37 |
38 | mkdocs_gen_files.set_edit_path(DOCS_EXAMPLE_INDEX, EXAMPLE_INDEX)
39 |
40 | # Now Iterate through each example folder
41 | for example_dir in EXAMPLE_FOLDER.iterdir():
42 | if not example_dir.is_dir():
43 | continue
44 |
45 | readme = next((p for p in example_dir.iterdir() if p.name == "README.md"), None)
46 | doc_example_dir = Path("examples", example_dir.name)
47 |
48 | # Copy the README.md file to the docs//index.md
49 | if readme is not None:
50 | doc_example_index = doc_example_dir / "index.md"
51 | with readme.open() as fd:
52 | contents = fd.read()
53 |
54 | with mkdocs_gen_files.open(doc_example_index, "w") as fd:
55 | fd.write(contents)
56 |
57 | mkdocs_gen_files.set_edit_path(doc_example_index, readme)
58 |
59 | # Copy the contents of all of the examples to the docs//examples/.md
60 | for path in example_dir.iterdir():
61 | if path.suffix != ".py":
62 | continue
63 |
64 | with path.open() as fd:
65 | contents = fd.readlines()
66 |
67 | # NOTE: We use quad backticks to escape the code blocks that are present in some of the examples
68 | escaped_contents = "".join(["````python\n", *contents, "\n````"])
69 |
70 | markdown_example_path = doc_example_dir / f"{path.stem}.md"
71 | with mkdocs_gen_files.open(markdown_example_path, "w") as fd:
72 | fd.write(escaped_contents)
73 |
74 | mkdocs_gen_files.set_edit_path(markdown_example_path, path)
75 |
--------------------------------------------------------------------------------
/docs/api/index.md:
--------------------------------------------------------------------------------
1 | # API
2 | Use the tree to navigate the API documentation.
3 |
--------------------------------------------------------------------------------
/docs/citations.md:
--------------------------------------------------------------------------------
1 | # Citations
2 |
3 | ## Citation of The Software
4 |
5 | For citing NePS, please refer to the following:
6 |
7 | ### APA Style
8 |
9 | ```apa
10 | Stoll, D., Mallik, N., Schrodi, S., Bergman, E., Janowski, M., Garibov, S., Abou Chakra, T., Rogalla, D., Bergman, E., Hvarfner, C., Binxin, R., & Hutter, F. (2023). Neural Pipeline Search (NePS) (Version 0.12.2) [Computer software]. https://github.com/automl/neps
11 | ```
12 |
13 | ### BibTex Style
14 |
15 | ```bibtex
16 | @software{Stoll_Neural_Pipeline_Search_2023,
17 | author = {Stoll, Danny and Mallik, Neeratyoy and Schrodi, Simon and Bergmann, Eddie and Janowski, Maciej and Garibov, Samir and Abou Chakra, Tarek and Rogalla, Daniel and Bergman, Eddie and Hvarfner, Carl and Binxin, Ru and Hutter, Frank},
18 | month = oct,
19 | title = {{Neural Pipeline Search (NePS)}},
20 | url = {https://github.com/automl/neps},
21 | version = {0.12.2},
22 | year = {2024}
23 | }
24 | ```
25 |
26 | ## Citation of Papers
27 |
28 | ### PriorBand
29 |
30 | If you have used [PriorBand](https://openreview.net/forum?id=uoiwugtpCH) as the optimizer, please use the bibtex below:
31 |
32 | ```bibtex
33 | @inproceedings{mallik2023priorband,
34 | title = {PriorBand: Practical Hyperparameter Optimization in the Age of Deep Learning},
35 | author = {Neeratyoy Mallik and Eddie Bergman and Carl Hvarfner and Danny Stoll and Maciej Janowski and Marius Lindauer and Luigi Nardi and Frank Hutter},
36 | year = {2023},
37 | booktitle = {Thirty-seventh Conference on Neural Information Processing Systems (NeurIPS 2023)},
38 | keywords = {}
39 | }
40 | ```
41 |
42 | ### Hierarchichal NAS with Context-free Grammars
43 |
44 | If you have used the context-free grammar search space and the graph kernels implemented in NePS for the paper [Hierarchical NAS](https://openreview.net/forum?id=Hpt1i5j6wh), please use the bibtex below:
45 |
46 | ```bibtex
47 | @inproceedings{schrodi2023hierarchical,
48 | title = {Construction of Hierarchical Neural Architecture Search Spaces based on Context-free Grammars},
49 | author = {Simon Schrodi and Danny Stoll and Binxin Ru and Rhea Sanjay Sukthanker and Thomas Brox and Frank Hutter},
50 | year = {2023},
51 | booktitle = {Thirty-seventh Conference on Neural Information Processing Systems (NeurIPS 2023)},
52 | keywords = {}
53 | }
54 | ```
55 |
--------------------------------------------------------------------------------
/docs/dev_docs/contributing.md:
--------------------------------------------------------------------------------
1 | --8<-- "CONTRIBUTING.md"
2 |
--------------------------------------------------------------------------------
/docs/doc_images/examples/val_loss_image_segmentation.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/automl/neps/0ef3731fa5049336e9a2bb61be7093fc902a96ff/docs/doc_images/examples/val_loss_image_segmentation.jpg
--------------------------------------------------------------------------------
/docs/doc_images/optimizers/bo_acqu.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/automl/neps/0ef3731fa5049336e9a2bb61be7093fc902a96ff/docs/doc_images/optimizers/bo_acqu.jpg
--------------------------------------------------------------------------------
/docs/doc_images/optimizers/bo_surrogate.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/automl/neps/0ef3731fa5049336e9a2bb61be7093fc902a96ff/docs/doc_images/optimizers/bo_surrogate.jpg
--------------------------------------------------------------------------------
/docs/doc_images/optimizers/freeze_thaw_fantasizing.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/automl/neps/0ef3731fa5049336e9a2bb61be7093fc902a96ff/docs/doc_images/optimizers/freeze_thaw_fantasizing.jpg
--------------------------------------------------------------------------------
/docs/doc_images/optimizers/freeze_thawing.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/automl/neps/0ef3731fa5049336e9a2bb61be7093fc902a96ff/docs/doc_images/optimizers/freeze_thawing.jpg
--------------------------------------------------------------------------------
/docs/doc_images/optimizers/pibo_acqus.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/automl/neps/0ef3731fa5049336e9a2bb61be7093fc902a96ff/docs/doc_images/optimizers/pibo_acqus.jpg
--------------------------------------------------------------------------------
/docs/doc_images/optimizers/priorband_sampler.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/automl/neps/0ef3731fa5049336e9a2bb61be7093fc902a96ff/docs/doc_images/optimizers/priorband_sampler.jpg
--------------------------------------------------------------------------------
/docs/doc_images/optimizers/sh_wikipedia.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/automl/neps/0ef3731fa5049336e9a2bb61be7093fc902a96ff/docs/doc_images/optimizers/sh_wikipedia.jpg
--------------------------------------------------------------------------------
/docs/doc_images/tensorboard/tblogger_hparam1.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/automl/neps/0ef3731fa5049336e9a2bb61be7093fc902a96ff/docs/doc_images/tensorboard/tblogger_hparam1.jpg
--------------------------------------------------------------------------------
/docs/doc_images/tensorboard/tblogger_hparam2.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/automl/neps/0ef3731fa5049336e9a2bb61be7093fc902a96ff/docs/doc_images/tensorboard/tblogger_hparam2.jpg
--------------------------------------------------------------------------------
/docs/doc_images/tensorboard/tblogger_hparam3.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/automl/neps/0ef3731fa5049336e9a2bb61be7093fc902a96ff/docs/doc_images/tensorboard/tblogger_hparam3.jpg
--------------------------------------------------------------------------------
/docs/doc_images/tensorboard/tblogger_image.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/automl/neps/0ef3731fa5049336e9a2bb61be7093fc902a96ff/docs/doc_images/tensorboard/tblogger_image.jpg
--------------------------------------------------------------------------------
/docs/doc_images/tensorboard/tblogger_scalar.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/automl/neps/0ef3731fa5049336e9a2bb61be7093fc902a96ff/docs/doc_images/tensorboard/tblogger_scalar.jpg
--------------------------------------------------------------------------------
/docs/getting_started.md:
--------------------------------------------------------------------------------
1 | # Getting Started
2 |
3 | Getting started with NePS involves a straightforward yet powerful process, centering around its three main components.
4 | This approach ensures flexibility and efficiency in evaluating different architecture and hyperparameter configurations
5 | for your problem.
6 |
7 | NePS requires Python 3.10 or higher.
8 | You can install it via `pip` or from [source](https://github.com/automl/neps/).
9 |
10 | ```bash
11 | pip install neural-pipeline-search
12 | ```
13 |
14 | ## The 3 Main Components
15 |
16 | 1. **Establish a [`pipeline_space=`](reference/pipeline_space.md)**:
17 |
18 | ```python
19 | pipeline_space={
20 | "some_parameter": (0.0, 1.0), # float
21 | "another_parameter": (0, 10), # integer
22 | "optimizer": ["sgd", "adam"], # categorical
23 | "epoch": neps.Integer(lower=1, upper=100, is_fidelity=True),
24 | "learning_rate": neps.Float(lower=1e-5, upper=1, log=True),
25 | "alpha": neps.Float(lower=0.1, upper=1.0, prior=0.99, prior_confidence="high")
26 | }
27 |
28 | ```
29 |
30 | 2. **Define an `evaluate_pipeline()` function**:
31 |
32 | ```python
33 | def evaluate_pipeline(some_parameter: float,
34 | another_parameter: float,
35 | optimizer: str, epoch: int,
36 | learning_rate: float, alpha: float) -> float:
37 | model = make_model(...)
38 | loss = eval_model(model)
39 | return loss
40 | ```
41 |
42 | 3. **Execute with [`neps.run()`](reference/neps_run.md)**:
43 |
44 | ```python
45 | neps.run(evaluate_pipeline, pipeline_space)
46 | ```
47 |
48 | ---
49 |
50 | ## What's Next?
51 |
52 | The [reference](reference/neps_run.md) section provides detailed information on the individual components of NePS.
53 |
54 | 1. How to use the [**`neps.run()`** function](reference/neps_run.md) to start the optimization process.
55 | 2. The different [search space](reference/pipeline_space.md) options available.
56 | 3. How to choose and configure the [optimizer](reference/optimizers.md) used.
57 | 4. How to define the [`evaluate_pipeline()` function](reference/evaluate_pipeline.md).
58 | 5. How to [analyze](reference/analyse.md) the optimization runs.
59 |
60 | Or discover the features of NePS through these practical examples:
61 |
62 | * **[Hyperparameter Optimization (HPO)](examples/basic_usage/hyperparameters.md)**:
63 | Learn the essentials of hyperparameter optimization with NePS.
64 |
65 | * **[Multi-Fidelity Optimization](examples/efficiency/multi_fidelity.md)**:
66 | Understand how to leverage multi-fidelity optimization for efficient model tuning.
67 |
68 | * **[Utilizing Expert Priors for Hyperparameters](examples/efficiency/expert_priors_for_hyperparameters.md)**:
69 | Learn how to incorporate expert priors for more efficient hyperparameter selection.
70 |
71 | * **[Additional NePS Examples](examples/index.md)**:
72 | Explore more examples, including various use cases and advanced configurations in NePS.
73 |
--------------------------------------------------------------------------------
/docs/index.md:
--------------------------------------------------------------------------------
1 | # Neural Pipeline Search (NePS)
2 |
3 | [](https://pypi.org/project/neural-pipeline-search/)
4 | [](https://pypi.org/project/neural-pipeline-search/)
5 | [](https://github.com/automl/neps/blob/master/LICENSE)
6 | [](https://github.com/automl/neps/actions)
7 |
8 | Welcome to NePS, a powerful and flexible Python library for hyperparameter optimization (HPO) and neural architecture search (NAS) with its primary goal: **make HPO and NAS usable for deep learners in practice**.
9 |
10 | NePS houses recently published and also well-established algorithms that can all be run massively parallel on distributed setups, with tools to analyze runs, restart runs, etc., all **tailored to the needs of deep learning experts**.
11 |
12 | ## Key Features
13 |
14 | In addition to the features offered by traditional HPO and NAS libraries, NePS stands out with:
15 |
16 | 1. **Hyperparameter Optimization (HPO) Efficient Enough For Deep Learning:**
17 | NePS excels in efficiently tuning hyperparameters using algorithms that enable users to make use of their prior knowledge, while also using many other efficiency boosters.
18 | - [PriorBand: Practical Hyperparameter Optimization in the Age of Deep Learning (NeurIPS 2023)](https://arxiv.org/abs/2306.12370)
19 | - [πBO: Augmenting Acquisition Functions with User Beliefs for Bayesian Optimization (ICLR 2022)](https://arxiv.org/abs/2204.11051)
20 | 1. **Neural Architecture Search (NAS) with Expressive Search Spaces:**
21 | NePS provides capabilities for designing and optimizing architectures in an expressive and natural fashion.
22 | - [Construction of Hierarchical Neural Architecture Search Spaces based on Context-free Grammars (NeurIPS 2023)](https://arxiv.org/abs/2211.01842)
23 | 1. **Zero-effort Parallelization and an Experience Tailored to DL:**
24 | NePS simplifies the process of parallelizing optimization tasks both on individual computers and in distributed
25 | computing environments. As NePS is made for deep learners, all technical choices are made with DL in mind and common
26 | DL tools such as Tensorboard are [embraced](https://automl.github.io/neps/latest/reference/analyse/#visualizing-results).
27 |
28 | !!! tip
29 |
30 | Check out:
31 |
32 | * [Reference documentation](./reference/neps_run.md) for a quick overview.
33 | * [API](api/neps/api.md) for a more detailed reference.
34 | * [Colab Tutorial](https://colab.research.google.com/drive/11IOhkmMKsIUhWbHyMYzT0v786O9TPWlH?usp=sharing) walking through NePS's main features.
35 | * [Examples](examples/index.md) for basic code snippets to get started.
36 |
37 | ## Installation
38 |
39 | To install the latest release from PyPI run
40 |
41 | ```bash
42 | pip install neural-pipeline-search
43 | ```
44 |
45 | ## Basic Usage
46 |
47 | Using `neps` always follows the same pattern:
48 |
49 | 1. Define a `evaluate_pipeline` function capable of evaluating different architectural and/or hyperparameter configurations
50 | for your problem.
51 | 1. Define a search space named `pipeline_space` of those Parameters e.g. via a dictionary
52 | 1. Call `neps.run` to optimize `evaluate_pipeline` over `pipeline_space`
53 |
54 | In code, the usage pattern can look like this:
55 |
56 | ```python
57 | import neps
58 | import logging
59 |
60 |
61 | # 1. Define a function that accepts hyperparameters and computes the validation error
62 | def evaluate_pipeline(
63 | hyperparameter_a: float, hyperparameter_b: int, architecture_parameter: str
64 | ) -> dict:
65 | # Create your model
66 | model = MyModel(architecture_parameter)
67 |
68 | # Train and evaluate the model with your training pipeline
69 | validation_error = train_and_eval(
70 | model, hyperparameter_a, hyperparameter_b
71 | )
72 | return validation_error
73 |
74 |
75 | # 2. Define a search space of parameters; use the same parameter names as in evaluate_pipeline
76 | pipeline_space = dict(
77 | hyperparameter_a=neps.Float(
78 | lower=0.001, upper=0.1, log=True # The search space is sampled in log space
79 | ),
80 | hyperparameter_b=neps.Integer(lower=1, upper=42),
81 | architecture_parameter=neps.Categorical(["option_a", "option_b"]),
82 | )
83 |
84 | # 3. Run the NePS optimization
85 | logging.basicConfig(level=logging.INFO)
86 | neps.run(
87 | evaluate_pipeline=evaluate_pipeline,
88 | pipeline_space=pipeline_space,
89 | root_directory="path/to/save/results", # Replace with the actual path.
90 | max_evaluations_total=100,
91 | )
92 | ```
93 |
94 | ## Examples
95 |
96 | Discover how NePS works through these examples:
97 |
98 | - **[Hyperparameter Optimization](examples/basic_usage/hyperparameters.md)**: Learn the essentials of hyperparameter optimization with NePS.
99 |
100 | - **[Multi-Fidelity Optimization](examples/efficiency/multi_fidelity.md)**: Understand how to leverage multi-fidelity optimization for efficient model tuning.
101 |
102 | - **[Utilizing Expert Priors for Hyperparameters](examples/efficiency/expert_priors_for_hyperparameters.md)**: Learn how to incorporate expert priors for more efficient hyperparameter selection.
103 |
104 | - **[Additional NePS Examples](examples/index.md)**: Explore more examples, including various use cases and advanced configurations in NePS.
105 |
106 | ## Contributing
107 |
108 | Please see the [documentation for contributors](dev_docs/contributing.md).
109 |
110 | ## Citations
111 |
112 | For pointers on citing the NePS package and papers refer to our [documentation on citations](citations.md).
113 |
--------------------------------------------------------------------------------
/docs/reference/evaluate_pipeline.md:
--------------------------------------------------------------------------------
1 | # The evaluate function
2 |
3 | ## Introduction
4 |
5 | The `evaluate_pipeline=` function is crucial for NePS. It encapsulates the objective function to be minimized, which could range from a regular equation to a full training and evaluation pipeline for a neural network.
6 |
7 | This function receives the configuration to be utilized from the parameters defined in the search space. Consequently, it executes the same set of instructions or equations based on the provided configuration to minimize the objective function.
8 |
9 | We will show some basic usages and some functionalites this function would require for successful implementation.
10 |
11 | ## Types of Returns
12 |
13 | ### 1. Single Value
14 |
15 | Assuming the `pipeline_space=` was already created (have a look at [pipeline space](./pipeline_space.md) for more details).
16 | A `evaluate_pipeline=` function with an objective of minimizing the loss will resemble the following:
17 |
18 | ```python
19 | def evaluate_pipeline(
20 | **config, # The hyperparameters to be used in the pipeline
21 | ):
22 | element_1 = config["element_1"]
23 | element_2 = config["element_2"]
24 | element_3 = config["element_3"]
25 |
26 | loss = element_1 - element_2 + element_3
27 |
28 | return loss
29 | ```
30 |
31 | ### 2. Dictionary
32 |
33 | In this section, we will outline the special variables that are expected to be returned when the `evaluate_pipeline=` function returns a dictionary.
34 |
35 | #### Loss
36 |
37 | One crucial return variable is the `loss`. This metric serves as a fundamental indicator for the optimizer. One option is to return a dictionary with the `loss` as a key, along with other user-chosen metrics.
38 |
39 | !!! note
40 |
41 | Loss can be any value that is to be minimized by the objective function.
42 |
43 | ```python
44 | def evaluate_pipeline(
45 | **config, # The hyperparameters to be used in the pipeline
46 | ):
47 |
48 | element_1 = config["element_1"]
49 | element_2 = config["element_2"]
50 | element_3 = config["element_3"]
51 |
52 | loss = element_1 - element_2 + element_3
53 | reverse_loss = -loss
54 |
55 | return {
56 | "objective_to_minimize": loss,
57 | "info_dict": {
58 | "reverse_loss": reverse_loss
59 | ...
60 | }
61 | }
62 | ```
63 |
64 | #### Cost
65 |
66 | Along with the return of the `loss`, the `evaluate_pipeline=` function would optionally need to return a `cost` in certain cases. Specifically when the `max_cost_total` parameter is being utilized in the `neps.run` function.
67 |
68 |
69 | !!! note
70 |
71 | `max_cost_total` sums the cost from all returned configuration results and checks whether the maximum allowed cost has been reached (if so, the search will come to an end).
72 |
73 | ```python
74 | import neps
75 | import logging
76 |
77 |
78 | def evaluate_pipeline(
79 | **config, # The hyperparameters to be used in the pipeline
80 | ):
81 |
82 | element_1 = config["element_1"]
83 | element_2 = config["element_2"]
84 | element_3 = config["element_3"]
85 |
86 | loss = element_1 - element_2 + element_3
87 | cost = 2
88 |
89 | return {
90 | "objective_to_minimize": loss,
91 | "cost": cost,
92 | }
93 |
94 | if __name__ == "__main__":
95 | logging.basicConfig(level=logging.INFO)
96 | neps.run(
97 | evaluate_pipeline=evaluate_pipeline,
98 | pipeline_space=pipeline_space, # Assuming the pipeline space is defined
99 | root_directory="results/bo",
100 | max_cost_total=10,
101 | optimizer="bayesian_optimization",
102 | )
103 | ```
104 |
105 | Each evaluation carries a cost of 2. Hence in this example, the Bayesian optimization search is set to perform 5 evaluations.
106 |
107 | ## Arguments for Convenience
108 |
109 | NePS also provides the `pipeline_directory` and the `previous_pipeline_directory` as arguments in the `evaluate_pipeline=` function for user convenience.
110 |
111 | Regard an example to be run with a multi-fidelity optimizer, some checkpointing would be advantageous such that one does not have to train the configuration from scratch when the configuration qualifies to higher fidelity brackets.
112 |
113 | ```python
114 | def evaluate_pipeline(
115 | pipeline_directory, # The directory where the config is saved
116 | previous_pipeline_directory, # The directory of the immediate lower fidelity config
117 | **config, # The hyperparameters to be used in the pipeline
118 | ):
119 | # Assume the third element is our fidelity element
120 | element_1 = config["element_1"]
121 | element_2 = config["element_2"]
122 | fidelity = config["fidelity"]
123 |
124 | # Load any saved checkpoints
125 | checkpoint_name = "checkpoint.pth"
126 | start_fidelity = 0
127 |
128 | if previous_pipeline_directory is not None:
129 | # Read in state of the model after the previous fidelity rung
130 | checkpoint = torch.load(previous_pipeline_directory / checkpoint_name)
131 | prev_fidelity = checkpoint["fidelity"]
132 | else:
133 | prev_fidelity = 0
134 |
135 | start_fidelity += prev_fidelity
136 |
137 | loss = 0
138 | for i in range(start_fidelity, fidelity):
139 | loss += element_1 - element_2
140 |
141 | torch.save(
142 | {
143 | "fidelity": fidelity,
144 | },
145 | pipeline_directory / checkpoint_name,
146 | )
147 |
148 | return loss
149 | ```
150 |
151 | This could allow the proper navigation to the trained models and further train them on higher fidelities without repeating the entire training process.
152 |
--------------------------------------------------------------------------------
/docs/reference/pipeline_space.md:
--------------------------------------------------------------------------------
1 | # Initializing the Pipeline Space
2 |
3 | In NePS, we need to define a `pipeline_space`.
4 | This space can be structured through various approaches, including a Python dictionary, or ConfigSpace.
5 | Each of these methods allows you to specify a set of parameter types, ranging from Float and Categorical to specialized architecture parameters.
6 | Whether you choose a dictionary, or ConfigSpace, your selected method serves as a container or framework
7 | within which these parameters are defined and organized. This section not only guides you through the process of
8 | setting up your `pipeline_space` using these methods but also provides detailed instructions and examples on how to
9 | effectively incorporate various parameter types, ensuring that NePS can utilize them in the optimization process.
10 |
11 |
12 | ## Parameters
13 | NePS currently features 4 primary hyperparameter types:
14 |
15 | * [`Categorical`][neps.space.Categorical]
16 | * [`Float`][neps.space.Float]
17 | * [`Integer`][neps.space.Integer]
18 | * [`Constant`][neps.space.Constant]
19 |
20 | Using these types, you can define the parameters that NePS will optimize during the search process.
21 | The most basic way to pass these parameters is through a Python dictionary, where each key-value
22 | pair represents a parameter name and its respective type.
23 | For example, the following Python dictionary defines a `pipeline_space` with four parameters
24 | for optimizing a deep learning model:
25 |
26 | ```python
27 | pipeline_space = {
28 | "learning_rate": neps.Float(0.00001, 0.1, log=True),
29 | "num_epochs": neps.Integer(3, 30, is_fidelity=True),
30 | "optimizer": ["adam", "sgd", "rmsprop"], # Categorical
31 | "dropout_rate": 0.5, # Constant
32 | }
33 |
34 | neps.run(.., pipeline_space=pipeline_space)
35 | ```
36 |
37 | ??? example "Quick Parameter Reference"
38 |
39 | === "`Categorical`"
40 |
41 | ::: neps.space.Categorical
42 |
43 | === "`Float`"
44 |
45 | ::: neps.space.Float
46 |
47 | === "`Integer`"
48 |
49 | ::: neps.space.Integer
50 |
51 | === "`Constant`"
52 |
53 | ::: neps.space.Constant
54 |
55 |
56 | ## Using your knowledge, providing a Prior
57 | When optimizing, you can provide your own knowledge using the parameter `prior=`.
58 | By indicating a `prior=` we take this to be your user prior,
59 | **your knowledge about where a good value for this parameter lies**.
60 |
61 | You can also specify a `prior_confidence=` to indicate how strongly you want NePS,
62 | to focus on these, one of either `"low"`, `"medium"`, or `"high"`.
63 |
64 | ```python
65 | import neps
66 |
67 | neps.run(
68 | ...,
69 | pipeline_space={
70 | "learning_rate": neps.Float(1e-4, 1e-1, log=True, prior=1e-2, prior_confidence="medium"),
71 | "num_epochs": neps.Integer(3, 30, is_fidelity=True),
72 | "optimizer": neps.Categorical(["adam", "sgd", "rmsprop"], prior="adam", prior_confidence="low"),
73 | "dropout_rate": neps.Constant(0.5),
74 | }
75 | )
76 | ```
77 |
78 | !!! warning "Interaction with `is_fidelity`"
79 |
80 | If you specify `is_fidelity=True` and `prior=` for one parameter, this will raise an error.
81 |
82 | Currently the two major algorithms that exploit this in NePS are `PriorBand`
83 | (prior-based `HyperBand`) and `PiBO`, a version of Bayesian Optimization which uses Priors. For more information on priors and algorithms using them, please refer to the [prior documentation](../reference/search_algorithms/prior.md).
84 |
85 | ## Using ConfigSpace
86 |
87 | For users familiar with the [`ConfigSpace`](https://automl.github.io/ConfigSpace/main/) library,
88 | can also define the `pipeline_space` through `ConfigurationSpace()`
89 |
90 | ```python
91 | from configspace import ConfigurationSpace, Float
92 |
93 | configspace = ConfigurationSpace(
94 | {
95 | "learning_rate": Float("learning_rate", bounds=(1e-4, 1e-1), log=True)
96 | "optimizer": ["adam", "sgd", "rmsprop"],
97 | "dropout_rate": 0.5,
98 | }
99 | )
100 | ```
101 |
102 | !!! warning
103 |
104 | Parameters you wish to use as a **fidelity** are not support through ConfigSpace
105 | at this time.
106 |
107 | For additional information on ConfigSpace and its features, please visit the following
108 | [link](https://github.com/automl/ConfigSpace).
109 |
--------------------------------------------------------------------------------
/docs/reference/search_algorithms/bayesian_optimization.md:
--------------------------------------------------------------------------------
1 | # Bayesian Optimization
2 |
3 | ## What is Bayesian Optimization?
4 |
5 | `Bayesian Optimization`/`BO` is a fundamental optimization technique for finding (local) optima of expensive-to-evaluate functions. The main idea of `BO` is an interplay of a model (the [`surrogate function`](../search_algorithms/bayesian_optimization.md#the-surrogate-function)) of the objective function, built from the data collected during the optimization process, and an [`acquisition function`](../search_algorithms/bayesian_optimization.md#the-acquisition-function) that guides the search for the next evaluation point.
6 |
7 | ### The surrogate function
8 |
9 | For each dimension of the search space, the surrogate function models the objective function as a `Gaussian Process` (GP). A GP consists of a mean function and a covariance function, which are both learned from the data. The mean function represents the expected value of the objective function, while the covariance function models the uncertainty of the predictions.
10 | The following image shows a GP with its mean function and the 95% confidence interval:
11 |
12 | ||
13 | |:--:|
14 | |The dashed line represents the (hidden) objective function, while the solid line is the surrogate mean function. The shaded area around the mean function is its confidence interval. Note that the confidence interval collapses where observations have been made and gets large in regions where no data is available yet. (Image Source: [Medium.com](https://towardsdatascience.com/shallow-understanding-on-bayesian-optimization-324b6c1f7083), Jan 27, 2025)|
15 |
16 | ### The acquisition function
17 |
18 | The acquisition function is the guiding force in `BO`. From the information contained in the surrogate function, the acquisition function suggests the next evaluation point. It balances the trade-off between exploration (sampling points where the surrogate function is uncertain) and exploitation (sampling points where the surrogate function is promising).
19 |
20 | ||
21 | |:--:|
22 | |The image shows the surrogate function from before, now with the acquisition function plotted below. The maximum of the acquisition function is the point that will usually be evaluated next. (Image Source: [Medium.com](https://towardsdatascience.com/shallow-understanding-on-bayesian-optimization-324b6c1f7083), Jan 27, 2025)|
23 |
24 | There are numerous acquisition functions, with the most popular being `Expected Improvement` (EI):
25 |
26 | - EI is defined as the expected improvement over the current best observation:
27 |
28 | $$EI(\boldsymbol{x}) = \mathbb{E}[\max(0, f(\boldsymbol{x}) - f(\boldsymbol{x}^+))]$$
29 |
30 | And `Probability of Improvement` (PI):
31 |
32 | - PI is defined as the probability that the surrogate function is better than the current best observation:
33 |
34 | $$PI(\boldsymbol{x}) = P(f(\boldsymbol{x}) > f(\boldsymbol{x}^+))$$
35 |
36 | where $f(\boldsymbol{x})$ is the surrogate function and $f(\boldsymbol{x}^+)$ is the best observation so far.
37 |
38 | To read more about `BO`, please refer to this [`Bayesian Optimization` tutorial](https://arxiv.org/abs/1807.02811) or this article on [Towards Data Science](https://towardsdatascience.com/bayesian-optimization-concept-explained-in-layman-terms-1d2bcdeaf12f).
39 |
40 | !!! example "Practical Tips"
41 |
42 | - `BO` can handle expensive-to-evaluate, noisy, high-dimensional and black-box objectives and can be used in the optimization of hyperparameters, neural architectures, and the entire pipeline.
43 | - It is highly costumizable with many choices for the surrogate and acquisition functions, but even the basic settings work well in many cases.
44 |
45 | !!! info
46 | Therefore, `BO` is chosen as the [default optimizer](../../reference/optimizers.md#21-automatic-optimizer-selection) in NePS when there is no [Prior](../search_algorithms/prior.md) or [Multi-Fidelity](../search_algorithms/multifidelity.md) information available.
47 |
--------------------------------------------------------------------------------
/docs/reference/search_algorithms/landing_page_algo.md:
--------------------------------------------------------------------------------
1 | # Algorithms
2 |
3 | Algorithms are the search strategies determining what configurations to evaluate next. In NePS, we provide a variety of pre-implemented algorithms and offer the possibility to implement custom algorithms. This chapter gives an overview of the different algorithms available in NePS and practical tips for their usage.
4 |
5 | We distinguish between algorithms that use different types of information and strategies to guide the search process:
6 |
7 | ✅ = supported/necessary, ❌ = not supported, ✔️* = optional, click for details, ✖️\* ignorable, click for details
8 |
9 | | Algorithm | [Multi-Fidelity](../search_algorithms/multifidelity.md) | [Priors](../search_algorithms/prior.md) | Model-based |
10 | | :- | :------------: | :----: | :---------: |
11 | | `Grid Search`|[️️✖️*][neps.optimizers.algorithms.grid_search]|❌|❌|
12 | | `Random Search`|[️️✖️*][neps.optimizers.algorithms.random_search]|[✔️*][neps.optimizers.algorithms.random_search]|❌|
13 | | [`Bayesian Optimization`](../search_algorithms/bayesian_optimization.md)|[️️✖️*][neps.optimizers.algorithms.bayesian_optimization]|❌|✅|
14 | | [`Successive Halving`](../search_algorithms/multifidelity.md#1-successive-halfing)|✅|[✔️*][neps.optimizers.algorithms.successive_halving]|❌|
15 | | [`ASHA`](../search_algorithms/multifidelity.md#asynchronous-successive-halving)|✅|[✔️*][neps.optimizers.algorithms.asha]|❌|
16 | | [`Hyperband`](../search_algorithms/multifidelity.md#2-hyperband)|✅|[✔️*][neps.optimizers.algorithms.hyperband]|❌|
17 | | [`Asynch HB`](../search_algorithms/multifidelity.md)|✅|[✔️*][neps.optimizers.algorithms.async_hb]|❌|
18 | | [`IfBO`](../search_algorithms/multifidelity.md#3-in-context-freeze-thaw-bayesian-optimization)|✅|[✔️*][neps.optimizers.algorithms.ifbo]|✅|
19 | | [`PiBO`](../search_algorithms/prior.md#1-pibo)|[️️✖️*][neps.optimizers.algorithms.pibo]|✅|✅|
20 | | [`PriorBand`](../search_algorithms/multifidelity_prior.md#1-priorband)|✅|✅|✅|
21 |
22 | ## What is Multi-Fidelity Optimization?
23 |
24 | Multi-Fidelity (MF) optimization leverages the idea of running an AutoML problem on a small scale, which is cheaper and faster, and then using this information to train full-scale models. The _low-fidelity_ runs could be on a smaller dataset, a smaller model, or for shorter training times. MF-algorithms then infer which configurations are likely to perform well on the full problem, before investing larger compute amounts.
25 |
26 | !!! tip "Advantages of Multi-Fidelity"
27 |
28 | - **Parallelization**: MF-algorithms can use the information from many parallel low-fidelity runs to guide the search in the few high-fidelity runs.
29 | - **Exploration**: By using low-fidelity runs, the optimizer can explore more of the search space.
30 |
31 | !!! warning "Disadvantages of Multi-Fidelity"
32 |
33 | - **Variance**: The performance of a configuration on a low-fidelity run might not correlate well with its performance on a high-fidelity run. This can result in misguided decisions.
34 |
35 | We present a collection of MF-algorithms [here](./multifidelity.md) and algorithms that combine MF with priors [here](./multifidelity_prior.md).
36 |
37 | ## What are Priors?
38 |
39 | Priors are used when there exists some information about the search space, that can be used to guide the optimization process. This information could come from expert domain knowledge or previous experiments. A Prior is provided in the form of a distribution over one dimension of the search space, with a `mean` (the suspected optimum) and a `confidence level`, or `variance`. We discuss how Priors can be included in your NePS-search space [here](../../reference/pipeline_space.md#using-your-knowledge-providing-a-prior).
40 |
41 | !!! tip "Advantages of using Priors"
42 |
43 | - **Less compute**: By providing a Prior, the optimizer can focus on the most promising regions of the search space, potentially saving a lot of compute.
44 | - **More exploitation**: By focusing on these regions, the optimizer might find a better final solution.
45 |
46 | !!! warning "Disadvantages of using Priors"
47 |
48 | - **Less exploration**: By focusing on these regions, the optimizer _might_ miss out on other regions that could potentially be better.
49 | - **Bad priors**: If the Prior is not a good representation of the search space, the optimizer might deliver suboptimal results, compared to a search without Priors. The optimizers we provide in NePS are specifically designed to handle bad priors, but they still slow down the search process.
50 |
51 | We present a collection of algorithms that use Priors [here](./prior.md) and algorithms that combine priors with Multi-Fidelity [here](./multifidelity_prior.md).
52 |
--------------------------------------------------------------------------------
/docs/reference/search_algorithms/multifidelity_prior.md:
--------------------------------------------------------------------------------
1 | # Multi-Fidelity and Prior Optimizers
2 |
3 | This section concerns optimizers that use both Multi-Fidelity and Priors. They combine the advantages and disadvantages of both methods to exploit all available information.
4 | For a detailed explanation of Multi-Fidelity and Priors, please refer [here](landing_page_algo.md).
5 |
6 | ## Optimizers using Multi-Fidelity and Priors
7 |
8 | ### 1 `PriorBand`
9 |
10 | `PriorBand` (see [paper](https://openreview.net/pdf?id=uoiwugtpCH)) is an extension of [`HyperBand`](../../reference/search_algorithms/multifidelity.md#2-hyperband) that utilizes expert Priors to choose the next configuration.
11 |
12 | ``PriorBand``'s sampling module $\mathcal{E}_\pi$ balances the influence of the Prior, the incumbent configurations and randomness to select configurations.
13 |
14 | ||
15 | |:--:|
16 | |The ``PriorBand`` sampling module balances the influence of the Prior, the $1/\eta$ incumbent configurations and randomness to select configurations. (Image Source: [PriorBand-paper](https://openreview.net/pdf?id=uoiwugtpCH), Jan 27, 2025)|
17 |
18 | The Prior sampling $p_\pi$ is most meaningful at full fidelity and when not much data is available yet, while the incumbent sampling $p_{\hat{\lambda}}$, coming from actual data, is most significant but sparse, and random sampling $p_{\mathcal{U}}$ is needed for exploration, especially at lower fidelities. This results in these inital sampling probabilities when there is no incumbent yet:
19 |
20 | $$
21 | p_{\mathcal{U}}=1/(1+\eta^r)
22 | $$
23 |
24 | $$
25 | p_\pi=1-p_{\mathcal{U}}
26 | $$
27 |
28 | $$
29 | p_{\hat{\lambda}}=0
30 | $$
31 |
32 | where $\eta$ is the promotion-hyperparameter from [`HyperBand`](../../reference/search_algorithms/multifidelity.md#2-hyperband) and $r$ is the current fidelity level (_rung_), showing the decay of the random sampling probability with increasing fidelity.
33 |
34 | When there is an incumbent, the probabilities are adjusted to:
35 |
36 | $$
37 | p_{\mathcal{U}}=1/(1+\eta^r)
38 | $$
39 |
40 | $$
41 | p_\pi=p_\pi\cdot\mathcal{S}_{\hat{\lambda}}/(\mathcal{S}_\pi+\mathcal{S}_{\hat{\lambda}})
42 | $$
43 |
44 | $$
45 | p_{\hat{\lambda}}=p_{\hat{\lambda}}\cdot\mathcal{S}_{\pi}/(\mathcal{S}_\pi+\mathcal{S}_{\hat{\lambda}})
46 | $$
47 |
48 | where $\mathcal{S}_\pi$ and $\mathcal{S}_{\hat{\lambda}}$ are the summed probabilities of the top $1/\eta$ configurations under Prior and incumbent sampling, respectively. This way, the balance is shifted towards the distribution that would have yielded the best configurations so far. Crucially, this compensates for potentially bad Priors, as the incumbent sampling will take over when it has proven to be better.
49 |
50 | See the algorithm's implementation details in the [api][neps.optimizers.algorithms.priorband].
51 |
52 | ??? example "Practical Tips"
53 |
54 | - ``PriorBand`` is a good choice when you have a Prior but are wary of its quality and you can utilize Multi-Fidelity.
55 |
56 | !!! info
57 |
58 | `PriorBand` is chosen as the [default optimizer](../../reference/optimizers.md#21-automatic-optimizer-selection) in NePS when there is both [Prior](../search_algorithms/prior.md) and [Multi-Fidelity](../search_algorithms/multifidelity.md) information available.
59 |
60 | #### _Model-based_ `PriorBand`
61 |
62 | `PriorBand` can also be extended with a model, where after $n$ evaluations, a [`BO`](../search_algorithms/bayesian_optimization.md) model is trained to advise the sampling module.
63 |
--------------------------------------------------------------------------------
/docs/reference/search_algorithms/prior.md:
--------------------------------------------------------------------------------
1 | # Prior Optimizers
2 |
3 | This section concerns optimizers that utilize priors to guide the search process. Priors are explained in detail [here](./landing_page_algo.md#what-are-priors).
4 |
5 | ## 1 `PiBO`
6 |
7 | `PiBO` (see [paper](https://arxiv.org/pdf/2204.11051)) is an extension of [`Bayesian Optimization` (`BO`)](../search_algorithms/bayesian_optimization.md) that uses a specific `acquisition function` that incorporates Priors, by including a `Prior-factor` that decays over time. This way, the optimizer first relies on the Prior knowledge, before shifting focus to the data acquired during the optimization process.
8 | The altered acquisition function takes this form:
9 |
10 | $$\boldsymbol{x}_n\in \underset{\boldsymbol{x}\in\mathcal{X}}{\operatorname{argmax}}\alpha(\boldsymbol{x},\mathcal{D}_n)\pi(\boldsymbol{x})^{\beta/n}$$
11 |
12 | where after $n$ evaluations, the Prior-function $\pi(\boldsymbol{x})$ is decayed by the factor $\beta/n$ and multiplied with the acquisition function $\alpha(\boldsymbol{x},\mathcal{D}_n)$. In our `PiBO` implementation, we use [`Expected Improvement`](../search_algorithms/bayesian_optimization.md#the-acquisition-function) as the acquisition function.
13 |
14 | The following illustration from the `PiBO`-paper shows the influence of a well-chosen and a bad, decaying Prior on the optimization process:
15 |
16 | ||
17 | |:--:|
18 | |Left: A well-located Prior influences the acquisition function leading to quicker convergence and even more exploration. Right: An off-center Prior slows down, but does not prevent convergence. (Image Source: [PiBO-paper](https://arxiv.org/pdf/2204.11051), Jan 27, 2025)|
19 |
20 | In both cases, the optimization process uses the additional information provided by the Prior to arrive at the solution, however, the bad Prior (right) results in a slower convergence to the optimum.
21 |
22 | See the algorithm's implementation details in the [api][neps.optimizers.algorithms.pibo].
23 |
24 | ??? example "Practical Tips"
25 |
26 | TODO
27 |
28 | !!! info
29 | ``PiBO`` is chosen as the [default optimizer](../../reference/optimizers.md#21-automatic-optimizer-selection) in NePS when there is only Prior, but no [Multi-Fidelity](../search_algorithms/multifidelity.md) information available.
30 | ___
31 |
32 | For optimizers using both Priors and Multi-Fidelity, please refer [here](multifidelity_prior.md).
33 |
--------------------------------------------------------------------------------
/docs/reference/seeding.md:
--------------------------------------------------------------------------------
1 | # Seeding
2 |
3 | Seeding is only rudimentarily supported in NePS, as we provide a function to capture the global rng state of `Python`, `numpy` and `torch`. It is not yet possible to seed only NePS internally.
4 |
5 | See the [Seeding API][neps.state.seed_snapshot.SeedSnapshot] for the details on how to [capture][neps.state.seed_snapshot.SeedSnapshot.new_capture] and [use][neps.state.seed_snapshot.SeedSnapshot.set_as_global_seed_state] this global rng state.
6 |
--------------------------------------------------------------------------------
/docs/stylesheets/custom.css:
--------------------------------------------------------------------------------
1 | /* If anything is breaking from this css, please feel free to
2 | * remove it.
3 | */
4 |
5 | /* Highlight None with color inside code blocks */
6 | code.highlight.language-python span.kc {
7 | color: var(--md-code-hl-keyword-color);
8 | }
9 | /* Make tool tip annotations wider */
10 | :root {
11 | --md-tooltip-width: 500px;
12 | }
13 | /* api doc attribute cards
14 | div.doc-class > div.doc-contents > div.doc-children > div.doc-object {
15 | padding-right: 20px;
16 | padding-left: 20px;
17 | border-radius: 15px;
18 | margin-top: 20px;
19 | margin-bottom: 20px;
20 | box-shadow: 2px 2px 2px rgba(0, 0, 0, 0.2);
21 | margin-right: 0px;
22 | border-color: rgba(0, 0, 0, 0.2);
23 | border-width: 1px;
24 | border-style: solid;
25 | } */
26 |
--------------------------------------------------------------------------------
/mkdocs.yml:
--------------------------------------------------------------------------------
1 | site_name: NePS
2 | docs_dir: docs
3 | repo_url: https://github.com/automl/neps
4 | repo_name: automl/neps
5 | edit_uri: ""
6 |
7 | theme:
8 | name: material
9 | icon:
10 | repo: fontawesome/brands/github
11 | features:
12 | - content.code.annotate
13 | - content.code.copy
14 | - navigation.footer
15 | - navigation.sections
16 | - toc.follow
17 | - toc.integrate
18 | - navigation.tabs
19 | - navigation.tabs.sticky
20 | - header.autohide
21 | - search.suggest
22 | - search.highlight
23 | - search.share
24 | palette:
25 | # Palette toggle for light mode
26 | - scheme: default
27 | toggle:
28 | icon: material/weather-night
29 | name: Switch to dark mode
30 | # Palette toggle for dark mode
31 | - scheme: slate
32 | toggle:
33 | icon: material/weather-sunny
34 | name: Switch to light mode
35 |
36 | # We do have some extra custom css
37 | # If for whatever reason you think this is breaking something,
38 | # please feel free to remove it.
39 | extra_css:
40 | - stylesheets/custom.css
41 |
42 | markdown_extensions:
43 | - admonition
44 | - tables
45 | - attr_list
46 | - md_in_html
47 | - toc:
48 | permalink: "#"
49 | - pymdownx.highlight:
50 | anchor_linenums: true
51 | - pymdownx.magiclink:
52 | hide_protocol: true
53 | repo_url_shortener: true
54 | repo_url_shorthand: true
55 | user: automl
56 | repo: neps
57 | - pymdownx.highlight
58 | - pymdownx.inlinehilite
59 | - pymdownx.snippets
60 | - pymdownx.details
61 | - pymdownx.tabbed:
62 | alternate_style: true
63 | - pymdownx.superfences:
64 | custom_fences:
65 | - name: mermaid
66 | class: mermaid
67 | format: !!python/name:pymdownx.superfences.fence_code_format
68 | - pymdownx.emoji:
69 | emoji_index: !!python/name:material.extensions.emoji.twemoji
70 | emoji_generator: !!python/name:material.extensions.emoji.to_svg
71 | - pymdownx.arithmatex:
72 | generic: true
73 |
74 | extra:
75 | version:
76 | provider: mike
77 |
78 | extra_javascript: # Add MathJax for math rendering
79 | - javascripts/mathjax.js
80 | - https://unpkg.com/mathjax@3/es5/tex-mml-chtml.js
81 |
82 | plugins:
83 | - search
84 | - autorefs
85 | - gen-files:
86 | scripts:
87 | - docs/_code/api_generator.py
88 | - docs/_code/example_generator.py
89 | - literate-nav
90 | - mkdocstrings:
91 | default_handler: python
92 | enable_inventory: true
93 | handlers:
94 | python:
95 | paths: [neps]
96 | # Extra objects which allow for linking to external docs
97 | inventories:
98 | - 'https://docs.python.org/3/objects.inv'
99 | - 'https://numpy.org/doc/stable/objects.inv'
100 | - 'https://pandas.pydata.org/docs/objects.inv'
101 | - 'https://pytorch.org/docs/stable/objects.inv'
102 | # Please do not try to change these without having
103 | # looked at all of the documentation and seeing if it
104 | # causes the API docs to look weird anywhere.
105 | options: # https://mkdocstrings.github.io/python/usage/
106 | docstring_section_style: spacy
107 | docstring_options:
108 | ignore_init_summary: true
109 | trim_doctest_flags: true
110 | returns_multiple_items: false
111 | show_docstring_attributes: true
112 | show_docstring_description: true
113 | show_root_heading: false
114 | show_root_toc_entry: false
115 | show_object_full_path: false
116 | show_root_members_full_path: false
117 | signature_crossrefs: true
118 | merge_init_into_class: true
119 | show_symbol_type_heading: true
120 | show_symbol_type_toc: true
121 | docstring_style: google
122 | inherited_members: true
123 | line_length: 60
124 | show_if_no_docstring: false
125 | show_bases: true
126 | show_source: true
127 | members_order: "alphabetical"
128 | group_by_category: true
129 | show_signature: true
130 | separate_signature: true
131 | show_signature_annotations: true
132 | filters:
133 | - "!^_[^_]"
134 |
135 |
136 |
137 | nav:
138 | - Home: 'index.md'
139 | - Getting Started: 'getting_started.md'
140 | - Reference:
141 | - Run: 'reference/neps_run.md'
142 | - Search Space: 'reference/pipeline_space.md'
143 | - The Evaluate Function: 'reference/evaluate_pipeline.md'
144 | - Analysing Runs: 'reference/analyse.md'
145 | - Optimizer: 'reference/optimizers.md'
146 | - Seeding: 'reference/seeding.md'
147 | - Examples: "examples/" # auto-generated
148 | - Algorithms:
149 | - Algorithms: 'reference/search_algorithms/landing_page_algo.md'
150 | - Multi-Fidelity Optimizers: 'reference/search_algorithms/multifidelity.md'
151 | - Prior Optimizers: 'reference/search_algorithms/prior.md'
152 | - Multi-Fidelity & Prior Optimizers: 'reference/search_algorithms/multifidelity_prior.md'
153 | - Bayesian Optimization: 'reference/search_algorithms/bayesian_optimization.md'
154 | - API: 'api/' # auto-generated
155 | - Contributing: 'dev_docs/contributing.md'
156 | - Cite: 'citations.md'
157 |
--------------------------------------------------------------------------------
/neps/__init__.py:
--------------------------------------------------------------------------------
1 | from neps.api import run
2 | from neps.optimizers import algorithms
3 | from neps.optimizers.ask_and_tell import AskAndTell
4 | from neps.optimizers.optimizer import SampledConfig
5 | from neps.plot.plot import plot
6 | from neps.plot.tensorboard_eval import tblogger
7 | from neps.space import Categorical, Constant, Float, Integer, SearchSpace
8 | from neps.state import BudgetInfo, Trial
9 | from neps.status.status import status
10 | from neps.utils.files import load_and_merge_yamls as load_yamls
11 |
12 | __all__ = [
13 | "AskAndTell",
14 | "BudgetInfo",
15 | "Categorical",
16 | "Constant",
17 | "Float",
18 | "Integer",
19 | "SampledConfig",
20 | "SearchSpace",
21 | "Trial",
22 | "algorithms",
23 | "load_yamls",
24 | "plot",
25 | "run",
26 | "status",
27 | "tblogger",
28 | ]
29 |
--------------------------------------------------------------------------------
/neps/env.py:
--------------------------------------------------------------------------------
1 | """Environment variable parsing for the state."""
2 |
3 | from __future__ import annotations
4 |
5 | import os
6 | from collections.abc import Callable
7 | from typing import Any, Literal, TypeVar
8 |
9 | T = TypeVar("T")
10 | V = TypeVar("V")
11 |
12 | ENV_VARS_USED: dict[str, tuple[Any, Any]] = {}
13 |
14 |
15 | def get_env(key: str, parse: Callable[[str], T], default: V) -> T | V:
16 | """Get an environment variable or return a default value."""
17 | if (e := os.environ.get(key)) is not None:
18 | value = parse(e)
19 | ENV_VARS_USED[key] = (e, value)
20 | return value
21 |
22 | ENV_VARS_USED[key] = (default, default)
23 | return default
24 |
25 |
26 | def is_nullable(e: str) -> bool:
27 | """Check if an environment variable is nullable."""
28 | return e.lower() in ("none", "n", "null")
29 |
30 |
31 | def yaml_or_json(e: str) -> Literal["yaml", "json"]:
32 | """Check if an environment variable is either yaml or json."""
33 | if e.lower() in ("yaml", "json"):
34 | return e.lower() # type: ignore
35 | raise ValueError(f"Expected 'yaml' or 'json', got '{e}'.")
36 |
37 |
38 | LINUX_FILELOCK_FUNCTION = get_env(
39 | "NEPS_LINUX_FILELOCK_FUNCTION",
40 | parse=str,
41 | default="lockf",
42 | )
43 | MAX_RETRIES_GET_NEXT_TRIAL = get_env(
44 | "NEPS_MAX_RETRIES_GET_NEXT_TRIAL",
45 | parse=int,
46 | default=10,
47 | )
48 | MAX_RETRIES_SET_EVALUATING = get_env(
49 | "NEPS_MAX_RETRIES_SET_EVALUATING",
50 | parse=int,
51 | default=10,
52 | )
53 | MAX_RETRIES_CREATE_LOAD_STATE = get_env(
54 | "NEPS_MAX_RETRIES_CREATE_LOAD_STATE",
55 | parse=int,
56 | default=10,
57 | )
58 | MAX_RETRIES_WORKER_CHECK_SHOULD_STOP = get_env(
59 | "NEPS_MAX_RETRIES_WORKER_CHECK_SHOULD_STOP",
60 | parse=int,
61 | default=3,
62 | )
63 | TRIAL_FILELOCK_POLL = get_env(
64 | "NEPS_TRIAL_FILELOCK_POLL",
65 | parse=float,
66 | default=0.05,
67 | )
68 | TRIAL_FILELOCK_TIMEOUT = get_env(
69 | "NEPS_TRIAL_FILELOCK_TIMEOUT",
70 | parse=lambda e: None if is_nullable(e) else float(e),
71 | default=120,
72 | )
73 | FS_SYNC_GRACE_BASE = get_env(
74 | "NEPS_FS_SYNC_GRACE_BASE",
75 | parse=float,
76 | default=0.00, # Keep it low initially to not punish synced os
77 | )
78 | FS_SYNC_GRACE_INC = get_env(
79 | "NEPS_FS_SYNC_GRACE_INC",
80 | parse=float,
81 | default=0.1,
82 | )
83 |
84 | # NOTE: We want this to be greater than the trials filelock, so that
85 | # anything requesting to just update the trials is more likely to obtain it
86 | # as those operations tend to be faster than something that requires optimizer
87 | # state.
88 | STATE_FILELOCK_POLL = get_env(
89 | "NEPS_STATE_FILELOCK_POLL",
90 | parse=float,
91 | default=0.20,
92 | )
93 | STATE_FILELOCK_TIMEOUT = get_env(
94 | "NEPS_STATE_FILELOCK_TIMEOUT",
95 | parse=lambda e: None if is_nullable(e) else float(e),
96 | default=120,
97 | )
98 | GLOBAL_ERR_FILELOCK_POLL = get_env(
99 | "NEPS_GLOBAL_ERR_FILELOCK_POLL",
100 | parse=float,
101 | default=0.05,
102 | )
103 | GLOBAL_ERR_FILELOCK_TIMEOUT = get_env(
104 | "NEPS_GLOBAL_ERR_FILELOCK_TIMEOUT",
105 | parse=lambda e: None if is_nullable(e) else float(e),
106 | default=120,
107 | )
108 | TRIAL_CACHE_MAX_UPDATES_BEFORE_CONSOLIDATION = get_env(
109 | "NEPS_TRIAL_CACHE_MAX_UPDATES_BEFORE_CONSOLIDATION",
110 | parse=int,
111 | default=10,
112 | )
113 | CONFIG_SERIALIZE_FORMAT: Literal["yaml", "json"] = get_env( # type: ignore
114 | "NEPS_CONFIG_SERIALIZE_FORMAT",
115 | parse=yaml_or_json,
116 | default="yaml",
117 | )
118 |
--------------------------------------------------------------------------------
/neps/exceptions.py:
--------------------------------------------------------------------------------
1 | """Exceptions for NePS that don't belong in a specific module."""
2 |
3 | from __future__ import annotations
4 |
5 | from typing import Any
6 |
7 |
8 | class NePSError(Exception):
9 | """Base class for all NePS exceptions.
10 |
11 | This allows an easier way to catch all NePS exceptions
12 | if we inherit all exceptions from this class.
13 | """
14 |
15 |
16 | class LockFailedError(NePSError):
17 | """Raised when a lock cannot be acquired."""
18 |
19 |
20 | class TrialAlreadyExistsError(NePSError):
21 | """Raised when a trial already exists in the store."""
22 |
23 | def __init__(self, trial_id: str, *args: Any) -> None:
24 | """Initialize the exception with the trial id."""
25 | super().__init__(trial_id, *args)
26 | self.trial_id = trial_id
27 |
28 | def __str__(self) -> str:
29 | return f"Trial with id {self.trial_id} already exists!"
30 |
31 |
32 | class TrialNotFoundError(NePSError):
33 | """Raised when a trial already exists in the store."""
34 |
35 |
36 | class WorkerFailedToGetPendingTrialsError(NePSError):
37 | """Raised when a worker failed to get pending trials."""
38 |
39 |
40 | class WorkerRaiseError(NePSError):
41 | """Raised from a worker when an error is raised.
42 |
43 | Includes additional information on how to recover
44 | """
45 |
--------------------------------------------------------------------------------
/neps/optimizers/__init__.py:
--------------------------------------------------------------------------------
1 | from __future__ import annotations
2 |
3 | from collections.abc import Callable, Mapping
4 | from typing import TYPE_CHECKING, Any, Concatenate, Literal
5 |
6 | from neps.optimizers.algorithms import (
7 | CustomOptimizer,
8 | OptimizerChoice,
9 | PredefinedOptimizers,
10 | determine_optimizer_automatically,
11 | )
12 | from neps.optimizers.optimizer import AskFunction, OptimizerInfo
13 | from neps.utils.common import extract_keyword_defaults
14 |
15 | if TYPE_CHECKING:
16 | from neps.space import SearchSpace
17 |
18 |
19 | def _load_optimizer_from_string(
20 | optimizer: OptimizerChoice | Literal["auto"],
21 | space: SearchSpace,
22 | *,
23 | optimizer_kwargs: Mapping[str, Any] | None = None,
24 | ) -> tuple[AskFunction, OptimizerInfo]:
25 | if optimizer == "auto":
26 | _optimizer = determine_optimizer_automatically(space)
27 | else:
28 | _optimizer = optimizer
29 |
30 | optimizer_build = PredefinedOptimizers.get(_optimizer)
31 | if optimizer_build is None:
32 | raise ValueError(
33 | f"Unrecognized `optimizer` of type {type(optimizer)}."
34 | f" {optimizer}. Available optimizers are:"
35 | f" {PredefinedOptimizers.keys()}"
36 | )
37 |
38 | keywords = extract_keyword_defaults(optimizer_build)
39 | optimizer_kwargs = optimizer_kwargs or {}
40 | opt = optimizer_build(space, **optimizer_kwargs)
41 | info = OptimizerInfo(name=_optimizer, info={**keywords, **optimizer_kwargs})
42 | return opt, info
43 |
44 |
45 | def load_optimizer(
46 | optimizer: (
47 | OptimizerChoice
48 | | Mapping[str, Any]
49 | | tuple[OptimizerChoice, Mapping[str, Any]]
50 | | Callable[Concatenate[SearchSpace, ...], AskFunction]
51 | | CustomOptimizer
52 | | Literal["auto"]
53 | ),
54 | space: SearchSpace,
55 | ) -> tuple[AskFunction, OptimizerInfo]:
56 | match optimizer:
57 | # Predefined string (including "auto")
58 | case str():
59 | return _load_optimizer_from_string(optimizer, space)
60 |
61 | # Predefined string with kwargs
62 | case (opt, kwargs) if isinstance(opt, str):
63 | return _load_optimizer_from_string(opt, space, optimizer_kwargs=kwargs) # type: ignore
64 |
65 | # Mapping with a name
66 | case {"name": name, **_kwargs}:
67 | return _load_optimizer_from_string(name, space, optimizer_kwargs=_kwargs) # type: ignore
68 |
69 | # Provided optimizer initializer
70 | case _ if callable(optimizer):
71 | keywords = extract_keyword_defaults(optimizer)
72 | _optimizer = optimizer(space)
73 | info = OptimizerInfo(name=optimizer.__name__, info=keywords)
74 | return _optimizer, info
75 |
76 | # Custom optimizer, we create it
77 | case CustomOptimizer(initialized=False):
78 | _optimizer = optimizer.create(space)
79 | keywords = extract_keyword_defaults(optimizer.optimizer)
80 | info = OptimizerInfo(
81 | name=optimizer.name, info={**keywords, **optimizer.kwargs}
82 | )
83 | return _optimizer, info
84 |
85 | # Custom (already initialized) optimizer
86 | case CustomOptimizer(initialized=True):
87 | preinit_opt = optimizer.optimizer
88 | info = OptimizerInfo(name=optimizer.name, info=optimizer.kwargs)
89 | return preinit_opt, info # type: ignore
90 |
91 | case _:
92 | raise ValueError(
93 | f"Unrecognized `optimizer` of type {type(optimizer)}."
94 | f" {optimizer}. Must either be a string, callable or"
95 | " a `CustomOptimizer` instance."
96 | )
97 |
--------------------------------------------------------------------------------
/neps/optimizers/acquisition/__init__.py:
--------------------------------------------------------------------------------
1 | from neps.optimizers.acquisition.cost_cooling import cost_cooled_acq
2 | from neps.optimizers.acquisition.pibo import pibo_acquisition
3 | from neps.optimizers.acquisition.weighted_acquisition import WeightedAcquisition
4 |
5 | __all__ = ["WeightedAcquisition", "cost_cooled_acq", "pibo_acquisition"]
6 |
--------------------------------------------------------------------------------
/neps/optimizers/acquisition/cost_cooling.py:
--------------------------------------------------------------------------------
1 | from __future__ import annotations
2 |
3 | from typing import TYPE_CHECKING
4 |
5 | import torch
6 | from botorch.acquisition.logei import partial
7 |
8 | from neps.optimizers.acquisition.weighted_acquisition import WeightedAcquisition
9 |
10 | if TYPE_CHECKING:
11 | from botorch.acquisition import AcquisitionFunction
12 | from botorch.acquisition.analytic import GPyTorchModel
13 | from torch import Tensor
14 |
15 |
16 | def apply_cost_cooling(
17 | acq_values: Tensor,
18 | X: Tensor,
19 | acq: AcquisitionFunction,
20 | cost_model: GPyTorchModel,
21 | alpha: float,
22 | ) -> Tensor:
23 | # NOTE: We expect **positive** costs from model
24 | cost = cost_model.posterior(X).mean
25 | cost = cost.squeeze(dim=-1) if cost_model.num_outputs == 1 else cost.sum(dim=-1)
26 |
27 | if acq._log:
28 | # Take log of both sides, acq is already log scaled
29 | # -- x = acq / cost^alpha
30 | # -- log(x) = log(acq) - alpha * log(cost)
31 | w = alpha * cost.log()
32 | return acq_values - w # type: ignore
33 |
34 | # https://github.com/pytorch/botorch/discussions/2194
35 | w = cost.pow(alpha)
36 | return torch.where(acq_values > 0, acq_values / w, acq_values * w)
37 |
38 |
39 | def cost_cooled_acq(
40 | acq_fn: AcquisitionFunction,
41 | model: GPyTorchModel,
42 | used_max_cost_total_percentage: float,
43 | ) -> WeightedAcquisition:
44 | assert 0 <= used_max_cost_total_percentage <= 1
45 | return WeightedAcquisition(
46 | acq=acq_fn,
47 | apply_weight=partial(
48 | apply_cost_cooling,
49 | cost_model=model,
50 | alpha=1 - used_max_cost_total_percentage,
51 | ),
52 | )
53 |
--------------------------------------------------------------------------------
/neps/optimizers/acquisition/pibo.py:
--------------------------------------------------------------------------------
1 | """# Copyright (c) Meta Platforms, Inc. and affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 | Prior-Guided Acquisition Functions
6 |
7 | References:
8 |
9 | .. [Hvarfner2022]
10 | C. Hvarfner, D. Stoll, A. Souza, M. Lindauer, F. Hutter, L. Nardi. PiBO:
11 | Augmenting Acquisition Functions with User Beliefs for Bayesian Optimization.
12 | ICLR 2022.
13 | """
14 |
15 | from __future__ import annotations
16 |
17 | from typing import TYPE_CHECKING
18 |
19 | from botorch.acquisition.logei import partial
20 |
21 | from neps.optimizers.acquisition.weighted_acquisition import WeightedAcquisition
22 |
23 | if TYPE_CHECKING:
24 | from botorch.acquisition.acquisition import AcquisitionFunction
25 | from torch import Tensor
26 |
27 | from neps.sampling import Prior
28 | from neps.space import ConfigEncoder, Domain
29 |
30 |
31 | def apply_pibo_acquisition_weight(
32 | acq_values: Tensor,
33 | X: Tensor,
34 | acq: AcquisitionFunction,
35 | *,
36 | prior: Prior,
37 | x_domain: Domain | list[Domain] | ConfigEncoder,
38 | prior_exponent: float,
39 | ) -> Tensor:
40 | if acq._log:
41 | weighted_log_probs = prior.log_pdf(X, frm=x_domain) + prior_exponent
42 | return acq_values + weighted_log_probs
43 |
44 | weighted_probs = prior.pdf(X, frm=x_domain).pow(prior_exponent)
45 | return acq_values * weighted_probs
46 |
47 |
48 | def pibo_acquisition(
49 | acq_fn: AcquisitionFunction,
50 | prior: Prior,
51 | prior_exponent: float,
52 | x_domain: Domain | list[Domain] | ConfigEncoder,
53 | ) -> WeightedAcquisition:
54 | return WeightedAcquisition(
55 | acq=acq_fn,
56 | apply_weight=partial(
57 | apply_pibo_acquisition_weight,
58 | prior=prior,
59 | x_domain=x_domain,
60 | prior_exponent=prior_exponent,
61 | ),
62 | )
63 |
--------------------------------------------------------------------------------
/neps/optimizers/grid_search.py:
--------------------------------------------------------------------------------
1 | from __future__ import annotations
2 |
3 | from collections.abc import Mapping
4 | from dataclasses import dataclass
5 | from typing import TYPE_CHECKING, Any
6 |
7 | from neps.optimizers.optimizer import SampledConfig
8 |
9 | if TYPE_CHECKING:
10 | from neps.state import BudgetInfo, Trial
11 |
12 |
13 | @dataclass
14 | class GridSearch:
15 | """Evaluates a fixed list of configurations in order."""
16 |
17 | configs_list: list[dict[str, Any]]
18 | """The list of configurations to evaluate."""
19 |
20 | def __call__(
21 | self,
22 | trials: Mapping[str, Trial],
23 | budget_info: BudgetInfo | None,
24 | n: int | None = None,
25 | ) -> SampledConfig | list[SampledConfig]:
26 | assert n is None, "TODO"
27 | _num_previous_configs = len(trials)
28 | if _num_previous_configs > len(self.configs_list) - 1:
29 | raise ValueError("Grid search exhausted!")
30 |
31 | # TODO: Revisit this. Do we really need to shuffle the configs?
32 | configs = self.configs_list
33 |
34 | config = configs[_num_previous_configs]
35 | config_id = str(_num_previous_configs)
36 | return SampledConfig(config=config, id=config_id, previous_config_id=None)
37 |
--------------------------------------------------------------------------------
/neps/optimizers/models/__init__.py:
--------------------------------------------------------------------------------
1 | from neps.optimizers.models.ftpfn import FTPFNSurrogate
2 | from neps.optimizers.models.gp import make_default_single_obj_gp
3 |
4 | __all__ = ["FTPFNSurrogate", "make_default_single_obj_gp"]
5 |
--------------------------------------------------------------------------------
/neps/optimizers/optimizer.py:
--------------------------------------------------------------------------------
1 | """Optimizer interface.
2 |
3 | By implementing the [`AskFunction`][neps.optimizers.optimizer.AskFunction] protocol,
4 | you can inject your own optimizer into the neps runtime.
5 |
6 | ```python
7 | class MyOpt:
8 |
9 | def __init__(self, space: SearchSpace, ...): ...
10 |
11 | def __call__(
12 | self,
13 | trials: Mapping[str, Trial],
14 | budget_info: BudgetInfo | None,
15 | n: int | None = None,
16 | ) -> SampledConfig | list[SampledConfig]: ...
17 |
18 | neps.run(..., optimizer=MyOpt)
19 |
20 | # Or with optimizer hyperparameters
21 | neps.run(..., optimizer=(MyOpt, {"a": 1, "b": 2}))
22 | ```
23 | """
24 |
25 | from __future__ import annotations
26 |
27 | from abc import abstractmethod
28 | from collections.abc import Mapping
29 | from dataclasses import dataclass
30 | from typing import TYPE_CHECKING, Any, Protocol, TypedDict
31 |
32 | if TYPE_CHECKING:
33 | from neps.state.optimizer import BudgetInfo
34 | from neps.state.trial import Trial
35 |
36 |
37 | class OptimizerInfo(TypedDict):
38 | """Information about the optimizer, usually used for serialization."""
39 |
40 | name: str
41 | """The name of the optimizer."""
42 |
43 | info: Mapping[str, Any]
44 | """Additional information about the optimizer.
45 |
46 | Usually this will be the keyword arguments used to initialize the optimizer.
47 | """
48 |
49 |
50 | @dataclass
51 | class SampledConfig:
52 | id: str
53 | config: Mapping[str, Any]
54 | previous_config_id: str | None = None
55 |
56 |
57 | class AskFunction(Protocol):
58 | """Interface to implement the ask of optimizer."""
59 |
60 | @abstractmethod
61 | def __call__(
62 | self,
63 | trials: Mapping[str, Trial],
64 | budget_info: BudgetInfo | None,
65 | n: int | None = None,
66 | ) -> SampledConfig | list[SampledConfig]:
67 | """Sample a new configuration.
68 |
69 | Args:
70 | trials: All of the trials that are known about.
71 | budget_info: information about the budget constraints.
72 | n: The number of configurations to sample. If you do not support
73 | sampling multiple configurations at once, you should raise
74 | a `ValueError`.
75 |
76 | Returns:
77 | The sampled configuration(s)
78 | """
79 | ...
80 |
--------------------------------------------------------------------------------
/neps/optimizers/random_search.py:
--------------------------------------------------------------------------------
1 | from __future__ import annotations
2 |
3 | from collections.abc import Mapping
4 | from dataclasses import dataclass
5 | from typing import TYPE_CHECKING
6 |
7 | from neps.optimizers.optimizer import SampledConfig
8 |
9 | if TYPE_CHECKING:
10 | from neps.sampling import Sampler
11 | from neps.space import ConfigEncoder, SearchSpace
12 | from neps.state import BudgetInfo, Trial
13 |
14 |
15 | @dataclass
16 | class RandomSearch:
17 | """A simple random search optimizer."""
18 |
19 | space: SearchSpace
20 | encoder: ConfigEncoder
21 | sampler: Sampler
22 |
23 | def __call__(
24 | self,
25 | trials: Mapping[str, Trial],
26 | budget_info: BudgetInfo | None,
27 | n: int | None = None,
28 | ) -> SampledConfig | list[SampledConfig]:
29 | n_trials = len(trials)
30 | _n = 1 if n is None else n
31 | configs = self.sampler.sample(_n, to=self.encoder.domains)
32 |
33 | config_dicts = self.encoder.decode(configs)
34 | for config in config_dicts:
35 | config.update(self.space.constants)
36 | if self.space.fidelity is not None:
37 | config.update(
38 | {
39 | key: value.upper
40 | for key, value in self.space.fidelities.items()
41 | if key not in config
42 | }
43 | )
44 |
45 | if n is None:
46 | config = config_dicts[0]
47 | config_id = str(n_trials + 1)
48 | return SampledConfig(config=config, id=config_id, previous_config_id=None)
49 |
50 | return [
51 | SampledConfig(
52 | config=config,
53 | id=str(n_trials + i + 1),
54 | previous_config_id=None,
55 | )
56 | for i, config in enumerate(config_dicts)
57 | ]
58 |
--------------------------------------------------------------------------------
/neps/optimizers/utils/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/automl/neps/0ef3731fa5049336e9a2bb61be7093fc902a96ff/neps/optimizers/utils/__init__.py
--------------------------------------------------------------------------------
/neps/optimizers/utils/grid.py:
--------------------------------------------------------------------------------
1 | from __future__ import annotations
2 |
3 | from itertools import product
4 | from typing import Any
5 |
6 | import torch
7 |
8 | from neps.space import Categorical, Constant, Domain, Float, Integer, SearchSpace
9 |
10 |
11 | def make_grid(
12 | space: SearchSpace,
13 | *,
14 | size_per_numerical_hp: int = 10,
15 | ignore_fidelity: bool = True,
16 | ) -> list[dict[str, Any]]:
17 | """Get a grid of configurations from the search space.
18 |
19 | For [`Float`][neps.space.Float] and [`Integer`][neps.space.Integer]
20 | the parameter `size_per_numerical_hp=` is used to determine a grid.
21 |
22 | For [`Categorical`][neps.space.Categorical]
23 | hyperparameters, we include all the choices in the grid.
24 |
25 | For [`Constant`][neps.space.Constant] hyperparameters,
26 | we include the constant value in the grid.
27 |
28 | Args:
29 | size_per_numerical_hp: The size of the grid for each numerical hyperparameter.
30 |
31 | Returns:
32 | A list of configurations from the search space.
33 | """
34 | param_ranges: dict[str, list[Any]] = {}
35 | for name, hp in space.items():
36 | match hp:
37 | case Categorical():
38 | param_ranges[name] = list(hp.choices)
39 | case Constant():
40 | param_ranges[name] = [hp.value]
41 | case Integer() | Float():
42 | if hp.is_fidelity and ignore_fidelity:
43 | param_ranges[name] = [hp.upper]
44 | continue
45 |
46 | if hp.domain.cardinality is None:
47 | steps = size_per_numerical_hp
48 | else:
49 | steps = min(size_per_numerical_hp, hp.domain.cardinality)
50 |
51 | xs = torch.linspace(0, 1, steps=steps)
52 | numeric_values = hp.domain.cast(xs, frm=Domain.unit_float())
53 | uniq_values = torch.unique(numeric_values).tolist()
54 | param_ranges[name] = uniq_values
55 | case _:
56 | raise NotImplementedError(f"Unknown Parameter type: {type(hp)}\n{hp}")
57 | values = product(*param_ranges.values())
58 | keys = list(space.keys())
59 |
60 | return [dict(zip(keys, p, strict=False)) for p in values]
61 |
--------------------------------------------------------------------------------
/neps/optimizers/utils/initial_design.py:
--------------------------------------------------------------------------------
1 | from __future__ import annotations
2 |
3 | from collections.abc import Mapping
4 | from typing import TYPE_CHECKING, Any, Literal
5 |
6 | import torch
7 |
8 | from neps.sampling import Prior, Sampler
9 |
10 | if TYPE_CHECKING:
11 | from neps.space import ConfigEncoder
12 | from neps.space.parameters import Parameter
13 |
14 |
15 | def make_initial_design(
16 | *,
17 | parameters: Mapping[str, Parameter],
18 | encoder: ConfigEncoder,
19 | sampler: Literal["sobol", "prior", "uniform"] | Sampler,
20 | sample_size: int | Literal["ndim"] | None = "ndim",
21 | sample_prior_first: bool = True,
22 | seed: torch.Generator | None = None,
23 | ) -> list[dict[str, Any]]:
24 | """Generate the initial design of the optimization process.
25 |
26 | Args:
27 | space: The search space to use.
28 | encoder: The encoder to use for encoding/decoding configurations.
29 | sampler: The sampler to use for the initial design.
30 |
31 | If set to "sobol", a Sobol sequence will be used.
32 | If set to "uniform", a uniform random sampler will be used.
33 | If set to "prior", a prior sampler will be used, based on the defaults,
34 | and confidence scores of the hyperparameters.
35 | If set to a custom sampler, the sampler will be used directly.
36 |
37 | sample_size:
38 | The number of configurations to sample.
39 |
40 | If "ndim", the number of configs will be equal to the number of dimensions.
41 | If None, no configurations will be sampled.
42 |
43 | sample_prior_first: Whether to sample the prior configuration first.
44 | seed: The seed to use for the random number generation.
45 |
46 | """
47 | configs: list[dict[str, Any]] = []
48 | if sample_prior_first:
49 | configs.append(
50 | {
51 | name: p.prior if p.prior is not None else p.center
52 | for name, p in parameters.items()
53 | }
54 | )
55 |
56 | ndims = len(parameters)
57 | if sample_size == "ndim":
58 | sample_size = ndims
59 | elif sample_size is not None and not sample_size > 0:
60 | raise ValueError(
61 | "The sample size should be a positive integer if passing an int."
62 | )
63 |
64 | if sample_size is not None:
65 | match sampler:
66 | case "sobol":
67 | sampler = Sampler.sobol(ndim=ndims)
68 | case "uniform":
69 | sampler = Sampler.uniform(ndim=ndims)
70 | case "prior":
71 | sampler = Prior.from_parameters(parameters)
72 | case _:
73 | pass
74 |
75 | encoded_configs = sampler.sample(sample_size * 2, to=encoder.domains, seed=seed)
76 | uniq_x = torch.unique(encoded_configs, dim=0)
77 | sample_configs = encoder.decode(uniq_x[:sample_size])
78 | configs.extend(sample_configs)
79 |
80 | return configs
81 |
--------------------------------------------------------------------------------
/neps/optimizers/utils/multiobjective/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/automl/neps/0ef3731fa5049336e9a2bb61be7093fc902a96ff/neps/optimizers/utils/multiobjective/__init__.py
--------------------------------------------------------------------------------
/neps/optimizers/utils/multiobjective/epsnet.py:
--------------------------------------------------------------------------------
1 | """Implements an epsilon-net based multi-objective sort.
2 | Source: https://github.com/syne-tune/syne-tune/blob/main/syne_tune/optimizer/schedulers/multiobjective/non_dominated_priority.py
3 | Proposed in the paper: https://arxiv.org/pdf/2106.12639
4 | """
5 |
6 | from __future__ import annotations
7 |
8 | import numpy as np
9 |
10 |
11 | def pareto_efficient(X: np.ndarray) -> np.ndarray:
12 | """
13 | Evaluates for each allocation in the provided array whether it is Pareto efficient.
14 | The costs are assumed to be improved by lowering them (eg lower is better).
15 |
16 | Parameters
17 | ----------
18 | X: np.ndarray [N, D]
19 | The allocations to check where N is the number of allocations and D the number of
20 | costs per allocation.
21 |
22 | Returns
23 | -------
24 | np.ndarray [N]
25 | A boolean array, indicating for each allocation whether it is Pareto efficient.
26 | """
27 | # First, we assume that all allocations are Pareto efficient, i.e. not dominated
28 | mask = np.ones(X.shape[0], dtype=bool)
29 | # Then, we iterate over all allocations A and check which are dominated by then
30 | # current allocation A. If it is, we don't need to check it against another
31 | # allocation.
32 | for i, allocation in enumerate(X):
33 | # Only consider allocation if it hasn't been dominated yet
34 | if mask[i]:
35 | # An allocation is dominated by A if all costs are equal or lower
36 | # and at least one cost is strictly lower. Using that definition,
37 | # A cannot be dominated by itself.
38 | dominated = np.all(allocation <= X[mask], axis=1) * np.any(
39 | allocation < X[mask], axis=1
40 | )
41 | mask[mask] = ~dominated
42 |
43 | return mask
44 |
45 |
46 | def compute_epsilon_net(X: np.ndarray, dim: int | None = None) -> np.ndarray:
47 | """
48 | Outputs an order of the items in the provided array such that the items are
49 | spaced well. This means that after choosing a seed item, the next item is
50 | chosen to be the farthest from the seed item. The third item is then chosen
51 | to maximize the distance to the existing points and so on.
52 |
53 | This algorithm is taken from "Nearest-Neighbor Searching and Metric Space Dimensions"
54 | (Clarkson, 2005, p.17).
55 |
56 | Parameters
57 | ----------
58 | X: np.ndarray [N, D]
59 | The items to sparsify where N is the number of items and D their dimensionality.
60 | dim: Optional[int], default: None
61 | The index of the dimension which to use to choose the seed item.
62 | If ``None``, an item is chosen at random, otherwise the item with the
63 | lowest value in the specified dimension is used.
64 |
65 | Returns
66 | -------
67 | np.ndarray [N]
68 | A list of item indices, defining a sparsified order of the items.
69 | """
70 | indices = set(range(X.shape[0]))
71 |
72 | # Choose the seed item according to dim
73 | if dim is None:
74 | initial_index = np.random.choice(X.shape[0])
75 | else:
76 | initial_index = np.argmin(X, axis=0)[dim]
77 |
78 | # Initialize the order
79 | order = [initial_index]
80 | indices.remove(initial_index)
81 |
82 | # Iterate until all models have been chosen
83 | while indices:
84 | # Get the distance to all items that have already been chosen
85 | ordered_indices = list(indices)
86 | diff = X[ordered_indices][:, None, :].repeat(len(order), axis=1) - X[order]
87 | min_distances = np.linalg.norm(diff, axis=-1).min(-1)
88 |
89 | # Then, choose the one with the maximum distance to all points
90 | choice = ordered_indices[min_distances.argmax()]
91 | order.append(choice)
92 | indices.remove(choice)
93 |
94 | # convert argsort indices to rank
95 | ranks = np.empty(len(order), dtype=int)
96 | for rank, i in enumerate(order):
97 | ranks[i] = rank
98 | return np.array(ranks)
99 |
100 |
101 | def nondominated_sort(
102 | X: np.ndarray,
103 | dim: int | None = None,
104 | max_items: int | None = None,
105 | *,
106 | flatten: bool = True,
107 | ) -> list[int] | list[list[int]]:
108 | """
109 | Performs a multi-objective sort by iteratively computing the Pareto front
110 | and sparsifying the items within the Pareto front. This is a non-dominated sort
111 | leveraging an epsilon-net.
112 |
113 | Parameters
114 | ----------
115 | X: np.ndarray [N, D]
116 | The multi-dimensional items to sort.
117 | dim: Optional[int], default: None
118 | The feature (metric) to prefer when ranking items within the Pareto front.
119 | If ``None``, items are chosen randomly.
120 | max_items: Optional[int], default: None
121 | The maximum number of items that should be returned.
122 | When this is ``None``, all items are sorted.
123 | flatten: bool, default: True
124 | Whether to flatten the resulting array.
125 |
126 | Returns
127 | -------
128 | Union[List[int], List[List[int]]]
129 | The indices of the sorted items, either globally or within each of the
130 | Pareto front depending on the value of ``flatten``.
131 | """
132 | remaining = np.arange(X.shape[0])
133 | indices = []
134 | num_items = 0
135 |
136 | # Iterate until max_items are reached or there are no items left
137 | while remaining.size > 0 and (max_items is None or num_items < max_items):
138 | # Compute the Pareto front and sort the items within
139 | pareto_mask = pareto_efficient(X[remaining])
140 | pareto_front = remaining[pareto_mask]
141 | pareto_order = compute_epsilon_net(X[pareto_front], dim=dim)
142 |
143 | # Add order to the indices
144 | indices.append(pareto_front[pareto_order].tolist())
145 | num_items += len(pareto_front)
146 |
147 | # Remove items in the Pareto front from the remaining items
148 | remaining = remaining[~pareto_mask]
149 |
150 | # Restrict the number of items returned and optionally flatten
151 | if max_items is not None:
152 | limit = max_items - sum(len(x) for x in indices[:-1])
153 | indices[-1] = indices[-1][:limit]
154 | if not indices[-1]:
155 | indices = indices[:-1]
156 |
157 | if flatten:
158 | return [i for ix in indices for i in ix]
159 | return indices
160 |
--------------------------------------------------------------------------------
/neps/plot/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/automl/neps/0ef3731fa5049336e9a2bb61be7093fc902a96ff/neps/plot/__init__.py
--------------------------------------------------------------------------------
/neps/plot/__main__.py:
--------------------------------------------------------------------------------
1 | """Plot incumbent from a root directory of a neps.run.
2 |
3 | Usage:
4 | python -m neps.plot [-h] [--scientific_mode] [--key_to_extract] [--benchmarks]
5 | [--algorithms] [--consider_continuations] [--n_workers] [--x_range] [--log_x]
6 | [--log_y] [--filename] [--extension] [--dpi]
7 | root_directory
8 |
9 | Positional arguments:
10 | root_directory The root directory given to neps.run
11 |
12 | Optional arguments:
13 | -h, --help Show this help message and exit
14 | --scientific_mode If true, plot from a tree-structured root_directory: benchmark={}/algorithm={}/seed={}
15 | --key_to_extract The metric to be used on the x-axis (if active, make sure evaluate_pipeline returns the metric in the info_dict)
16 | --benchmarks List of benchmarks to plot
17 | --algorithms List of algorithms to plot
18 | --consider_continuations If true, toggle calculation of continuation costs
19 | --n_workers Number of parallel processes of neps.run
20 | --x_range Bound x-axis (e.g. 1 10)
21 | --log_x If true, toggle logarithmic scale on the x-axis
22 | --log_y If true, toggle logarithmic scale on the y-axis
23 | --filename Filename
24 | --extension Image format
25 | --dpi Image resolution
26 |
27 |
28 | Note:
29 | We have to use the __main__.py construct due to the issues explained in
30 | https://stackoverflow.com/questions/43393764/python-3-6-project-structure-leads-to-runtimewarning
31 |
32 | """ # noqa: E501
33 |
34 | from __future__ import annotations
35 |
36 | import argparse
37 | import logging
38 | from pathlib import Path
39 |
40 | from .plot import plot
41 |
42 | # fmt: off
43 | parser = argparse.ArgumentParser(
44 | prog="python -m neps.plot",
45 | description="Plot incumbent from a root directory of a neps.run"
46 | )
47 | parser.add_argument(
48 | "root_directory",
49 | type=Path,
50 | help="The root directory given to neps.run"
51 | )
52 | parser.add_argument(
53 | "--scientific_mode",
54 | action="store_true",
55 | help="If true, plot from a tree-structured root_directory"
56 | )
57 | parser.add_argument(
58 | "--key_to_extract",
59 | help="The metric to be used on the x-axis (if "
60 | "active, make sure evaluate_pipeline returns "
61 | "the metric in the info_dict)")
62 | parser.add_argument(
63 | "--benchmarks",
64 | default=["example"],
65 | nargs="+",
66 | help="List of benchmarks to plot",
67 | )
68 | parser.add_argument(
69 | "--algorithms",
70 | default=["neps"],
71 | nargs="+",
72 | help="List of algorithms to plot",
73 | )
74 | parser.add_argument(
75 | "--consider_continuations",
76 | action="store_true",
77 | help="If true, toggle calculation of continuation costs"
78 | )
79 | parser.add_argument(
80 | "--n_workers",
81 | type=int,
82 | default=1,
83 | help="Number of parallel processes of neps.run",
84 | )
85 | parser.add_argument(
86 | "--x_range",
87 | nargs="+",
88 | type=float,
89 | help="Bound x-axis (e.g. 1 10)"
90 | )
91 | parser.add_argument(
92 | "--log_x",
93 | action="store_true",
94 | help="If true, toggle logarithmic scale on the x-axis"
95 | )
96 | parser.add_argument(
97 | "--log_y",
98 | action="store_true",
99 | help="If true, toggle logarithmic scale on the y-axis"
100 | )
101 | parser.add_argument(
102 | "--filename",
103 | default="incumbent_trajectory",
104 | help="Filename",
105 | )
106 | parser.add_argument(
107 | "--extension",
108 | default="png",
109 | choices=["png", "pdf"],
110 | help="Image format",
111 | )
112 | parser.add_argument(
113 | "--dpi",
114 | type=int,
115 | default=100,
116 | help="Image resolution",
117 | )
118 |
119 | args = parser.parse_args()
120 | # fmt: on
121 |
122 | logging.basicConfig(level=logging.WARN)
123 | if args.x_range is not None and len(args.x_range) == 2:
124 | args.x_range = tuple(args.x_range)
125 | plot(
126 | root_directory=args.root_directory,
127 | scientific_mode=args.scientific_mode,
128 | key_to_extract=args.key_to_extract,
129 | benchmarks=args.benchmarks,
130 | algorithms=args.algorithms,
131 | consider_continuations=args.consider_continuations,
132 | n_workers=args.n_workers,
133 | x_range=args.x_range,
134 | log_x=args.log_x,
135 | log_y=args.log_y,
136 | filename=args.filename,
137 | extension=args.extension,
138 | dpi=args.dpi,
139 | )
140 |
--------------------------------------------------------------------------------
/neps/plot/plot.py:
--------------------------------------------------------------------------------
1 | """Plot results of a neural pipeline search run."""
2 |
3 | from __future__ import annotations
4 |
5 | import errno
6 | import logging
7 | import os
8 | from pathlib import Path
9 |
10 | import numpy as np
11 |
12 | from .plotting import _get_fig_and_axs, _map_axs, _plot_incumbent, _save_fig, _set_legend
13 | from .read_results import process_seed
14 |
15 |
16 | def plot( # noqa: C901, PLR0913
17 | root_directory: str | Path,
18 | *,
19 | scientific_mode: bool = False,
20 | key_to_extract: str | None = None,
21 | benchmarks: list[str] | None = None,
22 | algorithms: list[str] | None = None,
23 | consider_continuations: bool = False,
24 | n_workers: int = 1,
25 | x_range: tuple | None = None,
26 | log_x: bool = False,
27 | log_y: bool = True,
28 | filename: str = "incumbent_trajectory",
29 | extension: str = "png",
30 | dpi: int = 100,
31 | ) -> None:
32 | """Plot results of a neural pipeline search run.
33 |
34 | Args:
35 | root_directory: The directory with neps results (see below).
36 | scientific_mode: If true, plot from a tree-structured root_directory:
37 | benchmark={}/algorithm={}/seed={}
38 | key_to_extract: The metric to be used on the x-axis
39 | (if active, make sure evaluate_pipeline returns the metric in the info_dict)
40 | benchmarks: List of benchmarks to plot
41 | algorithms: List of algorithms to plot
42 | consider_continuations: If true, toggle calculation of continuation costs
43 | n_workers: Number of parallel processes of neps.run
44 | x_range: Bound x-axis (e.g. 1 10)
45 | log_x: If true, toggle logarithmic scale on the x-axis
46 | log_y: If true, toggle logarithmic scale on the y-axis
47 | filename: Filename
48 | extension: Image format
49 | dpi: Image resolution
50 |
51 | Raises:
52 | FileNotFoundError: If the data to be plotted is not present.
53 | """
54 | logger = logging.getLogger("neps")
55 | logger.info(f"Starting neps.plot using working directory {root_directory}")
56 |
57 | if benchmarks is None:
58 | benchmarks = ["example"]
59 | if algorithms is None:
60 | algorithms = ["neps"]
61 |
62 | logger.info(
63 | f"Processing {len(benchmarks)} benchmark(s) and {len(algorithms)} algorithm(s)..."
64 | )
65 |
66 | ncols = 1 if len(benchmarks) == 1 else 2
67 | nrows = np.ceil(len(benchmarks) / ncols).astype(int)
68 |
69 | fig, axs = _get_fig_and_axs(nrows=nrows, ncols=ncols)
70 |
71 | base_path = Path(root_directory)
72 |
73 | for benchmark_idx, benchmark in enumerate(benchmarks):
74 | if scientific_mode:
75 | _base_path = base_path / f"benchmark={benchmark}"
76 | if not _base_path.is_dir():
77 | raise FileNotFoundError(
78 | errno.ENOENT, os.strerror(errno.ENOENT), _base_path
79 | )
80 | else:
81 | _base_path = None
82 |
83 | for algorithm in algorithms:
84 | seeds = [None]
85 | if _base_path is not None:
86 | assert scientific_mode
87 | _path = _base_path / f"algorithm={algorithm}"
88 | if not _path.is_dir():
89 | raise FileNotFoundError(
90 | errno.ENOENT, os.strerror(errno.ENOENT), _path
91 | )
92 |
93 | seeds = sorted(os.listdir(_path)) # type: ignore
94 | else:
95 | _path = None
96 |
97 | incumbents = []
98 | costs = []
99 | max_costs = []
100 | for seed in seeds:
101 | incumbent, cost, max_cost = process_seed(
102 | path=_path if _path is not None else base_path,
103 | seed=seed,
104 | key_to_extract=key_to_extract,
105 | consider_continuations=consider_continuations,
106 | n_workers=n_workers,
107 | )
108 | incumbents.append(incumbent)
109 | costs.append(cost)
110 | max_costs.append(max_cost)
111 |
112 | is_last_row = benchmark_idx >= (nrows - 1) * ncols
113 | is_first_column = benchmark_idx % ncols == 0
114 | xlabel = "Evaluations" if key_to_extract is None else key_to_extract.upper()
115 | _plot_incumbent(
116 | ax=_map_axs(
117 | axs,
118 | benchmark_idx,
119 | len(benchmarks),
120 | ncols,
121 | ),
122 | x=costs,
123 | y=incumbents,
124 | scale_x=max(max_costs) if key_to_extract == "fidelity" else None,
125 | title=benchmark if scientific_mode else None,
126 | xlabel=xlabel if is_last_row else None,
127 | ylabel="Best error" if is_first_column else None,
128 | log_x=log_x,
129 | log_y=log_y,
130 | x_range=x_range,
131 | label=algorithm,
132 | )
133 |
134 | if scientific_mode:
135 | _set_legend(
136 | fig,
137 | axs,
138 | benchmarks=benchmarks,
139 | algorithms=algorithms,
140 | nrows=nrows,
141 | ncols=ncols,
142 | )
143 | _save_fig(fig, output_dir=base_path, filename=filename, extension=extension, dpi=dpi)
144 | logger.info(f"Saved to '{base_path}/{filename}.{extension}'")
145 |
--------------------------------------------------------------------------------
/neps/plot/plotting.py:
--------------------------------------------------------------------------------
1 | """Plotting functions for incumbent trajectory plots."""
2 |
3 | from __future__ import annotations
4 |
5 | from pathlib import Path
6 | from typing import Any
7 |
8 | import matplotlib.axes
9 | import matplotlib.figure
10 | import matplotlib.pyplot as plt
11 | import numpy as np
12 | import pandas as pd
13 | import seaborn as sns
14 | from scipy import stats
15 |
16 | _map_axs = (
17 | lambda axs, idx, length, ncols: axs
18 | if length == 1
19 | else (axs[idx] if length == ncols else axs[idx // ncols][idx % ncols])
20 | )
21 |
22 |
23 | def _set_general_plot_style() -> None:
24 | plt.rcParams.update(
25 | {
26 | "text.usetex": False, # True,
27 | # "pgf.texsystem": "pdflatex",
28 | # "pgf.rcfonts": False,
29 | # "font.family": "serif",
30 | # "font.serif": [],
31 | # "font.sans-serif": [],
32 | # "font.monospace": [],
33 | "font.size": "10.90",
34 | "legend.fontsize": "9.90",
35 | "xtick.labelsize": "small",
36 | "ytick.labelsize": "small",
37 | "legend.title_fontsize": "small",
38 | # "bottomlabel.weight": "normal",
39 | # "toplabel.weight": "normal",
40 | # "leftlabel.weight": "normal",
41 | # "tick.labelweight": "normal",
42 | # "title.weight": "normal",
43 | # "pgf.preamble": r"""
44 | # \usepackage[T1]{fontenc}
45 | # \usepackage[utf8x]{inputenc}
46 | # \usepackage{microtype}
47 | # """,
48 | }
49 | )
50 |
51 |
52 | def _get_fig_and_axs(
53 | nrows: int = 1,
54 | ncols: int = 1,
55 | ) -> tuple[matplotlib.figure.Figure, matplotlib.axes.Axes]:
56 | _set_general_plot_style()
57 |
58 | figsize = (4 * ncols, 3 * nrows)
59 |
60 | fig, axs = plt.subplots(
61 | nrows=nrows,
62 | ncols=ncols,
63 | figsize=figsize,
64 | )
65 |
66 | fig.tight_layout(pad=2.0, h_pad=2.5) # type: ignore
67 | sns.despine(fig)
68 |
69 | return fig, axs # type: ignore
70 |
71 |
72 | def _plot_incumbent(
73 | ax: matplotlib.axes.Axes,
74 | x: list | np.ndarray,
75 | y: list | np.ndarray,
76 | *,
77 | scale_x: float | None,
78 | xlabel: str | None = None,
79 | ylabel: str | None = None,
80 | title: str | None = None,
81 | log_x: bool = False,
82 | log_y: bool = False,
83 | x_range: tuple | None = None,
84 | **plotting_kwargs: Any,
85 | ) -> None:
86 | df = _interpolate_time(incumbents=y, costs=x, x_range=x_range, scale_x=scale_x)
87 | df = _df_to_x_range(df, x_range=x_range)
88 |
89 | x = df.index # type: ignore
90 | y_mean = df.mean(axis=1).to_numpy() # type: ignore
91 | ddof = 0 if len(df.columns) == 1 else 1
92 | std_error = stats.sem(df.values, axis=1, ddof=ddof)
93 |
94 | ax.plot(x, y_mean, linestyle="-", linewidth=0.7, **plotting_kwargs)
95 |
96 | ax.fill_between(x, y_mean - std_error, y_mean + std_error, alpha=0.2)
97 |
98 | ax.set_xlim(auto=True)
99 |
100 | if title is not None:
101 | ax.set_title(title, fontsize=20)
102 | if xlabel is not None:
103 | ax.set_xlabel(xlabel, fontsize=18, color=(0, 0, 0, 0.69))
104 | if ylabel is not None:
105 | ax.set_ylabel(ylabel, fontsize=18, color=(0, 0, 0, 0.69))
106 | if log_x:
107 | ax.set_xscale("log") # type: ignore
108 | if log_y:
109 | ax.set_yscale("symlog") # type: ignore
110 | if x_range is not None:
111 | ax.set_xlim(*x_range)
112 | ax.set_ylim(auto=True)
113 |
114 | # Black with some alpha
115 | ax.tick_params(axis="both", which="major", labelsize=18, labelcolor=(0, 0, 0, 0.69))
116 | ax.grid(visible=True, which="both", ls="-", alpha=0.8)
117 |
118 |
119 | def _interpolate_time(
120 | incumbents: list | np.ndarray,
121 | costs: list | np.ndarray,
122 | x_range: tuple | None = None,
123 | scale_x: float | None = None,
124 | ) -> pd.DataFrame:
125 | if isinstance(incumbents, list):
126 | incumbents = np.array(incumbents)
127 | if isinstance(costs, list):
128 | costs = np.array(costs)
129 |
130 | df_dict = {}
131 |
132 | for i, _ in enumerate(incumbents):
133 | _seed_info = pd.Series(incumbents[i], index=np.cumsum(costs[i]))
134 | df_dict[f"seed{i}"] = _seed_info
135 | df = pd.DataFrame.from_dict(df_dict)
136 |
137 | # important step to plot func evals on x-axis
138 | df.index = df.index if scale_x is None else df.index.to_numpy() / scale_x
139 |
140 | if x_range is not None:
141 | min_b, max_b = x_range
142 | new_entry = {c: np.nan for c in df.columns}
143 | _df = pd.DataFrame.from_dict(new_entry, orient="index").T
144 | _df.index = [min_b]
145 | df = pd.concat((df, _df)).sort_index()
146 | new_entry = {c: np.nan for c in df.columns}
147 | _df = pd.DataFrame.from_dict(new_entry, orient="index").T
148 | _df.index = [max_b]
149 | df = pd.concat((df, _df)).sort_index()
150 |
151 | df = df.fillna(method="backfill", axis=0).fillna(method="ffill", axis=0)
152 | if x_range is not None:
153 | df = df.query(f"{x_range[0]} <= index <= {x_range[1]}")
154 |
155 | return df
156 |
157 |
158 | def _df_to_x_range(df: pd.DataFrame, x_range: tuple | None = None) -> pd.DataFrame:
159 | x_max = np.inf if x_range is None else int(x_range[-1])
160 | new_entry = {c: np.nan for c in df.columns}
161 | _df = pd.DataFrame.from_dict(new_entry, orient="index").T
162 | _df.index = [x_max]
163 | df = pd.concat((df, _df)).sort_index()
164 | return df.fillna(method="backfill", axis=0).fillna(method="ffill", axis=0)
165 |
166 |
167 | def _set_legend(
168 | fig: matplotlib.figure.Figure,
169 | axs: matplotlib.axes.Axes,
170 | benchmarks: list[str],
171 | algorithms: list[str],
172 | nrows: int,
173 | ncols: int,
174 | ) -> None:
175 | bbox_y_mapping = {
176 | 1: -0.22,
177 | 2: -0.11,
178 | 3: -0.07,
179 | 4: -0.05,
180 | 5: -0.04,
181 | }
182 | anchor_y = bbox_y_mapping[nrows]
183 | bbox_to_anchor = (0.5, anchor_y)
184 |
185 | handles, labels = _map_axs(axs, 0, len(benchmarks), ncols).get_legend_handles_labels()
186 |
187 | legend = fig.legend(
188 | handles,
189 | labels,
190 | fontsize="large",
191 | loc="lower center",
192 | bbox_to_anchor=bbox_to_anchor,
193 | ncol=len(algorithms),
194 | frameon=True,
195 | )
196 |
197 | for legend_item in legend.legend_handles:
198 | if legend_item is not None:
199 | legend_item.set_linewidth(2.0) # type: ignore
200 |
201 |
202 | def _save_fig(
203 | fig: matplotlib.figure.Figure,
204 | output_dir: Path | str,
205 | filename: str = "incumbent_trajectory",
206 | extension: str = "png",
207 | dpi: int = 100,
208 | ) -> None:
209 | output_dir = Path(output_dir)
210 | output_dir.mkdir(parents=True, exist_ok=True)
211 | fig.savefig(
212 | output_dir / f"{filename}.{extension}",
213 | bbox_inches="tight",
214 | dpi=dpi,
215 | )
216 |
--------------------------------------------------------------------------------
/neps/plot/read_results.py:
--------------------------------------------------------------------------------
1 | """Utility functions for reading and processing results."""
2 |
3 | from __future__ import annotations
4 |
5 | from pathlib import Path
6 |
7 | import neps
8 |
9 |
10 | def process_seed(
11 | *,
12 | path: str | Path,
13 | seed: str | int | None,
14 | key_to_extract: str | None = None, # noqa: ARG001
15 | consider_continuations: bool = False, # noqa: ARG001
16 | n_workers: int = 1, # noqa: ARG001
17 | ) -> tuple[list[float], list[float], float]:
18 | """Reads and processes data per seed."""
19 | path = Path(path)
20 | if seed is not None:
21 | path = path / str(seed) / "neps_root_directory"
22 |
23 | _fulldf, _summary = neps.status(path, print_summary=False)
24 | raise NotImplementedError(
25 | "I'm sorry, I broke this. We now dump all the information neps has available"
26 | " into the above dataframe `fulldf`."
27 | )
28 | # > sorted_stats = sorted(sorted(stats.items()), key=lambda x: len(x[0]))
29 | # > stats = OrderedDict(sorted_stats)
30 |
31 | # > # max_cost only relevant for scaling x-axis when using fidelity on the x-axis
32 | # > max_cost: float = -1.0
33 | # > if key_to_extract == "fidelity":
34 | # > # TODO(eddiebergman): This can crash for a number of reasons, namely if the
35 | # > # config crased and it's result is an error, or if the `"info_dict"` and/or
36 | # > # `key_to_extract` doesn't exist
37 | # > max_cost = max(s.result["info_dict"][key_to_extract] for s in stats.values())
38 |
39 | # > global_start = stats[min(stats.keys())].metadata["time_sampled"]
40 |
41 | # > def get_cost(idx: str) -> float:
42 | # > if key_to_extract is not None:
43 | # > # TODO(eddiebergman): This can crash for a number of reasons, namely if
44 | # > # the config crased and it's result is an error, or if the `"info_dict"`
45 | # > # and/or `key_to_extract` doesn't exist
46 | # > return float(stats[idx].result["info_dict"][key_to_extract])
47 |
48 | # > return 1.0
49 |
50 | # > losses = []
51 | # > costs = []
52 |
53 | # > for config_id, config_result in stats.items():
54 | # > config_cost = get_cost(config_id)
55 | # > if consider_continuations:
56 | # > if n_workers == 1:
57 | # > # calculates continuation costs for MF algorithms NOTE: assumes that
58 | # > # all recorded evaluations are black-box evaluations where
59 | # > # continuations or freeze-thaw was not accounted for during opt
60 | # > if "previous_config_id" in config_result.metadata:
61 | # > previous_config_id = config_result.metadata["previous_config_id"]
62 | # > config_cost -= get_cost(previous_config_id)
63 | # > else:
64 | # > config_cost = config_result.metadata["time_end"] - global_start
65 |
66 | # > # TODO(eddiebergman): Assumes it never crashed and there's a
67 | # > # objective_to_minimize available,not fixing now but it should be addressed
68 | # > losses.append(config_result.result["objective_to_minimize"]) # type: ignore
69 | # > costs.append(config_cost)
70 |
71 | # > return list(np.minimum.accumulate(losses)), costs, max_cost
72 |
--------------------------------------------------------------------------------
/neps/plot/tensorboard_eval.py:
--------------------------------------------------------------------------------
1 | """The tblogger module provides a simplified interface for logging to TensorBoard."""
2 |
3 | from __future__ import annotations
4 |
5 | import logging
6 | import time
7 | from collections.abc import Mapping
8 | from pathlib import Path
9 | from typing import TYPE_CHECKING, Any, ClassVar
10 |
11 | from torch.utils.tensorboard.writer import SummaryWriter
12 |
13 | from neps.runtime import (
14 | get_in_progress_trial,
15 | get_workers_neps_state,
16 | register_notify_trial_end,
17 | )
18 | from neps.status.status import status
19 | from neps.utils.common import get_initial_directory
20 |
21 | if TYPE_CHECKING:
22 | from neps.state.trial import Trial
23 |
24 | logger = logging.getLogger(__name__)
25 |
26 |
27 | class tblogger: # noqa: N801
28 | """The tblogger class provides a simplified interface for logging to tensorboard."""
29 |
30 | config_id: ClassVar[str | None] = None
31 | config: ClassVar[Mapping[str, Any] | None] = None
32 | config_working_directory: ClassVar[Path | None] = None
33 | optimizer_dir: ClassVar[Path | None] = None
34 | config_previous_directory: ClassVar[Path | None] = None
35 |
36 | write_incumbent: ClassVar[bool | None] = None
37 |
38 | config_writer: ClassVar[SummaryWriter | None] = None
39 | summary_writer: ClassVar[SummaryWriter | None] = None
40 |
41 | @staticmethod
42 | def _initiate_internal_configurations() -> None:
43 | """Track the Configuration space data from the way handled by neps runtime
44 | '_sample_config' to keep in sync with config ids and directories NePS is
45 | operating on.
46 | """
47 | trial = get_in_progress_trial()
48 | neps_state = get_workers_neps_state()
49 |
50 | register_notify_trial_end("NEPS_TBLOGGER", tblogger.end_of_config)
51 |
52 | # We are assuming that neps state is all filebased here
53 | root_dir = Path(neps_state.path)
54 | assert root_dir.exists()
55 |
56 | tblogger.config_working_directory = Path(trial.metadata.location)
57 | tblogger.config_previous_directory = (
58 | Path(trial.metadata.previous_trial_location)
59 | if trial.metadata.previous_trial_location is not None
60 | else None
61 | )
62 | tblogger.config_id = trial.metadata.id
63 | tblogger.optimizer_dir = root_dir
64 | tblogger.config = trial.config
65 |
66 | @staticmethod
67 | def WriteIncumbent() -> None: # noqa: N802
68 | """Allows for writing the incumbent of the current search."""
69 | tblogger._initiate_internal_configurations()
70 | tblogger.write_incumbent = True
71 |
72 | @staticmethod
73 | def ConfigWriter(*, write_summary_incumbent: bool = True) -> SummaryWriter: # noqa: N802
74 | """Creates and returns a TensorBoard SummaryWriter configured to write logs
75 | to the appropriate directory for NePS.
76 |
77 | Args:
78 | write_summary_incumbent (bool): Determines whether to write summaries
79 | for the incumbent configurations.
80 | Defaults to True.
81 |
82 | Returns:
83 | SummaryWriter: An instance of TensorBoard SummaryWriter pointing to the
84 | designated NePS directory.
85 | """
86 | tblogger.write_incumbent = write_summary_incumbent
87 | tblogger._initiate_internal_configurations()
88 | # This code runs only once per config, to assign that config a config_writer.
89 | if (
90 | tblogger.config_previous_directory is None
91 | and tblogger.config_working_directory is not None
92 | ):
93 | # If no fidelities are there yet, define the writer via the config_id
94 | tblogger.config_id = str(tblogger.config_working_directory).rsplit(
95 | "/", maxsplit=1
96 | )[-1]
97 | tblogger.config_writer = SummaryWriter(
98 | tblogger.config_working_directory / "tbevents"
99 | )
100 | return tblogger.config_writer
101 |
102 | # Searching for the initial directory where tensorboard events are stored.
103 | if tblogger.config_working_directory is not None:
104 | init_dir = get_initial_directory(
105 | pipeline_directory=tblogger.config_working_directory
106 | )
107 | tblogger.config_id = str(init_dir).rsplit("/", maxsplit=1)[-1]
108 | if (init_dir / "tbevents").exists():
109 | tblogger.config_writer = SummaryWriter(init_dir / "tbevents")
110 | return tblogger.config_writer
111 |
112 | raise FileNotFoundError(
113 | "'tbevents' was not found in the initial directory of the configuration."
114 | )
115 | return None
116 |
117 | @staticmethod
118 | def end_of_config(trial: Trial) -> None: # noqa: ARG004
119 | """Closes the writer."""
120 | if tblogger.config_writer:
121 | # Close and reset previous config writers for consistent logging.
122 | # Prevent conflicts by reinitializing writers when logging ongoing.
123 | tblogger.config_writer.close()
124 | tblogger.config_writer = None
125 |
126 | if tblogger.write_incumbent:
127 | tblogger._tracking_incumbent_api()
128 |
129 | @staticmethod
130 | def _tracking_incumbent_api() -> None:
131 | """Track the incumbent (best) objective_to_minimize and log it in the TensorBoard
132 | summary.
133 |
134 | Note:
135 | The function relies on the following global variables:
136 | - tblogger.optimizer_dir
137 | - tblogger.summary_writer
138 |
139 | The function logs the incumbent trajectory in TensorBoard.
140 | """
141 | assert tblogger.optimizer_dir is not None
142 | try:
143 | _, short = status(tblogger.optimizer_dir, print_summary=False)
144 |
145 | incum_tracker = short["num_success"] - 1
146 | incum_val = short["best_objective_to_minimize"]
147 |
148 | if tblogger.summary_writer is None and tblogger.optimizer_dir is not None:
149 | tblogger.summary_writer = SummaryWriter(
150 | tblogger.optimizer_dir / "summary_tb"
151 | )
152 |
153 | assert tblogger.summary_writer is not None
154 | tblogger.summary_writer.add_scalar(
155 | tag="Summary/Incumbent_graph",
156 | scalar_value=incum_val,
157 | global_step=incum_tracker,
158 | )
159 |
160 | # Frequent writer open/close creates new 'tfevent' files due to
161 | # parallelization needs. Simultaneous open writers risk conflicts,
162 | # so they're flushed and closed after use.
163 |
164 | tblogger.summary_writer.flush()
165 | tblogger.summary_writer.close()
166 | time.sleep(0.5)
167 |
168 | except: # noqa: E722
169 | logger.warning(
170 | "Incumbent tracking for TensorBoard with NePS has failed. "
171 | "This feature is now permanently disabled for the entire run."
172 | )
173 | tblogger.write_incumbent = False
174 |
--------------------------------------------------------------------------------
/neps/py.typed:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/automl/neps/0ef3731fa5049336e9a2bb61be7093fc902a96ff/neps/py.typed
--------------------------------------------------------------------------------
/neps/sampling/__init__.py:
--------------------------------------------------------------------------------
1 | from neps.sampling.priors import CenteredPrior, Prior, Uniform
2 | from neps.sampling.samplers import BorderSampler, Sampler, Sobol, WeightedSampler
3 |
4 | __all__ = [
5 | "BorderSampler",
6 | "CenteredPrior",
7 | "Prior",
8 | "Sampler",
9 | "Sobol",
10 | "Uniform",
11 | "WeightedSampler",
12 | ]
13 |
--------------------------------------------------------------------------------
/neps/space/__init__.py:
--------------------------------------------------------------------------------
1 | from neps.space.domain import Domain
2 | from neps.space.encoding import ConfigEncoder
3 | from neps.space.parameters import Categorical, Constant, Float, Integer, Parameter
4 | from neps.space.search_space import SearchSpace
5 |
6 | __all__ = [
7 | "Categorical",
8 | "ConfigEncoder",
9 | "Constant",
10 | "Domain",
11 | "Float",
12 | "Integer",
13 | "Parameter",
14 | "SearchSpace",
15 | ]
16 |
--------------------------------------------------------------------------------
/neps/space/search_space.py:
--------------------------------------------------------------------------------
1 | """Contains the [`SearchSpace`][neps.space.search_space.SearchSpace] class
2 | which contains the hyperparameters for the search space, as well as
3 | any fidelities and constants.
4 | """
5 |
6 | from __future__ import annotations
7 |
8 | from collections.abc import Iterator, Mapping
9 | from dataclasses import dataclass, field
10 | from typing import Any
11 |
12 | from neps.space.parameters import Categorical, Constant, Float, Integer, Parameter
13 |
14 |
15 | # NOTE: The use of `Mapping` instead of `dict` is so that type-checkers
16 | # can check if we accidetally mutate these as we pass the parameters around.
17 | # We really should not, and instead make a copy if we really need to.
18 | @dataclass
19 | class SearchSpace(Mapping[str, Parameter | Constant]):
20 | """A container for parameters."""
21 |
22 | elements: Mapping[str, Parameter | Constant] = field(default_factory=dict)
23 | """All items in the search space."""
24 |
25 | categoricals: Mapping[str, Categorical] = field(init=False)
26 | """The categorical hyperparameters in the search space."""
27 |
28 | numerical: Mapping[str, Integer | Float] = field(init=False)
29 | """The numerical hyperparameters in the search space.
30 |
31 | !!! note
32 |
33 | This does not include fidelities.
34 | """
35 |
36 | fidelities: Mapping[str, Integer | Float] = field(init=False)
37 | """The fidelities in the search space.
38 |
39 | Currently no optimizer supports multiple fidelities but it is defined here incase.
40 | """
41 |
42 | constants: Mapping[str, Any] = field(init=False, default_factory=dict)
43 | """The constants in the search space."""
44 |
45 | @property
46 | def searchables(self) -> Mapping[str, Parameter]:
47 | """The hyperparameters that can be searched over.
48 |
49 | !!! note
50 |
51 | This does not include either constants or fidelities.
52 | """
53 | return {**self.numerical, **self.categoricals}
54 |
55 | @property
56 | def fidelity(self) -> tuple[str, Float | Integer] | None:
57 | """The fidelity parameter for the search space."""
58 | return None if len(self.fidelities) == 0 else next(iter(self.fidelities.items()))
59 |
60 | def __post_init__(self) -> None:
61 | # Ensure that we have a consistent order for all our items.
62 | self.elements = dict(sorted(self.elements.items(), key=lambda x: x[0]))
63 |
64 | fidelities: dict[str, Float | Integer] = {}
65 | numerical: dict[str, Float | Integer] = {}
66 | categoricals: dict[str, Categorical] = {}
67 | constants: dict[str, Any] = {}
68 |
69 | # Process the hyperparameters
70 | for name, hp in self.elements.items():
71 | match hp:
72 | case Float() | Integer() if hp.is_fidelity:
73 | # We should allow this at some point, but until we do,
74 | # raise an error
75 | if len(fidelities) >= 1:
76 | raise ValueError(
77 | "neps only supports one fidelity parameter in the"
78 | " pipeline space, but multiple were given."
79 | f" Fidelities: {fidelities}, new: {name}"
80 | )
81 | fidelities[name] = hp
82 |
83 | case Float() | Integer():
84 | numerical[name] = hp
85 | case Categorical():
86 | categoricals[name] = hp
87 | case Constant():
88 | constants[name] = hp.value
89 |
90 | case _:
91 | raise ValueError(f"Unknown hyperparameter type: {hp}")
92 |
93 | self.categoricals = categoricals
94 | self.numerical = numerical
95 | self.constants = constants
96 | self.fidelities = fidelities
97 |
98 | def __getitem__(self, key: str) -> Parameter | Constant:
99 | return self.elements[key]
100 |
101 | def __iter__(self) -> Iterator[str]:
102 | return iter(self.elements)
103 |
104 | def __len__(self) -> int:
105 | return len(self.elements)
106 |
--------------------------------------------------------------------------------
/neps/state/__init__.py:
--------------------------------------------------------------------------------
1 | from neps.state.neps_state import NePSState
2 | from neps.state.optimizer import BudgetInfo, OptimizationState
3 | from neps.state.pipeline_eval import EvaluatePipelineReturn, UserResult, evaluate_trial
4 | from neps.state.seed_snapshot import SeedSnapshot
5 | from neps.state.settings import DefaultReportValues, OnErrorPossibilities, WorkerSettings
6 | from neps.state.trial import Trial
7 |
8 | __all__ = [
9 | "BudgetInfo",
10 | "DefaultReportValues",
11 | "EvaluatePipelineReturn",
12 | "NePSState",
13 | "OnErrorPossibilities",
14 | "OptimizationState",
15 | "SeedSnapshot",
16 | "Trial",
17 | "UserResult",
18 | "WorkerSettings",
19 | "evaluate_trial",
20 | ]
21 |
--------------------------------------------------------------------------------
/neps/state/err_dump.py:
--------------------------------------------------------------------------------
1 | """Error dump for serializing errors.
2 |
3 | This resource is used to store errors that can be serialized and deserialized,
4 | such that they can be shared between workers.
5 | """
6 |
7 | from __future__ import annotations
8 |
9 | from dataclasses import dataclass, field
10 | from typing import ClassVar
11 |
12 | from neps.exceptions import NePSError
13 |
14 |
15 | class SerializedError(NePSError):
16 | """An error the is serialized."""
17 |
18 |
19 | @dataclass
20 | class SerializableTrialError:
21 | """Error information for a trial."""
22 |
23 | trial_id: str
24 | """The ID of the trial."""
25 |
26 | worker_id: str
27 | """The ID of the worker that evaluated the trial which caused the error."""
28 |
29 | err_type: str
30 | """The type of the error."""
31 |
32 | err: str
33 | """The error msg."""
34 |
35 | tb: str | None
36 | """The traceback of the error."""
37 |
38 | def as_raisable(self) -> SerializedError:
39 | """Convert the error to a raisable error."""
40 | return SerializedError(
41 | f"An error occurred during the evaluation of a trial '{self.trial_id}' which"
42 | f" was evaluted by worker '{self.worker_id}'. The original error could not"
43 | " be deserialized but had the following information:"
44 | "\n"
45 | f"{self.err_type}: {self.err}"
46 | "\n\n"
47 | f"{self.tb}"
48 | )
49 |
50 |
51 | @dataclass
52 | class ErrDump:
53 | """A collection of errors that can be serialized and deserialized."""
54 |
55 | SerializableTrialError: ClassVar = SerializableTrialError
56 |
57 | errs: list[SerializableTrialError] = field(default_factory=list)
58 |
59 | def append(self, err: SerializableTrialError) -> None:
60 | """Append the an error to the reported errors."""
61 | return self.errs.append(err)
62 |
63 | def __len__(self) -> int:
64 | return len(self.errs)
65 |
66 | def __bool__(self) -> bool:
67 | return bool(self.errs)
68 |
69 | def empty(self) -> bool:
70 | """Check if the queue is empty."""
71 | return not self.errs
72 |
73 | def latest_err_as_raisable(self) -> SerializedError | None:
74 | """Get the latest error."""
75 | if self.errs:
76 | return self.errs[-1].as_raisable() # type: ignore
77 | return None
78 |
--------------------------------------------------------------------------------
/neps/state/optimizer.py:
--------------------------------------------------------------------------------
1 | """Optimizer state and info dataclasses."""
2 |
3 | from __future__ import annotations
4 |
5 | from dataclasses import dataclass, replace
6 | from typing import TYPE_CHECKING, Any
7 |
8 | if TYPE_CHECKING:
9 | from neps.state.seed_snapshot import SeedSnapshot
10 |
11 |
12 | @dataclass
13 | class BudgetInfo:
14 | """Information about the budget of an optimizer."""
15 |
16 | max_cost_total: float | None = None
17 | used_cost_budget: float = 0.0
18 | max_evaluations: int | None = None
19 | used_evaluations: int = 0
20 |
21 | def clone(self) -> BudgetInfo:
22 | """Create a copy of the budget info."""
23 | return replace(self)
24 |
25 |
26 | @dataclass
27 | class OptimizationState:
28 | """The current state of an optimizer."""
29 |
30 | budget: BudgetInfo | None
31 | """Information regarind the budget used by the optimization trajectory."""
32 |
33 | seed_snapshot: SeedSnapshot
34 | """The state of the random number generators at the time of the last sample."""
35 |
36 | shared_state: dict[str, Any] | None
37 | """Any information the optimizer wants to store between calls
38 | to sample and post evaluations.
39 |
40 | For example, an optimizer may wish to store running totals here or various other
41 | bits of information that may be expensive to recompute.
42 |
43 | Right now there's no support for tensors/arrays and almost no optimizer uses this
44 | feature. Only cost-cooling uses information out of `.budget`.
45 |
46 | Please reach out to @eddiebergman if you have a use case for this so we can make
47 | it more robust.
48 | """
49 |
--------------------------------------------------------------------------------
/neps/state/seed_snapshot.py:
--------------------------------------------------------------------------------
1 | """Snapshot of the global rng state."""
2 |
3 | from __future__ import annotations
4 |
5 | import contextlib
6 | import random
7 | from dataclasses import dataclass
8 | from typing import TYPE_CHECKING, Any, TypeAlias
9 |
10 | import numpy as np
11 |
12 | if TYPE_CHECKING:
13 | import torch
14 |
15 | NP_RNG_STATE: TypeAlias = tuple[str, np.ndarray, int, int, float]
16 | PY_RNG_STATE: TypeAlias = tuple[int, tuple[int, ...], int | None]
17 | TORCH_RNG_STATE: TypeAlias = torch.Tensor
18 | TORCH_CUDA_RNG_STATE: TypeAlias = list[torch.Tensor]
19 |
20 |
21 | @dataclass
22 | class SeedSnapshot:
23 | """State of the global rng.
24 |
25 | Primarly enables storing of the rng state to disk using a binary format
26 | native to each library, allowing for potential version mistmatches between
27 | processes loading the state, as long as they can read the binary format.
28 | """
29 |
30 | np_rng: NP_RNG_STATE
31 | py_rng: PY_RNG_STATE
32 | torch_rng: TORCH_RNG_STATE | None
33 | torch_cuda_rng: TORCH_CUDA_RNG_STATE | None
34 |
35 | @classmethod
36 | def new_capture(cls) -> SeedSnapshot:
37 | """Current state of the global rng.
38 |
39 | Takes a snapshot, including cloning or copying any arrays, tensors, etc.
40 | """
41 | self = cls(None, None, None, None) # type: ignore
42 | self.recapture()
43 | return self
44 |
45 | def recapture(self) -> None:
46 | """Reread the state of the global rng into this snapshot."""
47 | # https://numpy.org/doc/stable/reference/random/generated/numpy.random.get_state.html
48 |
49 | self.py_rng = random.getstate()
50 |
51 | np_keys = np.random.get_state(legacy=True)
52 | assert np_keys[0] == "MT19937" # type: ignore
53 | self.np_rng = (np_keys[0], np_keys[1].copy(), *np_keys[2:]) # type: ignore
54 |
55 | with contextlib.suppress(Exception):
56 | import torch
57 |
58 | self.torch_rng = torch.random.get_rng_state().clone()
59 | torch_cuda_keys: list[torch.Tensor] | None = None
60 | if torch.cuda.is_available():
61 | torch_cuda_keys = [c.clone() for c in torch.cuda.get_rng_state_all()]
62 | self.torch_cuda_rng = torch_cuda_keys
63 |
64 | def set_as_global_seed_state(self) -> None:
65 | """Set the global rng to the given state."""
66 | np.random.set_state(self.np_rng)
67 | random.setstate(self.py_rng)
68 |
69 | if self.torch_rng is not None or self.torch_cuda_rng is not None:
70 | import torch
71 |
72 | if self.torch_rng is not None:
73 | torch.random.set_rng_state(self.torch_rng)
74 |
75 | if self.torch_cuda_rng is not None and torch.cuda.is_available():
76 | torch.cuda.set_rng_state_all(self.torch_cuda_rng)
77 |
78 | def __eq__(self, other: Any, /) -> bool: # noqa: PLR0911
79 | if not isinstance(other, SeedSnapshot):
80 | return False
81 |
82 | if not (self.py_rng == other.py_rng):
83 | return False
84 |
85 | if not (
86 | self.np_rng[0] == other.np_rng[0]
87 | and self.np_rng[2] == other.np_rng[2]
88 | and self.np_rng[3] == other.np_rng[3]
89 | and self.np_rng[4] == other.np_rng[4]
90 | ):
91 | return False
92 |
93 | if not np.array_equal(self.np_rng[1], other.np_rng[1]):
94 | return False
95 |
96 | if self.torch_rng is not None and other.torch_rng is not None:
97 | import torch
98 |
99 | if not torch.equal(self.torch_rng, other.torch_rng):
100 | return False
101 |
102 | if self.torch_cuda_rng is not None and other.torch_cuda_rng is not None:
103 | import torch
104 |
105 | if not all(
106 | torch.equal(a, b)
107 | for a, b in zip(self.torch_cuda_rng, other.torch_cuda_rng, strict=False)
108 | ):
109 | return False
110 |
111 | if not isinstance(self.torch_rng, type(other.torch_rng)):
112 | return False
113 |
114 | return isinstance(self.torch_cuda_rng, type(other.torch_cuda_rng))
115 |
--------------------------------------------------------------------------------
/neps/state/settings.py:
--------------------------------------------------------------------------------
1 | """Settings for the worker and the global state of NePS."""
2 |
3 | from __future__ import annotations
4 |
5 | from dataclasses import dataclass
6 | from enum import Enum
7 | from typing import Literal
8 |
9 |
10 | @dataclass
11 | class DefaultReportValues:
12 | """Values to use when an error occurs."""
13 |
14 | objective_value_on_error: float | None = None
15 | """The value to use for the objective_to_minimize when an error occurs."""
16 |
17 | cost_value_on_error: float | None = None
18 | """The value to use for the cost when an error occurs."""
19 |
20 | cost_if_not_provided: float | None = None
21 | """The value to use for the cost when the evaluation function does not provide one."""
22 |
23 | learning_curve_on_error: list[float] | None = None
24 | """The value to use for the learning curve when an error occurs.
25 |
26 | If `'objective_to_minimize'`, the learning curve will be set to the
27 | objective_to_minimize value but as a list with a single value.
28 | """
29 |
30 | learning_curve_if_not_provided: (
31 | Literal["objective_to_minimize"] | list[float] | None
32 | ) = None
33 | """The value to use for the learning curve when the evaluation function does
34 | not provide one."""
35 |
36 |
37 | class OnErrorPossibilities(Enum):
38 | """Possible values for what to do when an error occurs."""
39 |
40 | RAISE_WORKER_ERROR = "raise_worker_error"
41 | """Raise an error only if the error occurs in the worker."""
42 |
43 | STOP_WORKER_ERROR = "stop_worker_error"
44 | """Stop the worker if an error occurs in the worker, without raising"""
45 |
46 | RAISE_ANY_ERROR = "raise_any_error"
47 | """Raise an error if there was an error from any worker, i.e. there is a trial in the
48 | NePSState that has an error."""
49 |
50 | STOP_ANY_ERROR = "stop_any_error"
51 | """Stop the workers if any error occured from any worker, i.e. there is a trial in the
52 | NePSState that has an error."""
53 |
54 | IGNORE = "ignore"
55 | """Ignore all errors and continue running."""
56 |
57 |
58 | # TODO: We can extend this over time
59 | # For now this is what was needed for the backend state and workers.
60 | @dataclass
61 | class WorkerSettings:
62 | """Settings for a running instance of NePS."""
63 |
64 | # --------- Evaluation ---------
65 | on_error: OnErrorPossibilities
66 | """What to do when an error occurs.
67 |
68 | - `'raise_worker_error'`: Raise an error only if the error occurs in the worker.
69 | - `'raise_any_error'`: Raise an error if any error occurs from any worker, i.e.
70 | there is a trial in the NePSState that has an error.
71 | - `'ignore'`: Ignore all errors and continue running.
72 | """
73 |
74 | default_report_values: DefaultReportValues
75 | """Values to use when an error occurs or was not specified."""
76 |
77 | batch_size: int | None
78 | """The number of configurations to sample in a single batch."""
79 |
80 | # --------- Global Stopping Criterion ---------
81 | max_evaluations_total: int | None
82 | """The maximum number of evaluations to run in total.
83 |
84 | Once this evaluation total is reached, **all** workers will stop evaluating
85 | new configurations.
86 |
87 | To control whether currently evaluating configurations are included in this
88 | total, see
89 | [`include_in_progress_evaluations_towards_maximum`][neps.state.settings.WorkerSettings.include_in_progress_evaluations_towards_maximum].
90 |
91 | If `None`, there is no limit and workers will continue to evaluate
92 | indefinitely.
93 | """
94 |
95 | include_in_progress_evaluations_towards_maximum: bool
96 | """Whether to include currently evaluating configurations towards the
97 | stopping criterion
98 | [`max_evaluations_total`][neps.state.settings.WorkerSettings.max_evaluations_total]
99 | """
100 |
101 | max_cost_total: float | None
102 | """The maximum cost to run in total.
103 |
104 | Once this cost total is reached, **all** workers will stop evaluating new
105 | configurations.
106 |
107 | This cost is the sum of `'cost'` values that are returned by evaluation
108 | of the target function.
109 |
110 | If `None`, there is no limit and workers will continue to evaluate
111 | indefinitely or until another stopping criterion is met.
112 | """
113 |
114 | max_evaluation_time_total_seconds: float | None
115 | """The maximum wallclock time allowed for evaluation in total.
116 |
117 | !!! note
118 | This does not include time for sampling new configurations.
119 |
120 | Once this wallclock time is reached, **all** workers will stop once their
121 | current evaluation is finished.
122 |
123 | If `None`, there is no limit and workers will continue to evaluate
124 | indefinitely or until another stopping criterion is met.
125 | """
126 |
127 | # --------- Local Worker Stopping Criterion ---------
128 | max_evaluations_for_worker: int | None
129 | """The maximum number of evaluations to run for the worker.
130 |
131 | This count is specific to each worker spawned by NePS.
132 | **only** the current worker will stop evaluating new configurations once
133 | this limit is reached.
134 |
135 | If `None`, there is no limit and this worker will continue to evaluate
136 | indefinitely or until another stopping criterion is met.
137 | """
138 |
139 | max_cost_for_worker: float | None
140 | """The maximum cost incurred by a worker before finisihng.
141 |
142 | Once this cost total is reached, **only** this worker will stop evaluating new
143 | configurations.
144 |
145 | This cost is the sum of `'cost'` values that are returned by evaluation
146 | of the target function.
147 |
148 | If `None`, there is no limit and the worker will continue to evaluate
149 | indefinitely or until another stopping criterion is met.
150 | """
151 |
152 | max_evaluation_time_for_worker_seconds: float | None
153 | """The maximum time to allow this worker for evaluating configurations.
154 |
155 | !!! note
156 | This does not include time for sampling new configurations.
157 |
158 | If `None`, there is no limit and this worker will continue to evaluate
159 | indefinitely or until another stopping criterion is met.
160 | """
161 |
162 | max_wallclock_time_for_worker_seconds: float | None
163 | """The maximum wallclock time to run for this worker.
164 |
165 | Once this wallclock time is reached, **only** this worker will stop evaluating
166 | new configurations.
167 |
168 | !!! warning
169 | This will not stop the worker if it is currently evaluating a configuration.
170 |
171 | This is useful when the worker is deployed on some managed resource where
172 | there is a time limit.
173 |
174 | If `None`, there is no limit and this worker will continue to evaluate
175 | indefinitely or until another stopping criterion is met.
176 | """
177 |
--------------------------------------------------------------------------------
/neps/status/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/automl/neps/0ef3731fa5049336e9a2bb61be7093fc902a96ff/neps/status/__init__.py
--------------------------------------------------------------------------------
/neps/status/__main__.py:
--------------------------------------------------------------------------------
1 | """Displays status information about a working directory of a neps.run.
2 |
3 | Usage:
4 | python -m neps.status [-h] [--best_objective_to_minimizees] [--best_configs]
5 | [--all_configs] working_directory
6 |
7 | Positional arguments:
8 | working_directory The working directory given to neps.run
9 |
10 | Optional arguments:
11 | -h, --help show this help message and exit
12 | --best_objective_to_minimizees Show the trajectory of the best
13 | objective_to_minimize across evaluations
14 | --best_configs Show the trajectory of the best configs and their
15 | objective_to_minimizees
16 | across evaluations
17 | --all_configs Show all configs and their objective_to_minimizees
18 |
19 | Note:
20 | We have to use the __main__.py construct due to the issues explained in
21 | https://stackoverflow.com/questions/43393764/python-3-6-project-structure-leads-to-runtimewarning
22 |
23 | """
24 |
25 | from __future__ import annotations
26 |
27 | import argparse
28 | import logging
29 | from pathlib import Path
30 |
31 | from .status import status
32 |
33 | # fmt: off
34 | parser = argparse.ArgumentParser(
35 | prog="python -m neps.status",
36 | description="Displays status information about a working directory of a neps.run",
37 | )
38 | parser.add_argument("root_directory", type=Path,
39 | help="The working directory given to neps.run")
40 | args = parser.parse_args()
41 |
42 | logging.basicConfig(level=logging.WARN)
43 | status(args.root_directory, print_summary=True)
44 |
--------------------------------------------------------------------------------
/neps/utils/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/automl/neps/0ef3731fa5049336e9a2bb61be7093fc902a96ff/neps/utils/__init__.py
--------------------------------------------------------------------------------
/neps/utils/files.py:
--------------------------------------------------------------------------------
1 | """Utilities for file operations."""
2 |
3 | from __future__ import annotations
4 |
5 | import dataclasses
6 | import io
7 | import os
8 | from collections.abc import Iterable, Iterator, Mapping
9 | from contextlib import contextmanager
10 | from enum import Enum
11 | from pathlib import Path
12 | from typing import IO, Any, Literal
13 |
14 | import yaml
15 |
16 | try:
17 | from yaml import (
18 | CDumper as YamlDumper, # type: ignore
19 | CSafeLoader as SafeLoader, # type: ignore
20 | )
21 | except ImportError:
22 | from yaml import SafeLoader, YamlDumper # type: ignore
23 |
24 |
25 | @contextmanager
26 | def atomic_write(file_path: Path | str, *args: Any, **kwargs: Any) -> Iterator[IO]:
27 | """Write to a file atomically.
28 |
29 | This means that the file will be flushed to disk and explicitly ask the operating
30 | systems to sync the contents to disk. This ensures that other processes that read
31 | from this file should see the contents immediately.
32 | """
33 | with open(file_path, *args, **kwargs) as file_stream: # noqa: PTH123
34 | yield file_stream
35 | file_stream.flush()
36 | os.fsync(file_stream.fileno())
37 | file_stream.close()
38 |
39 |
40 | def serializable_format(data: Any) -> Any: # noqa: PLR0911
41 | """Format data to be serializable."""
42 | if hasattr(data, "serialize"):
43 | return serializable_format(data.serialize())
44 |
45 | if dataclasses.is_dataclass(data) and not isinstance(data, type):
46 | return serializable_format(dataclasses.asdict(data)) # type: ignore
47 |
48 | if isinstance(data, Exception):
49 | return str(data)
50 |
51 | if isinstance(data, Enum):
52 | return data.value
53 |
54 | if isinstance(data, Mapping):
55 | return {key: serializable_format(val) for key, val in data.items()}
56 |
57 | if not isinstance(data, str) and isinstance(data, Iterable):
58 | return [serializable_format(val) for val in data]
59 |
60 | if type(data).__module__ in ["numpy", "torch"]:
61 | data = data.tolist() # type: ignore
62 | if type(data).__module__ == "numpy":
63 | data = data.item()
64 |
65 | return serializable_format(data)
66 |
67 | return data
68 |
69 |
70 | def serialize(
71 | data: Any,
72 | path: Path,
73 | *,
74 | check_serialized: bool = True,
75 | file_format: Literal["json", "yaml"] = "yaml",
76 | sort_keys: bool = True,
77 | ) -> None:
78 | """Serialize data to a yaml file."""
79 | if check_serialized:
80 | data = serializable_format(data)
81 |
82 | buf = io.StringIO()
83 | if file_format == "yaml":
84 | try:
85 | yaml.dump(data, buf, YamlDumper, sort_keys=sort_keys)
86 | except yaml.representer.RepresenterError as e:
87 | raise TypeError(
88 | "Could not serialize to yaml! The object "
89 | f"{e.args[1]} of type {type(e.args[1])} is not."
90 | ) from e
91 | elif file_format == "json":
92 | import json
93 |
94 | json.dump(data, buf, sort_keys=sort_keys)
95 | else:
96 | raise ValueError(f"Unknown format: {file_format}")
97 |
98 | _str = buf.getvalue()
99 | path.write_text(_str)
100 |
101 |
102 | def deserialize(
103 | path: Path | str,
104 | *,
105 | file_format: Literal["json", "yaml"] = "yaml",
106 | ) -> dict[str, Any]:
107 | """Deserialize data from a yaml file."""
108 | with Path(path).open("r") as file_stream:
109 | if file_format == "json":
110 | import json
111 |
112 | data = json.load(file_stream)
113 | elif file_format == "yaml":
114 | data = yaml.load(file_stream, SafeLoader)
115 | else:
116 | raise ValueError(f"Unknown format: {file_format}")
117 |
118 | if not isinstance(data, dict):
119 | raise TypeError(
120 | f"Deserialized data at {path} is not a dictionary!"
121 | f" Got {type(data)} instead.\n{data}"
122 | )
123 |
124 | return data
125 |
126 |
127 | def load_and_merge_yamls(*paths: str | Path | IO[str]) -> dict[str, Any]:
128 | """Load and merge yaml files into a single dictionary.
129 |
130 | Raises:
131 | ValueError: If there are duplicate keys in the yaml files.
132 | """
133 | config: dict[str, Any] = {}
134 | for path in paths:
135 | match path:
136 | case str() | Path():
137 | with Path(path).open("r") as file:
138 | read_config = yaml.safe_load(file)
139 |
140 | case _:
141 | read_config = yaml.safe_load(path)
142 |
143 | shared_keys = set(config) & set(read_config)
144 |
145 | if any(shared_keys):
146 | raise ValueError(f"Duplicate key(s) {shared_keys} in {paths}")
147 |
148 | config.update(read_config)
149 |
150 | return config
151 |
--------------------------------------------------------------------------------
/neps_examples/README.md:
--------------------------------------------------------------------------------
1 | # Overview
2 |
3 | 1. **Basic usage examples** demonstrate fundamental usage.
4 | Learn how to perform Hyperparameter Optimization (HPO) and analyze runs on a basic level.
5 |
6 | 2. **Convenience examples** show tensorboard compatibility and its integration, SLURM-scripting and understand file management within the evaluate pipeline function used in NePS.
7 |
8 | 3. **Efficiency examples** showcase how to enhance efficiency in NePS. Learn about expert priors, multi-fidelity, and parallelization to streamline your pipeline and optimize search processes.
9 |
10 | 4. **Experimental examples** tailored for NePS contributors. These examples provide insights and practices for experimental scenarios.
11 |
12 | 5. **Real-world examples** demonstrate how to apply NePS to real-world problems.
13 |
--------------------------------------------------------------------------------
/neps_examples/__init__.py:
--------------------------------------------------------------------------------
1 | all_main_examples = { # Used for printing in python -m neps_examples
2 | "basic_usage": [
3 | "analyse",
4 | "architecture",
5 | "architecture_and_hyperparameters",
6 | "hyperparameters",
7 | ],
8 | "convenience": [
9 | "logging_additional_info",
10 | "neps_tblogger_tutorial",
11 | "running_on_slurm_scripts",
12 | "neps_x_lightning",
13 | "running_on_slurm_scripts",
14 | "working_directory_per_pipeline",
15 | ],
16 | "efficiency": [
17 | "expert_priors_for_hyperparameters",
18 | "multi_fidelity",
19 | "multi_fidelity_and_expert_priors",
20 | "pytorch_native_ddp",
21 | "pytorch_lightning_ddp",
22 | ],
23 | }
24 |
25 | core_examples = [ # Run locally and on github actions
26 | "basic_usage/hyperparameters", # NOTE: This needs to be first for some tests to work
27 | "basic_usage/analyse",
28 | "experimental/expert_priors_for_architecture_and_hyperparameters",
29 | "efficiency/multi_fidelity",
30 | ]
31 |
32 | ci_examples = [ # Run on github actions
33 | "basic_usage/architecture_and_hyperparameters",
34 | "experimental/hierarchical_architecture",
35 | "efficiency/expert_priors_for_hyperparameters",
36 | "convenience/logging_additional_info",
37 | "convenience/working_directory_per_pipeline",
38 | "convenience/neps_tblogger_tutorial",
39 | ]
40 |
--------------------------------------------------------------------------------
/neps_examples/__main__.py:
--------------------------------------------------------------------------------
1 | from neps_examples import all_main_examples
2 | from pathlib import Path
3 |
4 | def print_examples():
5 | print("The following examples are available")
6 | for folder, examples in all_main_examples.items():
7 | print()
8 | for example in examples:
9 | print(f'python -m neps_examples.{folder}.{example}')
10 |
11 | def print_specific_example(example):
12 | neps_examples_dir = Path(__file__).parent
13 | print(neps_examples_dir)
14 | example_file = neps_examples_dir / f"{example.replace('.', '/')}.py"
15 | print(example_file.read_text())
16 |
17 |
18 | if __name__ == '__main__':
19 | import argparse
20 | parser = argparse.ArgumentParser()
21 | parser.add_argument("--print", default=None, help="Example name to print in form of 'basic_usage.hyperparameters'")
22 | args = parser.parse_args()
23 |
24 | if args.print:
25 | print_specific_example(args.print)
26 | else:
27 | print_examples()
28 |
--------------------------------------------------------------------------------
/neps_examples/basic_usage/analyse.py:
--------------------------------------------------------------------------------
1 | """How to generate a summary (neps.status) and visualizations (neps.plot) of a run.
2 |
3 | Before running this example analysis, run the hyperparameters example with:
4 |
5 | python -m neps_examples.basic_usage.hyperparameters
6 | """
7 |
8 | import neps
9 |
10 | # 1. At all times, NePS maintains several files in the root directory that are human
11 | # read-able and can be useful
12 |
13 | # 2. Printing a summary and reading in results.
14 | # Alternatively use `python -m neps.status results/hyperparameters_example`
15 | full, summary = neps.status("results/hyperparameters_example", print_summary=True)
16 | config_id = "1"
17 |
18 | print(full.head())
19 | print("")
20 | print(full.loc[config_id])
21 |
--------------------------------------------------------------------------------
/neps_examples/basic_usage/hyperparameters.py:
--------------------------------------------------------------------------------
1 | import logging
2 | import numpy as np
3 | import neps
4 |
5 | # This example demonstrates how to use NePS to optimize hyperparameters
6 | # of a pipeline. The pipeline is a simple function that takes in
7 | # five hyperparameters and returns their sum.
8 | # Neps uses the default optimizer to minimize this objective function.
9 |
10 | def evaluate_pipeline(float1, float2, categorical, integer1, integer2):
11 | objective_to_minimize = -float(
12 | np.sum([float1, float2, int(categorical), integer1, integer2])
13 | )
14 | return objective_to_minimize
15 |
16 |
17 | pipeline_space = dict(
18 | float1=neps.Float(lower=0, upper=1),
19 | float2=neps.Float(lower=-10, upper=10),
20 | categorical=neps.Categorical(choices=[0, 1]),
21 | integer1=neps.Integer(lower=0, upper=1),
22 | integer2=neps.Integer(lower=1, upper=1000, log=True),
23 | )
24 |
25 | logging.basicConfig(level=logging.INFO)
26 | neps.run(
27 | evaluate_pipeline=evaluate_pipeline,
28 | pipeline_space=pipeline_space,
29 | root_directory="results/hyperparameters_example",
30 | post_run_summary=True,
31 | max_evaluations_total=30,
32 | )
33 |
--------------------------------------------------------------------------------
/neps_examples/convenience/logging_additional_info.py:
--------------------------------------------------------------------------------
1 | import logging
2 | import time
3 | from warnings import warn
4 |
5 | import numpy as np
6 |
7 | import neps
8 |
9 |
10 | def evaluate_pipeline(float1, float2, categorical, integer1, integer2):
11 | start = time.time()
12 | objective_to_minimize = -float(
13 | np.sum([float1, float2, int(categorical), integer1, integer2])
14 | )
15 | end = time.time()
16 | return {
17 | "objective_to_minimize": objective_to_minimize,
18 | "info_dict": { # Optionally include additional information as an info_dict
19 | "train_time": end - start,
20 | },
21 | }
22 |
23 |
24 | pipeline_space = dict(
25 | float1=neps.Float(lower=0, upper=1),
26 | float2=neps.Float(lower=-10, upper=10),
27 | categorical=neps.Categorical(choices=[0, 1]),
28 | integer1=neps.Integer(lower=0, upper=1),
29 | integer2=neps.Integer(lower=1, upper=1000, log=True),
30 | )
31 |
32 | logging.basicConfig(level=logging.INFO)
33 | neps.run(
34 | evaluate_pipeline=evaluate_pipeline,
35 | pipeline_space=pipeline_space,
36 | root_directory="results/logging_additional_info",
37 | max_evaluations_total=5,
38 | )
39 |
--------------------------------------------------------------------------------
/neps_examples/convenience/running_on_slurm_scripts.py:
--------------------------------------------------------------------------------
1 | """Example that shows HPO with NePS based on a slurm script."""
2 |
3 | import logging
4 | import os
5 | import time
6 | from pathlib import Path
7 |
8 | import neps
9 |
10 |
11 | def _ask_to_submit_slurm_script(pipeline_directory: Path, script: str):
12 | script_path = pipeline_directory / "submit.sh"
13 | logging.info(f"Submitting the script {script_path} (see below): \n\n{script}")
14 |
15 | # You may want to remove the below check and not ask before submitting every time
16 | if input("Ok to submit? [Y|n] -- ").lower() in {"y", ""}:
17 | script_path.write_text(script)
18 | os.system(f"sbatch {script_path}")
19 | else:
20 | raise ValueError("We generated a slurm script that should not be submitted.")
21 |
22 |
23 | def _get_validation_error(pipeline_directory: Path):
24 | validation_error_file = pipeline_directory / "validation_error_from_slurm_job.txt"
25 | if validation_error_file.exists():
26 | return float(validation_error_file.read_text())
27 | return None
28 |
29 |
30 | def evaluate_pipeline_via_slurm(
31 | pipeline_directory: Path, optimizer: str, learning_rate: float
32 | ):
33 | script = f"""#!/bin/bash
34 | #SBATCH --time 0-00:05
35 | #SBATCH --job-name test
36 | #SBATCH --partition cpu-cascadelake
37 | #SBATCH --error "{pipeline_directory}/%N_%A_%x_%a.oe"
38 | #SBATCH --output "{pipeline_directory}/%N_%A_%x_%a.oe"
39 | # Plugin your python script here
40 | python -c "print('Learning rate {learning_rate} and optimizer {optimizer}')"
41 | # At the end of training and validation create this file
42 | echo -10 > {pipeline_directory}/validation_error_from_slurm_job.txt
43 | """
44 |
45 | # Now we submit and wait until the job has created validation_error_from_slurm_job.txt
46 | _ask_to_submit_slurm_script(pipeline_directory, script)
47 | while validation_error := _get_validation_error(pipeline_directory) is None:
48 | logging.info("Waiting until the job has finished.")
49 | time.sleep(60) # Adjust to something reasonable
50 | return validation_error
51 |
52 |
53 | pipeline_space = dict(
54 | optimizer=neps.Categorical(choices=["sgd", "adam"]),
55 | learning_rate=neps.Float(lower=10e-7, upper=10e-3, log=True),
56 | )
57 |
58 | logging.basicConfig(level=logging.INFO)
59 | neps.run(
60 | evaluate_pipeline=evaluate_pipeline_via_slurm,
61 | pipeline_space=pipeline_space,
62 | root_directory="results/slurm_script_example",
63 | max_evaluations_total=5,
64 | )
65 |
--------------------------------------------------------------------------------
/neps_examples/convenience/working_directory_per_pipeline.py:
--------------------------------------------------------------------------------
1 | import logging
2 | from pathlib import Path
3 | from warnings import warn
4 |
5 | import numpy as np
6 |
7 | import neps
8 |
9 |
10 | def evaluate_pipeline(pipeline_directory: Path, float1, categorical, integer1):
11 | # When adding pipeline_directory to evaluate_pipeline, neps detects its presence and
12 | # passes a directory unique for each pipeline configuration. You can then use this
13 | # pipeline_directory to create / save files pertaining to a specific pipeline, e.g.:
14 | pipeline_info = pipeline_directory / "info_file.txt"
15 | pipeline_info.write_text(f"{float1} - {categorical} - {integer1}")
16 |
17 | objective_to_minimize = -float(np.sum([float1, int(categorical), integer1]))
18 | return objective_to_minimize
19 |
20 |
21 | pipeline_space = dict(
22 | float1=neps.Float(lower=0, upper=1),
23 | categorical=neps.Categorical(choices=[0, 1]),
24 | integer1=neps.Integer(lower=0, upper=1),
25 | )
26 |
27 | logging.basicConfig(level=logging.INFO)
28 | neps.run(
29 | evaluate_pipeline=evaluate_pipeline,
30 | pipeline_space=pipeline_space,
31 | root_directory="results/working_directory_per_pipeline",
32 | max_evaluations_total=5,
33 | )
34 |
--------------------------------------------------------------------------------
/neps_examples/efficiency/README.md:
--------------------------------------------------------------------------------
1 | # Parallelization
2 |
3 | In order to run neps in parallel on multiple processes or multiple machines, simply call `neps.run` multiple times.
4 | All calls to `neps.run` need to use the same `root_directory` on the same filesystem to synchronize between the `neps.run`'s.
5 |
6 | For example, start the HPO example in two shells from the same directory as below.
7 |
8 | In shell 1:
9 |
10 | ```bash
11 | python -m neps_examples.basic_usage.hyperparameters
12 | ```
13 |
14 | In shell 2:
15 |
16 | ```bash
17 | python -m neps_examples.basic_usage.hyperparameters
18 | ```
19 |
--------------------------------------------------------------------------------
/neps_examples/efficiency/expert_priors_for_hyperparameters.py:
--------------------------------------------------------------------------------
1 | import logging
2 | import time
3 |
4 | import neps
5 |
6 |
7 | def evaluate_pipeline(some_float, some_integer, some_cat):
8 | start = time.time()
9 | if some_cat != "a":
10 | y = some_float + some_integer
11 | else:
12 | y = -some_float - some_integer
13 | end = time.time()
14 | return {
15 | "objective_to_minimize": y,
16 | "info_dict": {
17 | "test_score": y,
18 | "train_time": end - start,
19 | },
20 | }
21 |
22 |
23 | # neps uses the default values and a confidence in this default value to construct a prior
24 | # that speeds up the search
25 | pipeline_space = dict(
26 | some_float=neps.Float(
27 | lower=1,
28 | upper=1000,
29 | log=True,
30 | prior=900,
31 | prior_confidence="medium",
32 | ),
33 | some_integer=neps.Integer(
34 | lower=0,
35 | upper=50,
36 | prior=35,
37 | prior_confidence="low",
38 | ),
39 | some_cat=neps.Categorical(
40 | choices=["a", "b", "c"],
41 | prior="a",
42 | prior_confidence="high",
43 | ),
44 | )
45 |
46 | logging.basicConfig(level=logging.INFO)
47 | neps.run(
48 | evaluate_pipeline=evaluate_pipeline,
49 | pipeline_space=pipeline_space,
50 | root_directory="results/user_priors_example",
51 | max_evaluations_total=15,
52 | )
53 |
--------------------------------------------------------------------------------
/neps_examples/efficiency/multi_fidelity.py:
--------------------------------------------------------------------------------
1 | import logging
2 |
3 | import numpy as np
4 | from pathlib import Path
5 | import torch
6 | import torch.nn.functional as F
7 | from torch import nn, optim
8 |
9 | import neps
10 |
11 |
12 | class TheModelClass(nn.Module):
13 | """Taken from https://pytorch.org/tutorials/beginner/saving_loading_models.html"""
14 |
15 | def __init__(self):
16 | super().__init__()
17 | self.conv1 = nn.Conv2d(3, 6, 5)
18 | self.pool = nn.MaxPool2d(2, 2)
19 | self.conv2 = nn.Conv2d(6, 16, 5)
20 | self.fc1 = nn.Linear(16 * 5 * 5, 120)
21 | self.fc2 = nn.Linear(120, 84)
22 | self.fc3 = nn.Linear(84, 10)
23 |
24 | def forward(self, x):
25 | x = self.pool(F.relu(self.conv1(x)))
26 | x = self.pool(F.relu(self.conv2(x)))
27 | x = x.view(-1, 16 * 5 * 5)
28 | x = F.relu(self.fc1(x))
29 | x = F.relu(self.fc2(x))
30 | x = self.fc3(x)
31 | return x
32 |
33 |
34 | def get_model_and_optimizer(learning_rate):
35 | """Taken from https://pytorch.org/tutorials/beginner/saving_loading_models.html"""
36 | model = TheModelClass()
37 | optimizer = optim.SGD(model.parameters(), lr=learning_rate, momentum=0.9)
38 | return model, optimizer
39 |
40 |
41 | # Important: Include the "pipeline_directory" and "previous_pipeline_directory" arguments
42 | # in your evaluate_pipeline function. This grants access to NePS's folder system and is
43 | # critical for leveraging efficient multi-fidelity optimization strategies.
44 | # For more details, refer to the working_directory_per_pipeline example in convenience.
45 |
46 |
47 | def evaluate_pipeline(
48 | pipeline_directory: Path, # The path associated with this configuration
49 | previous_pipeline_directory: Path
50 | | None, # The path associated with any previous config
51 | learning_rate: float,
52 | epoch: int,
53 | ) -> dict:
54 | model, optimizer = get_model_and_optimizer(learning_rate)
55 | checkpoint_name = "checkpoint.pth"
56 |
57 | if previous_pipeline_directory is not None:
58 | # Read in state of the model after the previous fidelity rung
59 | checkpoint = torch.load(previous_pipeline_directory / checkpoint_name)
60 | model.load_state_dict(checkpoint["model_state_dict"])
61 | optimizer.load_state_dict(checkpoint["optimizer_state_dict"])
62 | epochs_previously_spent = checkpoint["epoch"]
63 | else:
64 | epochs_previously_spent = 0
65 |
66 | # Train model here ...
67 |
68 | # Save model to disk
69 | torch.save(
70 | {
71 | "epoch": epoch,
72 | "model_state_dict": model.state_dict(),
73 | "optimizer_state_dict": optimizer.state_dict(),
74 | },
75 | pipeline_directory / checkpoint_name,
76 | )
77 |
78 | objective_to_minimize = np.log(learning_rate / epoch) # Replace with actual error
79 | epochs_spent_in_this_call = epoch - epochs_previously_spent # Optional for stopping
80 | return dict(
81 | objective_to_minimize=objective_to_minimize, cost=epochs_spent_in_this_call
82 | )
83 |
84 |
85 | pipeline_space = dict(
86 | learning_rate=neps.Float(lower=1e-4, upper=1e0, log=True),
87 | epoch=neps.Integer(lower=1, upper=10, is_fidelity=True),
88 | )
89 |
90 | logging.basicConfig(level=logging.INFO)
91 | neps.run(
92 | evaluate_pipeline=evaluate_pipeline,
93 | pipeline_space=pipeline_space,
94 | root_directory="results/multi_fidelity_example",
95 | # Optional: Do not start another evaluation after <=50 epochs, corresponds to cost
96 | # field above.
97 | max_cost_total=50,
98 | )
99 |
--------------------------------------------------------------------------------
/neps_examples/efficiency/multi_fidelity_and_expert_priors.py:
--------------------------------------------------------------------------------
1 | import logging
2 |
3 | import numpy as np
4 | import neps
5 |
6 | # This example demonstrates NePS uses both fidelity and expert priors to
7 | # optimize hyperparameters of a pipeline.
8 |
9 | def evaluate_pipeline(float1, float2, integer1, fidelity):
10 | objective_to_minimize = -float(np.sum([float1, float2, integer1])) / fidelity
11 | return objective_to_minimize
12 |
13 |
14 | pipeline_space = dict(
15 | float1=neps.Float(
16 | lower=1,
17 | upper=1000,
18 | log=False,
19 | prior=600,
20 | prior_confidence="medium",
21 | ),
22 | float2=neps.Float(
23 | lower=-10,
24 | upper=10,
25 | prior=0,
26 | prior_confidence="medium",
27 | ),
28 | integer1=neps.Integer(
29 | lower=0,
30 | upper=50,
31 | prior=35,
32 | prior_confidence="low",
33 | ),
34 | fidelity=neps.Integer(
35 | lower=1,
36 | upper=10,
37 | is_fidelity=True,
38 | ),
39 | )
40 |
41 | logging.basicConfig(level=logging.INFO)
42 | neps.run(
43 | evaluate_pipeline=evaluate_pipeline,
44 | pipeline_space=pipeline_space,
45 | root_directory="results/multifidelity_priors",
46 | max_evaluations_total=25, # For an alternate stopping method see multi_fidelity.py
47 | )
48 |
--------------------------------------------------------------------------------
/neps_examples/efficiency/pytorch_lightning_ddp.py:
--------------------------------------------------------------------------------
1 | import logging
2 |
3 | import lightning as L
4 | import torch
5 | import torch.nn as nn
6 | import torch.nn.functional as F
7 | from torch.utils.data import DataLoader, random_split
8 | import neps
9 |
10 | NUM_GPU = 8 # Number of GPUs to use for DDP
11 |
12 |
13 | class ToyModel(nn.Module):
14 | """ Taken from https://pytorch.org/tutorials/intermediate/ddp_tutorial.html """
15 | def __init__(self):
16 | super(ToyModel, self).__init__()
17 | self.net1 = nn.Linear(10, 10)
18 | self.relu = nn.ReLU()
19 | self.net2 = nn.Linear(10, 5)
20 |
21 | def forward(self, x):
22 | return self.net2(self.relu(self.net1(x)))
23 |
24 | class LightningModel(L.LightningModule):
25 | def __init__(self, lr):
26 | super().__init__()
27 | self.lr = lr
28 | self.model = ToyModel()
29 |
30 | def training_step(self, batch, batch_idx):
31 | x, y = batch
32 | y_hat = self.model(x)
33 | loss = F.mse_loss(y_hat, y)
34 | self.log("train_loss", loss, prog_bar=True, sync_dist=True)
35 | return loss
36 |
37 | def validation_step(self, batch, batch_idx):
38 | x, y = batch
39 | y_hat = self.model(x)
40 | loss = F.mse_loss(y_hat, y)
41 | self.log("val_loss", loss, prog_bar=True, sync_dist=True)
42 | return loss
43 |
44 | def test_step(self, batch, batch_idx):
45 | x, y = batch
46 | y_hat = self.model(x)
47 | loss = F.mse_loss(y_hat, y)
48 | self.log("test_loss", loss, prog_bar=True, sync_dist=True)
49 | return loss
50 |
51 | def configure_optimizers(self):
52 | return torch.optim.SGD(self.parameters(), lr=self.lr)
53 |
54 | def evaluate_pipeline(lr=0.1, epoch=20):
55 | L.seed_everything(42)
56 | # Model
57 | model = LightningModel(lr=lr)
58 |
59 | # Generate random tensors for data and labels
60 | data = torch.rand((1000, 10))
61 | labels = torch.rand((1000, 5))
62 |
63 | dataset = list(zip(data, labels))
64 |
65 | train_dataset, val_dataset, test_dataset = random_split(dataset, [600, 200, 200])
66 |
67 | # Define simple data loaders using tensors and slicing
68 | train_dataloader = DataLoader(train_dataset, batch_size=20, shuffle=True)
69 | val_dataloader = DataLoader(val_dataset, batch_size=20, shuffle=False)
70 | test_dataloader = DataLoader(test_dataset, batch_size=20, shuffle=False)
71 |
72 | # Trainer with DDP Strategy
73 | trainer = L.Trainer(gradient_clip_val=0.25,
74 | max_epochs=epoch,
75 | fast_dev_run=False,
76 | strategy='ddp',
77 | devices=NUM_GPU
78 | )
79 | trainer.fit(model, train_dataloader, val_dataloader)
80 | trainer.validate(model, test_dataloader)
81 | return trainer.logged_metrics["val_loss"].item()
82 |
83 |
84 | pipeline_space = dict(
85 | lr=neps.Float(
86 | lower=0.001,
87 | upper=0.1,
88 | log=True,
89 | prior=0.01
90 | ),
91 | epoch=neps.Integer(
92 | lower=1,
93 | upper=3,
94 | is_fidelity=True
95 | )
96 | )
97 |
98 | logging.basicConfig(level=logging.INFO)
99 | neps.run(
100 | evaluate_pipeline=evaluate_pipeline,
101 | pipeline_space=pipeline_space,
102 | root_directory="results/pytorch_lightning_ddp",
103 | max_evaluations_total=5
104 | )
105 |
--------------------------------------------------------------------------------
/neps_examples/efficiency/pytorch_lightning_fsdp.py:
--------------------------------------------------------------------------------
1 | """Based on: https://lightning.ai/docs/pytorch/stable/advanced/model_parallel/fsdp.html
2 |
3 | Mind that this example does not run on Windows at the moment."""
4 |
5 | import torch
6 | import torch.nn.functional as F
7 | from torch.utils.data import DataLoader
8 |
9 | import lightning as L
10 | from lightning.pytorch.strategies import FSDPStrategy
11 | from lightning.pytorch.demos import Transformer, WikiText2
12 |
13 |
14 | class LanguageModel(L.LightningModule):
15 | def __init__(self, vocab_size, lr):
16 | super().__init__()
17 | self.model = Transformer( # 1B parameters
18 | vocab_size=vocab_size,
19 | nlayers=32,
20 | nhid=4096,
21 | ninp=1024,
22 | nhead=64,
23 | )
24 | self.lr = lr
25 |
26 | def training_step(self, batch):
27 | input, target = batch
28 | output = self.model(input, target)
29 | loss = F.nll_loss(output, target.view(-1))
30 | self.log("train_loss", loss, prog_bar=True)
31 | return loss
32 |
33 | def configure_optimizers(self):
34 | return torch.optim.Adam(self.parameters(), lr=self.lr)
35 |
36 |
37 | def evaluate_pipeline(lr=0.1, epoch=20):
38 | L.seed_everything(42)
39 |
40 | # Data
41 | dataset = WikiText2()
42 | train_dataloader = DataLoader(dataset)
43 |
44 | # Model
45 | model = LanguageModel(vocab_size=dataset.vocab_size, lr=lr)
46 |
47 | # Trainer
48 | trainer = L.Trainer(accelerator="cuda", strategy=FSDPStrategy())
49 | trainer.fit(model, train_dataloader, max_epochs=epoch)
50 | return trainer.logged_metrics["train_loss"].detach().item()
51 |
52 |
53 | if __name__ == "__main__":
54 | import neps
55 | import logging
56 |
57 | logging.basicConfig(level=logging.INFO)
58 |
59 | pipeline_space = dict(
60 | lr=neps.Float(
61 | lower=0.0001,
62 | upper=0.1,
63 | log=True,
64 | prior=0.01
65 | ),
66 | epoch=neps.Integer(
67 | lower=1,
68 | upper=3,
69 | is_fidelity=True
70 | )
71 | )
72 |
73 | neps.run(
74 | evaluate_pipeline=evaluate_pipeline,
75 | pipeline_space=pipeline_space,
76 | root_directory="results/pytorch_lightning_fsdp",
77 | max_evaluations_total=5
78 | )
79 |
--------------------------------------------------------------------------------
/neps_examples/efficiency/pytorch_native_ddp.py:
--------------------------------------------------------------------------------
1 | """ Some parts of this code are taken from https://pytorch.org/tutorials/intermediate/ddp_tutorial.html
2 |
3 | Mind that this example does not run on Windows at the moment."""
4 |
5 | import os
6 | import sys
7 | import tempfile
8 | import torch
9 | import torch.distributed as dist
10 | import torch.nn as nn
11 | import torch.optim as optim
12 | import torch.multiprocessing as mp
13 |
14 | from torch.nn.parallel import DistributedDataParallel as DDP
15 |
16 | import neps
17 | import logging
18 |
19 | NUM_GPU = 8 # Number of GPUs to use for DDP
20 |
21 | # On Windows platform, the torch.distributed package only
22 | # supports Gloo backend, FileStore and TcpStore.
23 | # For FileStore, set init_method parameter in init_process_group
24 | # to a local file. Example as follow:
25 | # init_method="file:///f:/libtmp/some_file"
26 | # dist.init_process_group(
27 | # "gloo",
28 | # rank=rank,
29 | # init_method=init_method,
30 | # world_size=world_size)
31 | # For TcpStore, same way as on Linux.
32 |
33 |
34 | def setup(rank, world_size):
35 | os.environ['MASTER_ADDR'] = 'localhost'
36 | os.environ['MASTER_PORT'] = '12355'
37 |
38 | # initialize the process group
39 | dist.init_process_group("gloo", rank=rank, world_size=world_size)
40 |
41 |
42 | def cleanup():
43 | dist.destroy_process_group()
44 |
45 |
46 | class ToyModel(nn.Module):
47 | """ Taken from https://pytorch.org/tutorials/intermediate/ddp_tutorial.html """
48 | def __init__(self):
49 | super(ToyModel, self).__init__()
50 | self.net1 = nn.Linear(10, 10)
51 | self.relu = nn.ReLU()
52 | self.net2 = nn.Linear(10, 5)
53 |
54 | def forward(self, x):
55 | return self.net2(self.relu(self.net1(x)))
56 |
57 |
58 | def demo_basic(rank, world_size, loss_dict, learning_rate, epochs):
59 | """ Taken from https://pytorch.org/tutorials/intermediate/ddp_tutorial.html (modified)"""
60 | print(f"Running basic DDP example on rank {rank}.")
61 | setup(rank, world_size)
62 |
63 | # create model and move it to GPU with id rank
64 | model = ToyModel().to(rank)
65 | ddp_model = DDP(model, device_ids=[rank])
66 |
67 | loss_fn = nn.MSELoss()
68 | optimizer = optim.SGD(ddp_model.parameters(), lr=learning_rate)
69 |
70 | total_loss = 0.0
71 | for epoch in range(epochs):
72 | optimizer.zero_grad()
73 | outputs = ddp_model(torch.randn(20, 10))
74 | labels = torch.randn(20, 5).to(rank)
75 | loss = loss_fn(outputs, labels)
76 | loss.backward()
77 | optimizer.step()
78 | total_loss += loss.item()
79 |
80 | if rank == 0:
81 | print(f"Epoch {epoch} complete")
82 |
83 | loss_dict[rank] = total_loss
84 |
85 | cleanup()
86 | print(f"Finished running basic DDP example on rank {rank}.")
87 |
88 |
89 | def evaluate_pipeline(learning_rate, epochs):
90 | from torch.multiprocessing import Manager
91 | world_size = NUM_GPU # Number of GPUs
92 |
93 | manager = Manager()
94 | loss_dict = manager.dict()
95 |
96 | mp.spawn(demo_basic,
97 | args=(world_size, loss_dict, learning_rate, epochs),
98 | nprocs=world_size,
99 | join=True)
100 |
101 | loss = sum(loss_dict.values()) // world_size
102 | return {'loss': loss}
103 |
104 |
105 | pipeline_space = dict(
106 | learning_rate=neps.Float(lower=10e-7, upper=10e-3, log=True),
107 | epochs=neps.Integer(lower=1, upper=3)
108 | )
109 |
110 | if __name__ == '__main__':
111 | logging.basicConfig(level=logging.INFO)
112 | neps.run(evaluate_pipeline=evaluate_pipeline,
113 | pipeline_space=pipeline_space,
114 | root_directory="results/pytorch_ddp",
115 | max_evaluations_total=25)
116 |
--------------------------------------------------------------------------------
/neps_examples/experimental/freeze_thaw.py:
--------------------------------------------------------------------------------
1 | import logging
2 | from pathlib import Path
3 | import torch
4 | import torch.nn as nn
5 | import torch.optim as optim
6 | from torch.utils.data import DataLoader
7 | from torchvision import datasets, transforms
8 |
9 | import neps
10 | from neps import tblogger
11 | from neps.plot.plot3D import Plotter3D
12 |
13 |
14 | class SimpleNN(nn.Module):
15 | def __init__(self, input_size, num_layers, num_neurons):
16 | super().__init__()
17 | layers = [nn.Flatten()]
18 |
19 | for _ in range(num_layers):
20 | layers.append(nn.Linear(input_size, num_neurons))
21 | layers.append(nn.ReLU())
22 | input_size = num_neurons # Set input size for the next layer
23 |
24 | layers.append(nn.Linear(num_neurons, 10)) # Output layer for 10 classes
25 | self.model = nn.Sequential(*layers)
26 |
27 | def forward(self, x):
28 | return self.model(x)
29 |
30 |
31 | def training_pipeline(
32 | pipeline_directory,
33 | previous_pipeline_directory,
34 | num_layers,
35 | num_neurons,
36 | epochs,
37 | learning_rate,
38 | weight_decay,
39 | ):
40 | """
41 | Trains and validates a simple neural network on the MNIST dataset.
42 |
43 | Args:
44 | num_layers (int): Number of hidden layers in the network.
45 | num_neurons (int): Number of neurons in each hidden layer.
46 | epochs (int): Number of training epochs.
47 | learning_rate (float): Learning rate for the optimizer.
48 | optimizer (str): Name of the optimizer to use ('adam' or 'sgd').
49 |
50 | Returns:
51 | float: The average objective_to_minimize over the validation set after training.
52 |
53 | Raises:
54 | KeyError: If the specified optimizer is not supported.
55 | """
56 | # Transformations applied on each image
57 | transform = transforms.Compose(
58 | [
59 | transforms.ToTensor(),
60 | transforms.Normalize(
61 | (0.1307,), (0.3081,)
62 | ), # Mean and Std Deviation for MNIST
63 | ]
64 | )
65 |
66 | # Loading MNIST dataset
67 | dataset = datasets.MNIST(
68 | root="./.data", train=True, download=True, transform=transform
69 | )
70 | train_set, val_set = torch.utils.data.random_split(dataset, [50000, 10000])
71 | train_loader = DataLoader(train_set, batch_size=64, shuffle=True)
72 | val_loader = DataLoader(val_set, batch_size=1000, shuffle=False)
73 |
74 | model = SimpleNN(28 * 28, num_layers, num_neurons)
75 | criterion = nn.CrossEntropyLoss()
76 |
77 | # Select optimizer
78 | optimizer = optim.AdamW(
79 | model.parameters(), lr=learning_rate, weight_decay=weight_decay
80 | )
81 |
82 | # Loading potential checkpoint
83 | start_epoch = 1
84 | if previous_pipeline_directory is not None:
85 | if (Path(previous_pipeline_directory) / "checkpoint.pt").exists():
86 | states = torch.load(
87 | Path(previous_pipeline_directory) / "checkpoint.pt",
88 | weights_only=False
89 | )
90 | model = states["model"]
91 | optimizer = states["optimizer"]
92 | start_epoch = states["epochs"]
93 |
94 | # Training loop
95 | for epoch in range(start_epoch, epochs + 1):
96 | model.train()
97 | for batch_idx, (data, target) in enumerate(train_loader):
98 | optimizer.zero_grad()
99 | output = model(data)
100 | objective_to_minimize = criterion(output, target)
101 | objective_to_minimize.backward()
102 | optimizer.step()
103 |
104 | # Validation loop
105 | model.eval()
106 | val_objective_to_minimize = 0
107 | val_correct = 0
108 | val_total = 0
109 | with torch.no_grad():
110 | for data, target in val_loader:
111 | output = model(data)
112 | val_objective_to_minimize += criterion(output, target).item()
113 |
114 | # Get the predicted class
115 | _, predicted = torch.max(output.data, 1)
116 |
117 | # Count correct predictions
118 | val_total += target.size(0)
119 | val_correct += (predicted == target).sum().item()
120 |
121 | val_objective_to_minimize /= len(val_loader.dataset)
122 | val_err = 1 - val_correct / val_total
123 |
124 | # Saving checkpoint
125 | states = {
126 | "model": model,
127 | "optimizer": optimizer,
128 | "epochs": epochs,
129 | }
130 | torch.save(states, Path(pipeline_directory) / "checkpoint.pt")
131 |
132 | # Logging
133 | # tblogger.log(
134 | # objective_to_minimize=val_objective_to_minimize,
135 | # current_epoch=epochs,
136 | # # Set to `True` for a live incumbent trajectory.
137 | # write_summary_incumbent=True,
138 | # # Set to `True` for a live objective_to_minimize trajectory for each config.
139 | # writer_config_scalar=True,
140 | # # Set to `True` for live parallel coordinate, scatter plot matrix, and table view.
141 | # writer_config_hparam=True,
142 | # # Appending extra data
143 | # extra_data={
144 | # "train_objective_to_minimize": tblogger.scalar_logging(
145 | # objective_to_minimize.item()
146 | # ),
147 | # "val_err": tblogger.scalar_logging(val_err),
148 | # },
149 | # )
150 |
151 | return val_err
152 |
153 |
154 | if __name__ == "__main__":
155 | logging.basicConfig(level=logging.INFO)
156 |
157 | pipeline_space = {
158 | "learning_rate": neps.Float(1e-5, 1e-1, log=True),
159 | "num_layers": neps.Integer(1, 5),
160 | "num_neurons": neps.Integer(64, 128),
161 | "weight_decay": neps.Float(1e-5, 0.1, log=True),
162 | "epochs": neps.Integer(1, 10, is_fidelity=True),
163 | }
164 |
165 | neps.run(
166 | pipeline_space=pipeline_space,
167 | evaluate_pipeline=training_pipeline,
168 | optimizer="ifbo",
169 | max_evaluations_total=50,
170 | root_directory="./results/ifbo-mnist/",
171 | overwrite_working_directory=False, # set to False for a multi-worker run
172 | )
173 |
174 | # NOTE: this is `experimental` and may not work as expected
175 | ## plotting a 3D plot for learning curves explored by ifbo
176 | plotter = Plotter3D(
177 | run_path="./results/ifbo-mnist/", # same as `root_directory` above
178 | fidelity_key="epochs", # same as `pipeline_space`
179 | )
180 | plotter.plot3D(filename="ifbo")
181 |
--------------------------------------------------------------------------------
/neps_examples/real_world/README.md:
--------------------------------------------------------------------------------
1 | # Real World Examples
2 |
3 | 1. **Image Segmentation Pipeline Hyperparameter Optimization**
4 |
5 | This example demonstrates how to perform hyperparameter optimization (HPO) for an image segmentation pipeline using NePS. The pipeline consists of a ResNet-50 model to segment images model trained on PASCAL Visual Object Classes (VOC) Dataset (http://host.robots.ox.ac.uk/pascal/VOC/).
6 |
7 | We compare the performance of the optimized hyperparameters with the default hyperparameters. using the validation loss achieved on the dataset after training the model with the respective hyperparameters.
8 |
9 | ```bash
10 | python image_segmentation_pipeline_hpo.py
11 | ```
12 |
13 | The search space has been set with the priors set to the hyperparameters found in this base example: https://lightning.ai/lightning-ai/studios/image-segmentation-with-pytorch-lightning
14 |
15 | We run the HPO process for 188 trials and obtain new set of hyperpamereters that outperform the default hyperparameters.
16 |
17 | | Hyperparameter | Prior | Optimized Value |
18 | |----------------|-------|-----------------|
19 | | learning_rate | 0.02 | 0.006745150778442621 |
20 | | batch_size | 4 | 5 |
21 | | momentum | 0.5 | 0.5844767093658447 |
22 | | weight_decay | 0.0001 | 0.00012664785026572645 |
23 |
24 |
25 | 
26 |
27 | The validation loss achieved on the dataset after training the model with the newly sampled hyperparameters is shown in the figure above.
28 |
29 | We compare the validation loss values when the model is trained with the default hyperparameters and the optimized hyperparameters:
30 |
31 | Validation Loss with Default Hyperparameters: 0.114094577729702
32 |
33 | Validation Loss with Optimized Hyperparameters: 0.0997161939740181
34 |
35 | The optimized hyperparameters outperform the default hyperparameters by 12.61%.
36 |
--------------------------------------------------------------------------------
/neps_examples/real_world/image_segmentation_hpo.py:
--------------------------------------------------------------------------------
1 | # Example pipeline used from; https://lightning.ai/lightning-ai/studios/image-segmentation-with-pytorch-lightning
2 |
3 | import os
4 |
5 | import torch
6 | from torchvision import transforms, datasets, models
7 | import lightning as L
8 | from lightning.pytorch.strategies import DDPStrategy
9 | from torch.optim.lr_scheduler import PolynomialLR
10 |
11 |
12 | class LitSegmentation(L.LightningModule):
13 | def __init__(self, iters_per_epoch, lr, momentum, weight_decay):
14 | super().__init__()
15 | self.model = models.segmentation.fcn_resnet50(num_classes=21, aux_loss=True)
16 | self.loss_fn = torch.nn.CrossEntropyLoss()
17 | self.iters_per_epoch = iters_per_epoch
18 | self.lr = lr
19 | self.momentum = momentum
20 | self.weight_decay = weight_decay
21 |
22 | def training_step(self, batch):
23 | images, targets = batch
24 | outputs = self.model(images)['out']
25 | loss = self.loss_fn(outputs, targets.long().squeeze(1))
26 | self.log("train_loss", loss, sync_dist=True)
27 | return loss
28 |
29 | def validation_step(self, batch):
30 | images, targets = batch
31 | outputs = self.model(images)['out']
32 | loss = self.loss_fn(outputs, targets.long().squeeze(1))
33 | self.log("val_loss", loss, sync_dist=True)
34 | return loss
35 |
36 | def configure_optimizers(self):
37 | optimizer = torch.optim.SGD(self.model.parameters(), lr=self.lr, momentum=self.momentum, weight_decay=self.weight_decay)
38 | scheduler = PolynomialLR(
39 | optimizer, total_iters=self.iters_per_epoch * self.trainer.max_epochs, power=0.9
40 | )
41 | return [optimizer], [scheduler]
42 |
43 |
44 |
45 | class SegmentationData(L.LightningDataModule):
46 | def __init__(self, batch_size=4):
47 | super().__init__()
48 | self.batch_size = batch_size
49 |
50 | def prepare_data(self):
51 | dataset_path = ".data/VOC/VOCtrainval_11-May-2012.tar"
52 | if not os.path.exists(dataset_path):
53 | datasets.VOCSegmentation(root=".data/VOC", download=True)
54 |
55 | def train_dataloader(self):
56 | transform = transforms.Compose([
57 | transforms.ToTensor(),
58 | transforms.Resize((256, 256), antialias=True),
59 | transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
60 | ])
61 | target_transform = transforms.Compose([transforms.ToTensor(), transforms.Resize((256, 256), antialias=True)])
62 | train_dataset = datasets.VOCSegmentation(root=".data/VOC", transform=transform, target_transform=target_transform)
63 | return torch.utils.data.DataLoader(train_dataset, batch_size=self.batch_size, shuffle=True, num_workers=16, persistent_workers=True)
64 |
65 | def val_dataloader(self):
66 | transform = transforms.Compose([
67 | transforms.ToTensor(),
68 | transforms.Resize((256, 256), antialias=True),
69 | transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
70 | ])
71 | target_transform = transforms.Compose([transforms.ToTensor(), transforms.Resize((256, 256), antialias=True)])
72 | val_dataset = datasets.VOCSegmentation(root=".data/VOC", year='2012', image_set='val', transform=transform, target_transform=target_transform)
73 | return torch.utils.data.DataLoader(val_dataset, batch_size=self.batch_size, shuffle=False, num_workers=16, persistent_workers=True)
74 |
75 |
76 | def evaluate_pipeline(**kwargs):
77 | data = SegmentationData(kwargs.get("batch_size", 4))
78 | data.prepare_data()
79 | iters_per_epoch = len(data.train_dataloader())
80 | model = LitSegmentation(iters_per_epoch, kwargs.get("lr", 0.02), kwargs.get("momentum", 0.9), kwargs.get("weight_decay", 1e-4))
81 | trainer = L.Trainer(max_epochs=kwargs.get("epoch", 30), strategy=DDPStrategy(find_unused_parameters=True), enable_checkpointing=False)
82 | trainer.fit(model, data)
83 | val_loss = trainer.logged_metrics["val_loss"].detach().item()
84 | return val_loss
85 |
86 |
87 | if __name__ == "__main__":
88 | import neps
89 | import logging
90 |
91 | logging.basicConfig(level=logging.INFO)
92 |
93 | # Search space for hyperparameters
94 | pipeline_space = dict(
95 | lr=neps.Float(
96 | lower=0.0001,
97 | upper=0.1,
98 | log=True,
99 | prior=0.02
100 | ),
101 | momentum=neps.Float(
102 | lower=0.1,
103 | upper=0.9,
104 | prior=0.5
105 | ),
106 | weight_decay=neps.Float(
107 | lower=1e-5,
108 | upper=1e-3,
109 | log=True,
110 | prior=1e-4
111 | ),
112 | epoch=neps.Integer(
113 | lower=10,
114 | upper=30,
115 | is_fidelity=True
116 | ),
117 | batch_size=neps.Integer(
118 | lower=4,
119 | upper=12,
120 | prior=4
121 | ),
122 | )
123 |
124 | neps.run(
125 | evaluate_pipeline=evaluate_pipeline,
126 | pipeline_space=pipeline_space,
127 | root_directory="results/hpo_image_segmentation",
128 | max_evaluations_total=500
129 | )
130 |
--------------------------------------------------------------------------------
/tests/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/automl/neps/0ef3731fa5049336e9a2bb61be7093fc902a96ff/tests/__init__.py
--------------------------------------------------------------------------------
/tests/test_config_encoder.py:
--------------------------------------------------------------------------------
1 | from __future__ import annotations
2 |
3 | import torch
4 |
5 | from neps.space import Categorical, ConfigEncoder, Float, Integer
6 |
7 |
8 | def test_config_encoder_pdist_calculation() -> None:
9 | parameters = {
10 | "a": Categorical(["cat", "mouse", "dog"]),
11 | "b": Integer(1, 10),
12 | "c": Float(1, 10),
13 | }
14 | encoder = ConfigEncoder.from_parameters(parameters)
15 | config1 = {"a": "cat", "b": 1, "c": 1.0}
16 | config2 = {"a": "mouse", "b": 10, "c": 10.0}
17 |
18 | # Same config, no distance
19 | x = encoder.encode([config1, config1])
20 | dist = encoder.pdist(x, square_form=False)
21 | assert dist.item() == 0.0
22 |
23 | # Opposite configs, max distance
24 | x = encoder.encode([config1, config2])
25 | dist = encoder.pdist(x, square_form=False)
26 |
27 | # The first config should have it's p2 euclidean distance as the norm
28 | # of the distances between these two configs, i.e. the distance along the
29 | # diagonal of a unit-square they belong to
30 | _first_config_numerical_encoding = torch.tensor([[0.0, 0.0]], dtype=torch.float64)
31 | _second_config_numerical_encoding = torch.tensor([[1.0, 1.0]], dtype=torch.float64)
32 | _expected_numerical_dist = torch.linalg.norm(
33 | _first_config_numerical_encoding - _second_config_numerical_encoding,
34 | ord=2,
35 | )
36 |
37 | # The categorical distance should just be one, as they are different
38 | _expected_categorical_dist = 1.0
39 |
40 | _expected_dist = _expected_numerical_dist + _expected_categorical_dist
41 | assert torch.isclose(dist, _expected_dist)
42 |
43 |
44 | def test_config_encoder_pdist_squareform() -> None:
45 | parameters = {
46 | "a": Categorical(["cat", "mouse", "dog"]),
47 | "b": Integer(1, 10),
48 | "c": Float(1, 10),
49 | }
50 | encoder = ConfigEncoder.from_parameters(parameters)
51 | config1 = {"a": "cat", "b": 1, "c": 1.0}
52 | config2 = {"a": "dog", "b": 5, "c": 5}
53 | config3 = {"a": "mouse", "b": 10, "c": 10.0}
54 |
55 | # Same config, no distance
56 | x = encoder.encode([config1, config2, config3])
57 | dist = encoder.pdist(x, square_form=False)
58 |
59 | # 3 possible distances
60 | assert dist.shape == (3,)
61 | torch.testing.assert_close(
62 | dist,
63 | torch.tensor([1.6285, 2.4142, 1.7857], dtype=torch.float64),
64 | atol=1e-4,
65 | rtol=1e-4,
66 | )
67 |
68 | dist_sq = encoder.pdist(x, square_form=True)
69 | assert dist_sq.shape == (3, 3)
70 |
71 | # Distance to self along diagonal should be 0
72 | torch.testing.assert_close(dist_sq.diagonal(), torch.zeros(3, dtype=torch.float64))
73 |
74 | # Should be symmetric
75 | torch.testing.assert_close(dist_sq, dist_sq.T)
76 |
--------------------------------------------------------------------------------
/tests/test_examples.py:
--------------------------------------------------------------------------------
1 | from __future__ import annotations
2 |
3 | import logging
4 | import os
5 | import runpy
6 | from pathlib import Path
7 |
8 | import pytest
9 | from neps_examples import ci_examples, core_examples
10 |
11 |
12 | @pytest.fixture(autouse=True)
13 | def use_tmpdir(tmp_path, request):
14 | os.chdir(tmp_path)
15 | yield
16 | os.chdir(request.config.invocation_dir)
17 |
18 |
19 | # https://stackoverflow.com/a/59745629
20 | # Fail tests if there is a logging.error
21 | @pytest.fixture(autouse=True)
22 | def no_logs_gte_error(caplog):
23 | yield
24 | errors = [
25 | record for record in caplog.get_records("call") if record.levelno >= logging.ERROR
26 | ]
27 | assert not errors
28 |
29 |
30 | examples_folder = Path(__file__, "..", "..", "neps_examples").resolve()
31 | core_examples_scripts = [examples_folder / f"{example}.py" for example in core_examples]
32 | ci_examples_scripts = [examples_folder / f"{example}.py" for example in ci_examples]
33 |
34 |
35 | @pytest.mark.core_examples
36 | @pytest.mark.parametrize("example", core_examples_scripts, ids=core_examples)
37 | def test_core_examples(example):
38 | if example.name == "analyse.py":
39 | # Run hyperparameters example to have something to analyse
40 | runpy.run_path(str(core_examples_scripts[0]), run_name="__main__")
41 |
42 | if example.name in (
43 | "architecture.py",
44 | "architecture_and_hyperparameters.py",
45 | "hierarchical_architecture.py",
46 | "expert_priors_for_architecture_and_hyperparameters.py",
47 | ):
48 | pytest.xfail("Architecture were removed temporarily")
49 |
50 | runpy.run_path(str(example), run_name="__main__")
51 |
52 |
53 | @pytest.mark.ci_examples
54 | @pytest.mark.parametrize("example", ci_examples_scripts, ids=ci_examples)
55 | def test_ci_examples(example):
56 | test_core_examples(example)
57 |
--------------------------------------------------------------------------------
/tests/test_runtime/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/automl/neps/0ef3731fa5049336e9a2bb61be7093fc902a96ff/tests/test_runtime/__init__.py
--------------------------------------------------------------------------------
/tests/test_runtime/test_default_report_values.py:
--------------------------------------------------------------------------------
1 | from __future__ import annotations
2 |
3 | from pathlib import Path
4 |
5 | from pytest_cases import fixture
6 |
7 | from neps.optimizers import OptimizerInfo
8 | from neps.optimizers.algorithms import random_search
9 | from neps.runtime import DefaultWorker
10 | from neps.space import Float, SearchSpace
11 | from neps.state import (
12 | DefaultReportValues,
13 | NePSState,
14 | OnErrorPossibilities,
15 | OptimizationState,
16 | SeedSnapshot,
17 | Trial,
18 | WorkerSettings,
19 | )
20 |
21 |
22 | @fixture
23 | def neps_state(tmp_path: Path) -> NePSState:
24 | return NePSState.create_or_load(
25 | path=tmp_path / "neps_state",
26 | optimizer_info=OptimizerInfo(name="blah", info={"nothing": "here"}),
27 | optimizer_state=OptimizationState(
28 | budget=None, seed_snapshot=SeedSnapshot.new_capture(), shared_state={}
29 | ),
30 | )
31 |
32 |
33 | def test_default_values_on_error(
34 | neps_state: NePSState,
35 | ) -> None:
36 | optimizer = random_search(pipeline_space=SearchSpace({"a": Float(0, 1)}))
37 | settings = WorkerSettings(
38 | on_error=OnErrorPossibilities.IGNORE,
39 | default_report_values=DefaultReportValues(
40 | objective_value_on_error=2.4, # <- Highlight
41 | cost_value_on_error=2.4, # <- Highlight
42 | learning_curve_on_error=[2.4, 2.5], # <- Highlight
43 | ),
44 | max_evaluations_total=None,
45 | include_in_progress_evaluations_towards_maximum=False,
46 | max_cost_total=None,
47 | max_evaluations_for_worker=1,
48 | max_evaluation_time_total_seconds=None,
49 | max_wallclock_time_for_worker_seconds=None,
50 | max_evaluation_time_for_worker_seconds=None,
51 | max_cost_for_worker=None,
52 | batch_size=None,
53 | )
54 |
55 | def eval_function(*args, **kwargs) -> float:
56 | raise ValueError("This is an error")
57 |
58 | worker = DefaultWorker.new(
59 | state=neps_state,
60 | optimizer=optimizer,
61 | evaluation_fn=eval_function,
62 | settings=settings,
63 | )
64 | worker.run()
65 |
66 | trials = neps_state.lock_and_read_trials()
67 | n_crashed = sum(
68 | trial.metadata.state == Trial.State.CRASHED is not None
69 | for trial in trials.values()
70 | )
71 | assert len(trials) == 1
72 | assert n_crashed == 1
73 |
74 | assert neps_state.lock_and_get_next_pending_trial() is None
75 | assert len(neps_state.lock_and_get_errors()) == 1
76 |
77 | trial = trials.popitem()[1]
78 | assert trial.metadata.state == Trial.State.CRASHED
79 | assert trial.report is not None
80 | assert trial.report.objective_to_minimize == 2.4
81 | assert trial.report.cost == 2.4
82 | assert trial.report.learning_curve == [2.4, 2.5]
83 |
84 |
85 | def test_default_values_on_not_specified(
86 | neps_state: NePSState,
87 | ) -> None:
88 | optimizer = random_search(SearchSpace({"a": Float(0, 1)}))
89 | settings = WorkerSettings(
90 | on_error=OnErrorPossibilities.IGNORE,
91 | default_report_values=DefaultReportValues(
92 | cost_if_not_provided=2.4,
93 | learning_curve_if_not_provided=[2.4, 2.5],
94 | ),
95 | max_evaluations_total=None,
96 | include_in_progress_evaluations_towards_maximum=False,
97 | max_cost_total=None,
98 | max_evaluations_for_worker=1,
99 | max_evaluation_time_total_seconds=None,
100 | max_wallclock_time_for_worker_seconds=None,
101 | max_evaluation_time_for_worker_seconds=None,
102 | max_cost_for_worker=None,
103 | batch_size=None,
104 | )
105 |
106 | def eval_function(*args, **kwargs) -> float:
107 | return 1.0
108 |
109 | worker = DefaultWorker.new(
110 | state=neps_state,
111 | optimizer=optimizer,
112 | evaluation_fn=eval_function,
113 | settings=settings,
114 | )
115 | worker.run()
116 |
117 | trials = neps_state.lock_and_read_trials()
118 | n_sucess = sum(
119 | trial.metadata.state == Trial.State.SUCCESS is not None
120 | for trial in trials.values()
121 | )
122 | assert len(trials) == 1
123 | assert n_sucess == 1
124 |
125 | assert neps_state.lock_and_get_next_pending_trial() is None
126 | assert len(neps_state.lock_and_get_errors()) == 0
127 |
128 | trial = trials.popitem()[1]
129 | assert trial.metadata.state == Trial.State.SUCCESS
130 | assert trial.report is not None
131 | assert trial.report.cost == 2.4
132 | assert trial.report.learning_curve == [2.4, 2.5]
133 |
134 |
135 | def test_default_value_objective_to_minimize_curve_take_objective_to_minimize_value(
136 | neps_state: NePSState,
137 | ) -> None:
138 | optimizer = random_search(SearchSpace({"a": Float(0, 1)}))
139 | settings = WorkerSettings(
140 | on_error=OnErrorPossibilities.IGNORE,
141 | default_report_values=DefaultReportValues(
142 | learning_curve_if_not_provided="objective_to_minimize"
143 | ),
144 | max_evaluations_total=None,
145 | include_in_progress_evaluations_towards_maximum=False,
146 | max_cost_total=None,
147 | max_evaluations_for_worker=1,
148 | max_evaluation_time_total_seconds=None,
149 | max_wallclock_time_for_worker_seconds=None,
150 | max_evaluation_time_for_worker_seconds=None,
151 | max_cost_for_worker=None,
152 | batch_size=None,
153 | )
154 |
155 | LOSS = 1.0
156 |
157 | def eval_function(*args, **kwargs) -> float:
158 | return LOSS
159 |
160 | worker = DefaultWorker.new(
161 | state=neps_state,
162 | optimizer=optimizer,
163 | evaluation_fn=eval_function,
164 | settings=settings,
165 | )
166 | worker.run()
167 |
168 | trials = neps_state.lock_and_read_trials()
169 | n_sucess = sum(
170 | trial.metadata.state == Trial.State.SUCCESS is not None
171 | for trial in trials.values()
172 | )
173 | assert len(trials) == 1
174 | assert n_sucess == 1
175 |
176 | assert neps_state.lock_and_get_next_pending_trial() is None
177 | assert len(neps_state.lock_and_get_errors()) == 0
178 |
179 | trial = trials.popitem()[1]
180 | assert trial.metadata.state == Trial.State.SUCCESS
181 | assert trial.report is not None
182 | assert trial.report.learning_curve == [LOSS]
183 |
--------------------------------------------------------------------------------
/tests/test_runtime/test_error_handling_strategies.py:
--------------------------------------------------------------------------------
1 | from __future__ import annotations
2 |
3 | import contextlib
4 | from dataclasses import dataclass
5 | from pathlib import Path
6 |
7 | import pytest
8 | from pytest_cases import fixture, parametrize
9 |
10 | from neps.exceptions import WorkerRaiseError
11 | from neps.optimizers import OptimizerInfo
12 | from neps.optimizers.algorithms import random_search
13 | from neps.runtime import DefaultWorker
14 | from neps.space import Float, SearchSpace
15 | from neps.state import (
16 | DefaultReportValues,
17 | NePSState,
18 | OnErrorPossibilities,
19 | OptimizationState,
20 | SeedSnapshot,
21 | Trial,
22 | WorkerSettings,
23 | )
24 |
25 |
26 | @fixture
27 | def neps_state(tmp_path: Path) -> NePSState:
28 | return NePSState.create_or_load(
29 | path=tmp_path / "neps_state",
30 | optimizer_info=OptimizerInfo(name="blah", info={"nothing": "here"}),
31 | optimizer_state=OptimizationState(
32 | budget=None,
33 | seed_snapshot=SeedSnapshot.new_capture(),
34 | shared_state=None,
35 | ),
36 | )
37 |
38 |
39 | @parametrize(
40 | "on_error",
41 | [OnErrorPossibilities.RAISE_ANY_ERROR, OnErrorPossibilities.RAISE_WORKER_ERROR],
42 | )
43 | def test_worker_raises_when_error_in_self(
44 | neps_state: NePSState,
45 | on_error: OnErrorPossibilities,
46 | ) -> None:
47 | optimizer = random_search(SearchSpace({"a": Float(0, 1)}))
48 | settings = WorkerSettings(
49 | on_error=on_error, # <- Highlight
50 | default_report_values=DefaultReportValues(),
51 | max_evaluations_total=None,
52 | include_in_progress_evaluations_towards_maximum=False,
53 | max_cost_total=None,
54 | max_evaluations_for_worker=1,
55 | max_evaluation_time_total_seconds=None,
56 | max_wallclock_time_for_worker_seconds=None,
57 | max_evaluation_time_for_worker_seconds=None,
58 | max_cost_for_worker=None,
59 | batch_size=None,
60 | )
61 |
62 | def eval_function(*args, **kwargs) -> float:
63 | raise ValueError("This is an error")
64 |
65 | worker = DefaultWorker.new(
66 | state=neps_state,
67 | optimizer=optimizer,
68 | evaluation_fn=eval_function,
69 | settings=settings,
70 | )
71 | with pytest.raises(WorkerRaiseError):
72 | worker.run()
73 |
74 | trials = neps_state.lock_and_read_trials()
75 | n_crashed = sum(
76 | trial.metadata.state == Trial.State.CRASHED is not None
77 | for trial in trials.values()
78 | )
79 | assert len(trials) == 1
80 | assert n_crashed == 1
81 |
82 | assert neps_state.lock_and_get_next_pending_trial() is None
83 | assert len(neps_state.lock_and_get_errors()) == 1
84 |
85 |
86 | def test_worker_raises_when_error_in_other_worker(neps_state: NePSState) -> None:
87 | optimizer = random_search(SearchSpace({"a": Float(0, 1)}))
88 | settings = WorkerSettings(
89 | on_error=OnErrorPossibilities.RAISE_ANY_ERROR, # <- Highlight
90 | default_report_values=DefaultReportValues(),
91 | max_evaluations_total=None,
92 | include_in_progress_evaluations_towards_maximum=False,
93 | max_cost_total=None,
94 | max_evaluations_for_worker=1,
95 | max_evaluation_time_total_seconds=None,
96 | max_wallclock_time_for_worker_seconds=None,
97 | max_evaluation_time_for_worker_seconds=None,
98 | max_cost_for_worker=None,
99 | batch_size=None,
100 | )
101 |
102 | def evaler(*args, **kwargs) -> float:
103 | raise ValueError("This is an error")
104 |
105 | worker1 = DefaultWorker.new(
106 | state=neps_state,
107 | optimizer=optimizer,
108 | evaluation_fn=evaler,
109 | settings=settings,
110 | )
111 | worker2 = DefaultWorker.new(
112 | state=neps_state,
113 | optimizer=optimizer,
114 | evaluation_fn=evaler,
115 | settings=settings,
116 | )
117 |
118 | # Worker1 should run 1 and error out
119 | with contextlib.suppress(WorkerRaiseError):
120 | worker1.run()
121 |
122 | # Worker2 should not run and immeditaly error out, however
123 | # it will have loaded in a serialized error
124 | with pytest.raises(WorkerRaiseError):
125 | worker2.run()
126 |
127 | trials = neps_state.lock_and_read_trials()
128 | n_crashed = sum(
129 | trial.metadata.state == Trial.State.CRASHED is not None
130 | for trial in trials.values()
131 | )
132 | assert len(trials) == 1
133 | assert n_crashed == 1
134 |
135 | assert neps_state.lock_and_get_next_pending_trial() is None
136 | assert len(neps_state.lock_and_get_errors()) == 1
137 |
138 |
139 | @pytest.mark.parametrize(
140 | "on_error",
141 | [OnErrorPossibilities.IGNORE, OnErrorPossibilities.RAISE_WORKER_ERROR],
142 | )
143 | def test_worker_does_not_raise_when_error_in_other_worker(
144 | neps_state: NePSState,
145 | on_error: OnErrorPossibilities,
146 | ) -> None:
147 | optimizer = random_search(SearchSpace({"a": Float(0, 1)}))
148 | settings = WorkerSettings(
149 | on_error=on_error, # <- Highlight
150 | default_report_values=DefaultReportValues(),
151 | max_evaluations_total=None,
152 | include_in_progress_evaluations_towards_maximum=False,
153 | max_cost_total=None,
154 | max_evaluations_for_worker=1,
155 | max_evaluation_time_total_seconds=None,
156 | max_wallclock_time_for_worker_seconds=None,
157 | max_evaluation_time_for_worker_seconds=None,
158 | max_cost_for_worker=None,
159 | batch_size=None,
160 | )
161 |
162 | @dataclass
163 | class _Eval:
164 | do_raise: bool
165 |
166 | def __call__(self, *args, **kwargs) -> float: # noqa: ARG002
167 | if self.do_raise:
168 | raise ValueError("This is an error")
169 | return 10
170 |
171 | evaler = _Eval(do_raise=True)
172 |
173 | worker1 = DefaultWorker.new(
174 | state=neps_state,
175 | optimizer=optimizer,
176 | evaluation_fn=evaler,
177 | settings=settings,
178 | )
179 | worker2 = DefaultWorker.new(
180 | state=neps_state,
181 | optimizer=optimizer,
182 | evaluation_fn=evaler,
183 | settings=settings,
184 | )
185 |
186 | # Worker1 should run 1 and error out
187 | evaler.do_raise = True
188 | with contextlib.suppress(WorkerRaiseError):
189 | worker1.run()
190 | assert worker1.worker_cumulative_eval_count == 1
191 |
192 | # Worker2 should run successfully
193 | evaler.do_raise = False
194 | worker2.run()
195 | assert worker2.worker_cumulative_eval_count == 1
196 |
197 | trials = neps_state.lock_and_read_trials()
198 | n_success = sum(
199 | trial.metadata.state == Trial.State.SUCCESS is not None
200 | for trial in trials.values()
201 | )
202 | n_crashed = sum(
203 | trial.metadata.state == Trial.State.CRASHED is not None
204 | for trial in trials.values()
205 | )
206 | assert n_success == 1
207 | assert n_crashed == 1
208 | assert len(trials) == 2
209 |
210 | assert neps_state.lock_and_get_next_pending_trial() is None
211 | assert len(neps_state.lock_and_get_errors()) == 1
212 |
--------------------------------------------------------------------------------
/tests/test_samplers.py:
--------------------------------------------------------------------------------
1 | from __future__ import annotations
2 |
3 | import torch
4 | from pytest_cases import parametrize
5 |
6 | from neps.sampling import BorderSampler, Prior, Sampler, Sobol, Uniform, WeightedSampler
7 | from neps.space import Domain
8 |
9 |
10 | def _make_centered_prior(ndim: int) -> Prior:
11 | return Prior.from_domains_and_centers(
12 | domains=[Domain.unit_float() for _ in range(ndim)],
13 | centers=[(0.5, 0.5) for _ in range(ndim)],
14 | )
15 |
16 |
17 | @parametrize(
18 | "sampler",
19 | [
20 | Sobol(ndim=3),
21 | BorderSampler(ndim=3),
22 | Uniform(ndim=3),
23 | # Convenence method for making a distribution around center points
24 | _make_centered_prior(ndim=3),
25 | WeightedSampler(
26 | [Uniform(ndim=3), _make_centered_prior(3), Sobol(ndim=3)],
27 | weights=torch.tensor([0.5, 0.25, 0.25]).tolist(),
28 | ),
29 | ],
30 | )
31 | def test_sampler_samples_into_domain(sampler: Sampler) -> None:
32 | assert sampler.ncols == 3
33 |
34 | domain_to_sample_into = Domain.integer(12, 15)
35 | for _ in range(10):
36 | x = sampler.sample(
37 | n=5,
38 | to=domain_to_sample_into,
39 | seed=None,
40 | )
41 |
42 | assert x.shape == (5, 3)
43 | assert (x >= 12).all()
44 | assert (x <= 15).all()
45 |
46 | x = sampler.sample(
47 | n=torch.Size((2, 1)),
48 | to=domain_to_sample_into,
49 | seed=None,
50 | )
51 | assert x.shape == (2, 1, 3)
52 | assert (x >= 12).all()
53 | assert (x <= 15).all()
54 |
55 |
56 | @parametrize(
57 | "prior",
58 | [
59 | Uniform(ndim=3),
60 | # Convenence method for making a distribution around center points
61 | _make_centered_prior(ndim=3),
62 | ],
63 | )
64 | def test_priors_give_positive_pdfs(prior: Prior) -> None:
65 | # NOTE: The uniform prior does not check that
66 | assert prior.ncols == 3
67 | domain = Domain.floating(10, 100)
68 |
69 | x = prior.sample(n=5, to=domain, seed=None)
70 | assert x.shape == (5, 3)
71 | assert (x >= 10).all()
72 | assert (x <= 100).all()
73 |
74 | probs = prior.pdf(x, frm=domain)
75 | assert (probs >= 0).all()
76 | assert probs.shape == (5,)
77 |
78 | x = prior.sample(n=torch.Size((2, 1)), to=domain, seed=None)
79 | assert x.shape == (2, 1, 3)
80 | assert (x >= 10).all()
81 | assert (x <= 100).all()
82 |
83 | probs = prior.pdf(x, frm=domain)
84 | assert (probs >= 0).all()
85 | assert probs.shape == (2, 1)
86 |
--------------------------------------------------------------------------------
/tests/test_search_space.py:
--------------------------------------------------------------------------------
1 | from __future__ import annotations
2 |
3 | import pytest
4 |
5 | from neps import Categorical, Constant, Float, Integer, SearchSpace
6 |
7 |
8 | def test_search_space_orders_parameters_by_name():
9 | unsorted = SearchSpace({"b": Float(0, 1), "c": Float(0, 1), "a": Float(0, 1)})
10 | expected = SearchSpace({"a": Float(0, 1), "b": Float(0, 1), "c": Float(0, 1)})
11 | assert unsorted == expected
12 |
13 |
14 | def test_multipe_fidelities_raises_error():
15 | # We should allow this at some point, but until we do, raise an error
16 | with pytest.raises(ValueError, match="neps only supports one fidelity parameter"):
17 | SearchSpace(
18 | {"a": Float(0, 1, is_fidelity=True), "b": Float(0, 1, is_fidelity=True)}
19 | )
20 |
21 |
22 | def test_sorting_of_parameters_into_subsets():
23 | elements = {
24 | "a": Float(0, 1),
25 | "b": Integer(0, 10),
26 | "c": Categorical(["a", "b", "c"]),
27 | "d": Float(0, 1, is_fidelity=True),
28 | "x": Constant("x"),
29 | }
30 | space = SearchSpace(elements)
31 | assert space.elements == elements
32 | assert space.categoricals == {"c": elements["c"]}
33 | assert space.numerical == {"a": elements["a"], "b": elements["b"]}
34 | assert space.fidelities == {"d": elements["d"]}
35 | assert space.constants == {"x": "x"}
36 |
37 | assert space.searchables == {
38 | "a": elements["a"],
39 | "b": elements["b"],
40 | "c": elements["c"],
41 | }
42 | assert space.fidelity == ("d", elements["d"])
43 |
--------------------------------------------------------------------------------
/tests/test_search_space_parsing.py:
--------------------------------------------------------------------------------
1 | from __future__ import annotations
2 |
3 | from typing import Any
4 |
5 | import pytest
6 |
7 | from neps.space import Categorical, Constant, Float, Integer, Parameter, parsing
8 |
9 |
10 | @pytest.mark.parametrize(
11 | ("config", "expected"),
12 | [
13 | (
14 | (0, 1),
15 | Integer(0, 1),
16 | ),
17 | (
18 | ("1e3", "1e5"),
19 | Integer(1e3, 1e5),
20 | ),
21 | (
22 | ("1e-3", "1e-1"),
23 | Float(1e-3, 1e-1),
24 | ),
25 | (
26 | (1e-5, 1e-1),
27 | Float(1e-5, 1e-1),
28 | ),
29 | (
30 | {"type": "float", "lower": 0.00001, "upper": "1e-1", "log": True},
31 | Float(0.00001, 0.1, log=True),
32 | ),
33 | (
34 | {"type": "int", "lower": 3, "upper": 30, "is_fidelity": True},
35 | Integer(3, 30, is_fidelity=True),
36 | ),
37 | (
38 | {
39 | "type": "int",
40 | "lower": "1e2",
41 | "upper": "3E4",
42 | "log": True,
43 | "is_fidelity": False,
44 | },
45 | Integer(100, 30000, log=True, is_fidelity=False),
46 | ),
47 | (
48 | {"type": "float", "lower": "3.3e-5", "upper": "1.5E-1"},
49 | Float(3.3e-5, 1.5e-1),
50 | ),
51 | (
52 | {"type": "cat", "choices": [2, "sgd", "10e-3"]},
53 | Categorical([2, "sgd", 0.01]),
54 | ),
55 | (
56 | 0.5,
57 | Constant(0.5),
58 | ),
59 | (
60 | "1e3",
61 | Constant(1000),
62 | ),
63 | (
64 | {"type": "cat", "choices": ["adam", "sgd", "rmsprop"]},
65 | Categorical(["adam", "sgd", "rmsprop"]),
66 | ),
67 | (
68 | {
69 | "lower": 0.00001,
70 | "upper": 0.1,
71 | "log": True,
72 | "prior": 3.3e-2,
73 | "prior_confidence": "high",
74 | },
75 | Float(0.00001, 0.1, log=True, prior=3.3e-2, prior_confidence="high"),
76 | ),
77 | ],
78 | )
79 | def test_type_deduction_succeeds(config: Any, expected: Parameter) -> None:
80 | parameter = parsing.as_parameter(config)
81 | assert parameter == expected
82 |
83 |
84 | @pytest.mark.parametrize(
85 | "config",
86 | [
87 | {"type": int, "lower": 0.00001, "upper": 0.1, "log": True}, # Invalid type
88 | (1, 2.5), # int and float
89 | (1, 2, 3), # too many values
90 | (1,), # too few values
91 | ],
92 | )
93 | def test_parsing_fails(config: dict[str, Any]) -> None:
94 | with pytest.raises(ValueError):
95 | parsing.as_parameter(config)
96 |
--------------------------------------------------------------------------------
/tests/test_state/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/automl/neps/0ef3731fa5049336e9a2bb61be7093fc902a96ff/tests/test_state/__init__.py
--------------------------------------------------------------------------------
/tests/test_state/test_filebased_neps_state.py:
--------------------------------------------------------------------------------
1 | """NOTE: These tests are pretty specific to the filebased state implementation.
2 | This could be generalized if we end up with a server based implementation but
3 | for now we're just testing the filebased implementation.
4 | """
5 |
6 | from __future__ import annotations
7 |
8 | from pathlib import Path
9 | from typing import Any
10 |
11 | import pytest
12 | from pytest_cases import fixture, parametrize
13 |
14 | from neps.exceptions import NePSError, TrialNotFoundError
15 | from neps.optimizers import OptimizerInfo
16 | from neps.state.err_dump import ErrDump
17 | from neps.state.neps_state import NePSState
18 | from neps.state.optimizer import BudgetInfo, OptimizationState
19 | from neps.state.seed_snapshot import SeedSnapshot
20 |
21 |
22 | @fixture
23 | @parametrize("budget_info", [BudgetInfo(max_cost_total=10, used_cost_budget=0), None])
24 | @parametrize("shared_state", [{"a": "b"}, {}])
25 | def optimizer_state(
26 | budget_info: BudgetInfo | None,
27 | shared_state: dict[str, Any],
28 | ) -> OptimizationState:
29 | return OptimizationState(
30 | budget=budget_info,
31 | seed_snapshot=SeedSnapshot.new_capture(),
32 | shared_state=shared_state,
33 | )
34 |
35 |
36 | @fixture
37 | @parametrize(
38 | "optimizer_info",
39 | [OptimizerInfo(name="blah", info={"a": "b"})],
40 | )
41 | def optimizer_info(optimizer_info: OptimizerInfo) -> OptimizerInfo:
42 | return optimizer_info
43 |
44 |
45 | def test_create_with_new_filebased_neps_state(
46 | tmp_path: Path,
47 | optimizer_info: OptimizerInfo,
48 | optimizer_state: OptimizationState,
49 | ) -> None:
50 | new_path = tmp_path / "neps_state"
51 | neps_state = NePSState.create_or_load(
52 | path=new_path,
53 | optimizer_info=optimizer_info,
54 | optimizer_state=optimizer_state,
55 | )
56 | assert neps_state.lock_and_get_optimizer_info() == optimizer_info
57 | assert neps_state.lock_and_get_optimizer_state() == optimizer_state
58 | assert neps_state.all_trial_ids() == []
59 | assert neps_state.lock_and_read_trials() == {}
60 | assert neps_state.lock_and_get_errors() == ErrDump(errs=[])
61 | assert neps_state.lock_and_get_next_pending_trial() is None
62 | assert neps_state.lock_and_get_next_pending_trial(n=10) == []
63 |
64 | with pytest.raises(TrialNotFoundError):
65 | assert neps_state.lock_and_get_trial_by_id("1")
66 |
67 |
68 | def test_create_or_load_with_load_filebased_neps_state(
69 | tmp_path: Path,
70 | optimizer_info: OptimizerInfo,
71 | optimizer_state: OptimizationState,
72 | ) -> None:
73 | new_path = tmp_path / "neps_state"
74 | neps_state = NePSState.create_or_load(
75 | path=new_path,
76 | optimizer_info=optimizer_info,
77 | optimizer_state=optimizer_state,
78 | )
79 |
80 | # NOTE: This isn't a defined way to do this but we should check
81 | # that we prioritize what's in the existing data over what
82 | # was passed in.
83 | different_state = OptimizationState(
84 | budget=BudgetInfo(max_cost_total=20, used_cost_budget=10),
85 | seed_snapshot=SeedSnapshot.new_capture(),
86 | shared_state=None,
87 | )
88 | neps_state2 = NePSState.create_or_load(
89 | path=new_path,
90 | optimizer_info=optimizer_info,
91 | optimizer_state=different_state,
92 | )
93 | assert neps_state == neps_state2
94 |
95 |
96 | def test_load_on_existing_neps_state(
97 | tmp_path: Path,
98 | optimizer_info: OptimizerInfo,
99 | optimizer_state: OptimizationState,
100 | ) -> None:
101 | new_path = tmp_path / "neps_state"
102 | neps_state = NePSState.create_or_load(
103 | path=new_path,
104 | optimizer_info=optimizer_info,
105 | optimizer_state=optimizer_state,
106 | )
107 |
108 | neps_state2 = NePSState.create_or_load(path=new_path, load_only=True)
109 | assert neps_state == neps_state2
110 |
111 |
112 | def test_new_or_load_on_existing_neps_state_with_different_optimizer_info(
113 | tmp_path: Path,
114 | optimizer_info: OptimizerInfo,
115 | optimizer_state: OptimizationState,
116 | ) -> None:
117 | new_path = tmp_path / "neps_state"
118 | NePSState.create_or_load(
119 | path=new_path,
120 | optimizer_info=optimizer_info,
121 | optimizer_state=optimizer_state,
122 | )
123 |
124 | with pytest.raises(NePSError):
125 | NePSState.create_or_load(
126 | path=new_path,
127 | optimizer_info=OptimizerInfo(name="randomlll", info={"e": "f"}),
128 | optimizer_state=optimizer_state,
129 | )
130 |
--------------------------------------------------------------------------------
/tests/test_state/test_rng.py:
--------------------------------------------------------------------------------
1 | from __future__ import annotations
2 |
3 | import random
4 | from collections.abc import Callable
5 | from pathlib import Path
6 |
7 | import numpy as np
8 | import pytest
9 | import torch
10 |
11 | from neps.state.seed_snapshot import SeedSnapshot
12 |
13 |
14 | @pytest.mark.parametrize(
15 | "make_ints",
16 | [
17 | lambda: [random.randint(0, 100) for _ in range(10)],
18 | lambda: list(np.random.randint(0, 100, (10,))),
19 | lambda: list(torch.randint(0, 100, (10,))),
20 | ],
21 | )
22 | def test_randomstate_consistent(
23 | tmp_path: Path, make_ints: Callable[[], list[int]]
24 | ) -> None:
25 | random.seed(42)
26 | np.random.seed(42)
27 | torch.manual_seed(42)
28 |
29 | seed_dir = tmp_path / "seed_dir"
30 | seed_dir.mkdir(exist_ok=True, parents=True)
31 |
32 | seed_state = SeedSnapshot.new_capture()
33 | integers_1 = make_ints()
34 |
35 | seed_state.set_as_global_seed_state()
36 |
37 | integers_2 = make_ints()
38 | assert integers_1 == integers_2
39 |
--------------------------------------------------------------------------------