├── .github
└── workflows
│ ├── docs.yml
│ └── release.yml
├── .gitignore
├── CHANGELOG.md
├── CODE_OF_CONDUCT.md
├── LICENSE
├── MANIFEST.in
├── README.md
├── docs
├── example_runner.py
├── examples
│ └── index.md
├── index.md
└── reference
│ ├── index.md
│ ├── optimizers
│ ├── index.md
│ ├── optimizer.md
│ ├── quick.md
│ └── random.md
│ ├── predictors
│ ├── cost.md
│ ├── index.md
│ ├── perf.md
│ └── predictor.md
│ └── tuners
│ ├── image_cls_tuner.md
│ ├── index.md
│ └── quicktuner.md
├── examples
├── define_search_space.py
├── index.md
├── metatrain.py
├── quicktuning.py
└── step_by_step.py
├── mkdocs.yml
├── pyproject.toml
└── src
└── qtt
├── __init__.py
├── finetune
└── image
│ └── classification
│ ├── __init__.py
│ ├── fn.py
│ └── utils.py
├── optimizers
├── __init__.py
├── optimizer.py
├── quick.py
└── rndm.py
├── predictors
├── __init__.py
├── cost.py
├── data.py
├── models.py
├── perf.py
├── predictor.py
└── utils.py
├── tuners
├── __init__.py
├── image
│ ├── __init__.py
│ └── classification
│ │ ├── __init__.py
│ │ └── tuner.py
└── quick.py
└── utils
├── __init__.py
├── config.py
├── log_utils.py
├── pretrained.py
└── setup.py
/.github/workflows/docs.yml:
--------------------------------------------------------------------------------
1 | name: docs
2 | on:
3 | workflow_dispatch:
4 | permissions:
5 | contents: write
6 | jobs:
7 | deploy:
8 | runs-on: ubuntu-latest
9 | steps:
10 | - uses: actions/checkout@v4
11 | - uses: actions/setup-python@v5
12 | with:
13 | python-version: 3.x
14 | - uses: actions/cache@v2
15 | with:
16 | key: ${{ github.ref }}
17 | path: .cache
18 | - name: "Install dependencies"
19 | run: python -m pip install mkdocs-material mkdocs-autorefs mkdocs-glightbox mkdocs-literate-nav mkdocstrings[python] mkdocs-gen-files mkdocs-awesome-pages-plugin typing-extensions more-itertools pillow cairosvg mike markdown-exec
20 | - name: "Build docs"
21 | run: mkdocs gh-deploy --force
--------------------------------------------------------------------------------
/.github/workflows/release.yml:
--------------------------------------------------------------------------------
1 | name: release
2 | on:
3 | workflow_dispatch:
4 | permissions:
5 | contents: write
6 | jobs:
7 | test-docs:
8 | runs-on: ubuntu-latest
9 | steps:
10 | - uses: actions/checkout@v4
11 | - uses: actions/setup-python@v5
12 | with:
13 | python-version: "3.11"
14 | cache: pip
15 | - run: python -m pip install ".[dev]"
16 | - run: mkdocs build --clean --strict
17 | # Need to ensure we bump before we create any artifacts
18 | bump:
19 | runs-on: ubuntu-latest
20 | needs: [test-docs]
21 | steps:
22 | - uses: actions/checkout@v4
23 | with:
24 | fetch-tags: 1 # Essential to later commitizen
25 | fetch-depth: 0 # Recommended by the action
26 | token: ${{ secrets.PUSH_ACCESS }}
27 | - run: git tag # Debug statement
28 | - name: Create bump and changelog
29 | uses: commitizen-tools/commitizen-action@master
30 | id: cz
31 | with:
32 | github_token: ${{ secrets.PUSH_ACCESS }}
33 | debug: true
34 | changelog_increment_filename: changelog-increment.md
35 | - run: |
36 | mkdir changelog-output
37 | mv changelog-increment.md changelog-output/changelog-increment.md
38 | cat changelog-output/changelog-increment.md
39 | - name: Upload the changelog
40 | uses: actions/upload-artifact@v4
41 | with:
42 | name: changelog
43 | path: changelog-output
44 |
45 | build:
46 | needs: [bump]
47 | runs-on: ubuntu-latest
48 | steps:
49 | - uses: actions/checkout@v4
50 | with:
51 | ref: "main" # Necessary to download the latest of main as this will have been updated on the step before
52 | fetch-tags: 1
53 | fetch-depth: 0
54 | - uses: actions/setup-python@v5
55 | with:
56 | python-version: "3.11"
57 | cache: pip
58 | - run: python -m pip install build
59 | - run: python -m build --sdist
60 | - name: Store the distribution packages
61 | uses: actions/upload-artifact@v4
62 | with:
63 | name: build-output
64 | path: dist
65 |
66 | docs:
67 | needs: [bump]
68 | runs-on: ubuntu-latest
69 | steps:
70 | - uses: actions/checkout@v4
71 | - uses: actions/setup-python@v5
72 | with:
73 | python-version: 3.11
74 | - uses: actions/cache@v2
75 | with:
76 | key: ${{ github.ref }}
77 | path: .cache
78 | - run: python -m pip install mkdocs-material mkdocs-autorefs mkdocs-glightbox mkdocs-literate-nav mkdocstrings[python] mkdocs-gen-files mkdocs-awesome-pages-plugin typing-extensions more-itertools pillow cairosvg mike markdown-exec
79 | - run: mkdocs gh-deploy --force
80 | release:
81 | needs: [docs, build]
82 | runs-on: ubuntu-latest
83 | steps:
84 | - uses: actions/checkout@v4
85 | with:
86 | ref: "main" # Necessary to download the latest of main as this will have been updated on the step before
87 | fetch-tags: 1
88 | fetch-depth: 0
89 | - name: Download the build artifiact
90 | uses: actions/download-artifact@v4
91 | with:
92 | name: build-output
93 | path: dist
94 | - run: ls -R dist
95 | - name: Download the changelog
96 | uses: actions/download-artifact@v4
97 | with:
98 | name: changelog
99 | path: changelog-output
100 | - run: |
101 | ls -R changelog-output
102 | mv changelog-output/changelog-increment.md changelog-increment.md
103 | cat changelog-increment.md
104 | - name: "Create Github Release"
105 | env:
106 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
107 | run: |
108 | current_version=$(git tag | sort --version-sort | tail -n 1)
109 | echo "Release for ${current_version}"
110 | gh release create \
111 | --generate-notes \
112 | --notes-file changelog-increment.md \
113 | --verify-tag \
114 | "${current_version}" "dist/quicktunetool-${current_version}.tar.gz"
115 | publish:
116 | needs: [release]
117 | runs-on: ubuntu-latest
118 | permissions:
119 | id-token: write
120 | steps:
121 | - uses: actions/checkout@v4
122 | with:
123 | ref: "main" # Necessary to download the latest of main as this will have been updated on the step before
124 | - uses: actions/download-artifact@v4
125 | with:
126 | name: build-output
127 | path: dist
128 | - name: Publish distribution 📦 to PyPI
129 | uses: pypa/gh-action-pypi-publish@release/v1
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # Byte-compiled / optimized / DLL files
2 | __pycache__/
3 | *.py[cod]
4 | *$py.class
5 |
6 | # C extensions
7 | *.so
8 |
9 | # Distribution / packaging
10 | .Python
11 | build/
12 | develop-eggs/
13 | dist/
14 | downloads/
15 | eggs/
16 | .eggs/
17 | lib/
18 | lib64/
19 | parts/
20 | sdist/
21 | var/
22 | wheels/
23 | share/python-wheels/
24 | *.egg-info/
25 | .installed.cfg
26 | *.egg
27 | MANIFEST
28 |
29 | # PyInstaller
30 | # Usually these files are written by a python script from a template
31 | # before PyInstaller builds the exe, so as to inject date/other infos into it.
32 | *.manifest
33 | *.spec
34 |
35 | # Installer logs
36 | pip-log.txt
37 | pip-delete-this-directory.txt
38 |
39 | # Unit test / coverage reports
40 | htmlcov/
41 | .tox/
42 | .nox/
43 | .coverage
44 | .coverage.*
45 | .cache
46 | nosetests.xml
47 | coverage.xml
48 | *.cover
49 | *.py,cover
50 | .hypothesis/
51 | .pytest_cache/
52 | cover/
53 |
54 | # Translations
55 | *.mo
56 | *.pot
57 |
58 | # Django stuff:
59 | *.log
60 | local_settings.py
61 | db.sqlite3
62 | db.sqlite3-journal
63 |
64 | # Flask stuff:
65 | instance/
66 | .webassets-cache
67 |
68 | # Scrapy stuff:
69 | .scrapy
70 |
71 | # Sphinx documentation
72 | docs/_build/
73 |
74 | # PyBuilder
75 | .pybuilder/
76 | target/
77 |
78 | # Jupyter Notebook
79 | .ipynb_checkpoints
80 |
81 | # IPython
82 | profile_default/
83 | ipython_config.py
84 |
85 | # pyenv
86 | # For a library or package, you might want to ignore these files since the code is
87 | # intended to run in multiple environments; otherwise, check them in:
88 | # .python-version
89 |
90 | # pipenv
91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies
93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not
94 | # install all needed dependencies.
95 | #Pipfile.lock
96 |
97 | # poetry
98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
99 | # This is especially recommended for binary packages to ensure reproducibility, and is more
100 | # commonly ignored for libraries.
101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
102 | #poetry.lock
103 |
104 | # pdm
105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
106 | #pdm.lock
107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
108 | # in version control.
109 | # https://pdm.fming.dev/#use-with-ide
110 | .pdm.toml
111 |
112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
113 | __pypackages__/
114 |
115 | # Celery stuff
116 | celerybeat-schedule
117 | celerybeat.pid
118 |
119 | # SageMath parsed files
120 | *.sage.py
121 |
122 | # Environments
123 | .env
124 | .venv
125 | env/
126 | venv/
127 | ENV/
128 | env.bak/
129 | venv.bak/
130 |
131 | # Spyder project settings
132 | .spyderproject
133 | .spyproject
134 |
135 | # Rope project settings
136 | .ropeproject
137 |
138 | # mkdocs documentation
139 | /site
140 |
141 | # mypy
142 | .mypy_cache/
143 | .dmypy.json
144 | dmypy.json
145 |
146 | # Pyre type checker
147 | .pyre/
148 |
149 | # pytype static type analyzer
150 | .pytype/
151 |
152 | # Cython debug symbols
153 | cython_debug/
154 |
155 | # PyCharm
156 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can
157 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
158 | # and can be added to the global gitignore or merged into this file. For a more nuclear
159 | # option (not recommended) you can uncomment the following to ignore the entire idea folder.
160 | #.idea/
161 |
--------------------------------------------------------------------------------
/CHANGELOG.md:
--------------------------------------------------------------------------------
1 | ## 0.0.4 (2025-01-09)
2 |
3 | ### Fix
4 |
5 | - remove changelog_start_rev (pyproject.toml)
6 | - **pyproject.toml**: wrong version numbers
7 |
8 | ## 0.0.3 (2025-01-09)
9 |
10 | ### Feat
11 |
12 | - added versioning
13 |
14 | ### Fix
15 |
16 | - bug in release script
17 | - docs-workflow
18 | - docs-workflow
19 | - pyproject
20 | - workflow
21 |
--------------------------------------------------------------------------------
/CODE_OF_CONDUCT.md:
--------------------------------------------------------------------------------
1 | # Code of Conduct
2 |
3 | We are a community based on openness, as well as friendly and didactic discussions.
4 |
5 | We aspire to treat everybody equally, and value their contributions.
6 |
7 | We think scikit-learn is a great open source project and copied their code of conduct.
8 |
9 | Decisions are made based on technical merit and consensus.
10 |
11 | Code is not the only way to help the project. Reviewing pull requests,
12 | answering questions to help others on mailing lists or issues, organizing and
13 | teaching tutorials, working on the website, improving the documentation, are
14 | all priceless contributions.
15 |
16 | We abide by the principles of openness, respect, and consideration of others of
17 | the Python Software Foundation: https://www.python.org/psf/codeofconduct/
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | BSD 3-Clause License
2 |
3 | Copyright (c) 2024, AutoML-Freiburg-Hannover
4 |
5 | Redistribution and use in source and binary forms, with or without
6 | modification, are permitted provided that the following conditions are met:
7 |
8 | 1. Redistributions of source code must retain the above copyright notice, this
9 | list of conditions and the following disclaimer.
10 |
11 | 2. Redistributions in binary form must reproduce the above copyright notice,
12 | this list of conditions and the following disclaimer in the documentation
13 | and/or other materials provided with the distribution.
14 |
15 | 3. Neither the name of the copyright holder nor the names of its
16 | contributors may be used to endorse or promote products derived from
17 | this software without specific prior written permission.
18 |
19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
23 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
25 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
26 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
27 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 |
--------------------------------------------------------------------------------
/MANIFEST.in:
--------------------------------------------------------------------------------
1 | recursive-include *
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Quick-Tune-Tool
2 |
3 | [](https://pypi.python.org/pypi/quicktunetool)
4 | [](https://pypi.python.org/pypi/quicktunetool)
5 | [](https://github.com/astral-sh/ruff)
6 | [](CONTRIBUTING.md)
7 |
8 | **A Practical Tool and User Guide for Automatically Finetuning Pretrained Models**
9 |
10 | > Quick-Tune-Tool is an automated solution for selecting and finetuning pretrained models across various machine learning domains. Built upon the Quick-Tune algorithm, this tool bridges the gap between research-code and practical applications, making model finetuning accessible and efficient for practitioners.
11 |
12 |
13 | ## Installation
14 | ```bash
15 | pip install quicktunetool
16 | # or
17 | git clone https://github.com/automl/quicktunetool
18 | pip install -e quicktunetool # Use -e for editable mode
19 | ```
20 |
21 |
22 | ## Usage
23 |
24 | A simple example for using Quick-Tune-Tool with a pretrained optimizer for image classification:
25 |
26 | ```python
27 | from qtt import QuickTuner, get_pretrained_optimizer
28 | from qtt.finetune.image.classification import fn
29 |
30 | # Load task information and meta-features
31 | task_info, metafeat = extract_task_info_metafeat("path/to/dataset")
32 |
33 | # Initialize the optimizer
34 | optimizer = get_pretrained_optimizer("mtlbm/full")
35 | optimizer.setup(128, metafeat)
36 |
37 | # Create QuickTuner instance and run
38 | qt = QuickTuner(optimizer, fn)
39 | qt.run(task_info, time_budget=3600)
40 | ```
41 |
42 | This code snippet demonstrates how to run QTT on an image dataset in just a few lines of code.
43 |
44 | ## Contributing
45 |
46 | Contributions are welcome! Please follow these steps:
47 |
48 | 1. Fork the repository
49 | 2. Create a new branch (`git checkout -b feature/YourFeature`)
50 | 3. Commit your changes (`git commit -m 'Add your feature'`)
51 | 4. Push to the branch (`git push origin feature/YourFeature`)
52 | 5. Open a pull request
53 |
54 | For any questions or suggestions, please contact the maintainers.
55 |
56 | ## Project Status
57 |
58 | - ✅ Active development
59 |
60 | ## Support
61 |
62 | - 📝 [Documentation](https://automl.github.io/quicktunetool/)
63 | - 🐛 [Issue Tracker](https://github.com/automl/quicktunetool/issues)
64 | - 💬 [Discussions](https://github.com/automl/quicktunetool/discussions)
65 |
66 | ## License
67 |
68 | This project is licensed under the BSD License - see the LICENSE file for details.
69 |
70 | ## References
71 |
72 | The concepts and methodologies of QuickTuneTool are detailed in the following workshop paper:
73 |
74 | ```
75 | @inproceedings{
76 | rapant2024quicktunetool,
77 | title={Quick-Tune-Tool: A Practical Tool and its User Guide for Automatically Finetuning Pretrained Models},
78 | author={Ivo Rapant and Lennart Purucker and Fabio Ferreira and Sebastian Pineda Arango and Arlind Kadra and Josif Grabocka and Frank Hutter},
79 | booktitle={AutoML Conference 2024 (Workshop Track)},
80 | year={2024},
81 | url={https://openreview.net/forum?id=d0Hapti3Uc}
82 | }
83 | ```
84 |
85 | If you use QuickTuneTool in your research, please also cite the following paper:
86 |
87 | ```
88 | @inproceedings{
89 | arango2024quicktune,
90 | title={Quick-Tune: Quickly Learning Which Pretrained Model to Finetune and How},
91 | author={Sebastian Pineda Arango and Fabio Ferreira and Arlind Kadra and Frank Hutter and Josif Grabocka},
92 | booktitle={The Twelfth International Conference on Learning Representations},
93 | year={2024},
94 | url={https://openreview.net/forum?id=tqh1zdXIra}
95 | }
96 | ```
97 |
98 | ---
99 |
100 | Made with ❤️ by https://github.com/automl
--------------------------------------------------------------------------------
/docs/example_runner.py:
--------------------------------------------------------------------------------
1 | """Generates the examples pages."""
2 |
3 | from __future__ import annotations
4 |
5 | import logging
6 | import os
7 | import textwrap
8 | from dataclasses import dataclass
9 | from itertools import takewhile
10 | from pathlib import Path
11 | from typing import Any
12 | from typing_extensions import override
13 |
14 | import mkdocs_gen_files
15 | from more_itertools import first_true, peekable
16 |
17 | logger = logging.getLogger("mkdocs")
18 |
19 | RUN_EXAMPLES_ENV_VAR = "DOC_RENDER_EXAMPLES"
20 |
21 |
22 | @dataclass
23 | class CodeSegment:
24 | lines: list[str]
25 | session: str
26 | exec: bool
27 |
28 | def code(self, code: list[str]) -> str:
29 | points_start = first_true(code, pred=lambda _l: _l.startswith("# 1."))
30 | if points_start is not None:
31 | points_start_index = code.index(points_start)
32 |
33 | points = [""]
34 | points.extend([_l.lstrip("#")[1:] for _l in code[points_start_index:]])
35 | points.append("")
36 |
37 | body = code[:points_start_index]
38 | else:
39 | points = []
40 | body = code
41 |
42 | # Trim off any excess leading lines which only have whitespace
43 | while body and body[0].strip() == "":
44 | body.pop(0)
45 |
46 | hl_lines = []
47 | for i, line in enumerate(body):
48 | _l = line.rstrip()
49 | if "" in _l:
50 | hl_lines.append(str(i + 1))
51 | _l = _l.replace("", "").rstrip()
52 |
53 | for strip in ["# !", "#!", "#"]:
54 | if _l.endswith(strip):
55 | _l = _l[: -len(strip)]
56 |
57 | body[i] = _l
58 |
59 | if any(hl_lines):
60 | hl_lines = " ".join(hl_lines)
61 | hl_string = f'hl_lines="{hl_lines}"'
62 | else:
63 | hl_string = ""
64 |
65 | # We generate two tabs if executing
66 | if self.exec:
67 | indented_body = "\n".join(f" {_l}" for _l in body)
68 |
69 | code_annotations = " ".join(
70 | [
71 | "{",
72 | ".python",
73 | ".annotate",
74 | hl_string,
75 | "}",
76 | ],
77 | )
78 | tab1 = "\n".join(
79 | [
80 | '=== "Code"',
81 | "",
82 | f" ``` {code_annotations}",
83 | indented_body,
84 | " ```",
85 | *[f" {point}" for point in points],
86 | "",
87 | ],
88 | )
89 |
90 | run_annotations = " ".join(
91 | [
92 | "{",
93 | ".python",
94 | f"session='{self.session}'",
95 | 'exec="True"',
96 | 'result="python"',
97 | "}",
98 | ],
99 | )
100 |
101 | tab2 = "\n".join(
102 | [
103 | '=== "Run"',
104 | "",
105 | f" ``` {run_annotations}",
106 | indented_body,
107 | " ```",
108 | ],
109 | )
110 |
111 | return "\n".join([tab1, "", tab2])
112 |
113 | annotations = " ".join(["{", ".python", ".annotate", hl_string, "}"])
114 | top = f"```{annotations}"
115 | bottom = "```"
116 |
117 | s = [top, *body, bottom, *points]
118 | body = "\n".join(s)
119 | return body
120 |
121 | @override
122 | def __str__(self) -> str:
123 | return self.code(self.lines)
124 |
125 |
126 | @dataclass
127 | class CommentSegment:
128 | lines: list[str]
129 |
130 | @override
131 | def __str__(self) -> str:
132 | return "\n".join(self.lines)
133 |
134 |
135 | @dataclass
136 | class Example:
137 | name: str
138 | filepath: Path
139 | description: str
140 | segments: list[CodeSegment | CommentSegment]
141 |
142 | @classmethod
143 | def should_execute(cls, *, name: str, runnable: bool) -> bool:
144 | if not runnable:
145 | return False
146 |
147 | env_var = os.environ.get(RUN_EXAMPLES_ENV_VAR, "all")
148 | if env_var in ("false", "", "0", "no", "off"):
149 | return False
150 |
151 | if env_var == "all":
152 | return True
153 |
154 | examples_to_exec = [
155 | example.lstrip().rstrip() for example in env_var.lower().split(",")
156 | ]
157 | return name.lower() in examples_to_exec
158 |
159 | @classmethod
160 | def header_flags(cls, line: str) -> dict[str, Any] | None:
161 | prefix = "# Flags:"
162 | if not line.startswith(prefix):
163 | return None
164 |
165 | line = line[len(prefix) :]
166 | flags = [line.strip() for line in line.split(",")]
167 |
168 | results = {}
169 |
170 | results["doc-runnable"] = any(flag.lower() == "doc-runnable" for flag in flags)
171 | return results
172 |
173 | @classmethod
174 | def from_file(cls, path: Path) -> Example:
175 | with path.open() as f:
176 | lines = f.readlines()
177 |
178 | lines = iter(lines)
179 |
180 | # First line is the name of the example to show
181 | name = next(lines).strip().replace('"""', "")
182 | potential_flag_line = next(lines)
183 | flags = cls.header_flags(potential_flag_line)
184 | if flags is None:
185 | # Prepend the potential flag line back to the lines
186 | lines = iter([potential_flag_line, *lines])
187 | flags = {}
188 |
189 | # Lines leading up to the second triple quote are the description
190 | description = "".join(takewhile(lambda _l: not _l.startswith('"""'), lines))
191 |
192 | segments: list[CodeSegment | CommentSegment] = []
193 |
194 | # The rest is interspersed with triple quotes and code blocks
195 | # We need to wrap the code blocks in triple backticks while
196 | # removing the triple quotes for the comment blocks
197 | remaining = peekable(lines)
198 | while remaining.peek(None) is not None:
199 | # If we encounter triple backticks we remove them and just add the lines
200 | # in, up until the point we hit the next set of backticks
201 | if remaining.peek().startswith('"""'):
202 | # Skip the triple quotes
203 | next(remaining)
204 | ls = list(takewhile(lambda _l: not _l.startswith('"""'), remaining))
205 | comment_segment = CommentSegment([line.rstrip() for line in ls])
206 | segments.append(comment_segment)
207 |
208 | # Otherwise we wrap the line in triple backticks until we hit the next
209 | # set of triple quotes
210 | else:
211 | ls = list(takewhile(lambda _l: not _l.startswith('"""'), remaining))
212 | code_segment = CodeSegment(
213 | [line.rstrip() for line in ls],
214 | session=name,
215 | exec=cls.should_execute(
216 | name=name,
217 | runnable=flags.get("doc-runnable", False),
218 | ),
219 | )
220 | segments.append(code_segment)
221 |
222 | remaining.prepend('"""') # Stick back in so we can find it next itr
223 |
224 | return cls(name, path, description, segments)
225 |
226 | def header(self) -> str:
227 | return f"# {self.name}"
228 |
229 | def description_header(self) -> str:
230 | return "\n".join(
231 | [
232 | "## Description",
233 | self.description,
234 | ],
235 | )
236 |
237 | def generate_doc(self) -> str:
238 | return "\n".join(
239 | [
240 | self.header(),
241 | self.copy_section(),
242 | self.description_header(),
243 | *map(str, self.segments),
244 | ],
245 | )
246 |
247 | def copy_section(self) -> str:
248 | body = "\n".join(
249 | [
250 | "```python",
251 | *[
252 | "\n".join(segment.lines)
253 | for segment in self.segments
254 | if isinstance(segment, CodeSegment)
255 | ],
256 | "```",
257 | ],
258 | )
259 | indented_body = textwrap.indent(body, " " * 4)
260 | header = (
261 | f'??? quote "Expand to copy'
262 | f' `{self.filepath}` :material-content-copy: (top right)"'
263 | )
264 | return "\n".join(
265 | [
266 | header,
267 | "",
268 | indented_body,
269 | "",
270 | ],
271 | )
272 |
273 |
274 | if os.environ.get(RUN_EXAMPLES_ENV_VAR, "all") in ("false", "", "0", "no", "off"):
275 | logger.warning(
276 | f"Env variable {RUN_EXAMPLES_ENV_VAR} not set - not running examples."
277 | " Use `just docs-full` to run and render examples.",
278 | )
279 |
280 | for path in sorted(Path("examples").rglob("*.py")):
281 | module_path = path.relative_to("examples").with_suffix("")
282 | doc_path = path.relative_to("examples").with_suffix(".md")
283 | full_doc_path = Path("examples", doc_path)
284 |
285 | parts = tuple(module_path.parts)
286 | filename = parts[-1]
287 |
288 | if filename.startswith("_"):
289 | continue
290 |
291 | example = Example.from_file(path)
292 | with mkdocs_gen_files.open(full_doc_path, "w") as f:
293 | f.write(example.generate_doc())
294 |
295 | toc_name = example.name
296 | mkdocs_gen_files.set_edit_path(full_doc_path, path)
297 |
--------------------------------------------------------------------------------
/docs/examples/index.md:
--------------------------------------------------------------------------------
1 | # Examples
2 | This houses the examples for the project.
3 | Use the navigation bar to the left to view more.
4 |
--------------------------------------------------------------------------------
/docs/index.md:
--------------------------------------------------------------------------------
1 | # Quick-Tune-Tool
2 |
3 | [](https://pypi.python.org/pypi/quicktunetool)
4 | [](https://pypi.python.org/pypi/quicktunetool)
5 | [](https://github.com/astral-sh/ruff)
6 | 
7 |
8 | **A Practical Tool and User Guide for Automatically Finetuning Pretrained Models**
9 |
10 | > Quick-Tune-Tool is an automated solution for selecting and finetuning pretrained models across various machine learning domains. Built upon the Quick-Tune algorithm, this tool bridges the gap between research-code and practical applications, making model finetuning accessible and efficient for practitioners.
11 |
12 | ## Installation
13 | ```bash
14 | pip install quicktunetool
15 | # or
16 | git clone https://github.com/automl/quicktunetool
17 | pip install -e quicktunetool # Use -e for editable mode
18 | ```
19 |
20 | ## Usage
21 |
22 | A simple example for using Quick-Tune-Tool with a pretrained optimizer for image classification:
23 |
24 | ```python
25 | from qtt import QuickTuner, get_pretrained_optimizer
26 | from qtt.finetune.image.classification import fn
27 |
28 | # Load task information and meta-features
29 | task_info, metafeat = extract_task_info_metafeat("path/to/dataset")
30 |
31 | # Initialize the optimizer
32 | optimizer = get_pretrained_optimizer("mtlbm/micro")
33 | optimizer.setup(128, metafeat)
34 |
35 | # Create QuickTuner instance and run
36 | qt = QuickTuner(optimizer, fn)
37 | qt.run(task_info, time_budget=3600)
38 | ```
39 |
40 | This code snippet demonstrates how to run QTT on an image dataset in just a few lines of code.
41 |
42 | ## Contributing
43 |
44 | Contributions are welcome! Please follow these steps:
45 |
46 | 1. Fork the repository
47 | 2. Create a new branch (`git checkout -b feature/YourFeature`)
48 | 3. Commit your changes (`git commit -m 'Add your feature'`)
49 | 4. Push to the branch (`git push origin feature/YourFeature`)
50 | 5. Open a pull request
51 |
52 | For any questions or suggestions, please contact the maintainers.
53 |
54 | ## Project Status
55 |
56 | - ✅ Active development
57 |
58 | ## Support
59 |
60 | - 📝 [Documentation](https://automl.github.io/quicktunetool/)
61 | - 🐛 [Issue Tracker](https://github.com/automl/quicktunetool/issues)
62 | - 💬 [Discussions](https://github.com/automl/quicktunetool/discussions)
63 |
64 | ## License
65 |
66 | This project is licensed under the BSD License - see the LICENSE file for details.
67 |
68 | ## References
69 |
70 | The concepts and methodologies of QuickTuneTool are detailed in the following workshop paper:
71 |
72 | ```
73 | @inproceedings{
74 | rapant2024quicktunetool,
75 | title={Quick-Tune-Tool: A Practical Tool and its User Guide for Automatically Finetuning Pretrained Models},
76 | author={Ivo Rapant and Lennart Purucker and Fabio Ferreira and Sebastian Pineda Arango and Arlind Kadra and Josif Grabocka and Frank Hutter},
77 | booktitle={AutoML Conference 2024 (Workshop Track)},
78 | year={2024},
79 | url={https://openreview.net/forum?id=d0Hapti3Uc}
80 | }
81 | ```
82 |
83 | If you use QuickTuneTool in your research, please also cite the following paper:
84 |
85 | ```
86 | @inproceedings{
87 | arango2024quicktune,
88 | title={Quick-Tune: Quickly Learning Which Pretrained Model to Finetune and How},
89 | author={Sebastian Pineda Arango and Fabio Ferreira and Arlind Kadra and Frank Hutter and Josif Grabocka},
90 | booktitle={The Twelfth International Conference on Learning Representations},
91 | year={2024},
92 | url={https://openreview.net/forum?id=tqh1zdXIra}
93 | }
94 | ```
95 |
96 | ---
97 |
98 | Made with ❤️ by https://github.com/automl
--------------------------------------------------------------------------------
/docs/reference/index.md:
--------------------------------------------------------------------------------
1 | # Code References
2 |
3 | This section provides references for the core components of Quick-Tune-Tool, describing the primary modules and classes that make up the tool's architecture. The code is organized into three main parts: **Optimizers**, **Predictors**, and **Tuners**.
4 |
5 | ---
6 |
7 | ## 1. Optimizers
8 |
9 | The Optimizers module is responsible for suggesting configurations for evaluation, using various optimization strategies. Available optimizers include:
10 |
11 | - **QuickTune Optimizer**
12 | - File: `optimizers/quick.py`
13 | - Implements the QuickTune algorithm, balancing multi-fidelity expected improvement with cost estimation to select configurations.
14 |
15 | - **Random Search Optimizer**
16 | - File: `optimizers/random.py`
17 | - Provides a basic random search optimizer as a baseline for comparison with other optimization strategies.
18 |
19 | ---
20 |
21 | ## 2. Predictors
22 |
23 | The Predictors module includes components that estimate model performance and finetuning costs, enabling efficient configuration selection.
24 |
25 | - **Performance Predictor**
26 | - File: `predictors/perf.py`
27 | - Uses meta-learning to estimate the potential performance of a model configuration based on historical data and auxiliary task information.
28 |
29 | - **Cost Predictor**
30 | - File: `predictors/cost.py`
31 | - Evaluates the computational cost associated with different finetuning configurations, helping to balance resource efficiency with optimization goals.
32 |
33 | ---
34 |
35 | ## 3. Tuners
36 |
37 | The Tuners module coordinates the tuning process, managing environment setup, experiment flow, and result handling.
38 |
39 | - **QuickTuner**
40 | - File: `tuners/quick.py`
41 | - Serves as the central class that manages the tuning process, integrating optimizers and predictors to manage iterative evaluations and updates.
42 |
43 | - **Image-Classification**
44 | - File: `tuners/image/classification/tuner.py`
45 | - A specialized tuner for image classification, offering a reduced interface where users simply provide the path to the image dataset.
46 |
47 | ---
48 |
49 | ## Additional Resources
50 |
51 | - **Finetuning Scripts**
52 | - Directory: `scripts/`
53 | - Functions used to evaluate configurations, returning performance metrics for each step.
54 |
55 | - **Utility Scripts**
56 | - Directory: `utils/`
57 | - A collection of helper functions and utilities to support data processing, result logging, and other ancillary tasks.
58 |
59 | ---
60 |
61 | Refer to each module's in-code documentation for further details on function arguments, usage examples, and dependencies.
62 |
--------------------------------------------------------------------------------
/docs/reference/optimizers/index.md:
--------------------------------------------------------------------------------
1 | # Overview
2 |
3 | The `Optimizer` class serves as a base class within the Quick-Tune-Tool, providing low-level functionality. It is designed to support flexible configuration management and interact with tuners during the optimization process. Key aspects of the class include directory setup, model saving, and interfacing methods for requesting and reporting trial evaluations.
4 |
5 | Here’s an overview of the [`Optimizer`][qtt.optimizers.optimizer] class:
6 |
7 | #### Core Methods
8 |
9 | - **`ask`**: Abstract method that must be implemented in subclasses. It requests a new configuration trial from the optimizer, returning it as a dictionary. Raises `NotImplementedError` if not overridden.
10 |
11 | - **`tell`**: Accepts and processes a report (result) from a trial evaluation. This method allows the optimizer to record outcomes for each configuration and adjust future suggestions accordingly. Supports both single and multiple trial reports.
12 |
13 | - **`ante`**: A placeholder method for pre-processing tasks to be performed before requesting a configuration trial (used by tuners). Can be overridden in subclasses for custom pre-processing.
14 |
15 | - **`post`**: A placeholder for post-processing tasks, executed after a trial evaluation has been submitted. Designed
16 |
17 | This class is intended to be extended for specific optimization strategies, with `ask` and `tell` as the primary methods for interaction with tuners.
18 |
19 | ---
20 |
21 | ### Available Optimizers
22 |
23 | - [**`RandomOptimizer`**][qtt.optimizers.rndm]
24 | - [**`QuickOptimizer`**][qtt.optimizers.quick]
--------------------------------------------------------------------------------
/docs/reference/optimizers/optimizer.md:
--------------------------------------------------------------------------------
1 | ::: qtt.optimizers.optimizer
--------------------------------------------------------------------------------
/docs/reference/optimizers/quick.md:
--------------------------------------------------------------------------------
1 | ::: qtt.optimizers.quick
--------------------------------------------------------------------------------
/docs/reference/optimizers/random.md:
--------------------------------------------------------------------------------
1 | ::: qtt.optimizers.rndm
--------------------------------------------------------------------------------
/docs/reference/predictors/cost.md:
--------------------------------------------------------------------------------
1 | ::: qtt.predictors.cost
--------------------------------------------------------------------------------
/docs/reference/predictors/index.md:
--------------------------------------------------------------------------------
1 | # Overview
2 |
3 | The `Predictor` class serves as a base class for implementing predictive models within the Quick-Tune-Tool. It provides core functionality for model setup, data handling, training, and persistence (saving/loading), allowing specific predictive models to extend and customize these methods.
4 |
5 | #### Core Methods
6 |
7 | - **`fit`** and **`_fit`**:
8 | - `fit`: Public method for training the model. It takes feature data `X`, target labels `y`, verbosity level, and any additional arguments.
9 | - `_fit`: Abstract method where specific model training logic is implemented. Models inheriting from `Predictor` should override `_fit` to implement their own fitting procedures.
10 |
11 | - **`preprocess`** and **`_preprocess`**:
12 | - `preprocess`: Wrapper method that calls `_preprocess` to prepare data for fitting or prediction.
13 | - `_preprocess`: Abstract method where data transformation logic should be added. Designed to clean and structure input data before model training or inference.
14 |
15 | - **`load`** and **`save`**:
16 | - `load`: Class method to load a saved model from disk, optionally resetting its path and logging the location.
17 | - `save`: Saves the current model to disk in a specified path, providing persistence for trained models.
18 |
19 | - **`predict`**:
20 | Abstract method for generating predictions on new data. Specific predictive models should implement this method based on their inference logic.
21 |
22 | This `Predictor` class offers a foundation for different predictive models, providing essential methods for data handling, training, and saving/loading, with extensibility for custom implementations.
23 |
24 | ---
25 |
26 | #### Available Predictors
27 |
28 | - [**`PerfPredictor`**][qtt.predictors.perf]
29 | Predicts the performance of a configuration on a new dataset.
30 | - [**`CostPredictor`**][qtt.predictors.cost]
31 | Predicts the cost of training a configuration on a new dataset.
--------------------------------------------------------------------------------
/docs/reference/predictors/perf.md:
--------------------------------------------------------------------------------
1 | ::: qtt.predictors.perf
--------------------------------------------------------------------------------
/docs/reference/predictors/predictor.md:
--------------------------------------------------------------------------------
1 | ::: qtt.predictors.predictor
--------------------------------------------------------------------------------
/docs/reference/tuners/image_cls_tuner.md:
--------------------------------------------------------------------------------
1 | # QuickTuner - Image Classification
2 |
3 | ::: qtt.tuners.image.classification.tuner
--------------------------------------------------------------------------------
/docs/reference/tuners/index.md:
--------------------------------------------------------------------------------
1 | # Overview
2 |
3 | The `QuickTuner` class is a high-level tuner designed to optimize a given objective function by managing iterative evaluations and coordinating with an `Optimizer`. It provides comprehensive functionality for logging, result tracking, checkpointing, and handling evaluation budgets.
4 |
5 | #### Core Methods
6 |
7 | - **`run`**: Executes the optimization process within a specified budget of function evaluations (`fevals`) or time (`time_budget`). This method iteratively:
8 | - Requests new configurations from the optimizer.
9 | - Evaluates configurations using the objective function `f`.
10 | - Updates the optimizer with evaluation results and logs progress.
11 | - Saves results based on the specified `save_freq` setting.
12 |
13 | - **`save`** and **`load`**:
14 | - `save`: Saves the current state of the tuner, including the incumbent, evaluation history, and tuner state.
15 | - `load`: Loads a previously saved tuner state to resume optimization from where it left off.
16 |
17 | #### Usage Example
18 |
19 | The `QuickTuner` is typically used to optimize an objective function with the support of an optimizer, managing configuration sampling, evaluation, and tracking. It is particularly suited for iterative optimization tasks where tracking the best configuration and logging results are essential.
20 |
--------------------------------------------------------------------------------
/docs/reference/tuners/quicktuner.md:
--------------------------------------------------------------------------------
1 | # QuickTuner
2 |
3 | ::: qtt.tuners.quick
--------------------------------------------------------------------------------
/examples/define_search_space.py:
--------------------------------------------------------------------------------
1 | """Define Search Space
2 |
3 | This examples shows how to define a search space. We use
4 | [ConfigSpace](https://github.com/automl/ConfigSpace).
5 |
6 | This search space is defined for a image classification task and includes
7 | various hyperparameters that can be optimized.
8 |
9 | First import the necessary modules:
10 | """
11 |
12 | from ConfigSpace import (
13 | Categorical,
14 | ConfigurationSpace,
15 | EqualsCondition,
16 | OrConjunction,
17 | OrdinalHyperparameter,
18 | )
19 |
20 | cs = ConfigurationSpace("image-classification")
21 |
22 | """
23 | ## Finetuning Parameters
24 | The finetuning parameters in this configuration space are designed to control how a
25 | pre-trained model is fine-tuned on a new dataset. Here's a breakdown of each finetuning
26 | parameter:
27 |
28 | 1. **`pct-to-freeze`** (Percentage of Model to Freeze):
29 | This parameter controls the fraction of the model's layers that will be frozen during
30 | training. Freezing a layer means that its weights will not be updated. Where `0.0`
31 | means no layers are frozen, and `1.0` means all layers are frozen, except for the
32 | final classification layer.
33 |
34 | 2. **`layer-decay`** (Layer-wise Learning Rate Decay):
35 | Layer-wise decay is a technique where deeper layers of the model use lower learning
36 | rates than layers closer to the output.
37 |
38 | 3. **`linear-probing`**:
39 | When linear probing is enabled, it means the training is focused on updating only the
40 | final classification layer (linear layer), while keeping the rest of the model frozen.
41 |
42 | 4. **`stoch-norm`** ([Stochastic Normalization](https://proceedings.neurips.cc//paper_files/paper/2020/hash/bc573864331a9e42e4511de6f678aa83-Abstract.html)):
43 | Enabling stochastic normalization during training.
44 |
45 | 5. **`sp-reg`** ([Starting Point Regularization](https://arxiv.org/abs/1802.01483)):
46 | This parameter controls the amount of regularization applied to the weights of the model
47 | towards the pretrained model.
48 |
49 | 6. **`delta-reg`** ([DELTA Regularization](https://arxiv.org/abs/1901.09229)):
50 | DELTA regularization aims to preserve the outer layer outputs of the target network.
51 |
52 | 7. **`bss-reg`** ([Batch Spectral Shrinkage Regularization](https://proceedings.neurips.cc/paper_files/paper/2019/hash/c6bff625bdb0393992c9d4db0c6bbe45-Abstract.html)):
53 | Batch Spectral Shrinkage (BSS) regularization penalizes the spectral norm of the model's weight matrices.
54 |
55 | 8. **`cotuning-reg`** ([Co-tuning Regularization](https://proceedings.neurips.cc/paper/2020/hash/c8067ad1937f728f51288b3eb986afaa-Abstract.html)):
56 | This parameter controls the strength of co-tuning, a method that aligns the
57 | representation of new data with the pre-trained model's representations
58 | """
59 | freeze = OrdinalHyperparameter("pct-to-freeze", [0.0, 0.2, 0.4, 0.6, 0.8, 1.0])
60 | ld = OrdinalHyperparameter("layer-decay", [0.0, 0.65, 0.75])
61 | lp = OrdinalHyperparameter("linear-probing", [False, True])
62 | sn = OrdinalHyperparameter("stoch-norm", [False, True])
63 | sr = OrdinalHyperparameter("sp-reg", [0.0, 0.0001, 0.001, 0.01, 0.1])
64 | d_reg = OrdinalHyperparameter("delta-reg", [0.0, 0.0001, 0.001, 0.01, 0.1])
65 | bss = OrdinalHyperparameter("bss-reg", [0.0, 0.0001, 0.001, 0.01, 0.1])
66 | cot = OrdinalHyperparameter("cotuning-reg", [0.0, 0.5, 1.0, 2.0, 4.0])
67 |
68 | """
69 | ## Regularization Parameters
70 |
71 | - **`mixup`**: A data augmentation technique that mixes two training samples and their labels. The value determines the strength of mixing between samples.
72 |
73 | - **`mixup_prob`**: Specifies the probability of applying mixup augmentation to a given batch. A value of 0 means mixup is never applied, while 1 means it is applied to every batch.
74 |
75 | - **`cutmix`**: Another data augmentation method that combines portions of two images and their labels.
76 |
77 | - **`drop`** (Dropout): Dropout is a regularization technique where random neurons in a layer are "dropped out" (set to zero) during training.
78 |
79 | - **`smoothing`** (Label Smoothing): A technique that smooths the true labels, assigning a small probability to incorrect classes.
80 |
81 | - **`clip_grad`**: This controls the gradient clipping, which constrains the magnitude of gradients during backpropagation.
82 |
83 | """
84 | mix = OrdinalHyperparameter("mixup", [0.0, 0.2, 0.4, 1.0, 2.0, 4.0, 8.0])
85 | mix_p = OrdinalHyperparameter("mixup-prob", [0.0, 0.25, 0.5, 0.75, 1.0])
86 | cut = OrdinalHyperparameter("cutmix", [0.0, 0.1, 0.25, 0.5, 1.0, 2.0, 4.0])
87 | drop = OrdinalHyperparameter("drop", [0.0, 0.1, 0.2, 0.3, 0.4])
88 | smooth = OrdinalHyperparameter("smoothing", [0.0, 0.05, 0.1])
89 | clip = OrdinalHyperparameter("clip-grad", [0, 1, 10])
90 |
91 | """
92 | ## Optimization Parameters
93 | """
94 | amp = OrdinalHyperparameter("amp", [False, True])
95 | opt = Categorical("opt", ["sgd", "momentum", "adam", "adamw", "adamp"])
96 | betas = Categorical("opt-betas", ["0.9 0.999", "0.0 0.99", "0.9 0.99", "0.0 0.999"])
97 | lr = OrdinalHyperparameter("lr", [1e-05, 5e-05, 0.0001, 0.0005, 0.001, 0.005, 0.01])
98 | w_ep = OrdinalHyperparameter("warmup_epochs", [0, 5, 10])
99 | w_lr = OrdinalHyperparameter("warmup-lr", [0.0, 1e-05, 1e-06])
100 | wd = OrdinalHyperparameter("weight-decay", [0, 1e-05, 0.0001, 0.001, 0.01, 0.1])
101 | bs = OrdinalHyperparameter("batch-size", [2, 4, 8, 16, 32, 64, 128, 256, 512])
102 | mom = OrdinalHyperparameter("momentum", [0.0, 0.8, 0.9, 0.95, 0.99])
103 | sched = Categorical("sched", ["cosine", "step", "multistep", "plateau"])
104 | pe = OrdinalHyperparameter("patience-epochs", [2, 5, 10])
105 | dr = OrdinalHyperparameter("decay-rate", [0.1, 0.5])
106 | de = OrdinalHyperparameter("decay-epochs", [10, 20])
107 | da = Categorical(
108 | "data-augmentation",
109 | ["auto-augment", "random-augment", "trivial-augment", "none"],
110 | )
111 | aa = Categorical("auto-augment", ["v0", "original"])
112 | ra_nops = OrdinalHyperparameter("ra-num-ops", [2, 3])
113 | ra_mag = OrdinalHyperparameter("ra-magnitude", [9, 17])
114 | cond_1 = EqualsCondition(pe, sched, "plateau")
115 | cond_2 = OrConjunction(
116 | EqualsCondition(dr, sched, "step"),
117 | EqualsCondition(dr, sched, "multistep"),
118 | )
119 | cond_3 = OrConjunction(
120 | EqualsCondition(de, sched, "step"),
121 | EqualsCondition(de, sched, "multistep"),
122 | )
123 | cond_4 = EqualsCondition(mom, opt, "momentum")
124 | cond_5 = OrConjunction(
125 | EqualsCondition(betas, opt, "adam"),
126 | EqualsCondition(betas, opt, "adamw"),
127 | EqualsCondition(betas, opt, "adamp"),
128 | )
129 | cond_6 = EqualsCondition(ra_nops, da, "random-augment")
130 | cond_7 = EqualsCondition(ra_mag, da, "random-augment")
131 | cond_8 = EqualsCondition(aa, da, "auto-augment")
132 | cs.add(
133 | mix,
134 | mix_p,
135 | cut,
136 | drop,
137 | smooth,
138 | clip,
139 | freeze,
140 | ld,
141 | lp,
142 | sn,
143 | sr,
144 | d_reg,
145 | bss,
146 | cot,
147 | amp,
148 | opt,
149 | betas,
150 | lr,
151 | w_ep,
152 | w_lr,
153 | wd,
154 | bs,
155 | mom,
156 | sched,
157 | pe,
158 | dr,
159 | de,
160 | da,
161 | aa,
162 | ra_nops,
163 | ra_mag,
164 | cond_1,
165 | cond_2,
166 | cond_3,
167 | cond_4,
168 | cond_5,
169 | cond_6,
170 | cond_7,
171 | cond_8,
172 | )
173 |
174 | """
175 | ## Model Choices
176 |
177 | The **model choices** represent a range of state-of-the-art deep learning architectures
178 | for image classification tasks. Each model has different characteristics in terms of
179 | architecture, size, and computational efficiency, providing flexibility to users
180 | depending on their specific needs and resources. Here's an overview:
181 |
182 | - **Transformer-based models**: These models, such as BEiT and DeiT, use the transformer
183 | architecture that has become popular in computer vision tasks. They are highly scalable
184 | and effective for large datasets and benefit from pre-training on extensive image
185 | corpora.
186 |
187 | - **ConvNet-based models**: Models like ConvNeXt and EfficientNet are based on
188 | convolutional neural networks (CNNs), which have long been the standard for image
189 | classification.
190 |
191 | - **Lightweight models**: Options such as MobileViT and EdgeNeXt are designed for
192 | resource-constrained environments like mobile devices or edge computing. These models
193 | prioritize smaller size and lower computational costs.
194 | """
195 | model = Categorical(
196 | "model",
197 | [
198 | "beit_base_patch16_384",
199 | "beit_large_patch16_512",
200 | "convnext_small_384_in22ft1k",
201 | "deit3_small_patch16_384_in21ft1k",
202 | "dla46x_c",
203 | "edgenext_small",
204 | "edgenext_x_small",
205 | "edgenext_xx_small",
206 | "mobilevit_xs",
207 | "mobilevit_xxs",
208 | "mobilevitv2_075",
209 | "swinv2_base_window12to24_192to384_22kft1k",
210 | "tf_efficientnet_b4_ns",
211 | "tf_efficientnet_b6_ns",
212 | "tf_efficientnet_b7_ns",
213 | "volo_d1_384",
214 | "volo_d3_448",
215 | "volo_d4_448",
216 | "volo_d5_448",
217 | "volo_d5_512",
218 | "xcit_nano_12_p8_384_dist",
219 | "xcit_small_12_p8_384_dist",
220 | "xcit_tiny_12_p8_384_dist",
221 | "xcit_tiny_24_p8_384_dist",
222 | ],
223 | )
224 | cs.add(model)
225 |
226 | cs.to_yaml("space.yaml")
227 |
--------------------------------------------------------------------------------
/examples/index.md:
--------------------------------------------------------------------------------
1 | # Examples
--------------------------------------------------------------------------------
/examples/metatrain.py:
--------------------------------------------------------------------------------
1 | """Metatrain"""
2 |
3 | from qtt.predictors import PerfPredictor, CostPredictor
4 | import pandas as pd
5 |
6 |
7 | """
8 | The `fit`-method of the predictors takes tabular data as input. If the data is stored in
9 | a CSV file, the expected format of the CSV is shown below:
10 |
11 | ## Configurations
12 |
13 | Hyperparammeter configurations of previous evaluations. Do not apply any preprocessing
14 | to the data. Use native data types as much as possible.
15 |
16 | | | model | opt | lr | sched | batch_size |
17 | |-----------|---------------|-------|--------|---------|------------|
18 | | 1 | xcit_abc | adam | 0.001 | cosine | 64 |
19 | | 2 | beit_def | sgd | 0.0005 | step | 128 |
20 | | 3 | mobilevit_xyz | adamw | 0.01 | plateau | 32 |
21 | | ... |
22 |
23 | ## Meta-Features
24 |
25 | Meta-features are optional. Meta-features refer to features that describe or summarize
26 | other features in a dataset. They are higher-level characteristics or properties of the
27 | dataset that can provide insight into its structure or complexity.
28 |
29 | | | num-features | num-classes |
30 | |---|--------------|-------------|
31 | | 1 | 128 | 42 |
32 | | 2 | 256 | 123 |
33 | | 3 | 384 | 1000 |
34 |
35 | ## Learning Curves
36 |
37 | Learning curves show the performance of a model over time or over iterations as it
38 | learns from training data. For the vision classification task, the learning curves
39 | are the validation accuracy on the validation set.
40 |
41 | | | 1 | 2 | 3 | 4 | 5 | ... |
42 | |---|------|------|------|------|------|-----|
43 | | 1 | 0.11 | 0.12 | 0.13 | 0.14 | 0.15 | ... |
44 | | 2 | 0.21 | 0.22 | 0.23 | 0.24 | 0.25 | ... |
45 | | 3 | 0.31 | 0.32 | 0.33 | 0.34 | 0.35 | ... |
46 |
47 | ## Cost
48 |
49 | The cost of running a pipeline (per fidelity). This refers to the total runtime required
50 | to complete the pipeline. This includes both the training and evaluation phases. We use
51 | the total runtime as the cost measure for each pipeline execution.
52 |
53 | | | cost |
54 | |---|-------|
55 | | 1 | 12.3 |
56 | | 2 | 45.6 |
57 | | 3 | 78.9 |
58 |
59 | Ensure that the CSV files follow this structure for proper processing.
60 | """
61 | config = pd.read_csv("config.csv", index_col=0) # pipeline configurations
62 | meta = pd.read_csv("meta.csv", index_col=0) # if meta-features are available
63 | curve = pd.read_csv("curve.csv", index_col=0) # learning curves
64 | cost = pd.read_csv("cost.csv", index_col=0) # runtime costs
65 |
66 | X = pd.concat([config, meta], axis=1)
67 | curve = curve.values # predictors expect curves as numpy arrays
68 | cost = cost.values # predictors expect costs as numpy arrays
69 |
70 | perf_predictor = PerfPredictor().fit(X, curve)
71 | cost_predictor = CostPredictor().fit(X, cost)
72 |
--------------------------------------------------------------------------------
/examples/quicktuning.py:
--------------------------------------------------------------------------------
1 | """A quick example of using a special QuickTuner to tune image classifiers on a new dataset."""
2 |
3 | from qtt import QuickImageCLSTuner
4 |
5 | tuner = QuickImageCLSTuner("path/to/dataset")
--------------------------------------------------------------------------------
/examples/step_by_step.py:
--------------------------------------------------------------------------------
1 | from qtt import QuickOptimizer, QuickTuner
2 | from qtt.finetune.image.classification import extract_image_dataset_metafeat, fn
3 | import pandas as pd
4 | from ConfigSpace import ConfigurationSpace
5 |
6 | pipeline = pd.read_csv("pipeline.csv", index_col=0)
7 | curve = pd.read_csv("curve.csv", index_col=0)
8 | cost = pd.read_csv("cost.csv", index_col=0)
9 | meta = pd.read_csv("meta.csv", index_col=0)
10 | cs = ConfigurationSpace.from_yaml("space.yaml")
11 |
12 | config = pd.merge(pipeline, meta, on="dataset")
13 | config.drop(("dataset"), axis=1, inplace=True)
14 | opt = QuickOptimizer(cs, 50, cost_aware=True)
15 |
16 | ti, mf = extract_image_dataset_metafeat("path/to/dataset")
17 | opt.setup(128, mf)
18 |
19 | qt = QuickTuner(opt, fn)
20 | qt.run(100, trial_info=ti)
21 |
--------------------------------------------------------------------------------
/mkdocs.yml:
--------------------------------------------------------------------------------
1 | site_name: Quick-Tune-Tool
2 | repo_url: https://github.com/automl/quicktunetool
3 | repo_name: automl/quicktunetool
4 |
5 | theme:
6 | name: material
7 | features:
8 | - content.code.annotate
9 | - content.code.copy
10 | - navigation.footer
11 | - navigation.sections
12 | - toc.follow
13 | - toc.integrate
14 | - navigation.tabs
15 | - navigation.tabs.sticky
16 | - header.autohide
17 | - search.suggest
18 | - search.highlight
19 | - search.share
20 |
21 | extra:
22 | social:
23 | - icon: fontawesome/brands/github
24 | link: https://github.com/automl
25 | - icon: fontawesome/brands/twitter
26 | link: https://twitter.com/automl_org
27 |
28 | watch:
29 | - src/
30 | - examples/
31 |
32 | markdown_extensions:
33 | - admonition
34 | - tables
35 | - attr_list
36 | - md_in_html
37 | - toc:
38 | permalink: "#"
39 | - pymdownx.highlight:
40 | anchor_linenums: true
41 | - pymdownx.magiclink:
42 | hide_protocol: true
43 | repo_url_shortener: true
44 | repo_url_shorthand: true
45 | user: automl
46 | repo: qtt
47 | - pymdownx.highlight
48 | - pymdownx.inlinehilite
49 | - pymdownx.snippets
50 | - pymdownx.details
51 | - pymdownx.tabbed:
52 | alternate_style: true
53 | - pymdownx.superfences:
54 | custom_fences:
55 | - name: mermaid
56 | class: mermaid
57 | format: !!python/name:pymdownx.superfences.fence_code_format
58 | - pymdownx.emoji:
59 | emoji_index: !!python/name:material.extensions.emoji.twemoji
60 | emoji_generator: !!python/name:material.extensions.emoji.to_svg
61 |
62 | plugins:
63 | - search
64 | - autorefs
65 | - glightbox
66 | - offline:
67 | enabled: !ENV [DOCS_OFFLINE, false]
68 | - markdown-exec
69 | - literate-nav:
70 | nav_file: SUMMARY.md
71 | - mkdocstrings:
72 | default_handler: python
73 | enable_inventory: true
74 | handlers:
75 | python:
76 | paths: [src]
77 | # Extra objects which allow for linking to external docs
78 | import:
79 | - 'https://docs.python.org/3/objects.inv'
80 | - 'https://numpy.org/doc/stable/objects.inv'
81 | - 'https://pandas.pydata.org/docs/objects.inv'
82 | - 'https://optuna.readthedocs.io/en/stable/objects.inv'
83 | - 'https://scikit-learn.org/stable/objects.inv'
84 | - 'https://pytorch.org/docs/stable/objects.inv'
85 | options: # https://mkdocstrings.github.io/python/usage/
86 | docstring_section_style: list
87 | docstring_options:
88 | ignore_init_summary: true
89 | trim_doctest_flags: true
90 | returns_multiple_items: false
91 | show_docstring_attributes: true
92 | show_docstring_description: true
93 | # show_root_heading: true
94 | show_root_toc_entry: true
95 | show_object_full_path: false
96 | show_root_members_full_path: false
97 | signature_crossrefs: true
98 | # merge_init_into_class: true
99 | show_symbol_type_heading: true
100 | show_symbol_type_toc: true
101 | docstring_style: google
102 | inherited_members: true
103 | show_if_no_docstring: false
104 | show_bases: true
105 | # members_order: "alphabetical"
106 | # group_by_category: true
107 | # show_signature: true
108 | # separate_signature: true
109 | # show_signature_annotations: true
110 | # filters:
111 | # - "!^_[^_]"
112 | - gen-files:
113 | scripts:
114 | - docs/example_runner.py
115 |
116 | nav:
117 | - Home: index.md
118 | - Reference:
119 | - reference/index.md
120 | - Optimizers:
121 | - reference/optimizers/index.md
122 | - reference/optimizers/optimizer.md
123 | - QuickOptimizer: reference/optimizers/quick.md
124 | - RandomOptimizer: reference/optimizers/random.md
125 | - Predictors:
126 | - reference/predictors/index.md
127 | - reference/predictors/predictor.md
128 | - PerfPredictor: reference/predictors/perf.md
129 | - CostPredictor: reference/predictors/cost.md
130 | - Tuners:
131 | - reference/tuners/index.md
132 | - Image-Classification-Tuner: reference/tuners/image_cls_tuner.md
133 | - reference/tuners/quicktuner.md
134 | # Auto generated with docs/examples_runner.py
135 | - Examples: "examples/"
136 |
--------------------------------------------------------------------------------
/pyproject.toml:
--------------------------------------------------------------------------------
1 | [project]
2 | name = "quicktunetool"
3 | version = "0.0.4"
4 | description = "A Framework for Efficient Model Selection and Hyperparameter Optimization"
5 | readme = "README.md"
6 | requires-python = ">=3.11"
7 | license = { file = "LICENSE" }
8 | keywords = ["Machine Learning", "AutoML", "HPO", "Fine-Tuning", "Meta-Learning"]
9 | authors = [
10 | { name = "Ivo Rapant", email = "rapanti@cs.uni-freiburg.de" }
11 | ]
12 |
13 | classifiers = [
14 | "Development Status :: 3 - Alpha",
15 | "Intended Audience :: Developers",
16 | "Intended Audience :: Science/Research",
17 | "License :: OSI Approved :: BSD License",
18 | "Operating System :: Unix",
19 | "Programming Language :: Python",
20 | "Programming Language :: Python :: 3.11",
21 | "Topic :: Software Development :: Build Tools",
22 | ]
23 |
24 | dependencies = [
25 | "torch > 2.0.0",
26 | "torchvision >= 0.15.1",
27 | "ConfigSpace >= 1.2.0",
28 | "gpytorch >= 1.9.0",
29 | "numpy < 2",
30 | "pandas >= 2.0.0",
31 | "pyyaml >= 6.0.1",
32 | "scikit-learn == 1.5.0",
33 | "tqdm >= 4.66.0",
34 | "fimm",
35 | "commitizen"
36 | ]
37 |
38 | [project.optional-dependencies]
39 | dev = ["quicktunetool[docs, tooling]"]
40 | tooling = ["commitizen", "pre-commit", "ruff", "mypy", "types-psutil", "types-pyyaml"]
41 | docs = [
42 | "mkdocs",
43 | "mkdocs-material",
44 | "mkdocs-autorefs",
45 | "mkdocs-awesome-pages-plugin",
46 | "mkdocs-gen-files",
47 | "mkdocs-literate-nav",
48 | "mkdocs-glightbox",
49 | "mkdocstrings[python]",
50 | "markdown-exec[ansi]",
51 | "matplotlib",
52 | "more-itertools",
53 | "rich",
54 | "typing-extensions",
55 | "mike",
56 | "pillow",
57 | "cairosvg",
58 | "black",
59 | ]
60 |
61 |
62 | [project.urls]
63 | "Homepage" = "https://github.com/automl/quicktunetool"
64 | "Bug Reports" = "https://github.com/automl/quicktunetool/issues"
65 | "Source" = "https://github.com/automl/quicktunetool"
66 |
67 | [tool.setuptools.packages.find]
68 | where = ["src"]
69 |
70 | [tool.commitizen]
71 | name = "cz_conventional_commits"
72 | version = "0.0.4"
73 | update_changelog_on_bump = true
74 | version_files = ["pyproject.toml:version"]
--------------------------------------------------------------------------------
/src/qtt/__init__.py:
--------------------------------------------------------------------------------
1 | from .optimizers import Optimizer, QuickOptimizer, RandomOptimizer
2 | from .predictors import PerfPredictor, CostPredictor, Predictor
3 | from .tuners import QuickTuner, QuickImageCLSTuner
4 | from .utils.pretrained import get_pretrained_optimizer
5 | from .utils.log_utils import setup_default_logging
6 |
7 | __all__ = [
8 | "Optimizer",
9 | "QuickImageCLSTuner",
10 | "QuickOptimizer",
11 | "QuickTuner",
12 | "RandomOptimizer",
13 | "PerfPredictor",
14 | "CostPredictor",
15 | "Predictor",
16 | "get_pretrained_optimizer",
17 | ]
18 |
19 | setup_default_logging()
20 |
--------------------------------------------------------------------------------
/src/qtt/finetune/image/classification/__init__.py:
--------------------------------------------------------------------------------
1 | from .fn import fn
2 | from .utils import extract_image_dataset_metafeat
3 |
4 | __all__ = ["fn", "extract_image_dataset_metafeat"]
5 |
--------------------------------------------------------------------------------
/src/qtt/finetune/image/classification/fn.py:
--------------------------------------------------------------------------------
1 | import os
2 | import subprocess
3 | import time
4 | from importlib.util import find_spec
5 |
6 | import pandas as pd
7 |
8 | hp_list = [
9 | "batch-size",
10 | "bss-reg",
11 | "cotuning-reg",
12 | "cutmix",
13 | "decay-rate",
14 | "decay-epochs",
15 | "delta-reg",
16 | "drop",
17 | "layer-decay",
18 | "lr",
19 | "mixup",
20 | "mixup-prob",
21 | "model",
22 | "opt",
23 | "patience-epochs",
24 | "pct-to-freeze",
25 | "sched",
26 | "smoothing",
27 | "sp-reg",
28 | "warmup-epochs",
29 | "warmup-lr",
30 | "weight-decay",
31 | ]
32 | num_hp_list = ["clip-grad", "layer-decay"]
33 | bool_hp_list = ["amp", "linear-probing", "stoch-norm"]
34 | static_args = ["--pretrained", "--checkpoint-hist", "1", "--epochs", "50", "--workers", "8"]
35 | trial_args = ["train-split", "val-split", "num-classes"]
36 |
37 |
38 | def fn(trial: dict, trial_info: dict) -> dict:
39 | """
40 | Fine-tune a pretrained model on a image dataset.
41 | Using the [fimm library]().
42 |
43 | Args:
44 | trial (dict): A dictionary containing trial-specific configurations. Mandatory keys include:
45 | - "config-id" (str): Unique identifier for the trial configuration.
46 | - "config" (dict): Hyperparameter settings for the trial, as a dictionary.
47 | - "fidelity" (int): Specifies the fidelity level for the trial (e.g., epoch count or number of samples).
48 | trial_info (dict): A dictionary with additional trial metadata. Mandatory keys include:
49 | - "data-dir" (str): Path to the directory containing the image dataset.
50 | - "output-dir" (str): Path to the directory where training results and logs are saved.
51 | - "train-split" (str): Path to the training split folder.
52 | - "val-split" (str): Path to the validation split folder.
53 | - "num-classes" (int): Number of classes in the dataset.
54 |
55 | Returns:
56 | dict: Updated trial dictionary with:
57 | - "status" (bool): Indicates whether the training process was successful.
58 | - "score" (float): Final evaluation score (top-1 accuracy as a decimal).
59 | - "cost" (float): Time taken for the training process in seconds.
60 | """
61 |
62 | if not find_spec("fimm"):
63 | raise ImportError(
64 | "You need to install fimm to run this script. Run `pip install fimm` in your console."
65 | )
66 |
67 | config: dict = trial["config"]
68 | fidelity = str(trial["fidelity"])
69 | config_id = str(trial["config-id"])
70 |
71 | data_dir: str = trial_info["data-dir"]
72 | output_dir: str = trial_info["output-dir"]
73 |
74 | args = ["train", "--data-dir", data_dir, "--output", output_dir, "--experiment", str(config_id)]
75 | args += ["--fidelity", fidelity]
76 | args += static_args
77 | for arg in trial_args:
78 | args += [f"--{arg}", str(trial_info[arg])]
79 |
80 | # DATA AUGMENTATIONS
81 | match config.get("data-augmentation"):
82 | case "auto-augment":
83 | args += ["--aa", config["auto-augment"]]
84 | case "trivial-augment":
85 | args += ["--ta"]
86 | case "random-augment":
87 | args += ["--ra"]
88 | args += ["--ra-num-ops", str(config["ra-num-ops"])]
89 | args += ["--ra-magnitude", str(config["ra-magnitude"])]
90 |
91 | for k, v in config.items():
92 | if k in hp_list:
93 | args += [f"--{k}", str(v)]
94 | elif k in num_hp_list:
95 | if v > 0:
96 | args += [f"--{k}", str(v)]
97 | elif k in bool_hp_list:
98 | if v:
99 | args += [f"--{k}"]
100 |
101 | if os.path.exists(os.path.join(output_dir, str(config_id))):
102 | resume_path = os.path.join(output_dir, str(config_id), "last.pth.tar")
103 | args += ["--resume", resume_path]
104 |
105 | start = time.time()
106 | process = subprocess.Popen(args)
107 | try:
108 | process.wait()
109 | except KeyboardInterrupt:
110 | process.terminate()
111 | finally:
112 | if process.poll() is None:
113 | process.terminate()
114 | end = time.time()
115 |
116 | report: dict = {}
117 | if process.returncode == 0:
118 | output_path = os.path.join(output_dir, str(config_id))
119 | df = pd.read_csv(os.path.join(output_path, "summary.csv"))
120 | report["status"] = True
121 | report["score"] = df["eval_top1"].values[-1] / 100
122 | report["cost"] = end - start
123 | else:
124 | report["status"] = False
125 | report["score"] = 0.0
126 | report["cost"] = float("inf")
127 |
128 | trial.update(report)
129 | return trial
130 |
--------------------------------------------------------------------------------
/src/qtt/finetune/image/classification/utils.py:
--------------------------------------------------------------------------------
1 | import os
2 | from pathlib import Path
3 |
4 | from torchvision.datasets import ImageFolder # type: ignore
5 |
6 |
7 | def extract_image_dataset_metafeat(
8 | path_root: str | Path, train_split: str = "train", val_split: str = "val"
9 | ):
10 | """
11 | Extracts metadata features from an image dataset for classification tasks.
12 |
13 | This function analyzes the specified dataset directory to compute metadata
14 | features, such as the number of samples, number of classes, average number
15 | of features (image size), and number of channels.
16 |
17 | Args:
18 | path_root (str | Path): The root directory of the dataset.
19 | train_split (str, optional): The subdirectory name for training data. Defaults to "train".
20 | val_split (str, optional): The subdirectory name for validation data. Defaults to "val".
21 |
22 | Returns:
23 | tuple: A tuple containing:
24 | - trial_info (dict): Information about the dataset directory and splits.
25 | - metafeat (dict): Metadata features including:
26 | - "num-samples": Total number of samples in the dataset.
27 | - "num-classes": Number of classes in the dataset.
28 | - "num-features": Average number of features (image size).
29 | - "num-channels": Number of channels in the images.
30 |
31 | Raises:
32 | ValueError: If the specified path does not exist or is not a directory.
33 | """
34 | # handle path
35 | path_root = Path(path_root)
36 | path_root = path_root.expanduser() # expands ~ to home directory
37 | path_root = Path(path_root) # ensure type safety
38 | path_root = path_root.resolve() # convert to an absolute path
39 | if not path_root.exists():
40 | raise ValueError(f"The specified path does not exist: {path_root}")
41 | if not path_root.is_dir():
42 | raise ValueError(f"The specified path is not a directory: {path_root}")
43 |
44 | num_samples = 0
45 | num_classes = 0
46 | num_features = 224
47 | num_channels = 3
48 |
49 | # trainset
50 | train_path = os.path.join(path_root, train_split)
51 | if os.path.exists(train_path):
52 | trainset = ImageFolder(train_path)
53 | num_samples += len(trainset)
54 | num_channels = 3 if trainset[0][0].mode == "RGB" else 1
55 | num_classes = len(trainset.classes)
56 |
57 | for img, _ in trainset:
58 | num_features += img.size[0]
59 | num_features //= len(trainset)
60 |
61 | # valset
62 | val_path = os.path.join(path_root, val_split)
63 | if os.path.exists(val_path):
64 | valset = ImageFolder(val_path)
65 | num_samples += len(valset)
66 |
67 | metafeat = {
68 | "num-samples": num_samples,
69 | "num-classes": num_classes,
70 | "num-features": num_features,
71 | "num-channels": num_channels,
72 | }
73 |
74 | trial_info = {
75 | "data-dir": str(path_root),
76 | "train-split": train_split,
77 | "val-split": val_split,
78 | "num-classes": num_classes,
79 | }
80 |
81 | return trial_info, metafeat
82 |
--------------------------------------------------------------------------------
/src/qtt/optimizers/__init__.py:
--------------------------------------------------------------------------------
1 | from .optimizer import Optimizer
2 | from .quick import QuickOptimizer
3 | from .rndm import RandomOptimizer
4 |
5 | __all__ = [
6 | "Optimizer",
7 | "QuickOptimizer",
8 | "RandomOptimizer",
9 | ]
10 |
--------------------------------------------------------------------------------
/src/qtt/optimizers/optimizer.py:
--------------------------------------------------------------------------------
1 | import logging
2 | import os
3 | import pickle
4 |
5 | from ..utils import setup_outputdir
6 |
7 | logger = logging.getLogger(__name__)
8 |
9 |
10 | class Optimizer:
11 | """Base class. Implements all low-level functionality.
12 |
13 | Args:
14 | name (str):
15 | Name of the subdirectory inside path where model will be saved.
16 | The final model directory will be os.path.join(path, name)
17 | If None, defaults to the model's class name: self.__class__.__name__
18 | path (str):
19 | Directory location to store all outputs.
20 | If None, a new unique time-stamped directory is chosen.
21 | """
22 |
23 | model_file_name = "model.pkl"
24 |
25 | def __init__(
26 | self,
27 | name: str | None = None,
28 | path: str | None = None,
29 | ):
30 | if name is None:
31 | self.name = self.__class__.__name__
32 | logger.info(
33 | f"No name was specified for model, defaulting to class name: {self.name}",
34 | )
35 | else:
36 | self.name = name
37 |
38 | if path is None:
39 | self.path = setup_outputdir(path=self.name.lower())
40 | logger.info(
41 | f"No path was specified for predictor, defaulting to: {self.path}",
42 | )
43 | else:
44 | self.path = setup_outputdir(path)
45 |
46 | self._is_initialized = False
47 |
48 | def ante(self):
49 | """This method is intended for the use with a tuner.
50 | It allows to perform some pre-processing steps before each ask."""
51 | pass
52 |
53 | def post(self):
54 | """This method is intended for the use with a tuner.
55 | It allows to perform some post-processing steps after each tell."""
56 | pass
57 |
58 | def ask(self) -> dict | None:
59 | """Ask the optimizer for a trial to evaluate.
60 |
61 | Returns:
62 | A config to sample.
63 | """
64 | raise NotImplementedError
65 |
66 | def tell(self, report: dict | list[dict]):
67 | """Tell the optimizer the result for an asked trial.
68 |
69 | Args:
70 | report (dict): The result for a trial
71 | """
72 | raise NotImplementedError
73 |
74 | @classmethod
75 | def load(cls, path: str, reset_paths: bool = True, verbose: bool = True):
76 | """
77 | Loads the model from disk to memory.
78 |
79 | Args:
80 | path (str):
81 | Path to the saved model, minus the file name.
82 | This should generally be a directory path ending with a '/' character (or appropriate path separator value depending on OS).
83 | The model file is typically located in os.path.join(path, cls.model_file_name).
84 | reset_paths (bool):
85 | Whether to reset the self.path value of the loaded model to be equal to path.
86 | It is highly recommended to keep this value as True unless accessing the original self.path value is important.
87 | If False, the actual valid path and self.path may differ, leading to strange behaviour and potential exceptions if the model needs to load any other files at a later time.
88 | verbose (bool):
89 | Whether to log the location of the loaded file.
90 |
91 | Returns:
92 | model (Optimizer): The loaded model object.
93 | """
94 | file_path = os.path.join(path, cls.model_file_name)
95 | with open(file_path, "rb") as f:
96 | model = pickle.load(f)
97 | if reset_paths:
98 | model.path = path
99 | if verbose:
100 | logger.info(f"Model loaded from: {file_path}")
101 | return model
102 |
103 | def save(self, path: str | None = None, verbose: bool = True) -> str:
104 | """
105 | Saves the model to disk.
106 |
107 | Args:
108 | path (str): Path to the saved model, minus the file name. This should generally
109 | be a directory path ending with a '/' character (or appropriate path separator
110 | value depending on OS). If None, self.path is used. The final model file is
111 | typically saved to os.path.join(path, self.model_file_name).
112 | verbose (bool): Whether to log the location of the saved file.
113 |
114 | Returns:
115 | str: Path to the saved model, minus the file name. Use this value to load the
116 | model from disk via cls.load(path), where cls is the class of the model
117 | object (e.g., model = Model.load(path)).
118 | """
119 | if path is None:
120 | path = self.path
121 | os.makedirs(path, exist_ok=True)
122 | file_path = os.path.join(path, self.model_file_name)
123 | with open(file_path, "wb") as f:
124 | pickle.dump(self, f, protocol=pickle.HIGHEST_PROTOCOL)
125 | if verbose:
126 | logger.info(f"Model saved to: {file_path}")
127 | return path
128 |
129 | def reset_path(self, path: str | None = None):
130 | """Reset the path of the model.
131 |
132 | Args:
133 | path (str):
134 | Directory location to store all outputs.
135 | If None, a new unique time-stamped directory is chosen.
136 | """
137 | if path is None:
138 | path = setup_outputdir(path=self.name.lower(), path_suffix=self.name)
139 | self.path = path
140 |
--------------------------------------------------------------------------------
/src/qtt/optimizers/quick.py:
--------------------------------------------------------------------------------
1 | import logging
2 | from typing import Literal, Mapping
3 |
4 | import numpy as np
5 | import pandas as pd
6 | from ConfigSpace import ConfigurationSpace
7 | from scipy.stats import norm # type: ignore
8 |
9 | from ..predictors import CostPredictor, PerfPredictor
10 | from ..utils import fix_random_seeds, set_logger_verbosity
11 | from .optimizer import Optimizer
12 |
13 | logger = logging.getLogger(__name__)
14 |
15 |
16 | class QuickOptimizer(Optimizer):
17 | """QuickOptimizer implements a cost-aware Bayesian optimization. It builds upon the
18 | DyHPO algorithm, adding cost-awareness to the optimization process.
19 |
20 | Args:
21 | cs (ConfigurationSpace): The configuration space to optimize over.
22 | max_fidelity (int): The maximum fidelity to optimize. Fidelity is a measure of
23 | a resource used by a configuration, such as the number of epochs.
24 | perf_predictor (PerfPredictor, optional): The performance predictor to use. If
25 | None, a new predictor is created.
26 | cost_predictor (CostPredictor, optional): The cost predictor to use. If None,
27 | a new CostPredictor is created if `cost_aware` is True.
28 | cost_aware (bool, optional): Whether to use the cost predictor. Defaults to False.
29 | cost_factor (float, optional): A factor to control the scaling of cost values.
30 | Values must be in the range `[0.0, inf)`. A cost factor smaller than 1
31 | compresses the cost values closer together (with 0 equalizing them), while
32 | values larger than 1 expand them. Defaults to 1.0.
33 | acq_fn (str, optional): The acquisition function to use. One of ["ei", "ucb",
34 | "thompson", "exploit"]. Defaults to "ei".
35 | explore_factor (float, optional): The exploration factor in the acquisition
36 | function. Defaults to 1.0.
37 | patience (int, optional): Determines if early stopping should be applied for a
38 | single configuration. If the score does not improve for `patience` steps,
39 | the configuration is stopped. Defaults to None.
40 | tol (float, optional): Tolerance for early stopping. Training stops if the score
41 | does not improve by at least `tol` for `patience` iterations (if set). Values
42 | must be in the range `[0.0, inf)`. Defaults to 0.0.
43 | score_thresh (float, optional): Threshold for early stopping. If the score is
44 | above `1 - score_thresh`, the configuration is stopped. Defaults to 0.0.
45 | init_random_search_steps (int, optional): Number of configurations to evaluate
46 | randomly at the beginning of the optimization (with fidelity 1) before using
47 | predictors/acquisition function. Defaults to 10.
48 | refit_init_steps (int, optional): Number of steps (successful evaluations) before
49 | refitting the predictors. Defaults to 0.
50 | refit (bool, optional): Whether to refit the predictors with observed data.
51 | Defaults to False.
52 | refit_interval (int, optional): Interval for refitting the predictors. Defaults
53 | to 1.
54 | path (str, optional): Path to save the optimizer state. Defaults to None.
55 | seed (int, optional): Seed for reproducibility. Defaults to None.
56 | verbosity (int, optional): Verbosity level for logging. Defaults to 2.
57 | """
58 |
59 | def __init__(
60 | self,
61 | cs: ConfigurationSpace,
62 | max_fidelity: int,
63 | perf_predictor: PerfPredictor | None = None,
64 | cost_predictor: CostPredictor | None = None,
65 | *,
66 | cost_aware: bool = False,
67 | cost_factor: float = 1.0,
68 | acq_fn: Literal["ei", "ucb", "thompson", "exploit"] = "ei",
69 | explore_factor: float = 0.0,
70 | patience: int | None = None,
71 | tol: float = 1e-4,
72 | score_thresh: float = 0.0,
73 | init_random_search_steps: int = 3,
74 | refit_init_steps: int = 0,
75 | refit: bool = False,
76 | refit_interval: int = 1,
77 | #
78 | path: str | None = None,
79 | seed: int | None = None,
80 | verbosity: int = 2,
81 | ):
82 | super().__init__(path=path)
83 | set_logger_verbosity(verbosity, logger)
84 | self.verbosity = verbosity
85 |
86 | if seed is not None:
87 | fix_random_seeds(seed)
88 | self.seed = seed
89 |
90 | # configuration space
91 | self.cs = cs
92 | self.max_fidelity = max_fidelity
93 |
94 | # optimizer related parameters
95 | self.acq_fn = acq_fn
96 | self.explore_factor = explore_factor
97 | self.cost_aware = cost_aware
98 | self.cost_factor = cost_factor
99 | self.patience = patience
100 | self.tol = tol
101 | self.scr_thr = score_thresh
102 | self.refit_init_steps = refit_init_steps
103 | self.refit = refit
104 | self.refit_interval = refit_interval
105 |
106 | # predictors
107 | self.perf_predictor = perf_predictor
108 | if self.perf_predictor is None:
109 | self.perf_predictor = PerfPredictor(path=path)
110 | self.cost_predictor = cost_predictor
111 | if self.cost_aware and self.cost_predictor is None:
112 | self.cost_predictor = CostPredictor(path=path)
113 |
114 | # trackers
115 | self.init_random_search_steps = init_random_search_steps
116 | self.ask_count = 0
117 | self.tell_count = 0
118 | self.init_count = 0
119 | self.eval_count = 0
120 | self.configs: list[dict] = []
121 | self.evaled: set[int] = set()
122 | self.stoped: set[int] = set()
123 | self.failed: set[int] = set()
124 | self.history: list = []
125 |
126 | # placeholders
127 | self.pipelines: pd.DataFrame
128 | self.curves: np.ndarray
129 | self.fidelities: np.ndarray
130 | self.costs: np.ndarray | None = None
131 | self.score_history: np.ndarray | None = None
132 |
133 | # flags
134 | self.ready = False
135 | self.finished = False
136 |
137 | def setup(
138 | self,
139 | n: int,
140 | metafeat: Mapping[str, int | float] | None = None,
141 | ) -> None:
142 | """Setup the optimizer for optimization.
143 |
144 | Create the configurations to evaluate. The configurations are sampled from the
145 | configuration space. Optionally, metafeatures of the dataset can be provided.
146 |
147 | Args:
148 | n (int): The number of configurations to create.
149 | metafeat (Mapping[str, int | float], optional): The metafeatures of the dataset.
150 | """
151 | self.N = n
152 | self.fidelities = np.zeros(n, dtype=int)
153 | self.curves = np.full((n, self.max_fidelity), np.nan, dtype=float)
154 | self.costs = None
155 | if self.patience is not None:
156 | self.score_history = np.zeros((n, self.patience), dtype=float)
157 |
158 | if self.seed is not None:
159 | self.cs.seed(self.seed)
160 | _configs = self.cs.sample_configuration(n)
161 | self.configs = [dict(c) for c in _configs]
162 | self.pipelines = pd.DataFrame(self.configs)
163 |
164 | self.metafeat = None
165 | if metafeat is not None:
166 | self.metafeat = pd.DataFrame([metafeat] * self.N)
167 | self.pipelines = pd.concat([self.pipelines, self.metafeat], axis=1)
168 |
169 | self.ready = True
170 |
171 | def setup_pandas(
172 | self,
173 | df: pd.DataFrame,
174 | metafeat: Mapping[str, int | float] | None = None,
175 | ):
176 | """Setup the optimizer for optimization.
177 |
178 | Use an existing DataFrame to create the configurations to evaluate. Optionally,
179 | metafeatures of the dataset can be provided.
180 |
181 | Args:
182 | df (pd.DataFrame): The DataFrame with the configurations to evaluate.
183 | metafeat (Mapping[str, int | float], optional): The metafeatures of the dataset.
184 | """
185 | self.pipelines = df
186 | self.N = len(df)
187 | self.fidelities = np.zeros(self.N, dtype=int)
188 | self.curves = np.full((self.N, self.max_fidelity), np.nan, dtype=float)
189 | self.costs = None
190 | if self.patience is not None:
191 | self.score_history = np.zeros((self.N, self.patience), dtype=float)
192 |
193 | self.metafeat = None
194 | if metafeat is not None:
195 | self.metafeat = pd.DataFrame([metafeat] * self.N)
196 | self.pipelines = pd.concat([self.pipelines, self.metafeat], axis=1)
197 | self.configs = self.pipelines.to_dict(orient="records")
198 |
199 | self.ready = True
200 |
201 | def _predict(self) -> tuple[np.ndarray, np.ndarray, np.ndarray | None]:
202 | """Predict the performance and cost of the configurations.
203 |
204 | Returns:
205 | The mean and standard deviation of the performance of the pipelines and their costs.
206 | """
207 | pipeline, curve = self.pipelines, self.curves
208 |
209 | if self.perf_predictor is None:
210 | raise AssertionError("PerfPredictor is not fitted yet")
211 | pred = self.perf_predictor.predict(X=pipeline, curve=curve)
212 | pred_mean, pred_std = pred
213 |
214 | costs = None
215 | if self.cost_aware:
216 | if self.cost_predictor is None:
217 | raise AssertionError("CostPredictor is not fitted yet")
218 | if self.costs is None:
219 | c = self.cost_predictor.predict(X=pipeline)
220 | c = np.clip(c, 1e-6, None) # avoid division by zero
221 | c /= c.max() # normalize
222 | c = np.power(c, self.cost_factor) # rescale
223 | self.costs = c
224 | costs = self.costs
225 |
226 | return pred_mean, pred_std, costs
227 |
228 | def _calc_acq_val(self, mean, std, y_max):
229 | """Calculate the acquisition value.
230 |
231 | Args:
232 | mean (np.ndarray): The mean of the predictions.
233 | std (np.ndarray): The standard deviation of the predictions.
234 | cost (np.ndarray): The cost of the pipeline.
235 |
236 | Returns:
237 | The acquisition values.
238 | """
239 | fn = self.acq_fn
240 | xi = self.explore_factor
241 | match fn:
242 | # Expected Improvement
243 | case "ei":
244 | mask = std == 0
245 | std = std + mask * 1.0
246 | z = (mean - y_max - xi) / std
247 | acq_value = (mean - y_max) * norm.cdf(z) + std * norm.pdf(z)
248 | acq_value[mask] = 0.0
249 | # Upper Confidence Bound
250 | case "ucb":
251 | acq_value = mean + xi * std
252 | # Thompson Sampling
253 | case "thompson":
254 | acq_value = np.random.normal(mean, std)
255 | # Exploitation
256 | case "exploit":
257 | acq_value = mean
258 | case _:
259 | raise ValueError
260 | return acq_value
261 |
262 | def _optimize_acq_fn(self, mean, std, cost) -> list[int]:
263 | """
264 | Optimize the acquisition function.
265 |
266 | Args:
267 | mean (np.ndarray): The mean of the predictions.
268 | std (np.ndarray): The standard deviation of the predictions.
269 | cost (np.ndarray): The cost of the pipeline.
270 |
271 | Returns:
272 | list[int]: A sorted list of indices of the pipeline.
273 | """
274 | # maximum score per fidelity
275 | curves = np.nan_to_num(self.curves)
276 | y_max = curves.max(axis=0)
277 | y_max = np.maximum.accumulate(y_max)
278 |
279 | # get the ymax for the next fidelity of the pipelines
280 | next_fidelitys = np.minimum(self.fidelities + 1, self.max_fidelity)
281 | y_max_next = y_max[next_fidelitys - 1]
282 |
283 | acq_values = self._calc_acq_val(mean, std, y_max_next)
284 | if self.cost_aware:
285 | acq_values /= cost
286 |
287 | return np.argsort(acq_values).tolist()
288 |
289 | def _ask(self):
290 | pred_mean, pred_std, cost = self._predict()
291 | ranks = self._optimize_acq_fn(pred_mean, pred_std, cost)
292 | ranks = [r for r in ranks if r not in self.stoped | self.failed]
293 | index = ranks[-1]
294 | logger.debug(f"predicted score: {pred_mean[index]:.4f}")
295 | return index
296 |
297 | def ask(self) -> dict | None:
298 | """Ask the optimizer for a configuration to evaluate.
299 |
300 | Returns:
301 | A dictionary with the configuration to evaluate.
302 | """
303 | if not self.ready:
304 | raise RuntimeError("Call setup() before ask()")
305 |
306 | if self.finished:
307 | return None
308 |
309 | self.ask_count += 1
310 | if len(self.evaled) < self.init_random_search_steps:
311 | left = set(range(self.N)) - self.evaled - self.failed - self.stoped
312 | index = left.pop()
313 | fidelity = 1
314 | else:
315 | index = self._ask()
316 | fidelity = self.fidelities[index] + 1
317 |
318 | return {
319 | "config-id": index,
320 | "config": self.configs[index],
321 | "fidelity": fidelity,
322 | }
323 |
324 | def tell(self, result: dict | list):
325 | """Tell the result of a trial to the optimizer.
326 |
327 | Args:
328 | result (dict | list[dict]): The result(s) for a trial.
329 | """
330 | if isinstance(result, dict):
331 | result = [result]
332 | for res in result:
333 | self._tell(res)
334 |
335 | def _tell(self, result: dict):
336 | self.tell_count += 1
337 |
338 | index = result["config-id"]
339 | fidelity = result["fidelity"]
340 | # cost = result["cost"]
341 | score = result["score"]
342 | status = result["status"]
343 |
344 | if not status:
345 | self.failed.add(index)
346 | return
347 |
348 | if score >= 1.0 - self.scr_thr or fidelity == self.max_fidelity:
349 | self.stoped.add(index)
350 |
351 | # update trackers
352 | self.curves[index, fidelity - 1] = score
353 | self.fidelities[index] = fidelity
354 | # self.costs[index] = cost
355 | self.history.append(result)
356 | self.evaled.add(index)
357 | self.eval_count += 1
358 |
359 | if self.patience is not None:
360 | assert self.score_history is not None
361 | if not np.any(self.score_history[index] < (score - self.tol)):
362 | self.stoped.add(index)
363 | self.score_history[index][fidelity % self.patience] = score
364 |
365 | self.finished = self._check_is_finished()
366 |
367 | def _check_is_finished(self):
368 | """Check if there is no more configurations to evaluate."""
369 | left = set(range(self.N)) - self.evaled - self.failed - self.stoped
370 | if not left:
371 | return True
372 | return False
373 |
374 | def ante(self):
375 | """Some operations to perform by the tuner before the optimization loop.
376 |
377 | Here: refit the predictors with observed data.
378 | """
379 | if (
380 | self.refit
381 | and not self.eval_count % self.refit_interval
382 | and self.eval_count >= self.refit_init_steps
383 | ):
384 | self.fit_extra()
385 |
386 | def fit_extra(self):
387 | """Refit the predictors with observed data."""
388 | pipeline, curve = self.pipelines, self.curves
389 | self.perf_predictor.fit_extra(pipeline, curve) # type: ignore
390 |
391 | def fit(self, X, curve, cost):
392 | """Fit the predictors with the given training data."""
393 | self.perf_predictor.fit(X, curve) # type: ignore
394 | if self.cost_predictor is not None:
395 | self.cost_predictor.fit(X, cost)
396 |
397 | def reset_path(self, path: str | None = None):
398 | """
399 | Reset the path of the model.
400 |
401 | Args:
402 | path (str, optional): Directory location to store all outputs. If None, a new unique
403 | time-stamped directory is chosen.
404 | """
405 | super().reset_path(path)
406 | if self.perf_predictor is not None:
407 | self.perf_predictor.reset_path(path)
408 | if self.cost_predictor is not None:
409 | self.cost_predictor.reset_path(path)
410 |
--------------------------------------------------------------------------------
/src/qtt/optimizers/rndm.py:
--------------------------------------------------------------------------------
1 | import logging
2 | import random
3 |
4 | import numpy as np
5 | from ConfigSpace import ConfigurationSpace
6 |
7 | from ..utils import fix_random_seeds, set_logger_verbosity
8 | from .optimizer import Optimizer
9 |
10 | logger = logging.getLogger(__name__)
11 |
12 |
13 | class RandomOptimizer(Optimizer):
14 | """A basic implementation of a random search optimizer.
15 |
16 | Args:
17 | cs (ConfigurationSpace): Configuration space object.
18 | max_fidelity (int): Maximum fidelity level.
19 | n (int): Number of configurations to sample.
20 | patience (int, optional): Determines if early stopping should be applied for a
21 | single configuration. If the score does not improve for `patience` steps,
22 | the configuration is stopped. Defaults to None.
23 | tol (float, optional): Tolerance for early stopping. Training stops if the score
24 | does not improve by at least `tol` for `patience` iterations (if set). Values
25 | must be in the range `[0.0, inf)`. Defaults to 0.0.
26 | score_thresh (float, optional): Score threshold for early stopping. Defaults to 0.0.
27 | path (str, optional): Path to save the optimizer. Defaults to None.
28 | seed (int, optional): Random seed. Defaults to None.
29 | verbosity (int, optional): Verbosity level. Defaults to 2.
30 | """
31 |
32 | def __init__(
33 | self,
34 | cs: ConfigurationSpace,
35 | max_fidelity: int,
36 | n: int,
37 | *,
38 | patience: int | None = None,
39 | tol: float = 0.0,
40 | score_thresh: float = 0.0,
41 | #
42 | path: str | None = None,
43 | seed: int | None = None,
44 | verbosity: int = 2,
45 | ):
46 | super().__init__(path=path)
47 | set_logger_verbosity(verbosity, logger)
48 | self.verbosity = verbosity
49 |
50 | if seed is not None:
51 | fix_random_seeds(seed)
52 | self.seed = seed
53 |
54 | self.cs = cs
55 | self.max_fidelity = max_fidelity
56 | self.candidates = cs.sample_configuration(n)
57 | self.N = n
58 | self.patience = patience
59 | self.tol = tol
60 | self.scr_thr = score_thresh
61 |
62 | self.reset()
63 |
64 | def reset(self) -> None:
65 | # trackers
66 | self.iteration = 0
67 | self.ask_count = 0
68 | self.tell_count = 0
69 | self.init_count = 0
70 | self.eval_count = 0
71 | self.evaled: set[int] = set()
72 | self.stoped: set[int] = set()
73 | self.failed: set[int] = set()
74 | self.history: list = []
75 |
76 | self.fidelities: np.ndarray = np.zeros(self.N, dtype=int)
77 | self.curves: np.ndarray = np.zeros((self.N, self.max_fidelity), dtype=float)
78 | self.costs: np.ndarray = np.zeros(self.N, dtype=float)
79 |
80 | if self.patience is not None:
81 | self._score_history = np.zeros((self.N, self.patience), dtype=float)
82 |
83 | def ask(self) -> dict | None:
84 | left = set(range(self.N)) - self.failed - self.stoped
85 | if not left:
86 | return None
87 | index = random.choice(list(left))
88 |
89 | fidelity = self.fidelities[index] + 1
90 |
91 | return {
92 | "config_id": index,
93 | "config": self.candidates[index],
94 | "fidelity": fidelity,
95 | }
96 |
97 | def tell(self, reports: dict | list):
98 | if isinstance(reports, dict):
99 | reports = [reports]
100 | for report in reports:
101 | self._tell(report)
102 |
103 | def _tell(self, report: dict):
104 | self.tell_count += 1
105 |
106 | index = report["config-id"]
107 | fidelity = report["fidelity"]
108 | cost = report["cost"]
109 | score = report["score"]
110 | status = report["status"]
111 |
112 | if not status:
113 | self.failed.add(index)
114 | return
115 |
116 | # update trackers
117 | self.curves[index, fidelity - 1] = score
118 | self.fidelities[index] = fidelity
119 | self.costs[index] = cost
120 | self.history.append(report)
121 | self.evaled.add(index)
122 | self.eval_count += 1
123 |
124 | if score >= 1.0 - self.scr_thr or fidelity == self.max_fidelity:
125 | self.stoped.add(index)
126 |
127 | if self.patience is not None:
128 | if not np.any(self._score_history[index] < (score - self.tol)):
129 | self.stoped.add(index)
130 | self._score_history[index][fidelity % self.patience] = score
131 |
--------------------------------------------------------------------------------
/src/qtt/predictors/__init__.py:
--------------------------------------------------------------------------------
1 | from .cost import CostPredictor
2 | from .perf import PerfPredictor
3 | from .predictor import Predictor
4 |
5 | __all__ = ["PerfPredictor", "CostPredictor", "Predictor"]
6 |
--------------------------------------------------------------------------------
/src/qtt/predictors/cost.py:
--------------------------------------------------------------------------------
1 | import logging
2 | import os
3 | import random
4 |
5 | import numpy as np
6 | import pandas as pd
7 | import torch
8 | import torch.nn as nn
9 | from numpy.typing import ArrayLike
10 | from sklearn import preprocessing # type: ignore
11 | from torch.utils.data import DataLoader, random_split
12 |
13 | from ..utils.log_utils import set_logger_verbosity
14 | from .data import (
15 | SimpleTorchTabularDataset,
16 | create_preprocessor,
17 | get_feature_mapping,
18 | get_types_of_features,
19 | )
20 | from .models import MLP
21 | from .predictor import Predictor
22 | from .utils import MetricLogger, get_torch_device
23 |
24 | logger = logging.getLogger(__name__)
25 |
26 | DEFAULT_FIT_PARAMS = {
27 | "learning_rate_init": 0.0001,
28 | "batch_size": 1024,
29 | "max_iter": 100,
30 | "early_stop": True,
31 | "patience": 5,
32 | "validation_fraction": 0.1,
33 | "tol": 1e-4,
34 | }
35 |
36 |
37 | class CostPredictor(Predictor):
38 | """A predictor that predicts the cost of training a configuration on a new dataset."""
39 |
40 | temp_file_name = "temp_model.pt"
41 |
42 | def __init__(
43 | self,
44 | fit_params: dict = {},
45 | # refit_params: dict = {},
46 | path: str | None = None,
47 | seed: int | None = None,
48 | verbosity: int = 2,
49 | ) -> None:
50 | super().__init__(path=path)
51 |
52 | self.fit_params = self._validate_fit_params(fit_params, DEFAULT_FIT_PARAMS)
53 | self.seed = seed
54 | self.verbose = verbosity
55 |
56 | set_logger_verbosity(verbosity, logger)
57 |
58 | @staticmethod
59 | def _validate_fit_params(fit_params, default_params):
60 | if not isinstance(fit_params, dict):
61 | raise ValueError("fit_params must be a dictionary")
62 | for key in fit_params:
63 | if key not in default_params:
64 | raise ValueError(f"Unknown fit parameter: {key}")
65 | return {**default_params, **fit_params}
66 |
67 | def _get_model(self):
68 | params = {
69 | "in_dim": [
70 | len(self.types_of_features["continuous"]),
71 | len(self.types_of_features["categorical"]) + len(self.types_of_features["bool"]),
72 | ],
73 | "enc_out_dim": 16,
74 | "enc_nlayers": 3,
75 | "enc_hidden_dim": 128,
76 | }
77 | model = SimpleMLPRegressor(**params)
78 | return model
79 |
80 | def _validate_fit_data(self, X, y):
81 | if not isinstance(X, pd.DataFrame):
82 | raise ValueError("X must be a pandas.DataFrame instance")
83 |
84 | if not isinstance(y, np.ndarray):
85 | raise ValueError("y must be a numpy.ndarray instance")
86 |
87 | if X.shape[0] != y.shape[0]:
88 | raise ValueError("X and y must have the same number of samples")
89 |
90 | if y.shape[1] != 1:
91 | raise ValueError("y must have only one column")
92 |
93 | if len(set(X.columns)) < len(X.columns):
94 | raise ValueError(
95 | "Column names are not unique, please change duplicated column names (in pandas: train_data.rename(columns={'current_name':'new_name'})"
96 | )
97 |
98 | def _validate_predict_data(self, pipeline):
99 | if not isinstance(pipeline, pd.DataFrame):
100 | raise ValueError("pipeline and curve must be pandas.DataFrame instances")
101 |
102 | if len(set(pipeline.columns)) < len(pipeline.columns):
103 | raise ValueError(
104 | "Column names are not unique, please change duplicated column names (in pandas: train_data.rename(columns={'current_name':'new_name'})"
105 | )
106 |
107 | def _preprocess_fit_data(self, df: pd.DataFrame, array: np.ndarray):
108 | """
109 | Process data for fitting the model.
110 | """
111 | self._original_features = list(df.columns)
112 |
113 | df, self.types_of_features, self.features_to_drop = get_types_of_features(df)
114 | self._input_features = list(df.columns)
115 | continous_features = self.types_of_features["continuous"]
116 | categorical_features = self.types_of_features["categorical"]
117 | bool_features = self.types_of_features["bool"]
118 | self.preprocessor = create_preprocessor(
119 | continous_features, categorical_features, bool_features
120 | )
121 | out = self.preprocessor.fit_transform(df)
122 | self._feature_mapping = get_feature_mapping(self.preprocessor)
123 | if out.shape[1] != sum(len(v) for v in self._feature_mapping.values()):
124 | raise ValueError(
125 | "Error during one-hot encoding data processing for neural network. "
126 | "Number of columns in df array does not match feature_mapping."
127 | )
128 |
129 | self.label_scaler = preprocessing.StandardScaler() # MaxAbsScaler()
130 | out_array = self.label_scaler.fit_transform(array)
131 |
132 | return out, out_array
133 |
134 | def _preprocess_predict_data(self, df: pd.DataFrame, fill_missing=True):
135 | unexpected_columns = set(df.columns) - set(self._original_features)
136 | if len(unexpected_columns) > 0:
137 | logger.warning(
138 | "Data contains columns that were not present during fitting: "
139 | f"{unexpected_columns}"
140 | )
141 |
142 | df = df.drop(columns=self.features_to_drop, errors="ignore")
143 |
144 | missing_columns = set(self._input_features) - set(df.columns)
145 | if len(missing_columns) > 0:
146 | if fill_missing:
147 | logger.warning(
148 | "Data is missing columns that were present during fitting: "
149 | f"{missing_columns}. Trying to fill them with mean values / zeros."
150 | )
151 | for col in missing_columns:
152 | df[col] = None
153 | else:
154 | raise AssertionError(
155 | "Data is missing columns that were present during fitting: "
156 | f"{missing_columns}. Please fill them with appropriate values."
157 | )
158 | X = self.preprocessor.transform(df)
159 | X = np.array(X)
160 | X = np.nan_to_num(X)
161 | return X
162 |
163 | def _fit_model(
164 | self,
165 | dataset,
166 | learning_rate_init,
167 | batch_size,
168 | max_iter,
169 | early_stop,
170 | patience,
171 | validation_fraction,
172 | tol,
173 | ):
174 | if self.seed is not None:
175 | random.seed(self.seed)
176 | np.random.seed(self.seed)
177 | torch.manual_seed(self.seed)
178 |
179 | self.device = get_torch_device()
180 | _dev = self.device
181 | self.model.to(_dev)
182 |
183 | optimizer = torch.optim.AdamW(self.model.parameters(), learning_rate_init)
184 |
185 | patience_counter = 0
186 | best_iter = 0
187 | best_val_metric = np.inf
188 |
189 | if patience is not None:
190 | if early_stop:
191 | if validation_fraction < 0 or validation_fraction > 1:
192 | raise AssertionError(
193 | "validation_fraction must be between 0 and 1 when early_stop is True"
194 | )
195 | logger.info(
196 | f"Early stopping on validation loss with patience {patience} "
197 | f"using {validation_fraction} of the data for validation"
198 | )
199 | train_set, val_set = random_split(
200 | dataset=dataset,
201 | lengths=[1 - validation_fraction, validation_fraction],
202 | )
203 | else:
204 | logger.info(f"Early stopping on training loss with patience {patience}")
205 | train_set = dataset
206 | val_set = None
207 | else:
208 | train_set = dataset
209 | val_set = None
210 |
211 | bs = min(batch_size, int(2 ** (3 + np.floor(np.log10(len(train_set))))))
212 | train_loader = DataLoader(train_set, batch_size=bs, shuffle=True, drop_last=True)
213 | val_loader = None
214 | if val_set is not None:
215 | bs = min(batch_size, int(2 ** (3 + np.floor(np.log10(len(val_set))))))
216 | val_loader = DataLoader(val_set, batch_size=bs)
217 |
218 | cache_dir = os.path.expanduser("~/.cache")
219 | cache_dir = os.path.join(cache_dir, "qtt", self.name)
220 | os.makedirs(cache_dir, exist_ok=True)
221 | temp_save_file_path = os.path.join(cache_dir, self.temp_file_name)
222 | for it in range(1, max_iter + 1):
223 | self.model.train()
224 |
225 | train_loss = []
226 | header = f"TRAIN: ({it}/{max_iter})"
227 | metric_logger = MetricLogger(delimiter=" ")
228 | for batch in metric_logger.log_every(
229 | train_loader, len(train_loader) // 10, header, logger
230 | ):
231 | # forward
232 | batch = [item.to(_dev) for item in batch]
233 | X, y = batch
234 | loss = self.model.train_step(X, y)
235 | train_loss.append(loss.item())
236 |
237 | # update
238 | optimizer.zero_grad()
239 | loss.backward()
240 | optimizer.step()
241 |
242 | metric_logger.update(loss=loss.item())
243 | logger.info(f"Averaged stats: {str(metric_logger)}")
244 | val_metric = np.mean(train_loss)
245 |
246 | if val_loader is not None:
247 | self.model.eval()
248 |
249 | val_loss = []
250 | with torch.no_grad():
251 | for batch in val_loader:
252 | batch = [item.to(_dev) for item in batch]
253 | X, y = batch
254 | pred = self.model.predict(X)
255 | loss = torch.nn.functional.l1_loss(pred, y)
256 | val_loss.append(loss.item())
257 | val_metric = np.mean(val_loss)
258 |
259 | if patience is not None:
260 | if val_metric + tol < best_val_metric:
261 | patience_counter = 0
262 | best_val_metric = val_metric
263 | best_iter = it
264 | torch.save(self.model.state_dict(), temp_save_file_path)
265 | else:
266 | patience_counter += 1
267 | logger.info(
268 | f"VAL: {round(val_metric, 4)} "
269 | f"ITER: {it}/{max_iter} "
270 | f"BEST: {round(best_val_metric, 4)} ({best_iter})"
271 | )
272 | if patience_counter >= patience:
273 | logger.warning(
274 | "Early stopping triggered! "
275 | f"No improvement in the last {patience} iterations. "
276 | "Stopping training..."
277 | )
278 | break
279 |
280 | if early_stop:
281 | self.model.load_state_dict(torch.load(temp_save_file_path, weights_only=True))
282 |
283 | def _fit(
284 | self,
285 | X: pd.DataFrame,
286 | y: ArrayLike,
287 | **kwargs,
288 | ):
289 | if self.is_fit:
290 | raise AssertionError("Predictor is already fit! Create a new one.")
291 |
292 | y = np.array(y)
293 |
294 | self._validate_fit_data(X, y)
295 | _X, _y = self._preprocess_fit_data(X, y)
296 |
297 | train_dataset = SimpleTorchTabularDataset(_X, _y)
298 |
299 | self.model = self._get_model()
300 |
301 | self._fit_model(train_dataset, **self.fit_params)
302 |
303 | return self
304 |
305 | def _predict(self, **kwargs) -> np.ndarray:
306 | """Predict the costs of training a configuration on a new dataset.
307 |
308 | Args:
309 | X (pd.DataFrame): the configuration to predict.
310 | """
311 | if not self.is_fit or self.model is None:
312 | raise AssertionError("Model is not fitted yet")
313 |
314 | X: pd.DataFrame = kwargs.pop("X", None)
315 | if X is None:
316 | raise ValueError("X (pipeline configuration) must be provided")
317 |
318 | self._validate_predict_data(X)
319 | x = self._preprocess_predict_data(X)
320 |
321 | self.model.eval()
322 | self.model.to(self.device)
323 | x_t = torch.tensor(x, dtype=torch.float32).to(self.device)
324 |
325 | with torch.no_grad():
326 | pred = self.model.predict(x_t)
327 | out = pred.cpu().squeeze().numpy()
328 | return out
329 |
330 | def save(self, path: str | None = None, verbose=True) -> str:
331 | # Save on CPU to ensure the model can be loaded on a box without GPU
332 | if self.model is not None:
333 | self.model = self.model.to(torch.device("cpu"))
334 | path = super().save(path, verbose)
335 | # Put the model back to the device after the save
336 | if self.model is not None:
337 | self.model.to(self.device)
338 | return path
339 |
340 | @classmethod
341 | def load(cls, path: str, reset_paths=True, verbose=True):
342 | """
343 | Loads the model from disk to memory.
344 | The loaded model will be on the same device it was trained on (cuda/mps);
345 | if the device is it's not available (trained on GPU, deployed on CPU),
346 | then `cpu` will be used.
347 |
348 | Parameters
349 | ----------
350 | path : str
351 | Path to the saved model, minus the file name.
352 | This should generally be a directory path ending with a '/' character (or appropriate path separator value depending on OS).
353 | The model file is typically located in os.path.join(path, cls.model_file_name).
354 | reset_paths : bool, default True
355 | Whether to reset the self.path value of the loaded model to be equal to path.
356 | It is highly recommended to keep this value as True unless accessing the original self.path value is important.
357 | If False, the actual valid path and self.path may differ, leading to strange behaviour and potential exceptions if the model needs to load any other files at a later time.
358 | verbose : bool, default True
359 | Whether to log the location of the loaded file.
360 |
361 | Returns
362 | -------
363 | model : cls
364 | Loaded model object.
365 | """
366 | model: CostPredictor = super().load(path=path, reset_paths=reset_paths, verbose=verbose)
367 | return model
368 |
369 |
370 | class SimpleMLPRegressor(torch.nn.Module):
371 | def __init__(
372 | self,
373 | in_dim: int | list[int],
374 | enc_out_dim: int = 8,
375 | enc_nlayers: int = 3,
376 | enc_hidden_dim: int = 128,
377 | ):
378 | super().__init__()
379 | if isinstance(in_dim, int):
380 | in_dim = [in_dim]
381 | self.in_dim = in_dim
382 |
383 | # build config encoder
384 | encoder = nn.ModuleList()
385 | for dim in self.in_dim:
386 | encoder.append(MLP(dim, enc_out_dim, enc_nlayers, enc_hidden_dim))
387 | self.config_encoder = encoder
388 | enc_dims = len(self.config_encoder) * enc_out_dim
389 |
390 | self.head = MLP(enc_dims, 1, enc_nlayers, enc_hidden_dim, act_fn=nn.GELU)
391 |
392 | def forward(self, X) -> torch.Tensor:
393 | x = []
394 | start = 0
395 | for i, dim in enumerate(self.in_dim):
396 | end = start + dim
397 | output = self.config_encoder[i](X[:, start:end])
398 | x.append(output)
399 | start = end
400 | t = torch.cat(x, dim=1)
401 | t = self.head(t)
402 | return t
403 |
404 | def predict(self, X) -> torch.Tensor:
405 | return self(X)
406 |
407 | def train_step(self, X, y) -> torch.Tensor:
408 | pred = self(X)
409 | loss = torch.nn.functional.huber_loss(pred, y)
410 | return loss
411 |
--------------------------------------------------------------------------------
/src/qtt/predictors/data.py:
--------------------------------------------------------------------------------
1 | from collections import OrderedDict
2 |
3 | import logging
4 | import numpy as np
5 | import pandas as pd
6 | import torch
7 | from sklearn.compose import ColumnTransformer # type: ignore
8 | from sklearn.impute import SimpleImputer # type: ignore
9 | from sklearn.pipeline import Pipeline # type: ignore
10 | from sklearn.preprocessing import OneHotEncoder, StandardScaler # type: ignore
11 | from torch.utils.data import Dataset
12 |
13 | logger = logging.getLogger(__name__)
14 |
15 |
16 | def _custom_combiner(input_feature, category):
17 | return str(input_feature) + "=" + str(category)
18 |
19 |
20 | def create_preprocessor(continous_features, categorical_features, bool_features):
21 | transformers = []
22 | if continous_features:
23 | pipeline = Pipeline(
24 | steps=[
25 | ("imputer", SimpleImputer(strategy="constant", fill_value=0)),
26 | ("scaler", StandardScaler()),
27 | ]
28 | )
29 | transformers.append(("continuous", pipeline, continous_features))
30 | if categorical_features:
31 | pipeline = Pipeline(
32 | steps=[
33 | (
34 | "onehot",
35 | OneHotEncoder(
36 | handle_unknown="ignore",
37 | sparse_output=False,
38 | feature_name_combiner=_custom_combiner, # type: ignore
39 | ),
40 | )
41 | ]
42 | )
43 | transformers.append(("categorical", pipeline, categorical_features))
44 | if bool_features:
45 | pipeline = Pipeline(steps=[("scaler", StandardScaler())])
46 | transformers.append(("bool", pipeline, bool_features))
47 | return ColumnTransformer(
48 | transformers=transformers,
49 | remainder="passthrough",
50 | force_int_remainder_cols=False, # type: ignore
51 | )
52 |
53 |
54 | def get_types_of_features(df: pd.DataFrame) -> tuple[pd.DataFrame, dict, list]:
55 | """Returns dict with keys: 'continuous', 'categorical', 'bool'.
56 | Values = list of feature-names falling into each category.
57 | Each value is a list of feature-names corresponding to columns in original dataframe.
58 | """
59 | unique_features = [col for col in df.columns if df[col].nunique() == 1]
60 | if unique_features:
61 | logger.info(f"Features {unique_features} have only one unique value and are dropped")
62 | df.drop(columns=unique_features, inplace=True)
63 |
64 | nan_features = df.columns[df.isna().all()].tolist()
65 | if nan_features:
66 | logger.info(f"Features {nan_features} have only NaN values and are dropped")
67 | df = df.drop(columns=nan_features)
68 |
69 | continous_features = list(df.select_dtypes(include=["number"]).columns)
70 | categorical_features = list(df.select_dtypes(include=["object"]).columns)
71 | bool_features = list(df.select_dtypes(include=["bool"]).columns)
72 | valid_features = continous_features + categorical_features + bool_features
73 |
74 | unknown_features = [col for col in df.columns if col not in valid_features]
75 | if unknown_features:
76 | logger.info(f"Features {unknown_features} have unknown dtypes and are dropped")
77 | df = df.drop(columns=unknown_features)
78 |
79 | types_of_features: dict = {"continuous": [], "categorical": [], "bool": []}
80 | for col in df.columns:
81 | if col in continous_features:
82 | types_of_features["continuous"].append(col)
83 | elif col in categorical_features:
84 | types_of_features["categorical"].append(col)
85 | elif col in bool_features:
86 | types_of_features["bool"].append(col)
87 |
88 | features_to_drop = unique_features + nan_features + unknown_features
89 | return df, types_of_features, features_to_drop
90 |
91 |
92 | def get_feature_mapping(processor):
93 | feature_preserving_transforms = set(["continuous", "bool", "remainder"])
94 | feature_mapping = {}
95 | col_index = 0
96 | for tf_name, tf, transformed_features in processor.transformers_:
97 | if tf_name in feature_preserving_transforms:
98 | for feature in transformed_features:
99 | feature_mapping[feature] = [col_index]
100 | col_index += 1
101 | elif tf_name == "categorical":
102 | encoder = [step for (name, step) in tf.steps if name == "onehot"][0]
103 | for i in range(len(transformed_features)):
104 | feature = transformed_features[i]
105 | if feature in feature_mapping:
106 | raise ValueError(
107 | f"same feature is processed by two different column transformers: {feature}"
108 | )
109 | encoding_size = len(encoder.categories_[i])
110 | feature_mapping[feature] = list(range(col_index, col_index + encoding_size))
111 | col_index += encoding_size
112 | else:
113 | raise ValueError(f"Unknown transformer {tf_name}")
114 | return OrderedDict([(key, feature_mapping[key]) for key in feature_mapping])
115 |
116 |
117 | class SimpleTorchTabularDataset(Dataset):
118 | def __init__(self, *args):
119 | super().__init__()
120 | self.data = [torch.tensor(arg, dtype=torch.float32) for arg in args]
121 |
122 | def __len__(self):
123 | return self.data[0].shape[0]
124 |
125 | def __getitem__(self, idx):
126 | return [arg[idx] for arg in self.data]
127 |
128 |
129 | class CurveRegressionDataset(Dataset):
130 | """ """
131 |
132 | def __init__(self, x: np.ndarray, y: np.ndarray):
133 | super().__init__()
134 |
135 | true_indices = np.where(~np.isnan(y.flatten()))[0]
136 | self.mapping = list(true_indices)
137 | self.x = x
138 | self.y = y
139 | self.y_dim = y.shape[1]
140 |
141 | def __len__(self):
142 | return len(self.mapping)
143 |
144 | def __getitem__(self, idx):
145 | idx = self.mapping[idx]
146 | true_index = idx // self.y_dim
147 | fidelity = idx % self.y_dim
148 | curve = np.concatenate(
149 | (self.y[true_index, :fidelity], np.zeros(self.y.shape[1] - fidelity))
150 | )
151 | target = self.y[true_index, fidelity]
152 | x = torch.tensor(self.x[true_index], dtype=torch.float32)
153 | curve = torch.tensor(curve, dtype=torch.float32)
154 | target = torch.tensor(target, dtype=torch.float32)
155 | return x, curve, target
156 |
157 |
158 | def make_regression_from_series_dataset(pipeline: pd.DataFrame, curve: np.ndarray):
159 | """
160 | This method takes a pandas DataFrame `pipeline` containing features and a numpy
161 | array `curve` representing learning curves. For each point in the curve
162 | array the function creates a training sample.
163 |
164 | -> [pipeline] [y_0, ..., y_m]
165 | [pipeline] [0, ..., 0] -> [y_0]
166 | [pipeline] [y_0, 0, ..., 0] -> [y_1]
167 | [pipeline] [y_0, y_1, 0, ..., 0] -> [y_2]
168 | ...
169 | [pipeline] [y_0, y_1, ..., y_m-1, 0] -> [y_m]
170 |
171 | The transformation produces three outputs:
172 | 1. `X`: A DataFrame where each row corresponds to a repeated and filtered
173 | version of the original `pipeline` data.
174 | 2. `curve_out`: A 2D numpy array representing the padded versions of the
175 | sequences in `curve`.
176 | 3. `y`: A 1D numpy array representing the flattened and filtered values
177 | from `curve`.
178 | """
179 | _, m = curve.shape
180 | y = curve.flatten()
181 | curve_out = np.vstack(
182 | [np.concatenate((row[:i], np.zeros(m - i))) for row in curve for i in range(m)]
183 | )
184 | mask = np.isnan(y)
185 | y = y[~mask]
186 | curve_out = curve_out[~mask]
187 | X = pipeline.values.repeat(m, axis=0)[~mask]
188 |
189 | X = pd.DataFrame(X, columns=pipeline.columns)
190 | for col in pipeline.columns:
191 | X[col] = X[col].astype(pipeline[col].dtype)
192 | curve_out = np.array(curve_out)
193 | y = np.array(y)
194 |
195 | if X.shape[0] != curve_out.shape[0] or curve_out.shape[0] != y.shape[0]:
196 | raise ValueError("Data size mismatch")
197 |
198 | return X, curve_out, y
199 |
--------------------------------------------------------------------------------
/src/qtt/predictors/models.py:
--------------------------------------------------------------------------------
1 | from typing import Type
2 |
3 | import torch
4 | import torch.nn as nn
5 |
6 |
7 | class FeatureEncoder(nn.Module):
8 | def __init__(
9 | self,
10 | in_dim: int | list[int],
11 | in_curve_dim: int,
12 | out_dim: int = 16,
13 | enc_hidden_dim: int = 128,
14 | enc_out_dim: int = 8,
15 | enc_nlayers: int = 3,
16 | out_curve_dim: int = 8,
17 | ):
18 | super().__init__()
19 | if isinstance(in_dim, int):
20 | in_dim = [in_dim]
21 | self.in_dim = in_dim
22 | enc_dims = 0 # feature dimension after encoding
23 |
24 | # build pipeline encoder
25 | encoder = nn.ModuleList()
26 | for dim in self.in_dim:
27 | encoder.append(MLP(dim, enc_out_dim, enc_nlayers, enc_hidden_dim))
28 | self.config_encoder = encoder
29 | enc_dims += len(self.config_encoder) * enc_out_dim
30 |
31 | # build curve encoder
32 | self.curve_encoder = MLP(in_curve_dim, out_curve_dim, enc_nlayers, enc_hidden_dim)
33 | enc_dims += out_curve_dim
34 |
35 | self.head = MLP(enc_dims, out_dim, 3, enc_hidden_dim, act_fn=nn.GELU)
36 |
37 | def forward(self, pipeline, curve):
38 | # encode config
39 | start = 0
40 | x = []
41 | for i, dim in enumerate(self.in_dim):
42 | end = start + dim
43 | output = self.config_encoder[i](pipeline[:, start:end]) # type: ignore
44 | x.append(output)
45 | start = end
46 | x = torch.cat(x, dim=1)
47 |
48 | # budget = (curve > 0).sum(dim=-1, keepdim=True) + 1
49 | # budget /= curve.shape[-1]
50 | # x = torch.cat([x, budget], dim=1)
51 |
52 | # encode curve
53 | out = self.curve_encoder(curve)
54 | x = torch.cat([x, out], dim=1)
55 |
56 | x = self.head(x)
57 | # x = torch.softmax(x, dim=-1)
58 | return x
59 |
60 | def freeze(self):
61 | for param in self.parameters():
62 | param.requires_grad = False
63 |
64 |
65 | class CNN(nn.Module):
66 | def __init__(
67 | self,
68 | in_dim: int,
69 | in_channels: int,
70 | out_dim: int,
71 | act_fn: Type[nn.Module] = nn.ReLU,
72 | ):
73 | super().__init__()
74 | self.model = nn.Sequential(
75 | nn.Conv1d(in_channels, out_channels=8, kernel_size=3, padding="same"),
76 | act_fn(),
77 | nn.MaxPool1d(kernel_size=2),
78 | nn.Conv1d(8, out_channels=16, kernel_size=3, padding="same"),
79 | act_fn(),
80 | nn.MaxPool1d(kernel_size=2),
81 | nn.Conv1d(16, out_channels=32, kernel_size=3, padding="same"),
82 | act_fn(),
83 | nn.MaxPool1d(kernel_size=2),
84 | nn.Flatten(),
85 | nn.Linear(32 * (in_dim // 8), 32 * (in_dim // 8) // 2),
86 | act_fn(),
87 | nn.Linear(32 * (in_dim // 8) // 2, out_dim),
88 | )
89 |
90 | def forward(self, x):
91 | return self.model(x)
92 |
93 |
94 | class MLP(nn.Module):
95 | def __init__(
96 | self,
97 | in_dim: int,
98 | out_dim: int,
99 | nlayers: int = 2,
100 | hidden_dim: int = 128,
101 | bottleneck_dim: int = 8,
102 | act_fn: Type[nn.Module] = nn.ReLU,
103 | ):
104 | super().__init__()
105 | nlayers = max(nlayers, 1)
106 | if nlayers == 1:
107 | self.mlp: nn.Module = nn.Linear(in_dim, bottleneck_dim)
108 | else:
109 | layers = [nn.Linear(in_dim, hidden_dim), act_fn()]
110 | for _ in range(nlayers - 2):
111 | layers.append(nn.Linear(hidden_dim, hidden_dim))
112 | layers.append(act_fn())
113 | layers.append(nn.Linear(hidden_dim, bottleneck_dim))
114 | self.mlp = nn.Sequential(*layers)
115 | self.head = nn.Linear(bottleneck_dim, out_dim)
116 |
117 | def forward(self, x):
118 | x = self.mlp(x)
119 | x = nn.functional.normalize(x, dim=-1, p=2)
120 | x = self.head(x)
121 | return x
122 |
--------------------------------------------------------------------------------
/src/qtt/predictors/perf.py:
--------------------------------------------------------------------------------
1 | import copy
2 | import logging
3 | import os
4 | import random
5 | import shutil
6 | from typing import Tuple
7 |
8 | import gpytorch # type: ignore
9 | import numpy as np
10 | import pandas as pd
11 | import torch
12 | from numpy.typing import ArrayLike
13 | from torch.utils.data import DataLoader, random_split
14 |
15 | from qtt.predictors.models import FeatureEncoder
16 | from qtt.utils.log_utils import set_logger_verbosity
17 |
18 | from .data import (
19 | CurveRegressionDataset,
20 | create_preprocessor,
21 | get_feature_mapping,
22 | get_types_of_features,
23 | )
24 | from .predictor import Predictor
25 | from .utils import MetricLogger, get_torch_device
26 |
27 | logger = logging.getLogger(__name__)
28 |
29 | DEFAULT_FIT_PARAMS = {
30 | "learning_rate_init": 0.001,
31 | "batch_size": 2048,
32 | "max_iter": 100,
33 | "early_stop": True,
34 | "patience": 5,
35 | "validation_fraction": 0.1,
36 | "tol": 1e-4,
37 | }
38 |
39 | DEFAULT_REFIT_PARAMS = {
40 | "learning_rate_init": 0.001,
41 | "batch_size": 2048,
42 | "max_iter": 50,
43 | "early_stop": True,
44 | "patience": 5,
45 | "tol": 1e-4,
46 | }
47 |
48 |
49 | class PerfPredictor(Predictor):
50 | temp_file_name: str = "temp_model.pt"
51 | train_data_size: int = 4096
52 | _fit_data = None
53 |
54 | def __init__(
55 | self,
56 | fit_params: dict = {},
57 | refit_params: dict = {},
58 | path: str | None = None,
59 | seed: int | None = None,
60 | verbosity: int = 2,
61 | ) -> None:
62 | super().__init__(path=path)
63 | self.fit_params = self._validate_fit_params(fit_params, DEFAULT_FIT_PARAMS)
64 | self.refit_params = self._validate_fit_params(refit_params, DEFAULT_REFIT_PARAMS)
65 | self.seed = seed
66 | self.verbosity = verbosity
67 |
68 | set_logger_verbosity(verbosity, logger)
69 |
70 | @staticmethod
71 | def _validate_fit_params(fit_params, default_params):
72 | """
73 | Validate hyperparameters for fitting the model.
74 |
75 | Args:
76 | fit_params (dict): Hyperparameters for fitting the model.
77 | default_params (dict): Default hyperparameters for fitting the model.
78 |
79 | Raises:
80 | ValueError: If fit_params is not a dictionary or contains unknown hyperparameters.
81 |
82 | Returns:
83 | dict: Validated hyperparameters.
84 | """
85 | if not isinstance(fit_params, dict):
86 | raise ValueError("fit_params must be a dictionary")
87 | for key in fit_params:
88 | if key not in default_params:
89 | raise ValueError(f"Unknown fit parameter: {key}")
90 | return {**default_params, **fit_params}
91 |
92 | def _validate_fit_data(self, pipeline: pd.DataFrame, curve: np.ndarray):
93 | """
94 | Validate data for fitting the model.
95 |
96 | Args:
97 | pipeline (pandas.DataFrame): Pipeline data.
98 | curve (numpy.ndarray): Curve data.
99 |
100 | Raises:
101 | ValueError: If pipeline or curve is not a pandas.DataFrame or numpy.ndarray, or if
102 | pipeline and curve have different number of samples, or if column names are not
103 | unique.
104 |
105 | Returns:
106 | tuple: Validated pipeline and curve data.
107 | """
108 | if not isinstance(pipeline, pd.DataFrame):
109 | raise ValueError("pipeline must be a pandas.DataFrame instance")
110 |
111 | if not isinstance(curve, np.ndarray):
112 | raise ValueError("curve must be a numpy.ndarray instance")
113 |
114 | if pipeline.shape[0] != curve.shape[0]:
115 | raise ValueError("pipeline and curve must have the same number of samples")
116 |
117 | if len(set(pipeline.columns)) < len(pipeline.columns):
118 | raise ValueError(
119 | "Column names are not unique, please change duplicated column names (in pandas: train_data.rename(columns={'current_name':'new_name'})"
120 | )
121 |
122 | self._curve_dim = curve.shape[1]
123 |
124 | def _preprocess_fit_data(self, df: pd.DataFrame) -> np.ndarray:
125 | """
126 | Preprocess data for fitting the model.
127 |
128 | Args:
129 | df (pandas.DataFrame): Data to preprocess.
130 |
131 | Returns:
132 | numpy.ndarray: Preprocessed data.
133 | """
134 | self.original_features = list(df.columns)
135 |
136 | df, self.types_of_features, self.features_to_drop = get_types_of_features(df)
137 | self.input_features = list(df.columns)
138 |
139 | self.preprocessor = create_preprocessor(
140 | self.types_of_features["continuous"],
141 | self.types_of_features["categorical"],
142 | self.types_of_features["bool"],
143 | )
144 | out = self.preprocessor.fit_transform(df)
145 | self.feature_mapping = get_feature_mapping(self.preprocessor)
146 | if out.shape[1] != sum(len(v) for v in self.feature_mapping.values()):
147 | raise ValueError(
148 | "Error during one-hot encoding data processing for neural network. "
149 | "Number of columns in df array does not match feature_mapping."
150 | )
151 | return np.array(out)
152 |
153 | def _validate_predict_data(self, pipeline, curve):
154 | """Validate data for prediction. Applies the same steps as _validate_fit_data
155 |
156 | Args:
157 | pipeline (pandas.DataFrame): Pipeline data.
158 | curve (numpy.ndarray): Curve data.
159 |
160 | Raises:
161 | ValueError: If pipeline or curve is not a pandas.DataFrame or numpy.ndarray, or if
162 | pipeline and curve have different number of samples, or if column names are not
163 | unique.
164 |
165 | Returns:
166 | tuple: Validated pipeline and curve data.
167 | """
168 | if not isinstance(pipeline, pd.DataFrame) or not isinstance(curve, np.ndarray):
169 | raise ValueError("pipeline and curve must be pandas.DataFrame instances")
170 |
171 | if pipeline.shape[0] != curve.shape[0]:
172 | raise ValueError("pipeline and curve must have the same number of samples")
173 |
174 | if len(set(pipeline.columns)) < len(pipeline.columns):
175 | raise ValueError(
176 | "Column names are not unique, please change duplicated column names (in pandas: train_data.rename(columns={'current_name':'new_name'})"
177 | )
178 |
179 | if curve.shape[1] != self._curve_dim:
180 | raise ValueError(
181 | "curve must have the same number of features as the curve used for fitting"
182 | " (expected: {self._curve_length}, got: {curve.shape[1]})"
183 | )
184 |
185 | def _preprocess_predict_data(self, df: pd.DataFrame, fill_missing=True):
186 | extra_features = list(set(df.columns) - set(self.original_features))
187 | if extra_features:
188 | logger.warning(
189 | f"Features {extra_features} were not present in training data and are dropped"
190 | )
191 | df = df.drop(columns=extra_features, errors="ignore")
192 |
193 | df = df.drop(columns=self.features_to_drop, errors="ignore")
194 |
195 | missing_features = list(set(self.input_features) - set(df.columns))
196 | if missing_features:
197 | if fill_missing:
198 | logger.warning(
199 | f"Features {missing_features} missing in data. Missing values will be imputed."
200 | )
201 | for col in missing_features:
202 | df[col] = None
203 | else:
204 | raise AssertionError(f"Features {missing_features} missing in data.")
205 |
206 | # process data
207 | X = self.preprocessor.transform(df)
208 | X = np.array(X)
209 | X = np.nan_to_num(X)
210 | return X
211 |
212 | def _get_model(self):
213 | """
214 | Return a new instance of the model.
215 |
216 | Returns:
217 | SurrogateModel: A new instance of the model.
218 | """
219 | params = {
220 | "in_dim": [
221 | len(self.types_of_features["continuous"]),
222 | len(self.types_of_features["categorical"]) + len(self.types_of_features["bool"]),
223 | ],
224 | "in_curve_dim": self._curve_dim,
225 | }
226 | return SurrogateModel(**params)
227 |
228 | def _train_model(
229 | self,
230 | dataset,
231 | learning_rate_init: float,
232 | batch_size: int,
233 | max_iter: int,
234 | early_stop: bool,
235 | patience: int | None,
236 | validation_fraction: float,
237 | tol: float,
238 | ):
239 | """Train the model on the given dataset.
240 |
241 | Args:
242 | dataset: Dataset to train on.
243 | learning_rate_init (float): Initial learning rate.
244 | batch_size (int): Batch size to use.
245 | max_iter (int): Maximum number of iterations to train for.
246 | early_stop (bool): If True, stop training when validation loss stops improving.
247 | patience (int or None): Number of iterations to wait before stopping training
248 | if validation loss does not improve.
249 | validation_fraction (float): Fraction of the dataset to use for validation.
250 | tol (float): Tolerance for determining when to stop training.
251 | """
252 | if self.seed is not None:
253 | random.seed(self.seed)
254 | np.random.seed(self.seed)
255 | torch.manual_seed(self.seed)
256 |
257 | if self.model is None:
258 | raise ValueError("Model must be set before training")
259 |
260 | self.device = get_torch_device()
261 | dev = self.device
262 | self.model.to(dev)
263 |
264 | optimizer = torch.optim.AdamW(self.model.parameters(), learning_rate_init)
265 |
266 | patience_counter = 0
267 | best_iter = 0
268 | best_val_metric = np.inf
269 |
270 | if patience is not None:
271 | if early_stop:
272 | if validation_fraction <= 0 or validation_fraction >= 1:
273 | raise AssertionError(
274 | "validation_fraction must be between 0 and 1 when early_stop is True"
275 | )
276 | logger.info(
277 | f"Early stopping on validation loss with patience {patience} "
278 | f"using {validation_fraction} of the data for validation"
279 | )
280 | train_set, val_set = random_split(
281 | dataset=dataset,
282 | lengths=[1 - validation_fraction, validation_fraction],
283 | )
284 | else:
285 | logger.info(f"Early stopping on training loss with patience {patience}")
286 | train_set = dataset
287 | val_set = None
288 | else:
289 | train_set = dataset
290 | val_set = None
291 |
292 | train_loader = DataLoader(
293 | train_set,
294 | batch_size=min(batch_size, len(train_set)),
295 | shuffle=True,
296 | drop_last=True,
297 | num_workers=4,
298 | )
299 | val_loader = None
300 | if val_set is not None:
301 | val_loader = DataLoader(
302 | val_set,
303 | batch_size=min(batch_size, len(val_set)),
304 | num_workers=4,
305 | )
306 |
307 | cache_dir = os.path.join(self.path, ".tmp")
308 | os.makedirs(cache_dir, exist_ok=True)
309 | temp_save_file_path = os.path.join(cache_dir, self.temp_file_name)
310 | for it in range(1, max_iter + 1):
311 | self.model.train()
312 |
313 | train_loss = []
314 | header = f"TRAIN: ({it}/{max_iter})"
315 | metric_logger = MetricLogger(delimiter=" ")
316 | for batch in metric_logger.log_every(
317 | train_loader, max(len(train_loader) // 10, 1), header, logger
318 | ):
319 | # forward
320 | batch = (b.to(dev) for b in batch)
321 | X, curve, y = batch
322 | loss = self.model.train_step(X, curve, y)
323 | train_loss.append(loss.item())
324 |
325 | # update
326 | optimizer.zero_grad()
327 | loss.backward()
328 | optimizer.step()
329 |
330 | # log
331 | metric_logger.update(loss=loss.item())
332 | metric_logger.update(lengthscale=self.model.lengthscale)
333 | metric_logger.update(noise=self.model.noise) # type: ignore
334 | logger.info(f"({it}/{max_iter}) Averaged stats: {str(metric_logger)}")
335 | val_metric = np.mean(train_loss)
336 |
337 | if val_loader is not None:
338 | self.model.eval()
339 |
340 | val_loss = []
341 | with torch.no_grad():
342 | for batch in val_loader:
343 | batch = (b.to(dev) for b in batch)
344 | X, curve, y = batch
345 | pred = self.model.predict(X, curve)
346 | loss = torch.nn.functional.l1_loss(pred.mean, y)
347 | val_loss.append(loss.item())
348 | val_metric = np.mean(val_loss)
349 |
350 | if patience is not None:
351 | if val_metric + tol < best_val_metric:
352 | patience_counter = 0
353 | best_val_metric = val_metric
354 | best_iter = it
355 | torch.save(self.model.state_dict(), temp_save_file_path)
356 | else:
357 | patience_counter += 1
358 | logger.info(
359 | f"VAL: {round(val_metric, 4)} "
360 | f"ITER: {it}/{max_iter} "
361 | f"BEST: {round(best_val_metric, 4)} ({best_iter})"
362 | )
363 | if patience_counter >= patience:
364 | logger.log(
365 | 15,
366 | "Stopping training..."
367 | f"No improvement in the last {patience} iterations. ",
368 | )
369 | break
370 |
371 | if early_stop:
372 | logger.info(
373 | f"Loading best model from iteration {best_iter} with val score {best_val_metric}"
374 | )
375 | self.model.load_state_dict(torch.load(temp_save_file_path, weights_only=True))
376 |
377 | if os.path.exists(cache_dir):
378 | shutil.rmtree(cache_dir)
379 |
380 | # after training the gp, set its training data
381 | # TODO: check if this can be improved
382 | self.model.eval()
383 | size = min(self.train_data_size, len(dataset))
384 | loader = DataLoader(dataset, batch_size=size, shuffle=True)
385 | a, b, c = next(iter(loader))
386 | a, b, c = a.to(dev), b.to(dev), c.to(dev)
387 | self.model.set_train_data(a, b, c)
388 |
389 | def _fit(self, X: pd.DataFrame, y: ArrayLike, **kwargs):
390 | if self.is_fit:
391 | raise AssertionError("Predictor is already fit! Create a new one.")
392 | y = np.array(y)
393 |
394 | self._validate_fit_data(X, y)
395 | x = self._preprocess_fit_data(X)
396 | train_dataset = CurveRegressionDataset(x, y)
397 |
398 | self.model = self._get_model()
399 | self._train_model(train_dataset, **self.fit_params)
400 |
401 | self._model_fit = copy.deepcopy(self.model)
402 | self._fit_data = train_dataset
403 |
404 | return self
405 |
406 | def fit_extra(
407 | self,
408 | X: pd.DataFrame,
409 | curve: np.ndarray,
410 | fit_params: dict = {},
411 | ):
412 | if not self.is_fit:
413 | raise AssertionError("Model is not fitted yet")
414 |
415 | self._validate_predict_data(X, curve)
416 |
417 | x = self._preprocess_predict_data(X)
418 |
419 | tune_dataset = CurveRegressionDataset(x, curve)
420 |
421 | fit_params = self._validate_fit_params(fit_params, self.refit_params)
422 | self._refit_model(tune_dataset, **fit_params)
423 |
424 | def _refit_model(
425 | self,
426 | dataset,
427 | learning_rate_init,
428 | batch_size,
429 | max_iter,
430 | early_stop,
431 | patience,
432 | tol,
433 | ):
434 | learning_rate_init = 0.001
435 | logger.info("Refitting model...")
436 | if self.seed is not None:
437 | random.seed(self.seed)
438 | np.random.seed(self.seed)
439 | torch.manual_seed(self.seed)
440 | cache_dir = os.path.join(self.path, ".tmp")
441 | os.makedirs(cache_dir, exist_ok=True)
442 | temp_save_file_path = os.path.join(cache_dir, self.temp_file_name)
443 |
444 | num_workers = 4
445 | self.device = get_torch_device()
446 | dev = self.device
447 |
448 | self.model.to(dev)
449 | self.model.eval()
450 | torch.save(self.model.state_dict(), temp_save_file_path)
451 |
452 | # initial validation loss
453 | loader = DataLoader(
454 | dataset,
455 | batch_size=min(len(dataset), batch_size),
456 | num_workers=num_workers,
457 | )
458 | val_metric = []
459 | for batch in loader:
460 | batch = (b.to(dev) for b in batch)
461 | X, curve, y = batch
462 | pred = self.model.predict(X, curve)
463 | loss = torch.nn.functional.l1_loss(pred.mean, y)
464 | val_metric.append(loss.item())
465 | best_val_metric = np.mean(val_metric)
466 | logger.info(f"Initial validation loss: {best_val_metric}")
467 | patience_counter = 0
468 | best_iter = 0
469 |
470 | assert self._fit_data is not None
471 | fitting_set = self._fit_data
472 | logger.debug(f"Number of samples in the tuning set: {len(dataset)}")
473 | if len(dataset) < batch_size:
474 | logger.warning(
475 | f"Tuning-set size is small ({len(dataset)})."
476 | "Using all samples for training + validation. "
477 | f"Adding samples from training set to reach minimal sample size {batch_size}"
478 | )
479 |
480 | if patience is not None:
481 | if early_stop:
482 | logger.info(f"Early stopping on validation loss with patience {patience} ")
483 | else:
484 | logger.info(f"Early stopping on training loss with patience {patience}")
485 |
486 | loader_bs = min(int(2 ** np.floor(np.log2(len(dataset) - 1))), batch_size)
487 | train_loader = DataLoader(
488 | dataset,
489 | batch_size=loader_bs,
490 | shuffle=True,
491 | drop_last=True,
492 | num_workers=num_workers,
493 | )
494 | val_loader = DataLoader(
495 | dataset,
496 | batch_size=min(batch_size, len(dataset)),
497 | num_workers=num_workers,
498 | )
499 | extra_loader = None
500 | if loader_bs < self.train_data_size:
501 | extra_loader = DataLoader(
502 | fitting_set,
503 | batch_size=batch_size - loader_bs,
504 | shuffle=True,
505 | num_workers=num_workers,
506 | )
507 |
508 | optimizer = torch.optim.AdamW(self.model.parameters(), learning_rate_init)
509 | for it in range(1, max_iter + 1):
510 | self.model.train()
511 |
512 | train_loss = []
513 | header = f"TRAIN: ({it}/{max_iter})"
514 | metric_logger = MetricLogger(delimiter=" ")
515 | for batch in metric_logger.log_every(train_loader, 1, header, logger):
516 | # forward
517 | if extra_loader is not None:
518 | b1 = next(iter(extra_loader))
519 | batch = [torch.cat([b1, b2]) for b1, b2 in zip(batch, b1)]
520 | batch = (b.to(dev) for b in batch)
521 | X, curve, y = batch
522 | loss = self.model.train_step(X, curve, y)
523 | train_loss.append(loss.item())
524 |
525 | # update
526 | optimizer.zero_grad()
527 | loss.backward()
528 | optimizer.step()
529 |
530 | # log
531 | metric_logger.update(loss=loss.item())
532 | metric_logger.update(lengthscale=self.model.lengthscale)
533 | metric_logger.update(noise=self.model.noise) # type: ignore
534 | logger.info(f"[{it}/{max_iter}]Averaged stats: {str(metric_logger)}")
535 | val_metric = np.mean(train_loss)
536 |
537 | if val_loader is not None:
538 | self.model.eval()
539 |
540 | l1 = DataLoader(
541 | dataset,
542 | batch_size=len(dataset),
543 | shuffle=True,
544 | )
545 | batch = next(iter(l1))
546 | if len(dataset) < self.train_data_size:
547 | l2 = DataLoader(
548 | fitting_set,
549 | batch_size=self.train_data_size - loader_bs,
550 | shuffle=True,
551 | )
552 | b2 = next(iter(l2))
553 | batch = [torch.cat([p, q]) for p, q in zip(batch, b2)]
554 | batch = (b.to(dev) for b in batch)
555 | a, b, c = batch
556 | self.model.set_train_data(a, b, c)
557 |
558 | val_loss = []
559 | with torch.no_grad():
560 | for batch in val_loader:
561 | batch = (b.to(dev) for b in batch)
562 | X, curve, y = batch
563 | pred = self.model.predict(X, curve)
564 | loss = torch.nn.functional.l1_loss(pred.mean, y)
565 | val_loss.append(loss.item())
566 | val_metric = np.mean(val_loss)
567 |
568 | if patience is not None:
569 | if val_metric + tol < best_val_metric:
570 | patience_counter = 0
571 | best_val_metric = val_metric
572 | best_iter = it
573 | torch.save(self.model.state_dict(), temp_save_file_path)
574 | else:
575 | patience_counter += 1
576 | logger.info(
577 | f"[{it}/{max_iter}] "
578 | f"VAL: {round(val_metric, 4)} "
579 | f"BEST: {round(best_val_metric, 4)} ({best_iter})",
580 | )
581 | if patience_counter >= patience:
582 | logger.log(
583 | 15,
584 | "Stopping training..."
585 | f"No improvement in the last {patience} iterations. ",
586 | )
587 | break
588 |
589 | if patience:
590 | logger.info(f"Loading best model from iteration {best_iter}")
591 | self.model.load_state_dict(torch.load(temp_save_file_path))
592 |
593 | # remove cache dir
594 | if os.path.exists(cache_dir):
595 | shutil.rmtree(cache_dir)
596 |
597 | # after training the model, reset GPs training data
598 | self.model.eval()
599 | if len(dataset) < self.train_data_size:
600 | l1 = DataLoader(dataset, batch_size=len(dataset), shuffle=True)
601 | l2 = DataLoader(
602 | fitting_set,
603 | batch_size=self.train_data_size - len(dataset),
604 | shuffle=True,
605 | )
606 | b1 = next(iter(l1))
607 | b2 = next(iter(l2))
608 | batch = [torch.cat([a, b]) for a, b in zip(b1, b2)]
609 | batch = (b.to(dev) for b in batch)
610 | else:
611 | loader = DataLoader(dataset, batch_size=self.train_data_size, shuffle=True)
612 | batch = next(iter(loader))
613 | batch = (b.to(dev) for b in batch)
614 | a, b, c = batch
615 | self.model.set_train_data(a, b, c)
616 |
617 | def _predict(self, **kwargs) -> Tuple[np.ndarray, np.ndarray]:
618 | """
619 | Predict the performance of a configuration `X` on a new dataset `curve`.
620 |
621 | Args:
622 | X: the configuration to predict.
623 | curve: the dataset to predict on.
624 | fill_missing: whether to fill missing values in the dataset.
625 |
626 | Returns:
627 | The mean and standard deviation of the predicted performance.
628 | """
629 | if not self.is_fit or self.model is None:
630 | raise AssertionError("Model is not fitted yet")
631 |
632 | X: pd.DataFrame = kwargs.pop("X", None)
633 | curve: np.ndarray = kwargs.pop("curve", None)
634 | fill_missing: bool = kwargs.pop("fill_missing", False)
635 |
636 | if X is None:
637 | raise ValueError("X (pipeline configuration) is a required argument for this predictor")
638 | if curve is None:
639 | raise ValueError("curve is a required argument for this predictor")
640 |
641 | self._validate_predict_data(X, curve)
642 | x = self._preprocess_predict_data(X, fill_missing)
643 | curve = np.nan_to_num(curve)
644 |
645 | device = self.device
646 | self.model.eval()
647 | self.model.to(device)
648 | x = torch.tensor(x, dtype=torch.float32, device=device)
649 | c = torch.tensor(curve, dtype=torch.float32, device=device)
650 | mean = np.array([])
651 | std = np.array([])
652 | with torch.no_grad():
653 | bs = 4096 # TODO: make this a parameter
654 | for i in range(0, x.shape[0], bs):
655 | pred = self.model.predict(x[i : i + bs], c[i : i + bs])
656 | mean = np.append(mean, pred.mean.cpu().numpy())
657 | std = np.append(std, pred.stddev.cpu().numpy())
658 | return mean, std
659 |
660 | def save(self, path: str | None = None, verbose=True) -> str:
661 | # Save on CPU to ensure the model can be loaded on a box without GPU
662 | if self.model is not None:
663 | self.model = self.model.to(torch.device("cpu"))
664 | path = super().save(path, verbose)
665 | # Put the model back to the device after the save
666 | if self.model is not None:
667 | self.model.to(self.device)
668 | return path
669 |
670 | @classmethod
671 | def load(cls, path: str, reset_paths=True, verbose=True) -> "PerfPredictor":
672 | """
673 | Loads the model from disk to memory.
674 |
675 | The loaded model will be on the same device it was trained on (e.g., cuda/mps).
676 | If the device is unavailable (e.g., trained on GPU but deployed on CPU),
677 | the model will be loaded on `cpu`.
678 |
679 | Args:
680 | path (str): Path to the saved model, excluding the file name.
681 | This should typically be a directory path ending with a '/' character
682 | (or appropriate path separator based on OS). The model file is usually
683 | located at `os.path.join(path, cls.model_file_name)`.
684 | reset_paths (bool, optional): Whether to reset the `self.path` value of the loaded
685 | model to be equal to `path`. Defaults to True. Setting this to False may cause
686 | inconsistencies between the actual valid path and `self.path`, potentially leading
687 | to strange behavior and exceptions if the model needs to load other files later.
688 | verbose (bool, optional): Whether to log the location of the loaded file. Defaults to True.
689 |
690 | Returns:
691 | cls: The loaded model object.
692 | """
693 | model: PerfPredictor = super().load(path=path, reset_paths=reset_paths, verbose=verbose)
694 |
695 | verbosity = model.verbosity
696 | set_logger_verbosity(verbosity, logger)
697 | return model
698 |
699 |
700 | class GPRegressionModel(gpytorch.models.ExactGP):
701 | def __init__(
702 | self,
703 | train_x: torch.Tensor | None,
704 | train_y: torch.Tensor | None,
705 | likelihood: gpytorch.likelihoods.GaussianLikelihood,
706 | ):
707 | super().__init__(train_x, train_y, likelihood)
708 | self.mean_module = gpytorch.means.ConstantMean()
709 | self.covar_module = gpytorch.kernels.ScaleKernel(gpytorch.kernels.RBFKernel())
710 |
711 | def forward(self, x: torch.Tensor):
712 | mean = self.mean_module(x)
713 | covar = self.covar_module(x)
714 | return gpytorch.distributions.MultivariateNormal(mean, covar) # type: ignore
715 |
716 |
717 | class SurrogateModel(torch.nn.Module):
718 | def __init__(
719 | self,
720 | in_dim: int | list[int],
721 | in_curve_dim: int,
722 | out_dim: int = 32,
723 | enc_hidden_dim: int = 128,
724 | enc_out_dim: int = 32,
725 | enc_nlayers: int = 3,
726 | out_curve_dim: int = 16,
727 | ):
728 | super().__init__()
729 | self.encoder = FeatureEncoder(
730 | in_dim,
731 | in_curve_dim,
732 | out_dim,
733 | enc_hidden_dim,
734 | enc_out_dim,
735 | enc_nlayers,
736 | out_curve_dim,
737 | )
738 | self.likelihood = gpytorch.likelihoods.GaussianLikelihood()
739 | self.gp_model = GPRegressionModel(
740 | train_x=None,
741 | train_y=None,
742 | likelihood=self.likelihood,
743 | )
744 | self.mll = gpytorch.mlls.ExactMarginalLogLikelihood(
745 | self.likelihood,
746 | self.gp_model,
747 | )
748 |
749 | def forward(self, pipeline, curve):
750 | encoding = self.encoder(pipeline, curve)
751 | output = self.gp_model(encoding)
752 | return self.likelihood(output)
753 |
754 | @torch.no_grad()
755 | def predict(self, pipeline, curve):
756 | return self(pipeline, curve)
757 |
758 | def train_step(self, pipeline, curve, y) -> torch.Tensor:
759 | encoding = self.encoder(pipeline, curve)
760 | self.gp_model.set_train_data(encoding, y, False)
761 | output = self.gp_model(encoding)
762 | loss = -self.mll(output, y) # type: ignore
763 | return loss
764 |
765 | @torch.no_grad()
766 | def set_train_data(self, pipeline, curve, y) -> None:
767 | self.eval()
768 | encoding = self.encoder(pipeline, curve)
769 | self.gp_model.set_train_data(encoding, y, False)
770 |
771 | @property
772 | def lengthscale(self) -> float:
773 | return self.gp_model.covar_module.base_kernel.lengthscale.item()
774 |
775 | @property
776 | def noise(self) -> float:
777 | return self.gp_model.likelihood.noise.item() # type: ignore
778 |
--------------------------------------------------------------------------------
/src/qtt/predictors/predictor.py:
--------------------------------------------------------------------------------
1 | import logging
2 | import os
3 | import pickle
4 | from typing import Tuple
5 |
6 | import numpy as np
7 | import pandas as pd
8 | from numpy.typing import ArrayLike
9 |
10 | from qtt.utils import setup_outputdir
11 |
12 | logger = logging.getLogger(__name__)
13 |
14 |
15 | class Predictor:
16 | """Base class. Implements all low-level functionality.
17 |
18 | Args:
19 | path (str, optional): Directory location to store all outputs. Defaults to None.
20 | If None, a new unique time-stamped directory is chosen.
21 | name (str, optional): Name of the subdirectory inside `path` where the model will be saved.
22 | The final model directory will be `os.path.join(path, name)`.
23 | If None, defaults to the model's class name: `self.__class__.__name__`.
24 | """
25 |
26 | model_file_name = "model.pkl"
27 |
28 | def __init__(
29 | self,
30 | name: str | None = None,
31 | path: str | None = None,
32 | ):
33 | if name is None:
34 | self.name = self.__class__.__name__
35 | logger.info(
36 | f"No name was specified for model, defaulting to class name: {self.name}",
37 | )
38 | else:
39 | self.name = name
40 |
41 | if path is None:
42 | self.path: str = setup_outputdir(path=self.name.lower())
43 | logger.info(
44 | f"No path was specified for predictor, defaulting to: {self.path}",
45 | )
46 | else:
47 | self.path = setup_outputdir(path)
48 |
49 | self.model = None
50 |
51 | def reset_path(self, path: str | None = None):
52 | """
53 | Reset the path of the model.
54 |
55 | Args:
56 | path (str, optional):
57 | Directory location to store all outputs. If None, a new unique time-stamped directory is chosen.
58 | """
59 | if path is None:
60 | path = setup_outputdir(path=self.name.lower())
61 | self.path = path
62 |
63 | @property
64 | def is_fit(self) -> bool:
65 | """Returns True if the model has been fit."""
66 | return self.model is not None
67 |
68 | def fit(self, X: pd.DataFrame, y: ArrayLike, **kwargs):
69 | """
70 | Fit model to predict values in y based on X.
71 |
72 | Models should not override the `fit` method, but instead override the `_fit` method which has the same arguments.
73 |
74 | Args:
75 | X (pd.DataFrame):
76 | The training data features.
77 | y (ArrayLike):
78 | The training data ground truth labels.
79 | **kwargs :
80 | Any additional fit arguments a model supports.
81 | """
82 | out = self._fit(X=X, y=y, **kwargs)
83 | if out is None:
84 | out = self
85 | return out
86 |
87 | def _fit(self, X: pd.DataFrame, y: ArrayLike, **kwargs):
88 | """
89 | Fit model to predict values in y based on X.
90 |
91 | Models should override this method with their custom model fit logic.
92 | X should not be assumed to be in a state ready for fitting to the inner model, and models may require special preprocessing in this method.
93 | It is very important that `X = self.preprocess(X)` is called within `_fit`, or else `predict` and `predict_proba` may not work as intended.
94 | It is also important that `_preprocess` is overwritten to properly clean the data.
95 | Examples of logic that should be handled by a model include missing value handling, rescaling of features (if neural network), etc.
96 | If implementing a new model, it is recommended to refer to existing model implementations and experiment using toy datasets.
97 |
98 | Refer to `fit` method for documentation.
99 | """
100 | raise NotImplementedError
101 |
102 | def _preprocess(self, **kwargs):
103 | """
104 | Data transformation logic should be added here.
105 |
106 | Input data should not be trusted to be in a clean and ideal form, while the output should be in an ideal form for training/inference.
107 | Examples of logic that should be added here include missing value handling, rescaling of features (if neural network), etc.
108 | If implementing a new model, it is recommended to refer to existing model implementations and experiment using toy datasets.
109 |
110 | In bagged ensembles, preprocessing code that lives in `_preprocess` will be executed on each child model once per inference call.
111 | If preprocessing code could produce different output depending on the child model that processes the input data, then it must live here.
112 | When in doubt, put preprocessing code here instead of in `_preprocess_nonadaptive`.
113 | """
114 | raise NotImplementedError
115 |
116 | def preprocess(self, **kwargs):
117 | """
118 | Preprocesses the input data into internal form ready for fitting or inference.
119 | """
120 | return self._preprocess(**kwargs)
121 |
122 | @classmethod
123 | def load(cls, path: str, reset_paths: bool = True, verbose: bool = True):
124 | """
125 | Loads the model from disk to memory.
126 |
127 | Args:
128 | path (str):
129 | Path to the saved model, minus the file name.
130 | This should generally be a directory path ending with a '/' character (or appropriate path separator value depending on OS).
131 | The model file is typically located in os.path.join(path, cls.model_file_name).
132 | reset_paths (bool):
133 | Whether to reset the self.path value of the loaded model to be equal to path.
134 | It is highly recommended to keep this value as True unless accessing the original self.path value is important.
135 | If False, the actual valid path and self.path may differ, leading to strange behaviour and potential exceptions if the model needs to load any other files at a later time.
136 | verbose (bool):
137 | Whether to log the location of the loaded file.
138 |
139 | Returns:
140 | model (Predictor): Loaded model object.
141 | """
142 | file_path = os.path.join(path, cls.model_file_name)
143 | with open(file_path, "rb") as f:
144 | model = pickle.load(f)
145 | if reset_paths:
146 | model.path = path
147 | if verbose:
148 | logger.info(f"Model loaded from: {file_path}")
149 | return model
150 |
151 | def save(self, path: str | None = None, verbose: bool = True) -> str:
152 | """
153 | Saves the model to disk.
154 |
155 | Args:
156 | path (str): Path to the saved model, minus the file name.
157 | This should generally be a directory path ending with a '/' character (or appropriate path separator value depending on OS).
158 | If None, self.path is used.
159 | The final model file is typically saved to os.path.join(path, self.model_file_name).
160 | verbose (bool): Whether to log the location of the saved file.
161 |
162 | Returns:
163 | path: Path to the saved model, minus the file name. Use this value to load the model from disk via cls.load(path), cls being the class of the model object, such as ```model = PerfPredictor.load(path)```
164 | """
165 | if path is None:
166 | path = self.path
167 | path = setup_outputdir(path, create_dir=True, warn_if_exist=True)
168 | file_path = os.path.join(path, self.model_file_name)
169 | with open(file_path, "wb") as f:
170 | pickle.dump(self, f, protocol=pickle.HIGHEST_PROTOCOL)
171 | if verbose:
172 | logger.info(f"Model saved to: {file_path}")
173 | return path
174 |
175 | def predict(self, **kwargs) -> np.ndarray | Tuple[np.ndarray, ...]:
176 | """
177 | Predicts the output for the given input data.
178 |
179 | Models should not override the `predict` method, but instead override the `_predict` method
180 | which has the same arguments.
181 | """
182 | return self._predict(**kwargs)
183 |
184 | def _predict(self, **kwargs):
185 | """
186 | Predicts the output for the given input data.
187 |
188 | New predictors should override this method with their custom prediction logic.
189 | """
190 | raise NotImplementedError
191 |
--------------------------------------------------------------------------------
/src/qtt/predictors/utils.py:
--------------------------------------------------------------------------------
1 | import datetime
2 | from functools import partial
3 | import time
4 | from collections import defaultdict, deque
5 |
6 | import numpy as np
7 | import torch
8 |
9 |
10 | class SmoothedValue(object):
11 | """Track a series of values and provide access to smoothed values over a
12 | window or the global series average.
13 | """
14 |
15 | def __init__(self, window_size=20, fmt=None):
16 | if fmt is None:
17 | fmt = "{avg:.3f}"
18 | self.deque = deque(maxlen=window_size)
19 | self.total = 0.0
20 | self.count = 0
21 | self.fmt = fmt
22 |
23 | def update(self, value, n=1):
24 | self.deque.append(value)
25 | self.count += n
26 | self.total += value * n
27 |
28 | @property
29 | def median(self):
30 | d = np.array(list(self.deque))
31 | return np.median(d).item()
32 |
33 | @property
34 | def avg(self):
35 | d = np.array(list(self.deque), dtype=float)
36 | return d.mean().item()
37 |
38 | @property
39 | def global_avg(self):
40 | return self.total / self.count
41 |
42 | @property
43 | def max(self):
44 | return max(self.deque)
45 |
46 | @property
47 | def value(self):
48 | return self.deque[-1]
49 |
50 | def __str__(self):
51 | return self.fmt.format(
52 | median=self.median,
53 | avg=self.avg,
54 | global_avg=self.global_avg,
55 | max=self.max,
56 | value=self.value,
57 | )
58 |
59 |
60 | class MetricLogger(object):
61 | def __init__(self, delimiter="\t"):
62 | self.meters = defaultdict(SmoothedValue)
63 | self.delimiter = delimiter
64 |
65 | def update(self, **kwargs):
66 | for k, v in kwargs.items():
67 | if isinstance(v, torch.Tensor):
68 | v = v.item()
69 | assert isinstance(v, (float, int))
70 | self.meters[k].update(v)
71 |
72 | def __getattr__(self, attr):
73 | if attr in self.meters:
74 | return self.meters[attr]
75 | if attr in self.__dict__:
76 | return self.__dict__[attr]
77 | raise AttributeError("'{}' object has no attribute '{}'".format(type(self).__name__, attr))
78 |
79 | def __str__(self):
80 | loss_str = []
81 | for name, meter in self.meters.items():
82 | loss_str.append("{}: {}".format(name, str(meter)))
83 | return self.delimiter.join(loss_str)
84 |
85 | def add_meter(self, name, meter):
86 | self.meters[name] = meter
87 |
88 | def log_every(self, iterable, print_freq, header=None, logger=None):
89 | i = 1
90 | if not header:
91 | header = ""
92 | _print = print if logger is None else partial(logger.log, 15)
93 | start_time = time.time()
94 | end = time.time()
95 | iter_time = SmoothedValue(fmt="{avg:.3f}")
96 | space_fmt = ":" + str(len(str(len(iterable)))) + "d"
97 | log_msg = self.delimiter.join(
98 | [
99 | header,
100 | "[{0" + space_fmt + "}/{1}]",
101 | "eta: {eta}",
102 | "{meters}",
103 | "time: {time}",
104 | ]
105 | )
106 | for obj in iterable:
107 | yield obj
108 | iter_time.update(time.time() - end)
109 | if i % print_freq == 0 or i == len(iterable):
110 | eta_seconds = iter_time.global_avg * (len(iterable) - i)
111 | eta_string = str(datetime.timedelta(seconds=int(eta_seconds)))
112 | _print(
113 | log_msg.format(
114 | i,
115 | len(iterable),
116 | eta=eta_string,
117 | meters=str(self),
118 | time=str(iter_time),
119 | )
120 | )
121 | i += 1
122 | end = time.time()
123 | total_time = time.time() - start_time
124 | total_time_str = str(datetime.timedelta(seconds=int(total_time)))
125 | _print(f"Total time: {total_time_str} ({total_time / len(iterable):.3f} s / it)")
126 |
127 |
128 | def get_torch_device():
129 | return torch.device("cuda" if torch.cuda.is_available() else "cpu")
130 |
--------------------------------------------------------------------------------
/src/qtt/tuners/__init__.py:
--------------------------------------------------------------------------------
1 | from .image.classification.tuner import QuickImageCLSTuner
2 | from .quick import QuickTuner
3 |
4 | __all__ = ["QuickTuner", "QuickImageCLSTuner"]
5 |
--------------------------------------------------------------------------------
/src/qtt/tuners/image/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/automl/quicktunetool/0ce6fe17f0b3bbfcea24ec6e1882b77c5c82ceab/src/qtt/tuners/image/__init__.py
--------------------------------------------------------------------------------
/src/qtt/tuners/image/classification/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/automl/quicktunetool/0ce6fe17f0b3bbfcea24ec6e1882b77c5c82ceab/src/qtt/tuners/image/classification/__init__.py
--------------------------------------------------------------------------------
/src/qtt/tuners/image/classification/tuner.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 |
3 | from ....finetune.image.classification import extract_image_dataset_metafeat, fn
4 | from ....optimizers.quick import QuickOptimizer
5 | from ....utils.pretrained import get_pretrained_optimizer
6 | from ...quick import QuickTuner
7 |
8 |
9 | class QuickImageCLSTuner(QuickTuner):
10 | """QuickTuner for image classification.
11 |
12 | Args:
13 | data_path (str): Path to the dataset.
14 | path (str, optional): Path to save the optimizer. Defaults to None.
15 | verbosity (int, optional): Verbosity level. Defaults to 2.
16 | """
17 |
18 | def __init__(
19 | self,
20 | data_path: str,
21 | n: int = 512,
22 | path: str | None = None,
23 | verbosity: int = 2,
24 | ):
25 | quick_opt: QuickOptimizer = get_pretrained_optimizer("mtlbm/full")
26 |
27 | trial_info, metafeat = extract_image_dataset_metafeat(data_path)
28 | quick_opt.setup(n, metafeat=metafeat)
29 |
30 | self.trial_info = trial_info
31 |
32 | super().__init__(quick_opt, fn, path=path, verbosity=verbosity)
33 |
34 | def run(
35 | self,
36 | fevals: int | None = None,
37 | time_budget: float | None = None,
38 | trial_info: dict | None = None,
39 | ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
40 | """
41 |
42 | Args:
43 | fevals (int, optional): Number of function evaluations to run. Defaults to None.
44 | time_budget (float, optional): Time budget in seconds. Defaults to None.
45 | trial_info (dict, optional): Additional information to pass to the objective function. Defaults to None.
46 |
47 | Returns:
48 | - np.ndarray: Trajectory of the incumbent scores.
49 | - np.ndarray: Runtime of the incumbent evaluations.
50 | - np.ndarray: History of all evaluations.
51 | """
52 | if trial_info is not None:
53 | self.trial_info = trial_info
54 | return super().run(fevals=fevals, time_budget=time_budget, trial_info=self.trial_info)
55 |
--------------------------------------------------------------------------------
/src/qtt/tuners/quick.py:
--------------------------------------------------------------------------------
1 | import json
2 | import pickle
3 | import logging
4 | import os
5 | import time
6 | from typing import Callable, Any
7 |
8 | import numpy as np
9 | import pandas as pd
10 |
11 | from ..optimizers import Optimizer
12 | from ..utils import (
13 | add_log_to_file,
14 | config_to_serializible_dict,
15 | set_logger_verbosity,
16 | setup_outputdir,
17 | )
18 |
19 | logger = logging.getLogger(__name__)
20 |
21 |
22 | class QuickTuner:
23 | """
24 | QuickTuner is a simple tuner that can be used to optimize a given function
25 | using a given optimizer.
26 |
27 | Args:
28 | optimizer (Optimizer): An instance of an Optimizer class.
29 | f (Callable): A function that takes a configuration and returns a score.
30 | path (str, optional): Directory location to store all outputs. Defaults to None.
31 | If None, a new unique time-stamped directory is chosen.
32 | save_freq (str, optional): Frequency of saving the state of the tuner. Defaults to "step".
33 | - "step": save after each evaluation.
34 | - "incumbent": save only when the incumbent changes.
35 | - None: do not save.
36 | verbosity (int, optional): Verbosity level of the logger. Defaults to 2.
37 | resume (bool, optional): Whether to resume the tuner from a previous state. Defaults to False.
38 | """
39 |
40 | log_to_file: bool = True
41 | log_file_name: str = "quicktuner.log"
42 | log_file_path: str = "auto"
43 | path_suffix: str = "tuner"
44 |
45 | def __init__(
46 | self,
47 | optimizer: Optimizer,
48 | f: Callable,
49 | path: str | None = None,
50 | save_freq: str | None = "step",
51 | verbosity: int = 2,
52 | resume: bool = False,
53 | **kwargs,
54 | ):
55 | if resume and path is None:
56 | raise ValueError("Cannot resume without specifying a path.")
57 | self._validate_kwargs(kwargs)
58 |
59 | self.verbosity = verbosity
60 | set_logger_verbosity(verbosity, logger)
61 |
62 | self.output_dir = setup_outputdir(path, path_suffix=self.path_suffix)
63 | self._setup_log_to_file(self.log_to_file, self.log_file_path)
64 |
65 | if save_freq not in ["step", "incumbent"] and save_freq is not None:
66 | raise ValueError("Invalid value for 'save_freq'.")
67 | self.save_freq = save_freq
68 |
69 | self.optimizer = optimizer
70 | self.optimizer.reset_path(self.output_dir)
71 | self.f = f
72 |
73 | # trackers
74 | self.inc_score: float = 0.0
75 | self.inc_fidelity: int = -1
76 | self.inc_config: dict = {}
77 | self.inc_cost: float = 0.0
78 | self.inc_info: object = None
79 | self.inc_id: int = -1
80 | self.traj: list[object] = []
81 | self.history: list[object] = []
82 | self.runtime: list[object] = []
83 |
84 | self._remaining_fevals = None
85 | self._remaining_time = None
86 |
87 | if resume:
88 | self.load(os.path.join(self.output_dir, "qt.json"))
89 |
90 | def _setup_log_to_file(self, log_to_file: bool, log_file_path: str) -> None:
91 | """
92 | Set up the logging to file.
93 |
94 | Args:
95 | log_to_file (bool): Whether to log to file.
96 | log_file_path (str | Path): Path to the log file.
97 | """
98 | if not log_to_file:
99 | return
100 | if log_file_path == "auto":
101 | log_file_path = os.path.join(self.output_dir, "logs", self.log_file_name)
102 | log_file_path = os.path.abspath(os.path.normpath(log_file_path))
103 | os.makedirs(os.path.dirname(log_file_path), exist_ok=True)
104 | add_log_to_file(log_file_path, logger)
105 |
106 | def _is_budget_exhausted(
107 | self, fevals: int | None = None, time_budget: float | None = None
108 | ) -> bool:
109 | """
110 | Checks if the run budget has been exhausted. Returns whether the run should be terminated.
111 | Negative values translate to no budget. If no limit is desired, use None.
112 |
113 | Args:
114 | fevals (int, optional): Number of function evaluations. Defaults to None.
115 | time_budget (float, optional): Time budget in seconds. Defaults to None.
116 |
117 | Returns:
118 | bool: Whether the run should be terminated or continued.
119 | """
120 | if fevals is not None:
121 | evals_left = fevals - len(self.traj)
122 | if evals_left <= 0:
123 | return True
124 | logger.info(f"Evaluations left: {evals_left}")
125 | if time_budget is not None:
126 | time_left = time_budget - (time.time() - self.start)
127 | if time_left <= 0:
128 | return True
129 | logger.info(f"Time left: {time_left:.2f}s")
130 | return False
131 |
132 | def _save_incumbent(self, save: bool = True):
133 | """
134 | Saves the current incumbent configuration and its associated information to a JSON file.
135 |
136 | Args:
137 | save (bool, optional): Whether to save the incumbent. Defaults to True.
138 | """
139 | if not self.inc_config or not save:
140 | return
141 | try:
142 | out: dict[str, Any] = {}
143 | out["config"] = self.inc_config
144 | out["score"] = self.inc_score
145 | out["cost"] = self.inc_cost
146 | out["info"] = self.inc_info
147 | with open(os.path.join(self.output_dir, "incumbent.json"), "w") as f:
148 | json.dump(out, f, indent=2)
149 | except Exception as e:
150 | logger.error(f"Failed to save incumbent: {e}")
151 |
152 | def _save_history(self, save: bool = True):
153 | """
154 | Saves the history of evaluations to a CSV file.
155 |
156 | Args:
157 | save (bool, optional): Whether to save the history. Defaults to True.
158 | """
159 | if not self.history or not save:
160 | return
161 | try:
162 | history_path = os.path.join(self.output_dir, "history.csv")
163 | history_df = pd.DataFrame(self.history)
164 | history_df.to_csv(history_path)
165 | except Exception as e:
166 | logger.warning(f"History not saved: {e!r}")
167 | finally:
168 | logger.info("Saved history.")
169 |
170 | def _log_job_submission(self, trial_info: dict):
171 | """
172 | Logs a message when a job is submitted to the compute backend.
173 |
174 | Args:
175 | trial_info (dict): A dictionary containing the trial information.
176 | """
177 | fidelity = trial_info["fidelity"]
178 | config_id = trial_info["config-id"]
179 | logger.info(
180 | f"INCUMBENT: {self.inc_id} "
181 | f"SCORE: {self.inc_score} "
182 | f"FIDELITY: {self.inc_fidelity}",
183 | )
184 | logger.info(f"Evaluating configuration {config_id} with fidelity {fidelity}")
185 |
186 | def _get_state(self):
187 | """
188 | Returns the state of the QuickTuner as a dictionary.
189 |
190 | Returns:
191 | A dictionary containing the state of the QuickTuner.
192 | """
193 | state = self.__dict__.copy()
194 | state.pop("optimizer")
195 | state.pop("f")
196 | return state
197 |
198 | def _save_state(self, save: bool = True):
199 | """
200 | Saves the state of the QuickTuner to disk.
201 |
202 | The state of the Tuner is saved as a JSON file to disk, named 'qt.json' in the output
203 | directory. If the optimization is interrupted, the state can be loaded from disk to
204 | resume the optimization.
205 |
206 | Args:
207 | save (bool, optional): Whether to save the state. Defaults to True.
208 | """
209 | if not save:
210 | return
211 | # Get state
212 | state = self._get_state()
213 | # Write state to disk
214 | try:
215 | state_path = os.path.join(self.output_dir, "qt.json")
216 | with open(state_path, "wb") as f:
217 | pickle.dump(state, f, protocol=pickle.HIGHEST_PROTOCOL)
218 | except Exception as e:
219 | logger.warning(f"State not saved: {e!r}")
220 | finally:
221 | logger.info("State saved to disk.")
222 | try:
223 | opt_path = os.path.join(self.output_dir, "optimizer")
224 | self.optimizer.save(opt_path)
225 | except Exception as e:
226 | logger.warning(f"Optimizer state not saved: {e!r}")
227 |
228 | def save(self, incumbent: bool = True, history: bool = True, state: bool = True):
229 | logger.info("Saving current state to disk...")
230 | self._save_incumbent(incumbent)
231 | self._save_history(history)
232 | self._save_state(state)
233 |
234 | def load(self, path: str):
235 | logger.info(f"Loading state from {path}")
236 | with open(path, "rb") as f:
237 | state = pickle.load(f)
238 | self.__dict__.update(state)
239 | self.optimizer = Optimizer.load(os.path.join(self.output_dir, "optimizer"))
240 |
241 | def run(
242 | self,
243 | fevals: int | None = None,
244 | time_budget: float | None = None,
245 | trial_info: dict | None = None,
246 | ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
247 | """Run the tuner.
248 |
249 | Args:
250 | fevals (int, optional): Number of function evaluations to run. Defaults to None.
251 | time_budget (float, optional): Time budget in seconds. Defaults to None.
252 | trial_info (dict, optional): Additional information to pass to the objective function. Defaults to None.
253 |
254 | Returns:
255 | Tuple[np.ndarray, np.ndarray, np.ndarray]:
256 | - trajectory (np.ndarray): Trajectory of the incumbent scores.
257 | - runtime (np.ndarray): Runtime of the incumbent evaluations.
258 | - history (np.ndarray): History of all evaluations.
259 | """
260 | logger.info("Starting QuickTuner Run...")
261 | logger.info(f"QuickTuneTool will save results to {self.output_dir}")
262 |
263 | self.start = time.time()
264 | while True:
265 | self.optimizer.ante()
266 |
267 | # ask for a new configuration
268 | trial = self.optimizer.ask()
269 | if trial is None:
270 | break
271 | _trial_info = self._add_trial_info(trial_info)
272 |
273 | self._log_job_submission(trial)
274 | result = self.f(trial, trial_info=_trial_info)
275 |
276 | self._log_report(result)
277 | self.optimizer.tell(result)
278 |
279 | self.optimizer.post()
280 | if self._is_budget_exhausted(fevals, time_budget):
281 | logger.info("Budget exhausted. Stopping run...")
282 | break
283 |
284 | self._log_end()
285 | self.save()
286 |
287 | return (
288 | np.array(self.traj),
289 | np.array(self.runtime),
290 | np.array(self.history, dtype=object),
291 | )
292 |
293 | def _update_trackers(self, traj, runtime, history):
294 | self.traj.append(traj)
295 | self.runtime.append(runtime)
296 | self.history.append(history)
297 |
298 | def _log_report(self, reports):
299 | if isinstance(reports, dict):
300 | reports = [reports]
301 |
302 | inc_changed = False
303 | for report in reports:
304 | config_id = report["config-id"]
305 | score = report["score"]
306 | cost = report["cost"]
307 | fidelity = report["fidelity"]
308 | config = config_to_serializible_dict(report["config"])
309 |
310 | separator = "-" * 60
311 | logger.info(separator)
312 | logger.info(f"CONFIG ID : {config_id}")
313 | logger.info(f"FIDELITY : {fidelity}")
314 | logger.info(f"SCORE : {score:.3f}")
315 | logger.info(f"TIME : {cost:.3f}")
316 | logger.info(separator)
317 |
318 | if self.inc_score < score:
319 | self.inc_score = score
320 | self.inc_cost = cost
321 | self.inc_fidelity = fidelity
322 | self.inc_id = config_id
323 | self.inc_config = config
324 | self.inc_info = report.get("info")
325 | inc_changed = True
326 |
327 | report["config"] = config
328 | self._update_trackers(
329 | self.inc_score,
330 | time.time() - self.start,
331 | report,
332 | )
333 |
334 | if self.save_freq == "step" or (self.save_freq == "incumbent" and inc_changed):
335 | self.save()
336 |
337 | def _log_end(self):
338 | separator = "=" * 60
339 | logger.info(separator)
340 | logger.info("RUN COMPLETE - SUMMARY REPORT")
341 | logger.info(separator)
342 | logger.info(f"Best Score : {self.inc_score:.3f}")
343 | logger.info(f"Best Cost : {self.inc_cost:.3f} seconds")
344 | logger.info(f"Best Config ID : {self.inc_id}")
345 | logger.info(f"Best Configuration: {self.inc_config}")
346 | logger.info(separator)
347 |
348 | def _validate_kwargs(self, kwargs: dict) -> None:
349 | for key, value in kwargs.items():
350 | if hasattr(self, key):
351 | setattr(self, key, value)
352 | else:
353 | logger.warning(f"Unknown argument: {key}")
354 |
355 | def _add_trial_info(self, task_info: dict | None) -> dict:
356 | out = {} if task_info is None else task_info.copy()
357 | out["output-dir"] = self.output_dir
358 | out["remaining-fevals"] = self._remaining_fevals
359 | out["remaining-time"] = self._remaining_time
360 | return out
361 |
362 | def get_incumbent(self):
363 | return (
364 | self.inc_id,
365 | self.inc_config,
366 | self.inc_score,
367 | self.inc_fidelity,
368 | self.inc_cost,
369 | self.inc_info,
370 | )
371 |
--------------------------------------------------------------------------------
/src/qtt/utils/__init__.py:
--------------------------------------------------------------------------------
1 | from .config import config_to_serializible_dict, encode_config_space
2 | from .log_utils import add_log_to_file, set_logger_verbosity, setup_default_logging
3 | from .setup import fix_random_seeds, setup_outputdir
4 |
5 | __all__ = [
6 | "config_to_serializible_dict",
7 | "encode_config_space",
8 | "add_log_to_file",
9 | "set_logger_verbosity",
10 | "setup_default_logging",
11 | "fix_random_seeds",
12 | "setup_outputdir",
13 | ]
14 |
--------------------------------------------------------------------------------
/src/qtt/utils/config.py:
--------------------------------------------------------------------------------
1 | from collections import defaultdict
2 |
3 | import ConfigSpace as CS
4 |
5 | CATEGORICAL = CS.CategoricalHyperparameter
6 | NUMERICAL = (
7 | CS.UniformIntegerHyperparameter,
8 | CS.BetaIntegerHyperparameter,
9 | CS.UniformFloatHyperparameter,
10 | CS.BetaFloatHyperparameter,
11 | CS.NormalFloatHyperparameter,
12 | CS.NormalIntegerHyperparameter,
13 | )
14 | ORDINAL = CS.OrdinalHyperparameter
15 |
16 |
17 | def to_one_hot(hp_name, sequence):
18 | return [f"{hp_name}={c}" for c in sequence]
19 |
20 |
21 | def encode_config_space(cs: CS.ConfigurationSpace) -> tuple[list[str], list[list[str]]]:
22 | """Encode a ConfigSpace.ConfigurationSpace object into a list of one-hot
23 | encoded hyperparameters.
24 |
25 | Args:
26 | cs (CS.ConfigurationSpace): A ConfigSpace.ConfigurationSpace object.
27 | """
28 | type_dict = defaultdict(list)
29 | for hp in list(cs.values()):
30 | if isinstance(hp, CS.Constant):
31 | continue
32 | elif isinstance(hp, CATEGORICAL):
33 | one_hot = to_one_hot(hp.name, hp.choices)
34 | elif isinstance(hp, ORDINAL) and isinstance(hp.default_value, str):
35 | one_hot = to_one_hot(hp.name, hp.sequence)
36 | else:
37 | one_hot = [hp.name]
38 |
39 | _type = "none"
40 | if hp.meta is not None:
41 | _type = hp.meta.get("type", "none")
42 | type_dict[_type].extend(one_hot)
43 |
44 | encoding = []
45 | splits = []
46 | for key in sorted(type_dict.keys()):
47 | g = type_dict[key]
48 | g.sort()
49 | encoding.extend(g)
50 | splits.append(g)
51 | return encoding, splits
52 |
53 |
54 | def config_to_vector(configs: list[CS.Configuration], one_hot):
55 | """Convert a list of ConfigSpace.Configuration to a list of
56 | one-hot encoded dictionaries.
57 |
58 | Args:
59 | configs (list[CS.Configuration]): A list of ConfigSpace.Configuration objects.
60 | one_hot (list[str]): One-hot encodings of the hyperparameters.
61 | """
62 | encoded_configs = []
63 | for config in configs:
64 | config_dict = dict(config)
65 | enc_config = {}
66 | for hp in one_hot:
67 | # categorical hyperparameters
68 | if len(hp.split("=")) > 1:
69 | key, choice = hp.split("=")
70 | val = 1 if config_dict.get(key) == choice else 0
71 | else:
72 | val = config_dict.get(hp, 0) # NUM
73 | if isinstance(val, bool): # BOOL
74 | val = int(val)
75 | enc_config[hp] = val
76 | encoded_configs.append(enc_config)
77 | return encoded_configs
78 |
79 |
80 | def config_to_serializible_dict(config: CS.Configuration) -> dict:
81 | """Convert a ConfigSpace.Configuration to a serializable dictionary.
82 | Cast all values to basic types (int, float, str, bool) to ensure
83 | that the dictionary is JSON serializable.
84 |
85 | Args:
86 | config (CS.Configuration): A ConfigSpace.Configuration object.
87 | """
88 | serializable_dict = dict(config)
89 | for k, v in serializable_dict.items():
90 | if hasattr(v, "item"):
91 | serializable_dict[k] = v.item()
92 | return serializable_dict
93 |
--------------------------------------------------------------------------------
/src/qtt/utils/log_utils.py:
--------------------------------------------------------------------------------
1 | import logging
2 | from typing import Optional
3 |
4 |
5 | def verbosity2loglevel(verbosity: int):
6 | """Translates verbosity to logging level. Suppresses warnings if verbosity = 0.
7 |
8 | Args:
9 | verbosity (int): Verbosity level
10 |
11 | Returns:
12 | int: Logging level
13 | """
14 | if verbosity <= 0: # only errors
15 | # print("Caution: all warnings suppressed")
16 | log_level = 40
17 | elif verbosity == 1: # only warnings and critical print statements
18 | log_level = 25
19 | elif verbosity == 2: # key print statements which should be shown by default
20 | log_level = 20
21 | elif verbosity == 3: # more-detailed printing
22 | log_level = 15
23 | else:
24 | log_level = 10 # print everything (ie. debug mode)
25 | return log_level
26 |
27 |
28 | def set_logger_verbosity(verbosity: int, logger=None):
29 | """
30 | Set the verbosity of the logger. If no logger is provided, the root logger is used.
31 |
32 | Args:
33 | verbosity (int): Verbosity level
34 | logger (logging.Logger, optional): Logger to set verbosity for. Defaults to None.
35 | """
36 | if logger is None:
37 | logger = logging.root
38 | if verbosity < 0:
39 | verbosity = 0
40 | elif verbosity > 4:
41 | verbosity = 4
42 | logger.setLevel(verbosity2loglevel(verbosity))
43 |
44 |
45 | def add_log_to_file(
46 | file_path: str,
47 | logger: Optional[logging.Logger] = None,
48 | fmt: Optional[str] = None,
49 | datefmt: Optional[str] = None,
50 | ):
51 | """
52 | Add a FileHandler to the logger to log to a file in addition to the console.
53 | If no format is provided, the format is set to: asctime - name: levelname message
54 |
55 | Args:
56 | file_path (str): Path to the log file
57 | logger (logging.Logger, optional): Logger to add the file handler to. Defaults to None.
58 | fmt (str, optional): Format string. Defaults to None.
59 | datefmt (str, optional): Date format string. Defaults to None.
60 | """
61 | if logger is None:
62 | logger = logging.root
63 | fh = logging.FileHandler(file_path)
64 | if fmt is None:
65 | fmt = "%(asctime)s - %(name)16s: [%(levelname)s] %(message)s"
66 | if datefmt is None:
67 | datefmt = "%y.%m.%d %H:%M:%S"
68 | formatter = logging.Formatter(fmt, datefmt)
69 | fh.setFormatter(formatter)
70 | logger.addHandler(fh)
71 |
72 |
73 | def setup_default_logging(
74 | default_level=logging.INFO,
75 | fmt: Optional[str] = None,
76 | datefmt: Optional[str] = None,
77 | ):
78 | """Set up the default logging level and formatter for the root logger.
79 | If no format is provided, only the message is printed.
80 |
81 | Args:
82 | default_level (int, optional): Default logging level. Defaults to logging.INFO.
83 | fmt (str, optional): Format string. Defaults to None.
84 | datefmt (str, optional): Date format string. Defaults to None.
85 | """
86 | if fmt is None:
87 | fmt = "%(message)s"
88 | # fmt = "%(asctime)s - %(name)s: [%(levelname)s] %(message)s"
89 | if datefmt is None:
90 | datefmt = "%y.%m.%d %H:%M:%S"
91 | logging.basicConfig(format=fmt, datefmt=datefmt, level=default_level)
92 |
--------------------------------------------------------------------------------
/src/qtt/utils/pretrained.py:
--------------------------------------------------------------------------------
1 | import tarfile
2 | from pathlib import Path
3 |
4 | import requests
5 |
6 | from qtt.optimizers import QuickOptimizer
7 |
8 | VERSION_MAP = {
9 | "mtlbm/micro": dict(
10 | url="https://ml.informatik.uni-freiburg.de/research-artifacts/quicktunetool/mtlbm/micro/archive.tar.gz",
11 | name="archive",
12 | final_name="model",
13 | extension="pkl",
14 | ),
15 | "mtlbm/mini": dict(
16 | url="https://ml.informatik.uni-freiburg.de/research-artifacts/quicktunetool/mtlbm/mini/archive.tar.gz",
17 | name="archive",
18 | final_name="model",
19 | extension="pkl",
20 | ),
21 | "mtlbm/extended": dict(
22 | url="https://ml.informatik.uni-freiburg.de/research-artifacts/quicktunetool/mtlbm/extended/archive.tar.gz",
23 | name="archive",
24 | final_name="model",
25 | extension="pkl",
26 | ),
27 | "mtlbm/full": dict(
28 | url="https://ml.informatik.uni-freiburg.de/research-artifacts/quicktunetool/mtlbm/full/archive.tar.gz",
29 | name="archive",
30 | final_name="model",
31 | extension="pkl",
32 | ),
33 | }
34 |
35 |
36 | # Helper functions to generate the file names
37 | def FILENAME(version: str) -> str:
38 | return f"{VERSION_MAP[version].get('name')}.tar.gz"
39 |
40 |
41 | def FILE_URL(version: str) -> str:
42 | return f"{VERSION_MAP[version].get('url')}"
43 |
44 |
45 | def WEIGHTS_FILE_NAME(version: str) -> str:
46 | return f"{VERSION_MAP[version].get('name')}.{VERSION_MAP[version].get('extension')}"
47 |
48 |
49 | def WEIGHTS_FINAL_NAME(version: str) -> str:
50 | return f"{VERSION_MAP[version].get('final_name')}.{VERSION_MAP[version].get('extension')}"
51 |
52 |
53 | def get_pretrained_optimizer(
54 | version: str, download: bool = True, path: str = "~/.cache/qtt/pretrained"
55 | ) -> QuickOptimizer:
56 | """Get a pretrained optimizer.
57 |
58 | Args:
59 | version (str):
60 | Name of the pretrained optimizer version.
61 |
62 | Returns:
63 | Optimizer: A pretrained optimizer.
64 | """
65 | assert version in VERSION_MAP
66 |
67 | base_dir = Path(path).expanduser() / version
68 | model_path = base_dir / FILENAME(version)
69 |
70 | if download and not model_path.exists():
71 | base_dir.mkdir(parents=True, exist_ok=True)
72 | download_and_decompress(FILE_URL(version), model_path)
73 | elif not model_path.exists():
74 | raise ValueError(f"Pretrained optimizer '{version}' not found at {model_path}.")
75 |
76 | return QuickOptimizer.load(str(base_dir))
77 |
78 |
79 | def download_and_decompress(url: str, path: Path) -> None:
80 | """Helper function to download a file from a URL and decompress it and store by given name.
81 |
82 | Args:
83 | url (str): URL of the file to download
84 | path (Path): Path along with filename to save the downloaded file
85 |
86 | Returns:
87 | bool: Flag to indicate if the download and decompression was successful
88 | """
89 | # Check if the file already exists
90 | if path.exists():
91 | return
92 |
93 | # Send a HTTP request to the URL of the file
94 | response = requests.get(url, allow_redirects=True)
95 |
96 | # Check if the request is successful
97 | if response.status_code != 200:
98 | raise ValueError(
99 | f"Failed to download the surrogate from {url}. "
100 | f"Received HTTP status code: {response.status_code}."
101 | )
102 |
103 | # Save the .tar.gz file
104 | with open(path, "wb") as f:
105 | f.write(response.content)
106 |
107 | # Decompress the .tar.gz file
108 | with tarfile.open(path, "r:gz") as tar:
109 | tar.extractall(path.parent.absolute())
110 |
--------------------------------------------------------------------------------
/src/qtt/utils/setup.py:
--------------------------------------------------------------------------------
1 | import logging
2 | import os
3 | import random
4 | from datetime import datetime
5 |
6 | import numpy as np
7 | import torch
8 |
9 | logger = logging.getLogger(__name__)
10 |
11 |
12 | def setup_outputdir(path, create_dir=True, warn_if_exist=True, path_suffix=None) -> str:
13 | if path is not None:
14 | assert isinstance(
15 | path, str
16 | ), f"Only str is supported for path, got {path} of type {type(path)}."
17 |
18 | if path_suffix is None:
19 | path_suffix = ""
20 | if path is not None:
21 | path = os.path.join(path, path_suffix)
22 | if path is None:
23 | timestamp = datetime.now().strftime("%y%m%d-%H%M%S")
24 | path = os.path.join("qtt", timestamp, path_suffix)
25 | if create_dir:
26 | try:
27 | os.makedirs(path, exist_ok=False)
28 | logger.info(f"Created directory: {path}")
29 | except FileExistsError:
30 | logger.warning(f"'{path}' already exists! This may overwrite old data.")
31 | elif warn_if_exist:
32 | if os.path.isdir(path):
33 | logger.warning(f"'{path}' already exists! This may overwrite old data.")
34 | path = os.path.expanduser(path) # replace ~ with absolute path if it exists
35 | return path
36 |
37 |
38 | def fix_random_seeds(seed=None):
39 | """
40 | Fix random seeds.
41 | """
42 | if seed is None:
43 | seed = 42
44 | random.seed(seed)
45 | np.random.seed(seed)
46 | torch.manual_seed(seed)
47 | torch.cuda.manual_seed_all(seed)
48 |
--------------------------------------------------------------------------------