├── .dir-locals.el ├── .gitignore ├── README.org ├── addtree ├── __init__.py ├── acq.py ├── kernel_utils.py ├── parameter.py └── storage.py ├── assets ├── code-objective-function.png ├── explanation-construction.png ├── resnet50-cummax-median-95ci-1.png ├── resnet50-cummax-median-95ci.png ├── synthetic-function-1.png └── synthetic-function.png ├── examples ├── __init__.py ├── addtree_jenatton_small.py ├── addtree_model_compression_multiple.py ├── addtree_resnet20_nas.py ├── addtree_utils.py ├── common_utils.py ├── compression_common.py ├── makefile ├── models │ ├── __init__.py │ ├── resnet.py │ ├── resnet_cifar10.py │ └── vgg.py ├── nas_common.py ├── random_model_compression_multiple.py ├── random_resnet20_nas.py ├── resnet20_nas_optimized.py ├── smac_model_compression_multiple.py ├── smac_resnet20_nas.py ├── smac_utils.py ├── space_utils │ ├── __init__.py │ ├── addtree_cs_jenatton_small.py │ ├── addtree_cs_resnet20_nas.py │ ├── addtree_cs_resnet50.py │ ├── addtree_cs_resnet56.py │ ├── addtree_cs_vgg16.py │ ├── smac_bo_facade.py │ ├── smac_cs_nas_resnet20.py │ ├── smac_cs_resnet50_40d.py │ ├── smac_cs_resnet50_multiple.py │ ├── smac_cs_resnet56.py │ ├── smac_cs_single.py │ ├── smac_cs_vgg16_40d.py │ ├── smac_cs_vgg16_multiple.py │ ├── smac_hpo_facade.py │ ├── tpe_space_nas_resnet20.py │ ├── tpe_space_resnet50_multiple.py │ ├── tpe_space_resnet56.py │ └── tpe_space_vgg16_multiple.py ├── tpe_model_compression_multiple.py ├── tpe_resnet20_nas.py └── tpe_utils.py ├── pyproject.toml └── tests ├── __init__.py ├── test_addtree.py ├── test_parameter.py └── test_storage.py /.dir-locals.el: -------------------------------------------------------------------------------- 1 | ;;; Directory Local Variables 2 | ;;; For more information see (info "(emacs) Directory Variables") 3 | 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by https://www.gitignore.io/api/python 2 | # Edit at https://www.gitignore.io/?templates=python 3 | 4 | *.pdf 5 | *.csv 6 | *.zip 7 | 8 | ### Python ### 9 | # Byte-compiled / optimized / DLL files 10 | __pycache__/ 11 | *.py[cod] 12 | *$py.class 13 | 14 | # C extensions 15 | *.so 16 | 17 | # Distribution / packaging 18 | .Python 19 | build/ 20 | develop-eggs/ 21 | dist/ 22 | downloads/ 23 | eggs/ 24 | .eggs/ 25 | lib/ 26 | lib64/ 27 | parts/ 28 | sdist/ 29 | var/ 30 | wheels/ 31 | pip-wheel-metadata/ 32 | share/python-wheels/ 33 | *.egg-info/ 34 | .installed.cfg 35 | *.egg 36 | MANIFEST 37 | 38 | # PyInstaller 39 | # Usually these files are written by a python script from a template 40 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 41 | *.manifest 42 | *.spec 43 | 44 | # Installer logs 45 | pip-log.txt 46 | pip-delete-this-directory.txt 47 | 48 | # Unit test / coverage reports 49 | htmlcov/ 50 | .tox/ 51 | .nox/ 52 | .coverage 53 | .coverage.* 54 | .cache 55 | nosetests.xml 56 | coverage.xml 57 | *.cover 58 | .hypothesis/ 59 | .pytest_cache/ 60 | 61 | # Translations 62 | *.mo 63 | *.pot 64 | 65 | # Scrapy stuff: 66 | .scrapy 67 | 68 | # Sphinx documentation 69 | docs/_build/ 70 | 71 | # PyBuilder 72 | target/ 73 | 74 | # pyenv 75 | .python-version 76 | 77 | # pipenv 78 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 79 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 80 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 81 | # install all needed dependencies. 82 | #Pipfile.lock 83 | 84 | # celery beat schedule file 85 | celerybeat-schedule 86 | 87 | # SageMath parsed files 88 | *.sage.py 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | .spyproject 93 | 94 | # Rope project settings 95 | .ropeproject 96 | 97 | # Mr Developer 98 | .mr.developer.cfg 99 | .project 100 | .pydevproject 101 | 102 | # mkdocs documentation 103 | /site 104 | 105 | # mypy 106 | .mypy_cache/ 107 | .dmypy.json 108 | dmypy.json 109 | 110 | # Pyre type checker 111 | .pyre/ 112 | 113 | # End of https://www.gitignore.io/api/python 114 | /examples/script/ 115 | /examples/space_utils/haha.py 116 | haha 117 | /addtree/common.py 118 | /addtree/haha.py 119 | /examples/ssssingle/ 120 | /poetry.lock 121 | /tests/a_huge_profile 122 | /examples/exp_results/ 123 | *.json 124 | -------------------------------------------------------------------------------- /README.org: -------------------------------------------------------------------------------- 1 | #+TITLE: AddTree Covariance Function 2 | 3 | 4 | * Dependencies 5 | 6 | This project depends on a customized george, setup can be done via conda: 7 | 8 | #+begin_src sh 9 | conda create -n add python=3.11 10 | conda activate add 11 | conda install -y -c conda-forge libstdcxx-ng 12 | conda install -y -c anaconda pyyaml jinja2 numpy scipy pybind11 13 | git clone https://github.com/maxc01/addtree 14 | cd addtree 15 | pip install . 16 | #+end_src 17 | 18 | 19 | 20 | * How to code a tree given an objective function 21 | 22 | The first step is to represent an objective function at hand using a tree. For 23 | example, the synthetic function presented in Jenatton2017 can be encoded using 24 | a tree shown in the left part of the following figure and its corresponding 25 | code snippet is shown in the right part. We clarify this code snippet in the 26 | following. 27 | 28 | [[./assets/explanation-construction.png]] 29 | 30 | 1. =root= node doesn't contain a continuous parameter, thus the dimension of 31 | this parameter is set to be zero 32 | 2. =x2= node contains a one-dimensional continuous parameter, and this 33 | parameter is shared by two sub-functions associated with the first and 34 | second leaves (counting from left to right) 35 | 3. =x5= node contains a one-dimensional continuous parameter 36 | 4. build tree dependencies by adding =x7= as a child of =x3= 37 | 5. finish tree construction (precompute some quantities) 38 | 39 | * How to code a tree-structured function 40 | 41 | The general rule of coding a tree-structured function is using a hierarchical 42 | dictionary as the parameter of this function. For example, the code snippet 43 | corresponding to the above tree-structured function is shown below. 44 | 45 | #+begin_src python 46 | def obj_func(params): 47 | SHIFT = 0.5 48 | 49 | if params["L1"]["cat_value"] == "left" and params["L2"]["cat_value"] == "left": 50 | value = ( 51 | params["L1"]["cont_value"] + (params["L2"]["cont_value"] - SHIFT) ** 2 + 0.1 52 | ) 53 | elif params["L1"]["cat_value"] == "left" and params["L2"]["cat_value"] == "right": 54 | value = ( 55 | params["L1"]["cont_value"] + (params["L2"]["cont_value"] - SHIFT) ** 2 + 0.2 56 | ) 57 | elif params["L1"]["cat_value"] == "right" and params["L2"]["cat_value"] == "left": 58 | value = ( 59 | params["L1"]["cont_value"] + (params["L2"]["cont_value"] - SHIFT) ** 2 + 0.3 60 | ) 61 | elif params["L1"]["cat_value"] == "right" and params["L2"]["cat_value"] == "right": 62 | value = ( 63 | params["L1"]["cont_value"] + (params["L2"]["cont_value"] - SHIFT) ** 2 + 0.4 64 | ) 65 | else: 66 | raise ValueError("parameter names are not correct") 67 | 68 | info = dict() 69 | info["value"] = value 70 | info["value_sigma"] = 1e-9 71 | return info 72 | #+end_src 73 | 74 | For a more complex tree-structured function, see the [[https://github.com/maxc01/addtree/blob/6d2100597124bfe1b5fa5e433f827e7c80a427e2/examples/nas_common.py#L66][objective function of a simple NAS problem]]. 75 | 76 | * Examples 77 | 78 | ** synthetic function (from Jenatton2017) 79 | #+begin_src sh 80 | cd examples 81 | python addtree_jenatton_small.py 82 | #+end_src 83 | log files will be located in 84 | =exp_results/addtree/jenatton-small/a-unique-dir=. The following figure shows 85 | comparison of different algorithms on optimizaing the synthetic function. 86 | [[./assets/synthetic-function.png]] 87 | 88 | ** model compression 89 | 90 | #+begin_src sh 91 | cd examples 92 | python {{ algo }}_model_compression_multiple.py {{ model_name }} OUTPUT_PATH --pretrained PRETRAINED_PATH --prune_epochs 1 93 | #+end_src 94 | =model_name= can be "vgg16", "resnet50" or "resnet56". 95 | =algo= can be "addtree", "random" , "tpe" or "smac". 96 | 97 | For example, to compress resnet50 using =addtree=, 98 | #+begin_src sh 99 | python addtree_model_compression_multiple.py resnet50 OUTPUT_PATH --pretrained PRETRAINED_PATH --prune_epochs 1 100 | #+end_src 101 | 102 | The following picture shows comparison of different algorithms on compressing resnet50. 103 | [[./assets/resnet50-cummax-median-95ci.png]] 104 | 105 | 106 | * citation 107 | #+begin_src bibtex 108 | @inproceedings{ma2020, 109 | title = {Additive {{Tree}}-{{Structured Covariance Function}} for {{Conditional Parameter Spaces}} in {{Bayesian Optimization}}}, 110 | booktitle = {International {{Conference}} on {{Artificial Intelligence}} and {{Statistics}}}, 111 | author = {Ma, Xingchen and Blaschko, Matthew}, 112 | year = {2020}, 113 | month = jun, 114 | pages = {1015--1025}, 115 | publisher = {{PMLR}}, 116 | issn = {2640-3498}, 117 | language = {en} 118 | } 119 | 120 | @article{ma2020b, 121 | title = {Additive Tree-Structured Conditional Parameter Spaces in Bayesian Optimization: {{A}} Novel Covariance Function and a Fast Implementation}, 122 | author = {Ma, X. and Blaschko, M. B.}, 123 | year = {2020}, 124 | publisher = {{IEEE Computer Society}}, 125 | address = {{Los Alamitos, CA, USA}}, 126 | issn = {1939-3539}, 127 | doi = {10.1109/TPAMI.2020.3026019}, 128 | journal = {IEEE Transactions on Pattern Analysis \& Machine Intelligence}, 129 | keywords = {additives,bayes methods,data models,linear programming,mathematical model,neural networks,optimization} 130 | } 131 | #+end_src 132 | -------------------------------------------------------------------------------- /addtree/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = '0.1.0' 2 | -------------------------------------------------------------------------------- /addtree/acq.py: -------------------------------------------------------------------------------- 1 | import heapq 2 | import numpy as np 3 | from scipy.optimize import minimize 4 | 5 | 6 | def UCB(gp, X_new, Y_train, kappa=5.0): 7 | assert kappa > 0 8 | pred, pred_var = gp.predict(Y_train, X_new, return_var=True) 9 | pred_sigma = np.sqrt(pred_var) 10 | return pred + kappa * pred_sigma 11 | 12 | 13 | def LCB(gp, X_new, Y_train, kappa=5.0): 14 | assert kappa > 0 15 | pred, pred_var = gp.predict(Y_train, X_new, return_var=True) 16 | pred_sigma = np.sqrt(pred_var) 17 | return pred - kappa * pred_sigma 18 | 19 | 20 | def optimize_acq( 21 | acq_func, 22 | root, 23 | gp, 24 | Y_train, 25 | total_dim, 26 | grid_size=100, 27 | nb_seed=2, 28 | kappa=-1, 29 | return_full=False, 30 | quasi=False, 31 | pb_L=0.01, 32 | pb_R=0.99, 33 | ): 34 | 35 | info = [] 36 | for path_id in root.all_pathids(): 37 | path = root.select_path(path_id) 38 | eff_axes = path.axes() 39 | grid = path.rand(grid_size, total_dim, quasi=quasi) 40 | grid_acq = acq_func(gp, grid, Y_train, kappa=kappa) 41 | seeds_idx = np.argsort(grid_acq)[:nb_seed] 42 | bounds = [(pb_L, pb_R)] * len(eff_axes) 43 | ixgrid = np.ix_(seeds_idx, eff_axes) 44 | seeds = grid[ixgrid] 45 | 46 | def obj_func_acq(x): 47 | vec = path.set_data(x).path2vec(root.obs_dim) 48 | return acq_func(gp, vec[None], Y_train).item() 49 | 50 | # start minimization using these seeds 51 | _x_best = None 52 | _y_best = np.inf 53 | for seed in seeds: 54 | result = minimize(obj_func_acq, x0=seed, method="L-BFGS-B", bounds=bounds,) 55 | if result.fun < _y_best: 56 | _y_best = result.fun 57 | _x_best = result.x 58 | _x_best = np.clip(_x_best, *zip(*bounds)) 59 | heapq.heappush(info, (_y_best, path_id, _x_best, path)) 60 | 61 | if return_full: 62 | return info 63 | else: 64 | return info[0] 65 | -------------------------------------------------------------------------------- /addtree/kernel_utils.py: -------------------------------------------------------------------------------- 1 | import george.kernels as K 2 | from functools import reduce 3 | 4 | 5 | def build_addtree(root): 6 | obs_dim = root.obs_dim 7 | name2ker = {} 8 | ks = [] 9 | bfs_nodes = root.bfs_nodes() 10 | for node in bfs_nodes: 11 | kd = K.DeltaKernel(ndim=obs_dim, axes=node.bfs_index) 12 | axes = node.param_axes 13 | if len(axes) == 0: 14 | k = kd 15 | else: 16 | kc = K.ExpSquaredKernel( 17 | 0.5, ndim=obs_dim, axes=axes, metric_bounds=[(-7, 4.0)] 18 | ) 19 | name2ker[node.parameter.name] = kc 20 | k = kd * kc 21 | ks.append(k) 22 | 23 | kernel = reduce(lambda x, y: x + y, ks) 24 | 25 | return kernel 26 | 27 | 28 | def get_const_kernel(c, ndim): 29 | c = K.ConstantKernel(-0.69, ndim=ndim, bounds=[(-7.0, 4.0)]) 30 | return c 31 | -------------------------------------------------------------------------------- /addtree/parameter.py: -------------------------------------------------------------------------------- 1 | from typing import Dict, Sequence, Optional 2 | 3 | import numpy as np 4 | 5 | NAME2NODE = {} 6 | 7 | 8 | def clear_state(): 9 | global NAME2NODE 10 | NAME2NODE = {} 11 | 12 | 13 | def get_state(): 14 | return NAME2NODE 15 | 16 | 17 | class NodePath: 18 | """ This class represents a list of ParameterNode 19 | """ 20 | 21 | def __init__(self, path: Sequence["ParameterNode"]): 22 | self.path = path 23 | 24 | def path2dict(self) -> Dict[str, np.ndarray]: 25 | """transform a path to a dict, which is then passed to objective function. 26 | """ 27 | param_dict = {node.name: node.parameter.data for node in self.path} 28 | return param_dict 29 | 30 | def path2vec(self, obs_dim: int) -> np.ndarray: 31 | """transform a path to a vector, which will be added to a Storage. 32 | """ 33 | param_dict = self.path2dict() 34 | return self.dict2vec(param_dict, obs_dim) 35 | 36 | def dict2vec(self, param_dict: Dict[str, np.ndarray], obs_dim: int) -> np.ndarray: 37 | """transform a dict to a vector, which will be added to a Storage. 38 | # IMPORTANT: The basic patter is: first set flag, second set data 39 | """ 40 | bfs_repr = np.array([-1] * obs_dim, dtype="f") 41 | for k, v in param_dict.items(): 42 | node = NAME2NODE[k] 43 | bfs_repr[node.bfs_index] = node.local_id 44 | bfs_repr[node.param_axes] = v 45 | 46 | return bfs_repr 47 | 48 | def axes(self): 49 | _axes = [] 50 | for node in self.path: 51 | _axes.extend(node.param_axes) 52 | return _axes 53 | 54 | def set_data(self, x: np.ndarray) -> "NodePath": 55 | i = 0 56 | for node in self.path: 57 | node.parameter.data = x[i : (i + node.parameter.dim)] 58 | i += node.parameter.dim 59 | 60 | return self 61 | 62 | def rand(self, n, obs_dim, quasi=False): 63 | if quasi: 64 | import ghalton 65 | a = -np.ones((n, obs_dim)) 66 | for node in self.path: 67 | sequencer = ghalton.Halton(node.parameter.dim) 68 | a[:, node.bfs_index] = node.local_id 69 | a[:, node.param_axes] = sequencer.get(n) 70 | else: 71 | a = -np.random.rand(n, obs_dim) 72 | for node in self.path: 73 | a[:, node.bfs_index] = node.local_id 74 | a[:, node.param_axes] *= -1 75 | return a 76 | 77 | def __getitem__(self, idx): 78 | return self.path[idx] 79 | 80 | def __repr__(self): 81 | return "->".join([node.name for node in self.path]) 82 | 83 | 84 | class Parameter: 85 | def __init__(self, name: str, dim: int, data: Optional[np.ndarray] = None): 86 | self.name = name 87 | self.dim = dim 88 | self.data = data 89 | if data is None: 90 | self.set_rand() 91 | 92 | def set_rand(self): 93 | self.data = np.random.rand(self.dim) 94 | 95 | def __repr__(self): 96 | data_repr = ", ".join(["{:.2f}".format(i) for i in self.data]) 97 | return "{}: [{}]".format(self.name, data_repr) 98 | 99 | 100 | class ParameterNode: 101 | def __init__(self, parameter: Parameter): 102 | self.parameter = parameter 103 | self.name = parameter.name 104 | # TODO: use ID instead of name to index node 105 | # because name can duplicate, but ID cannot 106 | if self.name in NAME2NODE: 107 | raise ValueError(f"Node {self.name} already in the tree.") 108 | NAME2NODE[self.name] = self 109 | self.parant = None 110 | # a valid local_id start from 0, a float local_id indicates a missing 111 | # value, at this case, value takes no effect 112 | self.local_id = 0 if parameter.dim > 0 else -1 113 | self.depth = 0 114 | self.children = [] 115 | 116 | def add_child(self, child: "ParameterNode"): 117 | child.depth = self.depth + 1 118 | child.parent = self 119 | child.local_id = len(self.children) 120 | self.children.append(child) 121 | 122 | def __repr__(self): 123 | return self.parameter.name 124 | 125 | def __eq__(self, other: "ParameterNode"): 126 | return self.name == other.name 127 | 128 | def finish_add_child(self): 129 | 130 | bfs_template = [] 131 | 132 | def set_index(node): 133 | n_start = len(bfs_template) 134 | n_end = n_start + node.parameter.dim + 1 135 | node.bfs_index = n_start 136 | node.param_axes = list(range(n_start + 1, n_end)) 137 | bfs_template.extend([-1] * (1 + node.parameter.dim)) 138 | 139 | self._bfs(func=set_index) 140 | self.obs_dim = len(bfs_template) 141 | 142 | self.bfs_template = np.asarray(bfs_template, dtype="f") 143 | 144 | def _bfs(self, func=None, *func_args, **func_kwargs): 145 | bfs_queue = [] 146 | bfs_queue.append(self) 147 | while len(bfs_queue): 148 | node = bfs_queue.pop(0) 149 | # visit node and do something 150 | if callable(func): 151 | func(node, *func_args, **func_kwargs) 152 | for child in node.children: 153 | bfs_queue.append(child) 154 | 155 | def _dfs(self, func=None, *func_args, **func_kwargs): 156 | """pre-order DFS 157 | """ 158 | 159 | def preorder(node): 160 | # visit node and do something 161 | if callable(func): 162 | func(node, *func_args, **func_kwargs) 163 | for child in self.children: 164 | preorder(child) 165 | 166 | preorder(self) 167 | 168 | def is_leaf(self): 169 | return len(self.children) == 0 170 | 171 | def bfs_nodes(self): 172 | 173 | all_nodes = [] 174 | 175 | def collect_node(node): 176 | all_nodes.append(node) 177 | 178 | self._bfs(func=collect_node) 179 | 180 | return all_nodes 181 | 182 | def random_path(self, rand_data=False) -> NodePath: 183 | """generate a random path from current node. 184 | """ 185 | 186 | path = [] 187 | 188 | def _sample(node): 189 | if rand_data: 190 | node.parameter.set_rand() 191 | path.append(node) 192 | if node.is_leaf(): 193 | return 194 | else: 195 | child = np.random.choice(node.children) 196 | _sample(child) 197 | 198 | _sample(self) 199 | 200 | return NodePath(path) 201 | 202 | def select_path(self, path_id: str) -> NodePath: 203 | """select path from a string 204 | 205 | Note: path_id should not contain itself, e.g. '10' means select second 206 | child of current node, 207 | 208 | """ 209 | cur = self 210 | path = [cur] 211 | for local_id in path_id: 212 | local_id = int(local_id) 213 | path.append(cur.children[local_id]) 214 | cur = cur.children[local_id] 215 | 216 | return NodePath(path) 217 | 218 | def all_pathids(self) -> Sequence[str]: 219 | pathids = [] 220 | 221 | def preorder(node, cur_ids): 222 | cur_ids.append(str(node.local_id)) 223 | if node.is_leaf(): 224 | pathids.append("".join(cur_ids[1:])) 225 | for child in node.children: 226 | preorder(child, cur_ids) 227 | 228 | cur_ids.pop() 229 | 230 | cur_ids = [] 231 | preorder(self, cur_ids) 232 | 233 | return pathids 234 | 235 | def path_from_keys(self, keys) -> Sequence["ParameterNode"]: 236 | path = [self] 237 | for k in keys: 238 | path.append(NAME2NODE[k]) 239 | 240 | return path 241 | -------------------------------------------------------------------------------- /addtree/storage.py: -------------------------------------------------------------------------------- 1 | import warnings 2 | from collections import defaultdict 3 | 4 | import numpy as np 5 | from scipy.stats import beta 6 | from scipy.optimize import minimize 7 | 8 | import george 9 | 10 | 11 | class Storage: 12 | def __init__(self): 13 | self._X = [] 14 | self._Y = [] 15 | self._Yerr = [] 16 | self._path = [] 17 | 18 | def add(self, x, y, yerr=1e-5, path=None): 19 | self._X.append(x) 20 | self._Y.append(y) 21 | self._Yerr.append(yerr) 22 | self._path.append(path) 23 | 24 | @property 25 | def X(self): 26 | return np.asarray(self._X) 27 | 28 | @property 29 | def Y(self): 30 | return np.asarray(self._Y) 31 | 32 | @property 33 | def Yerr(self): 34 | return np.asarray(self._Yerr) 35 | 36 | def predict(self, gp, X_new): 37 | pred, pred_var = gp.predict(self.Y, X_new, return_var=True) 38 | pred_sd = np.sqrt(pred_var) 39 | return pred, pred_sd 40 | 41 | def optimize(self, kernel, n_restart=1, verbose=False, quasi=False): 42 | gp = george.GP(kernel, mean=self.Y.mean()) 43 | gp.compute(self.X, self.Yerr) 44 | 45 | if verbose: 46 | print("Initial ln-likelihood: {0:.2f}".format(gp.log_likelihood(self.Y))) 47 | 48 | def _neg_ln_like(p): 49 | gp.set_parameter_vector(p) 50 | return -gp.log_likelihood(self.Y) 51 | 52 | def _grad_neg_ln_like(p): 53 | gp.set_parameter_vector(p) 54 | return -gp.grad_log_likelihood(self.Y) 55 | 56 | bounds = kernel.get_parameter_bounds() 57 | x_best = None 58 | y_best = np.inf 59 | if quasi: 60 | import ghalton 61 | sequencer = ghalton.Halton(len(bounds)) 62 | seeds = sequencer.get(n_restart) 63 | else: 64 | seeds = np.random.uniform(*zip(*bounds), size=(n_restart, len(bounds))) 65 | for i in range(n_restart): 66 | result = minimize( 67 | _neg_ln_like, 68 | x0=seeds[i], 69 | jac=_grad_neg_ln_like, 70 | bounds=bounds, 71 | method="L-BFGS-B", 72 | ) 73 | if result.success is False: 74 | warnings.warn("Gaussian Process optimization is not successful.") 75 | if result.fun < y_best: 76 | y_best = result.fun 77 | x_best = result.x 78 | 79 | if x_best is None: 80 | raise RuntimeError("All optimizations are not successful.") 81 | 82 | gp.set_parameter_vector(x_best) 83 | if verbose: 84 | print("Best parameter of kernel: {}".format(x_best)) 85 | print("\nFinal ln-likelihood: {0:.2f}".format(gp.log_likelihood(self.Y))) 86 | 87 | return gp 88 | 89 | def best_xy(self): 90 | idx = np.argmin(self.Y) 91 | return (self.X[idx], self.Y[idx]) 92 | 93 | 94 | class GroupedStorage: 95 | def __init__(self): 96 | self.obs = defaultdict(list) 97 | 98 | def add(self, group, x, full_x, y, yerr): 99 | self.obs[group].append((x, full_x, y, yerr)) 100 | 101 | @property 102 | def groups(self): 103 | return self.obs.keys() 104 | 105 | def get_data(self, group_name): 106 | X, full_X, Y, Yerr = zip(*self.obs[group_name]) 107 | return np.asarray(X), np.asarray(full_X), np.asarray(Y), np.asarray(Yerr) 108 | -------------------------------------------------------------------------------- /assets/code-objective-function.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/maxc01/addtree/679f2f0c7ee3905385135e6693138e5733c6306b/assets/code-objective-function.png -------------------------------------------------------------------------------- /assets/explanation-construction.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/maxc01/addtree/679f2f0c7ee3905385135e6693138e5733c6306b/assets/explanation-construction.png -------------------------------------------------------------------------------- /assets/resnet50-cummax-median-95ci-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/maxc01/addtree/679f2f0c7ee3905385135e6693138e5733c6306b/assets/resnet50-cummax-median-95ci-1.png -------------------------------------------------------------------------------- /assets/resnet50-cummax-median-95ci.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/maxc01/addtree/679f2f0c7ee3905385135e6693138e5733c6306b/assets/resnet50-cummax-median-95ci.png -------------------------------------------------------------------------------- /assets/synthetic-function-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/maxc01/addtree/679f2f0c7ee3905385135e6693138e5733c6306b/assets/synthetic-function-1.png -------------------------------------------------------------------------------- /assets/synthetic-function.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/maxc01/addtree/679f2f0c7ee3905385135e6693138e5733c6306b/assets/synthetic-function.png -------------------------------------------------------------------------------- /examples/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/maxc01/addtree/679f2f0c7ee3905385135e6693138e5733c6306b/examples/__init__.py -------------------------------------------------------------------------------- /examples/addtree_jenatton_small.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import json 3 | import os 4 | import logging 5 | 6 | import numpy as np 7 | 8 | from addtree.kernel_utils import build_addtree 9 | from addtree.kernel_utils import get_const_kernel 10 | from addtree.storage import Storage 11 | from addtree.acq import optimize_acq, LCB 12 | 13 | from common_utils import setup_logger 14 | from common_utils import get_experiment_id 15 | 16 | from addtree_utils import build_tree_jenatton_small 17 | from addtree_utils import path2funcparam_jenatton_small 18 | 19 | 20 | def obj_func(params): 21 | SHIFT = 0.5 22 | 23 | if params["L1"]["cat_value"] == "left" and params["L2"]["cat_value"] == "left": 24 | value = ( 25 | params["L1"]["cont_value"] + (params["L2"]["cont_value"] - SHIFT) ** 2 + 0.1 26 | ) 27 | elif params["L1"]["cat_value"] == "left" and params["L2"]["cat_value"] == "right": 28 | value = ( 29 | params["L1"]["cont_value"] + (params["L2"]["cont_value"] - SHIFT) ** 2 + 0.2 30 | ) 31 | elif params["L1"]["cat_value"] == "right" and params["L2"]["cat_value"] == "left": 32 | value = ( 33 | params["L1"]["cont_value"] + (params["L2"]["cont_value"] - SHIFT) ** 2 + 0.3 34 | ) 35 | elif params["L1"]["cat_value"] == "right" and params["L2"]["cat_value"] == "right": 36 | value = ( 37 | params["L1"]["cont_value"] + (params["L2"]["cont_value"] - SHIFT) ** 2 + 0.4 38 | ) 39 | else: 40 | raise ValueError("parameter names are not correct") 41 | 42 | info = dict() 43 | info["value"] = value 44 | info["value_sigma"] = 1e-9 45 | return info 46 | 47 | 48 | def main(): 49 | 50 | try: 51 | logger = logging.getLogger("addtree-jenatton-small") 52 | logger.setLevel(logging.DEBUG) 53 | 54 | expid = get_experiment_id(6) 55 | output_dir = "./exp_results" 56 | output_dir = os.path.join(output_dir, "addtree", "jenatton-small", expid) 57 | os.makedirs(output_dir, exist_ok=False) 58 | log_path = os.path.join(output_dir, "addtree-jenatton-small.log") 59 | setup_logger(logger, log_path) 60 | 61 | logger.info(f"Experiment {expid} starts...") 62 | 63 | n_init = 5 64 | root = build_tree_jenatton_small() 65 | ss = Storage() 66 | ker = build_addtree(root) 67 | for i in range(n_init): 68 | logger.info("=" * 50) 69 | logger.info(f"Starting BO {i+1} iteration (Random Design)") 70 | path = root.random_path(rand_data=True) 71 | params = path2funcparam_jenatton_small(path[1:]) 72 | obj_info = obj_func(params) 73 | ss.add( 74 | path.path2vec(root.obs_dim), 75 | obj_info["value"], 76 | obj_info["value_sigma"], 77 | path, 78 | ) 79 | logger.info(f"Finishing BO {i+1} iteration") 80 | logger.info(params) 81 | logger.info(obj_info) 82 | 83 | all_info = {"iteration": i + 1, "params": params, "obj_info": obj_info} 84 | fn_path = os.path.join(output_dir, f"addtree_iter_{i+1}.json") 85 | with open(fn_path, "w") as f: 86 | json.dump(all_info, f) 87 | 88 | for i in range(n_init, 100): 89 | logger.info("=" * 50) 90 | logger.info(f"Starting BO {i+1} iteration (Optimization)") 91 | gp = ss.optimize(ker, n_restart=2, verbose=False) 92 | _, _, x_best, path = optimize_acq( 93 | LCB, 94 | root, 95 | gp, 96 | ss.Y, 97 | root.obs_dim, 98 | grid_size=100, 99 | nb_seed=2, 100 | kappa=1.0, 101 | pb_L=0.0, 102 | pb_R=1.0, 103 | ) 104 | path.set_data(x_best) 105 | params = path2funcparam_jenatton_small(path[1:]) 106 | obj_info = obj_func(params) 107 | ss.add( 108 | path.path2vec(root.obs_dim), 109 | obj_info["value"], 110 | obj_info["value_sigma"], 111 | path=path, 112 | ) 113 | logger.info(f"Finishing BO {i+1} iteration") 114 | logger.info(params) 115 | logger.info(obj_info) 116 | all_info = {"iteration": i + 1, "params": params, "obj_info": obj_info} 117 | fn_path = os.path.join(output_dir, f"addtree_iter_{i+1}.json") 118 | with open(fn_path, "w") as f: 119 | json.dump(all_info, f) 120 | 121 | except KeyboardInterrupt: 122 | print("Interrupted. You pressed Ctrl-C!!!") 123 | try: 124 | sys.exit(0) 125 | except SystemExit: 126 | os._exit(0) 127 | 128 | 129 | if __name__ == "__main__": 130 | main() 131 | -------------------------------------------------------------------------------- /examples/addtree_model_compression_multiple.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import json 3 | import os 4 | import logging 5 | 6 | import numpy as np 7 | 8 | from addtree.kernel_utils import build_addtree 9 | from addtree.kernel_utils import get_const_kernel 10 | from addtree.storage import Storage 11 | from addtree.acq import optimize_acq, LCB 12 | 13 | from common_utils import setup_logger 14 | from compression_common import setup_and_prune 15 | from compression_common import get_common_cmd_args 16 | from common_utils import get_experiment_id 17 | 18 | from addtree_utils import build_tree_vgg16 19 | from addtree_utils import path2funcparam_vgg16 20 | from addtree_utils import build_tree_resnet50 21 | from addtree_utils import path2funcparam_resnet50 22 | from addtree_utils import build_tree_resnet56 23 | from addtree_utils import path2funcparam_resnet56 24 | 25 | 26 | def main(): 27 | 28 | try: 29 | cmd_args, _ = get_common_cmd_args() 30 | 31 | output_basedir = cmd_args.output_basedir 32 | model_name = cmd_args.model_name 33 | if model_name == "vgg16": 34 | path2funcparam = path2funcparam_vgg16 35 | build_tree = build_tree_vgg16 36 | obs_sigma = 0.25 # TODO: this needs to be determined 37 | elif model_name == "resnet50": 38 | path2funcparam = path2funcparam_resnet50 39 | build_tree = build_tree_resnet50 40 | obs_sigma = 0.15 41 | elif model_name == "resnet56": 42 | path2funcparam = path2funcparam_resnet56 43 | build_tree = build_tree_resnet56 44 | obs_sigma = 0.03 45 | else: 46 | raise ValueError(f"model name {model_name} is wrong") 47 | 48 | logger = logging.getLogger(f"addtree-{model_name}") 49 | logger.setLevel(logging.DEBUG) 50 | 51 | expid = get_experiment_id(6) 52 | output_dir = os.path.join(output_basedir, "addtree", model_name, expid) 53 | os.makedirs(output_dir, exist_ok=False) 54 | log_path = os.path.join( 55 | output_dir, f"addtree-model-compression-{model_name}.log" 56 | ) 57 | setup_logger(logger, log_path) 58 | 59 | logger.info(f"Experiment {expid} starts...") 60 | logger.info("Experiment Configuration:") 61 | logger.info(vars(cmd_args)) 62 | logger.info("Observation sigma is: %f", obs_sigma) 63 | 64 | root = build_tree() 65 | ss = Storage() 66 | ker = build_addtree(root) 67 | const_ker = get_const_kernel(-0.69, root.obs_dim) 68 | ker = const_ker * ker 69 | n_init = cmd_args.n_init 70 | 71 | for i in range(n_init): 72 | logger.info("=" * 50) 73 | logger.info(f"Starting BO {i+1} iteration (Random Design)") 74 | path = root.random_path(rand_data=True) 75 | params = path2funcparam(path[1:]) 76 | obj_info = setup_and_prune( 77 | cmd_args, params, logger, prune_type="multiple", model_name=model_name 78 | ) 79 | ss.add( 80 | path.path2vec(root.obs_dim), obj_info["value"], 0.25, path, 81 | ) 82 | logger.info(f"Finishing BO {i+1} iteration") 83 | logger.info(params) 84 | logger.info(obj_info) 85 | 86 | all_info = {"iteration": i + 1, "params": params, "obj_info": obj_info} 87 | fn_path = os.path.join(output_dir, f"addtree_iter_{i+1}.json") 88 | with open(fn_path, "w") as f: 89 | json.dump(all_info, f) 90 | 91 | def get_kappa(t, max_iter): 92 | ks = np.linspace(1, 3, max_iter) 93 | return ks[t] 94 | 95 | for i in range(n_init, 300): 96 | logger.info("=" * 50) 97 | logger.info(f"Starting BO {i+1} iteration (Optimization)") 98 | gp = ss.optimize(ker, n_restart=5, verbose=False) 99 | _, _, x_best, path = optimize_acq( 100 | LCB, 101 | root, 102 | gp, 103 | ss.Y, 104 | root.obs_dim, 105 | grid_size=2000, 106 | nb_seed=8, 107 | kappa=get_kappa(i, 300), 108 | ) 109 | path.set_data(x_best) 110 | params = path2funcparam(path[1:]) 111 | obj_info = setup_and_prune( 112 | cmd_args, params, logger, prune_type="multiple", model_name=model_name 113 | ) 114 | ss.add( 115 | path.path2vec(root.obs_dim), obj_info["value"], obs_sigma, path=path, 116 | ) 117 | logger.info(f"Finishing BO {i+1} iteration") 118 | logger.info(params) 119 | logger.info(obj_info) 120 | all_info = {"iteration": i + 1, "params": params, "obj_info": obj_info} 121 | fn_path = os.path.join(output_dir, f"addtree_iter_{i+1}.json") 122 | with open(fn_path, "w") as f: 123 | json.dump(all_info, f) 124 | 125 | except KeyboardInterrupt: 126 | print("Interrupted. You pressed Ctrl-C!!!") 127 | try: 128 | sys.exit(0) 129 | except SystemExit: 130 | os._exit(0) 131 | 132 | 133 | if __name__ == "__main__": 134 | main() 135 | -------------------------------------------------------------------------------- /examples/addtree_resnet20_nas.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import json 3 | import os 4 | import logging 5 | 6 | import numpy as np 7 | 8 | from addtree.kernel_utils import build_addtree 9 | from addtree.storage import Storage 10 | from addtree.acq import optimize_acq, LCB 11 | 12 | from common_utils import setup_logger 13 | from common_utils import get_experiment_id 14 | from nas_common import get_common_cmd_args 15 | from nas_common import nas_train_test 16 | 17 | from addtree_utils import build_tree_resnet20_nas 18 | from addtree_utils import path2funcparam_resnet20_nas 19 | 20 | 21 | def main(): 22 | 23 | try: 24 | cmd_args, _ = get_common_cmd_args() 25 | 26 | output_basedir = cmd_args.output_basedir 27 | model_name = cmd_args.model_name 28 | if model_name == "resnet20": 29 | path2funcparam = path2funcparam_resnet20_nas 30 | build_tree = build_tree_resnet20_nas 31 | obs_sigma = 0.06 32 | else: 33 | raise ValueError(f"model name {model_name} is wrong") 34 | 35 | logger = logging.getLogger(f"ADDTREE-NAS-{model_name}") 36 | logger.setLevel(logging.DEBUG) 37 | 38 | expid = get_experiment_id(6) 39 | output_dir = os.path.join(output_basedir, "ADDTREE", model_name, expid) 40 | os.makedirs(output_dir, exist_ok=False) 41 | log_path = os.path.join(output_dir, f"ADDTREE-NAS-{model_name}.log") 42 | setup_logger(logger, log_path) 43 | 44 | logger.info(f"Experiment {expid} starts...") 45 | logger.info("Experiment Configuration:") 46 | logger.info(vars(cmd_args)) 47 | logger.info("Observation sigma is: %f", obs_sigma) 48 | 49 | root = build_tree() 50 | obs_dim = root.obs_dim 51 | ss = Storage() 52 | ker = build_addtree(root) 53 | n_init = cmd_args.n_init 54 | 55 | for i in range(n_init): 56 | logger.info("=" * 50) 57 | logger.info(f"Starting BO {i+1} iteration (Random Design)") 58 | path = root.random_path(rand_data=True) 59 | params = path2funcparam(path[1:]) 60 | obj_info = nas_train_test(cmd_args, params, logger, model_name=model_name) 61 | ss.add( 62 | path.path2vec(obs_dim), obj_info["value"], obs_sigma, path, 63 | ) 64 | logger.info(f"Finishing BO {i+1} iteration") 65 | logger.info(params) 66 | logger.info(obj_info) 67 | 68 | all_info = {"iteration": i + 1, "params": params, "obj_info": obj_info} 69 | fn_path = os.path.join(output_dir, f"addtree_iter_{i+1}.json") 70 | with open(fn_path, "w") as f: 71 | json.dump(all_info, f) 72 | 73 | def get_kappa(t, max_iter): 74 | ks = np.linspace(1, 3, max_iter) 75 | return ks[t] 76 | 77 | for i in range(n_init, 100): 78 | logger.info("=" * 50) 79 | logger.info(f"Starting BO {i+1} iteration (Optimization)") 80 | gp = ss.optimize(ker, n_restart=5, verbose=False) 81 | _, _, x_best, path = optimize_acq( 82 | LCB, 83 | root, 84 | gp, 85 | ss.Y, 86 | obs_dim, 87 | grid_size=2000, 88 | nb_seed=8, 89 | kappa=get_kappa(i, 300), 90 | ) 91 | path.set_data(x_best) 92 | params = path2funcparam(path[1:]) 93 | obj_info = nas_train_test(cmd_args, params, logger, model_name=model_name) 94 | ss.add( 95 | path.path2vec(root.obs_dim), obj_info["value"], obs_sigma, path=path, 96 | ) 97 | logger.info(f"Finishing BO {i+1} iteration") 98 | logger.info(params) 99 | logger.info(obj_info) 100 | all_info = {"iteration": i + 1, "params": params, "obj_info": obj_info} 101 | fn_path = os.path.join(output_dir, f"addtree_iter_{i+1}.json") 102 | with open(fn_path, "w") as f: 103 | json.dump(all_info, f) 104 | 105 | except KeyboardInterrupt: 106 | print("Interrupted. You pressed Ctrl-C!!!") 107 | try: 108 | sys.exit(0) 109 | except SystemExit: 110 | os._exit(0) 111 | 112 | 113 | if __name__ == "__main__": 114 | main() 115 | -------------------------------------------------------------------------------- /examples/addtree_utils.py: -------------------------------------------------------------------------------- 1 | from space_utils.addtree_cs_vgg16 import build_tree as build_tree_vgg16 2 | from space_utils.addtree_cs_vgg16 import path2funcparam as path2funcparam_vgg16 3 | 4 | # resnet 50 5 | from space_utils.addtree_cs_resnet50 import build_tree as build_tree_resnet50 6 | from space_utils.addtree_cs_resnet50 import path2funcparam as path2funcparam_resnet50 7 | 8 | # resnet56 9 | from space_utils.addtree_cs_resnet56 import build_tree as build_tree_resnet56 10 | from space_utils.addtree_cs_resnet56 import path2funcparam as path2funcparam_resnet56 11 | 12 | # jenatton small 13 | from space_utils.addtree_cs_jenatton_small import build_tree as build_tree_jenatton_small 14 | from space_utils.addtree_cs_jenatton_small import path2funcparam as path2funcparam_jenatton_small 15 | 16 | # resnet20 NAS 17 | from space_utils.addtree_cs_resnet20_nas import build_tree as build_tree_resnet20_nas 18 | from space_utils.addtree_cs_resnet20_nas import path2funcparam as path2funcparam_resnet20_nas 19 | 20 | -------------------------------------------------------------------------------- /examples/common_utils.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import math 3 | import os 4 | from datetime import datetime 5 | 6 | 7 | def setup_logger(main_logger, filename): 8 | ch = logging.StreamHandler() 9 | ch.setLevel(logging.DEBUG) 10 | fileHandler = logging.FileHandler(filename) 11 | fileHandler.setLevel(logging.DEBUG) 12 | 13 | formatter = logging.Formatter( 14 | "%(asctime)s - %(name)s - %(levelname)s - %(message)s" 15 | ) 16 | ch.setFormatter(formatter) 17 | fileHandler.setFormatter(formatter) 18 | main_logger.addHandler(ch) 19 | main_logger.addHandler(fileHandler) 20 | 21 | 22 | def char_map(index): 23 | if index < 26: 24 | return index + 97 25 | elif index < 52: 26 | return index - 26 + 65 27 | else: 28 | return index - 52 + 48 29 | 30 | 31 | def unique_string(n): 32 | """generate unique n-length string 33 | 34 | n: length of string 35 | """ 36 | from functools import reduce 37 | 38 | if n == 0: 39 | return "" 40 | byte_len = math.ceil(math.log2(52) + math.log2(62) * (n - 1)) 41 | num = reduce(lambda x, y: x * 256 + y, os.urandom(byte_len), 0) 42 | codes = [] 43 | codes.append(char_map(num % 52)) 44 | num = math.floor(num / 52) 45 | for i in range(1, n): 46 | codes.append(char_map(num % 62)) 47 | num = math.floor(num / 62) 48 | 49 | return "".join(map(chr, codes)) 50 | 51 | 52 | def get_experiment_id(n): 53 | expid = datetime.now().strftime("%Y-%m-%d-%H:%M:%S-") + unique_string(n) 54 | return expid 55 | -------------------------------------------------------------------------------- /examples/makefile: -------------------------------------------------------------------------------- 1 | .PHONY: run_addtree 2 | run_addtree: 3 | SINGULARITYENV_PYTHONPATH=/esat/vauxite/xma/sing_images/images_def/projects/addtree SINGULARITYENV_CUDA_VISIBLE_DEVICES=0 singularity exec --nv /esat/vauxite/xma/sing_images/images_def/working_images/automl.sif python addtree_model_compression_vgg16.py --prune_epochs 3 --checkpoints_dir ./checkpoints_addtree --pretrained ./cifar10_vgg16_best.pt 4 | 5 | .PHONY: run_smac 6 | run_smac: 7 | SINGULARITYENV_CUDA_VISIBLE_DEVICES=0 singularity exec --nv /esat/vauxite/xma/sing_images/images_def/working_images/automl.sif python smac_model_compression_vgg16.py --prune_epochs 3 --checkpoints_dir ./checkpoints_smac --pretrained ./cifar10_vgg16_best.pt 8 | 9 | .PHONY: run_random 10 | run_random: 11 | SINGULARITYENV_CUDA_VISIBLE_DEVICES=0 singularity exec --nv /esat/vauxite/xma/sing_images/images_def/working_images/automl.sif python random_model_compression_vgg16.py --prune_epochs 3 --checkpoints_dir ./checkpoints_random --pretrained ./cifar10_vgg16_best.pt 12 | 13 | .PHONY: clean 14 | clean: 15 | rm -rf ./checkpoints_random ./checkpoints_addtree ./checkpoints_smac ./smac3-output_2020* 16 | 17 | 18 | .PHONY: info 19 | info: 20 | bash ./script/stats.sh 21 | 22 | 23 | .PHONY: collect_multiple 24 | collect_multiple: 25 | @bash ./script/collect_multiple.sh 26 | 27 | .PHONY: collect_single 28 | collect_single: 29 | @bash ./script/collect_single.sh 30 | -------------------------------------------------------------------------------- /examples/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/maxc01/addtree/679f2f0c7ee3905385135e6693138e5733c6306b/examples/models/__init__.py -------------------------------------------------------------------------------- /examples/models/resnet.py: -------------------------------------------------------------------------------- 1 | '''ResNet in PyTorch. 2 | 3 | For Pre-activation ResNet, see 'preact_resnet.py'. 4 | 5 | Reference: 6 | [1] Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun 7 | Deep Residual Learning for Image Recognition. arXiv:1512.03385 8 | ''' 9 | import torch 10 | import torch.nn as nn 11 | import torch.nn.functional as F 12 | 13 | 14 | class BasicBlock(nn.Module): 15 | expansion = 1 16 | 17 | def __init__(self, in_planes, planes, stride=1): 18 | super(BasicBlock, self).__init__() 19 | self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) 20 | self.bn1 = nn.BatchNorm2d(planes) 21 | self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False) 22 | self.bn2 = nn.BatchNorm2d(planes) 23 | 24 | self.shortcut = nn.Sequential() 25 | if stride != 1 or in_planes != self.expansion*planes: 26 | self.shortcut = nn.Sequential( 27 | nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False), 28 | nn.BatchNorm2d(self.expansion*planes) 29 | ) 30 | 31 | def forward(self, x): 32 | out = F.relu(self.bn1(self.conv1(x))) 33 | out = self.bn2(self.conv2(out)) 34 | out += self.shortcut(x) 35 | out = F.relu(out) 36 | return out 37 | 38 | 39 | class Bottleneck(nn.Module): 40 | expansion = 4 41 | 42 | def __init__(self, in_planes, planes, stride=1): 43 | super(Bottleneck, self).__init__() 44 | self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, bias=False) 45 | self.bn1 = nn.BatchNorm2d(planes) 46 | self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) 47 | self.bn2 = nn.BatchNorm2d(planes) 48 | self.conv3 = nn.Conv2d(planes, self.expansion*planes, kernel_size=1, bias=False) 49 | self.bn3 = nn.BatchNorm2d(self.expansion*planes) 50 | 51 | self.shortcut = nn.Sequential() 52 | if stride != 1 or in_planes != self.expansion*planes: 53 | self.shortcut = nn.Sequential( 54 | nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False), 55 | nn.BatchNorm2d(self.expansion*planes) 56 | ) 57 | 58 | def forward(self, x): 59 | out = F.relu(self.bn1(self.conv1(x))) 60 | out = F.relu(self.bn2(self.conv2(out))) 61 | out = self.bn3(self.conv3(out)) 62 | out += self.shortcut(x) 63 | out = F.relu(out) 64 | return out 65 | 66 | 67 | class ResNet(nn.Module): 68 | def __init__(self, block, num_blocks, num_classes=10): 69 | super(ResNet, self).__init__() 70 | self.in_planes = 64 71 | 72 | self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False) 73 | self.bn1 = nn.BatchNorm2d(64) 74 | self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1) 75 | self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2) 76 | self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2) 77 | self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2) 78 | self.linear = nn.Linear(512*block.expansion, num_classes) 79 | 80 | def _make_layer(self, block, planes, num_blocks, stride): 81 | strides = [stride] + [1]*(num_blocks-1) 82 | layers = [] 83 | for stride in strides: 84 | layers.append(block(self.in_planes, planes, stride)) 85 | self.in_planes = planes * block.expansion 86 | return nn.Sequential(*layers) 87 | 88 | def forward(self, x): 89 | out = F.relu(self.bn1(self.conv1(x))) 90 | out = self.layer1(out) 91 | out = self.layer2(out) 92 | out = self.layer3(out) 93 | out = self.layer4(out) 94 | out = F.avg_pool2d(out, 4) 95 | out = out.view(out.size(0), -1) 96 | out = self.linear(out) 97 | return out 98 | 99 | 100 | def ResNet18(): 101 | return ResNet(BasicBlock, [2,2,2,2]) 102 | 103 | def ResNet34(): 104 | return ResNet(BasicBlock, [3,4,6,3]) 105 | 106 | def ResNet50(): 107 | return ResNet(Bottleneck, [3,4,6,3]) 108 | 109 | def ResNet101(): 110 | return ResNet(Bottleneck, [3,4,23,3]) 111 | 112 | def ResNet152(): 113 | return ResNet(Bottleneck, [3,8,36,3]) 114 | 115 | 116 | def test(): 117 | net = ResNet18() 118 | y = net(torch.randn(1,3,32,32)) 119 | print(y.size()) 120 | 121 | # test() 122 | -------------------------------------------------------------------------------- /examples/models/resnet_cifar10.py: -------------------------------------------------------------------------------- 1 | """ 2 | Properly implemented ResNet-s for CIFAR10 as described in paper [1]. 3 | 4 | The implementation and structure of this file is hugely influenced by [2] 5 | which is implemented for ImageNet and doesn't have option A for identity. 6 | Moreover, most of the implementations on the web is copy-paste from 7 | torchvision's resnet and has wrong number of params. 8 | 9 | Proper ResNet-s for CIFAR10 (for fair comparision and etc.) has following 10 | number of layers and parameters: 11 | 12 | name | layers | params 13 | ResNet20 | 20 | 0.27M 14 | ResNet32 | 32 | 0.46M 15 | ResNet44 | 44 | 0.66M 16 | ResNet56 | 56 | 0.85M 17 | ResNet110 | 110 | 1.7M 18 | ResNet1202| 1202 | 19.4m 19 | 20 | which this implementation indeed has. 21 | 22 | Reference: 23 | [1] Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun 24 | Deep Residual Learning for Image Recognition. arXiv:1512.03385 25 | [2] https://github.com/pytorch/vision/blob/master/torchvision/models/resnet.py 26 | 27 | If you use this implementation in you work, please don't forget to mention the 28 | author, Yerlan Idelbayev. 29 | 30 | https://www.reddit.com/r/deeplearning/comments/i3muc4/pushing_cifar10_sota_with_resnets/ 31 | https://github.com/akamaster/pytorch_resnet_cifar10 32 | """ 33 | import torch 34 | import torch.nn as nn 35 | import torch.nn.functional as F 36 | import torch.nn.init as init 37 | 38 | from torch.autograd import Variable 39 | 40 | __all__ = [ 41 | "ResNet", 42 | "resnet20", 43 | "resnet32", 44 | "resnet44", 45 | "resnet56", 46 | "resnet110", 47 | "resnet1202", 48 | ] 49 | 50 | 51 | def _weights_init(m): 52 | classname = m.__class__.__name__ 53 | # print(classname) 54 | if isinstance(m, nn.Linear) or isinstance(m, nn.Conv2d): 55 | init.kaiming_normal_(m.weight) 56 | 57 | 58 | class LambdaLayer(nn.Module): 59 | def __init__(self, lambd): 60 | super(LambdaLayer, self).__init__() 61 | self.lambd = lambd 62 | 63 | def forward(self, x): 64 | return self.lambd(x) 65 | 66 | 67 | class BasicBlock(nn.Module): 68 | expansion = 1 69 | 70 | def __init__( 71 | self, in_planes, planes, stride=1, option="A", act=None, act_params=None 72 | ): 73 | super(BasicBlock, self).__init__() 74 | self.conv1 = nn.Conv2d( 75 | in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False 76 | ) 77 | self.bn1 = nn.BatchNorm2d(planes) 78 | self.conv2 = nn.Conv2d( 79 | planes, planes, kernel_size=3, stride=1, padding=1, bias=False 80 | ) 81 | self.bn2 = nn.BatchNorm2d(planes) 82 | assert act in ["leaky", "elu"] 83 | if act == "leaky": 84 | self.act = F.leaky_relu 85 | elif act == "elu": 86 | self.act = F.elu 87 | self.act_params = act_params 88 | 89 | self.shortcut = nn.Sequential() 90 | if stride != 1 or in_planes != planes: 91 | if option == "A": 92 | """ 93 | For CIFAR10 ResNet paper uses option A. 94 | """ 95 | self.shortcut = LambdaLayer( 96 | lambda x: F.pad( 97 | x[:, :, ::2, ::2], 98 | (0, 0, 0, 0, planes // 4, planes // 4), 99 | "constant", 100 | 0, 101 | ) 102 | ) 103 | elif option == "B": 104 | self.shortcut = nn.Sequential( 105 | nn.Conv2d( 106 | in_planes, 107 | self.expansion * planes, 108 | kernel_size=1, 109 | stride=stride, 110 | bias=False, 111 | ), 112 | nn.BatchNorm2d(self.expansion * planes), 113 | ) 114 | 115 | def forward(self, x): 116 | out = self.act(self.bn1(self.conv1(x)), self.act_params[0]) 117 | out = self.bn2(self.conv2(out)) 118 | out += self.shortcut(x) 119 | out = self.act(out, self.act_params[1]) 120 | return out 121 | 122 | 123 | class ResNet(nn.Module): 124 | def __init__(self, block, num_blocks, num_classes=10, net_spec=None): 125 | super(ResNet, self).__init__() 126 | self.in_planes = 16 127 | 128 | self.conv1 = nn.Conv2d(3, 16, kernel_size=3, stride=1, padding=1, bias=False) 129 | self.bn1 = nn.BatchNorm2d(16) 130 | self.layer1 = self._make_layer( 131 | block, 132 | 16, 133 | num_blocks[0], 134 | stride=1, 135 | act=net_spec["b1"]["method"], 136 | act_params=net_spec["b1"]["amount"], 137 | ) 138 | self.layer2 = self._make_layer( 139 | block, 140 | 32, 141 | num_blocks[1], 142 | stride=2, 143 | act=net_spec["b2"]["method"], 144 | act_params=net_spec["b2"]["amount"], 145 | ) 146 | self.layer3 = self._make_layer( 147 | block, 148 | 64, 149 | num_blocks[2], 150 | stride=2, 151 | act=net_spec["b3"]["method"], 152 | act_params=net_spec["b3"]["amount"], 153 | ) 154 | self.linear = nn.Linear(64, num_classes) 155 | 156 | self.apply(_weights_init) 157 | 158 | def _make_layer(self, block, planes, num_blocks, stride, act, act_params): 159 | strides = [stride] + [1] * (num_blocks - 1) 160 | layers = [] 161 | for stride in strides: 162 | layers.append( 163 | block(self.in_planes, planes, stride, act=act, act_params=act_params) 164 | ) 165 | self.in_planes = planes * block.expansion 166 | 167 | return nn.Sequential(*layers) 168 | 169 | def forward(self, x): 170 | out = F.relu(self.bn1(self.conv1(x))) 171 | out = self.layer1(out) 172 | out = self.layer2(out) 173 | out = self.layer3(out) 174 | out = F.avg_pool2d(out, out.size()[3]) 175 | out = out.view(out.size(0), -1) 176 | out = self.linear(out) 177 | return out 178 | 179 | 180 | def resnet20(net_spec): 181 | return ResNet(BasicBlock, [3, 3, 3], net_spec=net_spec) 182 | 183 | 184 | def resnet32(): 185 | return ResNet(BasicBlock, [5, 5, 5]) 186 | 187 | 188 | def resnet44(): 189 | return ResNet(BasicBlock, [7, 7, 7]) 190 | 191 | 192 | def resnet56(): 193 | return ResNet(BasicBlock, [9, 9, 9]) 194 | 195 | 196 | def resnet110(): 197 | return ResNet(BasicBlock, [18, 18, 18]) 198 | 199 | 200 | def resnet1202(): 201 | return ResNet(BasicBlock, [200, 200, 200]) 202 | 203 | 204 | def test(net): 205 | import numpy as np 206 | 207 | total_params = 0 208 | 209 | for x in filter(lambda p: p.requires_grad, net.parameters()): 210 | total_params += np.prod(x.data.numpy().shape) 211 | print("Total number of params", total_params) 212 | print( 213 | "Total layers", 214 | len( 215 | list( 216 | filter( 217 | lambda p: p.requires_grad and len(p.data.size()) > 1, 218 | net.parameters(), 219 | ) 220 | ) 221 | ), 222 | ) 223 | 224 | 225 | if __name__ == "__main__": 226 | for net_name in __all__: 227 | if net_name.startswith("resnet"): 228 | print(net_name) 229 | test(globals()[net_name]()) 230 | print() 231 | -------------------------------------------------------------------------------- /examples/models/vgg.py: -------------------------------------------------------------------------------- 1 | '''VGG11/13/16/19 in Pytorch.''' 2 | import torch 3 | import torch.nn as nn 4 | 5 | 6 | cfg = { 7 | 'VGG11': [64, 'M', 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'], 8 | 'VGG13': [64, 64, 'M', 128, 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'], 9 | 'VGG16': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, 512, 512, 'M'], 10 | 'VGG19': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 256, 'M', 512, 512, 512, 512, 'M', 512, 512, 512, 512, 'M'], 11 | } 12 | 13 | 14 | class VGG(nn.Module): 15 | def __init__(self, vgg_name): 16 | super(VGG, self).__init__() 17 | self.features = self._make_layers(cfg[vgg_name]) 18 | self.classifier = nn.Linear(512, 10) 19 | 20 | def forward(self, x): 21 | out = self.features(x) 22 | out = out.view(out.size(0), -1) 23 | out = self.classifier(out) 24 | return out 25 | 26 | def _make_layers(self, cfg): 27 | layers = [] 28 | in_channels = 3 29 | for x in cfg: 30 | if x == 'M': 31 | layers += [nn.MaxPool2d(kernel_size=2, stride=2)] 32 | else: 33 | layers += [nn.Conv2d(in_channels, x, kernel_size=3, padding=1), 34 | nn.BatchNorm2d(x), 35 | nn.ReLU(inplace=True)] 36 | in_channels = x 37 | layers += [nn.AvgPool2d(kernel_size=1, stride=1)] 38 | return nn.Sequential(*layers) 39 | 40 | 41 | def test(): 42 | net = VGG('VGG11') 43 | x = torch.randn(2,3,32,32) 44 | y = net(x) 45 | print(y.size()) 46 | 47 | # test() 48 | -------------------------------------------------------------------------------- /examples/nas_common.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | import torch 4 | import torch.nn as nn 5 | import torch.optim as optim 6 | from torch.optim.lr_scheduler import MultiStepLR 7 | 8 | from compression_common import get_data_loaders 9 | from compression_common import train, test 10 | from models.resnet_cifar10 import resnet20 11 | 12 | 13 | def get_common_cmd_args(): 14 | """ Get parameters from command line 15 | 16 | common for all NAS , including random, smac, and adtree 17 | """ 18 | parser = argparse.ArgumentParser() 19 | parser.add_argument("model_name", type=str, help="model name, must be resnet20") 20 | parser.add_argument( 21 | "output_basedir", type=str, help="output base directory", 22 | ) 23 | parser.add_argument( 24 | "--n_init", 25 | type=int, 26 | default=50, 27 | metavar="N", 28 | help="number of random design (default: 50)", 29 | ) 30 | parser.add_argument( 31 | "--batch_size", 32 | type=int, 33 | default=128, 34 | metavar="N", 35 | help="input batch size for training (default: 128)", 36 | ) 37 | parser.add_argument( 38 | "--test_batch_size", 39 | type=int, 40 | default=1000, 41 | metavar="N", 42 | help="input batch size for testing (default: 1000)", 43 | ) 44 | parser.add_argument( 45 | "--lr", 46 | type=float, 47 | default=0.1, 48 | metavar="F", 49 | help="learning rate in finetuning (defualt: 0.1)", 50 | ) 51 | parser.add_argument( 52 | "--multi_gpu", action="store_true", help="Use multiple GPUs for training" 53 | ) 54 | parser.add_argument( 55 | "--log_interval", 56 | type=int, 57 | default=100, 58 | metavar="N", 59 | help="how many batches to wait before logging training status", 60 | ) 61 | args, extra = parser.parse_known_args() 62 | 63 | return args, extra 64 | 65 | 66 | def nas_train_test( 67 | cmd_args, params, main_logger, model_name, max_epoch=10, lr_ms=[7, 9] 68 | ): 69 | """construct a net based on params, train the net for several epoches, return 70 | validation accuracy 71 | 72 | """ 73 | assert model_name in ["resnet20"] 74 | 75 | device = torch.device("cuda" if torch.cuda.is_available() else "cpu") 76 | train_loader, test_loader = get_data_loaders(cmd_args) 77 | if model_name == "resnet20": 78 | model = resnet20(params).to(device=device) 79 | 80 | if isinstance(model, nn.DataParallel): 81 | model = model.module 82 | 83 | optimizer = optim.SGD( 84 | model.parameters(), lr=cmd_args.lr, momentum=0.9, weight_decay=1e-4 85 | ) 86 | 87 | best_top1 = 0 88 | scheduler = MultiStepLR(optimizer, milestones=lr_ms, gamma=0.1) 89 | main_logger.info("Training {} started...".format(model_name)) 90 | for epoch in range(max_epoch): 91 | main_logger.info("# Training Epoch {} #".format(epoch + 1)) 92 | train(cmd_args, model, device, train_loader, optimizer, epoch, main_logger) 93 | main_logger.info("Testing model...") 94 | top1 = test(cmd_args, model, device, test_loader, main_logger) 95 | scheduler.step() 96 | if top1 > best_top1: 97 | best_top1 = top1 98 | 99 | info = {} 100 | info["top1"] = best_top1 / 100 101 | info["value"] = -(best_top1 / 100) 102 | info["value_sigma"] = 0.01 103 | return info 104 | -------------------------------------------------------------------------------- /examples/random_model_compression_multiple.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import json 3 | import os 4 | import logging 5 | 6 | from common_utils import setup_logger 7 | from compression_common import setup_and_prune 8 | from compression_common import get_common_cmd_args 9 | from common_utils import get_experiment_id 10 | 11 | from smac_utils import cfg2funcparams_vgg16 12 | from smac_utils import get_cs_vgg16 13 | from smac_utils import cfg2funcparams_resnet50 14 | from smac_utils import get_cs_resnet50 15 | from smac_utils import cfg2funcparams_resnet56 16 | from smac_utils import get_cs_resnet56 17 | 18 | 19 | def main(): 20 | 21 | try: 22 | cmd_args, _ = get_common_cmd_args() 23 | 24 | output_basedir = cmd_args.output_basedir 25 | model_name = cmd_args.model_name 26 | if model_name == "vgg16": 27 | cfg2funcparams = cfg2funcparams_vgg16 28 | get_cs = get_cs_vgg16 29 | elif model_name == "resnet50": 30 | cfg2funcparams = cfg2funcparams_resnet50 31 | get_cs = get_cs_resnet50 32 | elif model_name == "resnet56": 33 | cfg2funcparams = cfg2funcparams_resnet56 34 | get_cs = get_cs_resnet56 35 | else: 36 | raise ValueError(f"model name {model_name} is wrong") 37 | 38 | logger = logging.getLogger(f"random-{model_name}") 39 | logger.setLevel(logging.DEBUG) 40 | 41 | expid = get_experiment_id(6) 42 | output_dir = os.path.join(output_basedir, "random", model_name, expid) 43 | os.makedirs(output_dir, exist_ok=True) 44 | log_path = os.path.join( 45 | output_dir, f"random-model-compression-{model_name}.log" 46 | ) 47 | setup_logger(logger, log_path) 48 | 49 | logger.info(f"Experiment {expid} starts...") 50 | logger.info("Experiment Configuration:") 51 | logger.info(vars(cmd_args)) 52 | 53 | def obj_func(cfg, opt_iter): 54 | logger.info("Starting BO iteration") 55 | params = cfg2funcparams(cfg) 56 | obj_info = setup_and_prune( 57 | cmd_args, params, logger, prune_type="multiple", model_name=model_name 58 | ) 59 | logger.info("Finishing BO iteration") 60 | logger.info(params) 61 | logger.info(obj_info) 62 | 63 | all_info = { 64 | "iteration": opt_iter, 65 | "params": params, 66 | "obj_info": obj_info, 67 | } 68 | fn_path = os.path.join(output_dir, f"random_iter_{opt_iter}.json") 69 | with open(fn_path, "w") as f: 70 | json.dump(all_info, f) 71 | 72 | return obj_info["value"] 73 | 74 | cs = get_cs() 75 | for i in range(300): 76 | cfg = cs.sample_configuration() 77 | obj_func(cfg, i + 1) 78 | 79 | except KeyboardInterrupt: 80 | print("Interrupted. You pressed Ctrl-C!!!") 81 | try: 82 | sys.exit(0) 83 | except SystemExit: 84 | os._exit(0) 85 | 86 | 87 | if __name__ == "__main__": 88 | main() 89 | -------------------------------------------------------------------------------- /examples/random_resnet20_nas.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import json 3 | import os 4 | import logging 5 | 6 | from common_utils import setup_logger 7 | from common_utils import get_experiment_id 8 | from nas_common import get_common_cmd_args 9 | from nas_common import nas_train_test 10 | 11 | 12 | from smac_utils import cfg2funcparams_nas_resnet20 13 | from smac_utils import get_cs_nas_resnet20 14 | 15 | 16 | def main(): 17 | 18 | try: 19 | cmd_args, _ = get_common_cmd_args() 20 | 21 | output_basedir = cmd_args.output_basedir 22 | model_name = cmd_args.model_name 23 | if model_name == "resnet20": 24 | cfg2funcparams = cfg2funcparams_nas_resnet20 25 | get_cs = get_cs_nas_resnet20 26 | else: 27 | raise ValueError(f"model name {model_name} is wrong") 28 | 29 | logger = logging.getLogger(f"RANDOM-NAS-{model_name}") 30 | logger.setLevel(logging.DEBUG) 31 | 32 | expid = get_experiment_id(6) 33 | output_dir = os.path.join(output_basedir, "RANDOM", model_name, expid) 34 | os.makedirs(output_dir, exist_ok=True) 35 | log_path = os.path.join(output_dir, f"RANDOM-NAS-{model_name}.log") 36 | setup_logger(logger, log_path) 37 | 38 | logger.info(f"Experiment {expid} starts...") 39 | logger.info("Experiment Configuration:") 40 | logger.info(vars(cmd_args)) 41 | 42 | def obj_func(cfg): 43 | logger.info("Starting BO iteration") 44 | params = cfg2funcparams(cfg) 45 | obj_info = nas_train_test(cmd_args, params, logger, model_name=model_name) 46 | logger.info("Finishing BO iteration") 47 | logger.info(params) 48 | logger.info(obj_info) 49 | 50 | all_info = { 51 | "params": params, 52 | "obj_info": obj_info, 53 | } 54 | fn_path = os.path.join(output_dir, "random_iter_hists.txt") 55 | with open(fn_path, "a") as f: 56 | json.dump(all_info, f) 57 | f.write("\n") 58 | 59 | return obj_info["value"] 60 | 61 | cs = get_cs() 62 | for i in range(100): 63 | cfg = cs.sample_configuration() 64 | obj_func(cfg) 65 | 66 | except KeyboardInterrupt: 67 | print("Interrupted. You pressed Ctrl-C!!!") 68 | try: 69 | sys.exit(0) 70 | except SystemExit: 71 | os._exit(0) 72 | 73 | 74 | if __name__ == "__main__": 75 | main() 76 | -------------------------------------------------------------------------------- /examples/resnet20_nas_optimized.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import sys 4 | 5 | from nas_common import get_common_cmd_args 6 | from nas_common import nas_train_test 7 | 8 | 9 | def main(): 10 | 11 | try: 12 | cmd_args, _ = get_common_cmd_args() 13 | 14 | model_name = cmd_args.model_name 15 | 16 | logger = logging.getLogger(f"ADDTREE-NAS-{model_name}") 17 | logger.setLevel(logging.DEBUG) 18 | 19 | ch = logging.StreamHandler() 20 | ch.setLevel(logging.DEBUG) 21 | formatter = logging.Formatter( 22 | "%(asctime)s - %(name)s - %(levelname)s - %(message)s" 23 | ) 24 | ch.setFormatter(formatter) 25 | logger.addHandler(ch) 26 | 27 | logger.info("Training Optimized starts...") 28 | logger.info("Experiment Configuration:") 29 | logger.info(vars(cmd_args)) 30 | 31 | params = { 32 | "b1": {"method": "elu", "amount": [0.01, 0.748403908117127]}, 33 | "b2": {"method": "leaky", "amount": [0.01, 0.99]}, 34 | "b3": {"method": "elu", "amount": [0.01, 0.01]}, 35 | } 36 | obj_info = nas_train_test( 37 | cmd_args, 38 | params, 39 | logger, 40 | model_name=model_name, 41 | max_epoch=200, 42 | lr_ms=[100, 150], 43 | ) 44 | logger.info(obj_info) 45 | except KeyboardInterrupt: 46 | print("Interrupted. You pressed Ctrl-C!!!") 47 | try: 48 | sys.exit(0) 49 | except SystemExit: 50 | os._exit(0) 51 | 52 | 53 | if __name__ == "__main__": 54 | main() 55 | -------------------------------------------------------------------------------- /examples/smac_model_compression_multiple.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import json 3 | import os 4 | import logging 5 | 6 | from common_utils import setup_logger 7 | from compression_common import setup_and_prune 8 | from compression_common import get_common_cmd_args 9 | from common_utils import get_experiment_id 10 | 11 | from smac.scenario.scenario import Scenario 12 | 13 | # from smac.facade.smac_hpo_facade import SMAC4HPO 14 | from smac_utils import SMAC4HPO 15 | 16 | from smac_utils import cfg2funcparams_vgg16_40d 17 | from smac_utils import get_cs_vgg16_40d 18 | from smac_utils import cfg2funcparams_resnet50_40d 19 | from smac_utils import get_cs_resnet50_40d 20 | from smac_utils import cfg2funcparams_resnet56 21 | from smac_utils import get_cs_resnet56 22 | 23 | 24 | def main(): 25 | 26 | try: 27 | cmd_args, _ = get_common_cmd_args() 28 | 29 | output_basedir = cmd_args.output_basedir 30 | model_name = cmd_args.model_name 31 | if model_name == "vgg16": 32 | cfg2funcparams = cfg2funcparams_vgg16_40d 33 | get_cs = get_cs_vgg16_40d 34 | elif model_name == "resnet50": 35 | cfg2funcparams = cfg2funcparams_resnet50_40d 36 | get_cs = get_cs_resnet50_40d 37 | elif model_name == "resnet56": 38 | cfg2funcparams = cfg2funcparams_resnet56 39 | get_cs = get_cs_resnet56 40 | else: 41 | raise ValueError(f"model name {model_name} is wrong") 42 | 43 | logger = logging.getLogger(f"smac-{model_name}") 44 | logger.setLevel(logging.DEBUG) 45 | 46 | expid = get_experiment_id(6) 47 | output_dir = os.path.join(output_basedir, "smac", model_name, expid) 48 | os.makedirs(output_dir, exist_ok=True) 49 | log_path = os.path.join(output_dir, f"smac-model-compression-{model_name}.log") 50 | setup_logger(logger, log_path) 51 | 52 | logger.info(f"Experiment {expid} starts...") 53 | logger.info("Experiment Configuration:") 54 | logger.info(vars(cmd_args)) 55 | 56 | def obj_func(cfg): 57 | logger.info("Starting BO iteration") 58 | params = cfg2funcparams(cfg) 59 | obj_info = setup_and_prune(cmd_args, params, logger, prune_type="multiple", model_name=model_name) 60 | logger.info("Finishing BO iteration") 61 | logger.info(params) 62 | logger.info(obj_info) 63 | 64 | all_info = { 65 | "params": params, 66 | "obj_info": obj_info, 67 | } 68 | fn_path = os.path.join(output_dir, "smac_iter_hists.txt") 69 | with open(fn_path, "a") as f: 70 | json.dump(all_info, f) 71 | f.write("\n") 72 | 73 | return obj_info["value"] 74 | 75 | # smac default do minimize 76 | cs = get_cs() 77 | scenario = Scenario( 78 | { 79 | "run_obj": "quality", # we optimize quality (alternatively runtime) 80 | "runcount_limit": 300, # maximum function evaluations 81 | "cs": cs, # configuration space 82 | "deterministic": "true", 83 | "initial_incumbent": "LHD", 84 | } 85 | ) 86 | 87 | smac = SMAC4HPO( 88 | scenario=scenario, 89 | tae_runner=obj_func, 90 | ) 91 | 92 | incumbent = smac.optimize() 93 | print(incumbent) 94 | 95 | except KeyboardInterrupt: 96 | print("Interrupted. You pressed Ctrl-C!!!") 97 | try: 98 | sys.exit(0) 99 | except SystemExit: 100 | os._exit(0) 101 | 102 | 103 | if __name__ == "__main__": 104 | main() 105 | -------------------------------------------------------------------------------- /examples/smac_resnet20_nas.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import json 3 | import os 4 | import logging 5 | 6 | from smac.scenario.scenario import Scenario 7 | from smac.facade.smac_hpo_facade import SMAC4HPO 8 | 9 | from common_utils import setup_logger 10 | from common_utils import get_experiment_id 11 | from nas_common import get_common_cmd_args 12 | from nas_common import nas_train_test 13 | 14 | 15 | from smac_utils import cfg2funcparams_nas_resnet20 16 | from smac_utils import get_cs_nas_resnet20 17 | 18 | 19 | def main(): 20 | 21 | try: 22 | cmd_args, _ = get_common_cmd_args() 23 | 24 | output_basedir = cmd_args.output_basedir 25 | model_name = cmd_args.model_name 26 | if model_name == "resnet20": 27 | cfg2funcparams = cfg2funcparams_nas_resnet20 28 | get_cs = get_cs_nas_resnet20 29 | else: 30 | raise ValueError(f"model name {model_name} is wrong") 31 | 32 | logger = logging.getLogger(f"SMAC-NAS-{model_name}") 33 | logger.setLevel(logging.DEBUG) 34 | 35 | expid = get_experiment_id(6) 36 | output_dir = os.path.join(output_basedir, "SMAC", model_name, expid) 37 | os.makedirs(output_dir, exist_ok=True) 38 | log_path = os.path.join(output_dir, f"SMAC-NAS-{model_name}.log") 39 | setup_logger(logger, log_path) 40 | 41 | logger.info(f"Experiment {expid} starts...") 42 | logger.info("Experiment Configuration:") 43 | logger.info(vars(cmd_args)) 44 | 45 | def obj_func(cfg): 46 | logger.info("Starting BO iteration") 47 | params = cfg2funcparams(cfg) 48 | obj_info = nas_train_test(cmd_args, params, logger, model_name=model_name) 49 | logger.info("Finishing BO iteration") 50 | logger.info(params) 51 | logger.info(obj_info) 52 | 53 | all_info = { 54 | "params": params, 55 | "obj_info": obj_info, 56 | } 57 | fn_path = os.path.join(output_dir, "smac_iter_hists.txt") 58 | with open(fn_path, "a") as f: 59 | json.dump(all_info, f) 60 | f.write("\n") 61 | 62 | return obj_info["value"] 63 | 64 | # smac default do minimize 65 | cs = get_cs() 66 | scenario = Scenario( 67 | { 68 | "run_obj": "quality", # we optimize quality (alternatively runtime) 69 | "runcount_limit": 100, # maximum function evaluations 70 | "cs": cs, # configuration space 71 | "deterministic": "true", 72 | "initial_incumbent": "LHD", 73 | } 74 | ) 75 | 76 | smac = SMAC4HPO(scenario=scenario, tae_runner=obj_func,) 77 | 78 | incumbent = smac.optimize() 79 | print(incumbent) 80 | 81 | except KeyboardInterrupt: 82 | print("Interrupted. You pressed Ctrl-C!!!") 83 | try: 84 | sys.exit(0) 85 | except SystemExit: 86 | os._exit(0) 87 | 88 | 89 | if __name__ == "__main__": 90 | main() 91 | -------------------------------------------------------------------------------- /examples/smac_utils.py: -------------------------------------------------------------------------------- 1 | # lower dimensional problem for SMAC 2 | # from smac_cs_40 import cfg2funcparams_40 # noqa 3 | # from smac_cs_40 import cs_40 # NOQA 4 | 5 | # from smac_cs_single import cfg2funcparams_single # NOQA 6 | # from smac_cs_single import cs_single # NOQA 7 | 8 | # resnet50 9 | from space_utils.smac_cs_resnet50_multiple import cfg2funcparams as cfg2funcparams_resnet50 # noqa 10 | from space_utils.smac_cs_resnet50_multiple import get_cs as get_cs_resnet50 # NOQA 11 | 12 | # vgg16 13 | from space_utils.smac_cs_vgg16_multiple import cfg2funcparams as cfg2funcparams_vgg16 # NOQA 14 | from space_utils.smac_cs_vgg16_multiple import get_cs as get_cs_vgg16 # noqa 15 | 16 | # resnet56 17 | from space_utils.smac_cs_resnet56 import cfg2funcparams as cfg2funcparams_resnet56 18 | from space_utils.smac_cs_resnet56 import get_cs as get_cs_resnet56 19 | 20 | # vgg16 40d 21 | from space_utils.smac_cs_vgg16_40d import cfg2funcparams as cfg2funcparams_vgg16_40d # NOQA 22 | from space_utils.smac_cs_vgg16_40d import get_cs as get_cs_vgg16_40d # noqa 23 | 24 | # resnet50 40d 25 | from space_utils.smac_cs_resnet50_40d import cfg2funcparams as cfg2funcparams_resnet50_40d # noqa 26 | from space_utils.smac_cs_resnet50_40d import get_cs as get_cs_resnet50_40d # NOQA 27 | 28 | # smac utils 29 | from space_utils.smac_hpo_facade import SMAC4HPO 30 | 31 | # resnet20, NAS 32 | from space_utils.smac_cs_nas_resnet20 import cfg2funcparams as cfg2funcparams_nas_resnet20 33 | from space_utils.smac_cs_nas_resnet20 import get_cs as get_cs_nas_resnet20 34 | -------------------------------------------------------------------------------- /examples/space_utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/maxc01/addtree/679f2f0c7ee3905385135e6693138e5733c6306b/examples/space_utils/__init__.py -------------------------------------------------------------------------------- /examples/space_utils/addtree_cs_jenatton_small.py: -------------------------------------------------------------------------------- 1 | from addtree.parameter import Parameter 2 | from addtree.parameter import ParameterNode 3 | 4 | NAME2METHOD = { 5 | "x2": "left", 6 | "x3": "right", 7 | "x4": "left", 8 | "x5": "right", 9 | "x6": "left", 10 | "x7": "right", 11 | } 12 | 13 | 14 | def build_tree(): 15 | root = ParameterNode(Parameter("root", 0)) 16 | x2 = ParameterNode(Parameter("x2", 1)) 17 | x3 = ParameterNode(Parameter("x3", 1)) 18 | x4 = ParameterNode(Parameter("x4", 1)) 19 | x5 = ParameterNode(Parameter("x5", 1)) 20 | x6 = ParameterNode(Parameter("x6", 1)) 21 | x7 = ParameterNode(Parameter("x7", 1)) 22 | 23 | root.add_child(x2) 24 | root.add_child(x3) 25 | x2.add_child(x4) 26 | x2.add_child(x5) 27 | x3.add_child(x6) 28 | x3.add_child(x7) 29 | 30 | root.finish_add_child() 31 | 32 | return root 33 | 34 | 35 | def path2funcparam(path): 36 | layer_names = ["L1", "L2"] 37 | params = {} 38 | for layer_name, node in zip(layer_names, path): 39 | params[layer_name] = {} 40 | params[layer_name]["cat_value"] = NAME2METHOD[node.name] 41 | if node.parameter.data.shape == (1,): 42 | params[layer_name]["cont_value"] = node.parameter.data.item() 43 | else: 44 | params[layer_name]["amount"] = node.parameter.data.tolist() 45 | 46 | return params 47 | -------------------------------------------------------------------------------- /examples/space_utils/addtree_cs_resnet20_nas.py: -------------------------------------------------------------------------------- 1 | from addtree.parameter import Parameter 2 | from addtree.parameter import ParameterNode 3 | 4 | NAME2METHOD = { 5 | "x1": "elu", 6 | "x2": "leaky", 7 | "x3": "elu", 8 | "x4": "leaky", 9 | "x5": "elu", 10 | "x6": "leaky", 11 | "x7": "elu", 12 | "x8": "leaky", 13 | "x9": "elu", 14 | "x10": "leaky", 15 | "x11": "elu", 16 | "x12": "leaky", 17 | "x13": "elu", 18 | "x14": "leaky", 19 | } 20 | 21 | 22 | def build_tree(): 23 | root = ParameterNode(Parameter("root", 0)) 24 | x1 = ParameterNode(Parameter("x1", 2)) 25 | x2 = ParameterNode(Parameter("x2", 2)) 26 | x3 = ParameterNode(Parameter("x3", 2)) 27 | x4 = ParameterNode(Parameter("x4", 2)) 28 | x5 = ParameterNode(Parameter("x5", 2)) 29 | x6 = ParameterNode(Parameter("x6", 2)) 30 | x7 = ParameterNode(Parameter("x7", 2)) 31 | x8 = ParameterNode(Parameter("x8", 2)) 32 | x9 = ParameterNode(Parameter("x9", 2)) 33 | x10 = ParameterNode(Parameter("x10", 2)) 34 | x11 = ParameterNode(Parameter("x11", 2)) 35 | x12 = ParameterNode(Parameter("x12", 2)) 36 | x13 = ParameterNode(Parameter("x13", 2)) 37 | x14 = ParameterNode(Parameter("x14", 2)) 38 | 39 | root.add_child(x1) 40 | root.add_child(x2) 41 | 42 | x1.add_child(x3) 43 | x1.add_child(x4) 44 | 45 | x2.add_child(x5) 46 | x2.add_child(x6) 47 | 48 | x3.add_child(x7) 49 | x3.add_child(x8) 50 | 51 | x4.add_child(x9) 52 | x4.add_child(x10) 53 | 54 | x5.add_child(x11) 55 | x5.add_child(x12) 56 | 57 | x6.add_child(x13) 58 | x6.add_child(x14) 59 | 60 | root.finish_add_child() 61 | 62 | return root 63 | 64 | 65 | def path2funcparam(path): 66 | b_names = ["b1", "b2", "b3"] 67 | params = {} 68 | for b_name, node in zip(b_names, path): 69 | params[b_name] = {} 70 | params[b_name]["method"] = NAME2METHOD[node.name] 71 | if node.parameter.data.shape == (1,): 72 | params[b_name]["amount"] = node.parameter.data.item() 73 | else: 74 | params[b_name]["amount"] = node.parameter.data.tolist() 75 | 76 | return params 77 | -------------------------------------------------------------------------------- /examples/space_utils/addtree_cs_resnet50.py: -------------------------------------------------------------------------------- 1 | from addtree.parameter import Parameter 2 | from addtree.parameter import ParameterNode 3 | 4 | 5 | NAME2METHOD = { 6 | "x1": "l1", 7 | "x2": "ln", 8 | "x3": "l1", 9 | "x4": "ln", 10 | "x5": "l1", 11 | "x6": "ln", 12 | "x7": "l1", 13 | "x8": "ln", 14 | "x9": "l1", 15 | "x10": "ln", 16 | "x11": "l1", 17 | "x12": "ln", 18 | "x13": "l1", 19 | "x14": "ln", 20 | } 21 | 22 | 23 | def build_tree(): 24 | root = ParameterNode(Parameter("root", 0)) 25 | x1 = ParameterNode(Parameter("x1", 4)) 26 | x2 = ParameterNode(Parameter("x2", 4)) 27 | x3 = ParameterNode(Parameter("x3", 4)) 28 | x4 = ParameterNode(Parameter("x4", 4)) 29 | x5 = ParameterNode(Parameter("x5", 4)) 30 | x6 = ParameterNode(Parameter("x6", 4)) 31 | x7 = ParameterNode(Parameter("x7", 4)) 32 | x8 = ParameterNode(Parameter("x8", 4)) 33 | x9 = ParameterNode(Parameter("x9", 4)) 34 | x10 = ParameterNode(Parameter("x10", 4)) 35 | x11 = ParameterNode(Parameter("x11", 4)) 36 | x12 = ParameterNode(Parameter("x12", 4)) 37 | x13 = ParameterNode(Parameter("x13", 4)) 38 | x14 = ParameterNode(Parameter("x14", 4)) 39 | 40 | root.add_child(x1) 41 | root.add_child(x2) 42 | 43 | x1.add_child(x3) 44 | x1.add_child(x4) 45 | 46 | x2.add_child(x5) 47 | x2.add_child(x6) 48 | 49 | x3.add_child(x7) 50 | x3.add_child(x8) 51 | 52 | x4.add_child(x9) 53 | x4.add_child(x10) 54 | 55 | x5.add_child(x11) 56 | x5.add_child(x12) 57 | 58 | x6.add_child(x13) 59 | x6.add_child(x14) 60 | 61 | root.finish_add_child() 62 | 63 | return root 64 | 65 | 66 | def path2funcparam(path): 67 | layer_names = ["layer2", "layer3", "layer4"] 68 | params = {} 69 | for layer_name, node in zip(layer_names, path): 70 | params[layer_name] = {} 71 | params[layer_name]["prune_method"] = NAME2METHOD[node.name] 72 | if node.parameter.data.shape == (1,): 73 | params[layer_name]["amount"] = node.parameter.data.item() 74 | else: 75 | params[layer_name]["amount"] = node.parameter.data.tolist() 76 | 77 | return params 78 | -------------------------------------------------------------------------------- /examples/space_utils/addtree_cs_resnet56.py: -------------------------------------------------------------------------------- 1 | from addtree.parameter import Parameter 2 | from addtree.parameter import ParameterNode 3 | 4 | 5 | NAME2METHOD = { 6 | "x1": "l1", 7 | "x2": "ln", 8 | "x3": "l1", 9 | "x4": "ln", 10 | "x5": "l1", 11 | "x6": "ln", 12 | "x7": "l1", 13 | "x8": "ln", 14 | "x9": "l1", 15 | "x10": "ln", 16 | "x11": "l1", 17 | "x12": "ln", 18 | "x13": "l1", 19 | "x14": "ln", 20 | } 21 | 22 | 23 | def build_tree(): 24 | root = ParameterNode(Parameter("root", 0)) 25 | x1 = ParameterNode(Parameter("x1", 3)) 26 | x2 = ParameterNode(Parameter("x2", 3)) 27 | x3 = ParameterNode(Parameter("x3", 3)) 28 | x4 = ParameterNode(Parameter("x4", 3)) 29 | x5 = ParameterNode(Parameter("x5", 3)) 30 | x6 = ParameterNode(Parameter("x6", 3)) 31 | x7 = ParameterNode(Parameter("x7", 3)) 32 | x8 = ParameterNode(Parameter("x8", 3)) 33 | x9 = ParameterNode(Parameter("x9", 3)) 34 | x10 = ParameterNode(Parameter("x10", 3)) 35 | x11 = ParameterNode(Parameter("x11", 3)) 36 | x12 = ParameterNode(Parameter("x12", 3)) 37 | x13 = ParameterNode(Parameter("x13", 3)) 38 | x14 = ParameterNode(Parameter("x14", 3)) 39 | 40 | root.add_child(x1) 41 | root.add_child(x2) 42 | 43 | x1.add_child(x3) 44 | x1.add_child(x4) 45 | 46 | x2.add_child(x5) 47 | x2.add_child(x6) 48 | 49 | x3.add_child(x7) 50 | x3.add_child(x8) 51 | 52 | x4.add_child(x9) 53 | x4.add_child(x10) 54 | 55 | x5.add_child(x11) 56 | x5.add_child(x12) 57 | 58 | x6.add_child(x13) 59 | x6.add_child(x14) 60 | 61 | root.finish_add_child() 62 | 63 | return root 64 | 65 | 66 | def path2funcparam(path): 67 | layer_names = ["layer1", "layer2", "layer3"] 68 | params = {} 69 | for layer_name, node in zip(layer_names, path): 70 | params[layer_name] = {} 71 | params[layer_name]["prune_method"] = NAME2METHOD[node.name] 72 | if node.parameter.data.shape == (1,): 73 | params[layer_name]["amount"] = node.parameter.data.item() 74 | else: 75 | params[layer_name]["amount"] = node.parameter.data.tolist() 76 | 77 | return params 78 | -------------------------------------------------------------------------------- /examples/space_utils/addtree_cs_vgg16.py: -------------------------------------------------------------------------------- 1 | from addtree.parameter import Parameter 2 | from addtree.parameter import ParameterNode 3 | 4 | NAME2METHOD = { 5 | "x1": "l1", 6 | "x2": "ln", 7 | "x3": "l1", 8 | "x4": "ln", 9 | "x5": "l1", 10 | "x6": "ln", 11 | "x7": "l1", 12 | "x8": "ln", 13 | "x9": "l1", 14 | "x10": "ln", 15 | "x11": "l1", 16 | "x12": "ln", 17 | "x13": "l1", 18 | "x14": "ln", 19 | } 20 | 21 | 22 | def build_tree(): 23 | root = ParameterNode(Parameter("root", 0)) 24 | x1 = ParameterNode(Parameter("x1", 3)) 25 | x2 = ParameterNode(Parameter("x2", 3)) 26 | x3 = ParameterNode(Parameter("x3", 3)) 27 | x4 = ParameterNode(Parameter("x4", 3)) 28 | x5 = ParameterNode(Parameter("x5", 3)) 29 | x6 = ParameterNode(Parameter("x6", 3)) 30 | x7 = ParameterNode(Parameter("x7", 3)) 31 | x8 = ParameterNode(Parameter("x8", 3)) 32 | x9 = ParameterNode(Parameter("x9", 3)) 33 | x10 = ParameterNode(Parameter("x10", 3)) 34 | x11 = ParameterNode(Parameter("x11", 3)) 35 | x12 = ParameterNode(Parameter("x12", 3)) 36 | x13 = ParameterNode(Parameter("x13", 3)) 37 | x14 = ParameterNode(Parameter("x14", 3)) 38 | 39 | root.add_child(x1) 40 | root.add_child(x2) 41 | 42 | x1.add_child(x3) 43 | x1.add_child(x4) 44 | 45 | x2.add_child(x5) 46 | x2.add_child(x6) 47 | 48 | x3.add_child(x7) 49 | x3.add_child(x8) 50 | 51 | x4.add_child(x9) 52 | x4.add_child(x10) 53 | 54 | x5.add_child(x11) 55 | x5.add_child(x12) 56 | 57 | x6.add_child(x13) 58 | x6.add_child(x14) 59 | 60 | root.finish_add_child() 61 | 62 | return root 63 | 64 | 65 | def path2funcparam(path): 66 | b_names = ["b1", "b2", "b3"] 67 | params = {} 68 | for b_name, node in zip(b_names, path): 69 | params[b_name] = {} 70 | params[b_name]["prune_method"] = NAME2METHOD[node.name] 71 | if node.parameter.data.shape == (1,): 72 | params[b_name]["amount"] = node.parameter.data.item() 73 | else: 74 | params[b_name]["amount"] = node.parameter.data.tolist() 75 | 76 | return params 77 | -------------------------------------------------------------------------------- /examples/space_utils/smac_bo_facade.py: -------------------------------------------------------------------------------- 1 | import typing 2 | 3 | import numpy as np 4 | 5 | from smac.facade.smac_ac_facade import SMAC4AC 6 | from smac.epm.base_gp import BaseModel 7 | from smac.epm.gaussian_process_mcmc import GaussianProcessMCMC, GaussianProcess 8 | from smac.epm.gp_base_prior import HorseshoePrior, LognormalPrior 9 | from smac.epm.gp_kernels import ConstantKernel, Matern, WhiteKernel, HammingKernel 10 | from smac.epm.util_funcs import get_types, get_rng 11 | from smac.initial_design.sobol_design import SobolDesign 12 | from smac.runhistory.runhistory2epm import RunHistory2EPM4Cost 13 | 14 | 15 | __author__ = "Marius Lindauer" 16 | __copyright__ = "Copyright 2018, ML4AAD" 17 | __license__ = "3-clause BSD" 18 | 19 | 20 | class SMAC4BO(SMAC4AC): 21 | """ 22 | Facade to use SMAC for BO using a GP 23 | 24 | see smac.facade.smac_Facade for API 25 | This facade overwrites options available via the SMAC facade 26 | 27 | Hyperparameters are chosen according to the best configuration for Gaussian process maximum likelihood found in 28 | "Towards Assessing the Impact of Bayesian Optimization's Own Hyperparameters" by Lindauer et al., presented at the 29 | DSO workshop 2019 (https://arxiv.org/abs/1908.06674). 30 | 31 | Changes are: 32 | * Instead of having an initial design of size 10*D as suggested by Jones et al. 1998 (actually, they suggested 33 | 10*D+1), we use an initial design of 8*D. 34 | * More restrictive lower and upper bounds on the length scale for the Matern and Hamming Kernel than the ones 35 | suggested by Klein et al. 2017 in the RoBO package. In practice, they are ``np.exp(-6.754111155189306)`` 36 | instead of ``np.exp(-10)`` for the lower bound and ``np.exp(0.0858637988771976)`` instead of 37 | ``np.exp(2)`` for the upper bound. 38 | * The initial design is set to be a Sobol grid 39 | * The random fraction is set to ``0.08447232371720552``, it was ``0.0`` before. 40 | 41 | Attributes 42 | ---------- 43 | logger 44 | stats : Stats 45 | solver : SMBO 46 | runhistory : RunHistory 47 | List with information about previous runs 48 | trajectory : list 49 | List of all incumbents 50 | 51 | """ 52 | 53 | def __init__(self, model_type: str = 'gp_mcmc', **kwargs: typing.Any): 54 | """ 55 | Constructor 56 | see ~smac.facade.smac_facade for documentation 57 | """ 58 | 59 | scenario = kwargs['scenario'] 60 | 61 | if len(scenario.cs.get_hyperparameters()) <= 80: 62 | kwargs['initial_design'] = kwargs.get('initial_design', SobolDesign) 63 | else: 64 | raise ValueError( 65 | 'The default initial design "Sobol sequence" can only handle up to 40 dimensions. ' 66 | 'Please use a different initial design, such as "the Latin Hypercube design".', 67 | ) 68 | kwargs['runhistory2epm'] = kwargs.get('runhistory2epm', RunHistory2EPM4Cost) 69 | 70 | init_kwargs = kwargs.get('initial_design_kwargs', dict()) or dict() 71 | init_kwargs['n_configs_x_params'] = init_kwargs.get('n_configs_x_params', 8) 72 | init_kwargs['max_config_fracs'] = init_kwargs.get('max_config_fracs', 0.25) 73 | kwargs['initial_design_kwargs'] = init_kwargs 74 | 75 | if kwargs.get('model') is None: 76 | 77 | model_kwargs = kwargs.get('model_kwargs', dict()) or dict() 78 | 79 | _, rng = get_rng(rng=kwargs.get("rng", None), run_id=kwargs.get("run_id", None), logger=None) 80 | 81 | types, bounds = get_types(kwargs['scenario'].cs, instance_features=None) 82 | 83 | cov_amp = ConstantKernel( 84 | 2.0, 85 | constant_value_bounds=(np.exp(-10), np.exp(2)), 86 | prior=LognormalPrior(mean=0.0, sigma=1.0, rng=rng), 87 | ) 88 | 89 | cont_dims = np.nonzero(types == 0)[0] 90 | cat_dims = np.nonzero(types != 0)[0] 91 | 92 | if len(cont_dims) > 0: 93 | exp_kernel = Matern( 94 | np.ones([len(cont_dims)]), 95 | [(np.exp(-6.754111155189306), np.exp(0.0858637988771976)) for _ in range(len(cont_dims))], 96 | nu=2.5, 97 | operate_on=cont_dims, 98 | ) 99 | 100 | if len(cat_dims) > 0: 101 | ham_kernel = HammingKernel( 102 | np.ones([len(cat_dims)]), 103 | [(np.exp(-6.754111155189306), np.exp(0.0858637988771976)) for _ in range(len(cat_dims))], 104 | operate_on=cat_dims, 105 | ) 106 | 107 | noise_kernel = WhiteKernel( 108 | noise_level=1e-8, 109 | noise_level_bounds=(np.exp(-25), np.exp(2)), 110 | prior=HorseshoePrior(scale=0.1, rng=rng), 111 | ) 112 | 113 | if len(cont_dims) > 0 and len(cat_dims) > 0: 114 | # both 115 | kernel = cov_amp * (exp_kernel * ham_kernel) + noise_kernel 116 | elif len(cont_dims) > 0 and len(cat_dims) == 0: 117 | # only cont 118 | kernel = cov_amp * exp_kernel + noise_kernel 119 | elif len(cont_dims) == 0 and len(cat_dims) > 0: 120 | # only cont 121 | kernel = cov_amp * ham_kernel + noise_kernel 122 | else: 123 | raise ValueError() 124 | 125 | if model_type == "gp": 126 | model_class = GaussianProcess # type: typing.Type[BaseModel] 127 | kwargs['model'] = model_class 128 | model_kwargs['kernel'] = kernel 129 | model_kwargs['normalize_y'] = True 130 | model_kwargs['seed'] = rng.randint(0, 2 ** 20) 131 | elif model_type == "gp_mcmc": 132 | model_class = GaussianProcessMCMC 133 | kwargs['model'] = model_class 134 | kwargs['integrate_acquisition_function'] = True 135 | 136 | model_kwargs['kernel'] = kernel 137 | 138 | n_mcmc_walkers = 3 * len(kernel.theta) 139 | if n_mcmc_walkers % 2 == 1: 140 | n_mcmc_walkers += 1 141 | model_kwargs['n_mcmc_walkers'] = n_mcmc_walkers 142 | model_kwargs['chain_length'] = 250 143 | model_kwargs['burnin_steps'] = 250 144 | model_kwargs['normalize_y'] = True 145 | model_kwargs['seed'] = rng.randint(0, 2**20) 146 | else: 147 | raise ValueError('Unknown model type %s' % model_type) 148 | kwargs['model_kwargs'] = model_kwargs 149 | 150 | if kwargs.get('random_configuration_chooser') is None: 151 | random_config_chooser_kwargs = kwargs.get( 152 | 'random_configuration_chooser_kwargs', 153 | dict(), 154 | ) or dict() 155 | random_config_chooser_kwargs['prob'] = random_config_chooser_kwargs.get('prob', 0.08447232371720552) 156 | kwargs['random_configuration_chooser_kwargs'] = random_config_chooser_kwargs 157 | 158 | if kwargs.get('acquisition_function_optimizer') is None: 159 | acquisition_function_optimizer_kwargs = kwargs.get( 160 | 'acquisition_function_optimizer_kwargs', 161 | dict(), 162 | ) or dict() 163 | acquisition_function_optimizer_kwargs['n_sls_iterations'] = 10 164 | kwargs['acquisition_function_optimizer_kwargs'] = acquisition_function_optimizer_kwargs 165 | 166 | # only 1 configuration per SMBO iteration 167 | intensifier_kwargs = kwargs.get('intensifier_kwargs', dict()) or dict() 168 | intensifier_kwargs['min_chall'] = 1 169 | kwargs['intensifier_kwargs'] = intensifier_kwargs 170 | scenario.intensification_percentage = 1e-10 171 | 172 | super().__init__(**kwargs) 173 | 174 | if self.solver.scenario.n_features > 0: 175 | raise NotImplementedError("BOGP cannot handle instances") 176 | 177 | self.logger.info(self.__class__) 178 | 179 | self.solver.scenario.acq_opt_challengers = 1000 # type: ignore[attr-defined] # noqa F821 180 | # activate predict incumbent 181 | self.solver.epm_chooser.predict_x_best = True 182 | -------------------------------------------------------------------------------- /examples/space_utils/smac_cs_nas_resnet20.py: -------------------------------------------------------------------------------- 1 | from smac.configspace import ConfigurationSpace 2 | from ConfigSpace.hyperparameters import ( 3 | CategoricalHyperparameter, 4 | UniformFloatHyperparameter, 5 | Constant, 6 | ) 7 | from ConfigSpace.conditions import InCondition 8 | 9 | 10 | def testing_cfg(): 11 | return { 12 | "root": "elu", 13 | "r1_1": 0.6866000529348212, 14 | "r1_2": 0.7538109366169266, 15 | "x1": "leaky", 16 | "r4_1": 0.9567669460238617, 17 | "r4_2": 0.6473958655250426, 18 | "x4": "elu", 19 | "r9_1": 0.5, 20 | "r9_2": 0.5, 21 | } 22 | 23 | 24 | def cfg2funcparams(cfg): 25 | def extract_one_value(keys): 26 | """this function extract ONE value with key in keys from cfg 27 | keys: e.g. ["x1","x2"] 28 | """ 29 | 30 | for k in keys: 31 | if k in cfg.keys(): 32 | return cfg[k] 33 | raise RuntimeError("key not exist") 34 | 35 | def extract_using_prefix(keys): 36 | """this function extract ONE value with key in keys from cfg 37 | keys: e.g. ["r1","r2"] 38 | """ 39 | 40 | cfg_prefix = set([i.split("_")[0] for i in cfg.keys()]) 41 | # e.g. r1->[r1_1, r1_2, r1_3] 42 | for k in keys: 43 | if k in cfg_prefix: 44 | return [cfg[k + "_1"], cfg[k + "_2"]] 45 | raise RuntimeError("key not exist") 46 | 47 | params = {} 48 | params["b1"] = {} 49 | params["b1"]["method"] = cfg["root"] 50 | params["b1"]["amount"] = extract_using_prefix(["r1", "r2"]) 51 | 52 | params["b2"] = {} 53 | params["b2"]["method"] = extract_one_value(["x1", "x2"]) 54 | params["b2"]["amount"] = extract_using_prefix(["r3", "r4", "r5", "r6"]) 55 | 56 | params["b3"] = {} 57 | params["b3"]["method"] = extract_one_value(["x3", "x4", "x5", "x6"]) 58 | params["b3"]["amount"] = extract_using_prefix( 59 | ["r7", "r8", "r9", "r10", "r11", "r12", "r13", "r14",] 60 | ) 61 | 62 | return params 63 | 64 | 65 | def get_cs(): 66 | cs = ConfigurationSpace() 67 | 68 | root = CategoricalHyperparameter("root", choices=["elu", "leaky"]) 69 | x1 = CategoricalHyperparameter("x1", choices=["elu", "leaky"]) 70 | x2 = CategoricalHyperparameter("x2", choices=["elu", "leaky"]) 71 | x3 = CategoricalHyperparameter("x3", choices=["elu", "leaky"]) 72 | x4 = CategoricalHyperparameter("x4", choices=["elu", "leaky"]) 73 | x5 = CategoricalHyperparameter("x5", choices=["elu", "leaky"]) 74 | x6 = CategoricalHyperparameter("x6", choices=["elu", "leaky"]) 75 | 76 | # r1 is the data associated in x1 77 | r1_1 = UniformFloatHyperparameter("r1_1", lower=0.01, upper=0.99, log=False) 78 | r1_2 = UniformFloatHyperparameter("r1_2", lower=0.01, upper=0.99, log=False) 79 | 80 | r2_1 = UniformFloatHyperparameter("r2_1", lower=0.01, upper=0.99, log=False) 81 | r2_2 = UniformFloatHyperparameter("r2_2", lower=0.01, upper=0.99, log=False) 82 | 83 | r3_1 = UniformFloatHyperparameter("r3_1", lower=0.01, upper=0.99, log=False) 84 | r3_2 = UniformFloatHyperparameter("r3_2", lower=0.01, upper=0.99, log=False) 85 | 86 | r4_1 = UniformFloatHyperparameter("r4_1", lower=0.01, upper=0.99, log=False) 87 | r4_2 = UniformFloatHyperparameter("r4_2", lower=0.01, upper=0.99, log=False) 88 | 89 | r5_1 = UniformFloatHyperparameter("r5_1", lower=0.01, upper=0.99, log=False) 90 | r5_2 = UniformFloatHyperparameter("r5_2", lower=0.01, upper=0.99, log=False) 91 | 92 | r6_1 = UniformFloatHyperparameter("r6_1", lower=0.01, upper=0.99, log=False) 93 | r6_2 = UniformFloatHyperparameter("r6_2", lower=0.01, upper=0.99, log=False) 94 | 95 | r7_1 = UniformFloatHyperparameter("r7_1", lower=0.01, upper=0.99, log=False) 96 | r7_2 = UniformFloatHyperparameter("r7_2", lower=0.01, upper=0.99, log=False) 97 | 98 | r8_1 = UniformFloatHyperparameter("r8_1", lower=0.01, upper=0.99, log=False) 99 | r8_2 = UniformFloatHyperparameter("r8_2", lower=0.01, upper=0.99, log=False) 100 | 101 | r9_1 = UniformFloatHyperparameter("r9_1", lower=0.01, upper=0.99, log=False) 102 | r9_2 = UniformFloatHyperparameter("r9_2", lower=0.01, upper=0.99, log=False) 103 | 104 | r10_1 = UniformFloatHyperparameter("r10_1", lower=0.01, upper=0.99, log=False) 105 | r10_2 = UniformFloatHyperparameter("r10_2", lower=0.01, upper=0.99, log=False) 106 | 107 | r11_1 = UniformFloatHyperparameter("r11_1", lower=0.01, upper=0.99, log=False) 108 | r11_2 = UniformFloatHyperparameter("r11_2", lower=0.01, upper=0.99, log=False) 109 | 110 | r12_1 = UniformFloatHyperparameter("r12_1", lower=0.01, upper=0.99, log=False) 111 | r12_2 = UniformFloatHyperparameter("r12_2", lower=0.01, upper=0.99, log=False) 112 | 113 | r13_1 = UniformFloatHyperparameter("r13_1", lower=0.01, upper=0.99, log=False) 114 | r13_2 = UniformFloatHyperparameter("r13_2", lower=0.01, upper=0.99, log=False) 115 | 116 | r14_1 = UniformFloatHyperparameter("r14_1", lower=0.01, upper=0.99, log=False) 117 | r14_2 = UniformFloatHyperparameter("r14_2", lower=0.01, upper=0.99, log=False) 118 | 119 | cs.add_hyperparameters( 120 | [ 121 | root, 122 | x1, 123 | x2, 124 | x3, 125 | x4, 126 | x5, 127 | x6, 128 | r1_1, 129 | r1_2, 130 | r2_1, 131 | r2_2, 132 | r3_1, 133 | r3_2, 134 | r4_1, 135 | r4_2, 136 | r5_1, 137 | r5_2, 138 | r6_1, 139 | r6_2, 140 | r7_1, 141 | r7_2, 142 | r8_1, 143 | r8_2, 144 | r9_1, 145 | r9_2, 146 | r10_1, 147 | r10_2, 148 | r11_1, 149 | r11_2, 150 | r12_1, 151 | r12_2, 152 | r13_1, 153 | r13_2, 154 | r14_1, 155 | r14_2, 156 | ] 157 | ) 158 | 159 | # add condition 160 | cs.add_condition(InCondition(x1, root, ["elu"])) 161 | cs.add_condition(InCondition(x2, root, ["leaky"])) 162 | cs.add_condition(InCondition(r1_1, root, ["elu"])) 163 | cs.add_condition(InCondition(r1_2, root, ["elu"])) 164 | cs.add_condition(InCondition(r2_1, root, ["leaky"])) 165 | cs.add_condition(InCondition(r2_2, root, ["leaky"])) 166 | 167 | cs.add_condition(InCondition(x3, x1, ["elu"])) 168 | cs.add_condition(InCondition(x4, x1, ["leaky"])) 169 | cs.add_condition(InCondition(r3_1, x1, ["elu"])) 170 | cs.add_condition(InCondition(r3_2, x1, ["elu"])) 171 | cs.add_condition(InCondition(r4_1, x1, ["leaky"])) 172 | cs.add_condition(InCondition(r4_2, x1, ["leaky"])) 173 | 174 | cs.add_condition(InCondition(x5, x2, ["elu"])) 175 | cs.add_condition(InCondition(x6, x2, ["leaky"])) 176 | cs.add_condition(InCondition(r5_1, x2, ["elu"])) 177 | cs.add_condition(InCondition(r5_2, x2, ["elu"])) 178 | cs.add_condition(InCondition(r6_1, x2, ["leaky"])) 179 | cs.add_condition(InCondition(r6_2, x2, ["leaky"])) 180 | 181 | cs.add_condition(InCondition(r7_1, x3, ["elu"])) 182 | cs.add_condition(InCondition(r7_2, x3, ["elu"])) 183 | cs.add_condition(InCondition(r8_1, x3, ["leaky"])) 184 | cs.add_condition(InCondition(r8_2, x3, ["leaky"])) 185 | 186 | cs.add_condition(InCondition(r9_1, x4, ["elu"])) 187 | cs.add_condition(InCondition(r9_2, x4, ["elu"])) 188 | cs.add_condition(InCondition(r10_1, x4, ["leaky"])) 189 | cs.add_condition(InCondition(r10_2, x4, ["leaky"])) 190 | 191 | cs.add_condition(InCondition(r11_1, x5, ["elu"])) 192 | cs.add_condition(InCondition(r11_2, x5, ["elu"])) 193 | cs.add_condition(InCondition(r12_1, x5, ["leaky"])) 194 | cs.add_condition(InCondition(r12_2, x5, ["leaky"])) 195 | 196 | cs.add_condition(InCondition(r13_1, x6, ["elu"])) 197 | cs.add_condition(InCondition(r13_2, x6, ["elu"])) 198 | cs.add_condition(InCondition(r14_1, x6, ["leaky"])) 199 | cs.add_condition(InCondition(r14_2, x6, ["leaky"])) 200 | 201 | return cs 202 | 203 | 204 | def testing_funcparams(): 205 | return { 206 | "b1": {"method": "elu", "amount": [0.637111302400592, 0.538971977797732]}, 207 | "b2": {"method": "elu", "amount": [0.738546994320347, 0.02924100126669598]}, 208 | "b3": {"method": "leaky", "amount": [0.9626645632442054, 0.6387380373273952]}, 209 | } 210 | 211 | 212 | if __name__ == "__main__": 213 | print("vgg16") 214 | cs = get_cs() 215 | print(len(cs.get_hyperparameters())) 216 | tt = cs.sample_configuration() 217 | print(tt) 218 | -------------------------------------------------------------------------------- /examples/space_utils/smac_cs_resnet50_40d.py: -------------------------------------------------------------------------------- 1 | from smac.configspace import ConfigurationSpace 2 | from smac.initial_design.random_configuration_design import RandomConfigurations 3 | from smac.initial_design.latin_hypercube_design import LHDesign 4 | from ConfigSpace.hyperparameters import ( 5 | CategoricalHyperparameter, 6 | UniformFloatHyperparameter, 7 | Constant, 8 | ) 9 | from ConfigSpace.conditions import InCondition 10 | 11 | 12 | def testing_cfg(): 13 | return { 14 | "root": "l1", 15 | "r1_1": 0.5745985057653132, 16 | "r1_2": 0.3914810073523109, 17 | "r1_3": 0.2976723864528515, 18 | "r1_4": 0.2212555286365667, 19 | "x1": "ln", 20 | "r4_1": 0.2268537928350621, 21 | "r4_2": 0.02334693439675707, 22 | "r4_3": 0.30347993498438, 23 | "r4_4": 0.6306302847154929, 24 | "x4": "ln", 25 | "r10_1": 0.5, 26 | "r10_2": 0.5, 27 | "r10_3": 0.5, 28 | "r10_4": 0.5, 29 | } 30 | 31 | 32 | def cfg2funcparams(cfg): 33 | def extract_one_value(keys): 34 | """this function extract ONE value with key in keys from cfg 35 | keys: e.g. ["x1","x2"] 36 | """ 37 | 38 | for k in keys: 39 | if k in cfg.keys(): 40 | return cfg[k] 41 | raise RuntimeError("key not exist") 42 | 43 | def extract_using_prefix(keys): 44 | """this function extract ONE value with key in keys from cfg 45 | keys: e.g. ["r1","r2"] 46 | """ 47 | 48 | cfg_prefix = set([i.split("_")[0] for i in cfg.keys()]) 49 | # e.g. r1->[r1_1, r1_2, r1_3] 50 | for k in keys: 51 | if k in cfg_prefix: 52 | return [cfg[k + "_1"], cfg[k + "_2"], cfg[k + "_3"], cfg[k + "_4"]] 53 | raise RuntimeError("key not exist") 54 | 55 | params = {} 56 | params["layer2"] = {} 57 | params["layer2"]["prune_method"] = cfg["root"] 58 | params["layer2"]["amount"] = extract_using_prefix(["r1", "r2"]) 59 | 60 | params["layer3"] = {} 61 | params["layer3"]["prune_method"] = extract_one_value(["x1", "x2"]) 62 | params["layer3"]["amount"] = extract_using_prefix(["r3", "r4", "r5", "r6"]) 63 | 64 | params["layer4"] = {} 65 | params["layer4"]["prune_method"] = extract_one_value(["x3", "x4", "x5", "x6"]) 66 | params["layer4"]["amount"] = extract_using_prefix( 67 | ["r7", "r8", "r9", "r10", "r11", "r12", "r13", "r14",] 68 | ) 69 | 70 | return params 71 | 72 | 73 | def get_cs(): 74 | cs = ConfigurationSpace() 75 | 76 | root = CategoricalHyperparameter("root", choices=["l1", "ln"]) 77 | x1 = CategoricalHyperparameter("x1", choices=["l1", "ln"]) 78 | x2 = CategoricalHyperparameter("x2", choices=["l1", "ln"]) 79 | x3 = CategoricalHyperparameter("x3", choices=["l1", "ln"]) 80 | x4 = CategoricalHyperparameter("x4", choices=["l1", "ln"]) 81 | x5 = CategoricalHyperparameter("x5", choices=["l1", "ln"]) 82 | x6 = CategoricalHyperparameter("x6", choices=["l1", "ln"]) 83 | 84 | # r1 is the data associated in x1 85 | r1_1 = UniformFloatHyperparameter("r1_1", lower=0.01, upper=0.99, log=False) 86 | r1_2 = UniformFloatHyperparameter("r1_2", lower=0.01, upper=0.99, log=False) 87 | r1_3 = UniformFloatHyperparameter("r1_3", lower=0.01, upper=0.99, log=False) 88 | r1_4 = UniformFloatHyperparameter("r1_4", lower=0.01, upper=0.99, log=False) 89 | 90 | r2_1 = UniformFloatHyperparameter("r2_1", lower=0.01, upper=0.99, log=False) 91 | r2_2 = UniformFloatHyperparameter("r2_2", lower=0.01, upper=0.99, log=False) 92 | r2_3 = UniformFloatHyperparameter("r2_3", lower=0.01, upper=0.99, log=False) 93 | r2_4 = UniformFloatHyperparameter("r2_4", lower=0.01, upper=0.99, log=False) 94 | 95 | r3_1 = UniformFloatHyperparameter("r3_1", lower=0.01, upper=0.99, log=False) 96 | r3_2 = UniformFloatHyperparameter("r3_2", lower=0.01, upper=0.99, log=False) 97 | r3_3 = Constant("r3_3", 0.5) 98 | r3_4 = Constant("r3_4", 0.5) 99 | 100 | r4_1 = UniformFloatHyperparameter("r4_1", lower=0.01, upper=0.99, log=False) 101 | r4_2 = UniformFloatHyperparameter("r4_2", lower=0.01, upper=0.99, log=False) 102 | r4_3 = Constant("r4_3", 0.5) 103 | r4_4 = Constant("r4_4", 0.5) 104 | 105 | r5_1 = UniformFloatHyperparameter("r5_1", lower=0.01, upper=0.99, log=False) 106 | r5_2 = UniformFloatHyperparameter("r5_2", lower=0.01, upper=0.99, log=False) 107 | r5_3 = Constant("r5_3", 0.5) 108 | r5_4 = Constant("r5_4", 0.5) 109 | 110 | r6_1 = UniformFloatHyperparameter("r6_1", lower=0.01, upper=0.99, log=False) 111 | r6_2 = UniformFloatHyperparameter("r6_2", lower=0.01, upper=0.99, log=False) 112 | r6_3 = UniformFloatHyperparameter("r6_3", lower=0.01, upper=0.99, log=False) 113 | r6_4 = Constant("r6_4", 0.5) 114 | 115 | r7_1 = UniformFloatHyperparameter("r7_1", lower=0.01, upper=0.99, log=False) 116 | r7_2 = UniformFloatHyperparameter("r7_2", lower=0.01, upper=0.99, log=False) 117 | r7_3 = Constant("r7_3", 0.5) 118 | r7_4 = Constant("r7_4", 0.5) 119 | 120 | r8_1 = UniformFloatHyperparameter("r8_1", lower=0.01, upper=0.99, log=False) 121 | r8_2 = UniformFloatHyperparameter("r8_2", lower=0.01, upper=0.99, log=False) 122 | r8_3 = Constant("r8_3", 0.5) 123 | r8_4 = Constant("r8_4", 0.5) 124 | 125 | r9_1 = UniformFloatHyperparameter("r9_1", lower=0.01, upper=0.99, log=False) 126 | r9_2 = UniformFloatHyperparameter("r9_2", lower=0.01, upper=0.99, log=False) 127 | r9_3 = Constant("r9_3", 0.5) 128 | r9_4 = Constant("r9_4", 0.5) 129 | 130 | r10_1 = UniformFloatHyperparameter("r10_1", lower=0.01, upper=0.99, log=False) 131 | r10_2 = UniformFloatHyperparameter("r10_2", lower=0.01, upper=0.99, log=False) 132 | r10_3 = Constant("r10_3", 0.5) 133 | r10_4 = Constant("r10_4", 0.5) 134 | 135 | r11_1 = UniformFloatHyperparameter("r11_1", lower=0.01, upper=0.99, log=False) 136 | r11_2 = UniformFloatHyperparameter("r11_2", lower=0.01, upper=0.99, log=False) 137 | r11_3 = Constant("r11_3", 0.5) 138 | r11_4 = Constant("r11_4", 0.5) 139 | 140 | r12_1 = UniformFloatHyperparameter("r12_1", lower=0.01, upper=0.99, log=False) 141 | r12_2 = UniformFloatHyperparameter("r12_2", lower=0.01, upper=0.99, log=False) 142 | r12_3 = Constant("r12_3", 0.5) 143 | r12_4 = Constant("r12_4", 0.5) 144 | 145 | r13_1 = UniformFloatHyperparameter("r13_1", lower=0.01, upper=0.99, log=False) 146 | r13_2 = UniformFloatHyperparameter("r13_2", lower=0.01, upper=0.99, log=False) 147 | r13_3 = Constant("r13_3", 0.5) 148 | r13_4 = Constant("r13_4", 0.5) 149 | 150 | r14_1 = UniformFloatHyperparameter("r14_1", lower=0.01, upper=0.99, log=False) 151 | r14_2 = UniformFloatHyperparameter("r14_2", lower=0.01, upper=0.99, log=False) 152 | r14_3 = Constant("r14_3", 0.5) 153 | r14_4 = Constant("r14_4", 0.5) 154 | 155 | cs.add_hyperparameters( 156 | [ 157 | root, 158 | x1, 159 | x2, 160 | x3, 161 | x4, 162 | x5, 163 | x6, 164 | r1_1, 165 | r1_2, 166 | r1_3, 167 | r1_4, 168 | r2_1, 169 | r2_2, 170 | r2_3, 171 | r2_4, 172 | r3_1, 173 | r3_2, 174 | r3_3, 175 | r3_4, 176 | r4_1, 177 | r4_2, 178 | r4_3, 179 | r4_4, 180 | r5_1, 181 | r5_2, 182 | r5_3, 183 | r5_4, 184 | r6_1, 185 | r6_2, 186 | r6_3, 187 | r6_4, 188 | r7_1, 189 | r7_2, 190 | r7_3, 191 | r7_4, 192 | r8_1, 193 | r8_2, 194 | r8_3, 195 | r8_4, 196 | r9_1, 197 | r9_2, 198 | r9_3, 199 | r9_4, 200 | r10_1, 201 | r10_2, 202 | r10_3, 203 | r10_4, 204 | r11_1, 205 | r11_2, 206 | r11_3, 207 | r11_4, 208 | r12_1, 209 | r12_2, 210 | r12_3, 211 | r12_4, 212 | r13_1, 213 | r13_2, 214 | r13_3, 215 | r13_4, 216 | r14_1, 217 | r14_2, 218 | r14_3, 219 | r14_4, 220 | ] 221 | ) 222 | 223 | # add condition 224 | cs.add_condition(InCondition(x1, root, ["l1"])) 225 | cs.add_condition(InCondition(x2, root, ["ln"])) 226 | cs.add_condition(InCondition(r1_1, root, ["l1"])) 227 | cs.add_condition(InCondition(r1_2, root, ["l1"])) 228 | cs.add_condition(InCondition(r1_3, root, ["l1"])) 229 | cs.add_condition(InCondition(r1_4, root, ["l1"])) 230 | cs.add_condition(InCondition(r2_1, root, ["ln"])) 231 | cs.add_condition(InCondition(r2_2, root, ["ln"])) 232 | cs.add_condition(InCondition(r2_3, root, ["ln"])) 233 | cs.add_condition(InCondition(r2_4, root, ["ln"])) 234 | 235 | cs.add_condition(InCondition(x3, x1, ["l1"])) 236 | cs.add_condition(InCondition(x4, x1, ["ln"])) 237 | cs.add_condition(InCondition(r3_1, x1, ["l1"])) 238 | cs.add_condition(InCondition(r3_2, x1, ["l1"])) 239 | cs.add_condition(InCondition(r3_3, x1, ["l1"])) 240 | cs.add_condition(InCondition(r3_4, x1, ["l1"])) 241 | cs.add_condition(InCondition(r4_1, x1, ["ln"])) 242 | cs.add_condition(InCondition(r4_2, x1, ["ln"])) 243 | cs.add_condition(InCondition(r4_3, x1, ["ln"])) 244 | cs.add_condition(InCondition(r4_4, x1, ["ln"])) 245 | 246 | cs.add_condition(InCondition(x5, x2, ["l1"])) 247 | cs.add_condition(InCondition(x6, x2, ["ln"])) 248 | cs.add_condition(InCondition(r5_1, x2, ["l1"])) 249 | cs.add_condition(InCondition(r5_2, x2, ["l1"])) 250 | cs.add_condition(InCondition(r5_3, x2, ["l1"])) 251 | cs.add_condition(InCondition(r5_4, x2, ["l1"])) 252 | cs.add_condition(InCondition(r6_1, x2, ["ln"])) 253 | cs.add_condition(InCondition(r6_2, x2, ["ln"])) 254 | cs.add_condition(InCondition(r6_3, x2, ["ln"])) 255 | cs.add_condition(InCondition(r6_4, x2, ["ln"])) 256 | 257 | cs.add_condition(InCondition(r7_1, x3, ["l1"])) 258 | cs.add_condition(InCondition(r7_2, x3, ["l1"])) 259 | cs.add_condition(InCondition(r7_3, x3, ["l1"])) 260 | cs.add_condition(InCondition(r7_4, x3, ["l1"])) 261 | cs.add_condition(InCondition(r8_1, x3, ["ln"])) 262 | cs.add_condition(InCondition(r8_2, x3, ["ln"])) 263 | cs.add_condition(InCondition(r8_3, x3, ["ln"])) 264 | cs.add_condition(InCondition(r8_4, x3, ["ln"])) 265 | 266 | cs.add_condition(InCondition(r9_1, x4, ["l1"])) 267 | cs.add_condition(InCondition(r9_2, x4, ["l1"])) 268 | cs.add_condition(InCondition(r9_3, x4, ["l1"])) 269 | cs.add_condition(InCondition(r9_4, x4, ["l1"])) 270 | cs.add_condition(InCondition(r10_1, x4, ["ln"])) 271 | cs.add_condition(InCondition(r10_2, x4, ["ln"])) 272 | cs.add_condition(InCondition(r10_3, x4, ["ln"])) 273 | cs.add_condition(InCondition(r10_4, x4, ["ln"])) 274 | 275 | cs.add_condition(InCondition(r11_1, x5, ["l1"])) 276 | cs.add_condition(InCondition(r11_2, x5, ["l1"])) 277 | cs.add_condition(InCondition(r11_3, x5, ["l1"])) 278 | cs.add_condition(InCondition(r11_4, x5, ["l1"])) 279 | cs.add_condition(InCondition(r12_1, x5, ["ln"])) 280 | cs.add_condition(InCondition(r12_2, x5, ["ln"])) 281 | cs.add_condition(InCondition(r12_3, x5, ["ln"])) 282 | cs.add_condition(InCondition(r12_4, x5, ["ln"])) 283 | 284 | cs.add_condition(InCondition(r13_1, x6, ["l1"])) 285 | cs.add_condition(InCondition(r13_2, x6, ["l1"])) 286 | cs.add_condition(InCondition(r13_3, x6, ["l1"])) 287 | cs.add_condition(InCondition(r13_4, x6, ["l1"])) 288 | cs.add_condition(InCondition(r14_1, x6, ["ln"])) 289 | cs.add_condition(InCondition(r14_2, x6, ["ln"])) 290 | cs.add_condition(InCondition(r14_3, x6, ["ln"])) 291 | cs.add_condition(InCondition(r14_4, x6, ["ln"])) 292 | 293 | return cs 294 | 295 | if __name__ == '__main__': 296 | print("resnet50") 297 | cs = get_cs() 298 | print(len(cs.get_hyperparameters())) 299 | -------------------------------------------------------------------------------- /examples/space_utils/smac_cs_resnet50_multiple.py: -------------------------------------------------------------------------------- 1 | from smac.configspace import ConfigurationSpace 2 | from smac.initial_design.random_configuration_design import RandomConfigurations 3 | from smac.initial_design.latin_hypercube_design import LHDesign 4 | from ConfigSpace.hyperparameters import ( 5 | CategoricalHyperparameter, 6 | UniformFloatHyperparameter, 7 | ) 8 | from ConfigSpace.conditions import InCondition 9 | 10 | 11 | def testing_cfg(): 12 | 13 | return { 14 | "root": "l1", 15 | "r1_1": 0.9124839121926617, 16 | "r1_2": 0.7678782085239745, 17 | "r1_3": 0.041539827904847074, 18 | "r1_4": 0.5809694732655409, 19 | "x1": "ln", 20 | "r4_1": 0.7405647986319563, 21 | "r4_2": 0.9477423229071988, 22 | "r4_3": 0.6850737142421288, 23 | "r4_4": 0.748929062010127, 24 | "x4": "ln", 25 | "r10_1": 0.3643284540251601, 26 | "r10_2": 0.2584210597988615, 27 | "r10_3": 0.42303151448861476, 28 | "r10_4": 0.5289414622975467, 29 | } 30 | 31 | 32 | def cfg2funcparams(cfg): 33 | def extract_one_value(keys): 34 | """this function extract ONE value with key in keys from cfg 35 | keys: e.g. ["x1","x2"] 36 | """ 37 | 38 | for k in keys: 39 | if k in cfg.keys(): 40 | return cfg[k] 41 | raise RuntimeError("key not exist") 42 | 43 | def extract_using_prefix(keys): 44 | """this function extract ONE value with key in keys from cfg 45 | keys: e.g. ["r1","r2"] 46 | """ 47 | 48 | cfg_prefix = set([i.split("_")[0] for i in cfg.keys()]) 49 | # e.g. r1->[r1_1, r1_2, r1_3] 50 | for k in keys: 51 | if k in cfg_prefix: 52 | return [cfg[k + "_1"], cfg[k + "_2"], cfg[k + "_3"], cfg[k + "_4"]] 53 | raise RuntimeError("key not exist") 54 | 55 | params = {} 56 | params["layer2"] = {} 57 | params["layer2"]["prune_method"] = cfg["root"] 58 | params["layer2"]["amount"] = extract_using_prefix(["r1", "r2"]) 59 | 60 | params["layer3"] = {} 61 | params["layer3"]["prune_method"] = extract_one_value(["x1", "x2"]) 62 | params["layer3"]["amount"] = extract_using_prefix(["r3", "r4", "r5", "r6"]) 63 | 64 | params["layer4"] = {} 65 | params["layer4"]["prune_method"] = extract_one_value(["x3", "x4", "x5", "x6"]) 66 | params["layer4"]["amount"] = extract_using_prefix( 67 | ["r7", "r8", "r9", "r10", "r11", "r12", "r13", "r14",] 68 | ) 69 | 70 | return params 71 | 72 | 73 | def get_cs(): 74 | cs = ConfigurationSpace() 75 | 76 | root = CategoricalHyperparameter("root", choices=["l1", "ln"]) 77 | x1 = CategoricalHyperparameter("x1", choices=["l1", "ln"]) 78 | x2 = CategoricalHyperparameter("x2", choices=["l1", "ln"]) 79 | x3 = CategoricalHyperparameter("x3", choices=["l1", "ln"]) 80 | x4 = CategoricalHyperparameter("x4", choices=["l1", "ln"]) 81 | x5 = CategoricalHyperparameter("x5", choices=["l1", "ln"]) 82 | x6 = CategoricalHyperparameter("x6", choices=["l1", "ln"]) 83 | 84 | # r1 is the data associated in x1 85 | r1_1 = UniformFloatHyperparameter("r1_1", lower=0.01, upper=0.99, log=False) 86 | r1_2 = UniformFloatHyperparameter("r1_2", lower=0.01, upper=0.99, log=False) 87 | r1_3 = UniformFloatHyperparameter("r1_3", lower=0.01, upper=0.99, log=False) 88 | r1_4 = UniformFloatHyperparameter("r1_4", lower=0.01, upper=0.99, log=False) 89 | 90 | r2_1 = UniformFloatHyperparameter("r2_1", lower=0.01, upper=0.99, log=False) 91 | r2_2 = UniformFloatHyperparameter("r2_2", lower=0.01, upper=0.99, log=False) 92 | r2_3 = UniformFloatHyperparameter("r2_3", lower=0.01, upper=0.99, log=False) 93 | r2_4 = UniformFloatHyperparameter("r2_4", lower=0.01, upper=0.99, log=False) 94 | 95 | r3_1 = UniformFloatHyperparameter("r3_1", lower=0.01, upper=0.99, log=False) 96 | r3_2 = UniformFloatHyperparameter("r3_2", lower=0.01, upper=0.99, log=False) 97 | r3_3 = UniformFloatHyperparameter("r3_3", lower=0.01, upper=0.99, log=False) 98 | r3_4 = UniformFloatHyperparameter("r3_4", lower=0.01, upper=0.99, log=False) 99 | 100 | r4_1 = UniformFloatHyperparameter("r4_1", lower=0.01, upper=0.99, log=False) 101 | r4_2 = UniformFloatHyperparameter("r4_2", lower=0.01, upper=0.99, log=False) 102 | r4_3 = UniformFloatHyperparameter("r4_3", lower=0.01, upper=0.99, log=False) 103 | r4_4 = UniformFloatHyperparameter("r4_4", lower=0.01, upper=0.99, log=False) 104 | 105 | r5_1 = UniformFloatHyperparameter("r5_1", lower=0.01, upper=0.99, log=False) 106 | r5_2 = UniformFloatHyperparameter("r5_2", lower=0.01, upper=0.99, log=False) 107 | r5_3 = UniformFloatHyperparameter("r5_3", lower=0.01, upper=0.99, log=False) 108 | r5_4 = UniformFloatHyperparameter("r5_4", lower=0.01, upper=0.99, log=False) 109 | 110 | r6_1 = UniformFloatHyperparameter("r6_1", lower=0.01, upper=0.99, log=False) 111 | r6_2 = UniformFloatHyperparameter("r6_2", lower=0.01, upper=0.99, log=False) 112 | r6_3 = UniformFloatHyperparameter("r6_3", lower=0.01, upper=0.99, log=False) 113 | r6_4 = UniformFloatHyperparameter("r6_4", lower=0.01, upper=0.99, log=False) 114 | 115 | r7_1 = UniformFloatHyperparameter("r7_1", lower=0.01, upper=0.99, log=False) 116 | r7_2 = UniformFloatHyperparameter("r7_2", lower=0.01, upper=0.99, log=False) 117 | r7_3 = UniformFloatHyperparameter("r7_3", lower=0.01, upper=0.99, log=False) 118 | r7_4 = UniformFloatHyperparameter("r7_4", lower=0.01, upper=0.99, log=False) 119 | 120 | r8_1 = UniformFloatHyperparameter("r8_1", lower=0.01, upper=0.99, log=False) 121 | r8_2 = UniformFloatHyperparameter("r8_2", lower=0.01, upper=0.99, log=False) 122 | r8_3 = UniformFloatHyperparameter("r8_3", lower=0.01, upper=0.99, log=False) 123 | r8_4 = UniformFloatHyperparameter("r8_4", lower=0.01, upper=0.99, log=False) 124 | 125 | r9_1 = UniformFloatHyperparameter("r9_1", lower=0.01, upper=0.99, log=False) 126 | r9_2 = UniformFloatHyperparameter("r9_2", lower=0.01, upper=0.99, log=False) 127 | r9_3 = UniformFloatHyperparameter("r9_3", lower=0.01, upper=0.99, log=False) 128 | r9_4 = UniformFloatHyperparameter("r9_4", lower=0.01, upper=0.99, log=False) 129 | 130 | r10_1 = UniformFloatHyperparameter("r10_1", lower=0.01, upper=0.99, log=False) 131 | r10_2 = UniformFloatHyperparameter("r10_2", lower=0.01, upper=0.99, log=False) 132 | r10_3 = UniformFloatHyperparameter("r10_3", lower=0.01, upper=0.99, log=False) 133 | r10_4 = UniformFloatHyperparameter("r10_4", lower=0.01, upper=0.99, log=False) 134 | 135 | r11_1 = UniformFloatHyperparameter("r11_1", lower=0.01, upper=0.99, log=False) 136 | r11_2 = UniformFloatHyperparameter("r11_2", lower=0.01, upper=0.99, log=False) 137 | r11_3 = UniformFloatHyperparameter("r11_3", lower=0.01, upper=0.99, log=False) 138 | r11_4 = UniformFloatHyperparameter("r11_4", lower=0.01, upper=0.99, log=False) 139 | 140 | r12_1 = UniformFloatHyperparameter("r12_1", lower=0.01, upper=0.99, log=False) 141 | r12_2 = UniformFloatHyperparameter("r12_2", lower=0.01, upper=0.99, log=False) 142 | r12_3 = UniformFloatHyperparameter("r12_3", lower=0.01, upper=0.99, log=False) 143 | r12_4 = UniformFloatHyperparameter("r12_4", lower=0.01, upper=0.99, log=False) 144 | 145 | r13_1 = UniformFloatHyperparameter("r13_1", lower=0.01, upper=0.99, log=False) 146 | r13_2 = UniformFloatHyperparameter("r13_2", lower=0.01, upper=0.99, log=False) 147 | r13_3 = UniformFloatHyperparameter("r13_3", lower=0.01, upper=0.99, log=False) 148 | r13_4 = UniformFloatHyperparameter("r13_4", lower=0.01, upper=0.99, log=False) 149 | 150 | r14_1 = UniformFloatHyperparameter("r14_1", lower=0.01, upper=0.99, log=False) 151 | r14_2 = UniformFloatHyperparameter("r14_2", lower=0.01, upper=0.99, log=False) 152 | r14_3 = UniformFloatHyperparameter("r14_3", lower=0.01, upper=0.99, log=False) 153 | r14_4 = UniformFloatHyperparameter("r14_4", lower=0.01, upper=0.99, log=False) 154 | 155 | cs.add_hyperparameters( 156 | [ 157 | root, 158 | x1, 159 | x2, 160 | x3, 161 | x4, 162 | x5, 163 | x6, 164 | r1_1, 165 | r1_2, 166 | r1_3, 167 | r1_4, 168 | r2_1, 169 | r2_2, 170 | r2_3, 171 | r2_4, 172 | r3_1, 173 | r3_2, 174 | r3_3, 175 | r3_4, 176 | r4_1, 177 | r4_2, 178 | r4_3, 179 | r4_4, 180 | r5_1, 181 | r5_2, 182 | r5_3, 183 | r5_4, 184 | r6_1, 185 | r6_2, 186 | r6_3, 187 | r6_4, 188 | r7_1, 189 | r7_2, 190 | r7_3, 191 | r7_4, 192 | r8_1, 193 | r8_2, 194 | r8_3, 195 | r8_4, 196 | r9_1, 197 | r9_2, 198 | r9_3, 199 | r9_4, 200 | r10_1, 201 | r10_2, 202 | r10_3, 203 | r10_4, 204 | r11_1, 205 | r11_2, 206 | r11_3, 207 | r11_4, 208 | r12_1, 209 | r12_2, 210 | r12_3, 211 | r12_4, 212 | r13_1, 213 | r13_2, 214 | r13_3, 215 | r13_4, 216 | r14_1, 217 | r14_2, 218 | r14_3, 219 | r14_4, 220 | ] 221 | ) 222 | 223 | # add condition 224 | cs.add_condition(InCondition(x1, root, ["l1"])) 225 | cs.add_condition(InCondition(x2, root, ["ln"])) 226 | cs.add_condition(InCondition(r1_1, root, ["l1"])) 227 | cs.add_condition(InCondition(r1_2, root, ["l1"])) 228 | cs.add_condition(InCondition(r1_3, root, ["l1"])) 229 | cs.add_condition(InCondition(r1_4, root, ["l1"])) 230 | cs.add_condition(InCondition(r2_1, root, ["ln"])) 231 | cs.add_condition(InCondition(r2_2, root, ["ln"])) 232 | cs.add_condition(InCondition(r2_3, root, ["ln"])) 233 | cs.add_condition(InCondition(r2_4, root, ["ln"])) 234 | 235 | cs.add_condition(InCondition(x3, x1, ["l1"])) 236 | cs.add_condition(InCondition(x4, x1, ["ln"])) 237 | cs.add_condition(InCondition(r3_1, x1, ["l1"])) 238 | cs.add_condition(InCondition(r3_2, x1, ["l1"])) 239 | cs.add_condition(InCondition(r3_3, x1, ["l1"])) 240 | cs.add_condition(InCondition(r3_4, x1, ["l1"])) 241 | cs.add_condition(InCondition(r4_1, x1, ["ln"])) 242 | cs.add_condition(InCondition(r4_2, x1, ["ln"])) 243 | cs.add_condition(InCondition(r4_3, x1, ["ln"])) 244 | cs.add_condition(InCondition(r4_4, x1, ["ln"])) 245 | 246 | cs.add_condition(InCondition(x5, x2, ["l1"])) 247 | cs.add_condition(InCondition(x6, x2, ["ln"])) 248 | cs.add_condition(InCondition(r5_1, x2, ["l1"])) 249 | cs.add_condition(InCondition(r5_2, x2, ["l1"])) 250 | cs.add_condition(InCondition(r5_3, x2, ["l1"])) 251 | cs.add_condition(InCondition(r5_4, x2, ["l1"])) 252 | cs.add_condition(InCondition(r6_1, x2, ["ln"])) 253 | cs.add_condition(InCondition(r6_2, x2, ["ln"])) 254 | cs.add_condition(InCondition(r6_3, x2, ["ln"])) 255 | cs.add_condition(InCondition(r6_4, x2, ["ln"])) 256 | 257 | cs.add_condition(InCondition(r7_1, x3, ["l1"])) 258 | cs.add_condition(InCondition(r7_2, x3, ["l1"])) 259 | cs.add_condition(InCondition(r7_3, x3, ["l1"])) 260 | cs.add_condition(InCondition(r7_4, x3, ["l1"])) 261 | cs.add_condition(InCondition(r8_1, x3, ["ln"])) 262 | cs.add_condition(InCondition(r8_2, x3, ["ln"])) 263 | cs.add_condition(InCondition(r8_3, x3, ["ln"])) 264 | cs.add_condition(InCondition(r8_4, x3, ["ln"])) 265 | 266 | cs.add_condition(InCondition(r9_1, x4, ["l1"])) 267 | cs.add_condition(InCondition(r9_2, x4, ["l1"])) 268 | cs.add_condition(InCondition(r9_3, x4, ["l1"])) 269 | cs.add_condition(InCondition(r9_4, x4, ["l1"])) 270 | cs.add_condition(InCondition(r10_1, x4, ["ln"])) 271 | cs.add_condition(InCondition(r10_2, x4, ["ln"])) 272 | cs.add_condition(InCondition(r10_3, x4, ["ln"])) 273 | cs.add_condition(InCondition(r10_4, x4, ["ln"])) 274 | 275 | cs.add_condition(InCondition(r11_1, x5, ["l1"])) 276 | cs.add_condition(InCondition(r11_2, x5, ["l1"])) 277 | cs.add_condition(InCondition(r11_3, x5, ["l1"])) 278 | cs.add_condition(InCondition(r11_4, x5, ["l1"])) 279 | cs.add_condition(InCondition(r12_1, x5, ["ln"])) 280 | cs.add_condition(InCondition(r12_2, x5, ["ln"])) 281 | cs.add_condition(InCondition(r12_3, x5, ["ln"])) 282 | cs.add_condition(InCondition(r12_4, x5, ["ln"])) 283 | 284 | cs.add_condition(InCondition(r13_1, x6, ["l1"])) 285 | cs.add_condition(InCondition(r13_2, x6, ["l1"])) 286 | cs.add_condition(InCondition(r13_3, x6, ["l1"])) 287 | cs.add_condition(InCondition(r13_4, x6, ["l1"])) 288 | cs.add_condition(InCondition(r14_1, x6, ["ln"])) 289 | cs.add_condition(InCondition(r14_2, x6, ["ln"])) 290 | cs.add_condition(InCondition(r14_3, x6, ["ln"])) 291 | cs.add_condition(InCondition(r14_4, x6, ["ln"])) 292 | 293 | return cs 294 | 295 | if __name__ == '__main__': 296 | print("resnet50") 297 | cs = get_cs() 298 | print(len(cs.get_hyperparameters())) 299 | -------------------------------------------------------------------------------- /examples/space_utils/smac_cs_resnet56.py: -------------------------------------------------------------------------------- 1 | from smac.configspace import ConfigurationSpace 2 | from smac.initial_design.random_configuration_design import RandomConfigurations 3 | from smac.initial_design.latin_hypercube_design import LHDesign 4 | from ConfigSpace.hyperparameters import ( 5 | CategoricalHyperparameter, 6 | UniformFloatHyperparameter, 7 | ) 8 | from ConfigSpace.conditions import InCondition 9 | 10 | 11 | def testing_cfg(): 12 | return { 13 | "root": "l1", 14 | "r1_1": 0.7325497717745826, 15 | "r1_2": 0.34164504389612943, 16 | "r1_3": 0.5211620038232283, 17 | "x1": "ln", 18 | "r4_1": 0.40540439341197976, 19 | "r4_2": 0.01476505741787909, 20 | "r4_3": 0.031323481210716375, 21 | "x4": "ln", 22 | "r10_1": 0.5298709384652244, 23 | "r10_2": 0.5616403538459042, 24 | "r10_3": 0.5043322521730372, 25 | } 26 | 27 | 28 | def cfg2funcparams(cfg): 29 | def extract_one_value(keys): 30 | """this function extract ONE value with key in keys from cfg 31 | keys: e.g. ["x1","x2"] 32 | """ 33 | 34 | for k in keys: 35 | if k in cfg.keys(): 36 | return cfg[k] 37 | raise RuntimeError("key not exist") 38 | 39 | def extract_using_prefix(keys): 40 | """this function extract ONE value with key in keys from cfg 41 | keys: e.g. ["r1","r2"] 42 | """ 43 | 44 | cfg_prefix = set([i.split("_")[0] for i in cfg.keys()]) 45 | # e.g. r1->[r1_1, r1_2, r1_3] 46 | for k in keys: 47 | if k in cfg_prefix: 48 | return [cfg[k + "_1"], cfg[k + "_2"], cfg[k + "_3"]] 49 | raise RuntimeError("key not exist") 50 | 51 | params = {} 52 | params["layer1"] = {} 53 | params["layer1"]["prune_method"] = cfg["root"] 54 | params["layer1"]["amount"] = extract_using_prefix(["r1", "r2"]) 55 | 56 | params["layer2"] = {} 57 | params["layer2"]["prune_method"] = extract_one_value(["x1", "x2"]) 58 | params["layer2"]["amount"] = extract_using_prefix(["r3", "r4", "r5", "r6"]) 59 | 60 | params["layer3"] = {} 61 | params["layer3"]["prune_method"] = extract_one_value(["x3", "x4", "x5", "x6"]) 62 | params["layer3"]["amount"] = extract_using_prefix( 63 | ["r7", "r8", "r9", "r10", "r11", "r12", "r13", "r14",] 64 | ) 65 | 66 | return params 67 | 68 | 69 | def get_cs(): 70 | cs = ConfigurationSpace() 71 | 72 | root = CategoricalHyperparameter("root", choices=["l1", "ln"]) 73 | x1 = CategoricalHyperparameter("x1", choices=["l1", "ln"]) 74 | x2 = CategoricalHyperparameter("x2", choices=["l1", "ln"]) 75 | x3 = CategoricalHyperparameter("x3", choices=["l1", "ln"]) 76 | x4 = CategoricalHyperparameter("x4", choices=["l1", "ln"]) 77 | x5 = CategoricalHyperparameter("x5", choices=["l1", "ln"]) 78 | x6 = CategoricalHyperparameter("x6", choices=["l1", "ln"]) 79 | 80 | # r1 is the data associated in x1 81 | r1_1 = UniformFloatHyperparameter("r1_1", lower=0.01, upper=0.99, log=False) 82 | r1_2 = UniformFloatHyperparameter("r1_2", lower=0.01, upper=0.99, log=False) 83 | r1_3 = UniformFloatHyperparameter("r1_3", lower=0.01, upper=0.99, log=False) 84 | 85 | r2_1 = UniformFloatHyperparameter("r2_1", lower=0.01, upper=0.99, log=False) 86 | r2_2 = UniformFloatHyperparameter("r2_2", lower=0.01, upper=0.99, log=False) 87 | r2_3 = UniformFloatHyperparameter("r2_3", lower=0.01, upper=0.99, log=False) 88 | 89 | r3_1 = UniformFloatHyperparameter("r3_1", lower=0.01, upper=0.99, log=False) 90 | r3_2 = UniformFloatHyperparameter("r3_2", lower=0.01, upper=0.99, log=False) 91 | r3_3 = UniformFloatHyperparameter("r3_3", lower=0.01, upper=0.99, log=False) 92 | 93 | r4_1 = UniformFloatHyperparameter("r4_1", lower=0.01, upper=0.99, log=False) 94 | r4_2 = UniformFloatHyperparameter("r4_2", lower=0.01, upper=0.99, log=False) 95 | r4_3 = UniformFloatHyperparameter("r4_3", lower=0.01, upper=0.99, log=False) 96 | 97 | r5_1 = UniformFloatHyperparameter("r5_1", lower=0.01, upper=0.99, log=False) 98 | r5_2 = UniformFloatHyperparameter("r5_2", lower=0.01, upper=0.99, log=False) 99 | r5_3 = UniformFloatHyperparameter("r5_3", lower=0.01, upper=0.99, log=False) 100 | 101 | r6_1 = UniformFloatHyperparameter("r6_1", lower=0.01, upper=0.99, log=False) 102 | r6_2 = UniformFloatHyperparameter("r6_2", lower=0.01, upper=0.99, log=False) 103 | r6_3 = UniformFloatHyperparameter("r6_3", lower=0.01, upper=0.99, log=False) 104 | 105 | r7_1 = UniformFloatHyperparameter("r7_1", lower=0.01, upper=0.99, log=False) 106 | r7_2 = UniformFloatHyperparameter("r7_2", lower=0.01, upper=0.99, log=False) 107 | r7_3 = UniformFloatHyperparameter("r7_3", lower=0.01, upper=0.99, log=False) 108 | 109 | r8_1 = UniformFloatHyperparameter("r8_1", lower=0.01, upper=0.99, log=False) 110 | r8_2 = UniformFloatHyperparameter("r8_2", lower=0.01, upper=0.99, log=False) 111 | r8_3 = UniformFloatHyperparameter("r8_3", lower=0.01, upper=0.99, log=False) 112 | 113 | r9_1 = UniformFloatHyperparameter("r9_1", lower=0.01, upper=0.99, log=False) 114 | r9_2 = UniformFloatHyperparameter("r9_2", lower=0.01, upper=0.99, log=False) 115 | r9_3 = UniformFloatHyperparameter("r9_3", lower=0.01, upper=0.99, log=False) 116 | 117 | r10_1 = UniformFloatHyperparameter("r10_1", lower=0.01, upper=0.99, log=False) 118 | r10_2 = UniformFloatHyperparameter("r10_2", lower=0.01, upper=0.99, log=False) 119 | r10_3 = UniformFloatHyperparameter("r10_3", lower=0.01, upper=0.99, log=False) 120 | 121 | r11_1 = UniformFloatHyperparameter("r11_1", lower=0.01, upper=0.99, log=False) 122 | r11_2 = UniformFloatHyperparameter("r11_2", lower=0.01, upper=0.99, log=False) 123 | r11_3 = UniformFloatHyperparameter("r11_3", lower=0.01, upper=0.99, log=False) 124 | 125 | r12_1 = UniformFloatHyperparameter("r12_1", lower=0.01, upper=0.99, log=False) 126 | r12_2 = UniformFloatHyperparameter("r12_2", lower=0.01, upper=0.99, log=False) 127 | r12_3 = UniformFloatHyperparameter("r12_3", lower=0.01, upper=0.99, log=False) 128 | 129 | r13_1 = UniformFloatHyperparameter("r13_1", lower=0.01, upper=0.99, log=False) 130 | r13_2 = UniformFloatHyperparameter("r13_2", lower=0.01, upper=0.99, log=False) 131 | r13_3 = UniformFloatHyperparameter("r13_3", lower=0.01, upper=0.99, log=False) 132 | 133 | r14_1 = UniformFloatHyperparameter("r14_1", lower=0.01, upper=0.99, log=False) 134 | r14_2 = UniformFloatHyperparameter("r14_2", lower=0.01, upper=0.99, log=False) 135 | r14_3 = UniformFloatHyperparameter("r14_3", lower=0.01, upper=0.99, log=False) 136 | 137 | cs.add_hyperparameters( 138 | [ 139 | root, 140 | x1, 141 | x2, 142 | x3, 143 | x4, 144 | x5, 145 | x6, 146 | r1_1, 147 | r1_2, 148 | r1_3, 149 | r2_1, 150 | r2_2, 151 | r2_3, 152 | r3_1, 153 | r3_2, 154 | r3_3, 155 | r4_1, 156 | r4_2, 157 | r4_3, 158 | r5_1, 159 | r5_2, 160 | r5_3, 161 | r6_1, 162 | r6_2, 163 | r6_3, 164 | r7_1, 165 | r7_2, 166 | r7_3, 167 | r8_1, 168 | r8_2, 169 | r8_3, 170 | r9_1, 171 | r9_2, 172 | r9_3, 173 | r10_1, 174 | r10_2, 175 | r10_3, 176 | r11_1, 177 | r11_2, 178 | r11_3, 179 | r12_1, 180 | r12_2, 181 | r12_3, 182 | r13_1, 183 | r13_2, 184 | r13_3, 185 | r14_1, 186 | r14_2, 187 | r14_3, 188 | ] 189 | ) 190 | 191 | # add condition 192 | cs.add_condition(InCondition(x1, root, ["l1"])) 193 | cs.add_condition(InCondition(x2, root, ["ln"])) 194 | cs.add_condition(InCondition(r1_1, root, ["l1"])) 195 | cs.add_condition(InCondition(r1_2, root, ["l1"])) 196 | cs.add_condition(InCondition(r1_3, root, ["l1"])) 197 | cs.add_condition(InCondition(r2_1, root, ["ln"])) 198 | cs.add_condition(InCondition(r2_2, root, ["ln"])) 199 | cs.add_condition(InCondition(r2_3, root, ["ln"])) 200 | 201 | cs.add_condition(InCondition(x3, x1, ["l1"])) 202 | cs.add_condition(InCondition(x4, x1, ["ln"])) 203 | cs.add_condition(InCondition(r3_1, x1, ["l1"])) 204 | cs.add_condition(InCondition(r3_2, x1, ["l1"])) 205 | cs.add_condition(InCondition(r3_3, x1, ["l1"])) 206 | cs.add_condition(InCondition(r4_1, x1, ["ln"])) 207 | cs.add_condition(InCondition(r4_2, x1, ["ln"])) 208 | cs.add_condition(InCondition(r4_3, x1, ["ln"])) 209 | 210 | cs.add_condition(InCondition(x5, x2, ["l1"])) 211 | cs.add_condition(InCondition(x6, x2, ["ln"])) 212 | cs.add_condition(InCondition(r5_1, x2, ["l1"])) 213 | cs.add_condition(InCondition(r5_2, x2, ["l1"])) 214 | cs.add_condition(InCondition(r5_3, x2, ["l1"])) 215 | cs.add_condition(InCondition(r6_1, x2, ["ln"])) 216 | cs.add_condition(InCondition(r6_2, x2, ["ln"])) 217 | cs.add_condition(InCondition(r6_3, x2, ["ln"])) 218 | 219 | cs.add_condition(InCondition(r7_1, x3, ["l1"])) 220 | cs.add_condition(InCondition(r7_2, x3, ["l1"])) 221 | cs.add_condition(InCondition(r7_3, x3, ["l1"])) 222 | cs.add_condition(InCondition(r8_1, x3, ["ln"])) 223 | cs.add_condition(InCondition(r8_2, x3, ["ln"])) 224 | cs.add_condition(InCondition(r8_3, x3, ["ln"])) 225 | 226 | cs.add_condition(InCondition(r9_1, x4, ["l1"])) 227 | cs.add_condition(InCondition(r9_2, x4, ["l1"])) 228 | cs.add_condition(InCondition(r9_3, x4, ["l1"])) 229 | cs.add_condition(InCondition(r10_1, x4, ["ln"])) 230 | cs.add_condition(InCondition(r10_2, x4, ["ln"])) 231 | cs.add_condition(InCondition(r10_3, x4, ["ln"])) 232 | 233 | cs.add_condition(InCondition(r11_1, x5, ["l1"])) 234 | cs.add_condition(InCondition(r11_2, x5, ["l1"])) 235 | cs.add_condition(InCondition(r11_3, x5, ["l1"])) 236 | cs.add_condition(InCondition(r12_1, x5, ["ln"])) 237 | cs.add_condition(InCondition(r12_2, x5, ["ln"])) 238 | cs.add_condition(InCondition(r12_3, x5, ["ln"])) 239 | 240 | cs.add_condition(InCondition(r13_1, x6, ["l1"])) 241 | cs.add_condition(InCondition(r13_2, x6, ["l1"])) 242 | cs.add_condition(InCondition(r13_3, x6, ["l1"])) 243 | cs.add_condition(InCondition(r14_1, x6, ["ln"])) 244 | cs.add_condition(InCondition(r14_2, x6, ["ln"])) 245 | cs.add_condition(InCondition(r14_3, x6, ["ln"])) 246 | 247 | return cs 248 | -------------------------------------------------------------------------------- /examples/space_utils/smac_cs_single.py: -------------------------------------------------------------------------------- 1 | from smac.configspace import ConfigurationSpace 2 | from smac.initial_design.random_configuration_design import RandomConfigurations 3 | from smac.initial_design.latin_hypercube_design import LHDesign 4 | from ConfigSpace.hyperparameters import ( 5 | CategoricalHyperparameter, 6 | UniformFloatHyperparameter, 7 | ) 8 | from ConfigSpace.conditions import InCondition 9 | 10 | 11 | def testing_cfg_single(): 12 | return { 13 | "root": "ln", 14 | "r2": 0.09001820541090022, 15 | "x2": "ln", 16 | "r6": 0.7571221577439017, 17 | "x6": "ln", 18 | "r14": 0.006530118449628031, 19 | } 20 | 21 | 22 | def cfg2funcparams_single(cfg): 23 | """ 24 | """ 25 | 26 | def extract_one_value(keys): 27 | """this function extract ONE value with key in keys from cfg 28 | keys: e.g. ["x1","x2"] 29 | """ 30 | 31 | for k in keys: 32 | if k in cfg.keys(): 33 | return cfg[k] 34 | raise RuntimeError("key not exist") 35 | 36 | params = {} 37 | params["b1"] = {} 38 | params["b1"]["prune_method"] = cfg["root"] 39 | params["b1"]["amount"] = extract_one_value(["r1", "r2"]) 40 | 41 | params["b2"] = {} 42 | params["b2"]["prune_method"] = extract_one_value(["x1", "x2"]) 43 | params["b2"]["amount"] = extract_one_value(["r3", "r4", "r5", "r6"]) 44 | 45 | params["b3"] = {} 46 | params["b3"]["prune_method"] = extract_one_value(["x3", "x4", "x5", "x6"]) 47 | params["b3"]["amount"] = extract_one_value( 48 | ["r7", "r8", "r9", "r10", "r11", "r12", "r13", "r14",] 49 | ) 50 | 51 | return params 52 | 53 | 54 | def cs_single(): 55 | # Build Configuration Space which defines all parameters and their ranges 56 | cs = ConfigurationSpace() 57 | 58 | root = CategoricalHyperparameter("root", choices=["l1", "ln"]) 59 | x1 = CategoricalHyperparameter("x1", choices=["l1", "ln"]) 60 | x2 = CategoricalHyperparameter("x2", choices=["l1", "ln"]) 61 | x3 = CategoricalHyperparameter("x3", choices=["l1", "ln"]) 62 | x4 = CategoricalHyperparameter("x4", choices=["l1", "ln"]) 63 | x5 = CategoricalHyperparameter("x5", choices=["l1", "ln"]) 64 | x6 = CategoricalHyperparameter("x6", choices=["l1", "ln"]) 65 | 66 | # r1 is the data associated in x1 67 | r1 = UniformFloatHyperparameter("r1", lower=0.01, upper=0.99, log=False) 68 | r2 = UniformFloatHyperparameter("r2", lower=0.01, upper=0.99, log=False) 69 | r3 = UniformFloatHyperparameter("r3", lower=0.01, upper=0.99, log=False) 70 | r4 = UniformFloatHyperparameter("r4", lower=0.01, upper=0.99, log=False) 71 | r5 = UniformFloatHyperparameter("r5", lower=0.01, upper=0.99, log=False) 72 | r6 = UniformFloatHyperparameter("r6", lower=0.01, upper=0.99, log=False) 73 | r7 = UniformFloatHyperparameter("r7", lower=0.01, upper=0.99, log=False) 74 | r8 = UniformFloatHyperparameter("r8", lower=0.01, upper=0.99, log=False) 75 | r9 = UniformFloatHyperparameter("r9", lower=0.01, upper=0.99, log=False) 76 | r10 = UniformFloatHyperparameter("r10", lower=0.01, upper=0.99, log=False) 77 | r11 = UniformFloatHyperparameter("r11", lower=0.01, upper=0.99, log=False) 78 | r12 = UniformFloatHyperparameter("r12", lower=0.01, upper=0.99, log=False) 79 | r13 = UniformFloatHyperparameter("r13", lower=0.01, upper=0.99, log=False) 80 | r14 = UniformFloatHyperparameter("r14", lower=0.01, upper=0.99, log=False) 81 | 82 | cs.add_hyperparameters( 83 | [ 84 | root, 85 | x1, 86 | x2, 87 | x3, 88 | x4, 89 | x5, 90 | x6, 91 | r1, 92 | r2, 93 | r3, 94 | r4, 95 | r5, 96 | r6, 97 | r7, 98 | r8, 99 | r9, 100 | r10, 101 | r11, 102 | r12, 103 | r13, 104 | r14, 105 | ] 106 | ) 107 | 108 | # add condition 109 | cs.add_condition(InCondition(x1, root, ["l1"])) 110 | cs.add_condition(InCondition(x2, root, ["ln"])) 111 | cs.add_condition(InCondition(r1, root, ["l1"])) 112 | cs.add_condition(InCondition(r2, root, ["ln"])) 113 | 114 | cs.add_condition(InCondition(x3, x1, ["l1"])) 115 | cs.add_condition(InCondition(x4, x1, ["ln"])) 116 | cs.add_condition(InCondition(r3, x1, ["l1"])) 117 | cs.add_condition(InCondition(r4, x1, ["ln"])) 118 | 119 | cs.add_condition(InCondition(x5, x2, ["l1"])) 120 | cs.add_condition(InCondition(x6, x2, ["ln"])) 121 | cs.add_condition(InCondition(r5, x2, ["l1"])) 122 | cs.add_condition(InCondition(r6, x2, ["ln"])) 123 | 124 | cs.add_condition(InCondition(r7, x3, ["l1"])) 125 | cs.add_condition(InCondition(r8, x3, ["ln"])) 126 | 127 | cs.add_condition(InCondition(r9, x4, ["l1"])) 128 | cs.add_condition(InCondition(r10, x4, ["ln"])) 129 | 130 | cs.add_condition(InCondition(r11, x5, ["l1"])) 131 | cs.add_condition(InCondition(r12, x5, ["ln"])) 132 | 133 | cs.add_condition(InCondition(r13, x6, ["l1"])) 134 | cs.add_condition(InCondition(r14, x6, ["ln"])) 135 | 136 | return cs 137 | -------------------------------------------------------------------------------- /examples/space_utils/smac_cs_vgg16_40d.py: -------------------------------------------------------------------------------- 1 | from smac.configspace import ConfigurationSpace 2 | from smac.initial_design.random_configuration_design import RandomConfigurations 3 | from smac.initial_design.latin_hypercube_design import LHDesign 4 | from ConfigSpace.hyperparameters import ( 5 | CategoricalHyperparameter, 6 | UniformFloatHyperparameter, 7 | Constant, 8 | ) 9 | from ConfigSpace.conditions import InCondition 10 | 11 | 12 | def testing_cfg(): 13 | return { 14 | "root": "l1", 15 | "r1_1": 0.6866000529348212, 16 | "r1_2": 0.7538109366169266, 17 | "r1_3": 0.9126215731867601, 18 | "x1": "ln", 19 | "r4_1": 0.9567669460238617, 20 | "r4_2": 0.6473958655250426, 21 | "r4_3": 0.9829140960106059, 22 | "x4": "l1", 23 | "r9_1": 0.5, 24 | "r9_2": 0.5, 25 | "r9_3": 0.5, 26 | } 27 | 28 | 29 | def cfg2funcparams(cfg): 30 | def extract_one_value(keys): 31 | """this function extract ONE value with key in keys from cfg 32 | keys: e.g. ["x1","x2"] 33 | """ 34 | 35 | for k in keys: 36 | if k in cfg.keys(): 37 | return cfg[k] 38 | raise RuntimeError("key not exist") 39 | 40 | def extract_using_prefix(keys): 41 | """this function extract ONE value with key in keys from cfg 42 | keys: e.g. ["r1","r2"] 43 | """ 44 | 45 | cfg_prefix = set([i.split("_")[0] for i in cfg.keys()]) 46 | # e.g. r1->[r1_1, r1_2, r1_3] 47 | for k in keys: 48 | if k in cfg_prefix: 49 | return [cfg[k + "_1"], cfg[k + "_2"], cfg[k + "_3"]] 50 | raise RuntimeError("key not exist") 51 | 52 | params = {} 53 | params["b1"] = {} 54 | params["b1"]["prune_method"] = cfg["root"] 55 | params["b1"]["amount"] = extract_using_prefix(["r1", "r2"]) 56 | 57 | params["b2"] = {} 58 | params["b2"]["prune_method"] = extract_one_value(["x1", "x2"]) 59 | params["b2"]["amount"] = extract_using_prefix(["r3", "r4", "r5", "r6"]) 60 | 61 | params["b3"] = {} 62 | params["b3"]["prune_method"] = extract_one_value(["x3", "x4", "x5", "x6"]) 63 | params["b3"]["amount"] = extract_using_prefix( 64 | ["r7", "r8", "r9", "r10", "r11", "r12", "r13", "r14",] 65 | ) 66 | 67 | return params 68 | 69 | 70 | def get_cs(): 71 | cs = ConfigurationSpace() 72 | 73 | root = CategoricalHyperparameter("root", choices=["l1", "ln"]) 74 | x1 = CategoricalHyperparameter("x1", choices=["l1", "ln"]) 75 | x2 = CategoricalHyperparameter("x2", choices=["l1", "ln"]) 76 | x3 = CategoricalHyperparameter("x3", choices=["l1", "ln"]) 77 | x4 = CategoricalHyperparameter("x4", choices=["l1", "ln"]) 78 | x5 = CategoricalHyperparameter("x5", choices=["l1", "ln"]) 79 | x6 = CategoricalHyperparameter("x6", choices=["l1", "ln"]) 80 | 81 | # r1 is the data associated in x1 82 | r1_1 = UniformFloatHyperparameter("r1_1", lower=0.01, upper=0.99, log=False) 83 | r1_2 = UniformFloatHyperparameter("r1_2", lower=0.01, upper=0.99, log=False) 84 | r1_3 = UniformFloatHyperparameter("r1_3", lower=0.01, upper=0.99, log=False) 85 | 86 | r2_1 = UniformFloatHyperparameter("r2_1", lower=0.01, upper=0.99, log=False) 87 | r2_2 = UniformFloatHyperparameter("r2_2", lower=0.01, upper=0.99, log=False) 88 | r2_3 = UniformFloatHyperparameter("r2_3", lower=0.01, upper=0.99, log=False) 89 | 90 | r3_1 = UniformFloatHyperparameter("r3_1", lower=0.01, upper=0.99, log=False) 91 | r3_2 = UniformFloatHyperparameter("r3_2", lower=0.01, upper=0.99, log=False) 92 | r3_3 = Constant("r3_3", 0.5) 93 | 94 | r4_1 = UniformFloatHyperparameter("r4_1", lower=0.01, upper=0.99, log=False) 95 | r4_2 = UniformFloatHyperparameter("r4_2", lower=0.01, upper=0.99, log=False) 96 | r4_3 = UniformFloatHyperparameter("r4_3", lower=0.01, upper=0.99, log=False) 97 | 98 | r5_1 = UniformFloatHyperparameter("r5_1", lower=0.01, upper=0.99, log=False) 99 | r5_2 = UniformFloatHyperparameter("r5_2", lower=0.01, upper=0.99, log=False) 100 | r5_3 = UniformFloatHyperparameter("r5_3", lower=0.01, upper=0.99, log=False) 101 | 102 | r6_1 = UniformFloatHyperparameter("r6_1", lower=0.01, upper=0.99, log=False) 103 | r6_2 = UniformFloatHyperparameter("r6_2", lower=0.01, upper=0.99, log=False) 104 | r6_3 = UniformFloatHyperparameter("r6_3", lower=0.01, upper=0.99, log=False) 105 | 106 | r7_1 = UniformFloatHyperparameter("r7_1", lower=0.01, upper=0.99, log=False) 107 | r7_2 = UniformFloatHyperparameter("r7_2", lower=0.01, upper=0.99, log=False) 108 | r7_3 = Constant("r7_3", 0.5) 109 | 110 | r8_1 = UniformFloatHyperparameter("r8_1", lower=0.01, upper=0.99, log=False) 111 | r8_2 = UniformFloatHyperparameter("r8_2", lower=0.01, upper=0.99, log=False) 112 | r8_3 = Constant("r8_3", 0.5) 113 | 114 | r9_1 = UniformFloatHyperparameter("r9_1", lower=0.01, upper=0.99, log=False) 115 | r9_2 = UniformFloatHyperparameter("r9_2", lower=0.01, upper=0.99, log=False) 116 | r9_3 = Constant("r9_3", 0.5) 117 | 118 | r10_1 = UniformFloatHyperparameter("r10_1", lower=0.01, upper=0.99, log=False) 119 | r10_2 = UniformFloatHyperparameter("r10_2", lower=0.01, upper=0.99, log=False) 120 | r10_3 = Constant("r10_3", 0.5) 121 | 122 | r11_1 = UniformFloatHyperparameter("r11_1", lower=0.01, upper=0.99, log=False) 123 | r11_2 = UniformFloatHyperparameter("r11_2", lower=0.01, upper=0.99, log=False) 124 | r11_3 = Constant("r11_3", 0.5) 125 | 126 | r12_1 = UniformFloatHyperparameter("r12_1", lower=0.01, upper=0.99, log=False) 127 | r12_2 = UniformFloatHyperparameter("r12_2", lower=0.01, upper=0.99, log=False) 128 | r12_3 = Constant("r12_3", 0.5) 129 | 130 | r13_1 = UniformFloatHyperparameter("r13_1", lower=0.01, upper=0.99, log=False) 131 | r13_2 = UniformFloatHyperparameter("r13_2", lower=0.01, upper=0.99, log=False) 132 | r13_3 = Constant("r13_3", 0.5) 133 | 134 | r14_1 = UniformFloatHyperparameter("r14_1", lower=0.01, upper=0.99, log=False) 135 | r14_2 = UniformFloatHyperparameter("r14_2", lower=0.01, upper=0.99, log=False) 136 | r14_3 = Constant("r14_3", 0.5) 137 | 138 | cs.add_hyperparameters( 139 | [ 140 | root, 141 | x1, 142 | x2, 143 | x3, 144 | x4, 145 | x5, 146 | x6, 147 | r1_1, 148 | r1_2, 149 | r1_3, 150 | r2_1, 151 | r2_2, 152 | r2_3, 153 | r3_1, 154 | r3_2, 155 | r3_3, 156 | r4_1, 157 | r4_2, 158 | r4_3, 159 | r5_1, 160 | r5_2, 161 | r5_3, 162 | r6_1, 163 | r6_2, 164 | r6_3, 165 | r7_1, 166 | r7_2, 167 | r7_3, 168 | r8_1, 169 | r8_2, 170 | r8_3, 171 | r9_1, 172 | r9_2, 173 | r9_3, 174 | r10_1, 175 | r10_2, 176 | r10_3, 177 | r11_1, 178 | r11_2, 179 | r11_3, 180 | r12_1, 181 | r12_2, 182 | r12_3, 183 | r13_1, 184 | r13_2, 185 | r13_3, 186 | r14_1, 187 | r14_2, 188 | r14_3, 189 | ] 190 | ) 191 | 192 | # add condition 193 | cs.add_condition(InCondition(x1, root, ["l1"])) 194 | cs.add_condition(InCondition(x2, root, ["ln"])) 195 | cs.add_condition(InCondition(r1_1, root, ["l1"])) 196 | cs.add_condition(InCondition(r1_2, root, ["l1"])) 197 | cs.add_condition(InCondition(r1_3, root, ["l1"])) 198 | cs.add_condition(InCondition(r2_1, root, ["ln"])) 199 | cs.add_condition(InCondition(r2_2, root, ["ln"])) 200 | cs.add_condition(InCondition(r2_3, root, ["ln"])) 201 | 202 | cs.add_condition(InCondition(x3, x1, ["l1"])) 203 | cs.add_condition(InCondition(x4, x1, ["ln"])) 204 | cs.add_condition(InCondition(r3_1, x1, ["l1"])) 205 | cs.add_condition(InCondition(r3_2, x1, ["l1"])) 206 | cs.add_condition(InCondition(r3_3, x1, ["l1"])) 207 | cs.add_condition(InCondition(r4_1, x1, ["ln"])) 208 | cs.add_condition(InCondition(r4_2, x1, ["ln"])) 209 | cs.add_condition(InCondition(r4_3, x1, ["ln"])) 210 | 211 | cs.add_condition(InCondition(x5, x2, ["l1"])) 212 | cs.add_condition(InCondition(x6, x2, ["ln"])) 213 | cs.add_condition(InCondition(r5_1, x2, ["l1"])) 214 | cs.add_condition(InCondition(r5_2, x2, ["l1"])) 215 | cs.add_condition(InCondition(r5_3, x2, ["l1"])) 216 | cs.add_condition(InCondition(r6_1, x2, ["ln"])) 217 | cs.add_condition(InCondition(r6_2, x2, ["ln"])) 218 | cs.add_condition(InCondition(r6_3, x2, ["ln"])) 219 | 220 | cs.add_condition(InCondition(r7_1, x3, ["l1"])) 221 | cs.add_condition(InCondition(r7_2, x3, ["l1"])) 222 | cs.add_condition(InCondition(r7_3, x3, ["l1"])) 223 | cs.add_condition(InCondition(r8_1, x3, ["ln"])) 224 | cs.add_condition(InCondition(r8_2, x3, ["ln"])) 225 | cs.add_condition(InCondition(r8_3, x3, ["ln"])) 226 | 227 | cs.add_condition(InCondition(r9_1, x4, ["l1"])) 228 | cs.add_condition(InCondition(r9_2, x4, ["l1"])) 229 | cs.add_condition(InCondition(r9_3, x4, ["l1"])) 230 | cs.add_condition(InCondition(r10_1, x4, ["ln"])) 231 | cs.add_condition(InCondition(r10_2, x4, ["ln"])) 232 | cs.add_condition(InCondition(r10_3, x4, ["ln"])) 233 | 234 | cs.add_condition(InCondition(r11_1, x5, ["l1"])) 235 | cs.add_condition(InCondition(r11_2, x5, ["l1"])) 236 | cs.add_condition(InCondition(r11_3, x5, ["l1"])) 237 | cs.add_condition(InCondition(r12_1, x5, ["ln"])) 238 | cs.add_condition(InCondition(r12_2, x5, ["ln"])) 239 | cs.add_condition(InCondition(r12_3, x5, ["ln"])) 240 | 241 | cs.add_condition(InCondition(r13_1, x6, ["l1"])) 242 | cs.add_condition(InCondition(r13_2, x6, ["l1"])) 243 | cs.add_condition(InCondition(r13_3, x6, ["l1"])) 244 | cs.add_condition(InCondition(r14_1, x6, ["ln"])) 245 | cs.add_condition(InCondition(r14_2, x6, ["ln"])) 246 | cs.add_condition(InCondition(r14_3, x6, ["ln"])) 247 | 248 | return cs 249 | 250 | 251 | if __name__ == '__main__': 252 | print("vgg16") 253 | cs = get_cs() 254 | print(len(cs.get_hyperparameters())) 255 | -------------------------------------------------------------------------------- /examples/space_utils/smac_cs_vgg16_multiple.py: -------------------------------------------------------------------------------- 1 | from smac.configspace import ConfigurationSpace 2 | from smac.initial_design.random_configuration_design import RandomConfigurations 3 | from smac.initial_design.latin_hypercube_design import LHDesign 4 | from ConfigSpace.hyperparameters import ( 5 | CategoricalHyperparameter, 6 | UniformFloatHyperparameter, 7 | ) 8 | from ConfigSpace.conditions import InCondition 9 | 10 | 11 | def testing_cfg(): 12 | return { 13 | "root": "l1", 14 | "r1_1": 0.7205429636854928, 15 | "r1_2": 0.8293958085088828, 16 | "r1_3": 0.22427746515912095, 17 | "x1": "l1", 18 | "r3_1": 0.2952837655706383, 19 | "r3_2": 0.8476091116459952, 20 | "r3_3": 0.988829316498358, 21 | "x3": "l1", 22 | "r7_1": 0.5112305846597317, 23 | "r7_2": 0.43618687843082593, 24 | "r7_3": 0.9085657036330497, 25 | } 26 | 27 | 28 | def cfg2funcparams(cfg): 29 | def extract_one_value(keys): 30 | """this function extract ONE value with key in keys from cfg 31 | keys: e.g. ["x1","x2"] 32 | """ 33 | 34 | for k in keys: 35 | if k in cfg.keys(): 36 | return cfg[k] 37 | raise RuntimeError("key not exist") 38 | 39 | def extract_using_prefix(keys): 40 | """this function extract ONE value with key in keys from cfg 41 | keys: e.g. ["r1","r2"] 42 | """ 43 | 44 | cfg_prefix = set([i.split("_")[0] for i in cfg.keys()]) 45 | # e.g. r1->[r1_1, r1_2, r1_3] 46 | for k in keys: 47 | if k in cfg_prefix: 48 | return [cfg[k + "_1"], cfg[k + "_2"], cfg[k + "_3"]] 49 | raise RuntimeError("key not exist") 50 | 51 | params = {} 52 | params["b1"] = {} 53 | params["b1"]["prune_method"] = cfg["root"] 54 | params["b1"]["amount"] = extract_using_prefix(["r1", "r2"]) 55 | 56 | params["b2"] = {} 57 | params["b2"]["prune_method"] = extract_one_value(["x1", "x2"]) 58 | params["b2"]["amount"] = extract_using_prefix(["r3", "r4", "r5", "r6"]) 59 | 60 | params["b3"] = {} 61 | params["b3"]["prune_method"] = extract_one_value(["x3", "x4", "x5", "x6"]) 62 | params["b3"]["amount"] = extract_using_prefix( 63 | ["r7", "r8", "r9", "r10", "r11", "r12", "r13", "r14",] 64 | ) 65 | 66 | return params 67 | 68 | 69 | def get_cs(): 70 | cs = ConfigurationSpace() 71 | 72 | root = CategoricalHyperparameter("root", choices=["l1", "ln"]) 73 | x1 = CategoricalHyperparameter("x1", choices=["l1", "ln"]) 74 | x2 = CategoricalHyperparameter("x2", choices=["l1", "ln"]) 75 | x3 = CategoricalHyperparameter("x3", choices=["l1", "ln"]) 76 | x4 = CategoricalHyperparameter("x4", choices=["l1", "ln"]) 77 | x5 = CategoricalHyperparameter("x5", choices=["l1", "ln"]) 78 | x6 = CategoricalHyperparameter("x6", choices=["l1", "ln"]) 79 | 80 | # r1 is the data associated in x1 81 | r1_1 = UniformFloatHyperparameter("r1_1", lower=0.01, upper=0.99, log=False) 82 | r1_2 = UniformFloatHyperparameter("r1_2", lower=0.01, upper=0.99, log=False) 83 | r1_3 = UniformFloatHyperparameter("r1_3", lower=0.01, upper=0.99, log=False) 84 | 85 | r2_1 = UniformFloatHyperparameter("r2_1", lower=0.01, upper=0.99, log=False) 86 | r2_2 = UniformFloatHyperparameter("r2_2", lower=0.01, upper=0.99, log=False) 87 | r2_3 = UniformFloatHyperparameter("r2_3", lower=0.01, upper=0.99, log=False) 88 | 89 | r3_1 = UniformFloatHyperparameter("r3_1", lower=0.01, upper=0.99, log=False) 90 | r3_2 = UniformFloatHyperparameter("r3_2", lower=0.01, upper=0.99, log=False) 91 | r3_3 = UniformFloatHyperparameter("r3_3", lower=0.01, upper=0.99, log=False) 92 | 93 | r4_1 = UniformFloatHyperparameter("r4_1", lower=0.01, upper=0.99, log=False) 94 | r4_2 = UniformFloatHyperparameter("r4_2", lower=0.01, upper=0.99, log=False) 95 | r4_3 = UniformFloatHyperparameter("r4_3", lower=0.01, upper=0.99, log=False) 96 | 97 | r5_1 = UniformFloatHyperparameter("r5_1", lower=0.01, upper=0.99, log=False) 98 | r5_2 = UniformFloatHyperparameter("r5_2", lower=0.01, upper=0.99, log=False) 99 | r5_3 = UniformFloatHyperparameter("r5_3", lower=0.01, upper=0.99, log=False) 100 | 101 | r6_1 = UniformFloatHyperparameter("r6_1", lower=0.01, upper=0.99, log=False) 102 | r6_2 = UniformFloatHyperparameter("r6_2", lower=0.01, upper=0.99, log=False) 103 | r6_3 = UniformFloatHyperparameter("r6_3", lower=0.01, upper=0.99, log=False) 104 | 105 | r7_1 = UniformFloatHyperparameter("r7_1", lower=0.01, upper=0.99, log=False) 106 | r7_2 = UniformFloatHyperparameter("r7_2", lower=0.01, upper=0.99, log=False) 107 | r7_3 = UniformFloatHyperparameter("r7_3", lower=0.01, upper=0.99, log=False) 108 | 109 | r8_1 = UniformFloatHyperparameter("r8_1", lower=0.01, upper=0.99, log=False) 110 | r8_2 = UniformFloatHyperparameter("r8_2", lower=0.01, upper=0.99, log=False) 111 | r8_3 = UniformFloatHyperparameter("r8_3", lower=0.01, upper=0.99, log=False) 112 | 113 | r9_1 = UniformFloatHyperparameter("r9_1", lower=0.01, upper=0.99, log=False) 114 | r9_2 = UniformFloatHyperparameter("r9_2", lower=0.01, upper=0.99, log=False) 115 | r9_3 = UniformFloatHyperparameter("r9_3", lower=0.01, upper=0.99, log=False) 116 | 117 | r10_1 = UniformFloatHyperparameter("r10_1", lower=0.01, upper=0.99, log=False) 118 | r10_2 = UniformFloatHyperparameter("r10_2", lower=0.01, upper=0.99, log=False) 119 | r10_3 = UniformFloatHyperparameter("r10_3", lower=0.01, upper=0.99, log=False) 120 | 121 | r11_1 = UniformFloatHyperparameter("r11_1", lower=0.01, upper=0.99, log=False) 122 | r11_2 = UniformFloatHyperparameter("r11_2", lower=0.01, upper=0.99, log=False) 123 | r11_3 = UniformFloatHyperparameter("r11_3", lower=0.01, upper=0.99, log=False) 124 | 125 | r12_1 = UniformFloatHyperparameter("r12_1", lower=0.01, upper=0.99, log=False) 126 | r12_2 = UniformFloatHyperparameter("r12_2", lower=0.01, upper=0.99, log=False) 127 | r12_3 = UniformFloatHyperparameter("r12_3", lower=0.01, upper=0.99, log=False) 128 | 129 | r13_1 = UniformFloatHyperparameter("r13_1", lower=0.01, upper=0.99, log=False) 130 | r13_2 = UniformFloatHyperparameter("r13_2", lower=0.01, upper=0.99, log=False) 131 | r13_3 = UniformFloatHyperparameter("r13_3", lower=0.01, upper=0.99, log=False) 132 | 133 | r14_1 = UniformFloatHyperparameter("r14_1", lower=0.01, upper=0.99, log=False) 134 | r14_2 = UniformFloatHyperparameter("r14_2", lower=0.01, upper=0.99, log=False) 135 | r14_3 = UniformFloatHyperparameter("r14_3", lower=0.01, upper=0.99, log=False) 136 | 137 | cs.add_hyperparameters( 138 | [ 139 | root, 140 | x1, 141 | x2, 142 | x3, 143 | x4, 144 | x5, 145 | x6, 146 | r1_1, 147 | r1_2, 148 | r1_3, 149 | r2_1, 150 | r2_2, 151 | r2_3, 152 | r3_1, 153 | r3_2, 154 | r3_3, 155 | r4_1, 156 | r4_2, 157 | r4_3, 158 | r5_1, 159 | r5_2, 160 | r5_3, 161 | r6_1, 162 | r6_2, 163 | r6_3, 164 | r7_1, 165 | r7_2, 166 | r7_3, 167 | r8_1, 168 | r8_2, 169 | r8_3, 170 | r9_1, 171 | r9_2, 172 | r9_3, 173 | r10_1, 174 | r10_2, 175 | r10_3, 176 | r11_1, 177 | r11_2, 178 | r11_3, 179 | r12_1, 180 | r12_2, 181 | r12_3, 182 | r13_1, 183 | r13_2, 184 | r13_3, 185 | r14_1, 186 | r14_2, 187 | r14_3, 188 | ] 189 | ) 190 | 191 | # add condition 192 | cs.add_condition(InCondition(x1, root, ["l1"])) 193 | cs.add_condition(InCondition(x2, root, ["ln"])) 194 | cs.add_condition(InCondition(r1_1, root, ["l1"])) 195 | cs.add_condition(InCondition(r1_2, root, ["l1"])) 196 | cs.add_condition(InCondition(r1_3, root, ["l1"])) 197 | cs.add_condition(InCondition(r2_1, root, ["ln"])) 198 | cs.add_condition(InCondition(r2_2, root, ["ln"])) 199 | cs.add_condition(InCondition(r2_3, root, ["ln"])) 200 | 201 | cs.add_condition(InCondition(x3, x1, ["l1"])) 202 | cs.add_condition(InCondition(x4, x1, ["ln"])) 203 | cs.add_condition(InCondition(r3_1, x1, ["l1"])) 204 | cs.add_condition(InCondition(r3_2, x1, ["l1"])) 205 | cs.add_condition(InCondition(r3_3, x1, ["l1"])) 206 | cs.add_condition(InCondition(r4_1, x1, ["ln"])) 207 | cs.add_condition(InCondition(r4_2, x1, ["ln"])) 208 | cs.add_condition(InCondition(r4_3, x1, ["ln"])) 209 | 210 | cs.add_condition(InCondition(x5, x2, ["l1"])) 211 | cs.add_condition(InCondition(x6, x2, ["ln"])) 212 | cs.add_condition(InCondition(r5_1, x2, ["l1"])) 213 | cs.add_condition(InCondition(r5_2, x2, ["l1"])) 214 | cs.add_condition(InCondition(r5_3, x2, ["l1"])) 215 | cs.add_condition(InCondition(r6_1, x2, ["ln"])) 216 | cs.add_condition(InCondition(r6_2, x2, ["ln"])) 217 | cs.add_condition(InCondition(r6_3, x2, ["ln"])) 218 | 219 | cs.add_condition(InCondition(r7_1, x3, ["l1"])) 220 | cs.add_condition(InCondition(r7_2, x3, ["l1"])) 221 | cs.add_condition(InCondition(r7_3, x3, ["l1"])) 222 | cs.add_condition(InCondition(r8_1, x3, ["ln"])) 223 | cs.add_condition(InCondition(r8_2, x3, ["ln"])) 224 | cs.add_condition(InCondition(r8_3, x3, ["ln"])) 225 | 226 | cs.add_condition(InCondition(r9_1, x4, ["l1"])) 227 | cs.add_condition(InCondition(r9_2, x4, ["l1"])) 228 | cs.add_condition(InCondition(r9_3, x4, ["l1"])) 229 | cs.add_condition(InCondition(r10_1, x4, ["ln"])) 230 | cs.add_condition(InCondition(r10_2, x4, ["ln"])) 231 | cs.add_condition(InCondition(r10_3, x4, ["ln"])) 232 | 233 | cs.add_condition(InCondition(r11_1, x5, ["l1"])) 234 | cs.add_condition(InCondition(r11_2, x5, ["l1"])) 235 | cs.add_condition(InCondition(r11_3, x5, ["l1"])) 236 | cs.add_condition(InCondition(r12_1, x5, ["ln"])) 237 | cs.add_condition(InCondition(r12_2, x5, ["ln"])) 238 | cs.add_condition(InCondition(r12_3, x5, ["ln"])) 239 | 240 | cs.add_condition(InCondition(r13_1, x6, ["l1"])) 241 | cs.add_condition(InCondition(r13_2, x6, ["l1"])) 242 | cs.add_condition(InCondition(r13_3, x6, ["l1"])) 243 | cs.add_condition(InCondition(r14_1, x6, ["ln"])) 244 | cs.add_condition(InCondition(r14_2, x6, ["ln"])) 245 | cs.add_condition(InCondition(r14_3, x6, ["ln"])) 246 | 247 | return cs 248 | 249 | 250 | if __name__ == "__main__": 251 | print("vgg16") 252 | cs = get_cs() 253 | print(len(cs.get_hyperparameters())) 254 | -------------------------------------------------------------------------------- /examples/space_utils/smac_hpo_facade.py: -------------------------------------------------------------------------------- 1 | import typing 2 | 3 | from smac.facade.smac_ac_facade import SMAC4AC 4 | from smac.runhistory.runhistory2epm import RunHistory2EPM4LogScaledCost 5 | from smac.optimizer.acquisition import LogEI 6 | from smac.epm.rf_with_instances import RandomForestWithInstances 7 | from smac.initial_design.sobol_design import SobolDesign 8 | from smac.initial_design.latin_hypercube_design import LHDesign 9 | 10 | __author__ = "Marius Lindauer" 11 | __copyright__ = "Copyright 2018, ML4AAD" 12 | __license__ = "3-clause BSD" 13 | 14 | 15 | class SMAC4HPO(SMAC4AC): 16 | """ 17 | Facade to use SMAC for hyperparameter optimization 18 | 19 | see smac.facade.smac_Facade for API 20 | This facade overwrites options available via the SMAC facade 21 | 22 | Attributes 23 | ---------- 24 | logger 25 | stats : Stats 26 | solver : SMBO 27 | runhistory : RunHistory 28 | List with information about previous runs 29 | trajectory : list 30 | List of all incumbents 31 | 32 | """ 33 | 34 | def __init__(self, **kwargs: typing.Any): 35 | """ 36 | Constructor 37 | see ~smac.facade.smac_facade for docu 38 | """ 39 | 40 | scenario = kwargs['scenario'] 41 | 42 | if len(scenario.cs.get_hyperparameters()) <= 70: 43 | kwargs['initial_design'] = kwargs.get('initial_design', LHDesign) 44 | else: 45 | raise ValueError( 46 | 'The default initial design "Sobol sequence" can only handle up to 40 dimensions. ' 47 | 'Please use a different initial design, such as "the Latin Hypercube design".', 48 | ) 49 | kwargs['runhistory2epm'] = kwargs.get('runhistory2epm', RunHistory2EPM4LogScaledCost) 50 | 51 | init_kwargs = kwargs.get('initial_design_kwargs', dict()) 52 | init_kwargs['n_configs_x_params'] = init_kwargs.get('n_configs_x_params', 10) 53 | init_kwargs['max_config_fracs'] = init_kwargs.get('max_config_fracs', 0.25) 54 | kwargs['initial_design_kwargs'] = init_kwargs 55 | 56 | # Intensification parameters - which intensifier to use and respective parameters 57 | intensifier_kwargs = kwargs.get('intensifier_kwargs', dict()) 58 | intensifier_kwargs['min_chall'] = 1 59 | kwargs['intensifier_kwargs'] = intensifier_kwargs 60 | scenario.intensification_percentage = 1e-10 61 | 62 | if kwargs.get('model') is None: 63 | model_class = RandomForestWithInstances 64 | kwargs['model'] = model_class 65 | 66 | # == static RF settings 67 | model_kwargs = kwargs.get('model_kwargs', dict()) 68 | model_kwargs['num_trees'] = model_kwargs.get('num_trees', 10) 69 | model_kwargs['do_bootstrapping'] = model_kwargs.get('do_bootstrapping', True) 70 | model_kwargs['ratio_features'] = model_kwargs.get('ratio_features', 1.0) 71 | model_kwargs['min_samples_split'] = model_kwargs.get('min_samples_split', 2) 72 | model_kwargs['min_samples_leaf'] = model_kwargs.get('min_samples_leaf', 1) 73 | model_kwargs['log_y'] = model_kwargs.get('log_y', True) 74 | kwargs['model_kwargs'] = model_kwargs 75 | 76 | # == Acquisition function 77 | kwargs['acquisition_function'] = kwargs.get('acquisition_function', LogEI) 78 | 79 | kwargs['runhistory2epm'] = kwargs.get('runhistory2epm', RunHistory2EPM4LogScaledCost) 80 | 81 | # assumes random chooser for random configs 82 | random_config_chooser_kwargs = kwargs.get('random_configuration_chooser_kwargs', dict()) 83 | random_config_chooser_kwargs['prob'] = random_config_chooser_kwargs.get('prob', 0.2) 84 | kwargs['random_configuration_chooser_kwargs'] = random_config_chooser_kwargs 85 | 86 | # better improve acquisition function optimization 87 | # 1. increase number of sls iterations 88 | acquisition_function_optimizer_kwargs = kwargs.get('acquisition_function_optimizer_kwargs', dict()) 89 | acquisition_function_optimizer_kwargs['n_sls_iterations'] = 10 90 | kwargs['acquisition_function_optimizer_kwargs'] = acquisition_function_optimizer_kwargs 91 | 92 | super().__init__(**kwargs) 93 | self.logger.info(self.__class__) 94 | 95 | # better improve acquisition function optimization 96 | # 2. more randomly sampled configurations 97 | self.solver.scenario.acq_opt_challengers = 10000 # type: ignore[attr-defined] # noqa F821 98 | 99 | # activate predict incumbent 100 | self.solver.epm_chooser.predict_x_best = True 101 | -------------------------------------------------------------------------------- /examples/space_utils/tpe_space_nas_resnet20.py: -------------------------------------------------------------------------------- 1 | from hyperopt import hp 2 | 3 | 4 | def get_space(): 5 | space = hp.choice( 6 | "root", 7 | [ 8 | { 9 | "method": "elu", 10 | "r1_1": hp.uniform("r1_1", 0.01, 0.99), 11 | "r1_2": hp.uniform("r1_2", 0.01, 0.99), 12 | "x1": hp.choice( 13 | "x1", 14 | [ 15 | { 16 | "method": "elu", 17 | "r3_1": hp.uniform("r3_1", 0.01, 0.99), 18 | "r3_2": hp.uniform("r3_2", 0.01, 0.99), 19 | "x3": hp.choice( 20 | "x3", 21 | [ 22 | { 23 | "method": "elu", 24 | "r7_1": hp.uniform("r7_1", 0.01, 0.99), 25 | "r7_2": hp.uniform("r7_2", 0.01, 0.99), 26 | }, 27 | { 28 | "method": "leaky", 29 | "r8_1": hp.uniform("r8_1", 0.01, 0.99), 30 | "r8_2": hp.uniform("r8_2", 0.01, 0.99), 31 | }, 32 | ], 33 | ), 34 | }, 35 | { 36 | "method": "leaky", 37 | "r4_1": hp.uniform("r4_1", 0.01, 0.99), 38 | "r4_2": hp.uniform("r4_2", 0.01, 0.99), 39 | "x4": hp.choice( 40 | "x4", 41 | [ 42 | { 43 | "method": "elu", 44 | "r9_1": hp.uniform("r9_1", 0.01, 0.99), 45 | "r9_2": hp.uniform("r9_2", 0.01, 0.99), 46 | }, 47 | { 48 | "method": "leaky", 49 | "r10_1": hp.uniform("r10_1", 0.01, 0.99), 50 | "r10_2": hp.uniform("r10_2", 0.01, 0.99), 51 | }, 52 | ], 53 | ), 54 | }, 55 | ], 56 | ), 57 | }, 58 | { 59 | "method": "leaky", 60 | "r2_1": hp.uniform("r2_1", 0.01, 0.99), 61 | "r2_2": hp.uniform("r2_2", 0.01, 0.99), 62 | "x2": hp.choice( 63 | "x2", 64 | [ 65 | { 66 | "method": "elu", 67 | "r5_1": hp.uniform("r5_1", 0.01, 0.99), 68 | "r5_2": hp.uniform("r5_2", 0.01, 0.99), 69 | "x5": hp.choice( 70 | "x5", 71 | [ 72 | { 73 | "method": "elu", 74 | "r11_1": hp.uniform("r11_1", 0.01, 0.99), 75 | "r11_2": hp.uniform("r11_2", 0.01, 0.99), 76 | }, 77 | { 78 | "method": "leaky", 79 | "r12_1": hp.uniform("r12_1", 0.01, 0.99), 80 | "r12_2": hp.uniform("r12_2", 0.01, 0.99), 81 | }, 82 | ], 83 | ), 84 | }, 85 | { 86 | "method": "leaky", 87 | "r6_1": hp.uniform("r6_1", 0.01, 0.99), 88 | "r6_2": hp.uniform("r6_2", 0.01, 0.99), 89 | "x6": hp.choice( 90 | "x6", 91 | [ 92 | { 93 | "method": "elu", 94 | "r13_1": hp.uniform("r13_1", 0.01, 0.99), 95 | "r13_2": hp.uniform("r13_2", 0.01, 0.99), 96 | }, 97 | { 98 | "method": "leaky", 99 | "r14_1": hp.uniform("r14_1", 0.01, 0.99), 100 | "r14_2": hp.uniform("r14_2", 0.01, 0.99), 101 | }, 102 | ], 103 | ), 104 | }, 105 | ], 106 | ), 107 | }, 108 | ], 109 | ) 110 | 111 | return space 112 | 113 | 114 | def testing_cfg(): 115 | return { 116 | "method": "leaky", 117 | "r2_1": 0.07067686282157104, 118 | "r2_2": 0.4890780211983336, 119 | "x2": { 120 | "method": "leaky", 121 | "r6_1": 0.5564595351825452, 122 | "r6_2": 0.9401366848897367, 123 | "x6": { 124 | "method": "elu", 125 | "r13_1": 0.012747480675794905, 126 | "r13_2": 0.07904304382102956, 127 | }, 128 | }, 129 | } 130 | 131 | 132 | def cfg2funcparams(cfg): 133 | """ 134 | """ 135 | 136 | def _extract_sufix(ks): 137 | target_ks = [] 138 | for k in ks: 139 | if k.endswith("_1") or k.endswith("_2"): 140 | target_ks.append(k) 141 | return sorted(target_ks) 142 | 143 | def extract_one_layer(dd): 144 | target_ks = _extract_sufix(dd.keys()) 145 | values = [dd[k] for k in target_ks] 146 | extra = [k for k in dd.keys() if k.startswith("x")] 147 | return dd["method"], values, extra 148 | 149 | params = {} 150 | 151 | b1_info = extract_one_layer(cfg) 152 | params["b1"] = {} 153 | params["b1"]["method"] = b1_info[0] 154 | params["b1"]["amount"] = b1_info[1] 155 | 156 | dd = cfg[b1_info[2][0]] 157 | b2_info = extract_one_layer(dd) 158 | params["b2"] = {} 159 | params["b2"]["method"] = b2_info[0] 160 | params["b2"]["amount"] = b2_info[1] 161 | 162 | dd = dd[b2_info[2][0]] 163 | b3_info = extract_one_layer(dd) 164 | params["b3"] = {} 165 | params["b3"]["method"] = b3_info[0] 166 | params["b3"]["amount"] = b3_info[1] 167 | 168 | return params 169 | 170 | 171 | if __name__ == "__main__": 172 | import hyperopt.pyll.stochastic 173 | 174 | space = get_space() 175 | print(hyperopt.pyll.stochastic.sample(space)) 176 | -------------------------------------------------------------------------------- /examples/space_utils/tpe_space_resnet50_multiple.py: -------------------------------------------------------------------------------- 1 | from hyperopt import hp 2 | 3 | 4 | def get_space(): 5 | space = hp.choice( 6 | "root", 7 | [ 8 | { 9 | "method": "l1", 10 | "r1_1": hp.uniform("r1_1", 0.01, 0.99), 11 | "r1_2": hp.uniform("r1_2", 0.01, 0.99), 12 | "r1_3": hp.uniform("r1_3", 0.01, 0.99), 13 | "r1_4": hp.uniform("r1_4", 0.01, 0.99), 14 | "x1": hp.choice( 15 | "x1", 16 | [ 17 | { 18 | "method": "l1", 19 | "r3_1": hp.uniform("r3_1", 0.01, 0.99), 20 | "r3_2": hp.uniform("r3_2", 0.01, 0.99), 21 | "r3_3": hp.uniform("r3_3", 0.01, 0.99), 22 | "r3_4": hp.uniform("r3_4", 0.01, 0.99), 23 | "x3": hp.choice( 24 | "x3", 25 | [ 26 | { 27 | "method": "l1", 28 | "r7_1": hp.uniform("r7_1", 0.01, 0.99), 29 | "r7_2": hp.uniform("r7_2", 0.01, 0.99), 30 | "r7_3": hp.uniform("r7_3", 0.01, 0.99), 31 | "r7_4": hp.uniform("r7_4", 0.01, 0.99), 32 | }, 33 | { 34 | "method": "ln", 35 | "r8_1": hp.uniform("r8_1", 0.01, 0.99), 36 | "r8_2": hp.uniform("r8_2", 0.01, 0.99), 37 | "r8_3": hp.uniform("r8_3", 0.01, 0.99), 38 | "r8_4": hp.uniform("r8_4", 0.01, 0.99), 39 | }, 40 | ], 41 | ), 42 | }, 43 | { 44 | "method": "ln", 45 | "r4_1": hp.uniform("r4_1", 0.01, 0.99), 46 | "r4_2": hp.uniform("r4_2", 0.01, 0.99), 47 | "r4_3": hp.uniform("r4_3", 0.01, 0.99), 48 | "r4_4": hp.uniform("r4_4", 0.01, 0.99), 49 | "x4": hp.choice( 50 | "x4", 51 | [ 52 | { 53 | "method": "l1", 54 | "r9_1": hp.uniform("r9_1", 0.01, 0.99), 55 | "r9_2": hp.uniform("r9_2", 0.01, 0.99), 56 | "r9_3": hp.uniform("r9_3", 0.01, 0.99), 57 | "r9_4": hp.uniform("r9_4", 0.01, 0.99), 58 | }, 59 | { 60 | "method": "ln", 61 | "r10_1": hp.uniform("r10_1", 0.01, 0.99), 62 | "r10_2": hp.uniform("r10_2", 0.01, 0.99), 63 | "r10_3": hp.uniform("r10_3", 0.01, 0.99), 64 | "r10_4": hp.uniform("r10_4", 0.01, 0.99), 65 | }, 66 | ], 67 | ), 68 | }, 69 | ], 70 | ), 71 | }, 72 | { 73 | "method": "ln", 74 | "r2_1": hp.uniform("r2_1", 0.01, 0.99), 75 | "r2_2": hp.uniform("r2_2", 0.01, 0.99), 76 | "r2_3": hp.uniform("r2_3", 0.01, 0.99), 77 | "r2_4": hp.uniform("r2_4", 0.01, 0.99), 78 | "x2": hp.choice( 79 | "x2", 80 | [ 81 | { 82 | "method": "l1", 83 | "r5_1": hp.uniform("r5_1", 0.01, 0.99), 84 | "r5_2": hp.uniform("r5_2", 0.01, 0.99), 85 | "r5_3": hp.uniform("r5_3", 0.01, 0.99), 86 | "r5_4": hp.uniform("r5_4", 0.01, 0.99), 87 | "x5": hp.choice( 88 | "x5", 89 | [ 90 | { 91 | "method": "l1", 92 | "r11_1": hp.uniform("r11_1", 0.01, 0.99), 93 | "r11_2": hp.uniform("r11_2", 0.01, 0.99), 94 | "r11_3": hp.uniform("r11_3", 0.01, 0.99), 95 | "r11_4": hp.uniform("r11_4", 0.01, 0.99), 96 | }, 97 | { 98 | "method": "ln", 99 | "r12_1": hp.uniform("r12_1", 0.01, 0.99), 100 | "r12_2": hp.uniform("r12_2", 0.01, 0.99), 101 | "r12_3": hp.uniform("r12_3", 0.01, 0.99), 102 | "r12_4": hp.uniform("r12_4", 0.01, 0.99), 103 | }, 104 | ], 105 | ), 106 | }, 107 | { 108 | "method": "ln", 109 | "r6_1": hp.uniform("r6_1", 0.01, 0.99), 110 | "r6_2": hp.uniform("r6_2", 0.01, 0.99), 111 | "r6_3": hp.uniform("r6_3", 0.01, 0.99), 112 | "r6_4": hp.uniform("r6_4", 0.01, 0.99), 113 | "x6": hp.choice( 114 | "x6", 115 | [ 116 | { 117 | "method": "l1", 118 | "r13_1": hp.uniform("r13_1", 0.01, 0.99), 119 | "r13_2": hp.uniform("r13_2", 0.01, 0.99), 120 | "r13_3": hp.uniform("r13_3", 0.01, 0.99), 121 | "r13_4": hp.uniform("r13_4", 0.01, 0.99), 122 | }, 123 | { 124 | "method": "ln", 125 | "r14_1": hp.uniform("r14_1", 0.01, 0.99), 126 | "r14_2": hp.uniform("r14_2", 0.01, 0.99), 127 | "r14_3": hp.uniform("r14_3", 0.01, 0.99), 128 | "r14_4": hp.uniform("r14_4", 0.01, 0.99), 129 | }, 130 | ], 131 | ), 132 | }, 133 | ], 134 | ), 135 | }, 136 | ], 137 | ) 138 | 139 | return space 140 | 141 | 142 | def testing_cfg(): 143 | return { 144 | "method": "ln", 145 | "r2_1": 0.37489201981717213, 146 | "r2_2": 0.8812799324415477, 147 | "r2_3": 0.49084648641951856, 148 | "r2_4": 0.9609524641001413, 149 | "x2": { 150 | "method": "l1", 151 | "r5_1": 0.20459838492316887, 152 | "r5_2": 0.6653838704923347, 153 | "r5_3": 0.5621462764801486, 154 | "r5_4": 0.7740010149152105, 155 | "x5": { 156 | "method": "ln", 157 | "r12_1": 0.7013464214437278, 158 | "r12_2": 0.27559665500940383, 159 | "r12_3": 0.5493753052043601, 160 | "r12_4": 0.8509727263261176, 161 | }, 162 | }, 163 | } 164 | 165 | 166 | def cfg2funcparams(cfg): 167 | """ 168 | """ 169 | 170 | def _extract_sufix(ks): 171 | target_ks = [] 172 | for k in ks: 173 | if ( 174 | k.endswith("_1") 175 | or k.endswith("_2") 176 | or k.endswith("_3") 177 | or k.endswith("_4") 178 | ): 179 | target_ks.append(k) 180 | return sorted(target_ks) 181 | 182 | def extract_one_layer(dd): 183 | target_ks = _extract_sufix(dd.keys()) 184 | values = [dd[k] for k in target_ks] 185 | extra = [k for k in dd.keys() if k.startswith("x")] 186 | return dd["method"], values, extra 187 | 188 | params = {} 189 | 190 | b1_info = extract_one_layer(cfg) 191 | params["layer2"] = {} 192 | params["layer2"]["prune_method"] = b1_info[0] 193 | params["layer2"]["amount"] = b1_info[1] 194 | 195 | dd = cfg[b1_info[2][0]] 196 | b2_info = extract_one_layer(dd) 197 | params["layer3"] = {} 198 | params["layer3"]["prune_method"] = b2_info[0] 199 | params["layer3"]["amount"] = b2_info[1] 200 | 201 | dd = dd[b2_info[2][0]] 202 | b3_info = extract_one_layer(dd) 203 | params["layer4"] = {} 204 | params["layer4"]["prune_method"] = b3_info[0] 205 | params["layer4"]["amount"] = b3_info[1] 206 | 207 | return params 208 | 209 | 210 | if __name__ == "__main__": 211 | import hyperopt.pyll.stochastic 212 | 213 | space = get_space() 214 | print(hyperopt.pyll.stochastic.sample(space)) 215 | -------------------------------------------------------------------------------- /examples/space_utils/tpe_space_resnet56.py: -------------------------------------------------------------------------------- 1 | from hyperopt import hp 2 | 3 | 4 | def get_space(): 5 | space = hp.choice( 6 | "root", 7 | [ 8 | { 9 | "method": "l1", 10 | "r1_1": hp.uniform("r1_1", 0.01, 0.99), 11 | "r1_2": hp.uniform("r1_2", 0.01, 0.99), 12 | "r1_3": hp.uniform("r1_3", 0.01, 0.99), 13 | "x1": hp.choice( 14 | "x1", 15 | [ 16 | { 17 | "method": "l1", 18 | "r3_1": hp.uniform("r3_1", 0.01, 0.99), 19 | "r3_2": hp.uniform("r3_2", 0.01, 0.99), 20 | "r3_3": hp.uniform("r3_3", 0.01, 0.99), 21 | "x3": hp.choice( 22 | "x3", 23 | [ 24 | { 25 | "method": "l1", 26 | "r7_1": hp.uniform("r7_1", 0.01, 0.99), 27 | "r7_2": hp.uniform("r7_2", 0.01, 0.99), 28 | "r7_3": hp.uniform("r7_3", 0.01, 0.99), 29 | }, 30 | { 31 | "method": "ln", 32 | "r8_1": hp.uniform("r8_1", 0.01, 0.99), 33 | "r8_2": hp.uniform("r8_2", 0.01, 0.99), 34 | "r8_3": hp.uniform("r8_3", 0.01, 0.99), 35 | }, 36 | ], 37 | ), 38 | }, 39 | { 40 | "method": "ln", 41 | "r4_1": hp.uniform("r4_1", 0.01, 0.99), 42 | "r4_2": hp.uniform("r4_2", 0.01, 0.99), 43 | "r4_3": hp.uniform("r4_3", 0.01, 0.99), 44 | "x4": hp.choice( 45 | "x4", 46 | [ 47 | { 48 | "method": "l1", 49 | "r9_1": hp.uniform("r9_1", 0.01, 0.99), 50 | "r9_2": hp.uniform("r9_2", 0.01, 0.99), 51 | "r9_3": hp.uniform("r9_3", 0.01, 0.99), 52 | }, 53 | { 54 | "method": "ln", 55 | "r10_1": hp.uniform("r10_1", 0.01, 0.99), 56 | "r10_2": hp.uniform("r10_2", 0.01, 0.99), 57 | "r10_3": hp.uniform("r10_3", 0.01, 0.99), 58 | }, 59 | ], 60 | ), 61 | }, 62 | ], 63 | ), 64 | }, 65 | { 66 | "method": "ln", 67 | "r2_1": hp.uniform("r2_1", 0.01, 0.99), 68 | "r2_2": hp.uniform("r2_2", 0.01, 0.99), 69 | "r2_3": hp.uniform("r2_3", 0.01, 0.99), 70 | "x2": hp.choice( 71 | "x2", 72 | [ 73 | { 74 | "method": "l1", 75 | "r5_1": hp.uniform("r5_1", 0.01, 0.99), 76 | "r5_2": hp.uniform("r5_2", 0.01, 0.99), 77 | "r5_3": hp.uniform("r5_3", 0.01, 0.99), 78 | "x5": hp.choice( 79 | "x5", 80 | [ 81 | { 82 | "method": "l1", 83 | "r11_1": hp.uniform("r11_1", 0.01, 0.99), 84 | "r11_2": hp.uniform("r11_2", 0.01, 0.99), 85 | "r11_3": hp.uniform("r11_3", 0.01, 0.99), 86 | }, 87 | { 88 | "method": "ln", 89 | "r12_1": hp.uniform("r12_1", 0.01, 0.99), 90 | "r12_2": hp.uniform("r12_2", 0.01, 0.99), 91 | "r12_3": hp.uniform("r12_3", 0.01, 0.99), 92 | }, 93 | ], 94 | ), 95 | }, 96 | { 97 | "method": "ln", 98 | "r6_1": hp.uniform("r6_1", 0.01, 0.99), 99 | "r6_2": hp.uniform("r6_2", 0.01, 0.99), 100 | "r6_3": hp.uniform("r6_3", 0.01, 0.99), 101 | "x6": hp.choice( 102 | "x6", 103 | [ 104 | { 105 | "method": "l1", 106 | "r13_1": hp.uniform("r13_1", 0.01, 0.99), 107 | "r13_2": hp.uniform("r13_2", 0.01, 0.99), 108 | "r13_3": hp.uniform("r13_3", 0.01, 0.99), 109 | }, 110 | { 111 | "method": "ln", 112 | "r14_1": hp.uniform("r14_1", 0.01, 0.99), 113 | "r14_2": hp.uniform("r14_2", 0.01, 0.99), 114 | "r14_3": hp.uniform("r14_3", 0.01, 0.99), 115 | }, 116 | ], 117 | ), 118 | }, 119 | ], 120 | ), 121 | }, 122 | ], 123 | ) 124 | 125 | return space 126 | 127 | 128 | def testing_cfg(): 129 | return { 130 | "method": "l1", 131 | "r1_1": 0.8240496446059529, 132 | "r1_2": 0.9776127085354424, 133 | "r1_3": 0.9331836454436203, 134 | "x1": { 135 | "method": "ln", 136 | "r4_1": 0.03395381604573712, 137 | "r4_2": 0.9083169481320289, 138 | "r4_3": 0.7829933759953129, 139 | "x4": { 140 | "method": "l1", 141 | "r9_1": 0.19563379141873094, 142 | "r9_2": 0.7764489202878972, 143 | "r9_3": 0.25423916992927137, 144 | }, 145 | }, 146 | } 147 | 148 | 149 | def cfg2funcparams(cfg): 150 | """ 151 | """ 152 | 153 | def _extract_sufix(ks): 154 | target_ks = [] 155 | for k in ks: 156 | if k.endswith("_1") or k.endswith("_2") or k.endswith("_3"): 157 | target_ks.append(k) 158 | return sorted(target_ks) 159 | 160 | def extract_one_layer(dd): 161 | target_ks = _extract_sufix(dd.keys()) 162 | values = [dd[k] for k in target_ks] 163 | extra = [k for k in dd.keys() if k.startswith("x")] 164 | return dd["method"], values, extra 165 | 166 | params = {} 167 | 168 | b1_info = extract_one_layer(cfg) 169 | params["layer1"] = {} 170 | params["layer1"]["prune_method"] = b1_info[0] 171 | params["layer1"]["amount"] = b1_info[1] 172 | 173 | dd = cfg[b1_info[2][0]] 174 | b2_info = extract_one_layer(dd) 175 | params["layer2"] = {} 176 | params["layer2"]["prune_method"] = b2_info[0] 177 | params["layer2"]["amount"] = b2_info[1] 178 | 179 | dd = dd[b2_info[2][0]] 180 | b3_info = extract_one_layer(dd) 181 | params["layer3"] = {} 182 | params["layer3"]["prune_method"] = b3_info[0] 183 | params["layer3"]["amount"] = b3_info[1] 184 | 185 | return params 186 | 187 | 188 | if __name__ == "__main__": 189 | import hyperopt.pyll.stochastic 190 | 191 | space = get_space() 192 | print(hyperopt.pyll.stochastic.sample(space)) 193 | -------------------------------------------------------------------------------- /examples/space_utils/tpe_space_vgg16_multiple.py: -------------------------------------------------------------------------------- 1 | from hyperopt import hp 2 | 3 | 4 | def get_space(): 5 | space = hp.choice( 6 | "root", 7 | [ 8 | { 9 | "method": "l1", 10 | "r1_1": hp.uniform("r1_1", 0.01, 0.99), 11 | "r1_2": hp.uniform("r1_2", 0.01, 0.99), 12 | "r1_3": hp.uniform("r1_3", 0.01, 0.99), 13 | "x1": hp.choice( 14 | "x1", 15 | [ 16 | { 17 | "method": "l1", 18 | "r3_1": hp.uniform("r3_1", 0.01, 0.99), 19 | "r3_2": hp.uniform("r3_2", 0.01, 0.99), 20 | "r3_3": hp.uniform("r3_3", 0.01, 0.99), 21 | "x3": hp.choice( 22 | "x3", 23 | [ 24 | { 25 | "method": "l1", 26 | "r7_1": hp.uniform("r7_1", 0.01, 0.99), 27 | "r7_2": hp.uniform("r7_2", 0.01, 0.99), 28 | "r7_3": hp.uniform("r7_3", 0.01, 0.99), 29 | }, 30 | { 31 | "method": "ln", 32 | "r8_1": hp.uniform("r8_1", 0.01, 0.99), 33 | "r8_2": hp.uniform("r8_2", 0.01, 0.99), 34 | "r8_3": hp.uniform("r8_3", 0.01, 0.99), 35 | }, 36 | ], 37 | ), 38 | }, 39 | { 40 | "method": "ln", 41 | "r4_1": hp.uniform("r4_1", 0.01, 0.99), 42 | "r4_2": hp.uniform("r4_2", 0.01, 0.99), 43 | "r4_3": hp.uniform("r4_3", 0.01, 0.99), 44 | "x4": hp.choice( 45 | "x4", 46 | [ 47 | { 48 | "method": "l1", 49 | "r9_1": hp.uniform("r9_1", 0.01, 0.99), 50 | "r9_2": hp.uniform("r9_2", 0.01, 0.99), 51 | "r9_3": hp.uniform("r9_3", 0.01, 0.99), 52 | }, 53 | { 54 | "method": "ln", 55 | "r10_1": hp.uniform("r10_1", 0.01, 0.99), 56 | "r10_2": hp.uniform("r10_2", 0.01, 0.99), 57 | "r10_3": hp.uniform("r10_3", 0.01, 0.99), 58 | }, 59 | ], 60 | ), 61 | }, 62 | ], 63 | ), 64 | }, 65 | { 66 | "method": "ln", 67 | "r2_1": hp.uniform("r2_1", 0.01, 0.99), 68 | "r2_2": hp.uniform("r2_2", 0.01, 0.99), 69 | "r2_3": hp.uniform("r2_3", 0.01, 0.99), 70 | "x2": hp.choice( 71 | "x2", 72 | [ 73 | { 74 | "method": "l1", 75 | "r5_1": hp.uniform("r5_1", 0.01, 0.99), 76 | "r5_2": hp.uniform("r5_2", 0.01, 0.99), 77 | "r5_3": hp.uniform("r5_3", 0.01, 0.99), 78 | "x5": hp.choice( 79 | "x5", 80 | [ 81 | { 82 | "method": "l1", 83 | "r11_1": hp.uniform("r11_1", 0.01, 0.99), 84 | "r11_2": hp.uniform("r11_2", 0.01, 0.99), 85 | "r11_3": hp.uniform("r11_3", 0.01, 0.99), 86 | }, 87 | { 88 | "method": "ln", 89 | "r12_1": hp.uniform("r12_1", 0.01, 0.99), 90 | "r12_2": hp.uniform("r12_2", 0.01, 0.99), 91 | "r12_3": hp.uniform("r12_3", 0.01, 0.99), 92 | }, 93 | ], 94 | ), 95 | }, 96 | { 97 | "method": "ln", 98 | "r6_1": hp.uniform("r6_1", 0.01, 0.99), 99 | "r6_2": hp.uniform("r6_2", 0.01, 0.99), 100 | "r6_3": hp.uniform("r6_3", 0.01, 0.99), 101 | "x6": hp.choice( 102 | "x6", 103 | [ 104 | { 105 | "method": "l1", 106 | "r13_1": hp.uniform("r13_1", 0.01, 0.99), 107 | "r13_2": hp.uniform("r13_2", 0.01, 0.99), 108 | "r13_3": hp.uniform("r13_3", 0.01, 0.99), 109 | }, 110 | { 111 | "method": "ln", 112 | "r14_1": hp.uniform("r14_1", 0.01, 0.99), 113 | "r14_2": hp.uniform("r14_2", 0.01, 0.99), 114 | "r14_3": hp.uniform("r14_3", 0.01, 0.99), 115 | }, 116 | ], 117 | ), 118 | }, 119 | ], 120 | ), 121 | }, 122 | ], 123 | ) 124 | 125 | return space 126 | 127 | 128 | def testing_cfg(): 129 | return { 130 | "method": "l1", 131 | "r1_1": 0.4787187980571228, 132 | "r1_2": 0.47581041577268224, 133 | "r1_3": 0.5228518172842436, 134 | "x1": { 135 | "method": "ln", 136 | "r4_1": 0.2990475437036728, 137 | "r4_2": 0.37493878522605695, 138 | "r4_3": 0.24950009451280078, 139 | "x4": { 140 | "method": "ln", 141 | "r10_1": 0.14595738097427577, 142 | "r10_2": 0.2275885307456954, 143 | "r10_3": 0.7269206516328793, 144 | }, 145 | }, 146 | } 147 | 148 | 149 | def cfg2funcparams(cfg): 150 | """ 151 | """ 152 | 153 | def _extract_sufix(ks): 154 | target_ks = [] 155 | for k in ks: 156 | if k.endswith("_1") or k.endswith("_2") or k.endswith("_3"): 157 | target_ks.append(k) 158 | return sorted(target_ks) 159 | 160 | def extract_one_layer(dd): 161 | target_ks = _extract_sufix(dd.keys()) 162 | values = [dd[k] for k in target_ks] 163 | extra = [k for k in dd.keys() if k.startswith("x")] 164 | return dd["method"], values, extra 165 | 166 | params = {} 167 | 168 | b1_info = extract_one_layer(cfg) 169 | params["b1"] = {} 170 | params["b1"]["prune_method"] = b1_info[0] 171 | params["b1"]["amount"] = b1_info[1] 172 | 173 | dd = cfg[b1_info[2][0]] 174 | b2_info = extract_one_layer(dd) 175 | params["b2"] = {} 176 | params["b2"]["prune_method"] = b2_info[0] 177 | params["b2"]["amount"] = b2_info[1] 178 | 179 | dd = dd[b2_info[2][0]] 180 | b3_info = extract_one_layer(dd) 181 | params["b3"] = {} 182 | params["b3"]["prune_method"] = b3_info[0] 183 | params["b3"]["amount"] = b3_info[1] 184 | 185 | return params 186 | 187 | 188 | if __name__ == "__main__": 189 | import hyperopt.pyll.stochastic 190 | 191 | space = get_space() 192 | print(hyperopt.pyll.stochastic.sample(space)) 193 | -------------------------------------------------------------------------------- /examples/tpe_model_compression_multiple.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import json 3 | import os 4 | import logging 5 | 6 | 7 | from common_utils import setup_logger 8 | from compression_common import setup_and_prune 9 | from hyperopt import fmin, tpe, STATUS_OK, Trials 10 | from compression_common import get_common_cmd_args 11 | from common_utils import get_experiment_id 12 | 13 | from tpe_utils import cfg2funcparams_vgg16 14 | from tpe_utils import get_space_vgg16 15 | from tpe_utils import cfg2funcparams_resnet50 16 | from tpe_utils import get_space_resnet50 17 | from tpe_utils import cfg2funcparams_resnet56 18 | from tpe_utils import get_space_resnet56 19 | 20 | 21 | def main(): 22 | 23 | try: 24 | cmd_args, _ = get_common_cmd_args() 25 | 26 | output_basedir = cmd_args.output_basedir 27 | model_name = cmd_args.model_name 28 | if model_name == "vgg16": 29 | cfg2funcparams = cfg2funcparams_vgg16 30 | get_space = get_space_vgg16 31 | elif model_name == "resnet50": 32 | cfg2funcparams = cfg2funcparams_resnet50 33 | get_space = get_space_resnet50 34 | elif model_name == "resnet56": 35 | cfg2funcparams = cfg2funcparams_resnet56 36 | get_space = get_space_resnet56 37 | else: 38 | raise ValueError(f"model name {model_name} is wrong") 39 | 40 | logger = logging.getLogger(f"TPE-{model_name}") 41 | logger.setLevel(logging.DEBUG) 42 | 43 | expid = get_experiment_id(6) 44 | output_dir = os.path.join(output_basedir, "tpe", model_name, expid) 45 | os.makedirs(output_dir, exist_ok=True) 46 | log_path = os.path.join(output_dir, f"tpe-model-compression-{model_name}.log") 47 | setup_logger(logger, log_path) 48 | 49 | logger.info(f"Experiment {expid} starts...") 50 | logger.info("Experiment Configuration:") 51 | logger.info(vars(cmd_args)) 52 | 53 | def obj_func(cfg): 54 | logger.info("Starting BO iteration") 55 | params = cfg2funcparams(cfg) 56 | obj_info = setup_and_prune( 57 | cmd_args, params, logger, prune_type="multiple", model_name=model_name 58 | ) 59 | logger.info("Finishing BO iteration") 60 | logger.info(params) 61 | logger.info(obj_info) 62 | 63 | all_info = { 64 | "params": params, 65 | "obj_info": obj_info, 66 | } 67 | fn_path = os.path.join(output_dir, "tpe_iter_hists.txt") 68 | with open(fn_path, "a") as f: 69 | json.dump(all_info, f) 70 | f.write("\n") 71 | 72 | return {"loss": obj_info["value"], "status": STATUS_OK} 73 | 74 | space = get_space() 75 | trials = Trials() 76 | best = fmin( 77 | obj_func, 78 | space=space, 79 | algo=tpe.suggest, 80 | max_evals=300, 81 | trials=trials, 82 | show_progressbar=False, 83 | ) 84 | print(best) 85 | logger.info("Finish TPE optimization") 86 | logger.info("Best is: ") 87 | logger.info(best) 88 | 89 | except KeyboardInterrupt: 90 | print("Interrupted. You pressed Ctrl-C!!!") 91 | try: 92 | sys.exit(0) 93 | except SystemExit: 94 | os._exit(0) 95 | 96 | 97 | if __name__ == "__main__": 98 | main() 99 | -------------------------------------------------------------------------------- /examples/tpe_resnet20_nas.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import json 3 | import os 4 | import logging 5 | 6 | from hyperopt import fmin, tpe, STATUS_OK, Trials 7 | 8 | from common_utils import setup_logger 9 | from common_utils import get_experiment_id 10 | from nas_common import get_common_cmd_args 11 | from nas_common import nas_train_test 12 | 13 | 14 | from tpe_utils import cfg2funcparams_nas_resnet20 15 | from tpe_utils import get_space_nas_resnet20 16 | 17 | 18 | def main(): 19 | 20 | try: 21 | cmd_args, _ = get_common_cmd_args() 22 | 23 | output_basedir = cmd_args.output_basedir 24 | model_name = cmd_args.model_name 25 | if model_name == "resnet20": 26 | cfg2funcparams = cfg2funcparams_nas_resnet20 27 | get_space = get_space_nas_resnet20 28 | else: 29 | raise ValueError(f"model name {model_name} is wrong") 30 | 31 | logger = logging.getLogger(f"TPE-NAS-{model_name}") 32 | logger.setLevel(logging.DEBUG) 33 | 34 | expid = get_experiment_id(6) 35 | output_dir = os.path.join(output_basedir, "TPE-NAS", model_name, expid) 36 | os.makedirs(output_dir, exist_ok=True) 37 | log_path = os.path.join(output_dir, f"TPE-NAS-{model_name}.log") 38 | setup_logger(logger, log_path) 39 | 40 | logger.info(f"Experiment {expid} starts...") 41 | logger.info("Experiment Configuration:") 42 | logger.info(vars(cmd_args)) 43 | 44 | def obj_func(cfg): 45 | logger.info("Starting BO iteration") 46 | params = cfg2funcparams(cfg) 47 | obj_info = nas_train_test(cmd_args, params, logger, model_name=model_name) 48 | logger.info("Finishing BO iteration") 49 | logger.info(params) 50 | logger.info(obj_info) 51 | 52 | all_info = { 53 | "params": params, 54 | "obj_info": obj_info, 55 | } 56 | fn_path = os.path.join(output_dir, "tpe_iter_hists.txt") 57 | with open(fn_path, "a") as f: 58 | json.dump(all_info, f) 59 | f.write("\n") 60 | 61 | return {"loss": obj_info["value"], "status": STATUS_OK} 62 | 63 | space = get_space() 64 | trials = Trials() 65 | best = fmin( 66 | obj_func, 67 | space=space, 68 | algo=tpe.suggest, 69 | max_evals=100, 70 | trials=trials, 71 | show_progressbar=False, 72 | ) 73 | print(best) 74 | logger.info("Finish TPE optimization") 75 | logger.info("Best is: ") 76 | logger.info(best) 77 | 78 | except KeyboardInterrupt: 79 | print("Interrupted. You pressed Ctrl-C!!!") 80 | try: 81 | sys.exit(0) 82 | except SystemExit: 83 | os._exit(0) 84 | 85 | 86 | if __name__ == "__main__": 87 | main() 88 | -------------------------------------------------------------------------------- /examples/tpe_utils.py: -------------------------------------------------------------------------------- 1 | from space_utils.tpe_space_vgg16_multiple import get_space as get_space_vgg16 2 | from space_utils.tpe_space_vgg16_multiple import cfg2funcparams as cfg2funcparams_vgg16 3 | 4 | # resnet50 5 | from space_utils.tpe_space_resnet50_multiple import get_space as get_space_resnet50 6 | from space_utils.tpe_space_resnet50_multiple import ( 7 | cfg2funcparams as cfg2funcparams_resnet50, 8 | ) 9 | 10 | # resnet56 11 | from space_utils.tpe_space_resnet56 import get_space as get_space_resnet56 12 | from space_utils.tpe_space_resnet56 import cfg2funcparams as cfg2funcparams_resnet56 13 | 14 | 15 | # resnet20, NAS 16 | from space_utils.tpe_space_nas_resnet20 import get_space as get_space_nas_resnet20 17 | from space_utils.tpe_space_nas_resnet20 import cfg2funcparams as cfg2funcparams_nas_resnet20 18 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "addtree" 3 | version = "0.1.0" 4 | description = "" 5 | authors = ["maxc01 "] 6 | 7 | [tool.poetry.dependencies] 8 | # python = "^3.11" 9 | # numpy = "^1.18.2" 10 | # scipy = "^1.4.1" 11 | # torch = "^1.4.0" 12 | # torchvision = "^0.5.0" 13 | george = { git = "https://github.com/maxc01/george" } 14 | 15 | [tool.poetry.dev-dependencies] 16 | pytest = "^5.2" 17 | 18 | [build-system] 19 | requires = ["poetry>=0.12"] 20 | build-backend = "poetry.masonry.api" 21 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/maxc01/addtree/679f2f0c7ee3905385135e6693138e5733c6306b/tests/__init__.py -------------------------------------------------------------------------------- /tests/test_addtree.py: -------------------------------------------------------------------------------- 1 | from addtree import __version__ 2 | 3 | 4 | def test_version(): 5 | assert __version__ == '0.1.0' 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /tests/test_parameter.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from pytest import approx 3 | 4 | import numpy as np 5 | 6 | from addtree.parameter import Parameter 7 | from addtree.parameter import ParameterNode 8 | from addtree.parameter import clear_state 9 | from addtree.parameter import NodePath 10 | 11 | 12 | @pytest.fixture 13 | def small_tree1(): 14 | clear_state() 15 | root = ParameterNode(Parameter("root", 2, np.array([1, 2]))) 16 | x1 = ParameterNode(Parameter("x1", 2, np.array([3.3, 4]))) 17 | x2 = ParameterNode(Parameter("x2", 3, np.array([5, 6.6, 7]))) 18 | root.add_child(x1) 19 | root.add_child(x2) 20 | root.finish_add_child() 21 | 22 | return root, x1, x2 23 | 24 | 25 | @pytest.fixture 26 | def large_tree(): 27 | clear_state() 28 | root = ParameterNode(Parameter("root", 0)) 29 | x1 = ParameterNode(Parameter("x1", 1)) 30 | x2 = ParameterNode(Parameter("x2", 1)) 31 | root.add_child(x1) 32 | root.add_child(x2) 33 | x3 = ParameterNode(Parameter("x3", 1)) 34 | x4 = ParameterNode(Parameter("x4", 1)) 35 | x1.add_child(x3) 36 | x1.add_child(x4) 37 | 38 | x5 = ParameterNode(Parameter("x5", 1)) 39 | x6 = ParameterNode(Parameter("x6", 1)) 40 | x2.add_child(x5) 41 | x2.add_child(x6) 42 | 43 | root.finish_add_child() 44 | 45 | return root 46 | 47 | 48 | def test_bfs_template(small_tree1): 49 | root = small_tree1[0] 50 | assert len(root.bfs_template) == 10 51 | assert root.bfs_template == approx(np.array([-1] * 10)) 52 | 53 | 54 | def test_obs_dim(small_tree1): 55 | root = small_tree1[0] 56 | assert root.obs_dim == 10 57 | 58 | 59 | def test_set_data(small_tree1): 60 | root, x1, x2 = small_tree1 61 | 62 | p1 = NodePath([root, x1]).set_data(np.array([9, 8, 7, 6], "f")) 63 | assert p1[0].parameter.data == approx(np.array([9, 8])) 64 | assert p1[1].parameter.data == approx(np.array([7, 6])) 65 | 66 | p2 = NodePath([root, x2]).set_data(np.array([8, 7, 6, 5, 4], "f")) 67 | assert p2[0].parameter.data == approx(np.array([8, 7])) 68 | assert p2[1].parameter.data == approx(np.array([6, 5, 4])) 69 | 70 | 71 | def test_path2dict(small_tree1): 72 | root, x1, x2 = small_tree1 73 | d1 = NodePath([root, x1]).path2dict() 74 | assert d1["root"] == approx(np.array([1, 2])) 75 | assert d1["x1"] == approx(np.array([3.3, 4])) 76 | 77 | d2 = NodePath([root, x2]).path2dict() 78 | assert d2["root"] == approx(np.array([1, 2])) 79 | assert d2["x2"] == approx(np.array([5, 6.6, 7])) 80 | 81 | 82 | def test_path2vec(small_tree1): 83 | root, x1, x2 = small_tree1 84 | assert NodePath([root, x1]).path2vec(root.obs_dim) == approx( 85 | np.array([0, 1, 2, 0, 3.3, 4, -1, -1, -1, -1]) 86 | ) 87 | 88 | assert NodePath([root, x2]).path2vec(root.obs_dim) == approx( 89 | np.array([0, 1, 2, -1, -1, -1, 1, 5, 6.6, 7]) 90 | ) 91 | 92 | 93 | def test_bfs_nodes(small_tree1): 94 | root, x1, x2 = small_tree1 95 | all_nodes = root.bfs_nodes() 96 | assert all_nodes[0] == root 97 | assert all_nodes[1] == x1 98 | assert all_nodes[2] == x2 99 | 100 | 101 | def test_select_path(small_tree1): 102 | root, x1, x2 = small_tree1 103 | 104 | path1 = root.select_path("0") 105 | assert path1[0] == root 106 | assert path1[1] == x1 107 | 108 | path2 = root.select_path("1") 109 | assert path2[0] == root 110 | assert path2[1] == x2 111 | 112 | 113 | def test_all_pathids(small_tree1): 114 | root, x1, x2 = small_tree1 115 | 116 | assert root.all_pathids() == ["0", "1"] 117 | 118 | 119 | def test_all_pathids_large(large_tree): 120 | root = large_tree 121 | assert root.all_pathids() == ["00", "01", "10", "11"] 122 | -------------------------------------------------------------------------------- /tests/test_storage.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import numpy as np 3 | from tqdm import tqdm 4 | 5 | from addtree.kernel_utils import build_addtree 6 | from addtree.storage import Storage 7 | from addtree.parameter import Parameter 8 | from addtree.parameter import ParameterNode 9 | from addtree.parameter import clear_state 10 | from addtree.parameter import get_state 11 | 12 | 13 | def large_tree(): 14 | root = ParameterNode(Parameter("root", 0)) 15 | x1 = ParameterNode(Parameter("x1", 1)) 16 | x2 = ParameterNode(Parameter("x2", 1)) 17 | root.add_child(x1) 18 | root.add_child(x2) 19 | x3 = ParameterNode(Parameter("x3", 1)) 20 | x4 = ParameterNode(Parameter("x4", 1)) 21 | x1.add_child(x3) 22 | x1.add_child(x4) 23 | 24 | x5 = ParameterNode(Parameter("x5", 1)) 25 | x6 = ParameterNode(Parameter("x6", 1)) 26 | x2.add_child(x5) 27 | x2.add_child(x6) 28 | 29 | root.finish_add_child() 30 | 31 | return root 32 | 33 | 34 | def obj_func(param_dict): 35 | SHIFT = 0.5 36 | if "x1" in param_dict and "x3" in param_dict: 37 | value = (param_dict["x3"] - SHIFT) ** 2 + param_dict["x1"] + 0.1 38 | elif "x1" in param_dict and "x4" in param_dict: 39 | value = (param_dict["x4"] - SHIFT) ** 2 + param_dict["x1"] + 0.2 40 | elif "x2" in param_dict and "x5" in param_dict: 41 | value = (param_dict["x5"] - SHIFT) ** 2 + param_dict["x2"] + 0.3 42 | elif "x2" in param_dict and "x6" in param_dict: 43 | value = (param_dict["x6"] - SHIFT) ** 2 + param_dict["x2"] + 0.4 44 | else: 45 | raise KeyError(f"{param_dict} don't contain the correct keys") 46 | 47 | info = dict() 48 | info["value"] = value.item() 49 | info["value_sigma"] = 1e-9 50 | return info 51 | 52 | 53 | clear_state() 54 | root = large_tree() 55 | ss = Storage() 56 | for i in range(5): 57 | path = root.random_path(rand_data=True) 58 | res = obj_func(path.path2dict()) 59 | ss.add(path.path2vec(root.obs_dim), res["value"], res["value_sigma"], path) 60 | 61 | ker = build_addtree(root) 62 | 63 | 64 | def LCB(gp, X_new, Y_train, kappa=1.0): 65 | pred, pred_var = gp.predict(Y_train, X_new, return_var=True) 66 | pred_sigma = np.sqrt(pred_var) 67 | return pred - kappa * pred_sigma 68 | 69 | 70 | # IMPORTANT: any parameter passed to acq MUST be effective, i.e. other 71 | # dimensions must be set to be invalid 72 | # BUG: the above is just the bug 73 | 74 | acq_func = LCB 75 | 76 | 77 | # BUG: the following is wrong, any parameter passed to acq MUST be effective 78 | def fill_x(x, total_dim, eff_axes): 79 | vec = np.empty(total_dim) 80 | vec[...] = -1 81 | vec[eff_axes] = x 82 | return vec 83 | 84 | 85 | def optimize_acq(gp, Y_train, total_dim, grid_size=100, nb_seed=2): 86 | 87 | info = [] 88 | for path_id in ["00", "01", "10", "11"]: 89 | path = root.select_path(path_id) 90 | eff_axes = path.axes() 91 | grid = path.rand(grid_size, total_dim) 92 | grid_acq = acq_func(gp, grid, Y_train) 93 | seeds_idx = np.argsort(grid_acq)[:nb_seed] 94 | bounds = [(0, 1)] * len(eff_axes) 95 | ixgrid = np.ix_(seeds_idx, eff_axes) 96 | seeds = grid[ixgrid] 97 | 98 | def obj_func_acq(x): 99 | # vec = fill_x(x, root.obs_dim, eff_axes) 100 | vec = path.set_data(x).path2vec(root.obs_dim) 101 | return acq_func(gp, vec[None], Y_train).item() 102 | 103 | # start minimization using these seeds 104 | _x_best = None 105 | _y_best = np.inf 106 | for seed in seeds: 107 | result = minimize(obj_func_acq, x0=seed, method="L-BFGS-B", bounds=bounds,) 108 | if result.fun < _y_best: 109 | _y_best = result.fun 110 | _x_best = result.x 111 | heapq.heappush(info, (_y_best, path_id, _x_best, path)) 112 | 113 | return info[0] 114 | 115 | 116 | for i in tqdm(range(95)): 117 | gp = ss.optimize(ker, n_restart=2, verbose=False) 118 | _, _, x_best, path = optimize_acq(gp, ss.Y, root.obs_dim) 119 | path.set_data(x_best) 120 | obj_info = obj_func(path.path2dict()) 121 | ss.add( 122 | path.path2vec(root.obs_dim), 123 | obj_info["value"], 124 | obj_info["value_sigma"], 125 | path=path, 126 | ) 127 | --------------------------------------------------------------------------------