├── test ├── __init__.py └── test_tabular_benchmarks │ ├── __init__.py │ ├── test_tabular_fcnet_benchmarks.py │ └── test_tabular_nas_cifar10.py ├── tabular_benchmarks ├── __init__.py ├── fcnet_benchmark.py └── nas_cifar10.py ├── example.py ├── setup.py ├── LICENSE ├── README.md ├── experiment_scripts ├── run_random_search.py ├── run_tpe.py ├── run_smac.py ├── run_hyperband.py ├── run_bohb.py ├── run_regularized_evolution.py └── run_rl.py └── data_collection └── train_fcnet.py /test/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/test_tabular_benchmarks/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tabular_benchmarks/__init__.py: -------------------------------------------------------------------------------- 1 | from .fcnet_benchmark import FCNetProteinStructureBenchmark 2 | from .fcnet_benchmark import FCNetSliceLocalizationBenchmark 3 | from .fcnet_benchmark import FCNetNavalPropulsionBenchmark 4 | from .fcnet_benchmark import FCNetParkinsonsTelemonitoringBenchmark 5 | from .nas_cifar10 import NASCifar10A, NASCifar10B, NASCifar10C 6 | -------------------------------------------------------------------------------- /example.py: -------------------------------------------------------------------------------- 1 | 2 | from tabular_benchmarks import FCNetProteinStructureBenchmark 3 | 4 | b = FCNetProteinStructureBenchmark(data_dir="./fcnet_tabular_benchmarks/") 5 | cs = b.get_configuration_space() 6 | config = cs.sample_configuration() 7 | 8 | print("Numpy representation: ", config.get_array()) 9 | print("Dict representation: ", config.get_dictionary()) 10 | 11 | max_epochs = 100 12 | y, cost = b.objective_function(config, budget=max_epochs) 13 | print(y, cost) 14 | 15 | 16 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | from setuptools import setup, find_packages 4 | here = os.path.abspath(os.path.dirname(__file__)) 5 | 6 | requires = [ 7 | 'h5py', 8 | 'numpy', 9 | 'ConfigSpace', 10 | 'pandas', # missing dependency of nasbench 11 | ] 12 | non_pypi_requires = [ 13 | 'git+https://github.com/google-research/nasbench.git@master#egg=nasbench' 14 | ] 15 | 16 | setup(name='nas_benchmarks', 17 | version='0.0.1', 18 | description='tabular benchmarks for feed forward neural networks', 19 | author='Aaron Klein', 20 | author_email='kleinaa@cs.uni-freiburg.de', 21 | keywords='hyperparameter optimization', 22 | packages=find_packages(), 23 | install_requires=requires, 24 | dependency_links=non_pypi_requires, 25 | ) 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2018, Aaron Klein 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /test/test_tabular_benchmarks/test_tabular_fcnet_benchmarks.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from tabular_benchmarks import FCNetProteinStructureBenchmark,\ 4 | FCNetSliceLocalizationBenchmark, FCNetNavalPropulsionBenchmark, FCNetParkinsonsTelemonitoringBenchmark 5 | 6 | 7 | class TestFCNetProteinStructure(unittest.TestCase): 8 | 9 | def setUp(self): 10 | self.b = FCNetProteinStructureBenchmark(data_dir="./fcnet_tabular_benchmarks/") 11 | 12 | def test_random_sampling(self): 13 | config = self.b.get_configuration_space().sample_configuration() 14 | self.b.objective_function(config) 15 | 16 | 17 | class TestFCNetSliceLocalization(unittest.TestCase): 18 | 19 | def setUp(self): 20 | self.b = FCNetSliceLocalizationBenchmark(data_dir="./fcnet_tabular_benchmarks/") 21 | 22 | def test_random_sampling(self): 23 | config = self.b.get_configuration_space().sample_configuration() 24 | self.b.objective_function(config) 25 | 26 | 27 | class TestFCNetNavalPropulsion(unittest.TestCase): 28 | 29 | def setUp(self): 30 | self.b = FCNetNavalPropulsionBenchmark(data_dir="./fcnet_tabular_benchmarks/") 31 | 32 | def test_random_sampling(self): 33 | config = self.b.get_configuration_space().sample_configuration() 34 | self.b.objective_function(config) 35 | 36 | 37 | class TestFCNetParkinsonsTelemonitoring(unittest.TestCase): 38 | 39 | def setUp(self): 40 | self.b = FCNetParkinsonsTelemonitoringBenchmark(data_dir="./fcnet_tabular_benchmarks/") 41 | 42 | def test_random_sampling(self): 43 | config = self.b.get_configuration_space().sample_configuration() 44 | self.b.objective_function(config) 45 | 46 | 47 | if __name__ == '__main__': 48 | unittest.main() 49 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Tabular Benchmarks for Hyperparameter Optimization and Neural Architecture Search 2 | 3 | This repository contains code of tabular benchmarks for 4 | - HPOBench: joint hyperparameter and architecture optimization of feed forward neural networks on regression problems (see [1]) 5 | - NASBench101: the architecture optimization of a convolutional neural network (see [2]) 6 | 7 | 8 | To download the datasets for the FC-Net benchmark: 9 | 10 | wget http://ml4aad.org/wp-content/uploads/2019/01/fcnet_tabular_benchmarks.tar.gz 11 | tar xf fcnet_tabular_benchmarks.tar.gz 12 | 13 | The data for NASBench is available [here](https://github.com/google-research/nasbench). 14 | 15 | To install it, type: 16 | 17 | git clone https://github.com/automl/nas_benchmarks.git 18 | cd nas_benchmarks 19 | python setup.py install 20 | 21 | The following example shows how to load the benchmark and to evaluate a random hyperparameter configuration: 22 | 23 | from tabular_benchmarks import FCNetProteinStructureBenchmark 24 | 25 | b = FCNetProteinStructureBenchmark(data_dir="./fcnet_tabular_benchmarks/") 26 | cs = b.get_configuration_space() 27 | config = cs.sample_configuration() 28 | 29 | print("Numpy representation: ", config.get_array()) 30 | print("Dict representation: ", config.get_dictionary()) 31 | 32 | max_epochs = 100 33 | y, cost = b.objective_function(config, budget=max_epochs) 34 | print(y, cost) 35 | 36 | 37 | To see how you can run different open-source optimizers from the literature, have a look on the python scripts in 'experiment_scripts' folder, which were also used to conducted the experiments in the papers. 38 | 39 | 40 | # References 41 | 42 | [1] Tabular Benchmarks for Joint Architecture and Hyperparameter Optimization 43 | A. Klein and F. Hutter 44 | arXiv:1905.04970 [cs.LG] 45 | 46 | [2] NAS-Bench-101: Towards Reproducible Neural Architecture Search 47 | C. Ying and A. Klein and E. Real and E. Christiansen and K. Murphy and F. Hutter 48 | arXiv:1902.09635 [cs.LG] 49 | -------------------------------------------------------------------------------- /experiment_scripts/run_random_search.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import argparse 4 | 5 | from tabular_benchmarks import FCNetProteinStructureBenchmark, FCNetSliceLocalizationBenchmark,\ 6 | FCNetNavalPropulsionBenchmark, FCNetParkinsonsTelemonitoringBenchmark 7 | from tabular_benchmarks import NASCifar10A, NASCifar10B, NASCifar10C 8 | 9 | parser = argparse.ArgumentParser() 10 | parser.add_argument('--run_id', default=0, type=int, nargs='?', help='unique number to identify this run') 11 | parser.add_argument('--benchmark', default="protein_structure", type=str, nargs='?', help='specifies the benchmark') 12 | parser.add_argument('--n_iters', default=100, type=int, nargs='?', help='number of iterations for optimization method') 13 | parser.add_argument('--output_path', default="./", type=str, nargs='?', 14 | help='specifies the path where the results will be saved') 15 | parser.add_argument('--data_dir', default="./", type=str, nargs='?', help='specifies the path to the tabular data') 16 | 17 | args = parser.parse_args() 18 | 19 | if args.benchmark == "nas_cifar10a": 20 | b = NASCifar10A(data_dir=args.data_dir) 21 | 22 | elif args.benchmark == "nas_cifar10b": 23 | b = NASCifar10B(data_dir=args.data_dir) 24 | 25 | elif args.benchmark == "nas_cifar10c": 26 | b = NASCifar10C(data_dir=args.data_dir) 27 | 28 | elif args.benchmark == "protein_structure": 29 | b = FCNetProteinStructureBenchmark(data_dir=args.data_dir) 30 | 31 | elif args.benchmark == "slice_localization": 32 | b = FCNetSliceLocalizationBenchmark(data_dir=args.data_dir) 33 | 34 | elif args.benchmark == "naval_propulsion": 35 | b = FCNetNavalPropulsionBenchmark(data_dir=args.data_dir) 36 | 37 | elif args.benchmark == "parkinsons_telemonitoring": 38 | b = FCNetParkinsonsTelemonitoringBenchmark(data_dir=args.data_dir) 39 | 40 | output_path = os.path.join(args.output_path, "random_search") 41 | os.makedirs(os.path.join(output_path), exist_ok=True) 42 | 43 | cs = b.get_configuration_space() 44 | 45 | runtime = [] 46 | regret = [] 47 | curr_incumbent = None 48 | curr_inc_value = None 49 | 50 | rt = 0 51 | X = [] 52 | for i in range(args.n_iters): 53 | config = cs.sample_configuration() 54 | 55 | b.objective_function(config) 56 | 57 | if args.benchmark == "nas_cifar10a" or args.benchmark == "nas_cifar10b" or args.benchmark == "nas_cifar10c": 58 | res = b.get_results(ignore_invalid_configs=True) 59 | else: 60 | res = b.get_results() 61 | 62 | fh = open(os.path.join(output_path, 'run_%d.json' % args.run_id), 'w') 63 | json.dump(res, fh) 64 | fh.close() 65 | -------------------------------------------------------------------------------- /experiment_scripts/run_tpe.py: -------------------------------------------------------------------------------- 1 | import os 2 | from copy import deepcopy 3 | import json 4 | import ConfigSpace 5 | import argparse 6 | 7 | from hyperopt import fmin, tpe, hp, STATUS_OK, Trials 8 | 9 | from tabular_benchmarks import FCNetProteinStructureBenchmark, FCNetSliceLocalizationBenchmark,\ 10 | FCNetNavalPropulsionBenchmark, FCNetParkinsonsTelemonitoringBenchmark 11 | from tabular_benchmarks import NASCifar10A, NASCifar10B, NASCifar10C 12 | 13 | parser = argparse.ArgumentParser() 14 | parser.add_argument('--run_id', default=0, type=int, nargs='?', help='unique number to identify this run') 15 | parser.add_argument('--benchmark', default="protein_structure", type=str, nargs='?', help='specifies the benchmark') 16 | parser.add_argument('--n_iters', default=100, type=int, nargs='?', help='number of iterations for optimization method') 17 | parser.add_argument('--output_path', default="./", type=str, nargs='?', 18 | help='specifies the path where the results will be saved') 19 | parser.add_argument('--data_dir', default="./", type=str, nargs='?', help='specifies the path to the tabular data') 20 | 21 | args = parser.parse_args() 22 | 23 | if args.benchmark == "nas_cifar10a": 24 | b = NASCifar10A(data_dir=args.data_dir) 25 | 26 | elif args.benchmark == "nas_cifar10b": 27 | b = NASCifar10B(data_dir=args.data_dir) 28 | 29 | elif args.benchmark == "nas_cifar10c": 30 | b = NASCifar10C(data_dir=args.data_dir) 31 | 32 | elif args.benchmark == "protein_structure": 33 | b = FCNetProteinStructureBenchmark(data_dir=args.data_dir) 34 | 35 | elif args.benchmark == "slice_localization": 36 | b = FCNetSliceLocalizationBenchmark(data_dir=args.data_dir) 37 | 38 | elif args.benchmark == "naval_propulsion": 39 | b = FCNetNavalPropulsionBenchmark(data_dir=args.data_dir) 40 | 41 | elif args.benchmark == "parkinsons_telemonitoring": 42 | b = FCNetParkinsonsTelemonitoringBenchmark(data_dir=args.data_dir) 43 | 44 | output_path = os.path.join(args.output_path, "tpe") 45 | os.makedirs(os.path.join(output_path), exist_ok=True) 46 | 47 | cs = b.get_configuration_space() 48 | 49 | space = {} 50 | for h in cs.get_hyperparameters(): 51 | if type(h) == ConfigSpace.hyperparameters.OrdinalHyperparameter: 52 | space[h.name] = hp.quniform(h.name, 0, len(h.sequence)-1, q=1) 53 | elif type(h) == ConfigSpace.hyperparameters.CategoricalHyperparameter: 54 | space[h.name] = hp.choice(h.name, h.choices) 55 | elif type(h) == ConfigSpace.hyperparameters.UniformIntegerHyperparameter: 56 | space[h.name] = hp.quniform(h.name, h.lower, h.upper, q=1) 57 | elif type(h) == ConfigSpace.hyperparameters.UniformFloatHyperparameter: 58 | space[h.name] = hp.uniform(h.name, h.lower, h.upper) 59 | 60 | 61 | def objective(x): 62 | config = deepcopy(x) 63 | for h in cs.get_hyperparameters(): 64 | if type(h) == ConfigSpace.hyperparameters.OrdinalHyperparameter: 65 | 66 | config[h.name] = h.sequence[int(x[h.name])] 67 | 68 | elif type(h) == ConfigSpace.hyperparameters.UniformIntegerHyperparameter: 69 | 70 | config[h.name] = int(x[h.name]) 71 | y, c = b.objective_function(config) 72 | 73 | return { 74 | 'config': config, 75 | 'loss': y, 76 | 'cost': c, 77 | 'status': STATUS_OK} 78 | 79 | 80 | trials = Trials() 81 | best = fmin(objective, 82 | space=space, 83 | algo=tpe.suggest, 84 | max_evals=args.n_iters, 85 | trials=trials) 86 | 87 | if args.benchmark == "nas_cifar10a" or args.benchmark == "nas_cifar10b" or args.benchmark == "nas_cifar10c": 88 | res = b.get_results(ignore_invalid_configs=True) 89 | else: 90 | res = b.get_results() 91 | 92 | fh = open(os.path.join(output_path, 'run_%d.json' % args.run_id), 'w') 93 | json.dump(res, fh) 94 | fh.close() 95 | -------------------------------------------------------------------------------- /experiment_scripts/run_smac.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | import os 4 | 5 | from smac.facade.smac_facade import SMAC 6 | from smac.scenario.scenario import Scenario 7 | from smac.tae.execute_func import ExecuteTAFuncDict 8 | from tabular_benchmarks import FCNetProteinStructureBenchmark, FCNetSliceLocalizationBenchmark,\ 9 | FCNetNavalPropulsionBenchmark, FCNetParkinsonsTelemonitoringBenchmark 10 | from tabular_benchmarks import NASCifar10A, NASCifar10B, NASCifar10C 11 | 12 | parser = argparse.ArgumentParser() 13 | parser.add_argument('--run_id', default=0, type=int, nargs='?', help='unique number to identify this run') 14 | parser.add_argument('--benchmark', default="protein_structure", type=str, nargs='?', help='specifies the benchmark') 15 | parser.add_argument('--n_iters', default=100, type=int, nargs='?', help='number of iterations for optimization method') 16 | parser.add_argument('--output_path', default="./", type=str, nargs='?', 17 | help='specifies the path where the results will be saved') 18 | parser.add_argument('--data_dir', default="./", type=str, nargs='?', help='specifies the path to the tabular data') 19 | parser.add_argument('--n_trees', default=10, type=int, nargs='?', help='number of trees for the random forest') 20 | parser.add_argument('--random_fraction', default=.33, type=float, nargs='?', help='fraction of random configurations') 21 | parser.add_argument('--max_feval', default=4, type=int, nargs='?', 22 | help='maximum number of function evaluation per configuration') 23 | 24 | args = parser.parse_args() 25 | 26 | if args.benchmark == "nas_cifar10a": 27 | b = NASCifar10A(data_dir=args.data_dir) 28 | 29 | elif args.benchmark == "nas_cifar10b": 30 | b = NASCifar10B(data_dir=args.data_dir) 31 | 32 | elif args.benchmark == "nas_cifar10c": 33 | b = NASCifar10C(data_dir=args.data_dir) 34 | 35 | elif args.benchmark == "protein_structure": 36 | b = FCNetProteinStructureBenchmark(data_dir=args.data_dir) 37 | 38 | elif args.benchmark == "slice_localization": 39 | b = FCNetSliceLocalizationBenchmark(data_dir=args.data_dir) 40 | 41 | elif args.benchmark == "naval_propulsion": 42 | b = FCNetNavalPropulsionBenchmark(data_dir=args.data_dir) 43 | 44 | elif args.benchmark == "parkinsons_telemonitoring": 45 | b = FCNetParkinsonsTelemonitoringBenchmark(data_dir=args.data_dir) 46 | 47 | output_path = os.path.join(args.output_path, "smac") 48 | os.makedirs(os.path.join(output_path), exist_ok=True) 49 | 50 | cs = b.get_configuration_space() 51 | 52 | scenario = Scenario({"run_obj": "quality", 53 | "runcount-limit": args.n_iters, 54 | "cs": cs, 55 | "deterministic": "false", 56 | "initial_incumbent": "RANDOM", 57 | "output_dir": ""}) 58 | 59 | 60 | def objective_function(config, **kwargs): 61 | y, c = b.objective_function(config) 62 | return float(y) 63 | 64 | 65 | tae = ExecuteTAFuncDict(objective_function, use_pynisher=False) 66 | smac = SMAC(scenario=scenario, tae_runner=tae) 67 | 68 | # probability for random configurations 69 | 70 | smac.solver.random_configuration_chooser.prob = args.random_fraction 71 | smac.solver.model.rf_opts.num_trees = args.n_trees 72 | # only 1 configuration per SMBO iteration 73 | smac.solver.scenario.intensification_percentage = 1e-10 74 | smac.solver.intensifier.min_chall = 1 75 | # maximum number of function evaluations per configuration 76 | smac.solver.intensifier.maxR = args.max_feval 77 | 78 | smac.optimize() 79 | 80 | if args.benchmark == "nas_cifar10a" or args.benchmark == "nas_cifar10b" or args.benchmark == "nas_cifar10c": 81 | res = b.get_results(ignore_invalid_configs=True) 82 | else: 83 | res = b.get_results() 84 | 85 | fh = open(os.path.join(output_path, 'run_%d.json' % args.run_id), 'w') 86 | json.dump(res, fh) 87 | fh.close() 88 | -------------------------------------------------------------------------------- /experiment_scripts/run_hyperband.py: -------------------------------------------------------------------------------- 1 | import os 2 | import ConfigSpace 3 | import json 4 | import argparse 5 | import logging 6 | logging.basicConfig(level=logging.ERROR) 7 | 8 | from hpbandster.optimizers.hyperband import HyperBand 9 | import hpbandster.core.nameserver as hpns 10 | from hpbandster.core.worker import Worker 11 | 12 | from tabular_benchmarks import FCNetProteinStructureBenchmark, FCNetSliceLocalizationBenchmark,\ 13 | FCNetNavalPropulsionBenchmark, FCNetParkinsonsTelemonitoringBenchmark 14 | from tabular_benchmarks import NASCifar10A, NASCifar10B, NASCifar10C 15 | 16 | parser = argparse.ArgumentParser() 17 | parser.add_argument('--run_id', default=0, type=int, nargs='?', help='unique number to identify this run') 18 | parser.add_argument('--benchmark', default="wrn_cifar10", type=str, nargs='?', help='specifies the benchmark') 19 | parser.add_argument('--n_iters', default=100, type=int, nargs='?', help='number of iterations for optimization method') 20 | parser.add_argument('--output_path', default="./", type=str, nargs='?', 21 | help='specifies the path where the results will be saved') 22 | parser.add_argument('--data_dir', default="./", type=str, nargs='?', help='specifies the path to the tabular data') 23 | parser.add_argument('--eta', default=3, type=int, nargs='?', help='eta parameter of successive halving') 24 | 25 | args = parser.parse_args() 26 | 27 | if args.benchmark == "nas_cifar10a": 28 | min_budget = 4 29 | max_budget = 108 30 | b = NASCifar10A(data_dir=args.data_dir) 31 | 32 | elif args.benchmark == "nas_cifar10b": 33 | b = NASCifar10B(data_dir=args.data_dir) 34 | min_budget = 4 35 | max_budget = 108 36 | 37 | elif args.benchmark == "nas_cifar10c": 38 | b = NASCifar10C(data_dir=args.data_dir) 39 | min_budget = 4 40 | max_budget = 108 41 | 42 | elif args.benchmark == "protein_structure": 43 | b = FCNetProteinStructureBenchmark(data_dir=args.data_dir) 44 | min_budget = 3 45 | max_budget = 100 46 | 47 | elif args.benchmark == "slice_localization": 48 | b = FCNetSliceLocalizationBenchmark(data_dir=args.data_dir) 49 | min_budget = 3 50 | max_budget = 100 51 | 52 | elif args.benchmark == "naval_propulsion": 53 | b = FCNetNavalPropulsionBenchmark(data_dir=args.data_dir) 54 | min_budget = 3 55 | max_budget = 100 56 | 57 | elif args.benchmark == "parkinsons_telemonitoring": 58 | b = FCNetParkinsonsTelemonitoringBenchmark(data_dir=args.data_dir) 59 | min_budget = 3 60 | max_budget = 100 61 | 62 | output_path = os.path.join(args.output_path, "hyperband") 63 | os.makedirs(os.path.join(output_path), exist_ok=True) 64 | 65 | cs = b.get_configuration_space() 66 | 67 | 68 | class MyWorker(Worker): 69 | def compute(self, config, budget, *args, **kwargs): 70 | c = ConfigSpace.Configuration(cs, values=config) 71 | y, cost = b.objective_function(c, budget=int(budget)) 72 | return ({ 73 | 'loss': float(y), 74 | 'info': float(cost)}) 75 | 76 | 77 | hb_run_id = '0' 78 | 79 | NS = hpns.NameServer(run_id=hb_run_id, host='localhost', port=0) 80 | ns_host, ns_port = NS.start() 81 | 82 | num_workers = 1 83 | 84 | workers = [] 85 | for i in range(num_workers): 86 | w = MyWorker(nameserver=ns_host, nameserver_port=ns_port, 87 | run_id=hb_run_id, 88 | id=i) 89 | w.run(background=True) 90 | workers.append(w) 91 | 92 | HB = HyperBand(configspace=cs, 93 | run_id=hb_run_id, 94 | eta=args.eta, min_budget=min_budget, max_budget=max_budget, 95 | nameserver=ns_host, 96 | nameserver_port=ns_port, 97 | ping_interval=10) 98 | 99 | results = HB.run(args.n_iters, min_n_workers=num_workers) 100 | 101 | HB.shutdown(shutdown_workers=True) 102 | NS.shutdown() 103 | 104 | if args.benchmark == "nas_cifar10a" or args.benchmark == "nas_cifar10b" or args.benchmark == "nas_cifar10c": 105 | res = b.get_results(ignore_invalid_configs=True) 106 | else: 107 | res = b.get_results() 108 | 109 | fh = open(os.path.join(output_path, 'run_%d.json' % args.run_id), 'w') 110 | json.dump(res, fh) 111 | fh.close() 112 | -------------------------------------------------------------------------------- /test/test_tabular_benchmarks/test_tabular_nas_cifar10.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from tabular_benchmarks import NASCifar10A, NASCifar10B, NASCifar10C 4 | 5 | 6 | class TestNASCifar10A(unittest.TestCase): 7 | 8 | def setUp(self): 9 | self.b = NASCifar10A(data_dir="./") 10 | 11 | def test_fix_configuration(self): 12 | cs = self.b.get_configuration_space() 13 | config = cs.sample_configuration() 14 | # inception architecture 15 | config["op_node_0"] = 'conv1x1-bn-relu' 16 | config["op_node_1"] = 'conv3x3-bn-relu' 17 | config["op_node_2"] = 'conv3x3-bn-relu' 18 | config["op_node_3"] = 'conv3x3-bn-relu' 19 | config["op_node_4"] = 'maxpool3x3' 20 | 21 | for i in range(21): 22 | config["edge_%d" % i] = 0 23 | 24 | config["edge_0"] = 1 25 | config["edge_1"] = 1 26 | config["edge_2"] = 1 27 | config["edge_4"] = 1 28 | config["edge_10"] = 1 29 | config["edge_14"] = 1 30 | config["edge_15"] = 1 31 | config["edge_19"] = 1 32 | config["edge_20"] = 1 33 | 34 | max_epochs = 108 35 | y, cost = self.b.objective_function(config, max_epochs) 36 | 37 | mean_test_error = self.b.y_star_test + self.b.get_results()['regret_test'][0] 38 | mean_test_acc = 1 - mean_test_error 39 | 40 | assert mean_test_acc == 0.9308560291926066 41 | 42 | def test_random_sampling(self): 43 | config = self.b.get_configuration_space().sample_configuration() 44 | self.b.objective_function(config) 45 | 46 | 47 | class TestNASCifar10B(unittest.TestCase): 48 | 49 | def setUp(self): 50 | self.b = NASCifar10B(data_dir="./") 51 | 52 | def test_fix_configuration(self): 53 | cs = self.b.get_configuration_space() 54 | config = cs.sample_configuration() 55 | # inception architecture 56 | config["op_node_0"] = 'conv1x1-bn-relu' 57 | config["op_node_1"] = 'conv3x3-bn-relu' 58 | config["op_node_2"] = 'conv3x3-bn-relu' 59 | config["op_node_3"] = 'conv3x3-bn-relu' 60 | config["op_node_4"] = 'maxpool3x3' 61 | 62 | config["edge_0"] = 0 63 | config["edge_1"] = 1 64 | config["edge_2"] = 2 65 | config["edge_3"] = 4 66 | config["edge_4"] = 10 67 | config["edge_5"] = 14 68 | config["edge_6"] = 15 69 | config["edge_7"] = 19 70 | config["edge_8"] = 20 71 | 72 | max_epochs = 108 73 | y, cost = self.b.objective_function(config, max_epochs) 74 | 75 | mean_test_error = self.b.y_star_test + self.b.get_results()['regret_test'][0] 76 | mean_test_acc = 1 - mean_test_error 77 | 78 | assert mean_test_acc == 0.9308560291926066 79 | 80 | def test_random_sampling(self): 81 | config = self.b.get_configuration_space().sample_configuration() 82 | self.b.objective_function(config) 83 | 84 | 85 | class TestNASCifar10C(unittest.TestCase): 86 | 87 | def setUp(self): 88 | self.b = NASCifar10C(data_dir="./") 89 | 90 | def test_fix_configuration(self): 91 | cs = self.b.get_configuration_space() 92 | config = cs.sample_configuration() 93 | # inception architecture 94 | config["op_node_0"] = 'conv1x1-bn-relu' 95 | config["op_node_1"] = 'conv3x3-bn-relu' 96 | config["op_node_2"] = 'conv3x3-bn-relu' 97 | config["op_node_3"] = 'conv3x3-bn-relu' 98 | config["op_node_4"] = 'maxpool3x3' 99 | 100 | from tabular_benchmarks.nas_cifar10 import VERTICES 101 | for i in range(VERTICES * (VERTICES - 1) // 2): 102 | config["edge_%d" % i] = 0 103 | 104 | config["edge_0"] = 1 105 | config["edge_1"] = 1 106 | config["edge_2"] = 1 107 | config["edge_4"] = 1 108 | config["edge_10"] = 1 109 | config["edge_14"] = 1 110 | config["edge_15"] = 1 111 | config["edge_19"] = 1 112 | config["edge_20"] = 1 113 | config["num_edges"] = 9 114 | 115 | max_epochs = 108 116 | 117 | y, cost = self.b.objective_function(config, max_epochs) 118 | 119 | mean_test_error = self.b.y_star_test + self.b.get_results()['regret_test'][0] 120 | mean_test_acc = 1 - mean_test_error 121 | 122 | assert mean_test_acc == 0.9308560291926066 123 | 124 | def test_random_sampling(self): 125 | config = self.b.get_configuration_space().sample_configuration() 126 | self.b.objective_function(config) 127 | 128 | 129 | if __name__ == '__main__': 130 | unittest.main() 131 | -------------------------------------------------------------------------------- /data_collection/train_fcnet.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | import json 4 | import numpy as np 5 | import functools 6 | 7 | import argparse 8 | 9 | from keras.models import Sequential 10 | from keras.layers import Dense, Dropout 11 | from keras.optimizers import Adam 12 | from keras.callbacks import LearningRateScheduler 13 | 14 | import keras.backend as K 15 | 16 | 17 | def get_data(path, dataset="protein_structure"): 18 | train = np.array(np.load(os.path.join(path, "%s_train_data.npy" % dataset)), dtype=np.float64) 19 | train_targets = np.array(np.load(os.path.join(path, "%s_train_targets.npy" % dataset)), dtype=np.float64) 20 | 21 | valid = np.array(np.load(os.path.join(path, "%s_valid_data.npy" % dataset)), dtype=np.float64) 22 | valid_targets = np.array(np.load(os.path.join(path, "%s_valid_targets.npy" % dataset)), dtype=np.float64) 23 | 24 | test = np.array(np.load(os.path.join(path, "%s_test_data.npy" % dataset)), dtype=np.float64) 25 | test_targets = np.array(np.load(os.path.join(path, "%s_test_targets.npy" % dataset)), dtype=np.float64) 26 | 27 | m = np.mean(train, axis=0) 28 | s = np.std(train, axis=0) 29 | 30 | train = (train - m) / s 31 | valid = (valid - m) / s 32 | test = (test - m) / s 33 | 34 | m = np.mean(train_targets, axis=0) 35 | s = np.std(train_targets, axis=0) 36 | 37 | train_targets = (train_targets - m) / s 38 | valid_targets = (valid_targets - m) / s 39 | test_targets = (test_targets - m) / s 40 | return train, train_targets, valid, valid_targets, test, test_targets 41 | 42 | 43 | def mean_squared_err(y_true, y_pred): 44 | return K.mean((y_true - y_pred)**2) 45 | 46 | 47 | def fix(epoch, initial_lr): 48 | return initial_lr 49 | 50 | 51 | def exponential(epoch, initial_lr, T_max, decay_rate=0.96): 52 | return initial_lr * decay_rate ** (epoch / T_max) 53 | 54 | 55 | def cosine(epoch, initial_lr, T_max): 56 | final_lr = 0 57 | return final_lr + (initial_lr - final_lr) / 2 * (1 + np.cos(np.pi * epoch / T_max)) 58 | 59 | 60 | def main(args): 61 | """Builds, trains, and evaluates the model.""" 62 | 63 | x_train, y_train, x_valid, y_valid, x_test, y_test = get_data(path=args["data_dir"], 64 | dataset=args["dataset"]) 65 | 66 | model = Sequential() 67 | model.add(Dense(args["n_units_1"], activation=args["activation_fn_1"], input_dim=x_train.shape[1])) 68 | model.add(Dropout(args["dropout_1"])) 69 | model.add(Dense(args["n_units_2"], activation=args["activation_fn_2"])) 70 | model.add(Dropout(args["dropout_2"])) 71 | model.add(Dense(1, activation='linear')) 72 | 73 | adam = Adam(lr=args["init_lr"]) 74 | model.compile(loss='mean_squared_error', 75 | optimizer=adam, 76 | metrics=[mean_squared_err]) 77 | 78 | if args["lr_schedule"] == "cosine": 79 | schedule = functools.partial(cosine, initial_lr=args["init_lr"], T_max=args["n_epochs"]) 80 | 81 | if args["lr_schedule"] == "exponential": 82 | schedule = functools.partial(exponential, initial_lr=args["init_lr"], T_max=args["n_epochs"]) 83 | 84 | elif args["lr_schedule"] == "const": 85 | schedule = functools.partial(fix, initial_lr=args["init_lr"]) 86 | 87 | lrate = LearningRateScheduler(schedule) 88 | 89 | callbacks_list = [lrate] 90 | st = time.time() 91 | 92 | hist = model.fit(x_train, y_train, 93 | epochs=args["n_epochs"], 94 | batch_size=args["batch_size"], 95 | validation_data=(x_valid, y_valid), 96 | callbacks=callbacks_list, 97 | verbose=2) 98 | 99 | final_perf = model.evaluate(x_test, y_test, batch_size=args["batch_size"])[1] 100 | 101 | config = { 102 | "n_units_1": args["n_units_1"], 103 | "n_units_2": args["n_units_2"], 104 | "dropout_1": args["dropout_1"], 105 | "dropout_2": args["dropout_2"], 106 | "activation_fn_1": args["activation_fn_1"], 107 | "activation_fn_2": args["activation_fn_2"], 108 | "init_lr": args["init_lr"], 109 | "lr_schedule": args["lr_schedule"], 110 | "batch_size": args["batch_size"]} 111 | 112 | r = dict() 113 | r["config"] = config 114 | r["runtime"] = time.time() - st 115 | r["train_loss"] = hist.history["loss"] 116 | r["valid_loss"] = hist.history["val_loss"] 117 | r["train_mse"] = hist.history["mean_squared_err"] 118 | r["valid_mse"] = hist.history["val_mean_squared_err"] 119 | r["final_test_error"] = final_perf 120 | r["n_params"] = int(model.count_params()) 121 | 122 | os.makedirs(args["model_dir"], exist_ok=True) 123 | json.dump(r, open(os.path.join(args["model_dir"], "result.json"), "w")) 124 | 125 | 126 | if __name__ == "__main__": 127 | parser = argparse.ArgumentParser() 128 | parser.add_argument('--batch_size', default=128, type=int, nargs='?', help='batch size') 129 | parser.add_argument('--n_units_1', default=16, type=int, nargs='?', help='number of units in first layer') 130 | parser.add_argument('--n_units_2', default=16, type=int, nargs='?', help='number of units in second layer') 131 | parser.add_argument('--dropout_1', default=0, type=float, nargs='?', help='dropout in first layer') 132 | parser.add_argument('--dropout_2', default=0, type=float, nargs='?', help='dropout in second layer') 133 | parser.add_argument('--activation_fn_1', default="tanh", nargs='?', type=str, help='type of activation function ' 134 | 'in first layer') 135 | parser.add_argument('--activation_fn_2', default="tanh", nargs='?', type=str, help='type of activation function ' 136 | 'in second layer') 137 | parser.add_argument('--init_lr', default=1e-3, type=float, nargs='?', help='initial learning rate') 138 | parser.add_argument('--lr_schedule', default="cosine", nargs='?', type=str, help='learning rate schedule') 139 | parser.add_argument('--n_epochs', default=100, type=int, nargs='?', help='total number of epochs for training') 140 | 141 | parser.add_argument('--model_dir', nargs='?', default="./model_dir", type=str, help='specifies the directory where' 142 | 'the results will be saved') 143 | parser.add_argument('--data_dir', nargs='?', default="./protein_structure", type=str, 144 | help='path to the dataset') 145 | parser.add_argument('--dataset', default="protein_structure", nargs='?', type=str, help='name of the dataset') 146 | 147 | args = vars(parser.parse_args()) 148 | 149 | main(args) 150 | -------------------------------------------------------------------------------- /experiment_scripts/run_bohb.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | import logging 4 | import os 5 | 6 | import ConfigSpace 7 | 8 | logging.basicConfig(level=logging.ERROR) 9 | 10 | from hpbandster.optimizers.bohb import BOHB 11 | import hpbandster.core.nameserver as hpns 12 | from hpbandster.core.worker import Worker 13 | 14 | from tabular_benchmarks import FCNetProteinStructureBenchmark, FCNetSliceLocalizationBenchmark, \ 15 | FCNetNavalPropulsionBenchmark, FCNetParkinsonsTelemonitoringBenchmark 16 | from tabular_benchmarks import NASCifar10A, NASCifar10B, NASCifar10C 17 | 18 | parser = argparse.ArgumentParser() 19 | parser.add_argument('--run_id', default=0, type=int, nargs='?', help='unique number to identify this run') 20 | parser.add_argument('--benchmark', default="nas_cifar10a", type=str, nargs='?', help='specifies the benchmark') 21 | parser.add_argument('--n_iters', default=100, type=int, nargs='?', help='number of iterations for optimization method') 22 | parser.add_argument('--output_path', default="./", type=str, nargs='?', 23 | help='specifies the path where the results will be saved') 24 | parser.add_argument('--data_dir', default="./", type=str, nargs='?', help='specifies the path to the tabular data') 25 | parser.add_argument('--strategy', default="sampling", type=str, nargs='?', 26 | help='optimization strategy for the acquisition function') 27 | parser.add_argument('--min_bandwidth', default=.3, type=float, nargs='?', help='minimum bandwidth for KDE') 28 | parser.add_argument('--num_samples', default=64, type=int, nargs='?', 29 | help='number of samples for the acquisition function') 30 | parser.add_argument('--random_fraction', default=.33, type=float, nargs='?', help='fraction of random configurations') 31 | parser.add_argument('--bandwidth_factor', default=3, type=int, nargs='?', help='factor multiplied to the bandwidth') 32 | 33 | args = parser.parse_args() 34 | 35 | if args.benchmark == "nas_cifar10a": 36 | min_budget = 4 37 | max_budget = 108 38 | b = NASCifar10A(data_dir=args.data_dir) 39 | 40 | elif args.benchmark == "nas_cifar10b": 41 | b = NASCifar10B(data_dir=args.data_dir) 42 | min_budget = 4 43 | max_budget = 108 44 | 45 | elif args.benchmark == "nas_cifar10c": 46 | b = NASCifar10C(data_dir=args.data_dir) 47 | min_budget = 4 48 | max_budget = 108 49 | 50 | elif args.benchmark == "protein_structure": 51 | b = FCNetProteinStructureBenchmark(data_dir=args.data_dir) 52 | min_budget = 3 53 | max_budget = 100 54 | 55 | elif args.benchmark == "slice_localization": 56 | b = FCNetSliceLocalizationBenchmark(data_dir=args.data_dir) 57 | min_budget = 3 58 | max_budget = 100 59 | 60 | elif args.benchmark == "naval_propulsion": 61 | b = FCNetNavalPropulsionBenchmark(data_dir=args.data_dir) 62 | min_budget = 3 63 | max_budget = 100 64 | 65 | elif args.benchmark == "parkinsons_telemonitoring": 66 | b = FCNetParkinsonsTelemonitoringBenchmark(data_dir=args.data_dir) 67 | min_budget = 3 68 | max_budget = 100 69 | 70 | output_path = os.path.join(args.output_path, "bohb") 71 | os.makedirs(os.path.join(output_path), exist_ok=True) 72 | 73 | if args.benchmark == "protein_structure" or \ 74 | args.benchmark == "slice_localization" or args.benchmark == "naval_propulsion" \ 75 | or args.benchmark == "parkinsons_telemonitoring": 76 | cs = ConfigSpace.ConfigurationSpace() 77 | 78 | cs.add_hyperparameter(ConfigSpace.UniformIntegerHyperparameter("n_units_1", lower=0, upper=5)) 79 | cs.add_hyperparameter(ConfigSpace.UniformIntegerHyperparameter("n_units_2", lower=0, upper=5)) 80 | cs.add_hyperparameter(ConfigSpace.UniformIntegerHyperparameter("dropout_1", lower=0, upper=2)) 81 | cs.add_hyperparameter(ConfigSpace.UniformIntegerHyperparameter("dropout_2", lower=0, upper=2)) 82 | cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("activation_fn_1", ["tanh", "relu"])) 83 | cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("activation_fn_2", ["tanh", "relu"])) 84 | cs.add_hyperparameter( 85 | ConfigSpace.UniformIntegerHyperparameter("init_lr", lower=0, upper=5)) 86 | cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("lr_schedule", ["cosine", "const"])) 87 | cs.add_hyperparameter(ConfigSpace.UniformIntegerHyperparameter("batch_size", lower=0, upper=3)) 88 | else: 89 | cs = b.get_configuration_space() 90 | 91 | 92 | class MyWorker(Worker): 93 | def compute(self, config, budget, **kwargs): 94 | if args.benchmark == "protein_structure" \ 95 | or args.benchmark == "slice_localization" or args.benchmark == "naval_propulsion" \ 96 | or args.benchmark == "parkinsons_telemonitoring": 97 | 98 | original_cs = b.get_configuration_space() 99 | c = original_cs.sample_configuration() 100 | c["n_units_1"] = original_cs.get_hyperparameter("n_units_1").sequence[config["n_units_1"]] 101 | c["n_units_2"] = original_cs.get_hyperparameter("n_units_2").sequence[config["n_units_2"]] 102 | c["dropout_1"] = original_cs.get_hyperparameter("dropout_1").sequence[config["dropout_1"]] 103 | c["dropout_2"] = original_cs.get_hyperparameter("dropout_2").sequence[config["dropout_2"]] 104 | c["init_lr"] = original_cs.get_hyperparameter("init_lr").sequence[config["init_lr"]] 105 | c["batch_size"] = original_cs.get_hyperparameter("batch_size").sequence[config["batch_size"]] 106 | c["activation_fn_1"] = config["activation_fn_1"] 107 | c["activation_fn_2"] = config["activation_fn_2"] 108 | c["lr_schedule"] = config["lr_schedule"] 109 | y, cost = b.objective_function(c, budget=int(budget)) 110 | 111 | else: 112 | y, cost = b.objective_function(config, budget=int(budget)) 113 | 114 | return ({ 115 | 'loss': float(y), 116 | 'info': float(cost)}) 117 | 118 | 119 | hb_run_id = '0' 120 | 121 | NS = hpns.NameServer(run_id=hb_run_id, host='localhost', port=0) 122 | ns_host, ns_port = NS.start() 123 | 124 | num_workers = 1 125 | 126 | workers = [] 127 | for i in range(num_workers): 128 | w = MyWorker(nameserver=ns_host, nameserver_port=ns_port, 129 | run_id=hb_run_id, 130 | id=i) 131 | w.run(background=True) 132 | workers.append(w) 133 | 134 | bohb = BOHB(configspace=cs, 135 | run_id=hb_run_id, 136 | eta=3, min_budget=min_budget, max_budget=max_budget, 137 | nameserver=ns_host, 138 | nameserver_port=ns_port, 139 | optimization_strategy=args.strategy, num_samples=args.num_samples, 140 | random_fraction=args.random_fraction, bandwidth_factor=args.bandwidth_factor, 141 | ping_interval=10, min_bandwidth=args.min_bandwidth) 142 | 143 | results = bohb.run(args.n_iters, min_n_workers=num_workers) 144 | 145 | bohb.shutdown(shutdown_workers=True) 146 | NS.shutdown() 147 | 148 | if args.benchmark == "nas_cifar10a" or args.benchmark == "nas_cifar10b" or args.benchmark == "nas_cifar10c": 149 | res = b.get_results(ignore_invalid_configs=True) 150 | else: 151 | res = b.get_results() 152 | 153 | fh = open(os.path.join(output_path, 'run_%d.json' % args.run_id), 'w') 154 | json.dump(res, fh) 155 | fh.close() 156 | -------------------------------------------------------------------------------- /tabular_benchmarks/fcnet_benchmark.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | 4 | import ConfigSpace 5 | import h5py 6 | import numpy as np 7 | 8 | 9 | class FCNetBenchmark(object): 10 | 11 | def __init__(self, path, dataset="fcnet_protein_structure_data.hdf5", seed=None): 12 | 13 | cs = self.get_configuration_space() 14 | self.names = [h.name for h in cs.get_hyperparameters()] 15 | 16 | self.data = h5py.File(os.path.join(path, dataset), "r") 17 | 18 | self.X = [] 19 | self.y = [] 20 | self.c = [] 21 | 22 | self.rng = np.random.RandomState(seed) 23 | 24 | def reset_tracker(self): 25 | # __init__() sans the data loading for multiple runs 26 | self.X = [] 27 | self.y = [] 28 | self.c = [] 29 | self.rng = np.random.RandomState(seed) 30 | 31 | def get_best_configuration(self): 32 | 33 | """ 34 | Returns the best configuration in the dataset that achieves the lowest test performance. 35 | 36 | :return: Returns tuple with the best configuration, its final validation performance and its test performance 37 | """ 38 | 39 | configs, te, ve = [], [], [] 40 | for k in self.data.keys(): 41 | configs.append(json.loads(k)) 42 | te.append(np.mean(self.data[k]["final_test_error"])) 43 | ve.append(np.mean(self.data[k]["valid_mse"][:, -1])) 44 | 45 | b = np.argmin(te) 46 | 47 | return configs[b], ve[b], te[b] 48 | 49 | def objective_function(self, config, budget=100, **kwargs): 50 | 51 | assert 0 < budget <= 100 # check whether budget is in the correct bounds 52 | 53 | i = self.rng.randint(4) 54 | 55 | if type(config) == ConfigSpace.Configuration: 56 | k = json.dumps(config.get_dictionary(), sort_keys=True) 57 | else: 58 | k = json.dumps(config, sort_keys=True) 59 | 60 | valid = self.data[k]["valid_mse"][i] 61 | runtime = self.data[k]["runtime"][i] 62 | 63 | time_per_epoch = runtime / 100 # divide by the maximum number of epochs 64 | 65 | rt = time_per_epoch * budget 66 | 67 | self.X.append(config) 68 | self.y.append(valid[budget - 1]) 69 | self.c.append(rt) 70 | 71 | return valid[budget - 1], rt 72 | 73 | def objective_function_learning_curve(self, config, budget=100): 74 | 75 | assert 0 < budget <= 100 # check whether budget is in the correct bounds 76 | 77 | index = self.rng.randint(4) 78 | 79 | if type(config) == ConfigSpace.Configuration: 80 | k = json.dumps(config.get_dictionary(), sort_keys=True) 81 | else: 82 | k = json.dumps(config, sort_keys=True) 83 | 84 | lc = [self.data[k]["valid_mse"][index][i] for i in range(budget)] 85 | runtime = self.data[k]["runtime"][index] 86 | 87 | time_per_epoch = runtime / 100 # divide by the maximum number of epochs 88 | 89 | rt = [time_per_epoch * (i + 1) for i in range(budget)] 90 | 91 | self.X.append(config) 92 | self.y.append(lc[-1]) 93 | self.c.append(rt[-1]) 94 | 95 | return lc, rt 96 | 97 | def objective_function_deterministic(self, config, budget=100, index=0, **kwargs): 98 | 99 | assert 0 < budget <= 100 # check whether budget is in the correct bounds 100 | 101 | if type(config) == ConfigSpace.Configuration: 102 | k = json.dumps(config.get_dictionary(), sort_keys=True) 103 | else: 104 | k = json.dumps(config, sort_keys=True) 105 | 106 | valid = self.data[k]["valid_mse"][index] 107 | runtime = self.data[k]["runtime"][index] 108 | 109 | time_per_epoch = runtime / 100 # divide by the maximum number of epochs 110 | 111 | rt = time_per_epoch * budget 112 | 113 | self.X.append(config) 114 | self.y.append(valid[budget - 1]) 115 | self.c.append(rt) 116 | 117 | return valid[budget - 1], rt 118 | 119 | def objective_function_test(self, config, **kwargs): 120 | 121 | if type(config) == ConfigSpace.Configuration: 122 | k = json.dumps(config.get_dictionary(), sort_keys=True) 123 | else: 124 | k = json.dumps(config, sort_keys=True) 125 | 126 | test = np.mean(self.data[k]["final_test_error"]) 127 | runtime = np.mean(self.data[k]["runtime"]) 128 | 129 | return test, runtime 130 | 131 | def get_results(self): 132 | 133 | inc, y_star_valid, y_star_test = self.get_best_configuration() 134 | 135 | regret_validation = [] 136 | regret_test = [] 137 | runtime = [] 138 | rt = 0 139 | 140 | inc_valid = np.inf 141 | inc_test = np.inf 142 | 143 | for i in range(len(self.X)): 144 | 145 | if inc_valid > self.y[i]: 146 | inc_valid = self.y[i] 147 | inc_test, _ = self.objective_function_test(self.X[i]) 148 | 149 | regret_validation.append(float(inc_valid - y_star_valid)) 150 | regret_test.append(float(inc_test - y_star_test)) 151 | rt += self.c[i] 152 | runtime.append(float(rt)) 153 | 154 | res = dict() 155 | res['regret_validation'] = regret_validation 156 | res['regret_test'] = regret_test 157 | res['runtime'] = runtime 158 | 159 | return res 160 | 161 | @staticmethod 162 | def get_configuration_space(): 163 | cs = ConfigSpace.ConfigurationSpace() 164 | 165 | cs.add_hyperparameter(ConfigSpace.OrdinalHyperparameter("n_units_1", [16, 32, 64, 128, 256, 512])) 166 | cs.add_hyperparameter(ConfigSpace.OrdinalHyperparameter("n_units_2", [16, 32, 64, 128, 256, 512])) 167 | cs.add_hyperparameter(ConfigSpace.OrdinalHyperparameter("dropout_1", [0.0, 0.3, 0.6])) 168 | cs.add_hyperparameter(ConfigSpace.OrdinalHyperparameter("dropout_2", [0.0, 0.3, 0.6])) 169 | cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("activation_fn_1", ["tanh", "relu"])) 170 | cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("activation_fn_2", ["tanh", "relu"])) 171 | cs.add_hyperparameter( 172 | ConfigSpace.OrdinalHyperparameter("init_lr", [5 * 1e-4, 1e-3, 5 * 1e-3, 1e-2, 5 * 1e-2, 1e-1])) 173 | cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("lr_schedule", ["cosine", "const"])) 174 | cs.add_hyperparameter(ConfigSpace.OrdinalHyperparameter("batch_size", [8, 16, 32, 64])) 175 | return cs 176 | 177 | 178 | class FCNetSliceLocalizationBenchmark(FCNetBenchmark): 179 | 180 | def __init__(self, data_dir="./"): 181 | super(FCNetSliceLocalizationBenchmark, self).__init__(path=data_dir, 182 | dataset="fcnet_slice_localization_data.hdf5") 183 | 184 | 185 | class FCNetProteinStructureBenchmark(FCNetBenchmark): 186 | 187 | def __init__(self, data_dir="./"): 188 | super(FCNetProteinStructureBenchmark, self).__init__(path=data_dir, dataset="fcnet_protein_structure_data.hdf5") 189 | 190 | 191 | class FCNetNavalPropulsionBenchmark(FCNetBenchmark): 192 | 193 | def __init__(self, data_dir="./"): 194 | super(FCNetNavalPropulsionBenchmark, self).__init__(path=data_dir, dataset="fcnet_naval_propulsion_data.hdf5") 195 | 196 | 197 | class FCNetParkinsonsTelemonitoringBenchmark(FCNetBenchmark): 198 | 199 | def __init__(self, data_dir="./"): 200 | super(FCNetParkinsonsTelemonitoringBenchmark, self).__init__(path=data_dir, 201 | dataset="fcnet_parkinsons_telemonitoring_data.hdf5") 202 | -------------------------------------------------------------------------------- /experiment_scripts/run_regularized_evolution.py: -------------------------------------------------------------------------------- 1 | """ 2 | Regularized evolution as described in: 3 | Real, E., Aggarwal, A., Huang, Y., and Le, Q. V. 4 | Regularized Evolution for Image Classifier Architecture Search. 5 | In Proceedings of the Conference on Artificial Intelligence (AAAI’19) 6 | 7 | The code is based one the original regularized evolution open-source implementation: 8 | https://colab.research.google.com/github/google-research/google-research/blob/master/evolution/regularized_evolution_algorithm/regularized_evolution.ipynb 9 | 10 | NOTE: This script has certain deviations from the original code owing to the search space of the benchmarks used: 11 | 1) The fitness function is not accuracy but error and hence the negative error is being maximized. 12 | 2) The architecture is a ConfigSpace object that defines the model architecture parameters. 13 | 14 | """ 15 | 16 | import argparse 17 | import collections 18 | import os 19 | import random 20 | import json 21 | from copy import deepcopy 22 | 23 | import ConfigSpace 24 | import numpy as np 25 | 26 | 27 | from tabular_benchmarks import FCNetProteinStructureBenchmark, FCNetSliceLocalizationBenchmark,\ 28 | FCNetNavalPropulsionBenchmark, FCNetParkinsonsTelemonitoringBenchmark 29 | from tabular_benchmarks import NASCifar10A, NASCifar10B 30 | 31 | 32 | class Model(object): 33 | """A class representing a model. 34 | 35 | It holds two attributes: `arch` (the simulated architecture) and `accuracy` 36 | (the simulated accuracy / fitness). See Appendix C for an introduction to 37 | this toy problem. 38 | 39 | In the real case of neural networks, `arch` would instead hold the 40 | architecture of the normal and reduction cells of a neural network and 41 | accuracy would be instead the result of training the neural net and 42 | evaluating it on the validation set. 43 | 44 | We do not include test accuracies here as they are not used by the algorithm 45 | in any way. In the case of real neural networks, the test accuracy is only 46 | used for the purpose of reporting / plotting final results. 47 | 48 | In the context of evolutionary algorithms, a model is often referred to as 49 | an "individual". 50 | 51 | Attributes: (as in the original code) 52 | arch: the architecture as an int representing a bit-string of length `DIM`. 53 | As a result, the integers are required to be less than `2**DIM`. They 54 | can be visualized as strings of 0s and 1s by calling `print(model)`, 55 | where `model` is an instance of this class. 56 | accuracy: the simulated validation accuracy. This is the sum of the 57 | bits in the bit-string, divided by DIM to produce a value in the 58 | interval [0.0, 1.0]. After that, a small amount of Gaussian noise is 59 | added with mean 0.0 and standard deviation `NOISE_STDEV`. The resulting 60 | number is clipped to within [0.0, 1.0] to produce the final validation 61 | accuracy of the model. A given model will have a fixed validation 62 | accuracy but two models that have the same architecture will generally 63 | have different validation accuracies due to this noise. In the context 64 | of evolutionary algorithms, this is often known as the "fitness". 65 | """ 66 | 67 | def __init__(self): 68 | self.arch = None 69 | self.accuracy = None 70 | 71 | def __str__(self): 72 | """Prints a readable version of this bitstring.""" 73 | return '{0:b}'.format(self.arch) 74 | 75 | 76 | def train_and_eval(config): 77 | y, cost = b.objective_function(config) 78 | # returns negative error (similar to maximizing accuracy) 79 | return -y 80 | 81 | 82 | def random_architecture(): 83 | config = cs.sample_configuration() 84 | return config 85 | 86 | 87 | def mutate_arch(parent_arch): 88 | # pick random parameter 89 | dim = np.random.randint(len(cs.get_hyperparameters())) 90 | hyper = cs.get_hyperparameters()[dim] 91 | 92 | if type(hyper) == ConfigSpace.OrdinalHyperparameter: 93 | choices = list(hyper.sequence) 94 | else: 95 | choices = list(hyper.choices) 96 | # drop current values from potential choices 97 | choices.remove(parent_arch[hyper.name]) 98 | 99 | # flip parameter 100 | idx = np.random.randint(len(choices)) 101 | 102 | child_arch = deepcopy(parent_arch) 103 | child_arch[hyper.name] = choices[idx] 104 | return child_arch 105 | 106 | 107 | def regularized_evolution(cycles, population_size, sample_size): 108 | """Algorithm for regularized evolution (i.e. aging evolution). 109 | 110 | Follows "Algorithm 1" in Real et al. "Regularized Evolution for Image 111 | Classifier Architecture Search". 112 | 113 | Args: 114 | cycles: the number of cycles the algorithm should run for. 115 | population_size: the number of individuals to keep in the population. 116 | sample_size: the number of individuals that should participate in each 117 | tournament. 118 | 119 | Returns: 120 | history: a list of `Model` instances, representing all the models computed 121 | during the evolution experiment. 122 | """ 123 | population = collections.deque() 124 | history = [] # Not used by the algorithm, only used to report results. 125 | 126 | # Initialize the population with random models. 127 | while len(population) < population_size: 128 | model = Model() 129 | model.arch = random_architecture() 130 | model.accuracy = train_and_eval(model.arch) 131 | population.append(model) 132 | history.append(model) 133 | 134 | # Carry out evolution in cycles. Each cycle produces a model and removes 135 | # another. 136 | while len(history) < cycles: 137 | # Sample randomly chosen models from the current population. 138 | sample = [] 139 | while len(sample) < sample_size: 140 | # Inefficient, but written this way for clarity. In the case of neural 141 | # nets, the efficiency of this line is irrelevant because training neural 142 | # nets is the rate-determining step. 143 | candidate = random.choice(list(population)) 144 | sample.append(candidate) 145 | 146 | # The parent is the best model in the sample. 147 | parent = max(sample, key=lambda i: i.accuracy) 148 | 149 | # Create the child model and store it. 150 | child = Model() 151 | child.arch = mutate_arch(parent.arch) 152 | child.accuracy = train_and_eval(child.arch) 153 | population.append(child) 154 | history.append(child) 155 | 156 | # Remove the oldest model. 157 | population.popleft() 158 | 159 | return history 160 | 161 | 162 | parser = argparse.ArgumentParser() 163 | parser.add_argument('--run_id', default=0, type=int, nargs='?', help='unique number to identify this run') 164 | parser.add_argument('--benchmark', default="protein_structure", type=str, nargs='?', help='specifies the benchmark') 165 | parser.add_argument('--n_iters', default=100, type=int, nargs='?', help='number of iterations for optimization method') 166 | parser.add_argument('--output_path', default="./", type=str, nargs='?', 167 | help='specifies the path where the results will be saved') 168 | parser.add_argument('--data_dir', default="./", type=str, nargs='?', help='specifies the path to the tabular data') 169 | parser.add_argument('--pop_size', default=100, type=int, nargs='?', help='population size') 170 | parser.add_argument('--sample_size', default=10, type=int, nargs='?', help='sample_size') 171 | 172 | 173 | args = parser.parse_args() 174 | 175 | if args.benchmark == "nas_cifar10a": 176 | b = NASCifar10A(data_dir=args.data_dir) 177 | 178 | elif args.benchmark == "nas_cifar10b": 179 | b = NASCifar10B(data_dir=args.data_dir) 180 | 181 | elif args.benchmark == "protein_structure": 182 | b = FCNetProteinStructureBenchmark(data_dir=args.data_dir) 183 | 184 | elif args.benchmark == "slice_localization": 185 | b = FCNetSliceLocalizationBenchmark(data_dir=args.data_dir) 186 | 187 | elif args.benchmark == "naval_propulsion": 188 | b = FCNetNavalPropulsionBenchmark(data_dir=args.data_dir) 189 | 190 | elif args.benchmark == "parkinsons_telemonitoring": 191 | b = FCNetParkinsonsTelemonitoringBenchmark(data_dir=args.data_dir) 192 | 193 | output_path = os.path.join(args.output_path, "regularized_evolution") 194 | os.makedirs(os.path.join(output_path), exist_ok=True) 195 | 196 | cs = b.get_configuration_space() 197 | 198 | history = regularized_evolution( 199 | cycles=args.n_iters, population_size=args.pop_size, sample_size=args.sample_size) 200 | 201 | if args.benchmark == "nas_cifar10a" or args.benchmark == "nas_cifar10b": 202 | res = b.get_results(ignore_invalid_configs=True) 203 | else: 204 | res = b.get_results() 205 | 206 | fh = open(os.path.join(output_path, 'run_%d.json' % args.run_id), 'w') 207 | json.dump(res, fh) 208 | fh.close() 209 | -------------------------------------------------------------------------------- /tabular_benchmarks/nas_cifar10.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import ConfigSpace 4 | import numpy as np 5 | from nasbench import api 6 | from nasbench.lib import graph_util 7 | 8 | MAX_EDGES = 9 9 | VERTICES = 7 10 | 11 | 12 | class NASCifar10(object): 13 | 14 | def __init__(self, data_dir, multi_fidelity=True): 15 | 16 | self.multi_fidelity = multi_fidelity 17 | if self.multi_fidelity: 18 | self.dataset = api.NASBench(os.path.join(data_dir, 'nasbench_full.tfrecord')) 19 | else: 20 | self.dataset = api.NASBench(os.path.join(data_dir, 'nasbench_only108.tfrecord')) 21 | self.X = [] 22 | self.y_valid = [] 23 | self.y_test = [] 24 | self.costs = [] 25 | 26 | self.y_star_valid = 0.04944576819737756 # lowest mean validation error 27 | self.y_star_test = 0.056824247042338016 # lowest mean test error 28 | 29 | def reset_tracker(self): 30 | # __init__() sans the data loading for multiple runs 31 | self.X = [] 32 | self.y_valid = [] 33 | self.y_test = [] 34 | self.costs = [] 35 | 36 | @staticmethod 37 | def objective_function(self, config): 38 | pass 39 | 40 | def record_invalid(self, config, valid, test, costs): 41 | self.X.append(config) 42 | self.y_valid.append(valid) 43 | self.y_test.append(test) 44 | self.costs.append(costs) 45 | 46 | def record_valid(self, config, data, model_spec): 47 | 48 | self.X.append(config) 49 | 50 | # compute mean test error for the final budget 51 | _, metrics = self.dataset.get_metrics_from_spec(model_spec) 52 | mean_test_error = 1 - np.mean([metrics[108][i]["final_test_accuracy"] for i in range(3)]) 53 | self.y_test.append(mean_test_error) 54 | 55 | # compute validation error for the chosen budget 56 | valid_error = 1 - data["validation_accuracy"] 57 | self.y_valid.append(valid_error) 58 | 59 | runtime = data["training_time"] 60 | self.costs.append(runtime) 61 | 62 | @staticmethod 63 | def get_configuration_space(): 64 | pass 65 | 66 | def get_results(self, ignore_invalid_configs=False): 67 | 68 | regret_validation = [] 69 | regret_test = [] 70 | runtime = [] 71 | rt = 0 72 | 73 | inc_valid = np.inf 74 | inc_test = np.inf 75 | 76 | for i in range(len(self.X)): 77 | 78 | if ignore_invalid_configs and self.costs[i] == 0: 79 | continue 80 | 81 | if inc_valid > self.y_valid[i]: 82 | inc_valid = self.y_valid[i] 83 | inc_test = self.y_test[i] 84 | 85 | regret_validation.append(float(inc_valid - self.y_star_valid)) 86 | regret_test.append(float(inc_test - self.y_star_test)) 87 | rt += self.costs[i] 88 | runtime.append(float(rt)) 89 | 90 | res = dict() 91 | res['regret_validation'] = regret_validation 92 | res['regret_test'] = regret_test 93 | res['runtime'] = runtime 94 | 95 | return res 96 | 97 | 98 | class NASCifar10A(NASCifar10): 99 | def objective_function(self, config, budget=108): 100 | if self.multi_fidelity is False: 101 | assert budget == 108 102 | 103 | matrix = np.zeros([VERTICES, VERTICES], dtype=np.int8) 104 | idx = np.triu_indices(matrix.shape[0], k=1) 105 | for i in range(VERTICES * (VERTICES - 1) // 2): 106 | row = idx[0][i] 107 | col = idx[1][i] 108 | matrix[row, col] = config["edge_%d" % i] 109 | 110 | # if not graph_util.is_full_dag(matrix) or graph_util.num_edges(matrix) > MAX_EDGES: 111 | if graph_util.num_edges(matrix) > MAX_EDGES: 112 | self.record_invalid(config, 1, 1, 0) 113 | return 1, 0 114 | 115 | labeling = [config["op_node_%d" % i] for i in range(5)] 116 | labeling = ['input'] + list(labeling) + ['output'] 117 | model_spec = api.ModelSpec(matrix, labeling) 118 | try: 119 | data = self.dataset.query(model_spec, epochs=budget) 120 | except api.OutOfDomainError: 121 | self.record_invalid(config, 1, 1, 0) 122 | return 1, 0 123 | 124 | self.record_valid(config, data, model_spec) 125 | return 1 - data["validation_accuracy"], data["training_time"] 126 | 127 | @staticmethod 128 | def get_configuration_space(): 129 | cs = ConfigSpace.ConfigurationSpace() 130 | 131 | ops_choices = ['conv1x1-bn-relu', 'conv3x3-bn-relu', 'maxpool3x3'] 132 | cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_0", ops_choices)) 133 | cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_1", ops_choices)) 134 | cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_2", ops_choices)) 135 | cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_3", ops_choices)) 136 | cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_4", ops_choices)) 137 | for i in range(VERTICES * (VERTICES - 1) // 2): 138 | cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("edge_%d" % i, [0, 1])) 139 | return cs 140 | 141 | 142 | class NASCifar10B(NASCifar10): 143 | def objective_function(self, config, budget=108): 144 | if self.multi_fidelity is False: 145 | assert budget == 108 146 | 147 | bitlist = [0] * (VERTICES * (VERTICES - 1) // 2) 148 | for i in range(MAX_EDGES): 149 | bitlist[config["edge_%d" % i]] = 1 150 | out = 0 151 | for bit in bitlist: 152 | out = (out << 1) | bit 153 | 154 | matrix = np.fromfunction(graph_util.gen_is_edge_fn(out), 155 | (VERTICES, VERTICES), 156 | dtype=np.int8) 157 | # if not graph_util.is_full_dag(matrix) or graph_util.num_edges(matrix) > MAX_EDGES: 158 | if graph_util.num_edges(matrix) > MAX_EDGES: 159 | self.record_invalid(config, 1, 1, 0) 160 | return 1, 0 161 | 162 | labeling = [config["op_node_%d" % i] for i in range(5)] 163 | labeling = ['input'] + list(labeling) + ['output'] 164 | model_spec = api.ModelSpec(matrix, labeling) 165 | try: 166 | data = self.dataset.query(model_spec, epochs=budget) 167 | except api.OutOfDomainError: 168 | self.record_invalid(config, 1, 1, 0) 169 | return 1, 0 170 | 171 | self.record_valid(config, data, model_spec) 172 | 173 | return 1 - data["validation_accuracy"], data["training_time"] 174 | 175 | @staticmethod 176 | def get_configuration_space(): 177 | cs = ConfigSpace.ConfigurationSpace() 178 | 179 | ops_choices = ['conv1x1-bn-relu', 'conv3x3-bn-relu', 'maxpool3x3'] 180 | cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_0", ops_choices)) 181 | cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_1", ops_choices)) 182 | cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_2", ops_choices)) 183 | cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_3", ops_choices)) 184 | cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_4", ops_choices)) 185 | cat = [i for i in range((VERTICES * (VERTICES - 1)) // 2)] 186 | for i in range(MAX_EDGES): 187 | cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("edge_%d" % i, cat)) 188 | return cs 189 | 190 | 191 | class NASCifar10C(NASCifar10): 192 | def objective_function(self, config, budget=108): 193 | if self.multi_fidelity is False: 194 | assert budget == 108 195 | 196 | edge_prob = [] 197 | for i in range(VERTICES * (VERTICES - 1) // 2): 198 | edge_prob.append(config["edge_%d" % i]) 199 | 200 | idx = np.argsort(edge_prob)[::-1][:config["num_edges"]] 201 | binay_encoding = np.zeros(len(edge_prob)) 202 | binay_encoding[idx] = 1 203 | matrix = np.zeros([VERTICES, VERTICES], dtype=np.int8) 204 | idx = np.triu_indices(matrix.shape[0], k=1) 205 | for i in range(VERTICES * (VERTICES - 1) // 2): 206 | row = idx[0][i] 207 | col = idx[1][i] 208 | matrix[row, col] = binay_encoding[i] 209 | 210 | if graph_util.num_edges(matrix) > MAX_EDGES: 211 | self.record_invalid(config, 1, 1, 0) 212 | return 1, 0 213 | 214 | labeling = [config["op_node_%d" % i] for i in range(5)] 215 | labeling = ['input'] + list(labeling) + ['output'] 216 | model_spec = api.ModelSpec(matrix, labeling) 217 | try: 218 | data = self.dataset.query(model_spec, epochs=budget) 219 | except api.OutOfDomainError: 220 | self.record_invalid(config, 1, 1, 0) 221 | return 1, 0 222 | 223 | self.record_valid(config, data, model_spec) 224 | 225 | return 1 - data["validation_accuracy"], data["training_time"] 226 | 227 | @staticmethod 228 | def get_configuration_space(): 229 | cs = ConfigSpace.ConfigurationSpace() 230 | 231 | ops_choices = ['conv1x1-bn-relu', 'conv3x3-bn-relu', 'maxpool3x3'] 232 | cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_0", ops_choices)) 233 | cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_1", ops_choices)) 234 | cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_2", ops_choices)) 235 | cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_3", ops_choices)) 236 | cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_4", ops_choices)) 237 | 238 | cs.add_hyperparameter(ConfigSpace.UniformIntegerHyperparameter("num_edges", 0, MAX_EDGES)) 239 | 240 | for i in range(VERTICES * (VERTICES - 1) // 2): 241 | cs.add_hyperparameter(ConfigSpace.UniformFloatHyperparameter("edge_%d" % i, 0, 1)) 242 | return cs 243 | -------------------------------------------------------------------------------- /experiment_scripts/run_rl.py: -------------------------------------------------------------------------------- 1 | """ 2 | REINFORCE baseline 3 | 4 | The code for the reinforcement learning agent was generously provided by Chris Ying. 5 | For the original implementation see here: 6 | 7 | https://github.com/google-research/nasbench 8 | 9 | For a more detailed description see the following paper: 10 | 11 | Ying, C., Klein, A., Real, E., Christiansen, E., Murphy, K., and Hutter, F. 12 | NAS-Bench-101: Towards reproducible neural architecture search. 13 | arXiv:1902.09635 [cs.LG]. 14 | 15 | 16 | """ 17 | import os 18 | import argparse 19 | import json 20 | import ConfigSpace 21 | import tensorflow as tf 22 | import tensorflow_probability as tfp 23 | 24 | from tabular_benchmarks import FCNetProteinStructureBenchmark, FCNetSliceLocalizationBenchmark,\ 25 | FCNetNavalPropulsionBenchmark, FCNetParkinsonsTelemonitoringBenchmark 26 | from tabular_benchmarks import NASCifar10A, NASCifar10B 27 | 28 | 29 | class ExponentialMovingAverage(object): 30 | """Class that maintains an exponential moving average.""" 31 | 32 | def __init__(self, momentum): 33 | self._numerator = tf.Variable(0.0, dtype=tf.float32, trainable=False) 34 | self._denominator = tf.Variable(0.0, dtype=tf.float32, trainable=False) 35 | self._momentum = momentum 36 | 37 | def update(self, value): 38 | """Update the moving average with a new sample.""" 39 | self._numerator.assign( 40 | self._momentum * self._numerator + (1 - self._momentum) * value) 41 | self._denominator.assign( 42 | self._momentum * self._denominator + (1 - self._momentum)) 43 | 44 | def value(self): 45 | """Return the current value of the moving average""" 46 | return self._numerator / self._denominator 47 | 48 | 49 | class Reward(object): 50 | """Computes the fitness of a sampled model by querying NASBench.""" 51 | 52 | def __init__(self, bench): 53 | self.bench = bench 54 | 55 | def compute_reward(self, sample): 56 | config = ConfigSpace.Configuration(b.get_configuration_space(), vector=sample) 57 | y, c = self.bench.objective_function(config) 58 | fitness = 1 - float(y) 59 | return fitness 60 | 61 | 62 | class REINFORCEOptimizer(object): 63 | """Class that optimizes a set of categorical variables using REINFORCE.""" 64 | 65 | def __init__(self, reward, cat_variables, momentum): 66 | # self._num_vertices = reward.num_vertices 67 | # self._num_operations = len(reward.available_ops) 68 | # self._num_edges = (self._num_vertices * (self._num_vertices - 1)) // 2 69 | # 70 | # self._edge_logits = tf.Variable(tf.zeros([self._num_edges, 2])) 71 | # self._op_logits = tf.Variable(tf.zeros([self._num_vertices - 2, 72 | # self._num_operations])) 73 | self._num_variables = len(cat_variables) 74 | self._logits = [tf.Variable(tf.zeros([1, ci])) for ci in cat_variables] 75 | self._baseline = ExponentialMovingAverage(momentum=momentum) 76 | self._reward = reward 77 | self._last_reward = 0.0 78 | self._test_acc = 0.0 79 | 80 | def step(self): 81 | """Helper function for a single step of the REINFORCE algorithm.""" 82 | # Obtain a single sample from the current distribution. 83 | # edge_dist = tfp.distributions.Categorical(logits=self._edge_logits) 84 | # op_dist = tfp.distributions.Categorical(logits=self._op_logits) 85 | dists = [tfp.distributions.Categorical(logits=li) for li in self._logits] 86 | attempts = 0 87 | while True: 88 | sample = [di.sample() for di in dists] 89 | 90 | # Compute the sample reward. Larger rewards are better. 91 | reward = self._reward.compute_reward(sample) 92 | attempts += 1 93 | if reward > 0.001: 94 | # print('num attempts: {}, reward: {}'.format(str(attempts), reward)) 95 | break 96 | 97 | self._last_reward = reward 98 | 99 | # Compute the log-likelihood the sample. 100 | log_prob = tf.reduce_sum([dists[i].log_prob(sample[i]) for i in range(len(sample))]) 101 | # log_prob = (tf.reduce_sum(edge_dist.log_prob(edge_sample)) + 102 | # tf.reduce_sum(op_dist.log_prob(op_sample))) 103 | 104 | # Update the baseline to reflect the current sample. 105 | self._baseline.update(reward) 106 | 107 | # Compute the advantage. This will be positive if the current sample is 108 | # better than average, and will be negative otherwise. 109 | advantage = reward - self._baseline.value() 110 | 111 | # Here comes the REINFORCE magic. We'll update the gradients by 112 | # differentiating with respect to this value. In practice, if advantage > 0 113 | # then the update will increase the log-probability, and if advantage < 0 114 | # then the update will decrease the log-probability. 115 | objective = tf.stop_gradient(advantage) * log_prob 116 | 117 | return objective 118 | 119 | def trainable_variables(self): 120 | # Return a list of trainable variables to update with gradient descent. 121 | # return [self._edge_logits, self._op_logits] 122 | return self._logits 123 | 124 | def baseline(self): 125 | """Return an exponential moving average of recent reward values.""" 126 | return self._baseline.value() 127 | 128 | def last_reward(self): 129 | """Returns the last reward earned.""" 130 | return self._last_reward 131 | 132 | def test_acc(self): 133 | """Returns the last test accuracy computed.""" 134 | return self._test_acc 135 | 136 | def probabilities(self): 137 | """Return a set of probabilities for each learned variable.""" 138 | # return [tf.nn.softmax(self._edge_logits), 139 | # tf.nn.softmax(self._op_logits)] 140 | # return tf.nn.softmax(self._op_logits) # More interesting to look at ops 141 | return [tf.nn.softmax(li).numpy() for li in self._logits] 142 | 143 | 144 | def run_reinforce(optimizer, learning_rate, max_time, bench, num_steps, log_every_n_steps=1000): 145 | """Run multiple steps of REINFORCE to optimize a fixed reward function.""" 146 | trainable_variables = optimizer.trainable_variables() 147 | trace = [] 148 | # run = [[0.0, 0.0, 0.0]] 149 | 150 | # step = 0 151 | for step in range(num_steps): 152 | # step += 1 153 | # Compute the gradient of the sample's log-probability w.r.t. the logits. 154 | with tf.GradientTape() as tape: 155 | objective = optimizer.step() 156 | 157 | # Update the logits using gradient ascent. 158 | gradients = tape.gradient(objective, trainable_variables) 159 | for grad, var in zip(gradients, trainable_variables): 160 | var.assign_add(learning_rate * grad) 161 | 162 | trace.append(optimizer.probabilities()) 163 | # run.append([nasbench.training_time_spent, 164 | # optimizer.last_reward(), # validation acc 165 | # optimizer.test_acc()]) # test acc (avg) 166 | if step % log_every_n_steps == 0: 167 | print('step = {:d}, baseline reward = {:.5f}'.format( 168 | step, optimizer.baseline().numpy())) 169 | # if nasbench.training_time_spent > max_time: 170 | # break 171 | 172 | return trace 173 | 174 | 175 | def scan_top_valid(data): 176 | """Selects top validation model but records the test accuracy.""" 177 | new_data = [] 178 | for run in data: 179 | new_run = [[0.0, 0.0, 0.0]] 180 | for r in run[1:]: 181 | if r[1] > new_run[-1][1]: 182 | new_run.append(r) 183 | else: 184 | new_run.append([r[0], new_run[-1][1], new_run[-1][2]]) 185 | new_data.append(new_run) 186 | 187 | return new_data 188 | 189 | 190 | parser = argparse.ArgumentParser() 191 | parser.add_argument('--run_id', default=0, type=int, nargs='?', help='unique number to identify this run') 192 | parser.add_argument('--benchmark', default="protein_structure", type=str, nargs='?', help='specifies the benchmark') 193 | parser.add_argument('--n_iters', default=100, type=int, nargs='?', help='number of iterations for optimization method') 194 | parser.add_argument('--output_path', default="./", type=str, nargs='?', 195 | help='specifies the path where the results will be saved') 196 | parser.add_argument('--data_dir', default="./", type=str, nargs='?', help='specifies the path to the tabular data') 197 | parser.add_argument('--lr', default=1e-1, type=float, nargs='?', help='learning rate') 198 | parser.add_argument('--momentum', default=0.9, type=float, nargs='?', 199 | help='momentum to compute the exponential averaging of the reward') 200 | 201 | args = parser.parse_args() 202 | 203 | 204 | if args.benchmark == "nas_cifar10a": 205 | b = NASCifar10A(data_dir=args.data_dir) 206 | 207 | elif args.benchmark == "nas_cifar10b": 208 | b = NASCifar10B(data_dir=args.data_dir) 209 | 210 | elif args.benchmark == "protein_structure": 211 | b = FCNetProteinStructureBenchmark(data_dir=args.data_dir) 212 | 213 | elif args.benchmark == "slice_localization": 214 | b = FCNetSliceLocalizationBenchmark(data_dir=args.data_dir) 215 | 216 | elif args.benchmark == "naval_propulsion": 217 | b = FCNetNavalPropulsionBenchmark(data_dir=args.data_dir) 218 | 219 | elif args.benchmark == "parkinsons_telemonitoring": 220 | b = FCNetParkinsonsTelemonitoringBenchmark(data_dir=args.data_dir) 221 | 222 | output_path = os.path.join(args.output_path, "rl") 223 | os.makedirs(os.path.join(output_path), exist_ok=True) 224 | 225 | # Eager mode used for RL baseline 226 | tf.enable_eager_execution() 227 | tf.enable_resource_variables() 228 | 229 | nb_reward = Reward(b) 230 | 231 | cat_variables = [] 232 | cs = b.get_configuration_space() 233 | for h in cs.get_hyperparameters(): 234 | if type(h) == ConfigSpace.hyperparameters.OrdinalHyperparameter: 235 | cat_variables.append(len(h.sequence)) 236 | elif type(h) == ConfigSpace.hyperparameters.CategoricalHyperparameter: 237 | cat_variables.append(len(h.choices)) 238 | 239 | optimizer = REINFORCEOptimizer(reward=nb_reward, cat_variables=cat_variables, momentum=args.momentum) 240 | trace = run_reinforce( 241 | optimizer=optimizer, 242 | learning_rate=args.lr, 243 | max_time=5e6, 244 | bench=b, 245 | num_steps=args.n_iters, 246 | log_every_n_steps=100) 247 | 248 | if args.benchmark == "nas_cifar10a" or args.benchmark == "nas_cifar10b" or args.benchmark == "nas_cifar10c": 249 | res = b.get_results(ignore_invalid_configs=True) 250 | else: 251 | res = b.get_results() 252 | 253 | fh = open(os.path.join(output_path, 'run_%d.json' % args.run_id), 'w') 254 | json.dump(res, fh) 255 | fh.close() 256 | 257 | --------------------------------------------------------------------------------