├── .gitignore ├── LICENSE ├── README.md ├── hyperkeras ├── __init__.py ├── benchmark │ ├── README.md │ ├── __init__.py │ └── nas_bench_101.py ├── examples │ └── __init__.py ├── hyper_keras.py ├── layer_weights_cache.py ├── layers.py ├── one_shot_model.py ├── search_space │ ├── __init__.py │ ├── cnn_search_space.py │ ├── dnn_search_space.py │ ├── enas_common_ops.py │ ├── enas_layers.py │ └── enas_micro.py ├── searchers │ ├── __init__.py │ └── enas_rl_searcher.py ├── tests │ ├── __init__.py │ ├── benchmark │ │ ├── __init__.py │ │ ├── nas_bench_101_test.py │ │ ├── nasbench101.ipynb │ │ └── run_nasbench101.py │ ├── enas_test.py │ ├── hyper_keras_test.py │ ├── keras_layer_test.py │ ├── layers_test.py │ ├── run_enas.py │ ├── run_enas_oneshot_rl.py │ ├── run_single_path_mcts.py │ ├── search_space_test.py │ ├── searchers │ │ ├── __init__.py │ │ └── enas_rnn_test.py │ └── sharing_weights_test.py └── utils.py ├── requirements.txt ├── setup.cfg └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by .ignore support plugin (hsz.mobi) 2 | .idea 3 | 4 | ### Python template 5 | # Byte-compiled / optimized / DLL files 6 | __pycache__/ 7 | *.py[cod] 8 | *$py.class 9 | 10 | # C extensions 11 | *.so 12 | 13 | # Distribution / packaging 14 | .Python 15 | build/ 16 | develop-eggs/ 17 | dist/ 18 | downloads/ 19 | eggs/ 20 | .eggs/ 21 | lib/ 22 | lib64/ 23 | parts/ 24 | sdist/ 25 | var/ 26 | wheels/ 27 | pip-wheel-metadata/ 28 | share/python-wheels/ 29 | *.egg-info/ 30 | .installed.cfg 31 | *.egg 32 | MANIFEST 33 | 34 | # PyInstaller 35 | # Usually these files are written by a python script from a template 36 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 37 | *.manifest 38 | *.spec 39 | 40 | # Installer logs 41 | pip-log.txt 42 | pip-delete-this-directory.txt 43 | 44 | # Unit test / coverage reports 45 | htmlcov/ 46 | .tox/ 47 | .nox/ 48 | .coverage 49 | .coverage.* 50 | .cache 51 | nosetests.xml 52 | coverage.xml 53 | *.cover 54 | *.py,cover 55 | .hypothesis/ 56 | .pytest_cache/ 57 | 58 | # Translations 59 | *.mo 60 | *.pot 61 | 62 | # Django stuff: 63 | *.log 64 | local_settings.py 65 | db.sqlite3 66 | db.sqlite3-journal 67 | 68 | # Flask stuff: 69 | instance/ 70 | .webassets-cache 71 | 72 | # Scrapy stuff: 73 | .scrapy 74 | 75 | # Sphinx documentation 76 | docs/_build/ 77 | 78 | # PyBuilder 79 | target/ 80 | 81 | # Jupyter Notebook 82 | .ipynb_checkpoints 83 | 84 | # IPython 85 | profile_default/ 86 | ipython_config.py 87 | 88 | # pyenv 89 | .python-version 90 | 91 | # pipenv 92 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 93 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 94 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 95 | # install all needed dependencies. 96 | #Pipfile.lock 97 | 98 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 99 | __pypackages__/ 100 | 101 | # Celery stuff 102 | celerybeat-schedule 103 | celerybeat.pid 104 | 105 | # SageMath parsed files 106 | *.sage.py 107 | 108 | # Environments 109 | .env 110 | .venv 111 | env/ 112 | venv/ 113 | ENV/ 114 | env.bak/ 115 | venv.bak/ 116 | 117 | # Spyder project settings 118 | .spyderproject 119 | .spyproject 120 | 121 | # Rope project settings 122 | .ropeproject 123 | 124 | # mkdocs documentation 125 | /site 126 | 127 | # mypy 128 | .mypy_cache/ 129 | .dmypy.json 130 | dmypy.json 131 | 132 | # Pyre type checker 133 | .pyre/ 134 | dt_output/ 135 | log/ 136 | trial_store/ 137 | tmp/ 138 | catboost_info/ 139 | 140 | #dispatchers 141 | logs/ 142 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # HyperKeras 2 | An AutoDL tool based on Tensorflow and Keras 3 | 4 | ## We Are Hiring！ 5 | Dear folks, we are opening several precious positions based in Beijing both for professionals and interns avid in AutoML/NAS, please send your resume/cv to yangjian@zetyun.com. (Application deadline: TBD.) 6 | 7 | -------------------------------------------------------------------------------- /hyperkeras/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | __author__ = 'yangjian' 3 | """ 4 | 5 | """ 6 | 7 | from hypernets.core.search_space import HyperSpace 8 | from tensorflow.keras.models import Model 9 | from .hyper_keras import HyperKeras 10 | 11 | def keras_model(self, deepcopy=True): 12 | compiled_space, outputs = self.compile_and_forward(deepcopy=deepcopy) 13 | inputs = compiled_space.get_inputs() 14 | model = Model(inputs=[input.output for input in inputs], 15 | outputs=outputs) 16 | return model 17 | 18 | 19 | HyperSpace.keras_model = keras_model -------------------------------------------------------------------------------- /hyperkeras/benchmark/README.md: -------------------------------------------------------------------------------- 1 | # NASBench: A Neural Architecture Search Dataset and Benchmark 2 | 3 | ## Setup 4 | 5 | 1. Clone this repo. 6 | 7 | ``` 8 | # git clone https://github.com/google-research/nasbench 9 | # clone from a fork to compatible with tensorflow 2.x 10 | git clone https://github.com/jackguagua/nasbench 11 | cd nasbench 12 | ``` 13 | 14 | 2. Install the project along with dependencies. 15 | 16 | ``` 17 | pip install -e . 18 | ``` 19 | 20 | **Note:** the only required dependency is TensorFlow. The above instructions 21 | will install the CPU version of TensorFlow to the virtualenv. For other install 22 | options, see https://www.tensorflow.org/install/. 23 | 24 | ## Download the dataset 25 | 26 | The full dataset (which includes all 5M data points at all 4 epoch lengths): 27 | 28 | https://storage.googleapis.com/nasbench/nasbench_full.tfrecord 29 | 30 | Size: ~1.95 GB, SHA256: `3d64db8180fb1b0207212f9032205064312b6907a3bbc81eabea10db2f5c7e9c` 31 | 32 | --- 33 | 34 | Subset of the dataset with only models trained at 108 epochs: 35 | 36 | https://storage.googleapis.com/nasbench/nasbench_only108.tfrecord 37 | 38 | Size: ~499 MB, SHA256: `4c39c3936e36a85269881d659e44e61a245babcb72cb374eacacf75d0e5f4fd1` 39 | 40 | 41 | ## Using the dataset 42 | 43 | Example usage (see `example.py` for a full runnable example): 44 | 45 | ```python 46 | # Load the data from file (this will take some time) 47 | from nasbench import api 48 | 49 | nasbench = api.NASBench('/path/to/nasbench.tfrecord') 50 | 51 | INPUT = 'input' 52 | OUTPUT = 'output' 53 | CONV1X1 = 'conv1x1-bn-relu' 54 | CONV3X3 = 'conv3x3-bn-relu' 55 | MAXPOOL3X3 = 'maxpool3x3' 56 | 57 | 58 | # Create an Inception-like module (5x5 convolution replaced with two 3x3 59 | # convolutions). 60 | model_spec = api.ModelSpec( 61 | # Adjacency matrix of the module 62 | matrix=[[0, 1, 1, 1, 0, 1, 0], # input layer 63 | [0, 0, 0, 0, 0, 0, 1], # 1x1 conv 64 | [0, 0, 0, 0, 0, 0, 1], # 3x3 conv 65 | [0, 0, 0, 0, 1, 0, 0], # 5x5 conv (replaced by two 3x3's) 66 | [0, 0, 0, 0, 0, 0, 1], # 5x5 conv (replaced by two 3x3's) 67 | [0, 0, 0, 0, 0, 0, 1], # 3x3 max-pool 68 | [0, 0, 0, 0, 0, 0, 0]], # output layer 69 | # Operations at the vertices of the module, matches order of matrix 70 | ops=[INPUT, CONV1X1, CONV3X3, CONV3X3, CONV3X3, MAXPOOL3X3, OUTPUT]) 71 | 72 | # Query this model from dataset, returns a dictionary containing the metrics 73 | # associated with this model. 74 | data = nasbench.query(model_spec) 75 | ``` 76 | 77 | ## Disclaimer 78 | 79 | This is not an official Google product. 80 | -------------------------------------------------------------------------------- /hyperkeras/benchmark/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | __author__ = 'yangjian' 3 | """ 4 | 5 | """ -------------------------------------------------------------------------------- /hyperkeras/benchmark/nas_bench_101.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | __author__ = 'yangjian' 3 | """ 4 | 5 | """ 6 | import numpy as np 7 | from nasbench import api 8 | 9 | from hypernets.core.meta_learner import MetaLearner 10 | from hypernets.core.ops import ModuleChoice, HyperInput, Identity, MultipleChoice, Choice 11 | from hypernets.core.search_space import HyperSpace 12 | from hypernets.core.trial import TrialHistory, DiskTrialStore, Trial 13 | 14 | 15 | class NasBench101(): 16 | def __init__(self, num_nodes, ops=None, input='input', 17 | output='output', nasbench_filepath=None, trial_store_path=None): 18 | 19 | self.num_nodes = num_nodes 20 | if ops is None: 21 | ops = ['conv3x3-bn-relu', 'conv1x1-bn-relu', 'maxpool3x3'] 22 | self.ops = ops 23 | self.num_ops = len(ops) 24 | self.input = input 25 | self.output = output 26 | self.nasbench = api.NASBench(nasbench_filepath) 27 | self.trial_store_path = trial_store_path 28 | 29 | def get_ops(self, node_no): 30 | ops = [Identity(name=f'node{node_no}_ops{i}') for i in range(self.num_ops)] 31 | return ops 32 | 33 | def get_space(self): 34 | space = HyperSpace() 35 | with space.as_default(): 36 | input0 = HyperInput() 37 | inputs = [input0] 38 | last = input0 39 | for node_no in range(1, self.num_nodes): 40 | input_i = Identity(hp_inputs=MultipleChoice(list(range(len(inputs))), 41 | num_chosen_most=len(inputs), 42 | num_chosen_least=0))(last) 43 | if node_no < self.num_nodes - 1: 44 | node_i = ModuleChoice(self.get_ops(node_no)) 45 | node_i(input_i) 46 | else: 47 | node_i = Identity(name=f'output')(input_i) 48 | last = node_i 49 | inputs.append(last) 50 | space.set_inputs(input0) 51 | return space 52 | 53 | def sample2spec(self, space_sample): 54 | assert space_sample.all_assigned 55 | edges_choice = [] 56 | ops_choice = [] 57 | for hp in space_sample.assigned_params_stack: 58 | if isinstance(hp, MultipleChoice): # 59 | edges_choice.append(hp.value) 60 | elif isinstance(hp, Choice): 61 | ops_choice.append(hp.value) 62 | else: 63 | raise ValueError(f'Unsupported hyper parameter:{hp}') 64 | 65 | assert len(edges_choice) == self.num_nodes - 1 66 | assert len(ops_choice) == self.num_nodes - 2 67 | 68 | matrix = np.zeros(shape=(self.num_nodes, self.num_nodes), dtype=int) 69 | col = 1 70 | for rows in edges_choice: 71 | for row in rows: 72 | matrix[row][col] = 1 73 | col += 1 74 | 75 | ops = [] 76 | ops.append(self.input) 77 | for op in ops_choice: 78 | ops.append(self.ops[op]) 79 | ops.append(self.output) 80 | 81 | return matrix, ops 82 | 83 | def valid_space_sample(self, space_sample): 84 | matrix, ops = self.sample2spec(space_sample) 85 | model_spec = api.ModelSpec(matrix=matrix, ops=ops) 86 | return self.nasbench.is_valid(model_spec) 87 | 88 | def run_searcher(self, searcher, max_trials=None, max_time_budget=5e6, use_meta_learner=True): 89 | history = TrialHistory('max') 90 | if use_meta_learner: 91 | disk_trial_store = DiskTrialStore(self.trial_store_path) 92 | disk_trial_store.clear_history() 93 | meta_learner = MetaLearner(history, 'nas_bench_101', disk_trial_store) 94 | searcher.set_meta_learner(meta_learner) 95 | 96 | self.nasbench.reset_budget_counters() 97 | times, best_valids, best_tests = [0.0], [0.0], [0.0] 98 | trial_no = 0 99 | while True: 100 | trial_no += 1 101 | if max_trials is not None and trial_no > max_trials: 102 | break 103 | 104 | sample = searcher.sample() 105 | matrix, ops = self.sample2spec(sample) 106 | model_spec = api.ModelSpec(matrix=matrix, ops=ops) 107 | data = self.nasbench.query(model_spec) 108 | 109 | if data['validation_accuracy'] > best_valids[-1]: 110 | best_valids.append(data['validation_accuracy']) 111 | best_tests.append(data['test_accuracy']) 112 | else: 113 | best_valids.append(best_valids[-1]) 114 | best_tests.append(best_tests[-1]) 115 | time_spent, _ = self.nasbench.get_budget_counters() 116 | times.append(time_spent) 117 | reward = data['test_accuracy'] 118 | trial = Trial(sample, trial_no, reward, data['training_time']) 119 | history.append(trial) 120 | searcher.update_result(sample, reward) 121 | 122 | if time_spent > max_time_budget: 123 | # Break the first time we exceed the budget. 124 | break 125 | 126 | return times, best_valids, best_tests 127 | -------------------------------------------------------------------------------- /hyperkeras/examples/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | __author__ = 'yangjian' 3 | """ 4 | 5 | """ -------------------------------------------------------------------------------- /hyperkeras/hyper_keras.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | __author__ = 'yangjian' 3 | """ 4 | 5 | """ 6 | from hypernets.model.hyper_model import HyperModel 7 | from hypernets.model.estimator import Estimator 8 | 9 | from tensorflow.keras import backend as K 10 | from tensorflow.keras.models import Model, load_model, save_model 11 | from tensorflow.keras import utils 12 | import tensorflow as tf 13 | import numpy as np 14 | import gc 15 | from .layer_weights_cache import LayerWeightsCache 16 | import logging 17 | 18 | logger = logging.getLogger(__name__) 19 | 20 | 21 | class KerasEstimator(Estimator): 22 | def __init__(self, space_sample, optimizer, loss, metrics, max_model_size=0, weights_cache=None, 23 | visualization=False): 24 | self.optimizer = optimizer 25 | self.loss = loss 26 | self.metrics = metrics 27 | self.max_model_size = max_model_size 28 | self.weights_cache = weights_cache 29 | self.visualization = visualization 30 | self.model = self._build_model(space_sample) 31 | Estimator.__init__(self, space_sample=space_sample) 32 | 33 | def _build_model(self, space_sample): 34 | K.clear_session() 35 | gc.collect() 36 | space_sample.weights_cache = self.weights_cache 37 | model = space_sample.keras_model(deepcopy=self.weights_cache is None) 38 | model.compile(optimizer=self.optimizer, loss=self.loss, metrics=self.metrics) 39 | if self.max_model_size > 0: 40 | model_size = compute_params_count(model) 41 | assert model_size <= self.max_model_size, f'Model size out of limit:{self.max_model_size}' 42 | if self.visualization: 43 | utils.plot_model(model, f'model_{space_sample.space_id}.png', show_shapes=True) 44 | return model 45 | 46 | def summary(self): 47 | self.model.summary() 48 | 49 | def fit(self, X, y, **kwargs): 50 | self.model.fit(X, y, **kwargs) 51 | 52 | def predict(self, X, **kwargs): 53 | return self.model.predict(X, **kwargs) 54 | 55 | def evaluate(self, X, y, metrics=None, **kwargs): 56 | scores = self.model.evaluate(X, y, **kwargs) 57 | result = {k: v for k, v in zip(self.model.metrics_names, scores)} 58 | return result 59 | 60 | def save(self, model_file): 61 | pass 62 | # save_model(self.model, model_file, save_format='h5') 63 | 64 | 65 | class HyperKeras(HyperModel): 66 | def __init__(self, searcher, optimizer, loss, metrics, dispatcher=None, callbacks=[], 67 | reward_metric=None, max_model_size=0, one_shot_mode=False, one_shot_train_sampler=None, 68 | visualization=False, task=None): 69 | self.optimizer = optimizer 70 | self.loss = loss 71 | self.metrics = metrics 72 | self.max_model_size = max_model_size 73 | if reward_metric is None: 74 | reward_metric = metrics[0] 75 | if one_shot_mode: 76 | self.weights_cache = LayerWeightsCache() 77 | else: 78 | self.weights_cache = None 79 | self.one_shot_mode = one_shot_mode 80 | self.one_shot_train_sampler = one_shot_train_sampler if one_shot_train_sampler is not None else searcher 81 | self.visualization = visualization 82 | HyperModel.__init__(self, searcher, dispatcher=dispatcher, callbacks=callbacks, reward_metric=reward_metric, 83 | task=task) 84 | 85 | def _get_estimator(self, space_sample): 86 | estimator = KerasEstimator(space_sample, optimizer=self.optimizer, loss=self.loss, metrics=self.metrics, 87 | max_model_size=self.max_model_size, weights_cache=self.weights_cache, 88 | visualization=self.visualization) 89 | return estimator 90 | 91 | def build_dataset_iter(self, X, y, batch_size=32, buffer_size=None, reshuffle_each_iteration=None, repeat_count=1): 92 | if buffer_size is None: 93 | buffer_size = len(X[-1]) 94 | dataset = tf.data.Dataset. \ 95 | from_tensor_slices((X, y)). \ 96 | shuffle(buffer_size=buffer_size, reshuffle_each_iteration=reshuffle_each_iteration). \ 97 | repeat(repeat_count). \ 98 | batch(batch_size) 99 | 100 | return iter(dataset) 101 | 102 | def fit_one_shot_model_epoch(self, X, y, batch_size=32, steps=None, epoch=0): 103 | step = 0 104 | dataset_iter = self.build_dataset_iter(X, y, batch_size=batch_size) 105 | for X_batch, y_batch in dataset_iter: 106 | sample = self.one_shot_train_sampler.sample() 107 | est = self._get_estimator(space_sample=sample) 108 | est.fit(X_batch, y_batch, batch_size=batch_size, epochs=1) 109 | step += 1 110 | print(f'One-shot model training, Epoch[{epoch}], Step[{step}]') 111 | if steps is not None and step >= steps: 112 | break 113 | print(f'One-shot model training finished. Epoch[{epoch}], Step[{step}]') 114 | 115 | def export_trial_configuration(self, trial): 116 | return None 117 | 118 | 119 | def compute_params_count(model): 120 | assert model.built, '' 121 | return int(np.sum([K.count_params(weights) for weights in model.trainable_weights])) 122 | -------------------------------------------------------------------------------- /hyperkeras/layer_weights_cache.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | __author__ = 'yangjian' 3 | """ 4 | 5 | """ 6 | import gc 7 | 8 | 9 | class LayerWeightsCache(): 10 | def __init__(self): 11 | self.reset() 12 | super(LayerWeightsCache, self).__init__() 13 | 14 | def reset(self): 15 | self.cache = dict() 16 | self.hit_counter = 0 17 | self.miss_counter = 0 18 | 19 | def clear(self): 20 | del self.cache 21 | gc.collect() 22 | self.reset() 23 | 24 | def hit(self): 25 | self.hit_counter += 1 26 | 27 | def miss(self): 28 | self.miss_counter += 1 29 | 30 | def put(self, key, layer): 31 | assert self.cache.get(key) is None, f'Duplicate keys are not allowed. key:{key}' 32 | self.cache[key] = layer 33 | 34 | def retrieve(self, key): 35 | item = self.cache.get(key) 36 | if item is None: 37 | self.miss() 38 | else: 39 | self.hit() 40 | return item 41 | -------------------------------------------------------------------------------- /hyperkeras/layers.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | """ 3 | 4 | """ 5 | 6 | from tensorflow.keras import layers as kl 7 | from hypernets.core.search_space import * 8 | from .utils import compile_layer 9 | 10 | 11 | class HyperLayer(ModuleSpace): 12 | def __init__(self, keras_layer_class, space=None, name=None, **hyperparams): 13 | self.keras_layer_class = keras_layer_class 14 | ModuleSpace.__init__(self, space, name, **hyperparams) 15 | 16 | def _compile(self): 17 | self.keras_layer = compile_layer(self.space, self.keras_layer_class, self.name, **self.param_values) 18 | 19 | def _forward(self, inputs): 20 | return self.keras_layer(inputs) 21 | 22 | def _on_params_ready(self): 23 | pass 24 | 25 | 26 | class Masking(HyperLayer): 27 | def __init__(self, mask_value=0., space=None, name=None, **kwargs): 28 | if mask_value != 0.: 29 | kwargs['mask_value'] = mask_value 30 | HyperLayer.__init__(self, kl.Masking, space, name, **kwargs) 31 | 32 | 33 | class Input(HyperLayer): 34 | def __init__(self, shape, dtype=None, space=None, name=None, **kwargs): 35 | kwargs['shape'] = shape 36 | if dtype is not None: 37 | kwargs['dtype'] = dtype 38 | HyperLayer.__init__(self, kl.Input, space, name, **kwargs) 39 | 40 | def _forward(self, inputs): 41 | return self.keras_layer 42 | 43 | 44 | class Dense(HyperLayer): 45 | def __init__(self, units, activation=None, use_bias=None, space=None, name=None, **kwargs): 46 | kwargs['units'] = units 47 | if activation is not None: 48 | kwargs['activation'] = activation 49 | if use_bias is not None: 50 | kwargs['use_bias'] = use_bias 51 | HyperLayer.__init__(self, kl.Dense, space, name, **kwargs) 52 | 53 | 54 | class Embedding(HyperLayer): 55 | def __init__(self, input_dim, output_dim, space=None, name=None, **kwargs): 56 | kwargs['input_dim'] = input_dim 57 | kwargs['output_dim'] = output_dim 58 | HyperLayer.__init__(self, kl.Embedding, space, name, **kwargs) 59 | 60 | 61 | class Dropout(HyperLayer): 62 | def __init__(self, rate, space=None, name=None, **kwargs): 63 | kwargs['rate'] = rate 64 | HyperLayer.__init__(self, kl.Dropout, space, name, **kwargs) 65 | 66 | 67 | class SpatialDropout1D(HyperLayer): 68 | def __init__(self, rate, space=None, name=None, **kwargs): 69 | kwargs['rate'] = rate 70 | HyperLayer.__init__(self, kl.SpatialDropout1D, space, name, **kwargs) 71 | 72 | 73 | class SpatialDropout2D(HyperLayer): 74 | def __init__(self, rate, data_format=None, space=None, name=None, **kwargs): 75 | kwargs['rate'] = rate 76 | if data_format is not None: 77 | kwargs['data_format'] = data_format 78 | HyperLayer.__init__(self, kl.SpatialDropout2D, space, name, **kwargs) 79 | 80 | 81 | class SpatialDropout3D(HyperLayer): 82 | def __init__(self, rate, data_format=None, space=None, name=None, **kwargs): 83 | kwargs['rate'] = rate 84 | if data_format is not None: 85 | kwargs['data_format'] = data_format 86 | HyperLayer.__init__(self, kl.SpatialDropout3D, space, name, **kwargs) 87 | 88 | 89 | class BatchNormalization(HyperLayer): 90 | def __init__(self, space=None, name=None, **kwargs): 91 | HyperLayer.__init__(self, kl.BatchNormalization, space, name, **kwargs) 92 | 93 | 94 | class Concatenate(HyperLayer): 95 | def __init__(self, axis=-1, space=None, name=None, **kwargs): 96 | if axis != -1: 97 | kwargs['axis'] = axis 98 | HyperLayer.__init__(self, kl.Concatenate, space, name, **kwargs) 99 | 100 | 101 | class Add(HyperLayer): 102 | def __init__(self, axis=-1, space=None, name=None, **kwargs): 103 | if axis != -1: 104 | kwargs['axis'] = axis 105 | HyperLayer.__init__(self, kl.Add, space, name, **kwargs) 106 | 107 | 108 | class Reshape(HyperLayer): 109 | def __init__(self, target_shape, space=None, name=None, **kwargs): 110 | kwargs['target_shape'] = target_shape 111 | HyperLayer.__init__(self, kl.Reshape, space, name, **kwargs) 112 | 113 | 114 | class Flatten(HyperLayer): 115 | def __init__(self, data_format=None, space=None, name=None, **kwargs): 116 | if data_format is not None: 117 | kwargs['data_format'] = data_format 118 | HyperLayer.__init__(self, kl.Flatten, space, name, **kwargs) 119 | 120 | 121 | class Activation(HyperLayer): 122 | def __init__(self, activation, space=None, name=None, **kwargs): 123 | kwargs['activation'] = activation 124 | HyperLayer.__init__(self, kl.Activation, space, name, **kwargs) 125 | 126 | 127 | class Conv2D(HyperLayer): 128 | def __init__(self, filters, kernel_size, strides=(1, 1), padding='valid', data_format=None, space=None, name=None, 129 | **kwargs): 130 | kwargs['filters'] = filters 131 | kwargs['kernel_size'] = kernel_size 132 | if strides is not None: 133 | kwargs['strides'] = strides 134 | if padding is not None: 135 | kwargs['padding'] = padding 136 | if data_format is not None: 137 | kwargs['data_format'] = data_format 138 | HyperLayer.__init__(self, kl.Conv2D, space, name, **kwargs) 139 | 140 | 141 | class SeparableConv2D(HyperLayer): 142 | def __init__(self, filters, kernel_size, strides=(1, 1), padding='valid', data_format=None, space=None, name=None, 143 | **kwargs): 144 | kwargs['filters'] = filters 145 | kwargs['kernel_size'] = kernel_size 146 | if strides is not None: 147 | kwargs['strides'] = strides 148 | if padding is not None: 149 | kwargs['padding'] = padding 150 | if data_format is not None: 151 | kwargs['data_format'] = data_format 152 | HyperLayer.__init__(self, kl.SeparableConv2D, space, name, **kwargs) 153 | 154 | 155 | class AveragePooling2D(HyperLayer): 156 | def __init__(self, pool_size=(2, 2), strides=None, padding='valid', data_format=None, space=None, name=None, 157 | **kwargs): 158 | if pool_size is not None: 159 | kwargs['pool_size'] = pool_size 160 | if strides is not None: 161 | kwargs['strides'] = strides 162 | if padding is not None: 163 | kwargs['padding'] = padding 164 | if data_format is not None: 165 | kwargs['data_format'] = data_format 166 | HyperLayer.__init__(self, kl.AveragePooling2D, space, name, **kwargs) 167 | 168 | 169 | class MaxPooling2D(HyperLayer): 170 | def __init__(self, pool_size=(2, 2), strides=None, padding='valid', data_format=None, space=None, name=None, 171 | **kwargs): 172 | if pool_size is not None: 173 | kwargs['pool_size'] = pool_size 174 | if strides is not None: 175 | kwargs['strides'] = strides 176 | if padding is not None: 177 | kwargs['padding'] = padding 178 | if data_format is not None: 179 | kwargs['data_format'] = data_format 180 | HyperLayer.__init__(self, kl.MaxPooling2D, space, name, **kwargs) 181 | 182 | 183 | class GlobalAveragePooling2D(HyperLayer): 184 | def __init__(self, data_format=None, space=None, name=None, **kwargs): 185 | if data_format is not None: 186 | kwargs['data_format'] = data_format 187 | HyperLayer.__init__(self, kl.GlobalAveragePooling2D, space, name, **kwargs) 188 | 189 | 190 | class GlobalMaxPooling2D(HyperLayer): 191 | def __init__(self, data_format=None, space=None, name=None, **kwargs): 192 | if data_format is not None: 193 | kwargs['data_format'] = data_format 194 | HyperLayer.__init__(self, kl.GlobalMaxPooling2D, space, name, **kwargs) 195 | 196 | 197 | class Cropping2D(HyperLayer): 198 | def __init__(self, cropping=((0, 0), (0, 0)), data_format=None, space=None, name=None, **kwargs): 199 | if cropping is not None: 200 | kwargs['cropping'] = cropping 201 | if data_format is not None: 202 | kwargs['data_format'] = data_format 203 | HyperLayer.__init__(self, kl.Cropping2D, space, name, **kwargs) 204 | 205 | 206 | class ZeroPadding2D(HyperLayer): 207 | def __init__(self, padding=(1, 1), data_format=None, space=None, name=None, **kwargs): 208 | if padding is not None: 209 | kwargs['padding'] = padding 210 | if data_format is not None: 211 | kwargs['data_format'] = data_format 212 | HyperLayer.__init__(self, kl.ZeroPadding2D, space, name, **kwargs) 213 | -------------------------------------------------------------------------------- /hyperkeras/one_shot_model.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | __author__ = 'yangjian' 3 | """ 4 | 5 | """ 6 | 7 | from .hyper_keras import HyperKeras 8 | from hypernets.core.trial import Trail 9 | from hypernets.core.callbacks import EarlyStoppingError 10 | 11 | import time 12 | 13 | 14 | class OneShotModel(HyperKeras): 15 | def __init__(self, searcher, optimizer, loss, metrics, 16 | epochs=320, 17 | batch_size=64, 18 | controller_train_per_epoch=True, 19 | controller_train_steps=50, 20 | dispatcher=None, 21 | callbacks=[], 22 | reward_metric=None, 23 | max_model_size=0, 24 | one_shot_train_sampler=None, 25 | visualization=False): 26 | self.epochs = epochs 27 | self.batch_size = batch_size 28 | self.train_controller_per_epoch = controller_train_per_epoch 29 | self.controller_train_steps = controller_train_steps 30 | super(OneShotModel, self).__init__(searcher, optimizer, loss, metrics, 31 | dispatcher=dispatcher, 32 | callbacks=callbacks, 33 | reward_metric=reward_metric, 34 | max_model_size=max_model_size, 35 | one_shot_mode=True, 36 | one_shot_train_sampler=one_shot_train_sampler, 37 | visualization=visualization) 38 | 39 | def search(self, X, y, X_eval, y_eval, max_trials=None, dataset_id=None, trial_store=None, **fit_kwargs): 40 | self.start_search_time = time.time() 41 | try: 42 | trial_no = 1 43 | for epoch in range(self.epochs): 44 | print(f'One-shot model epoch({epoch}) training...') 45 | self.fit_one_shot_model_epoch(X, y, batch_size=self.batch_size, epoch=epoch) 46 | if self.train_controller_per_epoch: 47 | trial_no = self.train_controller(X_eval, y_eval, self.controller_train_steps, max_trials, trial_no) 48 | 49 | if not self.train_controller_per_epoch: 50 | print(f'Architecture searching...') 51 | self.train_controller(X_eval, y_eval, max_trials, max_trials, trial_no) 52 | except EarlyStoppingError: 53 | print(f'Early stopping') 54 | # TODO: early stopping 55 | 56 | def train_controller(self, X_val, y_val, steps, max_trials, trial_from): 57 | trial_no = trial_from 58 | 59 | for con_step in range(steps): 60 | if max_trials is not None and trial_no >= max_trials: 61 | break 62 | start_time = time.time() 63 | space_sample = self.searcher.sample() 64 | estimator = self._get_estimator(space_sample) 65 | for callback in self.callbacks: 66 | callback.on_build_estimator(self, space_sample, estimator, trial_no) 67 | callback.on_trial_begin(self, space_sample, trial_no) 68 | 69 | metrics = estimator.evaluate(X_val, y_val, batch_size=self.batch_size, metrics=[self.reward_metric]) 70 | reward = self._get_reward(metrics, self.reward_metric) 71 | elapsed = time.time() - start_time 72 | trial = Trail(space_sample, trial_no, reward, elapsed) 73 | improved = self.history.append(trial) 74 | 75 | if improved: 76 | self.best_model = estimator 77 | self.searcher.update_result(space_sample, reward) 78 | 79 | for callback in self.callbacks: 80 | callback.on_trial_end(self, space_sample, trial_no, reward, improved, elapsed) 81 | trial_no += 1 82 | return trial_no 83 | 84 | def derive_arch(self, X_test, y_test, num=10): 85 | for i in range(num): 86 | space_sample = self.searcher.sample() 87 | estimator = self._get_estimator(space_sample) 88 | metrics = estimator.evaluate(X_test, y_test, batch_size=self.batch_size, metrics=[self.reward_metric]) 89 | reward = self._get_reward(metrics, self.reward_metric) 90 | print('>' * 50) 91 | estimator.summary() 92 | print(f'Reward on test set: {reward}') 93 | print('>' * 50) 94 | -------------------------------------------------------------------------------- /hyperkeras/search_space/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | __author__ = 'yangjian' 3 | """ 4 | 5 | """ -------------------------------------------------------------------------------- /hyperkeras/search_space/cnn_search_space.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | """ 3 | 4 | """ 5 | from hyperkeras.layers import Dense, Input, BatchNormalization, Activation, \ 6 | Conv2D, MaxPooling2D, AveragePooling2D, Flatten 7 | from hypernets.core.search_space import HyperSpace, Bool, Choice, Dynamic 8 | from hypernets.core.ops import Permutation, Sequential, Optional, Repeat, ModuleChoice 9 | import itertools 10 | 11 | 12 | def conv_block(block_no, hp_pooling, hp_filters, hp_kernel_size, hp_bn_act, hp_use_bn, hp_activation, strides=(1, 1)): 13 | def conv_bn(step): 14 | conv = Conv2D(filters=conv_filters, kernel_size=hp_kernel_size, strides=strides, padding='same') 15 | act = Activation(activation=hp_activation) 16 | optional_bn = Optional(BatchNormalization(), keep_link=True, hp_opt=hp_use_bn) 17 | 18 | # Use `Permutation` to try different arrangements of act, optional_bn 19 | # optional_bn is optional module and will be skipped when hp_use_bn is False 20 | perm_act_bn = Permutation([optional_bn, act], hp_seq=hp_bn_act) 21 | seq = Sequential([conv, perm_act_bn]) 22 | return seq 23 | 24 | if block_no < 2: 25 | repeat_num_choices = [2] 26 | multiplier = 1 27 | else: 28 | repeat_num_choices = [3, 4, 5] 29 | multiplier = 2 ** (block_no - 1) 30 | 31 | conv_filters = Dynamic(lambda filters: filters * multiplier, filters=hp_filters) 32 | conv = Repeat(conv_bn, repeat_times=repeat_num_choices) 33 | pooling = ModuleChoice([MaxPooling2D(padding='same'), AveragePooling2D(padding='same')], hp_or=hp_pooling) 34 | block = Sequential([conv, pooling]) 35 | return block 36 | 37 | 38 | def cnn_search_space(input_shape, output_units, output_activation='softmax', block_num_choices=[2, 3, 4, 5, 6], 39 | activation_choices=['relu'], filters_choices=[32, 64], kernel_size_choices=[(1, 1), (3, 3)]): 40 | space = HyperSpace() 41 | with space.as_default(): 42 | hp_use_bn = Bool() 43 | hp_pooling = Choice(list(range(2))) 44 | hp_filters = Choice(filters_choices) 45 | hp_kernel_size = Choice(kernel_size_choices) 46 | hp_fc_units = Choice([1024, 2048, 4096]) 47 | if len(activation_choices) == 1: 48 | hp_activation = activation_choices[0] 49 | else: 50 | hp_activation = Choice(activation_choices) 51 | hp_bn_act = Choice([seq for seq in itertools.permutations(range(2))]) 52 | 53 | input = Input(shape=input_shape) 54 | blocks = Repeat( 55 | lambda step: conv_block( 56 | block_no=step, 57 | hp_pooling=hp_pooling, 58 | hp_filters=hp_filters, 59 | hp_kernel_size=hp_kernel_size, 60 | hp_use_bn=hp_use_bn, 61 | hp_activation=hp_activation, 62 | hp_bn_act=hp_bn_act), 63 | repeat_times=block_num_choices)(input) 64 | x = Flatten()(blocks) 65 | x = Dense(units=hp_fc_units, activation=hp_activation, name='fc1')(x) 66 | x = Dense(units=hp_fc_units, activation=hp_activation, name='fc2')(x) 67 | x = Dense(output_units, activation=output_activation, name='predictions')(x) 68 | return space 69 | -------------------------------------------------------------------------------- /hyperkeras/search_space/dnn_search_space.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | __author__ = 'yangjian' 3 | """ 4 | 5 | """ 6 | from hyperkeras.layers import Dense, Input, BatchNormalization, Dropout, Activation 7 | from hypernets.core.search_space import HyperSpace, Bool, Choice, Real, Dynamic 8 | from hypernets.core.ops import Permutation, Sequential, Optional, Repeat 9 | import itertools 10 | 11 | def dnn_block(hp_dnn_units, hp_reduce_factor, hp_seq, hp_use_bn, hp_dropout, hp_activation, step): 12 | # The value of a `Dynamic` is computed as a function of the values of the `ParameterSpace` it depends on. 13 | block_units = Dynamic( 14 | lambda_fn=lambda units, reduce_factor: units if step == 0 else units * (reduce_factor ** step), 15 | units=hp_dnn_units, reduce_factor=hp_reduce_factor) 16 | dense = Dense(units=block_units) 17 | act = Activation(activation=hp_activation) 18 | optional_bn = Optional(BatchNormalization(), keep_link=True, hp_opt=hp_use_bn) 19 | dropout = Dropout(rate=hp_dropout) 20 | 21 | # Use `Permutation` to try different arrangements of act, optional_bn, dropout 22 | # optional_bn is optional module and will be skipped when hp_use_bn is False 23 | perm_act_bn_dropout = Permutation([act, optional_bn, dropout], hp_seq=hp_seq) 24 | 25 | # Use `Sequential` to connect dense and perm_act_bn_dropout in order 26 | seq = Sequential([dense, perm_act_bn_dropout]) 27 | return seq 28 | 29 | 30 | def dnn_search_space(input_shape, output_units, output_activation, units_choices=[200, 500, 1000], 31 | reduce_facotr_choices=[1, 0.8, 0.5], layer_num_choices=[2, 3, 4], 32 | activation_choices=['relu', 'tanh']): 33 | space = HyperSpace() 34 | with space.as_default(): 35 | hp_dnn_units = Choice(units_choices) 36 | hp_reduce_factor = Choice(reduce_facotr_choices) 37 | hp_use_bn = Bool() 38 | if len(activation_choices) == 1: 39 | hp_activation = activation_choices[0] 40 | else: 41 | hp_activation = Choice(activation_choices) 42 | 43 | hp_dropout = Real(0., 0.5, step=0.1) 44 | hp_seq = Choice([seq for seq in itertools.permutations(range(3))]) 45 | 46 | input = Input(shape=input_shape) 47 | backbone = Repeat( 48 | lambda step: dnn_block(hp_dnn_units, hp_reduce_factor, hp_seq, hp_use_bn, hp_dropout, hp_activation, step), 49 | repeat_times=layer_num_choices)(input) 50 | output = Dense(units=output_units, activation=output_activation)(backbone) 51 | return space 52 | -------------------------------------------------------------------------------- /hyperkeras/search_space/enas_common_ops.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | """ 3 | 4 | """ 5 | 6 | from .enas_layers import SafeMerge, Identity, CalibrateSize 7 | from hyperkeras.layers import BatchNormalization, Activation, Add, MaxPooling2D, AveragePooling2D, \ 8 | SeparableConv2D, Conv2D, GlobalAveragePooling2D, Dense, Dropout 9 | from hypernets.core.ops import ModuleChoice, InputChoice, ConnectLooseEnd 10 | from hypernets.core.search_space import ModuleSpace, MultipleChoice, Choice 11 | 12 | 13 | def sepconv2d_bn(no, name_prefix, kernel_size, filters, strides=(1, 1), data_format=None, x=None): 14 | relu = Activation(activation='relu', name=f'{name_prefix}relu_{no}_') 15 | if x is not None and isinstance(x, ModuleSpace): 16 | relu(x) 17 | sepconv2d = SeparableConv2D( 18 | filters=filters, 19 | kernel_size=kernel_size, 20 | strides=strides, 21 | padding='same', 22 | data_format=data_format, 23 | name=f'{name_prefix}sepconv2d_{no}' 24 | )(relu) 25 | bn = BatchNormalization(name=f'{name_prefix}bn_{no}_')(sepconv2d) 26 | return bn 27 | 28 | 29 | def sepconv3x3(name_prefix, filters, strides=(1, 1), data_format=None): 30 | name_prefix = name_prefix + 'sepconv3x3_' 31 | sep1 = sepconv2d_bn(0, name_prefix, kernel_size=(3, 3), filters=filters, strides=strides, data_format=data_format) 32 | sep2 = sepconv2d_bn(1, name_prefix, kernel_size=(3, 3), filters=filters, strides=strides, data_format=data_format, 33 | x=sep1) 34 | return sep2 35 | 36 | 37 | def sepconv5x5(name_prefix, filters, strides=(1, 1), data_format=None): 38 | name_prefix = name_prefix + 'sepconv5x5_' 39 | sep1 = sepconv2d_bn(0, name_prefix, kernel_size=(5, 5), filters=filters, strides=strides, data_format=data_format) 40 | sep2 = sepconv2d_bn(1, name_prefix, kernel_size=(5, 5), filters=filters, strides=strides, data_format=data_format, 41 | x=sep1) 42 | return sep2 43 | 44 | 45 | def maxpooling3x3(name_prefix, filters, strides=(1, 1), data_format=None): 46 | name_prefix = name_prefix + 'maxpooling3x3_' 47 | max = MaxPooling2D(pool_size=(3, 3), strides=strides, padding='same', data_format=data_format, 48 | name=f'{name_prefix}pool_') 49 | return max 50 | 51 | 52 | def avgpooling3x3(name_prefix, filters, strides=(1, 1), data_format=None): 53 | name_prefix = name_prefix + 'avgpooling3x3_' 54 | avg = AveragePooling2D(pool_size=(3, 3), strides=strides, padding='same', data_format=data_format, 55 | name=f'{name_prefix}pool_') 56 | return avg 57 | 58 | 59 | def identity(name_prefix): 60 | return Identity(name=f'{name_prefix}identity') 61 | 62 | 63 | def add(name_prefix, inputs): 64 | return Add(name=f'{name_prefix}add_')(inputs) 65 | 66 | 67 | def conv_cell(hp_dict, type, cell_no, node_no, left_or_right, inputs, filters, is_reduction=False, data_format=None): 68 | assert isinstance(inputs, list) 69 | assert all([isinstance(m, ModuleSpace) for m in inputs]) 70 | name_prefix = f'{type}_C{cell_no}_N{node_no}_{left_or_right}_' 71 | 72 | input_choice_key = f'{type[2:]}_N{node_no}_{left_or_right}_input_choice' 73 | op_choice_key = f'{type[2:]}_N{node_no}_{left_or_right}_op_choice' 74 | hp_choice = hp_dict.get(input_choice_key) 75 | if hp_choice is None: 76 | hp_choice = MultipleChoice(list(range(len(inputs))), 1, name=input_choice_key) 77 | hp_dict[input_choice_key] = hp_choice 78 | ic1 = InputChoice(inputs, 1, hp_choice=hp_choice)(inputs) 79 | if hp_choice is None: 80 | hp_dict[input_choice_key] = ic1.hp_choice 81 | 82 | # hp_strides = Dynamic(lambda_fn=lambda choice: (2, 2) if is_reduction and choice[0] <= 1 else (1, 1), 83 | # choice=ic1.hp_choice) 84 | hp_strides = (1, 1) 85 | hp_op_choice = hp_dict.get(op_choice_key) 86 | module_candidates = [sepconv5x5(name_prefix, filters, strides=hp_strides, data_format=data_format), 87 | sepconv3x3(name_prefix, filters, strides=hp_strides, data_format=data_format), 88 | avgpooling3x3(name_prefix, filters, strides=hp_strides, data_format=data_format), 89 | maxpooling3x3(name_prefix, filters, strides=hp_strides, data_format=data_format), 90 | identity(name_prefix)] 91 | if hp_op_choice is None: 92 | hp_op_choice = Choice(list(range(len(module_candidates))), name=op_choice_key) 93 | hp_dict[op_choice_key] = hp_op_choice 94 | op_choice = ModuleChoice(module_candidates, hp_or=hp_op_choice)(ic1) 95 | 96 | return op_choice 97 | 98 | 99 | def conv_node(hp_dict, type, cell_no, node_no, inputs, filters, is_reduction=False, data_format=None): 100 | op_left = conv_cell(hp_dict, type, cell_no, node_no, 'L', inputs, filters, is_reduction, data_format) 101 | op_right = conv_cell(hp_dict, type, cell_no, node_no, 'R', inputs, filters, is_reduction, data_format) 102 | name_prefix = f'{type}_C{cell_no}_N{node_no}_' 103 | return add(name_prefix, [op_left, op_right]) 104 | 105 | 106 | def conv_layer(hp_dict, type, cell_no, inputs, filters, node_num, is_reduction=False, data_format=None): 107 | name_prefix = f'{type}_C{cell_no}_' 108 | 109 | if inputs[0] == inputs[1]: 110 | c1 = c2 = CalibrateSize(0, filters, name_prefix, data_format)(inputs[0]) 111 | else: 112 | c1 = CalibrateSize(0, filters, name_prefix, data_format)(inputs) 113 | c2 = CalibrateSize(1, filters, name_prefix, data_format)(inputs) 114 | inputs = [c1, c2] 115 | all_nodes = [c1, c2] 116 | for node_no in range(node_num): 117 | node = conv_node(hp_dict, type, cell_no, node_no, inputs, filters, is_reduction, data_format) 118 | inputs.append(node) 119 | all_nodes.append(node) 120 | cle = ConnectLooseEnd(all_nodes)(all_nodes) 121 | 122 | # df = utils.conv_utils.normalize_data_format(data_format) 123 | # channel_axis = 1 if df == 'channels_first' else 3 124 | output = SafeMerge('add', filters, name_prefix, name=name_prefix + 'add_')(cle) 125 | return output 126 | 127 | 128 | def stem_op(input, filters, data_format=None): 129 | conv = Conv2D( 130 | filters=filters * 3, 131 | kernel_size=(3, 3), 132 | strides=(1, 1), 133 | padding='same', 134 | data_format=data_format, 135 | name=f'0_stem_conv2d' 136 | ) 137 | if input is None: 138 | input = conv 139 | else: 140 | conv(input) 141 | bn = BatchNormalization(name=f'0_stem_bn')(conv) 142 | return bn, input 143 | 144 | 145 | def auxiliary_head(x, data_format=None): 146 | pass 147 | 148 | 149 | def classification(x, classes, dropout_rate=0, data_format=None): 150 | name_prefix = 'classification' 151 | x = Activation('relu', name=f'{name_prefix}_relu')(x) 152 | x = GlobalAveragePooling2D(data_format=data_format, name=f'{name_prefix}_global_avgpool')(x) 153 | if dropout_rate > 0: 154 | x = Dropout(dropout_rate, name=f'{name_prefix}_dropout')(x) 155 | x = Dense(classes, activation='softmax', name=f'{name_prefix}_logit')(x) 156 | return x 157 | -------------------------------------------------------------------------------- /hyperkeras/search_space/enas_layers.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | """ 3 | 4 | """ 5 | from tensorflow.keras import layers as kl, utils 6 | from hypernets.core.search_space import ModuleSpace 7 | from hyperkeras.utils import compile_layer 8 | 9 | 10 | class FilterAlignment(kl.Layer): 11 | def __init__(self, name_prefix, filters, data_format, **kwargs): 12 | self.name_prefix = name_prefix 13 | self.filters = filters 14 | self.data_format = data_format 15 | self.relu = kl.ReLU(name=f'{name_prefix}_filters_aligning_relu_') 16 | self.conv2d = kl.Conv2D(filters, (1, 1), padding='same', name=f'{name_prefix}_filters_aligning_conv2d_', 17 | data_format=data_format) 18 | self.bn = kl.BatchNormalization(name=f'{name_prefix}_filters_aligning_bn_') 19 | super(FilterAlignment, self).__init__(**kwargs) 20 | 21 | def call(self, inputs, **kwargs): 22 | x = self.relu(inputs) 23 | x = self.conv2d(x) 24 | x = self.bn(x) 25 | return x 26 | 27 | def get_config(self, ): 28 | config = {'name_prefix': self.name_prefix, 'filters': self.filters, 'data_format': self.data_format} 29 | base_config = super(FilterAlignment, self).get_config() 30 | return dict(list(base_config.items()) + list(config.items())) 31 | 32 | 33 | class FactorizedReduction_K(kl.Layer): 34 | def __init__(self, name_prefix, filters, data_format, **kwargs): 35 | self.name_prefix = name_prefix 36 | self.filters = filters 37 | self.data_format = data_format 38 | self.relu = kl.ReLU(name=f'{name_prefix}relu_') 39 | self.x0_avgpool = kl.AveragePooling2D( 40 | pool_size=(1, 1), 41 | strides=(2, 2), 42 | padding="valid", 43 | data_format=self.data_format, 44 | name=f'{name_prefix}avepool2d_a_') 45 | self.x0_conv = kl.Conv2D( 46 | filters=filters, 47 | kernel_size=(1, 1), 48 | strides=(1, 1), 49 | padding="valid", 50 | data_format=self.data_format, 51 | name=f"{name_prefix}conv2d_a_") 52 | self.x1_zeropad = kl.ZeroPadding2D( 53 | padding=((0, 1), (0, 1)), 54 | data_format=self.data_format, 55 | name=f"{name_prefix}zeropad2d_b_") 56 | self.x1_cropping = kl.Cropping2D( 57 | cropping=((1, 0), (1, 0)), 58 | data_format=self.data_format, 59 | name=f"{name_prefix}crop2d_b_") 60 | self.x1_avgpool = kl.AveragePooling2D( 61 | pool_size=(1, 1), 62 | strides=(2, 2), 63 | padding="valid", 64 | data_format=self.data_format, 65 | name=f"{name_prefix}avepool2d_b_") 66 | self.x1_conv = kl.Conv2D( 67 | filters=filters, 68 | kernel_size=(1, 1), 69 | strides=(1, 1), 70 | padding="valid", 71 | data_format=self.data_format, 72 | name=f"{name_prefix}conv2d_b_") 73 | self.concat = kl.Concatenate(name=f"{name_prefix}concat_") 74 | self.bn = kl.BatchNormalization(name=f"{name_prefix}bn_") 75 | super(FactorizedReduction_K, self).__init__(**kwargs) 76 | 77 | def call(self, inputs, **kwargs): 78 | x = self.relu(inputs) 79 | x_0 = self.x0_avgpool(x) 80 | x_0 = self.x0_conv(x_0) 81 | x_1 = self.x1_zeropad(x) 82 | x_1 = self.x1_cropping(x_1) 83 | x_1 = self.x1_avgpool(x_1) 84 | x_1 = self.x0_conv(x_1) 85 | x = self.concat([x_0, x_1]) 86 | x = self.bn(x) 87 | return x 88 | 89 | def get_config(self, ): 90 | config = {'name_prefix': self.name_prefix, 'filters': self.filters, 'data_format': self.data_format} 91 | base_config = super(FactorizedReduction_K, self).get_config() 92 | return dict(list(base_config.items()) + list(config.items())) 93 | 94 | 95 | class EnasHyperLayer(ModuleSpace): 96 | def __init__(self, filters, name_prefix, data_format=None, space=None, name=None, **hyperparams): 97 | self.filters = filters 98 | self.name_prefix = name_prefix 99 | self.data_format = data_format 100 | ModuleSpace.__init__(self, space, name, **hyperparams) 101 | 102 | def _compile(self): 103 | pass 104 | 105 | def _on_params_ready(self): 106 | pass 107 | 108 | 109 | class Identity(EnasHyperLayer): 110 | def __init__(self, space=None, name=None, **hyperparams): 111 | EnasHyperLayer.__init__(self, None, None, None, space, name, **hyperparams) 112 | 113 | def _forward(self, inputs): 114 | return inputs 115 | 116 | 117 | class FactorizedReduction(EnasHyperLayer): 118 | def __init__(self, filters, name_prefix, data_format=None, space=None, name=None, **hyperparams): 119 | EnasHyperLayer.__init__(self, filters, name_prefix, data_format, space, name, 120 | **hyperparams) 121 | 122 | def _compile(self): 123 | self.factorized_reduction = compile_layer(search_space=self.space, 124 | layer_class=FactorizedReduction_K, 125 | name=f'{self.name_prefix}reduction_', 126 | name_prefix=f'{self.name_prefix}reduction_', 127 | filters=self.filters // 2, 128 | data_format=self.data_format) 129 | 130 | def _forward(self, inputs): 131 | return self.factorized_reduction(inputs) 132 | 133 | 134 | class CalibrateSize(EnasHyperLayer): 135 | def __init__(self, no, filters, name_prefix, data_format=None, space=None, name=None, **hyperparams): 136 | self.no = no 137 | EnasHyperLayer.__init__(self, filters, name_prefix, data_format, space, name, 138 | **hyperparams) 139 | 140 | def _compile(self): 141 | self.filter_alignment_input0 = compile_layer(search_space=self.space, 142 | layer_class=FilterAlignment, 143 | name=f'{self.name_prefix}input0_filter_alignment', 144 | name_prefix=self.name_prefix + 'input0', 145 | filters=self.filters, 146 | data_format=self.data_format) 147 | self.filter_alignment_input1 = compile_layer(search_space=self.space, 148 | layer_class=FilterAlignment, 149 | name=f'{self.name_prefix}input1_filter_alignment', 150 | name_prefix=self.name_prefix + 'input1', 151 | filters=self.filters, 152 | data_format=self.data_format) 153 | self.factorized_reduction = compile_layer(search_space=self.space, 154 | layer_class=FactorizedReduction_K, 155 | name=self.name_prefix + 'input0_factorized_reduction_', 156 | name_prefix=self.name_prefix + 'input0_factorized_reduction_', 157 | filters=self.filters // 2, 158 | data_format=self.data_format) 159 | 160 | def get_height_or_width(self, x): 161 | return x.get_shape().as_list()[2] 162 | 163 | def get_channels(self, x): 164 | if self.data_format in {None, 'channels_last'}: 165 | return x.get_shape().as_list()[3] 166 | else: 167 | return x.get_shape().as_list()[1] 168 | 169 | def _forward(self, inputs): 170 | if isinstance(inputs, list): 171 | assert len(inputs) == 2 172 | hw = [self.get_height_or_width(l) for l in inputs] 173 | c = [self.get_channels(l) for l in inputs] 174 | if self.no == 0: 175 | x = inputs[0] 176 | if hw[0] != hw[1]: 177 | assert hw[0] == 2 * hw[1] 178 | x = self.factorized_reduction(x) 179 | elif c[0] != self.filters: 180 | x = self.filter_alignment_input0(x) 181 | else: 182 | x = inputs[1] 183 | if c[1] != self.filters: 184 | x = self.filter_alignment_input1(x) 185 | return x 186 | else: 187 | x = inputs 188 | c = self.get_channels(x) 189 | if c != self.filters: 190 | x = self.filter_alignment_input0(x) 191 | return x 192 | 193 | 194 | class SafeMerge(EnasHyperLayer): 195 | def __init__(self, operation, filters, name_prefix, data_format=None, space=None, name=None, **hyperparams): 196 | self.operation = operation.lower() 197 | EnasHyperLayer.__init__(self, filters, name_prefix, data_format, space, name, **hyperparams) 198 | 199 | def _forward(self, inputs): 200 | if isinstance(inputs, list): 201 | pv = self.param_values 202 | if pv.get('name') is None: 203 | pv['name'] = self.name 204 | if self.operation == 'add': 205 | return kl.Add(name=pv['name'])(inputs) 206 | elif self.operation == 'concat': 207 | return kl.Concatenate(**pv)(inputs) 208 | else: 209 | raise ValueError(f'Not supported operation:{self.operation}') 210 | else: 211 | return inputs 212 | -------------------------------------------------------------------------------- /hyperkeras/search_space/enas_micro.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | __author__ = 'yangjian' 3 | """ 4 | 5 | """ 6 | from .enas_common_ops import * 7 | from hyperkeras.layers import Input 8 | from .enas_layers import FactorizedReduction 9 | from hypernets.core.search_space import HyperSpace 10 | 11 | 12 | def enas_micro_search_space(arch='NRNR', input_shape=(28, 28, 1), init_filters=64, node_num=4, data_format=None, 13 | classes=10, classification_dropout=0, 14 | hp_dict={}, use_input_placeholder=True, 15 | weights_cache=None): 16 | space = HyperSpace() 17 | with space.as_default(): 18 | if use_input_placeholder: 19 | input = Input(shape=input_shape, name='0_input') 20 | else: 21 | input = None 22 | stem, input = stem_op(input, init_filters, data_format) 23 | node0 = stem 24 | node1 = stem 25 | reduction_no = 0 26 | normal_no = 0 27 | 28 | for l in arch: 29 | if l == 'N': 30 | normal_no += 1 31 | type = 'normal' 32 | cell_no = normal_no 33 | is_reduction = False 34 | else: 35 | reduction_no += 1 36 | type = 'reduction' 37 | cell_no = reduction_no 38 | is_reduction = True 39 | filters = (2 ** reduction_no) * init_filters 40 | 41 | if is_reduction: 42 | node0 = FactorizedReduction(filters, f'{normal_no + reduction_no}_{type}_C{cell_no}_0', data_format)( 43 | node0) 44 | node1 = FactorizedReduction(filters, f'{normal_no + reduction_no}_{type}_C{cell_no}_1', data_format)( 45 | node1) 46 | x = conv_layer(hp_dict, f'{normal_no + reduction_no}_{type}', cell_no, [node0, node1], filters, node_num, 47 | is_reduction) 48 | node0 = node1 49 | node1 = x 50 | logit = classification(x, classes, classification_dropout, data_format) 51 | space.set_inputs(input) 52 | if weights_cache is not None: 53 | space.weights_cache = weights_cache 54 | 55 | return space 56 | -------------------------------------------------------------------------------- /hyperkeras/searchers/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | __author__ = 'yangjian' 3 | """ 4 | 5 | """ -------------------------------------------------------------------------------- /hyperkeras/searchers/enas_rl_searcher.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | __author__ = 'yangjian' 3 | """ 4 | 5 | """ 6 | from tensorflow.keras.models import Model 7 | from tensorflow.keras.optimizers import Adam 8 | from tensorflow.keras.layers import LSTMCell, RNN, Dense, Embedding 9 | from tensorflow.python.ops import clip_ops 10 | import tensorflow as tf 11 | from hypernets.core.search_space import MultipleChoice, Choice 12 | from hypernets.core.searcher import Searcher, OptimizeDirection 13 | 14 | 15 | class RnnController(Model): 16 | def __init__(self, search_space_fn, lstm_size=100, num_ops=5, temperature=None, tanh_constant=1.5, optimizer=None, 17 | baseline=0., baseline_decay=0.999, entropy_weight=None, **kwargs): 18 | super(RnnController, self).__init__(**kwargs) 19 | 20 | self.search_space_fn = search_space_fn 21 | self.num_ops = num_ops 22 | self.lstm_size = lstm_size 23 | self.tanh_constant = tanh_constant 24 | self.temperature = temperature 25 | self.lstm_rnn = RNN( 26 | cell=LSTMCell(units=lstm_size, use_bias=False, name='lstm_cell'), 27 | name='lstm_rnn', 28 | stateful=True) 29 | self.g_emb = tf.random.uniform(shape=(1, 1, lstm_size)) 30 | self.anchor = Dense(lstm_size, use_bias=False) 31 | self.q = Dense(lstm_size, use_bias=False) 32 | self.v = Dense(1, use_bias=False) 33 | 34 | self.log_prob = 0 35 | self.entropy = 0 36 | self.embedding = Embedding(num_ops + 1, lstm_size) 37 | self.dense_ops = Dense(num_ops, use_bias=False) 38 | 39 | self.baseline = baseline 40 | self.baseline_decay = baseline_decay 41 | self.entropy_weight = entropy_weight 42 | self.optimizer = optimizer if optimizer is not None else Adam(learning_rate=1e-3) 43 | self.clipnorm = 5.0 44 | 45 | def reset(self): 46 | self._inputs = self.g_emb 47 | self.attn_2 = {} 48 | self.log_prob = 0 49 | self.entropy = 0 50 | 51 | def _op_choice(self, op_id_true=None): 52 | logits = self.dense_ops(self.lstm_rnn(self._inputs)) 53 | if self.temperature is not None: 54 | logits /= self.temperature 55 | if self.tanh_constant is not None: 56 | logits = self.tanh_constant * tf.tanh(logits) 57 | 58 | op_id = tf.reshape(tf.random.categorical(logits, num_samples=1), [1]) 59 | if op_id_true: 60 | op_id_true = tf.reshape(op_id_true, [1]) 61 | cur_log_prob = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=op_id_true) 62 | else: 63 | if op_id[0] == 5: 64 | print(op_id) 65 | cur_log_prob = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=op_id) 66 | 67 | self.log_prob += cur_log_prob 68 | cur_entropy = tf.stop_gradient(cur_log_prob * tf.exp(-cur_log_prob)) 69 | self.entropy += cur_entropy 70 | self._inputs = tf.reshape(self.embedding(op_id), [1, 1, -1]) 71 | op_id = op_id.numpy()[0] 72 | return op_id 73 | 74 | def _input_choice(self, n_inputs, choice_id='ID_default', index_true=None): 75 | q, anchors = [], [] 76 | cell_type = '_'.join(choice_id.split('_')[:2]) 77 | for id in range(n_inputs): 78 | key = f'{cell_type}_{id}' 79 | if key not in self.attn_2: 80 | self.attn_2[key] = self.lstm_rnn(self._inputs) 81 | q.append(self.anchor(self.attn_2[key])) 82 | anchors.append(self.attn_2[key]) 83 | q = tf.concat(q, axis=0) 84 | q = tf.tanh(q + self.q(anchors[-1])) 85 | q = self.v(q) 86 | if self.temperature is not None: 87 | q /= self.temperature 88 | if self.tanh_constant is not None: 89 | q = self.tanh_constant * tf.tanh(q) 90 | logits = tf.reshape(q, [1, -1]) 91 | softmax_logits = tf.math.log(tf.nn.softmax(logits, axis=-1)) 92 | index = tf.reshape(tf.random.categorical(softmax_logits, 1), [1]) 93 | 94 | if index_true: 95 | index_true = tf.reshape(index_true, [1]) 96 | cur_log_prob = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=index_true) 97 | else: 98 | cur_log_prob = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=index) 99 | self.log_prob += cur_log_prob 100 | cur_entropy = tf.stop_gradient(cur_log_prob * tf.exp(-cur_log_prob)) 101 | self.entropy += cur_entropy 102 | self._inputs = tf.reshape(anchors[tf.reduce_sum(index)], [1, 1, -1]) 103 | 104 | return list(index.numpy()) 105 | 106 | def sample(self): 107 | self.reset() 108 | space_sample = self.search_space_fn() 109 | for hp in space_sample.params_iterator: 110 | if not isinstance(hp, (MultipleChoice, Choice)): 111 | raise ValueError('Does not support ParameterSpace other than `MultipleChoice` and `Choice` in ENAS.') 112 | if not hp.is_mutable: 113 | hp.random_sample() 114 | continue 115 | if isinstance(hp, MultipleChoice): # Param of InputChoice 116 | n_inputs = len(hp.options) 117 | input_id = self._input_choice(n_inputs, hp.id) 118 | hp.assign(input_id) 119 | else: # Param of ModuleChoice 120 | n_ops = len(hp.options) 121 | if n_ops != self.num_ops: 122 | raise ValueError('The number of modules in ModuleChoice is not equal to `num_ops`.') 123 | op_id = self._op_choice() 124 | hp.assign(op_id) 125 | return space_sample 126 | 127 | def calc_log_prob(self, space_sample): 128 | self.reset() 129 | for hp in space_sample.assigned_params_stack: 130 | if not isinstance(hp, (MultipleChoice, Choice)): 131 | raise ValueError('Does not support ParameterSpace other than `MultipleChoice` and `Choice` in ENAS.') 132 | if not hp.is_mutable: 133 | hp.random_sample() 134 | continue 135 | if isinstance(hp, MultipleChoice): # Param of InputChoice 136 | n_inputs = len(hp.options) 137 | self._input_choice(n_inputs, hp.id, hp.value[0]) 138 | else: # Param of ModuleChoice 139 | n_ops = len(hp.options) 140 | if n_ops != self.num_ops: 141 | raise ValueError('The number of modules in ModuleChoice is not equal to `num_ops`.') 142 | self._op_choice(hp.value) 143 | 144 | def train_one_sample(self, space_sample, reward): 145 | self.reset() 146 | with tf.GradientTape() as tape: 147 | self.reset() 148 | self.calc_log_prob(space_sample) 149 | if self.entropy_weight is not None: 150 | self.reward += self.entropy_weight * self.entropy 151 | self.baseline = self.baseline * self.baseline_decay + reward * (1 - self.baseline_decay) 152 | loss = self.log_prob * (reward - self.baseline) 153 | print(f'Reward: {reward}, Loss: {loss}') 154 | # loss += skip_weight * self.sample_skip_penalty 155 | grads = tape.gradient(loss, self.trainable_variables) 156 | 157 | if hasattr(self, "clipnorm"): 158 | grads = [clip_ops.clip_by_norm(g, self.clipnorm) for g in grads] 159 | if hasattr(self, "clipvalue"): 160 | grads = [ 161 | clip_ops.clip_by_value(g, -self.clipvalue, self.clipvalue) 162 | for g in grads 163 | ] 164 | self.optimizer.apply_gradients(zip(grads, self.trainable_variables)) 165 | return loss 166 | 167 | 168 | class EnasSearcher(Searcher): 169 | def __init__(self, space_fn, lstm_size=100, temperature=None, tanh_constant=1.5, optimizer=None, 170 | baseline=0., baseline_decay=0.999, entropy_weight=None, optimize_direction=OptimizeDirection.Minimize, 171 | use_meta_learner=True): 172 | Searcher.__init__(self, space_fn=space_fn, optimize_direction=optimize_direction, 173 | use_meta_learner=use_meta_learner) 174 | num_ops = self._get_num_ops(space_fn) 175 | self.controller = RnnController(space_fn, lstm_size, num_ops, temperature, tanh_constant, optimizer, baseline, 176 | baseline_decay, entropy_weight) 177 | 178 | @property 179 | def parallelizable(self): 180 | return False 181 | 182 | def sample(self): 183 | return self.controller.sample() 184 | 185 | def update_result(self, space_sample, result): 186 | return self.controller.train_one_sample(space_sample, result) 187 | 188 | def _get_num_ops(self, space_fn): 189 | space_sample = space_fn() 190 | num_ops = None 191 | for hp in space_sample.params_iterator: 192 | if not isinstance(hp, (MultipleChoice, Choice)): 193 | raise ValueError('Does not support ParameterSpace other than `MultipleChoice` and `Choice` in ENAS.') 194 | if not hp.is_mutable: 195 | hp.random_sample() 196 | continue 197 | if isinstance(hp, MultipleChoice): # Param of InputChoice 198 | hp.random_sample() 199 | else: # Param of ModuleChoice 200 | n_ops = len(hp.options) 201 | if num_ops is None: 202 | num_ops = n_ops 203 | else: 204 | if num_ops != n_ops: 205 | raise ValueError('The number of modules in all ModuleChoice must be the same.') 206 | if num_ops is None: 207 | num_ops = 5 208 | return num_ops 209 | -------------------------------------------------------------------------------- /hyperkeras/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | __author__ = 'yangjian' 3 | """ 4 | 5 | """ 6 | 7 | import tempfile 8 | import os 9 | 10 | test_output_dir = tempfile.mkdtemp() 11 | os.environ['DEEPTABLES_HOME'] = test_output_dir 12 | os.environ['HYPERNETS_HOME'] = test_output_dir -------------------------------------------------------------------------------- /hyperkeras/tests/benchmark/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | __author__ = 'yangjian' 3 | """ 4 | 5 | """ -------------------------------------------------------------------------------- /hyperkeras/tests/benchmark/nas_bench_101_test.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | __author__ = 'yangjian' 3 | """ 4 | 5 | """ 6 | 7 | try: 8 | from nasbench import api 9 | from hyperkeras.benchmark.nas_bench_101 import NasBench101 10 | 11 | run_test = True 12 | except: 13 | run_test = False 14 | 15 | 16 | class Test_NASBench101(): 17 | 18 | def test_get_space(self): 19 | if run_test: 20 | nasbench = NasBench101(7, ops=['conv3x3-bn-relu', 'conv1x1-bn-relu', 'maxpool3x3']) 21 | space = nasbench.get_space() 22 | space.random_sample() 23 | assert space.vectors 24 | 25 | matrix, ops = nasbench.sample2spec(space) 26 | assert matrix.shape == (7, 7) 27 | assert len(ops) == 7 28 | -------------------------------------------------------------------------------- /hyperkeras/tests/benchmark/run_nasbench101.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | __author__ = 'yangjian' 3 | """ 4 | 5 | """ 6 | 7 | from nasbench import api 8 | from hyperkeras.benchmark.nas_bench_101 import NasBench101 9 | from hypernets.searchers.mcts_searcher import MCTSSearcher 10 | from hypernets.core.meta_learner import MetaLearner 11 | from hypernets.core.trial import TrialHistory, DiskTrialStore, Trial 12 | from hyperkeras.tests import test_output_dir 13 | 14 | import argparse 15 | 16 | parser = argparse.ArgumentParser() 17 | parser.add_argument('input_file', help='Path to the file nasbench.tfrecord') 18 | args = parser.parse_args() 19 | 20 | nasbench = api.NASBench(args.input_file) 21 | hyn_nasbench = NasBench101(7, ops=['conv3x3-bn-relu', 'conv1x1-bn-relu', 'maxpool3x3']) 22 | 23 | 24 | def valid_space_sample(space_sample): 25 | matrix, ops = hyn_nasbench.sample2spec(space_sample) 26 | model_spec = api.ModelSpec(matrix=matrix, ops=ops) 27 | return nasbench.is_valid(model_spec) 28 | 29 | 30 | def run_searcher(searcher, max_trials=None, max_time_budget=5e6, use_meta_learner=True): 31 | history = TrialHistory('max') 32 | if use_meta_learner: 33 | disk_trial_store = DiskTrialStore(f'{test_output_dir}/trial_store') 34 | disk_trial_store.clear_history() 35 | meta_learner = MetaLearner(history, 'nas_bench_101', disk_trial_store) 36 | searcher.set_meta_learner(meta_learner) 37 | 38 | nasbench.reset_budget_counters() 39 | times, best_valids, best_tests = [0.0], [0.0], [0.0] 40 | trial_no = 0 41 | while True: 42 | trial_no += 1 43 | if max_trials is not None and trial_no > max_trials: 44 | break 45 | 46 | sample = searcher.sample() 47 | matrix, ops = hyn_nasbench.sample2spec(sample) 48 | model_spec = api.ModelSpec(matrix=matrix, ops=ops) 49 | data = nasbench.query(model_spec) 50 | 51 | if data['validation_accuracy'] > best_valids[-1]: 52 | best_valids.append(data['validation_accuracy']) 53 | best_tests.append(data['test_accuracy']) 54 | else: 55 | best_valids.append(best_valids[-1]) 56 | best_tests.append(best_tests[-1]) 57 | time_spent, _ = nasbench.get_budget_counters() 58 | times.append(time_spent) 59 | reward = data['test_accuracy'] 60 | trial = Trial(sample, trial_no, reward, data['training_time']) 61 | history.append(trial) 62 | searcher.update_result(sample, reward) 63 | 64 | if time_spent > max_time_budget: 65 | # Break the first time we exceed the budget. 66 | break 67 | 68 | return times, best_valids, best_tests 69 | 70 | 71 | searcher = MCTSSearcher(hyn_nasbench.get_space, optimize_direction='max', space_sample_validation_fn=valid_space_sample) 72 | 73 | # searcher = RandomSearcher(hyn_nasbench.get_space, space_sample_validation_fn=valid_space_sample) 74 | 75 | times, best_v, best_t = run_searcher(searcher) 76 | print(times) 77 | -------------------------------------------------------------------------------- /hyperkeras/tests/enas_test.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | """ 3 | 4 | """ 5 | from tensorflow.keras.utils import plot_model 6 | from hypernets.core.ops import * 7 | from hypernets.core.search_space import HyperSpace 8 | from hyperkeras.search_space.enas_common_ops import sepconv3x3, sepconv5x5, avgpooling3x3, \ 9 | maxpooling3x3, identity, conv_cell, conv_node, conv_layer 10 | from hyperkeras.search_space.enas_micro import enas_micro_search_space 11 | from hyperkeras.layers import Input 12 | from hyperkeras.tests import test_output_dir 13 | 14 | ids = [] 15 | 16 | 17 | def get_id(m): 18 | ids.append(m.id) 19 | return True 20 | 21 | 22 | class Test_Enas(): 23 | def test_enas_ops(self): 24 | def get_space(): 25 | space = HyperSpace() 26 | with space.as_default(): 27 | name_prefix = 'test_' 28 | filters = 64 29 | in1 = Input(shape=(28, 28, 1,)) 30 | in2 = Input(shape=(28, 28, 1,)) 31 | ic1 = InputChoice([in1, in2], 1)([in1, in2]) 32 | or1 = ModuleChoice([sepconv5x5(name_prefix, filters), 33 | sepconv3x3(name_prefix, filters), 34 | avgpooling3x3(name_prefix, filters), 35 | maxpooling3x3(name_prefix, filters), 36 | identity(name_prefix)])(ic1) 37 | space.set_inputs([in1, in2]) 38 | return space 39 | 40 | space = get_space() 41 | global ids 42 | ids = [] 43 | space.traverse(get_id) 44 | assert ids == ['Module_Input_1', 'Module_Input_2', 'Module_InputChoice_1', 'Module_ModuleChoice_1'] 45 | 46 | assert len(space.edges & {(space.Module_Input_1, space.Module_InputChoice_1), 47 | (space.Module_Input_2, space.Module_InputChoice_1)}) == 2 48 | space.Module_InputChoice_1.hp_choice.assign([0]) 49 | assert len(space.edges & {(space.Module_Input_1, space.Module_InputChoice_1), 50 | (space.Module_Input_2, space.Module_InputChoice_1)}) == 0 51 | assert len(space.edges & {(space.Module_Input_1, space.Module_ModuleChoice_1)}) == 1 52 | assert len(space.edges & {(space.Module_Input_2, space.Module_ModuleChoice_1)}) == 0 53 | 54 | space.Module_ModuleChoice_1.hp_or.assign(0) 55 | ids = [] 56 | space.traverse(get_id) 57 | assert ids == ['Module_Input_1', 'Module_Input_2', 'ID_test_sepconv5x5_relu_0_', 58 | 'ID_test_sepconv5x5_sepconv2d_0', 'ID_test_sepconv5x5_bn_0_', 59 | 'ID_test_sepconv5x5_relu_1_', 'ID_test_sepconv5x5_sepconv2d_1', 60 | 'ID_test_sepconv5x5_bn_1_'] 61 | 62 | space = get_space() 63 | space.Module_InputChoice_1.hp_choice.assign([0]) 64 | space.Module_ModuleChoice_1.hp_or.assign(1) 65 | ids = [] 66 | space.traverse(get_id) 67 | assert ids == ['Module_Input_1', 'Module_Input_2', 'ID_test_sepconv3x3_relu_0_', 68 | 'ID_test_sepconv3x3_sepconv2d_0', 'ID_test_sepconv3x3_bn_0_', 'ID_test_sepconv3x3_relu_1_', 69 | 'ID_test_sepconv3x3_sepconv2d_1', 'ID_test_sepconv3x3_bn_1_'] 70 | 71 | space = get_space() 72 | space.Module_InputChoice_1.hp_choice.assign([0]) 73 | space.Module_ModuleChoice_1.hp_or.assign(2) 74 | ids = [] 75 | space.traverse(get_id) 76 | assert ids == ['Module_Input_1', 'Module_Input_2', 'ID_test_avgpooling3x3_pool_'] 77 | 78 | space = get_space() 79 | space.Module_InputChoice_1.hp_choice.assign([0]) 80 | space.Module_ModuleChoice_1.hp_or.assign(3) 81 | ids = [] 82 | space.traverse(get_id) 83 | assert ids == ['Module_Input_1', 'Module_Input_2', 'ID_test_maxpooling3x3_pool_'] 84 | 85 | space = get_space() 86 | space.Module_InputChoice_1.hp_choice.assign([0]) 87 | space.Module_ModuleChoice_1.hp_or.assign(4) 88 | ids = [] 89 | space.traverse(get_id) 90 | assert ids == ['Module_Input_1', 'Module_Input_2', 'ID_test_identity'] 91 | 92 | def test_enas_op(self): 93 | hp_dict = {} 94 | 95 | def get_space(): 96 | space = HyperSpace() 97 | with space.as_default(): 98 | filters = 64 99 | in1 = Input(shape=(28, 28, 1,)) 100 | conv = conv_cell(hp_dict, 'normal', 0, 0, 'L', [in1, in1], filters) 101 | space.set_inputs([in1, in1]) 102 | space.set_outputs(conv) 103 | return space 104 | 105 | space = get_space() 106 | assert len(hp_dict.items()) == 2 107 | global ids 108 | ids = [] 109 | space.traverse(get_id) 110 | assert ids == ['Module_Input_1', 'Module_InputChoice_1', 'Module_ModuleChoice_1'] 111 | 112 | assert len(space.edges & {(space.Module_Input_1, space.Module_InputChoice_1)}) == 1 113 | space.Module_InputChoice_1.hp_choice.assign([0]) 114 | assert len(space.edges & {(space.Module_Input_1, space.Module_InputChoice_1)}) == 0 115 | assert len(space.edges & {(space.Module_Input_1, space.Module_ModuleChoice_1)}) == 1 116 | 117 | space.Module_ModuleChoice_1.hp_or.assign(0) 118 | ids = [] 119 | space.traverse(get_id) 120 | assert ids == ['Module_Input_1', 'ID_normal_C0_N0_L_sepconv5x5_relu_0_', 121 | 'ID_normal_C0_N0_L_sepconv5x5_sepconv2d_0', 'ID_normal_C0_N0_L_sepconv5x5_bn_0_', 122 | 'ID_normal_C0_N0_L_sepconv5x5_relu_1_', 'ID_normal_C0_N0_L_sepconv5x5_sepconv2d_1', 123 | 'ID_normal_C0_N0_L_sepconv5x5_bn_1_'] 124 | hp_dict = {} 125 | space = get_space() 126 | space.Module_InputChoice_1.hp_choice.assign([0]) 127 | space.Module_ModuleChoice_1.hp_or.assign(1) 128 | ids = [] 129 | space.traverse(get_id) 130 | assert ids == ['Module_Input_1', 'ID_normal_C0_N0_L_sepconv3x3_relu_0_', 131 | 'ID_normal_C0_N0_L_sepconv3x3_sepconv2d_0', 'ID_normal_C0_N0_L_sepconv3x3_bn_0_', 132 | 'ID_normal_C0_N0_L_sepconv3x3_relu_1_', 'ID_normal_C0_N0_L_sepconv3x3_sepconv2d_1', 133 | 'ID_normal_C0_N0_L_sepconv3x3_bn_1_'] 134 | # 135 | # model = space.keras_model() 136 | # plot_model(model) 137 | hp_dict = {} 138 | space = get_space() 139 | space.Module_InputChoice_1.hp_choice.assign([0]) 140 | space.Module_ModuleChoice_1.hp_or.assign(2) 141 | ids = [] 142 | space.traverse(get_id) 143 | assert ids == ['Module_Input_1', 'ID_normal_C0_N0_L_avgpooling3x3_pool_'] 144 | hp_dict = {} 145 | space = get_space() 146 | space.Module_InputChoice_1.hp_choice.assign([0]) 147 | space.Module_ModuleChoice_1.hp_or.assign(3) 148 | ids = [] 149 | space.traverse(get_id) 150 | assert ids == ['Module_Input_1', 'ID_normal_C0_N0_L_maxpooling3x3_pool_'] 151 | 152 | hp_dict = {} 153 | space = get_space() 154 | space.Module_InputChoice_1.hp_choice.assign([0]) 155 | space.Module_ModuleChoice_1.hp_or.assign(4) 156 | ids = [] 157 | space.traverse(get_id) 158 | assert ids == ['Module_Input_1', 'ID_normal_C0_N0_L_identity'] 159 | 160 | def test_enas_node(self): 161 | hp_dict = {} 162 | 163 | def get_space(): 164 | space = HyperSpace() 165 | with space.as_default(): 166 | filters = 64 167 | in1 = Input(shape=(28, 28, 1,), dtype='float32') 168 | conv_node(hp_dict, 'normal', 0, 0, [in1, in1], filters) 169 | space.set_inputs(in1) 170 | return space 171 | 172 | space = get_space() 173 | assert len(hp_dict.items()) == 4 174 | assert len(space.edges & {(space.Module_Input_1, space.Module_InputChoice_1), 175 | (space.Module_Input_1, space.Module_InputChoice_2)}) == 2 176 | global ids 177 | ids = [] 178 | space.traverse(get_id) 179 | assert ids == ['Module_Input_1', 'Module_InputChoice_1', 'Module_InputChoice_2', 180 | 'Module_ModuleChoice_1', 'Module_ModuleChoice_2', 'ID_normal_C0_N0_add_'] 181 | hp_dict = {} 182 | space = get_space() 183 | space.Module_InputChoice_1.hp_choice.assign([0]) 184 | space.Module_ModuleChoice_1.hp_or.assign(0) 185 | 186 | space.Module_InputChoice_2.hp_choice.assign([1]) 187 | space.Module_ModuleChoice_2.hp_or.assign(1) 188 | ids = [] 189 | space.traverse(get_id) 190 | assert ids == ['Module_Input_1', 'ID_normal_C0_N0_L_sepconv5x5_relu_0_', 191 | 'ID_normal_C0_N0_R_sepconv3x3_relu_0_', 'ID_normal_C0_N0_L_sepconv5x5_sepconv2d_0', 192 | 'ID_normal_C0_N0_R_sepconv3x3_sepconv2d_0', 'ID_normal_C0_N0_L_sepconv5x5_bn_0_', 193 | 'ID_normal_C0_N0_R_sepconv3x3_bn_0_', 'ID_normal_C0_N0_L_sepconv5x5_relu_1_', 194 | 'ID_normal_C0_N0_R_sepconv3x3_relu_1_', 'ID_normal_C0_N0_L_sepconv5x5_sepconv2d_1', 195 | 'ID_normal_C0_N0_R_sepconv3x3_sepconv2d_1', 'ID_normal_C0_N0_L_sepconv5x5_bn_1_', 196 | 'ID_normal_C0_N0_R_sepconv3x3_bn_1_', 'ID_normal_C0_N0_add_'] 197 | hp_dict = {} 198 | space = get_space() 199 | space.Module_InputChoice_1.hp_choice.assign([0]) 200 | space.Module_ModuleChoice_1.hp_or.assign(0) 201 | 202 | space.Module_InputChoice_2.hp_choice.assign([1]) 203 | space.Module_ModuleChoice_2.hp_or.assign(3) 204 | 205 | model = space.keras_model() 206 | plot_model(model, to_file=f'{test_output_dir}/test_enas_node.png', show_shapes=True) 207 | 208 | def test_enas_layer(self): 209 | hp_dict = {} 210 | 211 | def get_space(): 212 | space = HyperSpace() 213 | with space.as_default(): 214 | filters = 64 215 | in1 = Input(shape=(28, 28, 1,)) 216 | conv_layer(hp_dict, 'normal', 0, [in1, in1], filters, 5) 217 | space.set_inputs(in1) 218 | return space 219 | 220 | space = get_space() 221 | assert len(hp_dict.items()) == 20 222 | space.random_sample() 223 | model = space.keras_model() 224 | plot_model(model, to_file=f'{test_output_dir}/test_enas_cell.png', show_shapes=True) 225 | 226 | def test_enas_micro(self): 227 | hp_dict = {} 228 | space = enas_micro_search_space(arch='NNRNNR', hp_dict=hp_dict) 229 | assert len(hp_dict.items()) == 32 230 | assert space.combinations 231 | 232 | space.random_sample() 233 | model = space.keras_model() 234 | plot_model(model, to_file=f'{test_output_dir}/test_enas_micro.png', show_shapes=True) 235 | -------------------------------------------------------------------------------- /hyperkeras/tests/hyper_keras_test.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | """ 3 | 4 | """ 5 | 6 | from hyperkeras.layers import * 7 | from hyperkeras.hyper_keras import HyperKeras 8 | from hypernets.searchers import RandomSearcher 9 | from hypernets.core.callbacks import * 10 | from hypernets.core.searcher import OptimizeDirection 11 | 12 | 13 | class Test_HyperKeras(): 14 | def get_space(self): 15 | space = HyperSpace() 16 | with space.as_default(): 17 | in1 = Input(shape=(10,)) 18 | in2 = Input(shape=(20,)) 19 | in3 = Input(shape=(1,)) 20 | concat = Concatenate()([in1, in2, in3]) 21 | dense1 = Dense(10, activation=Choice(['relu', 'tanh', None]), use_bias=Bool())(concat) 22 | bn1 = BatchNormalization()(dense1) 23 | dropout1 = Dropout(Choice([0.3, 0.4, 0.5]))(bn1) 24 | output = Dense(2, activation='softmax', use_bias=True)(dropout1) 25 | return space 26 | 27 | def get_space_simple(self): 28 | space = HyperSpace() 29 | with space.as_default(): 30 | in1 = Input(shape=(10,)) 31 | dense1 = Dense(10, activation=Choice(['relu', 'tanh', None]), use_bias=Bool())(in1) 32 | bn1 = BatchNormalization()(dense1) 33 | dropout1 = Dropout(Choice([0.3, 0.4, 0.5]))(bn1) 34 | output = Dense(2, activation='softmax', use_bias=True)(dropout1) 35 | return space 36 | 37 | def get_x_y(self): 38 | x1 = np.random.randint(0, 10000, size=(100, 10)) 39 | x2 = np.random.randint(0, 100, size=(100, 20)) 40 | x3 = np.random.normal(1.0, 100.0, size=(100)) 41 | y = np.random.randint(0, 2, size=(100), dtype='int') 42 | x = [x1, x2, x3] 43 | return x, y 44 | 45 | def get_x_y_1(self): 46 | x = np.random.randint(0, 10000, size=(100, 10)) 47 | y = np.random.randint(0, 2, size=(100), dtype='int') 48 | return x, y 49 | 50 | def test_model_with_hp(self): 51 | rs = RandomSearcher(self.get_space, optimize_direction=OptimizeDirection.Maximize) 52 | hk = HyperKeras(rs, optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'], 53 | callbacks=[SummaryCallback()]) 54 | 55 | x, y = self.get_x_y() 56 | hk.search(x, y, x, y, max_trials=3) 57 | best_trial = hk.get_best_trial() 58 | 59 | estimator = hk.final_train(best_trial.space_sample, x, y) 60 | score = estimator.predict(x) 61 | result = estimator.evaluate(x, y) 62 | assert len(score) == 100 63 | assert result 64 | 65 | def test_build_dataset_iter(self): 66 | rs = RandomSearcher(self.get_space, optimize_direction=OptimizeDirection.Maximize) 67 | hk = HyperKeras(rs, optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'], 68 | callbacks=[SummaryCallback()]) 69 | x, y = self.get_x_y_1() 70 | 71 | ds_iter = hk.build_dataset_iter(x, y, batch_size=10) 72 | 73 | batch_counter = 0 74 | for x_b, y_b in ds_iter: 75 | # x_b, y_b = next() 76 | assert len(x_b) == 10 77 | assert len(y_b) == 10 78 | batch_counter += 1 79 | assert batch_counter == 10 80 | 81 | ds_iter = hk.build_dataset_iter(x, y, batch_size=32) 82 | 83 | batch_counter = 0 84 | for x_b, y_b in ds_iter: 85 | # x_b, y_b = next() 86 | if batch_counter < 3: 87 | assert len(x_b) == 32 88 | assert len(y_b) == 32 89 | else: 90 | assert len(x_b) == 4 91 | assert len(y_b) == 4 92 | batch_counter += 1 93 | assert batch_counter == 4 94 | 95 | ds_iter = hk.build_dataset_iter(x, y, batch_size=32, repeat_count=2) 96 | 97 | batch_counter = 0 98 | for x_b, y_b in ds_iter: 99 | # x_b, y_b = next() 100 | if batch_counter < 6: 101 | assert len(x_b) == 32 102 | assert len(y_b) == 32 103 | else: 104 | assert len(x_b) == 8 105 | assert len(y_b) == 8 106 | batch_counter += 1 107 | assert batch_counter == 7 108 | 109 | def test_fit_one_shot_model_epoch(self): 110 | rs = RandomSearcher(self.get_space_simple, optimize_direction=OptimizeDirection.Maximize) 111 | hk = HyperKeras(rs, 112 | optimizer='adam', 113 | loss='sparse_categorical_crossentropy', 114 | metrics=['accuracy'], 115 | callbacks=[SummaryCallback()], 116 | one_shot_mode=True, 117 | one_shot_train_sampler=rs) 118 | x, y = self.get_x_y_1() 119 | hk.fit_one_shot_model_epoch(x, y) 120 | -------------------------------------------------------------------------------- /hyperkeras/tests/keras_layer_test.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | __author__ = 'yangjian' 3 | """ 4 | 5 | """ 6 | import numpy as np 7 | import tensorflow as tf 8 | from tensorflow.keras import layers 9 | 10 | (x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data() 11 | # Rescale the images from [0,255] to the [0.0,1.0] range. 12 | x_train, x_test = x_train[..., np.newaxis] / 255.0, x_test[..., np.newaxis] / 255.0 13 | y_train = tf.keras.utils.to_categorical(y_train) 14 | y_test = tf.keras.utils.to_categorical(y_test) 15 | print("Number of original training examples:", len(x_train)) 16 | print("Number of original test examples:", len(x_test)) 17 | # sample for speed up 18 | samples = 10 19 | 20 | 21 | class Test_KerasLayer(): 22 | def test_conv2d(self): 23 | x = layers.Conv2D(32, (1, 1))(x_train[:samples]) 24 | assert x.shape == (10, 28, 28, 32) 25 | x = layers.Conv2D(32, (2, 2), padding='same')(x_train[:samples]) 26 | assert x.shape == (10, 28, 28, 32) 27 | x = layers.Conv2D(32, (2, 2), strides=(1, 1), padding='same')(x_train[:samples]) 28 | assert x.shape == (10, 28, 28, 32) 29 | x = layers.Conv2D(32, (2, 2), strides=(2, 2), padding='same')(x_train[:samples]) 30 | assert x.shape == (10, 14, 14, 32) 31 | x = layers.Conv2D(32, (2, 2), strides=(2, 1), padding='same')(x_train[:samples]) 32 | assert x.shape == (10, 14, 28, 32) 33 | 34 | def test_pooling(self): 35 | conv_x = layers.Conv2D(32, (1, 1))(x_train[:samples]) 36 | assert conv_x.shape == (10, 28, 28, 32) 37 | x = layers.AveragePooling2D(pool_size=3, strides=1, padding='same')(conv_x) 38 | assert x.shape == (10, 28, 28, 32) 39 | x = layers.MaxPooling2D(pool_size=3, strides=2, padding='same')(conv_x) 40 | assert x.shape == (10, 14, 14, 32) 41 | x = layers.MaxPooling2D(pool_size=3, strides=1, padding='same')(conv_x) 42 | assert x.shape == (10, 28, 28, 32) 43 | -------------------------------------------------------------------------------- /hyperkeras/tests/layers_test.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | """ 3 | 4 | """ 5 | 6 | from hyperkeras.layers import * 7 | from tensorflow.keras import models 8 | 9 | 10 | class Test_KerasLayers(): 11 | def test_input(self): 12 | space = HyperSpace() 13 | with space.as_default(): 14 | input1 = Input(shape=(3000,)) 15 | input2 = Input(shape=Choice([100, 200]), sparse=Bool()) 16 | assert space.Param_Constant_1.alias == 'Module_Input_1.shape' 17 | assert space.Param_Choice_1.alias == 'Module_Input_2.shape' 18 | assert space.Param_Bool_1.alias == 'Module_Input_2.sparse' 19 | output = input1.compile_and_forward() 20 | assert output.shape, (None, 3000) 21 | 22 | def test_hyper_dense(self): 23 | space = HyperSpace() 24 | with space.as_default(): 25 | input = Input(shape=(3000,)) 26 | dense = Dense(units=Int(100, 200)) 27 | dense(input) 28 | space.random_sample() 29 | assert dense.is_params_ready == True 30 | x = input.compile_and_forward() 31 | assert x.shape, (None, 3000) 32 | x = dense.compile_and_forward(x) 33 | assert x.shape, (None, dense.param_values['units']) 34 | 35 | def test_model_no_hp(self): 36 | space = HyperSpace() 37 | with space.as_default(): 38 | in1 = Input(shape=(10,)) 39 | in2 = Input(shape=(20,)) 40 | in3 = Input(shape=(1,)) 41 | concat = Concatenate()([in1, in2, in3]) 42 | dense1 = Dense(10, activation='relu', use_bias=True)(concat) 43 | bn1 = BatchNormalization()(dense1) 44 | dropout1 = Dropout(0.3)(bn1) 45 | output = Dense(2, activation='relu', use_bias=True)(dropout1) 46 | 47 | model = space.keras_model() 48 | model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy']) 49 | model.summary() 50 | 51 | x1 = np.random.randint(0, 10000, size=(100, 10)) 52 | x2 = np.random.randint(0, 100, size=(100, 20)) 53 | x3 = np.random.normal(1.0, 100.0, size=(100)) 54 | y = np.random.randint(0, 2, size=(100), dtype='int') 55 | x = [x1, x2, x3] 56 | history = model.fit(x=x, y=y) 57 | assert history 58 | 59 | def test_model_with_hp(self): 60 | space = HyperSpace() 61 | with space.as_default(): 62 | in1 = Input(shape=(10,)) 63 | in2 = Input(shape=(20,)) 64 | in3 = Input(shape=(1,)) 65 | concat = Concatenate()([in1, in2, in3]) 66 | dense1 = Dense(10, activation=Choice(['relu', 'tanh', None]), use_bias=Bool())(concat) 67 | bn1 = BatchNormalization()(dense1) 68 | dropout1 = Dropout(Choice([0.3, 0.4, 0.5]))(bn1) 69 | output = Dense(2, activation=Choice(['relu', 'softmax']), use_bias=True)(dropout1) 70 | 71 | space.random_sample() 72 | print(space.params_summary()) 73 | 74 | model = space.keras_model() 75 | model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy']) 76 | model.summary() 77 | 78 | x1 = np.random.randint(0, 10000, size=(100, 10)) 79 | x2 = np.random.randint(0, 100, size=(100, 20)) 80 | x3 = np.random.normal(1.0, 100.0, size=(100)) 81 | y = np.random.randint(0, 2, size=(100), dtype='int') 82 | x = [x1, x2, x3] 83 | history = model.fit(x=x, y=y) 84 | assert history 85 | 86 | def tests_compile_space(self): 87 | space = HyperSpace() 88 | with space.as_default(): 89 | input = Input(shape=(3000,)) 90 | dense = Dense(units=Int(100, 200)) 91 | dense(input) 92 | dense2 = Dense(units=2, name='dense_output') 93 | dense2(dense) 94 | 95 | space.random_sample() 96 | assert dense.is_params_ready == True 97 | compiled_space, _ = space.compile_and_forward() 98 | assert compiled_space.space_id == space.space_id 99 | 100 | outputs = compiled_space.get_outputs() 101 | assert len(outputs) == 1 102 | assert outputs[0].output.shape, (None, space.Module_Dense_2.param_values['units']) 103 | 104 | model = models.Model(inputs=compiled_space.Module_Input_1.output, outputs=outputs[0].output) 105 | model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy']) 106 | model.summary() 107 | assert model.get_layer('Module_Input_1').output.shape[1] == 3000 108 | assert model.get_layer('Module_Dense_1').output.shape[1] == compiled_space.Module_Dense_1.param_values['units'] 109 | assert model.get_layer('dense_output').output.shape[1] == 2 110 | -------------------------------------------------------------------------------- /hyperkeras/tests/run_enas.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | """ 3 | 4 | """ 5 | import tensorflow as tf 6 | import numpy as np 7 | from hypernets.core.callbacks import SummaryCallback 8 | from hypernets.searchers.random_searcher import RandomSearcher 9 | from hyperkeras.search_space.enas_micro import enas_micro_search_space 10 | from hyperkeras.hyper_keras import HyperKeras 11 | 12 | (x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data() 13 | # Rescale the images from [0,255] to the [0.0,1.0] range. 14 | x_train, x_test = x_train[..., np.newaxis] / 255.0, x_test[..., np.newaxis] / 255.0 15 | y_train = tf.keras.utils.to_categorical(y_train) 16 | y_test = tf.keras.utils.to_categorical(y_test) 17 | print("Number of original training examples:", len(x_train)) 18 | print("Number of original test examples:", len(x_test)) 19 | # sample for speed up 20 | samples = 100 21 | 22 | # 23 | # weights_cache = LayerWeightsCache() 24 | # space = enas_micro_search_space(arch='NR', hp_dict={}, use_input_placeholder=True, weights_cache=weights_cache) 25 | # space.random_sample() 26 | # model = space.keras_model(deepcopy=False) 27 | # model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']) 28 | # 29 | # model.fit(x_train[:samples], y_train[:samples], batch_size=32) 30 | # result = model.evaluate(x_train[:samples], y_train[:samples]) 31 | # 32 | # space = enas_micro_search_space(arch='NR', hp_dict={}, use_input_placeholder=True, weights_cache=weights_cache) 33 | # space.random_sample() 34 | # model2 = space.keras_model(deepcopy=False) 35 | # model2.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']) 36 | # model2.fit(x_train[:samples], y_train[:samples], batch_size=32) 37 | # result2 = model.evaluate(x_train[:samples], y_train[:samples]) 38 | # 39 | # weights_cache = LayerWeightsCache() 40 | # space = enas_micro_search_space(arch='NR', hp_dict={}, use_input_placeholder=False, weights_cache=weights_cache) 41 | # space.random_sample() 42 | # 43 | # model = SharingWeightModel(space) 44 | # model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']) 45 | # model.fit(x_train[:samples], y_train[:samples], batch_size=32) 46 | # result = model.evaluate(x_train[:samples], y_train[:samples]) 47 | # 48 | # space = enas_micro_search_space(arch='NR', hp_dict={}, use_input_placeholder=False, weights_cache=weights_cache) 49 | # space.random_sample() 50 | # model.update_search_space(space) 51 | # model.fit(x_train[:samples], y_train[:samples], batch_size=100) 52 | # result = model.evaluate(x_train[:samples], y_train[:samples]) 53 | 54 | rs = RandomSearcher( 55 | lambda: enas_micro_search_space(arch='NNRNNR', hp_dict={}), 56 | optimize_direction='max') 57 | hk = HyperKeras(rs, optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'], 58 | callbacks=[SummaryCallback()], one_shot_mode=True, visualization=False) 59 | 60 | # tenserboard = TensorBoard('./tensorboard/run_enas') 61 | hk.search(x_train[:samples], y_train[:samples], x_test[:int(samples / 10)], y_test[:int(samples / 10)], 62 | max_trials=100, epochs=1, callbacks=[]) 63 | assert hk.get 64 | -------------------------------------------------------------------------------- /hyperkeras/tests/run_enas_oneshot_rl.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | """ 3 | 4 | """ 5 | import tensorflow as tf 6 | 7 | from hypernets.core.callbacks import SummaryCallback 8 | from hypernets.core.ops import * 9 | from hyperkeras.search_space.enas_micro import enas_micro_search_space 10 | from hyperkeras.one_shot_model import OneShotModel 11 | from hyperkeras.searchers.enas_rl_searcher import EnasSearcher 12 | 13 | (x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data() 14 | # Rescale the images from [0,255] to the [0.0,1.0] range. 15 | x_train, x_test = x_train[..., np.newaxis] / 255.0, x_test[..., np.newaxis] / 255.0 16 | y_train = tf.keras.utils.to_categorical(y_train) 17 | y_test = tf.keras.utils.to_categorical(y_test) 18 | print("Number of original training examples:", len(x_train)) 19 | print("Number of original test examples:", len(x_test)) 20 | # sample for speed up 21 | samples = 100 22 | 23 | searcher = EnasSearcher(lambda: enas_micro_search_space(arch='NNRNNR', hp_dict={}), optimize_direction='max') 24 | 25 | model = OneShotModel(searcher, 26 | optimizer='adam', 27 | loss='categorical_crossentropy', 28 | metrics=['accuracy'], 29 | epochs=3, 30 | batch_size=64, 31 | controller_train_steps=2, 32 | callbacks=[SummaryCallback()], 33 | visualization=False) 34 | 35 | model.search(x_train[:samples], y_train[:samples], x_test[:int(samples / 10)], y_test[:int(samples / 10)], 36 | max_trials=100, epochs=1, callbacks=[]) 37 | assert model.best_model 38 | -------------------------------------------------------------------------------- /hyperkeras/tests/run_single_path_mcts.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | """ 3 | 4 | """ 5 | import tensorflow as tf 6 | from hypernets.core.callbacks import SummaryCallback 7 | from hypernets.core.ops import * 8 | from hypernets.searchers.mcts_searcher import MCTSSearcher 9 | from hypernets.searchers.random_searcher import RandomSearcher 10 | from hyperkeras.search_space.enas_micro import enas_micro_search_space 11 | from hyperkeras.one_shot_model import OneShotModel 12 | 13 | (x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data() 14 | # Rescale the images from [0,255] to the [0.0,1.0] range. 15 | x_train, x_test = x_train[..., np.newaxis] / 255.0, x_test[..., np.newaxis] / 255.0 16 | y_train = tf.keras.utils.to_categorical(y_train) 17 | y_test = tf.keras.utils.to_categorical(y_test) 18 | print("Number of original training examples:", len(x_train)) 19 | print("Number of original test examples:", len(x_test)) 20 | # sample for speed up 21 | samples = 100 22 | 23 | searcher = MCTSSearcher(lambda: enas_micro_search_space(arch='NNRNNR', hp_dict={}), optimize_direction='max') 24 | one_shot_sampler = RandomSearcher(lambda: enas_micro_search_space(arch='NNRNNR', hp_dict={}), optimize_direction='max') 25 | 26 | model = OneShotModel(searcher, 27 | optimizer='adam', 28 | loss='categorical_crossentropy', 29 | metrics=['accuracy'], 30 | epochs=3, 31 | batch_size=64, 32 | controller_train_per_epoch=False, # Single path 33 | callbacks=[SummaryCallback()], 34 | one_shot_train_sampler=one_shot_sampler, # uniform sampler 35 | visualization=False) 36 | # model.search(x_train[:samples], y_train[:samples], x_test[:int(samples / 10)], y_test[:int(samples / 10)], 37 | 38 | model.search(x_train, y_train, x_test, y_test, max_trials=1000, epochs=100, callbacks=[]) 39 | assert model.best_model 40 | -------------------------------------------------------------------------------- /hyperkeras/tests/search_space_test.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | """ 3 | 4 | """ 5 | 6 | import numpy as np 7 | import tensorflow as tf 8 | 9 | from hyperkeras import HyperKeras 10 | from hyperkeras.search_space.cnn_search_space import cnn_search_space 11 | from hyperkeras.search_space.dnn_search_space import dnn_search_space 12 | from hypernets.core.callbacks import SummaryCallback 13 | from hypernets.searchers.random_searcher import RandomSearcher 14 | 15 | 16 | class Test_Dnn_Space(): 17 | def test_dnn_space_hyper_model(self): 18 | rs = RandomSearcher(lambda: dnn_search_space(input_shape=10, output_units=2, output_activation='sigmoid'), 19 | optimize_direction='max') 20 | hk = HyperKeras(rs, optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'], 21 | callbacks=[SummaryCallback()]) 22 | 23 | x = np.random.randint(0, 10000, size=(100, 10)) 24 | y = np.random.randint(0, 2, size=(100), dtype='int') 25 | 26 | hk.search(x, y, x, y, max_trials=3) 27 | 28 | def test_dnn_space(self): 29 | space = dnn_search_space(input_shape=10, output_units=2, output_activation='sigmod') 30 | space.random_sample() 31 | ids = [] 32 | assert space.combinations 33 | 34 | def get_id(m): 35 | ids.append(m.id) 36 | return True 37 | 38 | space.traverse(get_id) 39 | assert ids 40 | 41 | def test_cnn_space_hyper_model(self): 42 | rs = RandomSearcher( 43 | lambda: cnn_search_space(input_shape=(28, 28, 1), 44 | output_units=10, 45 | output_activation='softmax', 46 | block_num_choices=[2, 3, 4, 5], 47 | filters_choices=[32, 64, 128], 48 | kernel_size_choices=[(1, 1), (3, 3)]), 49 | optimize_direction='max') 50 | hk = HyperKeras(rs, optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'], 51 | callbacks=[SummaryCallback()]) 52 | 53 | (x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data() 54 | 55 | # Rescale the images from [0,255] to the [0.0,1.0] range. 56 | x_train, x_test = x_train[..., np.newaxis] / 255.0, x_test[..., np.newaxis] / 255.0 57 | y_train = tf.keras.utils.to_categorical(y_train) 58 | y_test = tf.keras.utils.to_categorical(y_test) 59 | print("Number of original training examples:", len(x_train)) 60 | print("Number of original test examples:", len(x_test)) 61 | 62 | # sample for speed up 63 | samples = 100 64 | hk.search(x_train[:samples], y_train[:samples], x_test[:int(samples / 10)], y_test[:int(samples / 10)], 65 | max_trials=3, epochs=1) 66 | 67 | def test_cnn_space(self): 68 | space = cnn_search_space(input_shape=(50, 50), output_units=10, output_activation='softmax') 69 | space.random_sample() 70 | ids = [] 71 | assert space.combinations 72 | 73 | def get_id(m): 74 | ids.append(m.id) 75 | return True 76 | 77 | space.traverse(get_id) 78 | assert ids 79 | -------------------------------------------------------------------------------- /hyperkeras/tests/searchers/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | __author__ = 'yangjian' 3 | """ 4 | 5 | """ -------------------------------------------------------------------------------- /hyperkeras/tests/searchers/enas_rnn_test.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | __author__ = 'yangjian' 3 | """ 4 | 5 | """ 6 | from hypernets.core.search_space import HyperSpace 7 | from hyperkeras.search_space.enas_micro import enas_micro_search_space 8 | from hyperkeras.search_space.enas_common_ops import conv_layer 9 | from hyperkeras.layers import Input 10 | from hyperkeras.searchers.enas_rl_searcher import RnnController, EnasSearcher 11 | 12 | baseline_decay = 0.999 13 | 14 | 15 | class Test_EnasRnnController(): 16 | def get_space(self): 17 | hp_dict = {} 18 | space = HyperSpace() 19 | with space.as_default(): 20 | filters = 64 21 | in1 = Input(shape=(28, 28, 1,)) 22 | conv_layer(hp_dict, 'normal', 0, [in1, in1], filters, 5) 23 | space.set_inputs(in1) 24 | return space 25 | 26 | def test_sample(self): 27 | rc = RnnController(search_space_fn=self.get_space) 28 | rc.reset() 29 | out1 = rc.sample() 30 | out2 = rc.sample() 31 | assert out1 32 | assert out2 33 | 34 | def test_searcher(self): 35 | def enas_space_fn(): 36 | hp_dict = {} 37 | return enas_micro_search_space(arch='NNRNNR', hp_dict=hp_dict) 38 | 39 | enas_searcher = EnasSearcher(space_fn=enas_space_fn) 40 | sample = enas_searcher.sample() 41 | loss = enas_searcher.update_result(sample, 0.9) 42 | assert loss 43 | -------------------------------------------------------------------------------- /hyperkeras/tests/sharing_weights_test.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | __author__ = 'yangjian' 3 | """ 4 | 5 | """ 6 | import numpy as np 7 | from tensorflow.keras import layers, models, utils 8 | from hypernets.core.ops import * 9 | from hypernets.core.search_space import HyperSpace 10 | from hyperkeras.search_space.enas_common_ops import sepconv3x3, sepconv5x5, avgpooling3x3, \ 11 | maxpooling3x3, identity 12 | from hyperkeras.layer_weights_cache import LayerWeightsCache 13 | from hyperkeras.layers import Input 14 | from hyperkeras.tests import test_output_dir 15 | 16 | selection = 0 17 | 18 | 19 | class LayerChoice(layers.Layer): 20 | def __init__(self, options): 21 | super(LayerChoice, self).__init__() 22 | self.options = options 23 | self._compiled = False 24 | 25 | def call(self, *inputs): 26 | # if not self._compiled: 27 | # for layer in self.options: 28 | # if len(inputs) > 1: 29 | # layer.build([inp.shape for inp in inputs]) 30 | # elif len(inputs) == 1: 31 | # layer.build(inputs[0].shape) 32 | # self._compiled = True 33 | 34 | global selection 35 | choice = self.options[selection] 36 | x = choice(*inputs) 37 | return x 38 | 39 | 40 | class SW_Model(models.Model): 41 | def __init__(self): 42 | super(SW_Model, self).__init__() 43 | self.in_ = layers.Input(shape=(10,)) 44 | options = [layers.Dense(20, activation='relu', name='d1'), 45 | layers.Dense(20, activation='relu', name='d2'), 46 | layers.Dense(20, activation='relu', name='d3')] 47 | self.lc = LayerChoice(options) 48 | self.out_ = layers.Dense(2, activation='softmax') 49 | 50 | def call(self, x): 51 | # x = self.in_(inputs) 52 | x = self.lc(x) 53 | x = self.out_(x) 54 | return x 55 | 56 | 57 | class Test_SharingWeights(): 58 | 59 | def test_layer_cache(self): 60 | cache = LayerWeightsCache() 61 | 62 | def get_space(cache): 63 | space = HyperSpace() 64 | with space.as_default(): 65 | name_prefix = 'test_' 66 | filters = 64 67 | in1 = Input(shape=(28, 28, 1,)) 68 | in2 = Input(shape=(28, 28, 1,)) 69 | ic1 = InputChoice([in1, in2], 1)([in1, in2]) 70 | or1 = ModuleChoice([sepconv5x5(name_prefix, filters), 71 | sepconv3x3(name_prefix, filters), 72 | avgpooling3x3(name_prefix, filters), 73 | maxpooling3x3(name_prefix, filters), 74 | identity(name_prefix)])(ic1) 75 | space.set_inputs([in1, in2]) 76 | space.weights_cache = cache 77 | return space 78 | 79 | space = get_space(cache) 80 | space.assign_by_vectors([1, 0]) 81 | space = space.compile(deepcopy=False) 82 | assert len(space.weights_cache.cache.items()) == 8 83 | assert cache.hit_counter == 0 84 | assert cache.miss_counter == 8 85 | 86 | space = get_space(cache) 87 | space.assign_by_vectors([1, 1]) 88 | space = space.compile(deepcopy=False) 89 | assert len(space.weights_cache.cache.items()) == 14 90 | assert cache.hit_counter == 2 91 | assert cache.miss_counter == 14 92 | 93 | space = get_space(cache) 94 | space.assign_by_vectors([1, 0]) 95 | space = space.compile(deepcopy=False) 96 | assert len(space.weights_cache.cache.items()) == 14 97 | assert cache.hit_counter == 10 98 | assert cache.miss_counter == 14 99 | 100 | def test_model(self): 101 | model = SW_Model() 102 | utils.plot_model(model, to_file=f'{test_output_dir}/test_ops_0.png', show_shapes=True, expand_nested=True) 103 | x = np.random.normal(0.0, 1.0, size=(100, 10)) 104 | y = np.random.randint(0, 2, size=(100), dtype='int') 105 | 106 | global selection 107 | selection = 0 108 | 109 | model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy']) 110 | model.fit(x, y) 111 | result = model.evaluate(x, y) 112 | assert result 113 | out = model(x) 114 | assert out.shape == (100, 2) 115 | # assert model.layers[0].output.shape[1] == 10 116 | 117 | x = np.random.normal(0.0, 1.0, size=(1000, 10)) 118 | y = np.random.randint(0, 2, size=(1000), dtype='int') 119 | selection = 1 120 | out = model(x) 121 | model.fit(x, y) 122 | result = model.evaluate(x, y) 123 | assert out.shape == (1000, 2) 124 | -------------------------------------------------------------------------------- /hyperkeras/utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | __author__ = 'yangjian' 3 | """ 4 | 5 | """ 6 | 7 | 8 | def compile_layer(search_space, layer_class, name, **kwargs): 9 | if kwargs.get('name') is None: 10 | kwargs['name'] = name 11 | 12 | # In the weights sharing mode, the instance is first retrieved from the cache 13 | cache = search_space.__dict__.get('weights_cache') 14 | if cache is not None: 15 | layer = cache.retrieve(kwargs['name']) 16 | if layer is None: 17 | layer = layer_class(**kwargs) 18 | cache.put(kwargs['name'], layer) 19 | else: 20 | layer = layer_class(**kwargs) 21 | 22 | return layer 23 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | hypernets>=0.1.2 2 | tensorflow>=2.1.0 -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataCanvasIO/HyperKeras/05db33b8d26b90ea8b1143e165027eb4cb745363/setup.cfg -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | 3 | from __future__ import absolute_import 4 | 5 | from setuptools import find_packages 6 | from setuptools import setup 7 | 8 | version = '0.1.1' 9 | 10 | requirements = [ 11 | 'hypernets>=0.1.2', 12 | 'tensorflow>=2.1.0' 13 | ] 14 | 15 | MIN_PYTHON_VERSION = '>=3.6.*' 16 | 17 | long_description = open('README.md', encoding='utf-8').read() 18 | 19 | setup( 20 | name='hyperkeras', 21 | version=version, 22 | description='An AutoDL tool based on Tensorflow and Keras', 23 | long_description=long_description, 24 | long_description_content_type="text/markdown", 25 | url='', 26 | author='DataCanvas Community', 27 | author_email='yangjian@zetyun.com', 28 | license='Apache License 2.0', 29 | install_requires=requirements, 30 | python_requires=MIN_PYTHON_VERSION, 31 | extras_require={ 32 | 'hypernets_tests': ['pytest', ], 33 | 'gpu': ['tensorflow-gpu>=2.1.0', ] 34 | }, 35 | 36 | classifiers=[ 37 | 'Operating System :: OS Independent', 38 | 'Intended Audience :: Developers', 39 | 'Intended Audience :: Education', 40 | 'Intended Audience :: Science/Research', 41 | 'Programming Language :: Python', 42 | 'Programming Language :: Python :: 3.6', 43 | 'Programming Language :: Python :: 3.7', 44 | 'Topic :: Scientific/Engineering', 45 | 'Topic :: Scientific/Engineering :: Artificial Intelligence', 46 | 'Topic :: Software Development', 47 | 'Topic :: Software Development :: Libraries', 48 | 'Topic :: Software Development :: Libraries :: Python Modules', 49 | ], 50 | packages=find_packages(exclude=('docs', 'hypernets_tests')), 51 | package_data={ 52 | }, 53 | zip_safe=False, 54 | include_package_data=True, 55 | ) 56 | --------------------------------------------------------------------------------