├── .flake8 ├── .gitignore ├── CONTRIBUTING.md ├── LICENSE ├── NOTICE ├── README.md ├── example ├── config.py ├── config.yaml ├── config_new_allowed.yaml ├── config_new_allowed_bad.yaml ├── config_override.py ├── config_override_from_dict.py └── main.py ├── setup.py ├── tox.ini └── yacs ├── __init__.py ├── config.py └── tests.py /.flake8: -------------------------------------------------------------------------------- 1 | # This is an example .flake8 config, used when developing *Black* itself. 2 | # Keep in sync with setup.cfg which is used for source packages. 3 | 4 | [flake8] 5 | ignore = E203, E266, E501, W503 6 | max-line-length = 80 7 | max-complexity = 18 8 | select = B,C,E,F,W,T4,B9 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .watchmanconfig 2 | build 3 | yacs.egg-info 4 | yacs/__pycache__ 5 | example/__pycache__ 6 | .tox 7 | **/*.pyc 8 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to YACS 2 | We want to make contributing to this project as easy and transparent as 3 | possible. 4 | 5 | ## Our Development Process 6 | Minor changes and improvements will be released on an ongoing basis. Larger 7 | changes (e.g., changesets implementing a new paper) will be released on a more 8 | periodic basis. 9 | 10 | ## Pull Requests 11 | We actively welcome your pull requests. 12 | 13 | 1. Fork the repo and create your branch from `master`. 14 | 2. If you've added code that should be tested, add tests. 15 | 3. If you've changed APIs, update the documentation. 16 | 4. Ensure the test suite passes. 17 | 5. Make sure your code lints. 18 | 6. Ensure no regressions in baseline model speed and accuracy. 19 | 20 | ## Coding Style 21 | * Use automatic formatting: 22 | * `isort -rc -sl -t 1 --atomic .` 23 | * `black .` 24 | 25 | ## License 26 | By contributing to YACS, you agree that your contributions will be licensed 27 | under the LICENSE file in the root directory of this source tree. 28 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Portions of this software are derived from py-faster-rcnn. 2 | 3 | ============================================================================== 4 | py-faster-rcnn licence 5 | ============================================================================== 6 | 7 | Faster R-CNN 8 | 9 | The MIT License (MIT) 10 | 11 | Copyright (c) 2015 Microsoft Corporation 12 | 13 | Permission is hereby granted, free of charge, to any person obtaining a copy 14 | of this software and associated documentation files (the "Software"), to deal 15 | in the Software without restriction, including without limitation the rights 16 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 17 | copies of the Software, and to permit persons to whom the Software is 18 | furnished to do so, subject to the following conditions: 19 | 20 | The above copyright notice and this permission notice shall be included in 21 | all copies or substantial portions of the Software. 22 | 23 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 24 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 25 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 26 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 27 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 28 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 29 | THE SOFTWARE. 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## YACS 2 | 3 | ### Introduction 4 | 5 | YACS was created as a lightweight library to define and manage 6 | system configurations, such as those commonly found in software 7 | designed for scientific experimentation. These "configurations" 8 | typically cover concepts like hyperparameters used in training a 9 | machine learning model or configurable model hyperparameters, such 10 | as the depth of a convolutional neural network. Since you're doing 11 | science, **reproducibility is paramount** and thus you need a reliable 12 | way to serialize experimental configurations. YACS 13 | uses YAML as a simple, human readable serialization format. 14 | The paradigm is: `your code + a YACS config for experiment E (+ 15 | external dependencies + hardware + other nuisance terms ...) = 16 | reproducible experiment E`. While you might not be able to control 17 | everything, at least you can control your code and your experimental 18 | configuration. YACS is here to help you with that. 19 | 20 | YACS grew out of the experimental configuration systems used in: 21 | [py-faster-rcnn](https://github.com/rbgirshick/py-faster-rcnn) and 22 | [Detectron](https://github.com/facebookresearch/Detectron). 23 | 24 | ### Usage 25 | 26 | YACS can be used in a variety of flexible ways. There are two main 27 | paradigms: 28 | 29 | - Configuration as *local variable* 30 | - Configuration as a *global singleton* 31 | 32 | It's up to you which you prefer to use, though the local variable 33 | route is recommended. 34 | 35 | To use YACS in your project, you first create a project config 36 | file, typically called `config.py` or `defaults.py`. *This file 37 | is the one-stop reference point for all configurable options. 38 | It should be very well documented and provide sensible defaults 39 | for all options.* 40 | 41 | ```python 42 | # my_project/config.py 43 | from yacs.config import CfgNode as CN 44 | 45 | 46 | _C = CN() 47 | 48 | _C.SYSTEM = CN() 49 | # Number of GPUS to use in the experiment 50 | _C.SYSTEM.NUM_GPUS = 8 51 | # Number of workers for doing things 52 | _C.SYSTEM.NUM_WORKERS = 4 53 | 54 | _C.TRAIN = CN() 55 | # A very important hyperparameter 56 | _C.TRAIN.HYPERPARAMETER_1 = 0.1 57 | # The all important scales for the stuff 58 | _C.TRAIN.SCALES = (2, 4, 8, 16) 59 | 60 | 61 | def get_cfg_defaults(): 62 | """Get a yacs CfgNode object with default values for my_project.""" 63 | # Return a clone so that the defaults will not be altered 64 | # This is for the "local variable" use pattern 65 | return _C.clone() 66 | 67 | # Alternatively, provide a way to import the defaults as 68 | # a global singleton: 69 | # cfg = _C # users can `from config import cfg` 70 | ``` 71 | 72 | Next, you'll create YAML configuration files; typically you'll make 73 | one for each experiment. Each configuration file only overrides the 74 | options that are changing in that experiment. 75 | 76 | ```yaml 77 | # my_project/experiment.yaml 78 | SYSTEM: 79 | NUM_GPUS: 2 80 | TRAIN: 81 | SCALES: (1, 2) 82 | ``` 83 | 84 | Finally, you'll have your actual project code that uses the config 85 | system. After any initial setup it's a good idea to freeze it to 86 | prevent further modification by calling the `freeze()` method. As 87 | illustrated below, the config options can either be used a global 88 | set of options by importing `cfg` and accessing it directly, or 89 | the `cfg` can be copied and passed as an argument. 90 | 91 | ```python 92 | # my_project/main.py 93 | 94 | import my_project 95 | from config import get_cfg_defaults # local variable usage pattern, or: 96 | # from config import cfg # global singleton usage pattern 97 | 98 | 99 | if __name__ == "__main__": 100 | cfg = get_cfg_defaults() 101 | cfg.merge_from_file("experiment.yaml") 102 | cfg.freeze() 103 | print(cfg) 104 | 105 | # Example of using the cfg as global access to options 106 | if cfg.SYSTEM.NUM_GPUS > 0: 107 | my_project.setup_multi_gpu_support() 108 | 109 | model = my_project.create_model(cfg) 110 | ``` 111 | 112 | #### Command line overrides 113 | 114 | You can update a `CfgNode` using a list of fully-qualified key, value pairs. 115 | This makes it easy to consume override options from the command line. For example: 116 | 117 | ```python 118 | cfg.merge_from_file("experiment.yaml") 119 | # Now override from a list (opts could come from the command line) 120 | opts = ["SYSTEM.NUM_GPUS", 8, "TRAIN.SCALES", "(1, 2, 3, 4)"] 121 | cfg.merge_from_list(opts) 122 | ``` 123 | 124 | The following principle is recommended: "There is only one way to 125 | configure the same thing." This principle means that if an option 126 | is defined in a YACS config object, then your program should set 127 | that configuration option using `cfg.merge_from_list(opts)` and 128 | not by defining, for example, `--train-scales` as a command line 129 | argument that is then used to set `cfg.TRAIN.SCALES`. 130 | 131 | #### Python config files (instead of YAML) 132 | 133 | `yacs>= 0.1.4` supports loading `CfgNode` objects from Python source files. The 134 | convention is that the Python source must export a module variable named `cfg` of 135 | type `dict` or `CfgNode`. See examples using a [CfgNode](example/config_override.py) 136 | and a [dict](example/config_override_from_dict.py) as well as usage in the unit tests. 137 | -------------------------------------------------------------------------------- /example/config.py: -------------------------------------------------------------------------------- 1 | from yacs.config import CfgNode as CN 2 | 3 | _C = CN() 4 | 5 | _C.SYSTEM = CN() 6 | _C.SYSTEM.NUM_GPUS = 8 7 | _C.SYSTEM.NUM_WORKERS = 4 8 | 9 | _C.TRAIN = CN() 10 | _C.TRAIN.HYPERPARAMETER_1 = 0.1 11 | _C.TRAIN.SCALES = (2, 4, 8, 16) 12 | 13 | _C.DATASETS = CN(new_allowed=True) 14 | 15 | cfg = _C 16 | -------------------------------------------------------------------------------- /example/config.yaml: -------------------------------------------------------------------------------- 1 | SYSTEM: 2 | NUM_GPUS: 2 3 | TRAIN: 4 | SCALES: (1, 2) 5 | DATASETS: 6 | train_2017: 7 | 17: 1 8 | 18: 1 9 | -------------------------------------------------------------------------------- /example/config_new_allowed.yaml: -------------------------------------------------------------------------------- 1 | KWARGS: 2 | a: 1 # Test adding of basic value 3 | B: 4 | c: 2 # Test adding of another node 5 | D: 6 | e: '3' # Test adding of nested nodes 7 | -------------------------------------------------------------------------------- /example/config_new_allowed_bad.yaml: -------------------------------------------------------------------------------- 1 | KWARGS: 2 | Y: 3 | f: 4 # While `KWARGS` allows new nodes, `KWARGS.Y` doesn't, so this should raise an Exception 4 | -------------------------------------------------------------------------------- /example/config_override.py: -------------------------------------------------------------------------------- 1 | # NB: This file is used in unit tests in tests.py; do not change unless you know 2 | # what you're doing 3 | 4 | from yacs.config import CfgNode as CN 5 | 6 | HYPERPARAMETER_1_BASE_VALUE = 1.0 7 | 8 | cfg = CN() 9 | 10 | cfg.TRAIN = CN() 11 | cfg.TRAIN.HYPERPARAMETER_1 = 0.9 * HYPERPARAMETER_1_BASE_VALUE 12 | -------------------------------------------------------------------------------- /example/config_override_from_dict.py: -------------------------------------------------------------------------------- 1 | # NB: This file is used in unit tests in tests.py; do not change unless you know 2 | # what you're doing 3 | 4 | HYPERPARAMETER_1_BASE_VALUE = 1.0 5 | 6 | cfg = {"TRAIN": {"HYPERPARAMETER_1": 0.9 * HYPERPARAMETER_1_BASE_VALUE}} 7 | -------------------------------------------------------------------------------- /example/main.py: -------------------------------------------------------------------------------- 1 | from config import cfg 2 | 3 | if __name__ == "__main__": 4 | cfg.merge_from_file("config.yaml") 5 | cfg.freeze() 6 | 7 | cfg2 = cfg.clone() 8 | cfg2.defrost() 9 | cfg2.TRAIN.SCALES = (8, 16, 32) 10 | cfg2.freeze() 11 | 12 | print("cfg:") 13 | print(cfg) 14 | print("cfg2:") 15 | print(cfg2) 16 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name="yacs", 5 | version="0.1.8", 6 | author="Ross Girshick", 7 | author_email="ross.girshick@gmail.com", 8 | description="Yet Another Configuration System", 9 | url="https://github.com/rbgirshick/yacs", 10 | packages=["yacs"], 11 | long_description="A simple experiment configuration system for research", 12 | classifiers=[ 13 | "Programming Language :: Python :: 2.7", 14 | "Programming Language :: Python :: 3", 15 | "License :: OSI Approved :: Apache Software License", 16 | "Operating System :: OS Independent", 17 | "Intended Audience :: Science/Research", 18 | "Topic :: Scientific/Engineering :: Artificial Intelligence", 19 | ], 20 | install_requires=["PyYAML"], 21 | ) 22 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = {py27,py36}-pyyaml{3,4} 3 | 4 | [testenv] 5 | deps = 6 | pyyaml3: PyYAML>=3,<4 7 | pyyaml4: PyYAML==4.2b4 8 | commands = python yacs/tests.py 9 | 10 | -------------------------------------------------------------------------------- /yacs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rbgirshick/yacs/32d5e4ac300eca6cd3b839097dde39c4017a1070/yacs/__init__.py -------------------------------------------------------------------------------- /yacs/config.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018-present, Facebook, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | ############################################################################## 15 | 16 | """YACS -- Yet Another Configuration System is designed to be a simple 17 | configuration management system for academic and industrial research 18 | projects. 19 | 20 | See README.md for usage and examples. 21 | """ 22 | 23 | import copy 24 | import io 25 | import logging 26 | import os 27 | import sys 28 | from ast import literal_eval 29 | 30 | import yaml 31 | 32 | # Flag for py2 and py3 compatibility to use when separate code paths are necessary 33 | # When _PY2 is False, we assume Python 3 is in use 34 | _PY2 = sys.version_info.major == 2 35 | 36 | # Filename extensions for loading configs from files 37 | _YAML_EXTS = {"", ".yaml", ".yml"} 38 | _PY_EXTS = {".py"} 39 | 40 | # py2 and py3 compatibility for checking file object type 41 | # We simply use this to infer py2 vs py3 42 | if _PY2: 43 | _FILE_TYPES = (file, io.IOBase) 44 | else: 45 | _FILE_TYPES = (io.IOBase,) 46 | 47 | # CfgNodes can only contain a limited set of valid types 48 | _VALID_TYPES = {tuple, list, str, int, float, bool, type(None)} 49 | # py2 allow for str and unicode 50 | if _PY2: 51 | _VALID_TYPES = _VALID_TYPES.union({unicode}) # noqa: F821 52 | 53 | # Utilities for importing modules from file paths 54 | if _PY2: 55 | # imp is available in both py2 and py3 for now, but is deprecated in py3 56 | import imp 57 | else: 58 | import importlib.util 59 | 60 | logger = logging.getLogger(__name__) 61 | 62 | 63 | class CfgNode(dict): 64 | """ 65 | CfgNode represents an internal node in the configuration tree. It's a simple 66 | dict-like container that allows for attribute-based access to keys. 67 | """ 68 | 69 | IMMUTABLE = "__immutable__" 70 | DEPRECATED_KEYS = "__deprecated_keys__" 71 | RENAMED_KEYS = "__renamed_keys__" 72 | NEW_ALLOWED = "__new_allowed__" 73 | 74 | def __init__(self, init_dict=None, key_list=None, new_allowed=False): 75 | """ 76 | Args: 77 | init_dict (dict): the possibly-nested dictionary to initailize the CfgNode. 78 | key_list (list[str]): a list of names which index this CfgNode from the root. 79 | Currently only used for logging purposes. 80 | new_allowed (bool): whether adding new key is allowed when merging with 81 | other configs. 82 | """ 83 | # Recursively convert nested dictionaries in init_dict into CfgNodes 84 | init_dict = {} if init_dict is None else init_dict 85 | key_list = [] if key_list is None else key_list 86 | init_dict = self._create_config_tree_from_dict(init_dict, key_list) 87 | super(CfgNode, self).__init__(init_dict) 88 | # Manage if the CfgNode is frozen or not 89 | self.__dict__[CfgNode.IMMUTABLE] = False 90 | # Deprecated options 91 | # If an option is removed from the code and you don't want to break existing 92 | # yaml configs, you can add the full config key as a string to the set below. 93 | self.__dict__[CfgNode.DEPRECATED_KEYS] = set() 94 | # Renamed options 95 | # If you rename a config option, record the mapping from the old name to the new 96 | # name in the dictionary below. Optionally, if the type also changed, you can 97 | # make the value a tuple that specifies first the renamed key and then 98 | # instructions for how to edit the config file. 99 | self.__dict__[CfgNode.RENAMED_KEYS] = { 100 | # 'EXAMPLE.OLD.KEY': 'EXAMPLE.NEW.KEY', # Dummy example to follow 101 | # 'EXAMPLE.OLD.KEY': ( # A more complex example to follow 102 | # 'EXAMPLE.NEW.KEY', 103 | # "Also convert to a tuple, e.g., 'foo' -> ('foo',) or " 104 | # + "'foo:bar' -> ('foo', 'bar')" 105 | # ), 106 | } 107 | 108 | # Allow new attributes after initialisation 109 | self.__dict__[CfgNode.NEW_ALLOWED] = new_allowed 110 | 111 | @classmethod 112 | def _create_config_tree_from_dict(cls, dic, key_list): 113 | """ 114 | Create a configuration tree using the given dict. 115 | Any dict-like objects inside dict will be treated as a new CfgNode. 116 | 117 | Args: 118 | dic (dict): 119 | key_list (list[str]): a list of names which index this CfgNode from the root. 120 | Currently only used for logging purposes. 121 | """ 122 | dic = copy.deepcopy(dic) 123 | for k, v in dic.items(): 124 | if isinstance(v, dict): 125 | # Convert dict to CfgNode 126 | dic[k] = cls(v, key_list=key_list + [k]) 127 | else: 128 | # Check for valid leaf type or nested CfgNode 129 | _assert_with_logging( 130 | _valid_type(v, allow_cfg_node=False), 131 | "Key {} with value {} is not a valid type; valid types: {}".format( 132 | ".".join(key_list + [str(k)]), type(v), _VALID_TYPES 133 | ), 134 | ) 135 | return dic 136 | 137 | def __getattr__(self, name): 138 | if name in self: 139 | return self[name] 140 | else: 141 | raise AttributeError(name) 142 | 143 | def __setattr__(self, name, value): 144 | if self.is_frozen(): 145 | raise AttributeError( 146 | "Attempted to set {} to {}, but CfgNode is immutable".format( 147 | name, value 148 | ) 149 | ) 150 | 151 | _assert_with_logging( 152 | name not in self.__dict__, 153 | "Invalid attempt to modify internal CfgNode state: {}".format(name), 154 | ) 155 | _assert_with_logging( 156 | _valid_type(value, allow_cfg_node=True), 157 | "Invalid type {} for key {}; valid types = {}".format( 158 | type(value), name, _VALID_TYPES 159 | ), 160 | ) 161 | 162 | self[name] = value 163 | 164 | def __str__(self): 165 | def _indent(s_, num_spaces): 166 | s = s_.split("\n") 167 | if len(s) == 1: 168 | return s_ 169 | first = s.pop(0) 170 | s = [(num_spaces * " ") + line for line in s] 171 | s = "\n".join(s) 172 | s = first + "\n" + s 173 | return s 174 | 175 | r = "" 176 | s = [] 177 | for k, v in sorted(self.items()): 178 | seperator = "\n" if isinstance(v, CfgNode) else " " 179 | attr_str = "{}:{}{}".format(str(k), seperator, str(v)) 180 | attr_str = _indent(attr_str, 2) 181 | s.append(attr_str) 182 | r += "\n".join(s) 183 | return r 184 | 185 | def __repr__(self): 186 | return "{}({})".format(self.__class__.__name__, super(CfgNode, self).__repr__()) 187 | 188 | def dump(self, **kwargs): 189 | """Dump to a string.""" 190 | 191 | def convert_to_dict(cfg_node, key_list): 192 | if not isinstance(cfg_node, CfgNode): 193 | _assert_with_logging( 194 | _valid_type(cfg_node), 195 | "Key {} with value {} is not a valid type; valid types: {}".format( 196 | ".".join(key_list), type(cfg_node), _VALID_TYPES 197 | ), 198 | ) 199 | return cfg_node 200 | else: 201 | cfg_dict = dict(cfg_node) 202 | for k, v in cfg_dict.items(): 203 | cfg_dict[k] = convert_to_dict(v, key_list + [k]) 204 | return cfg_dict 205 | 206 | self_as_dict = convert_to_dict(self, []) 207 | return yaml.safe_dump(self_as_dict, **kwargs) 208 | 209 | def merge_from_file(self, cfg_filename): 210 | """Load a yaml config file and merge it this CfgNode.""" 211 | with open(cfg_filename, "r") as f: 212 | cfg = self.load_cfg(f) 213 | self.merge_from_other_cfg(cfg) 214 | 215 | def merge_from_other_cfg(self, cfg_other): 216 | """Merge `cfg_other` into this CfgNode.""" 217 | _merge_a_into_b(cfg_other, self, self, []) 218 | 219 | def merge_from_list(self, cfg_list): 220 | """Merge config (keys, values) in a list (e.g., from command line) into 221 | this CfgNode. For example, `cfg_list = ['FOO.BAR', 0.5]`. 222 | """ 223 | _assert_with_logging( 224 | len(cfg_list) % 2 == 0, 225 | "Override list has odd length: {}; it must be a list of pairs".format( 226 | cfg_list 227 | ), 228 | ) 229 | root = self 230 | for full_key, v in zip(cfg_list[0::2], cfg_list[1::2]): 231 | if root.key_is_deprecated(full_key): 232 | continue 233 | if root.key_is_renamed(full_key): 234 | root.raise_key_rename_error(full_key) 235 | key_list = full_key.split(".") 236 | d = self 237 | for subkey in key_list[:-1]: 238 | _assert_with_logging( 239 | subkey in d, "Non-existent key: {}".format(full_key) 240 | ) 241 | d = d[subkey] 242 | subkey = key_list[-1] 243 | _assert_with_logging(subkey in d, "Non-existent key: {}".format(full_key)) 244 | value = self._decode_cfg_value(v) 245 | value = _check_and_coerce_cfg_value_type(value, d[subkey], subkey, full_key) 246 | d[subkey] = value 247 | 248 | def freeze(self): 249 | """Make this CfgNode and all of its children immutable.""" 250 | self._immutable(True) 251 | 252 | def defrost(self): 253 | """Make this CfgNode and all of its children mutable.""" 254 | self._immutable(False) 255 | 256 | def is_frozen(self): 257 | """Return mutability.""" 258 | return self.__dict__[CfgNode.IMMUTABLE] 259 | 260 | def _immutable(self, is_immutable): 261 | """Set immutability to is_immutable and recursively apply the setting 262 | to all nested CfgNodes. 263 | """ 264 | self.__dict__[CfgNode.IMMUTABLE] = is_immutable 265 | # Recursively set immutable state 266 | for v in self.__dict__.values(): 267 | if isinstance(v, CfgNode): 268 | v._immutable(is_immutable) 269 | for v in self.values(): 270 | if isinstance(v, CfgNode): 271 | v._immutable(is_immutable) 272 | 273 | def clone(self): 274 | """Recursively copy this CfgNode.""" 275 | return copy.deepcopy(self) 276 | 277 | def register_deprecated_key(self, key): 278 | """Register key (e.g. `FOO.BAR`) a deprecated option. When merging deprecated 279 | keys a warning is generated and the key is ignored. 280 | """ 281 | _assert_with_logging( 282 | key not in self.__dict__[CfgNode.DEPRECATED_KEYS], 283 | "key {} is already registered as a deprecated key".format(key), 284 | ) 285 | self.__dict__[CfgNode.DEPRECATED_KEYS].add(key) 286 | 287 | def register_renamed_key(self, old_name, new_name, message=None): 288 | """Register a key as having been renamed from `old_name` to `new_name`. 289 | When merging a renamed key, an exception is thrown alerting to user to 290 | the fact that the key has been renamed. 291 | """ 292 | _assert_with_logging( 293 | old_name not in self.__dict__[CfgNode.RENAMED_KEYS], 294 | "key {} is already registered as a renamed cfg key".format(old_name), 295 | ) 296 | value = new_name 297 | if message: 298 | value = (new_name, message) 299 | self.__dict__[CfgNode.RENAMED_KEYS][old_name] = value 300 | 301 | def key_is_deprecated(self, full_key): 302 | """Test if a key is deprecated.""" 303 | if full_key in self.__dict__[CfgNode.DEPRECATED_KEYS]: 304 | logger.warning("Deprecated config key (ignoring): {}".format(full_key)) 305 | return True 306 | return False 307 | 308 | def key_is_renamed(self, full_key): 309 | """Test if a key is renamed.""" 310 | return full_key in self.__dict__[CfgNode.RENAMED_KEYS] 311 | 312 | def raise_key_rename_error(self, full_key): 313 | new_key = self.__dict__[CfgNode.RENAMED_KEYS][full_key] 314 | if isinstance(new_key, tuple): 315 | msg = " Note: " + new_key[1] 316 | new_key = new_key[0] 317 | else: 318 | msg = "" 319 | raise KeyError( 320 | "Key {} was renamed to {}; please update your config.{}".format( 321 | full_key, new_key, msg 322 | ) 323 | ) 324 | 325 | def is_new_allowed(self): 326 | return self.__dict__[CfgNode.NEW_ALLOWED] 327 | 328 | def set_new_allowed(self, is_new_allowed): 329 | """ 330 | Set this config (and recursively its subconfigs) to allow merging 331 | new keys from other configs. 332 | """ 333 | self.__dict__[CfgNode.NEW_ALLOWED] = is_new_allowed 334 | # Recursively set new_allowed state 335 | for v in self.__dict__.values(): 336 | if isinstance(v, CfgNode): 337 | v.set_new_allowed(is_new_allowed) 338 | for v in self.values(): 339 | if isinstance(v, CfgNode): 340 | v.set_new_allowed(is_new_allowed) 341 | 342 | @classmethod 343 | def load_cfg(cls, cfg_file_obj_or_str): 344 | """ 345 | Load a cfg. 346 | Args: 347 | cfg_file_obj_or_str (str or file): 348 | Supports loading from: 349 | - A file object backed by a YAML file 350 | - A file object backed by a Python source file that exports an attribute 351 | "cfg" that is either a dict or a CfgNode 352 | - A string that can be parsed as valid YAML 353 | """ 354 | _assert_with_logging( 355 | isinstance(cfg_file_obj_or_str, _FILE_TYPES + (str,)), 356 | "Expected first argument to be of type {} or {}, but it was {}".format( 357 | _FILE_TYPES, str, type(cfg_file_obj_or_str) 358 | ), 359 | ) 360 | if isinstance(cfg_file_obj_or_str, str): 361 | return cls._load_cfg_from_yaml_str(cfg_file_obj_or_str) 362 | elif isinstance(cfg_file_obj_or_str, _FILE_TYPES): 363 | return cls._load_cfg_from_file(cfg_file_obj_or_str) 364 | else: 365 | raise NotImplementedError("Impossible to reach here (unless there's a bug)") 366 | 367 | @classmethod 368 | def _load_cfg_from_file(cls, file_obj): 369 | """Load a config from a YAML file or a Python source file.""" 370 | _, file_extension = os.path.splitext(file_obj.name) 371 | if file_extension in _YAML_EXTS: 372 | return cls._load_cfg_from_yaml_str(file_obj.read()) 373 | elif file_extension in _PY_EXTS: 374 | return cls._load_cfg_py_source(file_obj.name) 375 | else: 376 | raise Exception( 377 | "Attempt to load from an unsupported file type {}; " 378 | "only {} are supported".format(file_obj, _YAML_EXTS.union(_PY_EXTS)) 379 | ) 380 | 381 | @classmethod 382 | def _load_cfg_from_yaml_str(cls, str_obj): 383 | """Load a config from a YAML string encoding.""" 384 | cfg_as_dict = yaml.safe_load(str_obj) 385 | return cls(cfg_as_dict) 386 | 387 | @classmethod 388 | def _load_cfg_py_source(cls, filename): 389 | """Load a config from a Python source file.""" 390 | module = _load_module_from_file("yacs.config.override", filename) 391 | _assert_with_logging( 392 | hasattr(module, "cfg"), 393 | "Python module from file {} must have 'cfg' attr".format(filename), 394 | ) 395 | VALID_ATTR_TYPES = {dict, CfgNode} 396 | _assert_with_logging( 397 | type(module.cfg) in VALID_ATTR_TYPES, 398 | "Imported module 'cfg' attr must be in {} but is {} instead".format( 399 | VALID_ATTR_TYPES, type(module.cfg) 400 | ), 401 | ) 402 | return cls(module.cfg) 403 | 404 | @classmethod 405 | def _decode_cfg_value(cls, value): 406 | """ 407 | Decodes a raw config value (e.g., from a yaml config files or command 408 | line argument) into a Python object. 409 | 410 | If the value is a dict, it will be interpreted as a new CfgNode. 411 | If the value is a str, it will be evaluated as literals. 412 | Otherwise it is returned as-is. 413 | """ 414 | # Configs parsed from raw yaml will contain dictionary keys that need to be 415 | # converted to CfgNode objects 416 | if isinstance(value, dict): 417 | return cls(value) 418 | # All remaining processing is only applied to strings 419 | if not isinstance(value, str): 420 | return value 421 | # Try to interpret `value` as a: 422 | # string, number, tuple, list, dict, boolean, or None 423 | try: 424 | value = literal_eval(value) 425 | # The following two excepts allow v to pass through when it represents a 426 | # string. 427 | # 428 | # Longer explanation: 429 | # The type of v is always a string (before calling literal_eval), but 430 | # sometimes it *represents* a string and other times a data structure, like 431 | # a list. In the case that v represents a string, what we got back from the 432 | # yaml parser is 'foo' *without quotes* (so, not '"foo"'). literal_eval is 433 | # ok with '"foo"', but will raise a ValueError if given 'foo'. In other 434 | # cases, like paths (v = 'foo/bar' and not v = '"foo/bar"'), literal_eval 435 | # will raise a SyntaxError. 436 | except ValueError: 437 | pass 438 | except SyntaxError: 439 | pass 440 | return value 441 | 442 | 443 | load_cfg = ( 444 | CfgNode.load_cfg 445 | ) # keep this function in global scope for backward compatibility 446 | 447 | 448 | def _valid_type(value, allow_cfg_node=False): 449 | return (type(value) in _VALID_TYPES) or ( 450 | allow_cfg_node and isinstance(value, CfgNode) 451 | ) 452 | 453 | 454 | def _merge_a_into_b(a, b, root, key_list): 455 | """Merge config dictionary a into config dictionary b, clobbering the 456 | options in b whenever they are also specified in a. 457 | """ 458 | _assert_with_logging( 459 | isinstance(a, CfgNode), 460 | "`a` (cur type {}) must be an instance of {}".format(type(a), CfgNode), 461 | ) 462 | _assert_with_logging( 463 | isinstance(b, CfgNode), 464 | "`b` (cur type {}) must be an instance of {}".format(type(b), CfgNode), 465 | ) 466 | 467 | for k, v_ in a.items(): 468 | full_key = ".".join(key_list + [k]) 469 | 470 | v = copy.deepcopy(v_) 471 | v = b._decode_cfg_value(v) 472 | 473 | if k in b: 474 | v = _check_and_coerce_cfg_value_type(v, b[k], k, full_key) 475 | # Recursively merge dicts 476 | if isinstance(v, CfgNode): 477 | try: 478 | _merge_a_into_b(v, b[k], root, key_list + [k]) 479 | except BaseException: 480 | raise 481 | else: 482 | b[k] = v 483 | elif b.is_new_allowed(): 484 | b[k] = v 485 | else: 486 | if root.key_is_deprecated(full_key): 487 | continue 488 | elif root.key_is_renamed(full_key): 489 | root.raise_key_rename_error(full_key) 490 | else: 491 | raise KeyError("Non-existent config key: {}".format(full_key)) 492 | 493 | 494 | def _check_and_coerce_cfg_value_type(replacement, original, key, full_key): 495 | """Checks that `replacement`, which is intended to replace `original` is of 496 | the right type. The type is correct if it matches exactly or is one of a few 497 | cases in which the type can be easily coerced. 498 | """ 499 | original_type = type(original) 500 | replacement_type = type(replacement) 501 | 502 | # The types must match (with some exceptions) 503 | if replacement_type == original_type: 504 | return replacement 505 | 506 | # If either of them is None, allow type conversion to one of the valid types 507 | if (replacement_type == type(None) and original_type in _VALID_TYPES) or ( 508 | original_type == type(None) and replacement_type in _VALID_TYPES 509 | ): 510 | return replacement 511 | 512 | # Cast replacement from from_type to to_type if the replacement and original 513 | # types match from_type and to_type 514 | def conditional_cast(from_type, to_type): 515 | if replacement_type == from_type and original_type == to_type: 516 | return True, to_type(replacement) 517 | else: 518 | return False, None 519 | 520 | # Conditionally casts 521 | # list <-> tuple 522 | casts = [(tuple, list), (list, tuple)] 523 | # For py2: allow converting from str (bytes) to a unicode string 524 | try: 525 | casts.append((str, unicode)) # noqa: F821 526 | except Exception: 527 | pass 528 | 529 | for (from_type, to_type) in casts: 530 | converted, converted_value = conditional_cast(from_type, to_type) 531 | if converted: 532 | return converted_value 533 | 534 | raise ValueError( 535 | "Type mismatch ({} vs. {}) with values ({} vs. {}) for config " 536 | "key: {}".format( 537 | original_type, replacement_type, original, replacement, full_key 538 | ) 539 | ) 540 | 541 | 542 | def _assert_with_logging(cond, msg): 543 | if not cond: 544 | logger.debug(msg) 545 | assert cond, msg 546 | 547 | 548 | def _load_module_from_file(name, filename): 549 | if _PY2: 550 | module = imp.load_source(name, filename) 551 | else: 552 | spec = importlib.util.spec_from_file_location(name, filename) 553 | module = importlib.util.module_from_spec(spec) 554 | spec.loader.exec_module(module) 555 | return module 556 | -------------------------------------------------------------------------------- /yacs/tests.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import tempfile 3 | import unittest 4 | 5 | import yacs.config 6 | import yaml 7 | from yacs.config import CfgNode as CN 8 | 9 | try: 10 | _ignore = unicode # noqa: F821 11 | PY2 = True 12 | except Exception as _ignore: 13 | PY2 = False 14 | 15 | 16 | class SubCN(CN): 17 | pass 18 | 19 | 20 | def get_cfg(cls=CN): 21 | cfg = cls() 22 | 23 | cfg.NUM_GPUS = 8 24 | 25 | cfg.TRAIN = cls() 26 | cfg.TRAIN.HYPERPARAMETER_1 = 0.1 27 | cfg.TRAIN.SCALES = (2, 4, 8, 16) 28 | 29 | cfg.MODEL = cls() 30 | cfg.MODEL.TYPE = "a_foo_model" 31 | 32 | # Some extra stuff to test CfgNode.__str__ 33 | cfg.STR = cls() 34 | cfg.STR.KEY1 = 1 35 | cfg.STR.KEY2 = 2 36 | cfg.STR.FOO = cls() 37 | cfg.STR.FOO.KEY1 = 1 38 | cfg.STR.FOO.KEY2 = 2 39 | cfg.STR.FOO.BAR = cls() 40 | cfg.STR.FOO.BAR.KEY1 = 1 41 | cfg.STR.FOO.BAR.KEY2 = 2 42 | 43 | cfg.register_deprecated_key("FINAL_MSG") 44 | cfg.register_deprecated_key("MODEL.DILATION") 45 | 46 | cfg.register_renamed_key( 47 | "EXAMPLE.OLD.KEY", 48 | "EXAMPLE.NEW.KEY", 49 | message="Please update your config fil config file.", 50 | ) 51 | 52 | cfg.KWARGS = cls(new_allowed=True) 53 | cfg.KWARGS.z = 0 54 | cfg.KWARGS.Y = cls() 55 | cfg.KWARGS.Y.X = 1 56 | 57 | return cfg 58 | 59 | 60 | class TestCfgNode(unittest.TestCase): 61 | def test_immutability(self): 62 | # Top level immutable 63 | a = CN() 64 | a.foo = 0 65 | a.freeze() 66 | with self.assertRaises(AttributeError): 67 | a.foo = 1 68 | a.bar = 1 69 | assert a.is_frozen() 70 | assert a.foo == 0 71 | a.defrost() 72 | assert not a.is_frozen() 73 | a.foo = 1 74 | assert a.foo == 1 75 | 76 | # Recursively immutable 77 | a.level1 = CN() 78 | a.level1.foo = 0 79 | a.level1.level2 = CN() 80 | a.level1.level2.foo = 0 81 | a.freeze() 82 | assert a.is_frozen() 83 | with self.assertRaises(AttributeError): 84 | a.level1.level2.foo = 1 85 | a.level1.bar = 1 86 | assert a.level1.level2.foo == 0 87 | 88 | 89 | class TestCfg(unittest.TestCase): 90 | def test_copy_cfg(self): 91 | cfg = get_cfg() 92 | cfg2 = cfg.clone() 93 | s = cfg.MODEL.TYPE 94 | cfg2.MODEL.TYPE = "dummy" 95 | assert cfg.MODEL.TYPE == s 96 | 97 | def test_merge_cfg_from_cfg(self): 98 | # Test: merge from clone 99 | cfg = get_cfg() 100 | s = "dummy0" 101 | cfg2 = cfg.clone() 102 | cfg2.MODEL.TYPE = s 103 | cfg.merge_from_other_cfg(cfg2) 104 | assert cfg.MODEL.TYPE == s 105 | 106 | # Test: merge from yaml 107 | s = "dummy1" 108 | cfg2 = CN.load_cfg(cfg.dump()) 109 | cfg2.MODEL.TYPE = s 110 | cfg.merge_from_other_cfg(cfg2) 111 | assert cfg.MODEL.TYPE == s 112 | 113 | # Test: merge with a valid key 114 | s = "dummy2" 115 | cfg2 = CN() 116 | cfg2.MODEL = CN() 117 | cfg2.MODEL.TYPE = s 118 | cfg.merge_from_other_cfg(cfg2) 119 | assert cfg.MODEL.TYPE == s 120 | 121 | # Test: merge with an invalid key 122 | s = "dummy3" 123 | cfg2 = CN() 124 | cfg2.FOO = CN() 125 | cfg2.FOO.BAR = s 126 | with self.assertRaises(KeyError): 127 | cfg.merge_from_other_cfg(cfg2) 128 | 129 | # Test: merge with converted type 130 | cfg2 = CN() 131 | cfg2.TRAIN = CN() 132 | cfg2.TRAIN.SCALES = [1] 133 | cfg.merge_from_other_cfg(cfg2) 134 | assert type(cfg.TRAIN.SCALES) is tuple 135 | assert cfg.TRAIN.SCALES[0] == 1 136 | 137 | # Test str (bytes) <-> unicode conversion for py2 138 | if PY2: 139 | cfg.A_UNICODE_KEY = u"foo" 140 | cfg2 = CN() 141 | cfg2.A_UNICODE_KEY = b"bar" 142 | cfg.merge_from_other_cfg(cfg2) 143 | assert type(cfg.A_UNICODE_KEY) == unicode # noqa: F821 144 | assert cfg.A_UNICODE_KEY == u"bar" 145 | 146 | # Test: merge with invalid type 147 | cfg2 = CN() 148 | cfg2.TRAIN = CN() 149 | cfg2.TRAIN.SCALES = 1 150 | with self.assertRaises(ValueError): 151 | cfg.merge_from_other_cfg(cfg2) 152 | 153 | def test_merge_cfg_from_file(self): 154 | with tempfile.NamedTemporaryFile(mode="wt") as f: 155 | cfg = get_cfg() 156 | f.write(cfg.dump()) 157 | f.flush() 158 | s = cfg.MODEL.TYPE 159 | cfg.MODEL.TYPE = "dummy" 160 | assert cfg.MODEL.TYPE != s 161 | cfg.merge_from_file(f.name) 162 | assert cfg.MODEL.TYPE == s 163 | 164 | def test_merge_cfg_from_list(self): 165 | cfg = get_cfg() 166 | opts = ["TRAIN.SCALES", "(100, )", "MODEL.TYPE", "foobar", "NUM_GPUS", 2] 167 | assert len(cfg.TRAIN.SCALES) > 0 168 | assert cfg.TRAIN.SCALES[0] != 100 169 | assert cfg.MODEL.TYPE != "foobar" 170 | assert cfg.NUM_GPUS != 2 171 | cfg.merge_from_list(opts) 172 | assert type(cfg.TRAIN.SCALES) is tuple 173 | assert len(cfg.TRAIN.SCALES) == 1 174 | assert cfg.TRAIN.SCALES[0] == 100 175 | assert cfg.MODEL.TYPE == "foobar" 176 | assert cfg.NUM_GPUS == 2 177 | 178 | def test_deprecated_key_from_list(self): 179 | # You should see logger messages like: 180 | # "Deprecated config key (ignoring): MODEL.DILATION" 181 | cfg = get_cfg() 182 | opts = ["FINAL_MSG", "foobar", "MODEL.DILATION", 2] 183 | with self.assertRaises(AttributeError): 184 | _ = cfg.FINAL_MSG # noqa 185 | with self.assertRaises(AttributeError): 186 | _ = cfg.MODEL.DILATION # noqa 187 | cfg.merge_from_list(opts) 188 | with self.assertRaises(AttributeError): 189 | _ = cfg.FINAL_MSG # noqa 190 | with self.assertRaises(AttributeError): 191 | _ = cfg.MODEL.DILATION # noqa 192 | 193 | def test_nonexistant_key_from_list(self): 194 | cfg = get_cfg() 195 | opts = ["MODEL.DOES_NOT_EXIST", "IGNORE"] 196 | with self.assertRaises(AssertionError): 197 | cfg.merge_from_list(opts) 198 | 199 | def test_load_cfg_invalid_type(self): 200 | class CustomClass(yaml.YAMLObject): 201 | """A custom class that yaml.safe_load can load.""" 202 | 203 | yaml_loader = yaml.SafeLoader 204 | yaml_tag = u"!CustomClass" 205 | 206 | # FOO.BAR.QUUX will have type CustomClass, which is not allowed 207 | cfg_string = "FOO:\n BAR:\n QUUX: !CustomClass {}" 208 | with self.assertRaises(AssertionError): 209 | yacs.config.load_cfg(cfg_string) 210 | 211 | def test_deprecated_key_from_file(self): 212 | # You should see logger messages like: 213 | # "Deprecated config key (ignoring): MODEL.DILATION" 214 | cfg = get_cfg() 215 | with tempfile.NamedTemporaryFile("wt") as f: 216 | cfg2 = cfg.clone() 217 | cfg2.MODEL.DILATION = 2 218 | f.write(cfg2.dump()) 219 | f.flush() 220 | with self.assertRaises(AttributeError): 221 | _ = cfg.MODEL.DILATION # noqa 222 | cfg.merge_from_file(f.name) 223 | with self.assertRaises(AttributeError): 224 | _ = cfg.MODEL.DILATION # noqa 225 | 226 | def test_renamed_key_from_list(self): 227 | cfg = get_cfg() 228 | opts = ["EXAMPLE.OLD.KEY", "foobar"] 229 | with self.assertRaises(AttributeError): 230 | _ = cfg.EXAMPLE.OLD.KEY # noqa 231 | with self.assertRaises(KeyError): 232 | cfg.merge_from_list(opts) 233 | 234 | def test_renamed_key_from_file(self): 235 | cfg = get_cfg() 236 | with tempfile.NamedTemporaryFile("wt") as f: 237 | cfg2 = cfg.clone() 238 | cfg2.EXAMPLE = CN() 239 | cfg2.EXAMPLE.RENAMED = CN() 240 | cfg2.EXAMPLE.RENAMED.KEY = "foobar" 241 | f.write(cfg2.dump()) 242 | f.flush() 243 | with self.assertRaises(AttributeError): 244 | _ = cfg.EXAMPLE.RENAMED.KEY # noqa 245 | with self.assertRaises(KeyError): 246 | cfg.merge_from_file(f.name) 247 | 248 | def test_load_cfg_from_file(self): 249 | cfg = get_cfg() 250 | with tempfile.NamedTemporaryFile("wt") as f: 251 | f.write(cfg.dump()) 252 | f.flush() 253 | with open(f.name, "rt") as f_read: 254 | yacs.config.load_cfg(f_read) 255 | 256 | def test_load_from_python_file(self): 257 | # Case 1: exports CfgNode 258 | cfg = get_cfg() 259 | cfg.merge_from_file("example/config_override.py") 260 | assert cfg.TRAIN.HYPERPARAMETER_1 == 0.9 261 | # Case 2: exports dict 262 | cfg = get_cfg() 263 | cfg.merge_from_file("example/config_override_from_dict.py") 264 | assert cfg.TRAIN.HYPERPARAMETER_1 == 0.9 265 | 266 | def test_invalid_type(self): 267 | cfg = get_cfg() 268 | with self.assertRaises(AssertionError): 269 | cfg.INVALID_KEY_TYPE = object() 270 | 271 | def test__str__(self): 272 | expected_str = """ 273 | KWARGS: 274 | Y: 275 | X: 1 276 | z: 0 277 | MODEL: 278 | TYPE: a_foo_model 279 | NUM_GPUS: 8 280 | STR: 281 | FOO: 282 | BAR: 283 | KEY1: 1 284 | KEY2: 2 285 | KEY1: 1 286 | KEY2: 2 287 | KEY1: 1 288 | KEY2: 2 289 | TRAIN: 290 | HYPERPARAMETER_1: 0.1 291 | SCALES: (2, 4, 8, 16) 292 | """.strip() 293 | cfg = get_cfg() 294 | assert str(cfg) == expected_str 295 | 296 | def test_new_allowed(self): 297 | cfg = get_cfg() 298 | cfg.merge_from_file("example/config_new_allowed.yaml") 299 | assert cfg.KWARGS.a == 1 300 | assert cfg.KWARGS.B.c == 2 301 | assert cfg.KWARGS.B.D.e == "3" 302 | 303 | def test_new_allowed_bad(self): 304 | cfg = get_cfg() 305 | with self.assertRaises(KeyError): 306 | cfg.merge_from_file("example/config_new_allowed_bad.yaml") 307 | 308 | cfg.set_new_allowed(True) 309 | cfg.merge_from_file("example/config_new_allowed_bad.yaml") 310 | assert cfg.KWARGS.Y.f == 4 311 | 312 | 313 | class TestCfgNodeSubclass(unittest.TestCase): 314 | def test_merge_cfg_from_file(self): 315 | with tempfile.NamedTemporaryFile(mode="wt") as f: 316 | cfg = get_cfg(SubCN) 317 | f.write(cfg.dump()) 318 | f.flush() 319 | s = cfg.MODEL.TYPE 320 | cfg.MODEL.TYPE = "dummy" 321 | assert cfg.MODEL.TYPE != s 322 | cfg.merge_from_file(f.name) 323 | assert cfg.MODEL.TYPE == s 324 | 325 | def test_merge_cfg_from_list(self): 326 | cfg = get_cfg(SubCN) 327 | opts = ["TRAIN.SCALES", "(100, )", "MODEL.TYPE", "foobar", "NUM_GPUS", 2] 328 | assert len(cfg.TRAIN.SCALES) > 0 329 | assert cfg.TRAIN.SCALES[0] != 100 330 | assert cfg.MODEL.TYPE != "foobar" 331 | assert cfg.NUM_GPUS != 2 332 | cfg.merge_from_list(opts) 333 | assert type(cfg.TRAIN.SCALES) is tuple 334 | assert len(cfg.TRAIN.SCALES) == 1 335 | assert cfg.TRAIN.SCALES[0] == 100 336 | assert cfg.MODEL.TYPE == "foobar" 337 | assert cfg.NUM_GPUS == 2 338 | 339 | def test_merge_cfg_from_cfg(self): 340 | cfg = get_cfg(SubCN) 341 | cfg2 = get_cfg(SubCN) 342 | s = "dummy0" 343 | cfg2.MODEL.TYPE = s 344 | cfg.merge_from_other_cfg(cfg2) 345 | assert cfg.MODEL.TYPE == s 346 | 347 | # Test: merge from yaml 348 | s = "dummy1" 349 | cfg2 = SubCN.load_cfg(cfg.dump()) 350 | cfg2.MODEL.TYPE = s 351 | cfg.merge_from_other_cfg(cfg2) 352 | assert cfg.MODEL.TYPE == s 353 | 354 | 355 | if __name__ == "__main__": 356 | logging.basicConfig() 357 | yacs_logger = logging.getLogger("yacs.config") 358 | yacs_logger.setLevel(logging.DEBUG) 359 | unittest.main() 360 | --------------------------------------------------------------------------------