├── .github └── workflows │ ├── apidocs.yml │ └── test.yml ├── .gitignore ├── CONTRIBUTING.rst ├── LICENSE ├── MANIFEST.in ├── README.rst ├── apidocs.sh ├── configargparse.py ├── setup.py ├── tests ├── __init__.py └── test_configargparse.py └── tox.ini /.github/workflows/apidocs.yml: -------------------------------------------------------------------------------- 1 | name: apidocs 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | tags: 7 | - '*' 8 | 9 | jobs: 10 | deploy: 11 | runs-on: macos-latest 12 | 13 | steps: 14 | - uses: actions/checkout@master 15 | - name: Set up Python 3.8 16 | uses: actions/setup-python@v2 17 | with: 18 | python-version: 3.8 19 | 20 | - name: Install requirements for documentation generation 21 | run: python -m pip install --upgrade pip setuptools wheel tox 22 | 23 | - name: Generate API documentation with pydoctor 24 | run: tox -e apidocs 25 | 26 | - name: Push API documentation to Github Pages 27 | uses: peaceiris/actions-gh-pages@v3 28 | with: 29 | github_token: ${{ secrets.GITHUB_TOKEN }} 30 | publish_dir: ./apidocs 31 | commit_message: "Generate API documentation" 32 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: unit tests 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | tags: 7 | - '*' 8 | pull_request: 9 | 10 | jobs: 11 | test: 12 | name: ${{ matrix.os }} py${{ matrix.python-version }} ${{ matrix.use-docker && '(docker)' || '' }} 13 | runs-on: ${{ matrix.os }} 14 | 15 | strategy: 16 | matrix: 17 | include: 18 | - name: Legacy Python on Ubuntu 19 | os: ubuntu-latest 20 | python-version: '3.6' 21 | use-docker: true 22 | - name: Legacy Python on Ubuntu 23 | os: ubuntu-latest 24 | python-version: '3.7' 25 | use-docker: true 26 | - name: Legacy Python on Ubuntu 27 | os: ubuntu-latest 28 | python-version: '3.8' 29 | use-docker: true 30 | 31 | os: [ubuntu-latest, windows-latest, macos-latest] 32 | python-version: ['3.9','3.10','3.11','3.12','3.13'] 33 | use-docker: [false] 34 | fail-fast: false 35 | 36 | steps: 37 | - uses: actions/checkout@v4 38 | 39 | - name: Set up Python ${{ matrix.python-version }} 40 | if: ${{ !matrix.use-docker }} 41 | uses: actions/setup-python@v5 42 | with: 43 | python-version: ${{ matrix.python-version }} 44 | 45 | - name: Run tests with Docker 46 | if: ${{ matrix.use-docker }} 47 | run: | 48 | docker run --rm -v ${{ github.workspace }}:/app -w /app python:${{ matrix.python-version }} bash -c " 49 | python -m pip install --upgrade pip setuptools wheel tox 50 | python -m pip install '.[test]' 51 | pytest 52 | " 53 | 54 | - name: Run tests 55 | if: ${{ !matrix.use-docker }} 56 | run: | 57 | python -m pip install --upgrade pip setuptools wheel tox 58 | python -m pip install '.[test]' 59 | pytest 60 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | apidocs 2 | 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Packages 9 | *.egg 10 | *.egg-info 11 | dist 12 | build 13 | eggs 14 | parts 15 | bin 16 | var 17 | sdist 18 | develop-eggs 19 | .installed.cfg 20 | lib 21 | lib64 22 | 23 | # Installer logs 24 | pip-log.txt 25 | 26 | # Unit test / coverage reports 27 | .coverage 28 | .tox 29 | htmlcov 30 | #tox.ini 31 | #nosetests.xml 32 | 33 | # Translations 34 | *.mo 35 | 36 | # Mr Developer 37 | .mr.developer.cfg 38 | .project 39 | .pydevproject 40 | 41 | # Complexity 42 | output/*.html 43 | output/*/index.html 44 | 45 | # Sphinx 46 | docs/_build 47 | 48 | # IDEs 49 | .idea 50 | 51 | .eggs/ 52 | -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | Contributing to ConfigArgParse 2 | ------------------------------ 3 | 4 | What you can do 5 | ~~~~~~~~~~~~~~~ 6 | 7 | If you like the project and think you could help with making it better, there are many ways you can do it: 8 | 9 | - Create an issue to report a bug or suggest a new feature 10 | - Triage old issues that need a refresh 11 | - Implement fixes for existing issues 12 | - Help with improving the documentation 13 | - Spread the word about the project to your colleagues, friends, blogs or any other channels 14 | - Any other things you could imagine 15 | 16 | Any contribution would be of great help and we'll highly appreciate it! 17 | If you have any questions, please create a new issue. 18 | 19 | Development process 20 | ~~~~~~~~~~~~~~~~~~~ 21 | 22 | Create a fork of the git repository and checkout a new branch from master branch. 23 | The branch name may start with an associated issue number so that we can easily cross-reference them. 24 | For example, use ``1234-some-brach-name`` as the name of the branch working to fix issue 1234. 25 | Once your changes are ready and are passing the automated tests, open a pull request. 26 | 27 | Don’t forget to sync your fork once in a while to work from the latest revision. 28 | 29 | Pre-commit checks 30 | ~~~~~~~~~~~~~~~~~ 31 | 32 | - Run unit tests: ``pytest``. 33 | 34 | Review process 35 | ~~~~~~~~~~~~~~ 36 | 37 | - Code changes and code added should have tests: untested code is buggy code and should 38 | not be accepted by reviewers. 39 | - All code changes must be reviewed by at least one maintainer who is not an author 40 | of the code being added. 41 | - When a reviewer completes a review, they should always say what the next step(s) should be: 42 | - Ok we can merge the patch as is 43 | - Some optional changes requested 44 | - Some required changes are requested 45 | - An issue should be opened to tackle another problem discovered during the coding process. 46 | - If a substantial change is pushed after a review, a follow-up review should be done. 47 | Small changes and nit-picks do not required follow-up reviews. 48 | 49 | Release process 50 | ~~~~~~~~~~~~~~~ 51 | 52 | The following is a high level overview and might not match the current state of things. 53 | 54 | - Create a branch: name it with release dash the name of the new version, i.e. ``release-1.5.1`` 55 | - On the branch, update the version and release notes. 56 | - Create a PR for that branch, wait for tests to pass and get an approval. 57 | - At this point, the *owner* should checkout the release branch, 58 | created and push a new tag named after the version i.e. ``1.5.1``, 59 | this will trigger the PyPi release process, monitor the process in the GitHub action UI... 60 | - Update the version and append ``.dev0`` to the current 61 | version number. In this way, stable versions only exist for a brief period of time 62 | (if someone tries to do a pip install from the git source, they will get a ``.dev0`` 63 | version instead of a misleading stable version number) 64 | - Wait for tests to pass and merge PR 65 | 66 | 67 | --- 68 | 69 | owner: people with administrative rights on the repo, only they are able to push a new tag. 70 | maintainers: people with push rights, they can merge PR if the requirements are met. -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 bw2 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include README.rst 3 | include tests/* 4 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ConfigArgParse 2 | -------------- 3 | 4 | .. image:: https://img.shields.io/pypi/v/ConfigArgParse.svg?style=flat 5 | :alt: PyPI version 6 | :target: https://pypi.python.org/pypi/ConfigArgParse 7 | 8 | .. image:: https://img.shields.io/pypi/pyversions/ConfigArgParse.svg 9 | :alt: Supported Python versions 10 | :target: https://pypi.python.org/pypi/ConfigArgParse 11 | 12 | .. image:: https://static.pepy.tech/badge/configargparse/week 13 | :alt: Downloads per week 14 | :target: https://pepy.tech/project/configargparse 15 | 16 | .. image:: https://img.shields.io/badge/-API_Documentation-blue 17 | :alt: API Documentation 18 | :target: https://bw2.github.io/ConfigArgParse/ 19 | 20 | 21 | Overview 22 | ~~~~~~~~ 23 | 24 | Applications with more than a handful of user-settable options are best 25 | configured through a combination of command line args, config files, 26 | hard-coded defaults, and in some cases, environment variables. 27 | 28 | Python's command line parsing modules such as argparse have very limited 29 | support for config files and environment variables, so this module 30 | extends argparse to add these features. 31 | 32 | Available on PyPI: http://pypi.python.org/pypi/ConfigArgParse 33 | 34 | 35 | Features 36 | ~~~~~~~~ 37 | 38 | - command-line, config file, env var, and default settings can now be 39 | defined, documented, and parsed in one go using a single API (if a 40 | value is specified in more than one way then: command line > 41 | environment variables > config file values > defaults) 42 | - config files can have .ini or .yaml style syntax (eg. key=value or 43 | key: value) 44 | - user can provide a config file via a normal-looking command line arg 45 | (eg. -c path/to/config.txt) rather than the argparse-style @config.txt 46 | - one or more default config file paths can be specified 47 | (eg. ['/etc/bla.conf', '~/.my_config'] ) 48 | - all argparse functionality is fully supported, so this module can 49 | serve as a drop-in replacement (verified by argparse unittests). 50 | - env vars and config file keys & syntax are automatically documented 51 | in the -h help message 52 | - new method :code:`print_values()` can report keys & values and where 53 | they were set (eg. command line, env var, config file, or default). 54 | - lite-weight (no 3rd-party library dependencies except (optionally) PyYAML) 55 | - extensible (:code:`ConfigFileParser` can be subclassed to define a new 56 | config file format) 57 | - unittested by running the unittests that came with argparse but on 58 | configargparse, and using tox to test with Python 3.5+ 59 | 60 | Example 61 | ~~~~~~~ 62 | 63 | *config_test.py*: 64 | 65 | Script that defines 4 options and a positional arg and then parses and prints the values. Also, 66 | it prints out the help message as well as the string produced by :code:`format_values()` to show 67 | what they look like. 68 | 69 | .. code:: py 70 | 71 | import configargparse 72 | 73 | p = configargparse.ArgParser(default_config_files=['/etc/app/conf.d/*.conf', '~/.my_settings']) 74 | p.add('-c', '--my-config', required=True, is_config_file=True, help='config file path') 75 | p.add('--genome', required=True, help='path to genome file') # this option can be set in a config file because it starts with '--' 76 | p.add('-v', help='verbose', action='store_true') 77 | p.add('-d', '--dbsnp', help='known variants .vcf', env_var='DBSNP_PATH') # this option can be set in a config file because it starts with '--' 78 | p.add('vcf', nargs='+', help='variant file(s)') 79 | 80 | options = p.parse_args() 81 | 82 | print(options) 83 | print("----------") 84 | print(p.format_help()) 85 | print("----------") 86 | print(p.format_values()) # useful for logging where different settings came from 87 | 88 | 89 | *config.txt:* 90 | 91 | Since the script above set the config file as required=True, lets create a config file to give it: 92 | 93 | .. code:: py 94 | 95 | # settings for config_test.py 96 | genome = HCMV # cytomegalovirus genome 97 | dbsnp = /data/dbsnp/variants.vcf 98 | 99 | 100 | *command line:* 101 | 102 | Now run the script and pass it the config file: 103 | 104 | .. code:: bash 105 | 106 | DBSNP_PATH=/data/dbsnp/variants_v2.vcf python config_test.py --my-config config.txt f1.vcf f2.vcf 107 | 108 | *output:* 109 | 110 | Here is the result: 111 | 112 | .. code:: bash 113 | 114 | Namespace(dbsnp='/data/dbsnp/variants_v2.vcf', genome='HCMV', my_config='config.txt', v=False, vcf=['f1.vcf', 'f2.vcf']) 115 | ---------- 116 | usage: config_test.py [-h] -c MY_CONFIG --genome GENOME [-v] [-d DBSNP] 117 | vcf [vcf ...] 118 | 119 | Args that start with '--' (eg. --genome) can also be set in a config file 120 | (/etc/app/conf.d/*.conf or ~/.my_settings or specified via -c). Config file 121 | syntax allows: key=value, flag=true, stuff=[a,b,c] (for details, see syntax at 122 | https://goo.gl/R74nmi). If an arg is specified in more than one place, then 123 | commandline values override environment variables which override config file 124 | values which override defaults. 125 | 126 | positional arguments: 127 | vcf variant file(s) 128 | 129 | optional arguments: 130 | -h, --help show this help message and exit 131 | -c MY_CONFIG, --my-config MY_CONFIG 132 | config file path 133 | --genome GENOME path to genome file 134 | -v verbose 135 | -d DBSNP, --dbsnp DBSNP 136 | known variants .vcf [env var: DBSNP_PATH] 137 | 138 | ---------- 139 | Command Line Args: --my-config config.txt f1.vcf f2.vcf 140 | Environment Variables: 141 | DBSNP_PATH: /data/dbsnp/variants_v2.vcf 142 | Config File (config.txt): 143 | genome: HCMV 144 | 145 | Special Values 146 | ~~~~~~~~~~~~~~ 147 | 148 | Under the hood, configargparse handles environment variables and config file 149 | values by converting them to their corresponding command line arg. For 150 | example, "key = value" will be processed as if "--key value" was specified 151 | on the command line. 152 | 153 | Also, the following special values (whether in a config file or an environment 154 | variable) are handled in a special way to support booleans and lists: 155 | 156 | - :code:`key = true` is handled as if "--key" was specified on the command line. 157 | In your python code this key must be defined as a boolean flag 158 | (eg. action="store_true" or similar). 159 | 160 | - :code:`key = [value1, value2, ...]` is handled as if "--key value1 --key value2" 161 | etc. was specified on the command line. In your python code this key must 162 | be defined as a list (eg. action="append"). 163 | 164 | Config File Syntax 165 | ~~~~~~~~~~~~~~~~~~ 166 | 167 | Only command line args that have a long version (eg. one that starts with '--') 168 | can be set in a config file. For example, "--color" can be set by putting 169 | "color=green" in a config file. The config file syntax depends on the constructor 170 | arg: :code:`config_file_parser_class` which can be set to one of the provided 171 | classes: :code:`DefaultConfigFileParser`, :code:`YAMLConfigFileParser`, 172 | :code:`ConfigparserConfigFileParser` or to your own subclass of the 173 | :code:`ConfigFileParser` abstract class. 174 | 175 | *DefaultConfigFileParser* - the full range of valid syntax is: 176 | 177 | .. code:: yaml 178 | 179 | # this is a comment 180 | ; this is also a comment (.ini style) 181 | --- # lines that start with --- are ignored (yaml style) 182 | ------------------- 183 | [section] # .ini-style section names are treated as comments 184 | 185 | # how to specify a key-value pair (all of these are equivalent): 186 | name value # key is case sensitive: "Name" isn't "name" 187 | name = value # (.ini style) (white space is ignored, so name = value same as name=value) 188 | name: value # (yaml style) 189 | --name value # (argparse style) 190 | 191 | # how to set a flag arg (eg. arg which has action="store_true") 192 | --name 193 | name 194 | name = True # "True" and "true" are the same 195 | 196 | # how to specify a list arg (eg. arg which has action="append") 197 | fruit = [apple, orange, lemon] 198 | indexes = [1, 12, 35 , 40] 199 | 200 | 201 | *YAMLConfigFileParser* - allows a subset of YAML syntax (http://goo.gl/VgT2DU) 202 | 203 | .. code:: yaml 204 | 205 | # a comment 206 | name1: value 207 | name2: true # "True" and "true" are the same 208 | 209 | fruit: [apple, orange, lemon] 210 | indexes: [1, 12, 35, 40] 211 | colors: 212 | - green 213 | - red 214 | - blue 215 | 216 | *ConfigparserConfigFileParser* - allows a subset of python's configparser 217 | module syntax (https://docs.python.org/3.7/library/configparser.html). In 218 | particular the following configparser options are set: 219 | 220 | .. code:: py 221 | 222 | config = configparser.ArgParser( 223 | delimiters=("=",":"), 224 | allow_no_value=False, 225 | comment_prefixes=("#",";"), 226 | inline_comment_prefixes=("#",";"), 227 | strict=True, 228 | empty_lines_in_values=False, 229 | ) 230 | 231 | Once configparser parses the config file all section names are removed, thus all 232 | keys must have unique names regardless of which INI section they are defined 233 | under. Also, any keys which have python list syntax are converted to lists by 234 | evaluating them as python code using ast.literal_eval 235 | (https://docs.python.org/3/library/ast.html#ast.literal_eval). To facilitate 236 | this all multi-line values are converted to single-line values. Thus multi-line 237 | string values will have all new-lines converted to spaces. Note, since key-value 238 | pairs that have python dictionary syntax are saved as single-line strings, even 239 | if formatted across multiple lines in the config file, dictionaries can be read 240 | in and converted to valid python dictionaries with PyYAML's safe_load. Example 241 | given below: 242 | 243 | .. code:: py 244 | 245 | # inside your config file (e.g. config.ini) 246 | [section1] # INI sections treated as comments 247 | system1_settings: { # start of multi-line dictionary 248 | 'a':True, 249 | 'b':[2, 4, 8, 16], 250 | 'c':{'start':0, 'stop':1000}, 251 | 'd':'experiment 32 testing simulation with parameter a on' 252 | } # end of multi-line dictionary value 253 | 254 | ....... 255 | 256 | # in your configargparse setup 257 | import configargparse 258 | import yaml 259 | 260 | parser = configargparse.ArgParser( 261 | config_file_parser_class=configargparse.ConfigparserConfigFileParser 262 | ) 263 | parser.add_argument('--system1_settings', type=yaml.safe_load) 264 | 265 | args = parser.parse_args() # now args.system1 is a valid python dict 266 | 267 | *IniConfigParser* - INI parser with support for sections. 268 | 269 | This parser somewhat ressembles ``ConfigparserConfigFileParser``. It uses configparser and apply the same kind of processing to 270 | values written with python list syntax. 271 | 272 | With the following additions: 273 | - Must be created with argument to bind the parser to a list of sections. 274 | - Does not convert multiline strings to single line. 275 | - Optional support for converting multiline strings to list (if ``split_ml_text_to_list=True``). 276 | - Optional support for quoting strings in config file 277 | (useful when text must not be converted to list or when text 278 | should contain trailing whitespaces). 279 | 280 | This config parser can be used to integrate with ``setup.cfg`` files. 281 | 282 | Example:: 283 | 284 | # this is a comment 285 | ; also a comment 286 | [my_super_tool] 287 | # how to specify a key-value pair 288 | format-string: restructuredtext 289 | # white space are ignored, so name = value same as name=value 290 | # this is why you can quote strings 291 | quoted-string = '\thello\tmom... ' 292 | # how to set an arg which has action="store_true" 293 | warnings-as-errors = true 294 | # how to set an arg which has action="count" or type=int 295 | verbosity = 1 296 | # how to specify a list arg (eg. arg which has action="append") 297 | repeatable-option = ["https://docs.python.org/3/objects.inv", 298 | "https://twistedmatrix.com/documents/current/api/objects.inv"] 299 | # how to specify a multiline text: 300 | multi-line-text = 301 | Lorem ipsum dolor sit amet, consectetur adipiscing elit. 302 | Vivamus tortor odio, dignissim non ornare non, laoreet quis nunc. 303 | Maecenas quis dapibus leo, a pellentesque leo. 304 | 305 | If you use ``IniConfigParser(sections, split_ml_text_to_list=True)``:: 306 | 307 | # the same rules are applicable with the following changes: 308 | [my-software] 309 | # how to specify a list arg (eg. arg which has action="append") 310 | repeatable-option = # Just enter one value per line (the list literal format can also be used) 311 | https://docs.python.org/3/objects.inv 312 | https://twistedmatrix.com/documents/current/api/objects.inv 313 | # how to specify a multiline text (you have to quote it): 314 | multi-line-text = ''' 315 | Lorem ipsum dolor sit amet, consectetur adipiscing elit. 316 | Vivamus tortor odio, dignissim non ornare non, laoreet quis nunc. 317 | Maecenas quis dapibus leo, a pellentesque leo. 318 | ''' 319 | 320 | Usage: 321 | 322 | .. code:: py 323 | 324 | import configargparse 325 | parser = configargparse.ArgParser( 326 | default_config_files=['setup.cfg', 'my_super_tool.ini'], 327 | config_file_parser_class=configargparse.IniConfigParser(['tool:my_super_tool', 'my_super_tool']), 328 | ) 329 | ... 330 | 331 | *TomlConfigParser* - TOML parser with support for sections. 332 | 333 | `TOML `_ parser. This config parser can be used to integrate with ``pyproject.toml`` files. 334 | 335 | Example:: 336 | 337 | # this is a comment 338 | [tool.my-software] # TOML section table. 339 | # how to specify a key-value pair 340 | format-string = "restructuredtext" # strings must be quoted 341 | # how to set an arg which has action="store_true" 342 | warnings-as-errors = true 343 | # how to set an arg which has action="count" or type=int 344 | verbosity = 1 345 | # how to specify a list arg (eg. arg which has action="append") 346 | repeatable-option = ["https://docs.python.org/3/objects.inv", 347 | "https://twistedmatrix.com/documents/current/api/objects.inv"] 348 | # how to specify a multiline text: 349 | multi-line-text = ''' 350 | Lorem ipsum dolor sit amet, consectetur adipiscing elit. 351 | Vivamus tortor odio, dignissim non ornare non, laoreet quis nunc. 352 | Maecenas quis dapibus leo, a pellentesque leo. 353 | ''' 354 | 355 | Usage: 356 | 357 | .. code:: py 358 | 359 | import configargparse 360 | parser = configargparse.ArgParser( 361 | default_config_files=['pyproject.toml', 'my_super_tool.toml'], 362 | config_file_parser_class=configargparse.TomlConfigParser(['tool.my_super_tool']), 363 | ) 364 | ... 365 | 366 | *CompositeConfigParser* - Create a config parser to understand multiple formats. 367 | 368 | This parser will successively try to parse the file with each parser, until it succeeds, 369 | else fail showing all encountered error messages. 370 | 371 | The following code will make configargparse understand both TOML and INI formats. 372 | Making it easy to integrate in both ``pyproject.toml`` and ``setup.cfg``. 373 | 374 | .. code:: py 375 | 376 | import configargparse 377 | my_tool_sections = ['tool.my_super_tool', 'tool:my_super_tool', 'my_super_tool'] 378 | # pyproject.toml like section, setup.cfg like section, custom section 379 | parser = configargparse.ArgParser( 380 | default_config_files=['setup.cfg', 'my_super_tool.ini'], 381 | config_file_parser_class=configargparse.CompositeConfigParser( 382 | [configargparse.TomlConfigParser(my_tool_sections), 383 | configargparse.IniConfigParser(my_tool_sections, split_ml_text_to_list=True)] 384 | ), 385 | ) 386 | ... 387 | 388 | Note that it's required to put the TOML parser first because the INI syntax basically would accept anything whereas TOML. 389 | 390 | ArgParser Singletons 391 | ~~~~~~~~~~~~~~~~~~~~~~~~~ 392 | 393 | To make it easier to configure different modules in an application, 394 | configargparse provides globally-available ArgumentParser instances 395 | via configargparse.get_argument_parser('name') (similar to 396 | logging.getLogger('name')). 397 | 398 | Here is an example of an application with a utils module that also 399 | defines and retrieves its own command-line args. 400 | 401 | *main.py* 402 | 403 | .. code:: py 404 | 405 | import configargparse 406 | import utils 407 | 408 | p = configargparse.get_argument_parser() 409 | p.add_argument("-x", help="Main module setting") 410 | p.add_argument("--m-setting", help="Main module setting") 411 | options = p.parse_known_args() # using p.parse_args() here may raise errors. 412 | 413 | *utils.py* 414 | 415 | .. code:: py 416 | 417 | import configargparse 418 | p = configargparse.get_argument_parser() 419 | p.add_argument("--utils-setting", help="Config-file-settable option for utils") 420 | 421 | if __name__ == "__main__": 422 | options = p.parse_known_args() 423 | 424 | Help Formatters 425 | ~~~~~~~~~~~~~~~ 426 | 427 | :code:`ArgumentDefaultsRawHelpFormatter` is a new HelpFormatter that both adds 428 | default values AND disables line-wrapping. It can be passed to the constructor: 429 | :code:`ArgParser(.., formatter_class=ArgumentDefaultsRawHelpFormatter)` 430 | 431 | 432 | Aliases 433 | ~~~~~~~ 434 | 435 | The configargparse.ArgumentParser API inherits its class and method 436 | names from argparse and also provides the following shorter names for 437 | convenience: 438 | 439 | - p = configargparse.get_arg_parser() # get global singleton instance 440 | - p = configargparse.get_parser() 441 | - p = configargparse.ArgParser() # create a new instance 442 | - p = configargparse.Parser() 443 | - p.add_arg(..) 444 | - p.add(..) 445 | - options = p.parse(..) 446 | 447 | HelpFormatters: 448 | 449 | - RawFormatter = RawDescriptionHelpFormatter 450 | - DefaultsFormatter = ArgumentDefaultsHelpFormatter 451 | - DefaultsRawFormatter = ArgumentDefaultsRawHelpFormatter 452 | 453 | API Documentation 454 | ~~~~~~~~~~~~~~~~~ 455 | 456 | You can review the generated API Documentation for the ``configargparse`` module: `HERE `_ 457 | 458 | Design Notes 459 | ~~~~~~~~~~~~ 460 | 461 | Unit tests: 462 | 463 | tests/test_configargparse.py contains custom unittests for features 464 | specific to this module (such as config file and env-var support), as 465 | well as a hook to load and run argparse unittests (see the built-in 466 | test.test_argparse module) but on configargparse in place of argparse. 467 | This ensures that configargparse will work as a drop in replacement for 468 | argparse in all usecases. 469 | 470 | Previously existing modules (PyPI search keywords: config argparse): 471 | 472 | - argparse (built-in module Python v2.7+) 473 | 474 | - Good: 475 | 476 | - fully featured command line parsing 477 | - can read args from files using an easy to understand mechanism 478 | 479 | - Bad: 480 | 481 | - syntax for specifying config file path is unusual (eg. 482 | @file.txt)and not described in the user help message. 483 | - default config file syntax doesn't support comments and is 484 | unintuitive (eg. --namevalue) 485 | - no support for environment variables 486 | 487 | - ConfArgParse v1.0.15 488 | (https://pypi.python.org/pypi/ConfArgParse) 489 | 490 | - Good: 491 | 492 | - extends argparse with support for config files parsed by 493 | ConfigParser 494 | - clear documentation in README 495 | 496 | - Bad: 497 | 498 | - config file values are processed using 499 | ArgumentParser.set_defaults(..) which means "required" and 500 | "choices" are not handled as expected. For example, if you 501 | specify a required value in a config file, you still have to 502 | specify it again on the command line. 503 | - doesn't work with Python 3 yet 504 | - no unit tests, code not well documented 505 | 506 | - appsettings v0.5 (https://pypi.python.org/pypi/appsettings) 507 | 508 | - Good: 509 | 510 | - supports config file (yaml format) and env_var parsing 511 | - supports config-file-only setting for specifying lists and 512 | dicts 513 | 514 | - Bad: 515 | 516 | - passes in config file and env settings via parse_args 517 | namespace param 518 | - tests not finished and don't work with Python 3 (import 519 | StringIO) 520 | 521 | - argparse_config v0.5.1 522 | (https://pypi.python.org/pypi/argparse_config) 523 | 524 | - Good: 525 | 526 | - similar features to ConfArgParse v1.0.15 527 | 528 | - Bad: 529 | 530 | - doesn't work with Python 3 (error during pip install) 531 | 532 | - yconf v0.3.2 - (https://pypi.python.org/pypi/yconf) - features 533 | and interface not that great 534 | - hieropt v0.3 - (https://pypi.python.org/pypi/hieropt) - doesn't 535 | appear to be maintained, couldn't find documentation 536 | 537 | - configurati v0.2.3 - (https://pypi.python.org/pypi/configurati) 538 | 539 | - Good: 540 | 541 | - JSON, YAML, or Python configuration files 542 | - handles rich data structures such as dictionaries 543 | - can group configuration names into sections (like .ini files) 544 | 545 | - Bad: 546 | 547 | - doesn't work with Python 3 548 | - 2+ years since last release to PyPI 549 | - apparently unmaintained 550 | 551 | 552 | Design choices: 553 | 554 | 1. all options must be settable via command line. Having options that 555 | can only be set using config files or env. vars adds complexity to 556 | the API, and is not a useful enough feature since the developer can 557 | split up options into sections and call a section "config file keys", 558 | with command line args that are just "--" plus the config key. 559 | 2. config file and env. var settings should be processed by appending 560 | them to the command line (another benefit of #1). This is an 561 | easy-to-implement solution and implicitly takes care of checking that 562 | all "required" args are provided, etc., plus the behavior should be 563 | easy for users to understand. 564 | 3. configargparse shouldn't override argparse's 565 | convert_arg_line_to_args method so that all argparse unit tests 566 | can be run on configargparse. 567 | 4. in terms of what to allow for config file keys, the "dest" value of 568 | an option can't serve as a valid config key because many options can 569 | have the same dest. Instead, since multiple options can't use the 570 | same long arg (eg. "--long-arg-x"), let the config key be either 571 | "--long-arg-x" or "long-arg-x". This means the developer can allow 572 | only a subset of the command-line args to be specified via config 573 | file (eg. short args like -x would be excluded). Also, that way 574 | config keys are automatically documented whenever the command line 575 | args are documented in the help message. 576 | 5. don't force users to put config file settings in the right .ini [sections]. 577 | This doesn't have a clear benefit since all options are command-line settable, 578 | and so have a globally unique key anyway. 579 | Enforcing sections just makes things harder for the user and adds complexity to the implementation. 580 | NOTE: This design choice was preventing configargparse from integrating with common Python project 581 | config files like setup.cfg or pyproject.toml, 582 | so additional parser classes were added that parse only a subset of the values defined in INI or 583 | TOML config files. 584 | 6. if necessary, config-file-only args can be added later by 585 | implementing a separate add method and using the namespace arg as in 586 | appsettings_v0.5 587 | 588 | Relevant sites: 589 | 590 | - http://stackoverflow.com/questions/6133517/parse-config-file-environment-and-command-line-arguments-to-get-a-single-coll 591 | - http://tricksntweaks.blogspot.com/2013_05_01_archive.html 592 | - http://www.youtube.com/watch?v=vvCwqHgZJc8#t=35 593 | 594 | 595 | 596 | Versioning 597 | ~~~~~~~~~~ 598 | 599 | This software follows `Semantic Versioning`_ 600 | 601 | .. _Semantic Versioning: http://semver.org/ 602 | -------------------------------------------------------------------------------- /apidocs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # This bash script builds the API documentation for ConfigArgParse. 3 | 4 | # Resolve source directory path. From https://stackoverflow.com/questions/59895/how-to-get-the-source-directory-of-a-bash-script-from-within-the-script-itself/246128#246128 5 | SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" 6 | cd $SCRIPT_DIR 7 | 8 | # Stop if errors 9 | set -euo pipefail 10 | IFS=$'\n\t,' 11 | 12 | # Figure the project version 13 | project_version="$(python3 setup.py -V)" 14 | 15 | # Figure commit ref 16 | git_sha="$(git rev-parse HEAD)" 17 | if ! git describe --exact-match --tags > /dev/null 2>&1 ; then 18 | is_tag=false 19 | else 20 | git_sha="$(git describe --exact-match --tags)" 21 | is_tag=true 22 | fi 23 | 24 | # Init output folder 25 | docs_folder="./apidocs/" 26 | rm -rf "${docs_folder}" 27 | mkdir -p "${docs_folder}" 28 | 29 | # We generate the docs for the argparse module too, such that we can document 30 | # the methods inherited from argparse.ArgumentParser, not only the methods that configargparse overrides. 31 | # And it help to have a better vision of the whole thing also. 32 | curl https://raw.githubusercontent.com/python/cpython/3.9/Lib/argparse.py > ./argparse.py 33 | echo "__docformat__ = 'restructuredtext'" >> ./argparse.py 34 | # Delete the file when the script exits 35 | trap "rm -f ./argparse.py" EXIT 36 | 37 | pydoctor \ 38 | --project-name="ConfigArgParse ${project_version}" \ 39 | --project-url="https://github.com/bw2/ConfigArgParse" \ 40 | --html-viewsource-base="https://github.com/bw2/ConfigArgParse/tree/${git_sha}" \ 41 | --intersphinx=https://docs.python.org/3/objects.inv \ 42 | --make-html \ 43 | --quiet \ 44 | --project-base-dir=.\ 45 | --docformat=google \ 46 | --html-output="${docs_folder}" \ 47 | ./argparse.py ./configargparse.py || true 48 | 49 | echo "API docs generated in ${docs_folder}" -------------------------------------------------------------------------------- /configargparse.py: -------------------------------------------------------------------------------- 1 | """ 2 | A drop-in replacement for `argparse` that allows options to also be set via config files and/or environment variables. 3 | 4 | :see: `configargparse.ArgumentParser`, `configargparse.add_argument` 5 | """ 6 | 7 | import argparse 8 | import ast 9 | import csv 10 | import functools 11 | import json 12 | import glob 13 | import os 14 | import re 15 | import sys 16 | import types 17 | from collections import OrderedDict 18 | import textwrap 19 | from io import StringIO 20 | 21 | 22 | ACTION_TYPES_THAT_DONT_NEED_A_VALUE = [ 23 | argparse._StoreTrueAction, 24 | argparse._StoreFalseAction, 25 | argparse._CountAction, 26 | argparse._StoreConstAction, 27 | argparse._AppendConstAction, 28 | ] 29 | 30 | if sys.version_info >= (3, 9): 31 | ACTION_TYPES_THAT_DONT_NEED_A_VALUE.append(argparse.BooleanOptionalAction) 32 | is_boolean_optional_action = lambda action: isinstance( 33 | action, argparse.BooleanOptionalAction 34 | ) 35 | else: 36 | is_boolean_optional_action = lambda action: False 37 | 38 | ACTION_TYPES_THAT_DONT_NEED_A_VALUE = tuple(ACTION_TYPES_THAT_DONT_NEED_A_VALUE) 39 | 40 | 41 | # global ArgumentParser instances 42 | _parsers = {} 43 | 44 | 45 | def init_argument_parser(name=None, **kwargs): 46 | """Creates a global ArgumentParser instance with the given name, 47 | passing any args other than "name" to the ArgumentParser constructor. 48 | This instance can then be retrieved using get_argument_parser(..) 49 | """ 50 | 51 | if name is None: 52 | name = "default" 53 | 54 | if name in _parsers: 55 | raise ValueError( 56 | ( 57 | "kwargs besides 'name' can only be passed in the" 58 | " first time. '%s' ArgumentParser already exists: %s" 59 | ) 60 | % (name, _parsers[name]) 61 | ) 62 | 63 | kwargs.setdefault("formatter_class", argparse.ArgumentDefaultsHelpFormatter) 64 | kwargs.setdefault("conflict_handler", "resolve") 65 | _parsers[name] = ArgumentParser(**kwargs) 66 | 67 | 68 | def get_argument_parser(name=None, **kwargs): 69 | """Returns the global ArgumentParser instance with the given name. The 1st 70 | time this function is called, a new ArgumentParser instance will be created 71 | for the given name, and any args other than "name" will be passed on to the 72 | ArgumentParser constructor. 73 | """ 74 | if name is None: 75 | name = "default" 76 | 77 | if len(kwargs) > 0 or name not in _parsers: 78 | init_argument_parser(name, **kwargs) 79 | 80 | return _parsers[name] 81 | 82 | 83 | class ArgumentDefaultsRawHelpFormatter( 84 | argparse.ArgumentDefaultsHelpFormatter, 85 | argparse.RawTextHelpFormatter, 86 | argparse.RawDescriptionHelpFormatter, 87 | ): 88 | """HelpFormatter that adds default values AND doesn't do line-wrapping""" 89 | 90 | pass 91 | 92 | 93 | class ConfigFileParser(object): 94 | """This abstract class can be extended to add support for new config file 95 | formats""" 96 | 97 | def get_syntax_description(self): 98 | """Returns a string describing the config file syntax.""" 99 | raise NotImplementedError("get_syntax_description(..) not implemented") 100 | 101 | def parse(self, stream): 102 | """Parses the keys and values from a config file. 103 | 104 | NOTE: For keys that were specified to configargparse as 105 | action="store_true" or "store_false", the config file value must be 106 | one of: "yes", "no", "on", "off", "true", "false". Otherwise an error will be raised. 107 | 108 | Args: 109 | stream (IO): A config file input stream (such as an open file object). 110 | 111 | Returns: 112 | OrderedDict: Items where the keys are strings and the 113 | values are either strings or lists (eg. to support config file 114 | formats like YAML which allow lists). 115 | """ 116 | raise NotImplementedError("parse(..) not implemented") 117 | 118 | def serialize(self, items): 119 | """Does the inverse of config parsing by taking parsed values and 120 | converting them back to a string representing config file contents. 121 | 122 | Args: 123 | items: an OrderedDict of items to be converted to the config file 124 | format. Keys should be strings, and values should be either strings 125 | or lists. 126 | 127 | Returns: 128 | Contents of config file as a string 129 | """ 130 | raise NotImplementedError("serialize(..) not implemented") 131 | 132 | 133 | class ConfigFileParserException(Exception): 134 | """Raised when config file parsing failed.""" 135 | 136 | 137 | class DefaultConfigFileParser(ConfigFileParser): 138 | """ 139 | Based on a simplified subset of INI and YAML formats. Here is the 140 | supported syntax 141 | 142 | .. code:: 143 | 144 | # this is a comment 145 | ; this is also a comment (.ini style) 146 | --- # lines that start with --- are ignored (yaml style) 147 | ------------------- 148 | [section] # .ini-style section names are treated as comments 149 | 150 | # how to specify a key-value pair (all of these are equivalent): 151 | name value # key is case sensitive: "Name" isn't "name" 152 | name = value # (.ini style) (white space is ignored, so name = value same as name=value) 153 | name: value # (yaml style) 154 | --name value # (argparse style) 155 | 156 | # how to set a flag arg (eg. arg which has action="store_true") 157 | --name 158 | name 159 | name = True # "True" and "true" are the same 160 | 161 | # how to specify a list arg (eg. arg which has action="append") 162 | fruit = [apple, orange, lemon] 163 | indexes = [1, 12, 35 , 40] 164 | 165 | """ 166 | 167 | def get_syntax_description(self): 168 | msg = ( 169 | "Config file syntax allows: key=value, flag=true, stuff=[a,b,c] " 170 | "(for details, see syntax at https://goo.gl/R74nmi)." 171 | ) 172 | return msg 173 | 174 | def parse(self, stream): 175 | # see ConfigFileParser.parse docstring 176 | 177 | items = OrderedDict() 178 | for i, line in enumerate(stream): 179 | line = line.strip() 180 | if not line or line[0] in ["#", ";", "["] or line.startswith("---"): 181 | continue 182 | 183 | match = re.match( 184 | r"^(?P[^:=;#\s]+)\s*" 185 | r'(?:(?P[:=\s])\s*([\'"]?)(?P.+?)?\3)?' 186 | r"\s*(?:\s[;#]\s*(?P.*?)\s*)?$", 187 | line, 188 | ) 189 | if match: 190 | key = match.group("key") 191 | equal = match.group("equal") 192 | value = match.group("value") 193 | comment = match.group("comment") 194 | if value is None and equal is not None and equal != " ": 195 | value = "" 196 | elif value is None: 197 | value = "true" 198 | if value.startswith("[") and value.endswith("]"): 199 | # handle special case of k=[1,2,3] or other json-like syntax 200 | try: 201 | value = json.loads(value) 202 | except Exception as e: 203 | # for backward compatibility with legacy format (eg. where config value is [a, b, c] instead of proper json ["a", "b", "c"] 204 | value = [elem.strip() for elem in value[1:-1].split(",")] 205 | if comment: 206 | comment = comment.strip()[1:].strip() 207 | items[key] = value 208 | else: 209 | raise ConfigFileParserException( 210 | "Unexpected line {} in {}: {}".format( 211 | i, getattr(stream, "name", "stream"), line 212 | ) 213 | ) 214 | return items 215 | 216 | def serialize(self, items): 217 | # see ConfigFileParser.serialize docstring 218 | r = StringIO() 219 | for key, value in items.items(): 220 | if isinstance(value, list): 221 | # handle special case of lists 222 | value = "[" + ", ".join(map(str, value)) + "]" 223 | r.write("{} = {}\n".format(key, value)) 224 | return r.getvalue() 225 | 226 | 227 | class ConfigparserConfigFileParser(ConfigFileParser): 228 | """parses INI files using pythons configparser.""" 229 | 230 | def get_syntax_description(self): 231 | msg = """Uses configparser module to parse an INI file which allows multi-line 232 | values. 233 | 234 | Allowed syntax is that for a ConfigParser with the following options: 235 | 236 | allow_no_value = False, 237 | inline_comment_prefixes = ("#",) 238 | strict = True 239 | empty_lines_in_values = False 240 | 241 | See https://docs.python.org/3/library/configparser.html for details. 242 | 243 | Note: INI file sections names are still treated as comments. 244 | """ 245 | return msg 246 | 247 | def parse(self, stream): 248 | # see ConfigFileParser.parse docstring 249 | import configparser 250 | from ast import literal_eval 251 | 252 | # parse with configparser to allow multi-line values 253 | config = configparser.ConfigParser( 254 | delimiters=("=", ":"), 255 | allow_no_value=False, 256 | comment_prefixes=("#", ";"), 257 | inline_comment_prefixes=("#", ";"), 258 | strict=True, 259 | empty_lines_in_values=False, 260 | ) 261 | try: 262 | config.read_string(stream.read()) 263 | except Exception as e: 264 | raise ConfigFileParserException("Couldn't parse config file: %s" % e) 265 | 266 | # convert to dict and remove INI section names 267 | result = OrderedDict() 268 | for section in config.sections(): 269 | for k, v in config[section].items(): 270 | multiLine2SingleLine = v.replace("\n", " ").replace("\r", " ") 271 | # handle special case for lists 272 | if "[" in multiLine2SingleLine and "]" in multiLine2SingleLine: 273 | # ensure not a dict with a list value 274 | prelist_string = multiLine2SingleLine.split("[")[0] 275 | if "{" not in prelist_string: 276 | result[k] = literal_eval(multiLine2SingleLine) 277 | else: 278 | result[k] = multiLine2SingleLine 279 | else: 280 | result[k] = multiLine2SingleLine 281 | return result 282 | 283 | def serialize(self, items): 284 | # see ConfigFileParser.serialize docstring 285 | import configparser 286 | import io 287 | 288 | config = configparser.ConfigParser( 289 | allow_no_value=False, 290 | inline_comment_prefixes=("#",), 291 | strict=True, 292 | empty_lines_in_values=False, 293 | ) 294 | items = {"DEFAULT": items} 295 | config.read_dict(items) 296 | stream = io.StringIO() 297 | config.write(stream) 298 | stream.seek(0) 299 | return stream.read() 300 | 301 | 302 | class YAMLConfigFileParser(ConfigFileParser): 303 | """Parses YAML config files. Depends on the PyYAML module. 304 | https://pypi.python.org/pypi/PyYAML 305 | """ 306 | 307 | def get_syntax_description(self): 308 | msg = ( 309 | "The config file uses YAML syntax and must represent a YAML " 310 | "'mapping' (for details, see http://learn.getgrav.org/advanced/yaml)." 311 | ) 312 | return msg 313 | 314 | def _load_yaml(self): 315 | """lazy-import PyYAML so that configargparse doesn't have to depend 316 | on it unless this parser is used.""" 317 | try: 318 | import yaml 319 | except ImportError: 320 | raise ConfigFileParserException( 321 | "Could not import yaml. " 322 | "It can be installed by running 'pip install PyYAML'" 323 | ) 324 | 325 | try: 326 | from yaml import CSafeLoader as SafeLoader 327 | from yaml import CDumper as Dumper 328 | except ImportError: 329 | from yaml import SafeLoader 330 | from yaml import Dumper 331 | 332 | return yaml, SafeLoader, Dumper 333 | 334 | def parse(self, stream): 335 | # see ConfigFileParser.parse docstring 336 | yaml, SafeLoader, _ = self._load_yaml() 337 | 338 | try: 339 | parsed_obj = yaml.load(stream, Loader=SafeLoader) 340 | except Exception as e: 341 | raise ConfigFileParserException("Couldn't parse config file: %s" % e) 342 | 343 | if not isinstance(parsed_obj, dict): 344 | raise ConfigFileParserException( 345 | "The config file doesn't appear to " 346 | "contain 'key: value' pairs (aka. a YAML mapping). " 347 | "yaml.load('%s') returned type '%s' instead of 'dict'." 348 | % (getattr(stream, "name", "stream"), type(parsed_obj).__name__) 349 | ) 350 | 351 | result = OrderedDict() 352 | for key, value in parsed_obj.items(): 353 | if isinstance(value, list): 354 | result[key] = value 355 | elif value is None: 356 | pass 357 | else: 358 | result[key] = str(value) 359 | 360 | return result 361 | 362 | def serialize(self, items, default_flow_style=False): 363 | # see ConfigFileParser.serialize docstring 364 | 365 | # lazy-import so there's no dependency on yaml unless this class is used 366 | yaml, _, Dumper = self._load_yaml() 367 | 368 | # it looks like ordering can't be preserved: http://pyyaml.org/ticket/29 369 | items = dict(items) 370 | return yaml.dump(items, default_flow_style=default_flow_style, Dumper=Dumper) 371 | 372 | 373 | """ 374 | Provides `configargparse.ConfigFileParser` classes to parse ``TOML`` and ``INI`` files with **mandatory** support for sections. 375 | Useful to integrate configuration into project files like ``pyproject.toml`` or ``setup.cfg``. 376 | 377 | `TomlConfigParser` usage: 378 | 379 | >>> TomlParser = TomlConfigParser(['tool.my_super_tool']) # Simple TOML parser. 380 | >>> parser = ArgumentParser(..., default_config_files=['./pyproject.toml'], config_file_parser_class=TomlParser) 381 | 382 | `IniConfigParser` works the same way (also it optionaly convert multiline strings to list with argument ``split_ml_text_to_list``). 383 | 384 | `CompositeConfigParser` usage: 385 | 386 | >>> MY_CONFIG_SECTIONS = ['tool.my_super_tool', 'tool:my_super_tool', 'my_super_tool'] 387 | >>> TomlParser = TomlConfigParser(MY_CONFIG_SECTIONS) 388 | >>> IniParser = IniConfigParser(MY_CONFIG_SECTIONS, split_ml_text_to_list=True) 389 | >>> MixedParser = CompositeConfigParser([TomlParser, IniParser]) # This parser supports both TOML and INI formats. 390 | >>> parser = ArgumentParser(..., default_config_files=['./pyproject.toml', 'setup.cfg', 'my_super_tool.ini'], config_file_parser_class=MixedParser) 391 | 392 | """ 393 | 394 | # I did not invented these regex, just put together some stuff from: 395 | # - https://stackoverflow.com/questions/11859442/how-to-match-string-in-quotes-using-regex 396 | # - and https://stackoverflow.com/a/41005190 397 | 398 | _QUOTED_STR_REGEX = re.compile(r"(^\"(?:\\.|[^\"\\])*\"$)|" r"(^\'(?:\\.|[^\'\\])*\'$)") 399 | 400 | _TRIPLE_QUOTED_STR_REGEX = re.compile( 401 | r"(^\"\"\"(\s+)?(([^\"]|\"([^\"]|\"[^\"]))*(\"\"?)?)?(\s+)?(?:\\.|[^\"\\])?\"\"\"$)|" 402 | # Unescaped quotes at the end of a string generates 403 | # "SyntaxError: EOL while scanning string literal", 404 | # so we don't account for those kind of strings as quoted. 405 | r"(^\'\'\'(\s+)?(([^\']|\'([^\']|\'[^\']))*(\'\'?)?)?(\s+)?(?:\\.|[^\'\\])?\'\'\'$)", 406 | flags=re.DOTALL, 407 | ) 408 | 409 | 410 | @functools.lru_cache(maxsize=256, typed=True) 411 | def is_quoted(text, triple=True): 412 | """ 413 | Detect whether a string is a quoted representation. 414 | 415 | :param triple: Also match tripple quoted strings. 416 | """ 417 | return bool(_QUOTED_STR_REGEX.match(text)) or ( 418 | triple and bool(_TRIPLE_QUOTED_STR_REGEX.match(text)) 419 | ) 420 | 421 | 422 | def unquote_str(text, triple=True): 423 | """ 424 | Unquote a maybe quoted string representation. 425 | If the string is not detected as being a quoted representation, it returns the same string as passed. 426 | It supports all kinds of python quotes: ``\"\"\"``, ``'''``, ``"`` and ``'``. 427 | 428 | :param triple: Also unquote tripple quoted strings. 429 | @raises ValueError: If the string is detected as beeing quoted but literal_eval() fails to evaluate it as string. 430 | This would be a bug in the regex. 431 | """ 432 | if is_quoted(text, triple=triple): 433 | try: 434 | s = ast.literal_eval(text) 435 | assert isinstance(s, str) 436 | except Exception as e: 437 | raise ValueError( 438 | f"Error trying to unquote the quoted string: {text}: {e}" 439 | ) from e 440 | return s 441 | return text 442 | 443 | 444 | def parse_toml_section_name(section_name): 445 | """ 446 | Parse a TOML section name to a sequence of strings. 447 | 448 | The following names are all valid: 449 | 450 | .. python:: 451 | 452 | "a.b.c" # this is best practice -> returns ("a", "b", "c") 453 | " d.e.f " # same as [d.e.f] -> returns ("d", "e", "f") 454 | " g . h . i " # same as [g.h.i] -> returns ("g", "h", "i") 455 | ' j . "ʞ" . "l" ' # same as [j."ʞ"."l"], double or simple quotes here are supported. -> returns ("j", "ʞ", "l") 456 | """ 457 | section = [] 458 | for row in csv.reader([section_name], delimiter="."): 459 | for a in row: 460 | section.append(unquote_str(a.strip(), triple=False)) 461 | return tuple(section) 462 | 463 | 464 | def get_toml_section(data, section): 465 | """ 466 | Given some TOML data (as loaded with `toml.load()`), returns the requested section of the data. 467 | Returns ``None`` if the section is not found. 468 | """ 469 | sections = parse_toml_section_name(section) if isinstance(section, str) else section 470 | itemdata = data.get(sections[0]) 471 | if not itemdata: 472 | return None 473 | sections = sections[1:] 474 | if sections: 475 | return get_toml_section(itemdata, sections) 476 | else: 477 | if not isinstance(itemdata, dict): 478 | return None 479 | return itemdata 480 | 481 | 482 | class TomlConfigParser(ConfigFileParser): 483 | """ 484 | Create a TOML parser bounded to the list of provided sections. 485 | 486 | Example:: 487 | # this is a comment 488 | [tool.my-software] # TOML section table. 489 | # how to specify a key-value pair 490 | format-string = "restructuredtext" # strings must be quoted 491 | # how to set an arg which has action="store_true" 492 | warnings-as-errors = true 493 | # how to set an arg which has action="count" or type=int 494 | verbosity = 1 495 | # how to specify a list arg (eg. arg which has action="append") 496 | repeatable-option = ["https://docs.python.org/3/objects.inv", 497 | "https://twistedmatrix.com/documents/current/api/objects.inv"] 498 | # how to specify a multiline text: 499 | multi-line-text = ''' 500 | Lorem ipsum dolor sit amet, consectetur adipiscing elit. 501 | Vivamus tortor odio, dignissim non ornare non, laoreet quis nunc. 502 | Maecenas quis dapibus leo, a pellentesque leo. 503 | ''' 504 | 505 | Note that the config file fragment above is also valid for the `IniConfigParser` class and would be parsed the same manner. 506 | Thought, any valid TOML config file will not be necessarly parsable with `IniConfigParser` (INI files must be rigorously indented whereas TOML files). 507 | 508 | See the `TOML specification <>`_ for details. 509 | """ 510 | 511 | def __init__(self, sections): 512 | """ 513 | :param sections: The section names bounded to the new parser. 514 | """ 515 | super().__init__() 516 | self.sections = sections 517 | 518 | def __call__(self): 519 | return self 520 | 521 | def parse(self, stream): 522 | """Parses the keys and values from a TOML config file.""" 523 | # parse with configparser to allow multi-line values 524 | import toml 525 | 526 | try: 527 | config = toml.load(stream) 528 | except Exception as e: 529 | raise ConfigFileParserException("Couldn't parse TOML file: %s" % e) 530 | 531 | # convert to dict and filter based on section names 532 | result = OrderedDict() 533 | 534 | for section in self.sections: 535 | data = get_toml_section(config, section) 536 | if data: 537 | # Seems a little weird, but anything that is not a list is converted to string, 538 | # It will be converted back to boolean, int or whatever after. 539 | # Because config values are still passed to argparser for computation. 540 | for key, value in data.items(): 541 | if isinstance(value, list): 542 | result[key] = value 543 | elif value is None: 544 | pass 545 | else: 546 | result[key] = str(value) 547 | break 548 | 549 | return result 550 | 551 | def get_syntax_description(self): 552 | return ( 553 | "Config file syntax is Tom's Obvious, Minimal Language. " 554 | "See https://github.com/toml-lang/toml/blob/v0.5.0/README.md for details." 555 | ) 556 | 557 | 558 | class IniConfigParser(ConfigFileParser): 559 | """ 560 | Create a INI parser bounded to the list of provided sections. 561 | Optionaly convert multiline strings to list. 562 | 563 | Example (if split_ml_text_to_list=False):: 564 | 565 | # this is a comment 566 | ; also a comment 567 | [my-software] 568 | # how to specify a key-value pair 569 | format-string: restructuredtext 570 | # white space are ignored, so name = value same as name=value 571 | # this is why you can quote strings 572 | quoted-string = '\thello\tmom... ' 573 | # how to set an arg which has action="store_true" 574 | warnings-as-errors = true 575 | # how to set an arg which has action="count" or type=int 576 | verbosity = 1 577 | # how to specify a list arg (eg. arg which has action="append") 578 | repeatable-option = ["https://docs.python.org/3/objects.inv", 579 | "https://twistedmatrix.com/documents/current/api/objects.inv"] 580 | # how to specify a multiline text: 581 | multi-line-text = 582 | Lorem ipsum dolor sit amet, consectetur adipiscing elit. 583 | Vivamus tortor odio, dignissim non ornare non, laoreet quis nunc. 584 | Maecenas quis dapibus leo, a pellentesque leo. 585 | 586 | Example (if split_ml_text_to_list=True):: 587 | 588 | # the same rules are applicable with the following changes: 589 | [my-software] 590 | # how to specify a list arg (eg. arg which has action="append") 591 | repeatable-option = # Just enter one value per line (the list literal format can also be used) 592 | https://docs.python.org/3/objects.inv 593 | https://twistedmatrix.com/documents/current/api/objects.inv 594 | # how to specify a multiline text (you have to quote it): 595 | multi-line-text = ''' 596 | Lorem ipsum dolor sit amet, consectetur adipiscing elit. 597 | Vivamus tortor odio, dignissim non ornare non, laoreet quis nunc. 598 | Maecenas quis dapibus leo, a pellentesque leo. 599 | ''' 600 | """ 601 | 602 | def __init__(self, sections, split_ml_text_to_list): 603 | """ 604 | :param sections: The section names bounded to the new parser. 605 | :split_ml_text_to_list: Wether to convert multiline strings to list 606 | """ 607 | super().__init__() 608 | self.sections = sections 609 | self.split_ml_text_to_list = split_ml_text_to_list 610 | 611 | def __call__(self): 612 | return self 613 | 614 | def parse(self, stream): 615 | """Parses the keys and values from an INI config file.""" 616 | # parse with configparser to allow multi-line values 617 | import configparser 618 | 619 | config = configparser.ConfigParser() 620 | try: 621 | config.read_string(stream.read()) 622 | except Exception as e: 623 | raise ConfigFileParserException("Couldn't parse INI file: %s" % e) 624 | 625 | # convert to dict and filter based on INI section names 626 | result = OrderedDict() 627 | for section in config.sections() + [configparser.DEFAULTSECT]: 628 | if section not in self.sections: 629 | continue 630 | for k, v in config[section].items(): 631 | strip_v = v.strip() 632 | if not strip_v: 633 | # ignores empty values, anyway allow_no_value=False by default so this should not happend. 634 | continue 635 | # evaluate lists 636 | if strip_v.startswith("[") and strip_v.endswith("]"): 637 | try: 638 | result[k] = ast.literal_eval(strip_v) 639 | except ValueError as e: 640 | # error evaluating object 641 | raise ConfigFileParserException( 642 | "Error evaluating list: " 643 | + str(e) 644 | + ". Put quotes around your text if it's meant to be a string." 645 | ) from e 646 | else: 647 | if is_quoted(strip_v): 648 | # evaluate quoted string 649 | try: 650 | result[k] = unquote_str(strip_v) 651 | except ValueError as e: 652 | # error unquoting string 653 | raise ConfigFileParserException(str(e)) from e 654 | # split multi-line text into list of strings if split_ml_text_to_list is enabled. 655 | elif self.split_ml_text_to_list and "\n" in v.rstrip("\n"): 656 | try: 657 | result[k] = [ 658 | unquote_str(i) for i in strip_v.split("\n") if i 659 | ] 660 | except ValueError as e: 661 | # error unquoting string 662 | raise ConfigFileParserException(str(e)) from e 663 | else: 664 | result[k] = v 665 | return result 666 | 667 | def get_syntax_description(self): 668 | msg = ( 669 | "Uses configparser module to parse an INI file which allows multi-line values. " 670 | "See https://docs.python.org/3/library/configparser.html for details. " 671 | "This parser includes support for quoting strings literal as well as python list syntax evaluation. " 672 | ) 673 | if self.split_ml_text_to_list: 674 | msg += ( 675 | "Alternatively lists can be constructed with a plain multiline string, " 676 | "each non-empty line will be converted to a list item." 677 | ) 678 | return msg 679 | 680 | 681 | class CompositeConfigParser(ConfigFileParser): 682 | """ 683 | Createa a config parser composed by others `ConfigFileParser`s. 684 | 685 | The composite parser will successively try to parse the file with each parser, 686 | until it succeeds, else raise execption with all encountered errors. 687 | """ 688 | 689 | def __init__(self, config_parser_types): 690 | super().__init__() 691 | self.parsers = [p() for p in config_parser_types] 692 | 693 | def __call__(self): 694 | return self 695 | 696 | def parse(self, stream): 697 | errors = [] 698 | for p in self.parsers: 699 | try: 700 | return p.parse(stream) # type: ignore[no-any-return] 701 | except Exception as e: 702 | stream.seek(0) 703 | errors.append(e) 704 | raise ConfigFileParserException( 705 | f"Error parsing config: {', '.join(repr(str(e)) for e in errors)}" 706 | ) 707 | 708 | def get_syntax_description(self): 709 | def guess_format_name(classname): 710 | strip = ( 711 | classname.lower() 712 | .strip("_") 713 | .replace("parser", "") 714 | .replace("config", "") 715 | .replace("file", "") 716 | ) 717 | return strip.upper() if strip else "??" 718 | 719 | msg = "Uses multiple config parser settings (in order): \n" 720 | for i, parser in enumerate(self.parsers): 721 | msg += f"[{i+1}] {guess_format_name(parser.__class__.__name__)}: {parser.get_syntax_description()} \n" 722 | return msg 723 | 724 | 725 | # used while parsing args to keep track of where they came from 726 | _COMMAND_LINE_SOURCE_KEY = "command_line" 727 | _ENV_VAR_SOURCE_KEY = "environment_variables" 728 | _CONFIG_FILE_SOURCE_KEY = "config_file" 729 | _DEFAULTS_SOURCE_KEY = "defaults" 730 | 731 | 732 | class ArgumentParser(argparse.ArgumentParser): 733 | """Drop-in replacement for `argparse.ArgumentParser` that adds support for 734 | environment variables and ``.ini`` or ``.yaml-style`` config files. 735 | """ 736 | 737 | def __init__(self, *args, **kwargs): 738 | r"""Supports args of the `argparse.ArgumentParser` constructor 739 | as \*\*kwargs, as well as the following additional args. 740 | 741 | Arguments: 742 | add_config_file_help: Whether to add a description of config file 743 | syntax to the help message. 744 | add_env_var_help: Whether to add something to the help message for 745 | args that can be set through environment variables. 746 | auto_env_var_prefix: If set to a string instead of None, all config- 747 | file-settable options will become also settable via environment 748 | variables whose names are this prefix followed by the config 749 | file key, all in upper case. (eg. setting this to ``foo_`` will 750 | allow an arg like ``--my-arg`` to also be set via the FOO_MY_ARG 751 | environment variable) 752 | default_config_files: When specified, this list of config files will 753 | be parsed in order, with the values from each config file 754 | taking precedence over previous ones. This allows an application 755 | to look for config files in multiple standard locations such as 756 | the install directory, home directory, and current directory. 757 | Also, shell \* syntax can be used to specify all conf files in a 758 | directory. For example:: 759 | 760 | ["/etc/conf/app_config.ini", 761 | "/etc/conf/conf-enabled/*.ini", 762 | "~/.my_app_config.ini", 763 | "./app_config.txt"] 764 | 765 | ignore_unknown_config_file_keys: If true, settings that are found 766 | in a config file but don't correspond to any defined 767 | configargparse args will be ignored. If false, they will be 768 | processed and appended to the commandline like other args, and 769 | can be retrieved using parse_known_args() instead of parse_args() 770 | config_file_open_func: function used to open a config file for reading 771 | or writing. Needs to return a file-like object. 772 | config_file_parser_class: configargparse.ConfigFileParser subclass 773 | which determines the config file format. configargparse comes 774 | with DefaultConfigFileParser and YAMLConfigFileParser. 775 | args_for_setting_config_path: A list of one or more command line 776 | args to be used for specifying the config file path 777 | (eg. ["-c", "--config-file"]). Default: [] 778 | config_arg_is_required: When args_for_setting_config_path is set, 779 | set this to True to always require users to provide a config path. 780 | config_arg_help_message: the help message to use for the 781 | args listed in args_for_setting_config_path. 782 | args_for_writing_out_config_file: A list of one or more command line 783 | args to use for specifying a config file output path. If 784 | provided, these args cause configargparse to write out a config 785 | file with settings based on the other provided commandline args, 786 | environment variants and defaults, and then to exit. 787 | (eg. ["-w", "--write-out-config-file"]). Default: [] 788 | write_out_config_file_arg_help_message: The help message to use for 789 | the args in args_for_writing_out_config_file. 790 | """ 791 | # This is the only way to make positional args (tested in the argparse 792 | # main test suite) and keyword arguments work across both Python 2 and 793 | # 3. This could be refactored to not need extra local variables. 794 | add_config_file_help = kwargs.pop("add_config_file_help", True) 795 | add_env_var_help = kwargs.pop("add_env_var_help", True) 796 | auto_env_var_prefix = kwargs.pop("auto_env_var_prefix", None) 797 | default_config_files = kwargs.pop("default_config_files", []) 798 | ignore_unknown_config_file_keys = kwargs.pop( 799 | "ignore_unknown_config_file_keys", False 800 | ) 801 | config_file_parser_class = kwargs.pop( 802 | "config_file_parser_class", DefaultConfigFileParser 803 | ) 804 | args_for_setting_config_path = kwargs.pop("args_for_setting_config_path", []) 805 | config_arg_is_required = kwargs.pop("config_arg_is_required", False) 806 | config_arg_help_message = kwargs.pop( 807 | "config_arg_help_message", "config file path" 808 | ) 809 | args_for_writing_out_config_file = kwargs.pop( 810 | "args_for_writing_out_config_file", [] 811 | ) 812 | write_out_config_file_arg_help_message = kwargs.pop( 813 | "write_out_config_file_arg_help_message", 814 | "takes the current " 815 | "command line args and writes them out to a config file at the " 816 | "given path, then exits", 817 | ) 818 | 819 | self._config_file_open_func = kwargs.pop("config_file_open_func", open) 820 | 821 | self._add_config_file_help = add_config_file_help 822 | self._add_env_var_help = add_env_var_help 823 | self._auto_env_var_prefix = auto_env_var_prefix 824 | 825 | argparse.ArgumentParser.__init__(self, *args, **kwargs) 826 | 827 | # parse the additional args 828 | if config_file_parser_class is None: 829 | self._config_file_parser = DefaultConfigFileParser() 830 | else: 831 | self._config_file_parser = config_file_parser_class() 832 | 833 | self._default_config_files = default_config_files 834 | self._ignore_unknown_config_file_keys = ignore_unknown_config_file_keys 835 | if args_for_setting_config_path: 836 | self.add_argument( 837 | *args_for_setting_config_path, 838 | dest="config_file", 839 | required=config_arg_is_required, 840 | help=config_arg_help_message, 841 | is_config_file_arg=True, 842 | ) 843 | 844 | if args_for_writing_out_config_file: 845 | self.add_argument( 846 | *args_for_writing_out_config_file, 847 | dest="write_out_config_file_to_this_path", 848 | metavar="CONFIG_OUTPUT_PATH", 849 | help=write_out_config_file_arg_help_message, 850 | is_write_out_config_file_arg=True, 851 | ) 852 | 853 | # TODO: delete me! 854 | if sys.version_info < (3, 9): 855 | self.exit_on_error = True 856 | 857 | def parse_args( 858 | self, args=None, namespace=None, config_file_contents=None, env_vars=os.environ 859 | ): 860 | """Supports all the same args as the `argparse.ArgumentParser.parse_args()`, 861 | as well as the following additional args. 862 | 863 | Arguments: 864 | args: a list of args as in argparse, or a string (eg. "-x -y bla") 865 | config_file_contents: String. Used for testing. 866 | env_vars: Dictionary. Used for testing. 867 | 868 | Returns: 869 | argparse.Namespace: namespace 870 | """ 871 | args, argv = self.parse_known_args( 872 | args=args, 873 | namespace=namespace, 874 | config_file_contents=config_file_contents, 875 | env_vars=env_vars, 876 | ignore_help_args=False, 877 | ) 878 | 879 | if argv: 880 | msg = "unrecognized arguments: %s" % " ".join(argv) 881 | if self.exit_on_error: 882 | self.error(msg) 883 | else: 884 | raise ArgumentError(None, msg) 885 | return args 886 | 887 | def parse_known_args( 888 | self, 889 | args=None, 890 | namespace=None, 891 | config_file_contents=None, 892 | env_vars=os.environ, 893 | ignore_help_args=False, 894 | ): 895 | """Supports all the same args as the `argparse.ArgumentParser.parse_args()`, 896 | as well as the following additional args. 897 | 898 | Arguments: 899 | args: a list of args as in argparse, or a string (eg. "-x -y bla") 900 | config_file_contents (str). Used for testing. 901 | env_vars (dict). Used for testing. 902 | ignore_help_args (bool): This flag determines behavior when user specifies ``--help`` or ``-h``. If False, 903 | it will have the default behavior - printing help and exiting. If True, it won't do either. 904 | 905 | Returns: 906 | tuple[argparse.Namespace, list[str]]: tuple namescpace, unknown_args 907 | """ 908 | if args is None: 909 | args = sys.argv[1:] 910 | elif isinstance(args, str): 911 | args = args.split() 912 | else: 913 | args = list(args) 914 | 915 | for a in self._actions: 916 | a.is_positional_arg = not a.option_strings 917 | 918 | if ignore_help_args: 919 | args = [arg for arg in args if arg not in ("-h", "--help")] 920 | 921 | # maps a string describing the source (eg. env var) to a settings dict 922 | # to keep track of where values came from (used by print_values()). 923 | # The settings dicts for env vars and config files will then map 924 | # the config key to an (argparse Action obj, string value) 2-tuple. 925 | self._source_to_settings = OrderedDict() 926 | if args: 927 | a_v_pair = (None, list(args)) # copy args list to isolate changes 928 | self._source_to_settings[_COMMAND_LINE_SOURCE_KEY] = {"": a_v_pair} 929 | 930 | # handle auto_env_var_prefix __init__ arg by setting a.env_var as needed 931 | if self._auto_env_var_prefix is not None: 932 | for a in self._actions: 933 | config_file_keys = self.get_possible_config_keys(a) 934 | if config_file_keys and not ( 935 | a.env_var 936 | or a.is_positional_arg 937 | or a.is_config_file_arg 938 | or a.is_write_out_config_file_arg 939 | or isinstance(a, argparse._VersionAction) 940 | or isinstance(a, argparse._HelpAction) 941 | ): 942 | stripped_config_file_key = config_file_keys[0].strip( 943 | self.prefix_chars 944 | ) 945 | a.env_var = ( 946 | (self._auto_env_var_prefix + stripped_config_file_key) 947 | .replace("-", "_") 948 | .upper() 949 | ) 950 | 951 | # add env var settings to the commandline that aren't there already 952 | env_var_args = [] 953 | nargs = False 954 | actions_with_env_var_values = [ 955 | a 956 | for a in self._actions 957 | if not a.is_positional_arg 958 | and a.env_var 959 | and a.env_var in env_vars 960 | and not already_on_command_line(args, a.option_strings, self.prefix_chars) 961 | ] 962 | for action in actions_with_env_var_values: 963 | key = action.env_var 964 | value = env_vars[key] 965 | # Make list-string into list. 966 | if action.nargs or isinstance(action, argparse._AppendAction): 967 | nargs = True 968 | if value.startswith("[") and value.endswith("]"): 969 | # handle special case of k=[1,2,3] or other json-like syntax 970 | try: 971 | value = json.loads(value) 972 | except Exception: 973 | # for backward compatibility with legacy format (eg. where config value is [a, b, c] instead of proper json ["a", "b", "c"] 974 | value = [elem.strip() for elem in value[1:-1].split(",")] 975 | env_var_args += self.convert_item_to_command_line_arg(action, key, value) 976 | 977 | if nargs: 978 | args = args + env_var_args 979 | else: 980 | args = env_var_args + args 981 | 982 | if env_var_args: 983 | self._source_to_settings[_ENV_VAR_SOURCE_KEY] = OrderedDict( 984 | [ 985 | (a.env_var, (a, env_vars[a.env_var])) 986 | for a in actions_with_env_var_values 987 | ] 988 | ) 989 | 990 | # before parsing any config files, check if -h was specified. 991 | supports_help_arg = any( 992 | a for a in self._actions if isinstance(a, argparse._HelpAction) 993 | ) 994 | skip_config_file_parsing = supports_help_arg and ( 995 | "-h" in args or "--help" in args 996 | ) 997 | 998 | # prepare for reading config file(s) 999 | known_config_keys = { 1000 | config_key: action 1001 | for action in self._actions 1002 | for config_key in self.get_possible_config_keys(action) 1003 | } 1004 | 1005 | # open the config file(s) 1006 | config_streams = [] 1007 | if config_file_contents is not None: 1008 | stream = StringIO(config_file_contents) 1009 | stream.name = "method arg" 1010 | config_streams = [stream] 1011 | elif not skip_config_file_parsing: 1012 | config_streams = self._open_config_files(args) 1013 | 1014 | # parse each config file 1015 | for stream in reversed(config_streams): 1016 | try: 1017 | config_items = self._config_file_parser.parse(stream) 1018 | except ConfigFileParserException as e: 1019 | self.error(str(e)) 1020 | finally: 1021 | if hasattr(stream, "close"): 1022 | stream.close() 1023 | 1024 | # add each config item to the commandline unless it's there already 1025 | config_args = [] 1026 | nargs = False 1027 | for key, value in config_items.items(): 1028 | if key in known_config_keys: 1029 | action = known_config_keys[key] 1030 | discard_this_key = already_on_command_line( 1031 | args, action.option_strings, self.prefix_chars 1032 | ) 1033 | else: 1034 | action = None 1035 | discard_this_key = ( 1036 | self._ignore_unknown_config_file_keys 1037 | or already_on_command_line( 1038 | args, 1039 | [ 1040 | self.get_command_line_key_for_unknown_config_file_setting( 1041 | key 1042 | ) 1043 | ], 1044 | self.prefix_chars, 1045 | ) 1046 | ) 1047 | 1048 | if not discard_this_key: 1049 | config_args += self.convert_item_to_command_line_arg( 1050 | action, key, value 1051 | ) 1052 | source_key = "%s|%s" % (_CONFIG_FILE_SOURCE_KEY, stream.name) 1053 | if source_key not in self._source_to_settings: 1054 | self._source_to_settings[source_key] = OrderedDict() 1055 | self._source_to_settings[source_key][key] = (action, value) 1056 | if ( 1057 | action 1058 | and action.nargs 1059 | or isinstance(action, argparse._AppendAction) 1060 | ): 1061 | nargs = True 1062 | 1063 | if nargs: 1064 | args = args + config_args 1065 | else: 1066 | args = config_args + args 1067 | 1068 | # save default settings for use by print_values() 1069 | default_settings = OrderedDict() 1070 | for action in self._actions: 1071 | cares_about_default_value = ( 1072 | not action.is_positional_arg or action.nargs in [OPTIONAL, ZERO_OR_MORE] 1073 | ) 1074 | if ( 1075 | already_on_command_line(args, action.option_strings, self.prefix_chars) 1076 | or not cares_about_default_value 1077 | or action.default is None 1078 | or action.default == SUPPRESS 1079 | or isinstance(action, ACTION_TYPES_THAT_DONT_NEED_A_VALUE) 1080 | ): 1081 | continue 1082 | else: 1083 | if action.option_strings: 1084 | key = action.option_strings[-1] 1085 | else: 1086 | key = action.dest 1087 | default_settings[key] = (action, str(action.default)) 1088 | 1089 | if default_settings: 1090 | self._source_to_settings[_DEFAULTS_SOURCE_KEY] = default_settings 1091 | 1092 | # parse all args (including commandline, config file, and env var) 1093 | namespace, unknown_args = argparse.ArgumentParser.parse_known_args( 1094 | self, args=args, namespace=namespace 1095 | ) 1096 | # handle any args that have is_write_out_config_file_arg set to true 1097 | # check if the user specified this arg on the commandline 1098 | output_file_paths = [ 1099 | getattr(namespace, a.dest, None) 1100 | for a in self._actions 1101 | if getattr(a, "is_write_out_config_file_arg", False) 1102 | ] 1103 | output_file_paths = [a for a in output_file_paths if a is not None] 1104 | self.write_config_file(namespace, output_file_paths, exit_after=True) 1105 | return namespace, unknown_args 1106 | 1107 | def get_source_to_settings_dict(self): 1108 | """ 1109 | If called after `parse_args()` or `parse_known_args()`, returns a dict that contains up to 4 keys corresponding 1110 | to where a given option's value is coming from: 1111 | - "command_line" 1112 | - "environment_variables" 1113 | - "config_file" 1114 | - "defaults" 1115 | Each such key, will be mapped to another dictionary containing the options set via that method. Here the key 1116 | will be the option name, and the value will be a 2-tuple of the form (`argparse.Action` obj, `str` value). 1117 | 1118 | Returns: 1119 | dict[str, dict[str, tuple[argparse.Action, str]]]: source to settings dict 1120 | """ 1121 | # _source_to_settings is set in parse_know_args(). 1122 | return self._source_to_settings # type:ignore[attribute-error] 1123 | 1124 | def write_config_file(self, parsed_namespace, output_file_paths, exit_after=False): 1125 | """Write the given settings to output files. 1126 | 1127 | Args: 1128 | parsed_namespace: namespace object created within parse_known_args() 1129 | output_file_paths: any number of file paths to write the config to 1130 | exit_after: whether to exit the program after writing the config files 1131 | """ 1132 | for output_file_path in output_file_paths: 1133 | # validate the output file path 1134 | try: 1135 | with self._config_file_open_func(output_file_path, "w") as output_file: 1136 | pass 1137 | except IOError as e: 1138 | raise ValueError( 1139 | "Couldn't open {} for writing: {}".format(output_file_path, e) 1140 | ) 1141 | if output_file_paths: 1142 | # generate the config file contents 1143 | config_items = self.get_items_for_config_file_output( 1144 | self._source_to_settings, parsed_namespace 1145 | ) 1146 | file_contents = self._config_file_parser.serialize(config_items) 1147 | for output_file_path in output_file_paths: 1148 | with self._config_file_open_func(output_file_path, "w") as output_file: 1149 | output_file.write(file_contents) 1150 | 1151 | print("Wrote config file to " + ", ".join(output_file_paths)) 1152 | if exit_after: 1153 | self.exit(0) 1154 | 1155 | def get_command_line_key_for_unknown_config_file_setting(self, key): 1156 | """Compute a commandline arg key to be used for a config file setting 1157 | that doesn't correspond to any defined configargparse arg (and so 1158 | doesn't have a user-specified commandline arg key). 1159 | 1160 | Args: 1161 | key: The config file key that was being set. 1162 | 1163 | Returns: 1164 | str: command line key 1165 | """ 1166 | key_without_prefix_chars = key.strip(self.prefix_chars) 1167 | command_line_key = self.prefix_chars[0] * 2 + key_without_prefix_chars 1168 | 1169 | return command_line_key 1170 | 1171 | def get_items_for_config_file_output(self, source_to_settings, parsed_namespace): 1172 | """Converts the given settings back to a dictionary that can be passed 1173 | to ConfigFormatParser.serialize(..). 1174 | 1175 | Args: 1176 | source_to_settings: the dictionary described in parse_known_args() 1177 | parsed_namespace: namespace object created within parse_known_args() 1178 | Returns: 1179 | OrderedDict: where keys are strings and values are either strings 1180 | or lists 1181 | """ 1182 | config_file_items = OrderedDict() 1183 | for source, settings in source_to_settings.items(): 1184 | if source == _COMMAND_LINE_SOURCE_KEY: 1185 | _, existing_command_line_args = settings[""] 1186 | for action in self._actions: 1187 | config_file_keys = self.get_possible_config_keys(action) 1188 | if ( 1189 | config_file_keys 1190 | and not action.is_positional_arg 1191 | and already_on_command_line( 1192 | existing_command_line_args, 1193 | action.option_strings, 1194 | self.prefix_chars, 1195 | ) 1196 | ): 1197 | value = getattr(parsed_namespace, action.dest, None) 1198 | if value is not None: 1199 | if isinstance(value, bool): 1200 | value = str(value).lower() 1201 | config_file_items[config_file_keys[0]] = value 1202 | 1203 | elif source == _ENV_VAR_SOURCE_KEY: 1204 | for key, (action, value) in settings.items(): 1205 | config_file_keys = self.get_possible_config_keys(action) 1206 | if config_file_keys: 1207 | value = getattr(parsed_namespace, action.dest, None) 1208 | if value is not None: 1209 | config_file_items[config_file_keys[0]] = value 1210 | elif source.startswith(_CONFIG_FILE_SOURCE_KEY): 1211 | for key, (action, value) in settings.items(): 1212 | config_file_items[key] = value 1213 | elif source == _DEFAULTS_SOURCE_KEY: 1214 | for key, (action, value) in settings.items(): 1215 | config_file_keys = self.get_possible_config_keys(action) 1216 | if config_file_keys: 1217 | value = getattr(parsed_namespace, action.dest, None) 1218 | if value is not None: 1219 | config_file_items[config_file_keys[0]] = value 1220 | return config_file_items 1221 | 1222 | def convert_item_to_command_line_arg(self, action, key, value): 1223 | """Converts a config file or env var key + value to a list of 1224 | commandline args to append to the commandline. 1225 | 1226 | Args: 1227 | action: The argparse Action object for this setting, or None if this 1228 | config file setting doesn't correspond to any defined 1229 | configargparse arg. 1230 | key: string (config file key or env var name) 1231 | value: parsed value of type string or list 1232 | 1233 | Returns: 1234 | list[str]: args 1235 | """ 1236 | args = [] 1237 | 1238 | if action is None: 1239 | command_line_key = ( 1240 | self.get_command_line_key_for_unknown_config_file_setting(key) 1241 | ) 1242 | else: 1243 | if not is_boolean_optional_action(action): 1244 | command_line_key = action.option_strings[-1] 1245 | 1246 | # handle boolean value 1247 | if action is not None and isinstance( 1248 | action, ACTION_TYPES_THAT_DONT_NEED_A_VALUE 1249 | ): 1250 | assert isinstance( 1251 | value, str 1252 | ), "config parser should convert anything that is not a list to string." 1253 | if value.lower() in ("true", "yes", "on", "1"): 1254 | if not is_boolean_optional_action(action): 1255 | args.append(command_line_key) 1256 | else: 1257 | # --foo 1258 | args.append(action.option_strings[0]) 1259 | elif value.lower() in ("false", "no", "off", "0"): 1260 | # don't append when set to "false" / "no" 1261 | if not is_boolean_optional_action(action): 1262 | pass 1263 | else: 1264 | # --no-foo 1265 | args.append(action.option_strings[1]) 1266 | elif isinstance(action, argparse._CountAction): 1267 | for arg in args: 1268 | if any([arg.startswith(s) for s in action.option_strings]): 1269 | value = 0 1270 | args += [action.option_strings[0]] * int(value) 1271 | else: 1272 | self.error( 1273 | "Unexpected value for %s: '%s'. Expecting 'true', " 1274 | "'false', 'yes', 'no', 'on', 'off', '1' or '0'" % (key, value) 1275 | ) 1276 | elif isinstance(value, list): 1277 | accepts_list_and_has_nargs = ( 1278 | action is not None 1279 | and action.nargs is not None 1280 | and ( 1281 | isinstance(action, argparse._StoreAction) 1282 | or isinstance(action, argparse._AppendAction) 1283 | ) 1284 | and ( 1285 | action.nargs in ("+", "*") 1286 | or (isinstance(action.nargs, int) and action.nargs > 1) 1287 | ) 1288 | ) 1289 | 1290 | if action is None or isinstance(action, argparse._AppendAction): 1291 | for list_elem in value: 1292 | if accepts_list_and_has_nargs and isinstance(list_elem, list): 1293 | args.append(command_line_key) 1294 | for sub_elem in list_elem: 1295 | args.append(str(sub_elem)) 1296 | else: 1297 | args.append("%s=%s" % (command_line_key, str(list_elem))) 1298 | elif accepts_list_and_has_nargs: 1299 | args.append(command_line_key) 1300 | for list_elem in value: 1301 | args.append(str(list_elem)) 1302 | else: 1303 | self.error( 1304 | ( 1305 | "%s can't be set to a list '%s' unless its action type is changed " 1306 | "to 'append' or nargs is set to '*', '+', or > 1" 1307 | ) 1308 | % (key, value) 1309 | ) 1310 | elif isinstance(value, str): 1311 | args.append("%s=%s" % (command_line_key, value)) 1312 | else: 1313 | raise ValueError( 1314 | "Unexpected value type {} for value: {}".format(type(value), value) 1315 | ) 1316 | 1317 | return args 1318 | 1319 | def get_possible_config_keys(self, action): 1320 | """This method decides which actions can be set in a config file and 1321 | what their keys will be. It returns a list of 0 or more config keys that 1322 | can be used to set the given action's value in a config file. 1323 | 1324 | Returns: 1325 | list[str]: keys 1326 | """ 1327 | keys = [] 1328 | 1329 | # Do not write out the config options for writing out a config file 1330 | if getattr(action, "is_write_out_config_file_arg", None): 1331 | return keys 1332 | 1333 | for arg in action.option_strings: 1334 | if any(arg.startswith(2 * c) for c in self.prefix_chars): 1335 | keys += [arg[2:], arg] # eg. for '--bla' return ['bla', '--bla'] 1336 | 1337 | return keys 1338 | 1339 | def _open_config_files(self, command_line_args): 1340 | """Tries to parse config file path(s) from within command_line_args. 1341 | Returns a list of opened config files, including files specified on the 1342 | commandline as well as any default_config_files specified in the 1343 | constructor that are present on disk. 1344 | 1345 | Args: 1346 | command_line_args: List of all args 1347 | 1348 | Returns: 1349 | list[IO]: open config files 1350 | """ 1351 | # open any default config files 1352 | config_files = [] 1353 | for files in map( 1354 | glob.glob, map(os.path.expanduser, self._default_config_files) 1355 | ): 1356 | for f in files: 1357 | config_files.append(self._config_file_open_func(f)) 1358 | 1359 | # list actions with is_config_file_arg=True. Its possible there is more 1360 | # than one such arg. 1361 | user_config_file_arg_actions = [ 1362 | a for a in self._actions if getattr(a, "is_config_file_arg", False) 1363 | ] 1364 | 1365 | if not user_config_file_arg_actions: 1366 | return config_files 1367 | 1368 | for action in user_config_file_arg_actions: 1369 | # try to parse out the config file path by using a clean new 1370 | # ArgumentParser that only knows this one arg/action. 1371 | arg_parser = argparse.ArgumentParser( 1372 | prefix_chars=self.prefix_chars, add_help=False 1373 | ) 1374 | 1375 | arg_parser._add_action(action) 1376 | 1377 | # make parser not exit on error by replacing its error method. 1378 | # Otherwise it sys.exits(..) if, for example, config file 1379 | # is_required=True and user doesn't provide it. 1380 | def error_method(self, message): 1381 | pass 1382 | 1383 | arg_parser.error = types.MethodType(error_method, arg_parser) 1384 | 1385 | # check whether the user provided a value 1386 | parsed_arg = arg_parser.parse_known_args(args=command_line_args) 1387 | if not parsed_arg: 1388 | continue 1389 | namespace, _ = parsed_arg 1390 | user_config_file = getattr(namespace, action.dest, None) 1391 | 1392 | if not user_config_file: 1393 | continue 1394 | 1395 | # open user-provided config file 1396 | user_config_file = os.path.expanduser(user_config_file) 1397 | try: 1398 | stream = self._config_file_open_func(user_config_file) 1399 | except Exception as e: 1400 | if len(e.args) == 2: # OSError 1401 | errno, msg = e.args 1402 | else: 1403 | msg = str(e) 1404 | # close previously opened config files 1405 | for config_file in config_files: 1406 | try: 1407 | config_file.close() 1408 | except Exception: 1409 | pass 1410 | self.error( 1411 | "Unable to open config file: %s. Error: %s" 1412 | % (user_config_file, msg) 1413 | ) 1414 | 1415 | config_files += [stream] 1416 | 1417 | return config_files 1418 | 1419 | def format_values(self): 1420 | """Returns a string with all args and settings and where they came from 1421 | (eg. commandline, config file, environment variable or default) 1422 | 1423 | Returns: 1424 | str: source to settings string 1425 | """ 1426 | source_key_to_display_value_map = { 1427 | _COMMAND_LINE_SOURCE_KEY: "Command Line Args: ", 1428 | _ENV_VAR_SOURCE_KEY: "Environment Variables:\n", 1429 | _CONFIG_FILE_SOURCE_KEY: "Config File (%s):\n", 1430 | _DEFAULTS_SOURCE_KEY: "Defaults:\n", 1431 | } 1432 | 1433 | r = StringIO() 1434 | for ( 1435 | source, 1436 | settings, 1437 | ) in self._source_to_settings.items(): # type:ignore[argument-error] 1438 | source = source.split("|") 1439 | source = source_key_to_display_value_map[source[0]] % tuple(source[1:]) 1440 | r.write(source) 1441 | for key, (action, value) in settings.items(): 1442 | if key: 1443 | r.write(" {:<19}{}\n".format(key + ":", value)) 1444 | else: 1445 | if isinstance(value, str): 1446 | r.write(" %s\n" % value) 1447 | elif isinstance(value, list): 1448 | r.write(" %s\n" % " ".join(value)) 1449 | 1450 | return r.getvalue() 1451 | 1452 | def print_values(self, file=sys.stdout): 1453 | """Prints the format_values() string (to sys.stdout or another file).""" 1454 | file.write(self.format_values()) 1455 | 1456 | def format_help(self): 1457 | msg = "" 1458 | added_config_file_help = False 1459 | added_env_var_help = False 1460 | if self._add_config_file_help: 1461 | default_config_files = self._default_config_files 1462 | cc = 2 * self.prefix_chars[0] # eg. -- 1463 | config_settable_args = [ 1464 | (arg, a) 1465 | for a in self._actions 1466 | for arg in a.option_strings 1467 | if self.get_possible_config_keys(a) 1468 | and not ( 1469 | a.dest == "help" 1470 | or a.is_config_file_arg 1471 | or a.is_write_out_config_file_arg 1472 | ) 1473 | ] 1474 | config_path_actions = [ 1475 | a for a in self._actions if getattr(a, "is_config_file_arg", False) 1476 | ] 1477 | 1478 | if config_settable_args and (default_config_files or config_path_actions): 1479 | self._add_config_file_help = False # prevent duplication 1480 | added_config_file_help = True 1481 | 1482 | msg += ( 1483 | "Args that start with '%s' can also be set in " "a config file" 1484 | ) % cc 1485 | config_arg_string = " or ".join( 1486 | a.option_strings[0] for a in config_path_actions if a.option_strings 1487 | ) 1488 | if config_arg_string: 1489 | config_arg_string = "specified via " + config_arg_string 1490 | if default_config_files or config_arg_string: 1491 | msg += " (%s)." % " or ".join( 1492 | tuple(map(str, default_config_files)) 1493 | + tuple(filter(None, [config_arg_string])) 1494 | ) 1495 | msg += " " + self._config_file_parser.get_syntax_description() 1496 | 1497 | if self._add_env_var_help: 1498 | env_var_actions = [ 1499 | (a.env_var, a) for a in self._actions if getattr(a, "env_var", None) 1500 | ] 1501 | for env_var, a in env_var_actions: 1502 | if a.help == SUPPRESS: 1503 | continue 1504 | env_var_help_string = " [env var: %s]" % env_var 1505 | if not a.help: 1506 | a.help = "" 1507 | if env_var_help_string not in a.help: 1508 | a.help += env_var_help_string 1509 | added_env_var_help = True 1510 | self._add_env_var_help = False # prevent duplication 1511 | 1512 | if added_env_var_help or added_config_file_help: 1513 | value_sources = ["defaults"] 1514 | if added_config_file_help: 1515 | value_sources = ["config file values"] + value_sources 1516 | if added_env_var_help: 1517 | value_sources = ["environment variables"] + value_sources 1518 | msg += " In general, command-line values override %s." % ( 1519 | " which override ".join(value_sources) 1520 | ) 1521 | 1522 | text_width = max(self._get_formatter()._width, 11) 1523 | msg = textwrap.fill(msg, text_width) 1524 | 1525 | return argparse.ArgumentParser.format_help(self) + ( 1526 | "\n{}\n".format(msg) if msg != "" else "" 1527 | ) 1528 | 1529 | 1530 | def add_argument(self, *args, **kwargs): 1531 | """ 1532 | This method supports the same args as ArgumentParser.add_argument(..) 1533 | as well as the additional args below. 1534 | 1535 | Arguments: 1536 | env_var: If set, the value of this environment variable will override 1537 | any config file or default values for this arg (but can itself 1538 | be overridden on the commandline). Also, if auto_env_var_prefix is 1539 | set in the constructor, this env var name will be used instead of 1540 | the automatic name. 1541 | is_config_file_arg: If True, this arg is treated as a config file path 1542 | This provides an alternative way to specify config files in place of 1543 | the ArgumentParser(fromfile_prefix_chars=..) mechanism. 1544 | Default: False 1545 | is_write_out_config_file_arg: If True, this arg will be treated as a 1546 | config file path, and, when it is specified, will cause 1547 | configargparse to write all current commandline args to this file 1548 | as config options and then exit. 1549 | Default: False 1550 | 1551 | Returns: 1552 | argparse.Action: the new argparse action 1553 | """ 1554 | 1555 | env_var = kwargs.pop("env_var", None) 1556 | 1557 | is_config_file_arg = kwargs.pop("is_config_file_arg", None) or kwargs.pop( 1558 | "is_config_file", None 1559 | ) # for backward compat. 1560 | 1561 | is_write_out_config_file_arg = kwargs.pop("is_write_out_config_file_arg", None) 1562 | 1563 | action = self.original_add_argument_method(*args, **kwargs) 1564 | 1565 | action.is_positional_arg = not action.option_strings 1566 | action.env_var = env_var 1567 | action.is_config_file_arg = is_config_file_arg 1568 | action.is_write_out_config_file_arg = is_write_out_config_file_arg 1569 | 1570 | if action.is_positional_arg and env_var: 1571 | raise ValueError("env_var can't be set for a positional arg.") 1572 | if action.is_config_file_arg and not isinstance(action, argparse._StoreAction): 1573 | raise ValueError("arg with is_config_file_arg=True must have " "action='store'") 1574 | if action.is_write_out_config_file_arg: 1575 | error_prefix = "arg with is_write_out_config_file_arg=True " 1576 | if not isinstance(action, argparse._StoreAction): 1577 | raise ValueError(error_prefix + "must have action='store'") 1578 | if is_config_file_arg: 1579 | raise ValueError( 1580 | error_prefix + "can't also have " "is_config_file_arg=True" 1581 | ) 1582 | 1583 | return action 1584 | 1585 | 1586 | def already_on_command_line( 1587 | existing_args_list, potential_command_line_args, prefix_chars 1588 | ): 1589 | """Utility method for checking if any of the potential_command_line_args is 1590 | already present in existing_args. 1591 | 1592 | Returns: 1593 | bool: already on command line? 1594 | """ 1595 | arg_names = [] 1596 | for arg_string in existing_args_list: 1597 | if arg_string and arg_string[0] in prefix_chars and "=" in arg_string: 1598 | option_string, explicit_arg = arg_string.split("=", 1) 1599 | arg_names.append(option_string) 1600 | else: 1601 | arg_names.append(arg_string) 1602 | 1603 | return any( 1604 | potential_arg in arg_names for potential_arg in potential_command_line_args 1605 | ) 1606 | 1607 | 1608 | # TODO: Update to latest version of pydoctor when https://github.com/twisted/pydoctor/pull/414 has been merged 1609 | # such that the alises can be documented automatically. 1610 | 1611 | # wrap ArgumentParser's add_argument(..) method with the one above 1612 | argparse._ActionsContainer.original_add_argument_method = ( 1613 | argparse._ActionsContainer.add_argument 1614 | ) 1615 | argparse._ActionsContainer.add_argument = add_argument 1616 | 1617 | 1618 | # add all public classes and constants from argparse module's namespace to this 1619 | # module's namespace so that the 2 modules are truly interchangeable 1620 | Action = argparse.Action 1621 | ArgumentDefaultsHelpFormatter = argparse.ArgumentDefaultsHelpFormatter 1622 | ArgumentError = argparse.ArgumentError 1623 | ArgumentTypeError = argparse.ArgumentTypeError 1624 | FileType = argparse.FileType 1625 | HelpFormatter = argparse.HelpFormatter 1626 | MetavarTypeHelpFormatter = argparse.MetavarTypeHelpFormatter 1627 | Namespace = argparse.Namespace 1628 | RawDescriptionHelpFormatter = argparse.RawDescriptionHelpFormatter 1629 | RawTextHelpFormatter = argparse.RawTextHelpFormatter 1630 | ONE_OR_MORE = argparse.ONE_OR_MORE 1631 | OPTIONAL = argparse.OPTIONAL 1632 | PARSER = argparse.PARSER 1633 | REMAINDER = argparse.REMAINDER 1634 | SUPPRESS = argparse.SUPPRESS 1635 | ZERO_OR_MORE = argparse.ZERO_OR_MORE 1636 | 1637 | 1638 | # deprecated PEP-8 incompatible API names. 1639 | initArgumentParser = init_argument_parser 1640 | getArgumentParser = get_argument_parser 1641 | getArgParser = get_argument_parser 1642 | getParser = get_argument_parser 1643 | 1644 | # create shorter aliases for the key methods and class names 1645 | get_arg_parser = get_argument_parser 1646 | get_parser = get_argument_parser 1647 | 1648 | ArgParser = ArgumentParser 1649 | Parser = ArgumentParser 1650 | 1651 | argparse._ActionsContainer.add_arg = argparse._ActionsContainer.add_argument 1652 | argparse._ActionsContainer.add = argparse._ActionsContainer.add_argument 1653 | 1654 | ArgumentParser.parse = ArgumentParser.parse_args 1655 | ArgumentParser.parse_known = ArgumentParser.parse_known_args 1656 | 1657 | RawFormatter = RawDescriptionHelpFormatter 1658 | DefaultsFormatter = ArgumentDefaultsHelpFormatter 1659 | DefaultsRawFormatter = ArgumentDefaultsRawHelpFormatter 1660 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import sys 4 | 5 | 6 | try: 7 | from setuptools import setup 8 | except ImportError: 9 | print("WARNING: setuptools not installed. Will try using distutils instead..") 10 | from distutils.core import setup 11 | 12 | 13 | def launch_http_server(directory): 14 | assert os.path.isdir(directory) 15 | 16 | try: 17 | try: 18 | from SimpleHTTPServer import SimpleHTTPRequestHandler 19 | except ImportError: 20 | from http.server import SimpleHTTPRequestHandler 21 | 22 | try: 23 | import SocketServer 24 | except ImportError: 25 | import socketserver as SocketServer 26 | 27 | import socket 28 | 29 | for port in [80] + list(range(8000, 8100)): 30 | try: 31 | s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 32 | s.bind(("localhost", port)) 33 | s.close() 34 | except socket.error as e: 35 | logging.debug("Can't use port %d: %s" % (port, e.strerror)) 36 | continue 37 | 38 | print( 39 | "HTML coverage report now available at http://{}{}".format( 40 | socket.gethostname(), (":%s" % port) if port != 80 else "" 41 | ) 42 | ) 43 | 44 | os.chdir(directory) 45 | SocketServer.TCPServer(("", port), SimpleHTTPRequestHandler).serve_forever() 46 | else: 47 | logging.debug("All network port. ") 48 | except Exception as e: 49 | logging.error( 50 | "ERROR: while starting an HTTP server to serve " 51 | "the coverage report: %s" % e 52 | ) 53 | 54 | 55 | command = sys.argv[-1] 56 | if command == "publish": 57 | os.system("rm -rf dist") 58 | os.system("python3 setup.py sdist") 59 | os.system("python3 setup.py bdist_wheel") 60 | os.system( 61 | "twine check dist/*" 62 | ) # check for formatting or other issues that would cause twine upload to error out 63 | os.system("twine upload dist/*whl dist/*gz") 64 | sys.exit() 65 | elif command == "coverage": 66 | try: 67 | import coverage 68 | except: 69 | sys.exit("coverage.py not installed (pip install --user coverage)") 70 | setup_py_path = os.path.abspath(__file__) 71 | os.system("coverage run --source=configargparse " + setup_py_path + " test") 72 | os.system("coverage report") 73 | os.system("coverage html") 74 | print("Done computing coverage") 75 | launch_http_server(directory="htmlcov") 76 | sys.exit() 77 | 78 | long_description = "" 79 | if command not in ["test", "coverage"]: 80 | long_description = open("README.rst").read() 81 | 82 | install_requires = [] 83 | tests_require = [ 84 | "mock", 85 | "PyYAML", 86 | "pytest", 87 | ] 88 | 89 | 90 | setup( 91 | name="ConfigArgParse", 92 | version="1.7.1", 93 | description="A drop-in replacement for argparse that allows options to " 94 | "also be set via config files and/or environment variables.", 95 | long_description=long_description, 96 | url="https://github.com/bw2/ConfigArgParse", 97 | py_modules=["configargparse"], 98 | include_package_data=True, 99 | license="MIT", 100 | keywords="options, argparse, ConfigArgParse, config, environment variables, " 101 | "envvars, ENV, environment, optparse, YAML, INI", 102 | classifiers=[ 103 | "Development Status :: 4 - Beta", 104 | "Intended Audience :: Developers", 105 | "License :: OSI Approved :: MIT License", 106 | "Natural Language :: English", 107 | "Programming Language :: Python :: 3", 108 | "Programming Language :: Python :: 3.6", 109 | "Programming Language :: Python :: 3.7", 110 | "Programming Language :: Python :: 3.8", 111 | "Programming Language :: Python :: 3.9", 112 | "Programming Language :: Python :: 3.10", 113 | "Programming Language :: Python :: 3.11", 114 | "Programming Language :: Python :: 3.12", 115 | "Programming Language :: Python :: 3.13", 116 | "Programming Language :: Python :: Implementation :: CPython", 117 | "Programming Language :: Python :: Implementation :: PyPy", 118 | ], 119 | test_suite="tests", 120 | python_requires=">=3.6", 121 | install_requires=install_requires, 122 | tests_require=tests_require, 123 | extras_require={ 124 | "yaml": ["PyYAML"], 125 | "test": tests_require, 126 | }, 127 | ) 128 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | -------------------------------------------------------------------------------- /tests/test_configargparse.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import configargparse 3 | from contextlib import contextmanager 4 | import inspect 5 | import logging 6 | import os 7 | import sys 8 | import tempfile 9 | import types 10 | import unittest 11 | from unittest import mock 12 | 13 | from io import StringIO 14 | 15 | if sys.version_info >= (3, 10): 16 | OPTIONAL_ARGS_STRING = "options" 17 | else: 18 | OPTIONAL_ARGS_STRING = "optional arguments" 19 | 20 | # set COLUMNS to get expected wrapping 21 | os.environ["COLUMNS"] = "80" 22 | 23 | # enable logging to simplify debugging 24 | logger = logging.getLogger() 25 | logger.level = logging.DEBUG 26 | stream_handler = logging.StreamHandler(sys.stdout) 27 | logger.addHandler(stream_handler) 28 | 29 | 30 | def replace_error_method(arg_parser): 31 | """Swap out arg_parser's error(..) method so that instead of calling 32 | sys.exit(..) it just raises an error. 33 | """ 34 | 35 | def error_method(self, message): 36 | raise argparse.ArgumentError(None, message) 37 | 38 | def exit_method(self, status, message=None): 39 | self._exit_method_called = True 40 | 41 | arg_parser._exit_method_called = False 42 | arg_parser.error = types.MethodType(error_method, arg_parser) 43 | arg_parser.exit = types.MethodType(exit_method, arg_parser) 44 | 45 | return arg_parser 46 | 47 | 48 | @contextmanager 49 | def captured_output(): 50 | """ 51 | swap stdout and stderr for StringIO so we can do asserts on outputs. 52 | """ 53 | new_out, new_err = StringIO(), StringIO() 54 | old_out, old_err = sys.stdout, sys.stderr 55 | try: 56 | sys.stdout, sys.stderr = new_out, new_err 57 | yield sys.stdout, sys.stderr 58 | finally: 59 | sys.stdout, sys.stderr = old_out, old_err 60 | 61 | 62 | class TestCase(unittest.TestCase): 63 | 64 | def initParser(self, *args, **kwargs): 65 | p = configargparse.ArgParser(*args, **kwargs) 66 | self.parser = replace_error_method(p) 67 | self.add_arg = self.parser.add_argument 68 | self.parse = self.parser.parse_args 69 | self.parse_known = self.parser.parse_known_args 70 | self.format_values = self.parser.format_values 71 | self.format_help = self.parser.format_help 72 | 73 | if not hasattr(self, "assertRegex"): 74 | self.assertRegex = self.assertRegexpMatches 75 | if not hasattr(self, "assertRaisesRegex"): 76 | self.assertRaisesRegex = self.assertRaisesRegexp 77 | 78 | return self.parser 79 | 80 | def assertParseArgsRaises(self, regex, args, **kwargs): 81 | self.assertRaisesRegex( 82 | argparse.ArgumentError, regex, self.parse, args=args, **kwargs 83 | ) 84 | 85 | 86 | class TestBasicUseCases(TestCase): 87 | def setUp(self): 88 | self.initParser(args_for_setting_config_path=[]) 89 | 90 | def testBasicCase1(self): 91 | ## Test command line and config file values 92 | self.add_arg("filenames", nargs="+", help="positional arg") 93 | self.add_arg("-x", "--arg-x", action="store_true") 94 | self.add_arg("-y", "--arg-y", dest="y1", type=int, required=True) 95 | self.add_arg("--arg-z", action="append", type=float, required=True) 96 | if sys.version_info >= (3, 9): 97 | self.add_arg("--foo", action=argparse.BooleanOptionalAction, default=False) 98 | else: 99 | self.add_arg("--foo", action="store_true", default=False) 100 | 101 | # make sure required args are enforced 102 | self.assertParseArgsRaises( 103 | ( 104 | "too few arg" 105 | if sys.version_info.major < 3 106 | else "the following arguments are required" 107 | ), 108 | args="", 109 | ) 110 | self.assertParseArgsRaises( 111 | ( 112 | "argument -y/--arg-y is required" 113 | if sys.version_info.major < 3 114 | else "the following arguments are required: -y/--arg-y" 115 | ), 116 | args="-x --arg-z 11 file1.txt", 117 | ) 118 | self.assertParseArgsRaises( 119 | ( 120 | "argument --arg-z is required" 121 | if sys.version_info.major < 3 122 | else "the following arguments are required: --arg-z" 123 | ), 124 | args="file1.txt file2.txt file3.txt -x -y 1", 125 | ) 126 | 127 | # check values after setting args on command line 128 | ns = self.parse( 129 | args="file1.txt --arg-x -y 3 --arg-z 10 --foo", config_file_contents="" 130 | ) 131 | self.assertListEqual(ns.filenames, ["file1.txt"]) 132 | self.assertEqual(ns.arg_x, True) 133 | self.assertEqual(ns.y1, 3) 134 | self.assertEqual(ns.arg_z, [10]) 135 | self.assertEqual(ns.foo, True) 136 | 137 | self.assertRegex( 138 | self.format_values(), 139 | "Command Line Args: file1.txt --arg-x -y 3 --arg-z 10", 140 | ) 141 | 142 | # check values after setting args in config file 143 | ns = self.parse( 144 | args="file1.txt file2.txt", 145 | config_file_contents=""" 146 | # set all required args in config file 147 | arg-x = True 148 | arg-y = 10 149 | arg-z = 30 150 | arg-z = 40 151 | foo = True 152 | """, 153 | ) 154 | self.assertListEqual(ns.filenames, ["file1.txt", "file2.txt"]) 155 | self.assertEqual(ns.arg_x, True) 156 | self.assertEqual(ns.y1, 10) 157 | self.assertEqual(ns.arg_z, [40]) 158 | self.assertEqual(ns.foo, True) 159 | 160 | self.assertRegex( 161 | self.format_values(), 162 | "Command Line Args: \\s+ file1.txt file2.txt\n" 163 | "Config File \\(method arg\\):\n" 164 | " arg-x: \\s+ True\n" 165 | " arg-y: \\s+ 10\n" 166 | " arg-z: \\s+ 40\n" 167 | " foo: \\s+ True\n", 168 | ) 169 | 170 | # check values after setting args in both command line and config file 171 | ns = self.parse( 172 | args="file1.txt file2.txt --arg-x -y 3 --arg-z 100 ", 173 | config_file_contents="""arg-y = 31.5 174 | arg-z = 30 175 | foo = True 176 | """, 177 | ) 178 | self.format_help() 179 | self.format_values() 180 | self.assertListEqual(ns.filenames, ["file1.txt", "file2.txt"]) 181 | self.assertEqual(ns.arg_x, True) 182 | self.assertEqual(ns.y1, 3) 183 | self.assertEqual(ns.arg_z, [100]) 184 | self.assertEqual(ns.foo, True) 185 | 186 | self.assertRegex( 187 | self.format_values(), 188 | "Command Line Args: file1.txt file2.txt --arg-x -y 3 --arg-z 100", 189 | ) 190 | 191 | def testBasicCase2(self, use_groups=False): 192 | 193 | ## Test command line, config file and env var values 194 | default_config_file = tempfile.NamedTemporaryFile(mode="w", delete=False) 195 | default_config_file.flush() 196 | 197 | p = self.initParser( 198 | default_config_files=[ 199 | "/etc/settings.ini", 200 | "/home/jeff/.user_settings", 201 | default_config_file.name, 202 | ] 203 | ) 204 | p.add_arg("vcf", nargs="+", help="Variant file(s)") 205 | if not use_groups: 206 | self.add_arg("--genome", help="Path to genome file", required=True) 207 | self.add_arg("-v", dest="verbose", action="store_true") 208 | self.add_arg("-g", "--my-cfg-file", required=True, is_config_file=True) 209 | self.add_arg("-d", "--dbsnp", env_var="DBSNP_PATH") 210 | self.add_arg( 211 | "-f", 212 | "--format", 213 | choices=["BED", "MAF", "VCF", "WIG", "R"], 214 | dest="fmt", 215 | metavar="FRMT", 216 | env_var="OUTPUT_FORMAT", 217 | default="BED", 218 | ) 219 | else: 220 | g = p.add_argument_group(title="g1") 221 | g.add_arg("--genome", help="Path to genome file", required=True) 222 | g.add_arg("-v", dest="verbose", action="store_true") 223 | g.add_arg("-g", "--my-cfg-file", required=True, is_config_file=True) 224 | g = p.add_argument_group(title="g2") 225 | g.add_arg("-d", "--dbsnp", env_var="DBSNP_PATH") 226 | g.add_arg( 227 | "-f", 228 | "--format", 229 | choices=["BED", "MAF", "VCF", "WIG", "R"], 230 | dest="fmt", 231 | metavar="FRMT", 232 | env_var="OUTPUT_FORMAT", 233 | default="BED", 234 | ) 235 | 236 | # make sure required args are enforced 237 | self.assertParseArgsRaises( 238 | ( 239 | "too few arg" 240 | if sys.version_info.major < 3 241 | else "the following arguments are required: vcf, -g/--my-cfg-file" 242 | ), 243 | args="--genome hg19", 244 | ) 245 | self.assertParseArgsRaises( 246 | "Unable to open config file: file.txt. Error: No such file or director", 247 | args="-g file.txt", 248 | ) 249 | 250 | # check values after setting args on command line 251 | config_file2 = tempfile.NamedTemporaryFile(mode="w", delete=False) 252 | config_file2.flush() 253 | 254 | ns = self.parse(args="--genome hg19 -g %s bla.vcf " % config_file2.name) 255 | self.assertEqual(ns.genome, "hg19") 256 | self.assertEqual(ns.verbose, False) 257 | self.assertIsNone(ns.dbsnp) 258 | self.assertEqual(ns.fmt, "BED") 259 | self.assertListEqual(ns.vcf, ["bla.vcf"]) 260 | 261 | self.assertRegex( 262 | self.format_values(), 263 | "Command Line Args: --genome hg19 -g [^\\s]+ bla.vcf\n" 264 | "Defaults:\n" 265 | " --format: \\s+ BED\n", 266 | ) 267 | 268 | # check precedence: args > env > config > default using the --format arg 269 | default_config_file.write("--format MAF") 270 | default_config_file.flush() 271 | ns = self.parse(args="--genome hg19 -g %s f.vcf " % config_file2.name) 272 | self.assertEqual(ns.fmt, "MAF") 273 | self.assertRegex( 274 | self.format_values(), 275 | "Command Line Args: --genome hg19 -g [^\\s]+ f.vcf\n" 276 | "Config File \\([^\\s]+\\):\n" 277 | " --format: \\s+ MAF\n", 278 | ) 279 | 280 | config_file2.write("--format VCF") 281 | config_file2.flush() 282 | ns = self.parse(args="--genome hg19 -g %s f.vcf " % config_file2.name) 283 | self.assertEqual(ns.fmt, "VCF") 284 | self.assertRegex( 285 | self.format_values(), 286 | "Command Line Args: --genome hg19 -g [^\\s]+ f.vcf\n" 287 | "Config File \\([^\\s]+\\):\n" 288 | " --format: \\s+ VCF\n", 289 | ) 290 | 291 | ns = self.parse( 292 | env_vars={"OUTPUT_FORMAT": "R", "DBSNP_PATH": "/a/b.vcf"}, 293 | args="--genome hg19 -g %s f.vcf " % config_file2.name, 294 | ) 295 | self.assertEqual(ns.fmt, "R") 296 | self.assertEqual(ns.dbsnp, "/a/b.vcf") 297 | self.assertRegex( 298 | self.format_values(), 299 | "Command Line Args: --genome hg19 -g [^\\s]+ f.vcf\n" 300 | "Environment Variables:\n" 301 | " DBSNP_PATH: \\s+ /a/b.vcf\n" 302 | " OUTPUT_FORMAT: \\s+ R\n", 303 | ) 304 | 305 | ns = self.parse( 306 | env_vars={ 307 | "OUTPUT_FORMAT": "R", 308 | "DBSNP_PATH": "/a/b.vcf", 309 | "ANOTHER_VAR": "something", 310 | }, 311 | args="--genome hg19 -g %s --format WIG f.vcf" % config_file2.name, 312 | ) 313 | self.assertEqual(ns.fmt, "WIG") 314 | self.assertEqual(ns.dbsnp, "/a/b.vcf") 315 | self.assertRegex( 316 | self.format_values(), 317 | "Command Line Args: --genome hg19 -g [^\\s]+ --format WIG f.vcf\n" 318 | "Environment Variables:\n" 319 | " DBSNP_PATH: \\s+ /a/b.vcf\n", 320 | ) 321 | 322 | if not use_groups: 323 | self.assertRegex( 324 | self.format_help(), 325 | "usage: .* \\[-h\\] --genome GENOME \\[-v\\] -g MY_CFG_FILE\n?" 326 | "\\s+\\[-d DBSNP\\]\\s+\\[-f FRMT\\]\\s+vcf \\[vcf ...\\]\n\n" 327 | "positional arguments:\n" 328 | " vcf \\s+ Variant file\\(s\\)\n\n" 329 | "%s:\n" 330 | " -h, --help \\s+ show this help message and exit\n" 331 | " --genome GENOME \\s+ Path to genome file\n" 332 | " -v\n" 333 | " -g(?: MY_CFG_FILE)?, --my-cfg-file MY_CFG_FILE\n" 334 | " -d(?: DBSNP)?, --dbsnp DBSNP\\s+\\[env var: DBSNP_PATH\\]\n" 335 | " -f(?: FRMT)?, --format FRMT\\s+\\[env var: OUTPUT_FORMAT\\]\n\n" 336 | % OPTIONAL_ARGS_STRING 337 | + 7 * r"(.+\s*)", 338 | ) 339 | else: 340 | self.assertRegex( 341 | self.format_help(), 342 | "usage: .* \\[-h\\] --genome GENOME \\[-v\\] -g MY_CFG_FILE\n?" 343 | "\\s+\\[-d DBSNP\\]\\s+\\[-f FRMT\\]\\s+vcf \\[vcf ...\\]\n\n" 344 | "positional arguments:\n" 345 | " vcf \\s+ Variant file\\(s\\)\n\n" 346 | "%s:\n" 347 | " -h, --help \\s+ show this help message and exit\n\n" 348 | "g1:\n" 349 | " --genome GENOME \\s+ Path to genome file\n" 350 | " -v\n" 351 | " -g(?: MY_CFG_FILE)?, --my-cfg-file MY_CFG_FILE\n\n" 352 | "g2:\n" 353 | " -d(?: DBSNP)?, --dbsnp DBSNP\\s+\\[env var: DBSNP_PATH\\]\n" 354 | " -f(?: FRMT)?, --format FRMT\\s+\\[env var: OUTPUT_FORMAT\\]\n\n" 355 | % OPTIONAL_ARGS_STRING 356 | + 7 * r"(.+\s*)", 357 | ) 358 | 359 | self.assertParseArgsRaises( 360 | "invalid choice: 'ZZZ'", 361 | args="--genome hg19 -g %s --format ZZZ f.vcf" % config_file2.name, 362 | ) 363 | self.assertParseArgsRaises( 364 | "unrecognized arguments: --bla", 365 | args="--bla --genome hg19 -g %s f.vcf" % config_file2.name, 366 | ) 367 | 368 | default_config_file.close() 369 | config_file2.close() 370 | 371 | def testBasicCase2_WithGroups(self): 372 | self.testBasicCase2(use_groups=True) 373 | 374 | def testCustomOpenFunction(self): 375 | expected_output = "dummy open called" 376 | 377 | def dummy_open(p): 378 | print(expected_output) 379 | return open(p) 380 | 381 | self.initParser(config_file_open_func=dummy_open) 382 | self.add_arg("--config", is_config_file=True) 383 | self.add_arg("--arg1", default=1, type=int) 384 | 385 | with tempfile.NamedTemporaryFile(mode="w", delete=False) as config_file: 386 | config_file.write("arg1 2") 387 | config_file_path = config_file.name 388 | 389 | with captured_output() as (out, _): 390 | args = self.parse("--config {}".format(config_file_path)) 391 | self.assertTrue(hasattr(args, "arg1")) 392 | self.assertEqual(args.arg1, 2) 393 | output = out.getvalue().strip() 394 | self.assertEqual(output, expected_output) 395 | 396 | def testIgnoreHelpArgs(self): 397 | p = self.initParser() 398 | self.add_arg("--arg1") 399 | args, _ = self.parse_known("--arg2 --help", ignore_help_args=True) 400 | self.assertEqual(args.arg1, None) 401 | self.add_arg("--arg2") 402 | args, _ = self.parse_known("--arg2 3 --help", ignore_help_args=True) 403 | self.assertEqual(args.arg2, "3") 404 | self.assertRaisesRegex( 405 | TypeError, 406 | "exit", 407 | self.parse_known, 408 | "--arg2 3 --help", 409 | ignore_help_args=False, 410 | ) 411 | 412 | def testPositionalAndConfigVarLists(self): 413 | self.initParser() 414 | self.add_arg("a") 415 | self.add_arg("-x", "--arg", nargs="+") 416 | 417 | ns = self.parse( 418 | "positional_value", 419 | config_file_contents="""arg = [Shell, someword, anotherword]""", 420 | ) 421 | 422 | self.assertEqual(ns.arg, ["Shell", "someword", "anotherword"]) 423 | self.assertEqual(ns.a, "positional_value") 424 | 425 | def testMutuallyExclusiveArgs(self): 426 | config_file = tempfile.NamedTemporaryFile(mode="w", delete=False) 427 | 428 | p = self.parser 429 | g = p.add_argument_group(title="group1") 430 | g.add_arg("--genome", help="Path to genome file", required=True) 431 | g.add_arg("-v", dest="verbose", action="store_true") 432 | 433 | g = p.add_mutually_exclusive_group(required=True) 434 | g.add_arg("-f1", "--type1-cfg-file", is_config_file=True) 435 | g.add_arg("-f2", "--type2-cfg-file", is_config_file=True) 436 | 437 | g = p.add_mutually_exclusive_group(required=True) 438 | g.add_arg( 439 | "-f", 440 | "--format", 441 | choices=["BED", "MAF", "VCF", "WIG", "R"], 442 | dest="fmt", 443 | metavar="FRMT", 444 | env_var="OUTPUT_FORMAT", 445 | default="BED", 446 | ) 447 | g.add_arg( 448 | "-b", 449 | "--bam", 450 | dest="fmt", 451 | action="store_const", 452 | const="BAM", 453 | env_var="BAM_FORMAT", 454 | ) 455 | 456 | ns = self.parse(args="--genome hg19 -f1 %s --bam" % config_file.name) 457 | self.assertEqual(ns.genome, "hg19") 458 | self.assertEqual(ns.verbose, False) 459 | self.assertEqual(ns.fmt, "BAM") 460 | 461 | ns = self.parse( 462 | env_vars={"BAM_FORMAT": "true"}, 463 | args="--genome hg19 -f1 %s" % config_file.name, 464 | ) 465 | self.assertEqual(ns.genome, "hg19") 466 | self.assertEqual(ns.verbose, False) 467 | self.assertEqual(ns.fmt, "BAM") 468 | self.assertRegex( 469 | self.format_values(), 470 | "Command Line Args: --genome hg19 -f1 [^\\s]+\n" 471 | "Environment Variables:\n" 472 | " BAM_FORMAT: \\s+ true\n" 473 | "Defaults:\n" 474 | " --format: \\s+ BED\n", 475 | ) 476 | 477 | self.assertRegex( 478 | self.format_help(), 479 | r"usage: .* \[-h\] --genome GENOME \[-v\]\s+\(-f1 TYPE1_CFG_FILE \|" 480 | r"\s+-f2 TYPE2_CFG_FILE\)\s+\(-f FRMT \| -b\)\n\n" 481 | "%s:\n" 482 | " -h, --help show this help message and exit\n" 483 | " -f1(?: TYPE1_CFG_FILE)?, --type1-cfg-file TYPE1_CFG_FILE\n" 484 | " -f2(?: TYPE2_CFG_FILE)?, --type2-cfg-file TYPE2_CFG_FILE\n" 485 | " -f(?: FRMT)?, --format FRMT\\s+\\[env var: OUTPUT_FORMAT\\]\n" 486 | " -b, --bam\\s+\\[env var: BAM_FORMAT\\]\n\n" 487 | "group1:\n" 488 | " --genome GENOME Path to genome file\n" 489 | " -v\n\n" % OPTIONAL_ARGS_STRING, 490 | ) 491 | config_file.close() 492 | 493 | def testSubParsers(self): 494 | config_file1 = tempfile.NamedTemporaryFile(mode="w", delete=False) 495 | config_file1.write("--i = B") 496 | config_file1.flush() 497 | 498 | config_file2 = tempfile.NamedTemporaryFile(mode="w", delete=False) 499 | config_file2.write("p = 10") 500 | config_file2.flush() 501 | 502 | parser = configargparse.ArgumentParser(prog="myProg") 503 | subparsers = parser.add_subparsers(title="actions") 504 | 505 | parent_parser = configargparse.ArgumentParser(add_help=False) 506 | parent_parser.add_argument( 507 | "-p", "--p", type=int, required=True, help="set db parameter" 508 | ) 509 | 510 | create_p = subparsers.add_parser( 511 | "create", parents=[parent_parser], help="create the orbix environment" 512 | ) 513 | create_p.add_argument("--i", env_var="INIT", choices=["A", "B"], default="A") 514 | create_p.add_argument("-config", is_config_file=True) 515 | 516 | update_p = subparsers.add_parser( 517 | "update", parents=[parent_parser], help="update the orbix environment" 518 | ) 519 | update_p.add_argument("-config2", is_config_file=True, required=True) 520 | 521 | ns = parser.parse_args(args="create -p 2 -config " + config_file1.name) 522 | self.assertEqual(ns.p, 2) 523 | self.assertEqual(ns.i, "B") 524 | 525 | ns = parser.parse_args(args="update -config2 " + config_file2.name) 526 | self.assertEqual(ns.p, 10) 527 | config_file1.close() 528 | config_file2.close() 529 | 530 | def testAddArgsErrors(self): 531 | self.assertRaisesRegex( 532 | ValueError, 533 | "arg with " 534 | "is_write_out_config_file_arg=True can't also have " 535 | "is_config_file_arg=True", 536 | self.add_arg, 537 | "-x", 538 | "--X", 539 | is_config_file=True, 540 | is_write_out_config_file_arg=True, 541 | ) 542 | self.assertRaisesRegex( 543 | ValueError, 544 | "arg with " "is_write_out_config_file_arg=True must have action='store'", 545 | self.add_arg, 546 | "-y", 547 | "--Y", 548 | action="append", 549 | is_write_out_config_file_arg=True, 550 | ) 551 | 552 | def testConfigFileSyntax(self): 553 | self.add_arg("-x", required=True, type=int) 554 | self.add_arg("--y", required=True, type=float) 555 | self.add_arg("--z") 556 | self.add_arg("--c") 557 | self.add_arg("--b", action="store_true") 558 | self.add_arg("--a", action="append", type=int) 559 | self.add_arg( 560 | "--m", 561 | action="append", 562 | nargs=3, 563 | metavar=("", "", ""), 564 | ) 565 | 566 | ns = self.parse( 567 | args="-x 1", 568 | env_vars={}, 569 | config_file_contents=""" 570 | 571 | #inline comment 1 572 | # inline comment 2 573 | # inline comment 3 574 | ;inline comment 4 575 | ; inline comment 5 576 | ;inline comment 6 577 | 578 | --- # separator 1 579 | ------------- # separator 2 580 | 581 | y=1.1 582 | y = 2.1 583 | y= 3.1 # with comment 584 | y= 4.1 ; with comment 585 | --- 586 | y:5.1 587 | y : 6.1 588 | y: 7.1 # with comment 589 | y: 8.1 ; with comment 590 | --- 591 | y \t 9.1 592 | y 10.1 593 | y 11.1 # with comment 594 | y 12.1 ; with comment 595 | --- 596 | b 597 | b = True 598 | b: True 599 | ---- 600 | a = 33 601 | --- 602 | z z 1 603 | --- 604 | m = [[1, 2, 3], [4, 5, 6]] 605 | """, 606 | ) 607 | 608 | self.assertEqual(ns.x, 1) 609 | self.assertEqual(ns.y, 12.1) 610 | self.assertEqual(ns.z, "z 1") 611 | self.assertIsNone(ns.c) 612 | self.assertEqual(ns.b, True) 613 | self.assertEqual(ns.a, [33]) 614 | self.assertRegex( 615 | self.format_values(), 616 | "Command Line Args: \\s+ -x 1\n" 617 | "Config File \\(method arg\\):\n" 618 | " y: \\s+ 12.1\n" 619 | " b: \\s+ True\n" 620 | " a: \\s+ 33\n" 621 | " z: \\s+ z 1\n", 622 | ) 623 | self.assertEqual(ns.m, [["1", "2", "3"], ["4", "5", "6"]]) 624 | 625 | # -x is not a long arg so can't be set via config file 626 | self.assertParseArgsRaises( 627 | ( 628 | "argument -x is required" 629 | if sys.version_info.major < 3 630 | else "the following arguments are required: -x, --y" 631 | ), 632 | args="", 633 | config_file_contents="-x 3", 634 | ) 635 | self.assertParseArgsRaises( 636 | "invalid float value: 'abc'", args="-x 5", config_file_contents="y: abc" 637 | ) 638 | self.assertParseArgsRaises( 639 | ( 640 | "argument --y is required" 641 | if sys.version_info.major < 3 642 | else "the following arguments are required: --y" 643 | ), 644 | args="-x 5", 645 | config_file_contents="z: 1", 646 | ) 647 | 648 | # test unknown config file args 649 | self.assertParseArgsRaises( 650 | "bla", args="-x 1 --y 2.3", config_file_contents="bla=3" 651 | ) 652 | 653 | ns, args = self.parse_known( 654 | "-x 10 --y 3.8", config_file_contents="bla=3", env_vars={"bla": "2"} 655 | ) 656 | self.assertListEqual(args, ["--bla=3"]) 657 | 658 | self.initParser(ignore_unknown_config_file_keys=False) 659 | ns, args = self.parse_known( 660 | args="-x 1", config_file_contents="bla=3", env_vars={"bla": "2"} 661 | ) 662 | self.assertEqual(set(args), {"--bla=3", "-x", "1"}) 663 | 664 | def testQuotedArgumentValues(self): 665 | self.initParser() 666 | self.add_arg("-a") 667 | self.add_arg("--b") 668 | self.add_arg("-c") 669 | self.add_arg("--d") 670 | self.add_arg("-e") 671 | self.add_arg("-q") 672 | self.add_arg("--quotes") 673 | 674 | # sys.argv equivalent of -a="1" --b "1" -c= --d "" -e=: -q "\"'" --quotes "\"'" 675 | ns = self.parse( 676 | args=[ 677 | "-a=1", 678 | "--b", 679 | "1", 680 | "-c=", 681 | "--d", 682 | "", 683 | "-e=:", 684 | "-q", 685 | "\"'", 686 | "--quotes", 687 | "\"'", 688 | ], 689 | env_vars={}, 690 | config_file_contents="", 691 | ) 692 | 693 | self.assertEqual(ns.a, "1") 694 | self.assertEqual(ns.b, "1") 695 | self.assertEqual(ns.c, "") 696 | self.assertEqual(ns.d, "") 697 | self.assertEqual(ns.e, ":") 698 | self.assertEqual(ns.q, "\"'") 699 | self.assertEqual(ns.quotes, "\"'") 700 | 701 | def testQuotedConfigFileValues(self): 702 | self.initParser() 703 | self.add_arg("--a") 704 | self.add_arg("--b") 705 | self.add_arg("--c") 706 | 707 | ns = self.parse( 708 | args="", 709 | env_vars={}, 710 | config_file_contents=""" 711 | a="1" 712 | b=: 713 | c= 714 | """, 715 | ) 716 | 717 | self.assertEqual(ns.a, "1") 718 | self.assertEqual(ns.b, ":") 719 | self.assertEqual(ns.c, "") 720 | 721 | def testBooleanValuesCanBeExpressedAsNumbers(self): 722 | self.initParser() 723 | store_true_env_var_name = "STORE_TRUE" 724 | self.add_arg( 725 | "--boolean_store_true", action="store_true", env_var=store_true_env_var_name 726 | ) 727 | 728 | result_namespace = self.parse( 729 | "", config_file_contents="""boolean_store_true = 1""" 730 | ) 731 | self.assertTrue(result_namespace.boolean_store_true) 732 | 733 | result_namespace = self.parse( 734 | "", config_file_contents="""boolean_store_true = 0""" 735 | ) 736 | self.assertFalse(result_namespace.boolean_store_true) 737 | 738 | result_namespace = self.parse("", env_vars={store_true_env_var_name: "1"}) 739 | self.assertTrue(result_namespace.boolean_store_true) 740 | 741 | result_namespace = self.parse("", env_vars={store_true_env_var_name: "0"}) 742 | self.assertFalse(result_namespace.boolean_store_true) 743 | 744 | self.initParser() 745 | store_false_env_var_name = "STORE_FALSE" 746 | self.add_arg( 747 | "--boolean_store_false", 748 | action="store_false", 749 | env_var=store_false_env_var_name, 750 | ) 751 | 752 | result_namespace = self.parse( 753 | "", config_file_contents="""boolean_store_false = 1""" 754 | ) 755 | self.assertFalse(result_namespace.boolean_store_false) 756 | 757 | result_namespace = self.parse( 758 | "", config_file_contents="""boolean_store_false = 0""" 759 | ) 760 | self.assertTrue(result_namespace.boolean_store_false) 761 | 762 | result_namespace = self.parse("", env_vars={store_false_env_var_name: "1"}) 763 | self.assertFalse(result_namespace.boolean_store_false) 764 | 765 | result_namespace = self.parse("", env_vars={store_false_env_var_name: "0"}) 766 | self.assertTrue(result_namespace.boolean_store_false) 767 | 768 | def testConfigOrEnvValueErrors(self): 769 | # error should occur when a flag arg is set to something other than "true" or "false" 770 | self.initParser() 771 | self.add_arg("--height", env_var="HEIGHT", required=True) 772 | self.add_arg("--do-it", dest="x", env_var="FLAG1", action="store_true") 773 | self.add_arg("--dont-do-it", dest="y", env_var="FLAG2", action="store_false") 774 | ns = self.parse("", env_vars={"HEIGHT": "tall", "FLAG1": "yes"}) 775 | self.assertEqual(ns.height, "tall") 776 | self.assertEqual(ns.x, True) 777 | ns = self.parse("", env_vars={"HEIGHT": "tall", "FLAG2": "yes"}) 778 | self.assertEqual(ns.y, False) 779 | ns = self.parse("", env_vars={"HEIGHT": "tall", "FLAG2": "no"}) 780 | self.assertEqual(ns.y, True) 781 | 782 | # error should occur when flag arg is given a value 783 | self.initParser() 784 | self.add_arg("-v", "--verbose", env_var="VERBOSE", action="store_true") 785 | self.assertParseArgsRaises( 786 | "Unexpected value for VERBOSE: 'bla'. " 787 | "Expecting 'true', 'false', 'yes', 'no', 'on', 'off', '1' or '0'", 788 | args="", 789 | env_vars={"VERBOSE": "bla"}, 790 | ) 791 | ns = self.parse( 792 | "", config_file_contents="verbose=true", env_vars={"HEIGHT": "true"} 793 | ) 794 | self.assertEqual(ns.verbose, True) 795 | ns = self.parse("", config_file_contents="verbose", env_vars={"HEIGHT": "true"}) 796 | self.assertEqual(ns.verbose, True) 797 | ns = self.parse("", env_vars={"HEIGHT": "true", "VERBOSE": "true"}) 798 | self.assertEqual(ns.verbose, True) 799 | ns = self.parse("", env_vars={"HEIGHT": "true", "VERBOSE": "false"}) 800 | self.assertEqual(ns.verbose, False) 801 | ns = self.parse( 802 | "", config_file_contents="--verbose", env_vars={"HEIGHT": "true"} 803 | ) 804 | self.assertEqual(ns.verbose, True) 805 | 806 | # error should occur is non-append arg is given a list value 807 | self.initParser() 808 | self.add_arg("-f", "--file", env_var="FILES", action="append", type=int) 809 | ns = self.parse("", env_vars={"file": "[1,2,3]", "VERBOSE": "true"}) 810 | self.assertIsNone(ns.file) 811 | 812 | def testValuesStartingWithDash(self): 813 | self.initParser() 814 | self.add_arg("--arg0") 815 | self.add_arg("--arg1", env_var="ARG1") 816 | self.add_arg("--arg2") 817 | self.add_arg("--arg3", action="append") 818 | self.add_arg("--arg4", action="append", env_var="ARG4") 819 | self.add_arg("--arg5", action="append") 820 | self.add_arg("--arg6") 821 | 822 | ns = self.parse( 823 | "--arg0=-foo --arg3=-foo --arg3=-bar --arg6=-test-more-dashes", 824 | config_file_contents="arg2: -foo\narg5: [-foo, -bar]", 825 | env_vars={"ARG1": "-foo", "ARG4": "[-foo, -bar]"}, 826 | ) 827 | self.assertEqual(ns.arg0, "-foo") 828 | self.assertEqual(ns.arg1, "-foo") 829 | self.assertEqual(ns.arg2, "-foo") 830 | self.assertEqual(ns.arg3, ["-foo", "-bar"]) 831 | self.assertEqual(ns.arg4, ["-foo", "-bar"]) 832 | self.assertEqual(ns.arg5, ["-foo", "-bar"]) 833 | self.assertEqual(ns.arg6, "-test-more-dashes") 834 | 835 | def testPriorityKnown(self): 836 | self.initParser() 837 | self.add_arg("--arg", env_var="ARG") 838 | 839 | ns = self.parse( 840 | "--arg command_line_val", 841 | config_file_contents="arg: config_val", 842 | env_vars={"ARG": "env_val"}, 843 | ) 844 | self.assertEqual(ns.arg, "command_line_val") 845 | 846 | ns = self.parse( 847 | "--arg=command_line_val", 848 | config_file_contents="arg: config_val", 849 | env_vars={"ARG": "env_val"}, 850 | ) 851 | self.assertEqual(ns.arg, "command_line_val") 852 | 853 | ns = self.parse( 854 | "", config_file_contents="arg: config_val", env_vars={"ARG": "env_val"} 855 | ) 856 | self.assertEqual(ns.arg, "env_val") 857 | 858 | def testPriorityUnknown(self): 859 | self.initParser() 860 | 861 | ns, args = self.parse_known( 862 | "--arg command_line_val", 863 | config_file_contents="arg: config_val", 864 | env_vars={"arg": "env_val"}, 865 | ) 866 | self.assertListEqual(args, ["--arg", "command_line_val"]) 867 | 868 | ns, args = self.parse_known( 869 | "--arg=command_line_val", 870 | config_file_contents="arg: config_val", 871 | ) 872 | self.assertListEqual(args, ["--arg=command_line_val"]) 873 | 874 | def testAutoEnvVarPrefix(self): 875 | self.initParser(auto_env_var_prefix="TEST_") 876 | self.add_arg("-a", "--arg0", is_config_file_arg=True) 877 | self.add_arg("-b", "--arg1", is_write_out_config_file_arg=True) 878 | self.add_arg("-x", "--arg2", env_var="TEST2", type=int) 879 | self.add_arg("-y", "--arg3", action="append", type=int) 880 | self.add_arg("-z", "--arg4", required=True) 881 | self.add_arg("-w", "--arg4-more", required=True) 882 | ns = self.parse( 883 | "", 884 | env_vars={ 885 | "TEST_ARG0": "0", 886 | "TEST_ARG1": "1", 887 | "TEST_ARG2": "2", 888 | "TEST2": "22", 889 | "TEST_ARG4": "arg4_value", 890 | "TEST_ARG4_MORE": "magic", 891 | }, 892 | ) 893 | self.assertIsNone(ns.arg0) 894 | self.assertIsNone(ns.arg1) 895 | self.assertEqual(ns.arg2, 22) 896 | self.assertEqual(ns.arg4, "arg4_value") 897 | self.assertEqual(ns.arg4_more, "magic") 898 | 899 | def testEnvVarLists(self): 900 | self.initParser() 901 | self.add_arg("-x", "--arg2", env_var="TEST2") 902 | self.add_arg("-y", "--arg3", env_var="TEST3", type=int) 903 | self.add_arg("-z", "--arg4", env_var="TEST4", nargs="+") 904 | self.add_arg("-u", "--arg5", env_var="TEST5", nargs="+", type=int) 905 | self.add_arg("--arg6", env_var="TEST6") 906 | self.add_arg("--arg7", env_var="TEST7", action="append") 907 | ns = self.parse( 908 | "", 909 | env_vars={ 910 | "TEST2": "22", 911 | "TEST3": "22", 912 | "TEST4": "[Shell, someword, anotherword]", 913 | "TEST5": "[22, 99, 33]", 914 | "TEST6": "[value6.1, value6.2, value6.3]", 915 | "TEST7": "[value7.1, value7.2, value7.3]", 916 | }, 917 | ) 918 | self.assertEqual(ns.arg2, "22") 919 | self.assertEqual(ns.arg3, 22) 920 | self.assertEqual(ns.arg4, ["Shell", "someword", "anotherword"]) 921 | self.assertEqual(ns.arg5, [22, 99, 33]) 922 | self.assertEqual(ns.arg6, "[value6.1, value6.2, value6.3]") 923 | self.assertEqual(ns.arg7, ["value7.1", "value7.2", "value7.3"]) 924 | 925 | def testPositionalAndEnvVarLists(self): 926 | self.initParser() 927 | self.add_arg("a") 928 | self.add_arg("-x", "--arg", env_var="TEST", nargs="+") 929 | 930 | ns = self.parse( 931 | "positional_value", env_vars={"TEST": "[Shell, someword, anotherword]"} 932 | ) 933 | 934 | self.assertEqual(ns.arg, ["Shell", "someword", "anotherword"]) 935 | self.assertEqual(ns.a, "positional_value") 936 | 937 | def testCounterCommandLine(self): 938 | self.initParser() 939 | self.add_arg("--verbose", "-v", action="count", default=0) 940 | 941 | ns = self.parse(args="-v -v -v", env_vars={}) 942 | self.assertEqual(ns.verbose, 3) 943 | 944 | ns = self.parse(args="-vvv", env_vars={}) 945 | self.assertEqual(ns.verbose, 3) 946 | 947 | def testCounterConfigFile(self): 948 | self.initParser() 949 | self.add_arg("--verbose", "-v", action="count", default=0) 950 | 951 | ns = self.parse( 952 | args="", 953 | env_vars={}, 954 | config_file_contents=""" 955 | verbose""", 956 | ) 957 | self.assertEqual(ns.verbose, 1) 958 | 959 | ns = self.parse( 960 | args="", 961 | env_vars={}, 962 | config_file_contents=""" 963 | verbose=3""", 964 | ) 965 | self.assertEqual(ns.verbose, 3) 966 | 967 | 968 | class TestMisc(TestCase): 969 | # TODO test different action types with config file, env var 970 | 971 | """Test edge cases""" 972 | 973 | def setUp(self): 974 | self.initParser(args_for_setting_config_path=[]) 975 | 976 | @mock.patch("argparse.ArgumentParser.__init__") 977 | def testKwrgsArePassedToArgParse(self, argparse_init): 978 | kwargs_for_argparse = {"allow_abbrev": False, "whatever_other_arg": "something"} 979 | 980 | parser = configargparse.ArgumentParser( 981 | add_config_file_help=False, **kwargs_for_argparse 982 | ) 983 | 984 | argparse_init.assert_called_with(parser, **kwargs_for_argparse) 985 | 986 | def testGlobalInstances(self, name=None): 987 | p = configargparse.getArgumentParser(name, prog="prog", usage="test") 988 | self.assertEqual(p.usage, "test") 989 | self.assertEqual(p.prog, "prog") 990 | self.assertRaisesRegex( 991 | ValueError, 992 | "kwargs besides 'name' can only be " "passed in the first time", 993 | configargparse.getArgumentParser, 994 | name, 995 | prog="prog", 996 | ) 997 | 998 | p2 = configargparse.getArgumentParser(name) 999 | self.assertEqual(p, p2) 1000 | 1001 | def testGlobalInstances_WithName(self): 1002 | self.testGlobalInstances("name1") 1003 | self.testGlobalInstances("name2") 1004 | 1005 | def testAddArguments_ArgValidation(self): 1006 | self.assertRaises(ValueError, self.add_arg, "positional", env_var="bla") 1007 | action = self.add_arg("positional") 1008 | self.assertIsNotNone(action) 1009 | self.assertEqual(action.dest, "positional") 1010 | 1011 | def testAddArguments_IsConfigFilePathArg(self): 1012 | self.assertRaises( 1013 | ValueError, self.add_arg, "c", action="store_false", is_config_file=True 1014 | ) 1015 | 1016 | self.add_arg("-c", "--config", is_config_file=True) 1017 | self.add_arg("--x", required=True) 1018 | 1019 | # verify parsing from config file 1020 | config_file = tempfile.NamedTemporaryFile(mode="w", delete=False) 1021 | config_file.write("x=bla") 1022 | config_file.flush() 1023 | 1024 | ns = self.parse(args="-c %s" % config_file.name) 1025 | self.assertEqual(ns.x, "bla") 1026 | 1027 | def testConstructor_ConfigFileArgs(self): 1028 | # Test constructor args: 1029 | # args_for_setting_config_path 1030 | # config_arg_is_required 1031 | # config_arg_help_message 1032 | temp_cfg = tempfile.NamedTemporaryFile(mode="w", delete=False) 1033 | temp_cfg.write("genome=hg19") 1034 | temp_cfg.flush() 1035 | 1036 | self.initParser( 1037 | args_for_setting_config_path=["-c", "--config"], 1038 | config_arg_is_required=True, 1039 | config_arg_help_message="my config file", 1040 | default_config_files=[temp_cfg.name], 1041 | ) 1042 | self.add_arg("--genome", help="Path to genome file", required=True) 1043 | self.assertParseArgsRaises( 1044 | ( 1045 | "argument -c/--config is required" 1046 | if sys.version_info.major < 3 1047 | else "arguments are required: -c/--config" 1048 | ), 1049 | args="", 1050 | ) 1051 | 1052 | temp_cfg2 = tempfile.NamedTemporaryFile(mode="w", delete=False) 1053 | ns = self.parse("-c " + temp_cfg2.name) 1054 | self.assertEqual(ns.genome, "hg19") 1055 | 1056 | # temp_cfg2 config file should override default config file values 1057 | temp_cfg2.write("genome=hg20") 1058 | temp_cfg2.flush() 1059 | ns = self.parse("-c " + temp_cfg2.name) 1060 | self.assertEqual(ns.genome, "hg20") 1061 | 1062 | self.assertRegex( 1063 | self.format_help(), 1064 | r"usage: .* \[-h\] -c CONFIG_FILE --genome GENOME\n\n" 1065 | r"%s:\n" 1066 | r" -h, --help\s+ show this help message and exit\n" 1067 | r" -c(?: CONFIG_FILE)?, --config CONFIG_FILE\s+ my config file\n" 1068 | r" --genome GENOME\s+ Path to genome file\n\n" % OPTIONAL_ARGS_STRING 1069 | + 5 * r"(.+\s*)", 1070 | ) 1071 | 1072 | # just run print_values() to make sure it completes and returns None 1073 | output = StringIO() 1074 | self.assertIsNone(self.parser.print_values(file=output)) 1075 | self.assertIn("Command Line Args:", output.getvalue()) 1076 | 1077 | # test ignore_unknown_config_file_keys=False 1078 | self.initParser(ignore_unknown_config_file_keys=False) 1079 | self.assertRaisesRegex( 1080 | argparse.ArgumentError, 1081 | "unrecognized arguments", 1082 | self.parse, 1083 | config_file_contents="arg1 = 3", 1084 | ) 1085 | ns, args = self.parse_known(config_file_contents="arg1 = 3") 1086 | self.assertEqual(getattr(ns, "arg1", ""), "") 1087 | 1088 | # test ignore_unknown_config_file_keys=True 1089 | self.initParser(ignore_unknown_config_file_keys=True) 1090 | ns = self.parse(args="", config_file_contents="arg1 = 3") 1091 | self.assertEqual(getattr(ns, "arg1", ""), "") 1092 | ns, args = self.parse_known(config_file_contents="arg1 = 3") 1093 | self.assertEqual(getattr(ns, "arg1", ""), "") 1094 | 1095 | def test_AbbrevConfigFileArgs(self): 1096 | """Tests that abbreviated values don't get pulled from config file.""" 1097 | temp_cfg = tempfile.NamedTemporaryFile(mode="w", delete=False) 1098 | temp_cfg.write("a2a = 0.5\n") 1099 | temp_cfg.write("a3a = 0.5\n") 1100 | temp_cfg.flush() 1101 | 1102 | self.initParser() 1103 | 1104 | self.add_arg( 1105 | "-c", 1106 | "--config_file", 1107 | required=False, 1108 | is_config_file=True, 1109 | help="config file path", 1110 | ) 1111 | 1112 | self.add_arg("--hello", type=int, required=False) 1113 | 1114 | command = "-c {} --hello 2".format(temp_cfg.name) 1115 | 1116 | known, unknown = self.parse_known(command) 1117 | 1118 | self.assertListEqual(unknown, ["--a2a=0.5", "--a3a=0.5"]) 1119 | 1120 | def test_FormatHelp(self): 1121 | self.initParser( 1122 | args_for_setting_config_path=["-c", "--config"], 1123 | config_arg_is_required=True, 1124 | config_arg_help_message="my config file", 1125 | default_config_files=["~/.myconfig"], 1126 | args_for_writing_out_config_file=["-w", "--write-config"], 1127 | ) 1128 | self.add_arg("--arg1", help="Arg1 help text", required=True) 1129 | self.add_arg("--flag", help="Flag help text", action="store_true") 1130 | 1131 | self.assertRegex( 1132 | self.format_help(), 1133 | r"usage: .* \[-h\] -c CONFIG_FILE\s+" 1134 | r"\[-w CONFIG_OUTPUT_PATH\]\s* --arg1\s+ARG1\s*\[--flag\]\s*" 1135 | "%s:\\s*" 1136 | "-h, --help \\s* show this help message and exit " 1137 | r"-c(?: CONFIG_FILE)?, --config CONFIG_FILE\s+my config file " 1138 | r"-w(?: CONFIG_OUTPUT_PATH)?, --write-config CONFIG_OUTPUT_PATH takes " 1139 | r"the current command line args and writes them " 1140 | r"out to a config file at the given path, then exits " 1141 | r"--arg1 ARG1 Arg1 help text " 1142 | r"--flag Flag help text " 1143 | "Args that start with '--' can also be set in a " 1144 | r"config file \(~/.myconfig or specified via -c\). " 1145 | r"Config file syntax allows: key=value, flag=true, stuff=\[a,b,c\] " 1146 | r"\(for details, see syntax at https://goo.gl/R74nmi\). " 1147 | r"In general, command-line values override config file values " 1148 | r"which override defaults. ".replace(" ", r"\s*") % OPTIONAL_ARGS_STRING, 1149 | ) 1150 | 1151 | def test_FormatHelpProg(self): 1152 | self.initParser("format_help_prog") 1153 | self.assertRegex(self.format_help(), "usage: format_help_prog .*") 1154 | 1155 | def test_FormatHelpProgLib(self): 1156 | parser = argparse.ArgumentParser("format_help_prog") 1157 | self.assertRegex(parser.format_help(), "usage: format_help_prog .*") 1158 | 1159 | class CustomClass(object): 1160 | def __init__(self, name): 1161 | self.name = name 1162 | 1163 | def __str__(self): 1164 | return self.name 1165 | 1166 | @staticmethod 1167 | def valid_custom(s): 1168 | if s == "invalid": 1169 | raise Exception("invalid name") 1170 | return TestMisc.CustomClass(s) 1171 | 1172 | def testConstructor_WriteOutConfigFileArgs(self): 1173 | # Test constructor args: 1174 | # args_for_writing_out_config_file 1175 | # write_out_config_file_arg_help_message 1176 | cfg_f = tempfile.NamedTemporaryFile(mode="w+", delete=False) 1177 | self.initParser( 1178 | args_for_writing_out_config_file=["-w"], 1179 | write_out_config_file_arg_help_message="write config", 1180 | ) 1181 | 1182 | self.add_arg("-not-config-file-settable") 1183 | self.add_arg("--config-file-settable-arg", type=int) 1184 | self.add_arg("--config-file-settable-arg2", type=int, default=3) 1185 | self.add_arg("--config-file-settable-flag", action="store_true") 1186 | self.add_arg("--config-file-settable-custom", type=TestMisc.valid_custom) 1187 | self.add_arg("-l", "--config-file-settable-list", action="append") 1188 | 1189 | # write out a config file 1190 | command_line_args = "-w %s " % cfg_f.name 1191 | command_line_args += "--config-file-settable-arg 1 " 1192 | command_line_args += "--config-file-settable-flag " 1193 | command_line_args += "--config-file-settable-custom custom_value " 1194 | command_line_args += "-l a -l b -l c -l d " 1195 | 1196 | self.assertFalse(self.parser._exit_method_called) 1197 | 1198 | ns = self.parse(command_line_args) 1199 | self.assertTrue(self.parser._exit_method_called) 1200 | 1201 | cfg_f.seek(0) 1202 | expected_config_file_contents = "config-file-settable-arg = 1\n" 1203 | expected_config_file_contents += "config-file-settable-flag = true\n" 1204 | expected_config_file_contents += "config-file-settable-custom = custom_value\n" 1205 | expected_config_file_contents += "config-file-settable-list = [a, b, c, d]\n" 1206 | expected_config_file_contents += "config-file-settable-arg2 = 3\n" 1207 | 1208 | self.assertEqual(cfg_f.read().strip(), expected_config_file_contents.strip()) 1209 | self.assertRaisesRegex( 1210 | ValueError, 1211 | "Couldn't open / for writing:", 1212 | self.parse, 1213 | args=command_line_args + " -w /", 1214 | ) 1215 | cfg_f.close() 1216 | 1217 | def testConstructor_WriteOutConfigFileArgs2(self): 1218 | # Test constructor args: 1219 | # args_for_writing_out_config_file 1220 | # write_out_config_file_arg_help_message 1221 | cfg_f = tempfile.NamedTemporaryFile(mode="w+", delete=False) 1222 | self.initParser( 1223 | args_for_writing_out_config_file=["-w"], 1224 | write_out_config_file_arg_help_message="write config", 1225 | ) 1226 | 1227 | self.add_arg("-not-config-file-settable") 1228 | self.add_arg("-a", "--arg1", type=int, env_var="ARG1") 1229 | self.add_arg("-b", "--arg2", type=int, default=3) 1230 | self.add_arg("-c", "--arg3") 1231 | self.add_arg("-d", "--arg4") 1232 | self.add_arg("-e", "--arg5") 1233 | self.add_arg( 1234 | "--config-file-settable-flag", action="store_true", env_var="FLAG_ARG" 1235 | ) 1236 | self.add_arg("-l", "--config-file-settable-list", action="append") 1237 | 1238 | # write out a config file 1239 | command_line_args = "-w %s " % cfg_f.name 1240 | command_line_args += "-l a -l b -l c -l d " 1241 | 1242 | self.assertFalse(self.parser._exit_method_called) 1243 | 1244 | ns = self.parse( 1245 | command_line_args, 1246 | env_vars={"ARG1": "10", "FLAG_ARG": "true", "SOME_OTHER_ENV_VAR": "2"}, 1247 | config_file_contents="arg3 = bla3\narg4 = bla4", 1248 | ) 1249 | self.assertTrue(self.parser._exit_method_called) 1250 | 1251 | cfg_f.seek(0) 1252 | expected_config_file_contents = "config-file-settable-list = [a, b, c, d]\n" 1253 | expected_config_file_contents += "arg1 = 10\n" 1254 | expected_config_file_contents += "config-file-settable-flag = True\n" 1255 | expected_config_file_contents += "arg3 = bla3\n" 1256 | expected_config_file_contents += "arg4 = bla4\n" 1257 | expected_config_file_contents += "arg2 = 3\n" 1258 | 1259 | self.assertEqual(cfg_f.read().strip(), expected_config_file_contents.strip()) 1260 | self.assertRaisesRegex( 1261 | ValueError, 1262 | "Couldn't open / for writing:", 1263 | self.parse, 1264 | args=command_line_args + " -w /", 1265 | ) 1266 | cfg_f.close() 1267 | 1268 | def testConstructor_WriteOutConfigFileArgsLong(self): 1269 | """Test config writing with long version of arg 1270 | 1271 | There was a bug where the long version of the 1272 | args_for_writing_out_config_file was being dumped into the resultant 1273 | output config file 1274 | """ 1275 | # Test constructor args: 1276 | # args_for_writing_out_config_file 1277 | # write_out_config_file_arg_help_message 1278 | cfg_f = tempfile.NamedTemporaryFile(mode="w+", delete=False) 1279 | self.initParser( 1280 | args_for_writing_out_config_file=["--write-config"], 1281 | write_out_config_file_arg_help_message="write config", 1282 | ) 1283 | 1284 | self.add_arg("-not-config-file-settable") 1285 | self.add_arg("--config-file-settable-arg", type=int) 1286 | self.add_arg("--config-file-settable-arg2", type=int, default=3) 1287 | self.add_arg("--config-file-settable-flag", action="store_true") 1288 | self.add_arg("-l", "--config-file-settable-list", action="append") 1289 | 1290 | # write out a config file 1291 | command_line_args = "--write-config %s " % cfg_f.name 1292 | command_line_args += "--config-file-settable-arg 1 " 1293 | command_line_args += "--config-file-settable-flag " 1294 | command_line_args += "-l a -l b -l c -l d " 1295 | 1296 | self.assertFalse(self.parser._exit_method_called) 1297 | 1298 | ns = self.parse(command_line_args) 1299 | self.assertTrue(self.parser._exit_method_called) 1300 | 1301 | cfg_f.seek(0) 1302 | expected_config_file_contents = "config-file-settable-arg = 1\n" 1303 | expected_config_file_contents += "config-file-settable-flag = true\n" 1304 | expected_config_file_contents += "config-file-settable-list = [a, b, c, d]\n" 1305 | expected_config_file_contents += "config-file-settable-arg2 = 3\n" 1306 | 1307 | self.assertEqual(cfg_f.read().strip(), expected_config_file_contents.strip()) 1308 | self.assertRaisesRegex( 1309 | ValueError, 1310 | "Couldn't open / for writing:", 1311 | self.parse, 1312 | args=command_line_args + " --write-config /", 1313 | ) 1314 | cfg_f.close() 1315 | 1316 | def testMethodAliases(self): 1317 | p = self.parser 1318 | p.add("-a", "--arg-a", default=3) 1319 | p.add_arg("-b", "--arg-b", required=True) 1320 | p.add_argument("-c") 1321 | 1322 | g1 = p.add_argument_group(title="group1", description="descr") 1323 | g1.add("-d", "--arg-d", required=True) 1324 | g1.add_arg("-e", "--arg-e", required=True) 1325 | g1.add_argument("-f", "--arg-f", default=5) 1326 | 1327 | g2 = p.add_mutually_exclusive_group(required=True) 1328 | g2.add("-x", "--arg-x") 1329 | g2.add_arg("-y", "--arg-y") 1330 | g2.add_argument("-z", "--arg-z", default=5) 1331 | 1332 | # verify that flags must be globally unique 1333 | g2 = p.add_argument_group(title="group2", description="descr") 1334 | self.assertRaises(argparse.ArgumentError, g1.add, "-c") 1335 | self.assertRaises(argparse.ArgumentError, g2.add, "-f") 1336 | 1337 | self.initParser() 1338 | p = self.parser 1339 | options = p.parse(args=[]) 1340 | self.assertDictEqual(vars(options), {}) 1341 | 1342 | def testConfigOpenFuncError(self): 1343 | # test OSError 1344 | def error_func(path): 1345 | raise OSError(9, "some error") 1346 | 1347 | self.initParser(config_file_open_func=error_func) 1348 | self.parser.add_argument("-g", is_config_file=True) 1349 | self.assertParseArgsRaises( 1350 | "Unable to open config file: file.txt. Error: some error", 1351 | args="-g file.txt", 1352 | ) 1353 | 1354 | # test other error 1355 | def error_func(path): 1356 | raise Exception("custom error") 1357 | 1358 | self.initParser(config_file_open_func=error_func) 1359 | self.parser.add_argument("-g", is_config_file=True) 1360 | self.assertParseArgsRaises( 1361 | "Unable to open config file: file.txt. Error: custom error", 1362 | args="-g file.txt", 1363 | ) 1364 | 1365 | 1366 | class TestConfigFileParsers(TestCase): 1367 | """Test ConfigFileParser subclasses in isolation""" 1368 | 1369 | def testDefaultConfigFileParser_Basic(self): 1370 | p = configargparse.DefaultConfigFileParser() 1371 | self.assertGreater(len(p.get_syntax_description()), 0) 1372 | 1373 | # test the simplest case 1374 | input_config_str = StringIO("""a: 3\n""") 1375 | parsed_obj = p.parse(input_config_str) 1376 | output_config_str = p.serialize(parsed_obj) 1377 | 1378 | self.assertEqual( 1379 | input_config_str.getvalue().replace(": ", " = "), output_config_str 1380 | ) 1381 | 1382 | self.assertDictEqual(parsed_obj, {"a": "3"}) 1383 | 1384 | def testDefaultConfigFileParser_All(self): 1385 | p = configargparse.DefaultConfigFileParser() 1386 | 1387 | # test the all syntax case 1388 | config_lines = [ 1389 | "# comment1 ", 1390 | "[ some section ]", 1391 | "----", 1392 | "---------", 1393 | "_a: 3", 1394 | "; comment2 ", 1395 | "_b = c", 1396 | "_list_arg1 = [a, b, c]", 1397 | "_str_arg = true", 1398 | "_list_arg2 = [1, 2, 3]", 1399 | ] 1400 | 1401 | # test parse 1402 | input_config_str = StringIO("\n".join(config_lines) + "\n") 1403 | parsed_obj = p.parse(input_config_str) 1404 | 1405 | # test serialize 1406 | output_config_str = p.serialize(parsed_obj) 1407 | self.assertEqual( 1408 | "\n".join(l.replace(": ", " = ") for l in config_lines if l.startswith("_")) 1409 | + "\n", 1410 | output_config_str, 1411 | ) 1412 | 1413 | self.assertDictEqual( 1414 | parsed_obj, 1415 | { 1416 | "_a": "3", 1417 | "_b": "c", 1418 | "_list_arg1": ["a", "b", "c"], 1419 | "_str_arg": "true", 1420 | "_list_arg2": [1, 2, 3], 1421 | }, 1422 | ) 1423 | 1424 | self.assertListEqual(parsed_obj["_list_arg1"], ["a", "b", "c"]) 1425 | self.assertListEqual(parsed_obj["_list_arg2"], [1, 2, 3]) 1426 | 1427 | def testDefaultConfigFileParser_BasicValues(self): 1428 | p = configargparse.DefaultConfigFileParser() 1429 | 1430 | # test the all syntax case 1431 | config_lines = [ 1432 | { 1433 | "line": "key = value # comment # comment", 1434 | "expected": ("key", "value", "comment # comment"), 1435 | }, 1436 | {"line": "key=value#comment ", "expected": ("key", "value#comment", None)}, 1437 | {"line": "key=value", "expected": ("key", "value", None)}, 1438 | {"line": "key =value", "expected": ("key", "value", None)}, 1439 | {"line": "key= value", "expected": ("key", "value", None)}, 1440 | {"line": "key = value", "expected": ("key", "value", None)}, 1441 | {"line": "key = value", "expected": ("key", "value", None)}, 1442 | {"line": " key = value ", "expected": ("key", "value", None)}, 1443 | {"line": "key:value", "expected": ("key", "value", None)}, 1444 | {"line": "key :value", "expected": ("key", "value", None)}, 1445 | {"line": "key: value", "expected": ("key", "value", None)}, 1446 | {"line": "key : value", "expected": ("key", "value", None)}, 1447 | {"line": "key : value", "expected": ("key", "value", None)}, 1448 | {"line": " key : value ", "expected": ("key", "value", None)}, 1449 | {"line": "key value", "expected": ("key", "value", None)}, 1450 | {"line": "key value", "expected": ("key", "value", None)}, 1451 | {"line": " key value ", "expected": ("key", "value", None)}, 1452 | ] 1453 | 1454 | for test in config_lines: 1455 | parsed_obj = p.parse(StringIO(test["line"])) 1456 | parsed_obj = dict(parsed_obj) 1457 | expected = {test["expected"][0]: test["expected"][1]} 1458 | self.assertDictEqual(parsed_obj, expected, msg="Line %r" % (test["line"])) 1459 | 1460 | def testDefaultConfigFileParser_QuotedValues(self): 1461 | p = configargparse.DefaultConfigFileParser() 1462 | 1463 | # test the all syntax case 1464 | config_lines = [ 1465 | {"line": 'key="value"', "expected": ("key", "value", None)}, 1466 | {"line": 'key = "value"', "expected": ("key", "value", None)}, 1467 | {"line": ' key = "value" ', "expected": ("key", "value", None)}, 1468 | {"line": 'key=" value "', "expected": ("key", " value ", None)}, 1469 | {"line": 'key = " value "', "expected": ("key", " value ", None)}, 1470 | {"line": ' key = " value " ', "expected": ("key", " value ", None)}, 1471 | {"line": "key='value'", "expected": ("key", "value", None)}, 1472 | {"line": "key = 'value'", "expected": ("key", "value", None)}, 1473 | {"line": " key = 'value' ", "expected": ("key", "value", None)}, 1474 | {"line": "key=' value '", "expected": ("key", " value ", None)}, 1475 | {"line": "key = ' value '", "expected": ("key", " value ", None)}, 1476 | {"line": " key = ' value ' ", "expected": ("key", " value ", None)}, 1477 | {"line": 'key="', "expected": ("key", '"', None)}, 1478 | {"line": 'key = "', "expected": ("key", '"', None)}, 1479 | {"line": ' key = " ', "expected": ("key", '"', None)}, 1480 | {"line": "key = '\"value\"'", "expected": ("key", '"value"', None)}, 1481 | {"line": "key = \"'value'\"", "expected": ("key", "'value'", None)}, 1482 | {"line": 'key = ""value""', "expected": ("key", '"value"', None)}, 1483 | {"line": "key = ''value''", "expected": ("key", "'value'", None)}, 1484 | {"line": 'key="value', "expected": ("key", '"value', None)}, 1485 | {"line": 'key = "value', "expected": ("key", '"value', None)}, 1486 | {"line": ' key = "value ', "expected": ("key", '"value', None)}, 1487 | {"line": 'key=value"', "expected": ("key", 'value"', None)}, 1488 | {"line": 'key = value"', "expected": ("key", 'value"', None)}, 1489 | {"line": ' key = value " ', "expected": ("key", 'value "', None)}, 1490 | {"line": "key='value", "expected": ("key", "'value", None)}, 1491 | {"line": "key = 'value", "expected": ("key", "'value", None)}, 1492 | {"line": " key = 'value ", "expected": ("key", "'value", None)}, 1493 | {"line": "key=value'", "expected": ("key", "value'", None)}, 1494 | {"line": "key = value'", "expected": ("key", "value'", None)}, 1495 | {"line": " key = value ' ", "expected": ("key", "value '", None)}, 1496 | ] 1497 | 1498 | for test in config_lines: 1499 | parsed_obj = p.parse(StringIO(test["line"])) 1500 | parsed_obj = dict(parsed_obj) 1501 | expected = {test["expected"][0]: test["expected"][1]} 1502 | self.assertDictEqual(parsed_obj, expected, msg="Line %r" % (test["line"])) 1503 | 1504 | def testDefaultConfigFileParser_BlankValues(self): 1505 | p = configargparse.DefaultConfigFileParser() 1506 | 1507 | # test the all syntax case 1508 | config_lines = [ 1509 | {"line": "key=", "expected": ("key", "", None)}, 1510 | {"line": "key =", "expected": ("key", "", None)}, 1511 | {"line": "key= ", "expected": ("key", "", None)}, 1512 | {"line": "key = ", "expected": ("key", "", None)}, 1513 | {"line": "key = ", "expected": ("key", "", None)}, 1514 | {"line": " key = ", "expected": ("key", "", None)}, 1515 | {"line": "key:", "expected": ("key", "", None)}, 1516 | {"line": "key :", "expected": ("key", "", None)}, 1517 | {"line": "key: ", "expected": ("key", "", None)}, 1518 | {"line": "key : ", "expected": ("key", "", None)}, 1519 | {"line": "key : ", "expected": ("key", "", None)}, 1520 | {"line": " key : ", "expected": ("key", "", None)}, 1521 | ] 1522 | 1523 | for test in config_lines: 1524 | parsed_obj = p.parse(StringIO(test["line"])) 1525 | parsed_obj = dict(parsed_obj) 1526 | expected = {test["expected"][0]: test["expected"][1]} 1527 | self.assertDictEqual(parsed_obj, expected, msg="Line %r" % (test["line"])) 1528 | 1529 | def testDefaultConfigFileParser_UnspecifiedValues(self): 1530 | p = configargparse.DefaultConfigFileParser() 1531 | 1532 | # test the all syntax case 1533 | config_lines = [ 1534 | {"line": "key ", "expected": ("key", "true", None)}, 1535 | {"line": "key", "expected": ("key", "true", None)}, 1536 | {"line": "key ", "expected": ("key", "true", None)}, 1537 | {"line": " key ", "expected": ("key", "true", None)}, 1538 | ] 1539 | 1540 | for test in config_lines: 1541 | parsed_obj = p.parse(StringIO(test["line"])) 1542 | parsed_obj = dict(parsed_obj) 1543 | expected = {test["expected"][0]: test["expected"][1]} 1544 | self.assertDictEqual(parsed_obj, expected, msg="Line %r" % (test["line"])) 1545 | 1546 | def testDefaultConfigFileParser_ColonEqualSignValue(self): 1547 | p = configargparse.DefaultConfigFileParser() 1548 | 1549 | # test the all syntax case 1550 | config_lines = [ 1551 | {"line": "key=:", "expected": ("key", ":", None)}, 1552 | {"line": "key =:", "expected": ("key", ":", None)}, 1553 | {"line": "key= :", "expected": ("key", ":", None)}, 1554 | {"line": "key = :", "expected": ("key", ":", None)}, 1555 | {"line": "key = :", "expected": ("key", ":", None)}, 1556 | {"line": " key = : ", "expected": ("key", ":", None)}, 1557 | {"line": "key:=", "expected": ("key", "=", None)}, 1558 | {"line": "key :=", "expected": ("key", "=", None)}, 1559 | {"line": "key: =", "expected": ("key", "=", None)}, 1560 | {"line": "key : =", "expected": ("key", "=", None)}, 1561 | {"line": "key : =", "expected": ("key", "=", None)}, 1562 | {"line": " key : = ", "expected": ("key", "=", None)}, 1563 | {"line": "key==", "expected": ("key", "=", None)}, 1564 | {"line": "key ==", "expected": ("key", "=", None)}, 1565 | {"line": "key= =", "expected": ("key", "=", None)}, 1566 | {"line": "key = =", "expected": ("key", "=", None)}, 1567 | {"line": "key = =", "expected": ("key", "=", None)}, 1568 | {"line": " key = = ", "expected": ("key", "=", None)}, 1569 | {"line": "key::", "expected": ("key", ":", None)}, 1570 | {"line": "key ::", "expected": ("key", ":", None)}, 1571 | {"line": "key: :", "expected": ("key", ":", None)}, 1572 | {"line": "key : :", "expected": ("key", ":", None)}, 1573 | {"line": "key : :", "expected": ("key", ":", None)}, 1574 | {"line": " key : : ", "expected": ("key", ":", None)}, 1575 | ] 1576 | 1577 | for test in config_lines: 1578 | parsed_obj = p.parse(StringIO(test["line"])) 1579 | parsed_obj = dict(parsed_obj) 1580 | expected = {test["expected"][0]: test["expected"][1]} 1581 | self.assertDictEqual(parsed_obj, expected, msg="Line %r" % (test["line"])) 1582 | 1583 | def testDefaultConfigFileParser_ValuesWithComments(self): 1584 | p = configargparse.DefaultConfigFileParser() 1585 | 1586 | # test the all syntax case 1587 | config_lines = [ 1588 | {"line": "key=value#comment ", "expected": ("key", "value#comment", None)}, 1589 | {"line": "key=value #comment", "expected": ("key", "value", "comment")}, 1590 | { 1591 | "line": " key = value # comment", 1592 | "expected": ("key", "value", "comment"), 1593 | }, 1594 | {"line": "key:value#comment", "expected": ("key", "value#comment", None)}, 1595 | {"line": "key:value #comment", "expected": ("key", "value", "comment")}, 1596 | { 1597 | "line": " key : value # comment", 1598 | "expected": ("key", "value", "comment"), 1599 | }, 1600 | {"line": "key=value;comment ", "expected": ("key", "value;comment", None)}, 1601 | {"line": "key=value ;comment", "expected": ("key", "value", "comment")}, 1602 | { 1603 | "line": " key = value ; comment", 1604 | "expected": ("key", "value", "comment"), 1605 | }, 1606 | {"line": "key:value;comment", "expected": ("key", "value;comment", None)}, 1607 | {"line": "key:value ;comment", "expected": ("key", "value", "comment")}, 1608 | { 1609 | "line": " key : value ; comment", 1610 | "expected": ("key", "value", "comment"), 1611 | }, 1612 | { 1613 | "line": "key = value # comment # comment", 1614 | "expected": ("key", "value", "comment # comment"), 1615 | }, 1616 | { 1617 | "line": 'key = "value # comment" # comment', 1618 | "expected": ("key", "value # comment", "comment"), 1619 | }, 1620 | {"line": 'key = "#" ; comment', "expected": ("key", "#", "comment")}, 1621 | {"line": 'key = ";" # comment', "expected": ("key", ";", "comment")}, 1622 | ] 1623 | 1624 | for test in config_lines: 1625 | parsed_obj = p.parse(StringIO(test["line"])) 1626 | parsed_obj = dict(parsed_obj) 1627 | expected = {test["expected"][0]: test["expected"][1]} 1628 | self.assertDictEqual(parsed_obj, expected, msg="Line %r" % (test["line"])) 1629 | 1630 | def testDefaultConfigFileParser_NegativeValues(self): 1631 | p = configargparse.DefaultConfigFileParser() 1632 | 1633 | # test the all syntax case 1634 | config_lines = [ 1635 | {"line": "key = -10", "expected": ("key", "-10", None)}, 1636 | {"line": "key : -10", "expected": ("key", "-10", None)}, 1637 | {"line": "key -10", "expected": ("key", "-10", None)}, 1638 | {"line": 'key = "-10"', "expected": ("key", "-10", None)}, 1639 | {"line": "key = '-10'", "expected": ("key", "-10", None)}, 1640 | {"line": "key=-10", "expected": ("key", "-10", None)}, 1641 | ] 1642 | 1643 | for test in config_lines: 1644 | parsed_obj = p.parse(StringIO(test["line"])) 1645 | parsed_obj = dict(parsed_obj) 1646 | expected = {test["expected"][0]: test["expected"][1]} 1647 | self.assertDictEqual(parsed_obj, expected, msg="Line %r" % (test["line"])) 1648 | 1649 | def testDefaultConfigFileParser_KeySyntax(self): 1650 | p = configargparse.DefaultConfigFileParser() 1651 | 1652 | # test the all syntax case 1653 | config_lines = [ 1654 | { 1655 | "line": "key_underscore = value", 1656 | "expected": ("key_underscore", "value", None), 1657 | }, 1658 | {"line": "key_underscore=", "expected": ("key_underscore", "", None)}, 1659 | {"line": "key_underscore", "expected": ("key_underscore", "true", None)}, 1660 | { 1661 | "line": "_key_underscore = value", 1662 | "expected": ("_key_underscore", "value", None), 1663 | }, 1664 | {"line": "_key_underscore=", "expected": ("_key_underscore", "", None)}, 1665 | {"line": "_key_underscore", "expected": ("_key_underscore", "true", None)}, 1666 | { 1667 | "line": "key_underscore_ = value", 1668 | "expected": ("key_underscore_", "value", None), 1669 | }, 1670 | {"line": "key_underscore_=", "expected": ("key_underscore_", "", None)}, 1671 | {"line": "key_underscore_", "expected": ("key_underscore_", "true", None)}, 1672 | {"line": "key-dash = value", "expected": ("key-dash", "value", None)}, 1673 | {"line": "key-dash=", "expected": ("key-dash", "", None)}, 1674 | {"line": "key-dash", "expected": ("key-dash", "true", None)}, 1675 | {"line": "key@word = value", "expected": ("key@word", "value", None)}, 1676 | {"line": "key@word=", "expected": ("key@word", "", None)}, 1677 | {"line": "key@word", "expected": ("key@word", "true", None)}, 1678 | {"line": "key$word = value", "expected": ("key$word", "value", None)}, 1679 | {"line": "key$word=", "expected": ("key$word", "", None)}, 1680 | {"line": "key$word", "expected": ("key$word", "true", None)}, 1681 | {"line": "key.word = value", "expected": ("key.word", "value", None)}, 1682 | {"line": "key.word=", "expected": ("key.word", "", None)}, 1683 | {"line": "key.word", "expected": ("key.word", "true", None)}, 1684 | ] 1685 | 1686 | for test in config_lines: 1687 | parsed_obj = p.parse(StringIO(test["line"])) 1688 | parsed_obj = dict(parsed_obj) 1689 | expected = {test["expected"][0]: test["expected"][1]} 1690 | self.assertDictEqual(parsed_obj, expected, msg="Line %r" % (test["line"])) 1691 | 1692 | def testYAMLConfigFileParser_Basic(self): 1693 | try: 1694 | import yaml 1695 | except: 1696 | logging.warning( 1697 | "WARNING: PyYAML not installed. " "Couldn't test YAMLConfigFileParser" 1698 | ) 1699 | return 1700 | 1701 | p = configargparse.YAMLConfigFileParser() 1702 | self.assertGreater(len(p.get_syntax_description()), 0) 1703 | 1704 | input_config_str = StringIO("""a: '3'\n""") 1705 | parsed_obj = p.parse(input_config_str) 1706 | output_config_str = p.serialize(dict(parsed_obj)) 1707 | 1708 | self.assertEqual(input_config_str.getvalue(), output_config_str) 1709 | 1710 | self.assertDictEqual(parsed_obj, {"a": "3"}) 1711 | 1712 | def testYAMLConfigFileParser_All(self): 1713 | try: 1714 | import yaml 1715 | except: 1716 | logging.warning( 1717 | "WARNING: PyYAML not installed. " "Couldn't test YAMLConfigFileParser" 1718 | ) 1719 | return 1720 | 1721 | p = configargparse.YAMLConfigFileParser() 1722 | 1723 | # test the all syntax case 1724 | config_lines = [ 1725 | "a: '3'", 1726 | "list_arg:", 1727 | "- 1", 1728 | "- 2", 1729 | "- 3", 1730 | ] 1731 | 1732 | # test parse 1733 | input_config_str = StringIO("\n".join(config_lines) + "\n") 1734 | parsed_obj = p.parse(input_config_str) 1735 | 1736 | # test serialize 1737 | output_config_str = p.serialize(parsed_obj) 1738 | self.assertEqual(input_config_str.getvalue(), output_config_str) 1739 | 1740 | self.assertDictEqual(parsed_obj, {"a": "3", "list_arg": [1, 2, 3]}) 1741 | 1742 | def testYAMLConfigFileParser_w_ArgumentParser_parsed_values(self): 1743 | try: 1744 | import yaml 1745 | except: 1746 | raise AssertionError( 1747 | "WARNING: PyYAML not installed. " "Couldn't test YAMLConfigFileParser" 1748 | ) 1749 | return 1750 | 1751 | parser = configargparse.ArgumentParser( 1752 | config_file_parser_class=configargparse.YAMLConfigFileParser 1753 | ) 1754 | parser.add_argument("-c", "--config", is_config_file=True) 1755 | parser.add_argument("--verbosity", action="count") 1756 | parser.add_argument("--verbose", action="store_true") 1757 | parser.add_argument("--level", type=int) 1758 | 1759 | config_lines = ["verbosity: 3", "verbose: true", "level: 35"] 1760 | config_str = "\n".join(config_lines) + "\n" 1761 | config_file = tempfile.gettempdir() + "/temp_YAMLConfigFileParser.cfg" 1762 | with open(config_file, "w") as f: 1763 | f.write(config_str) 1764 | args = parser.parse_args(["--config=%s" % config_file]) 1765 | assert args.verbosity == 3 1766 | assert args.verbose == True 1767 | assert args.level == 35 1768 | 1769 | 1770 | ################################################################################ 1771 | # since configargparse should work as a drop-in replacement for argparse 1772 | # in all situations, run argparse unittests on configargparse by modifying 1773 | # their source code to use configargparse.ArgumentParser 1774 | 1775 | try: 1776 | import test.test_argparse 1777 | 1778 | # Sig = test.test_argparse.Sig 1779 | # NS = test.test_argparse.NS 1780 | except ImportError: 1781 | logging.error( 1782 | "\n\n" 1783 | "============================\n" 1784 | "ERROR: Many tests couldn't be run because 'import test.test_argparse' " 1785 | "failed. Try building/installing python from source rather than through" 1786 | " a package manager.\n" 1787 | "============================\n" 1788 | ) 1789 | else: 1790 | test_argparse_source_code = inspect.getsource(test.test_argparse) 1791 | test_argparse_source_code = ( 1792 | test_argparse_source_code.replace( 1793 | "argparse.ArgumentParser", "configargparse.ArgumentParser" 1794 | ) 1795 | .replace("TestHelpFormattingMetaclass", "_TestHelpFormattingMetaclass") 1796 | .replace("test_main", "_test_main") 1797 | ) 1798 | 1799 | # pytest tries to collect tests from TestHelpFormattingMetaclass, and 1800 | # test_main, and raises a warning when it finds it's not a test class 1801 | # nor test function. Renaming TestHelpFormattingMetaclass and test_main 1802 | # prevents pytest from trying. 1803 | 1804 | # run or debug a subset of the argparse tests 1805 | # test_argparse_source_code = test_argparse_source_code.replace( 1806 | # "(TestCase)", "").replace( 1807 | # "(ParserTestCase)", "").replace( 1808 | # "(HelpTestCase)", "").replace( 1809 | # ", TestCase", "").replace( 1810 | # ", ParserTestCase", "") 1811 | # test_argparse_source_code = test_argparse_source_code.replace( 1812 | # "class TestMessageContentError", "class TestMessageContentError(TestCase)") 1813 | 1814 | exec(test_argparse_source_code) 1815 | 1816 | # print argparse unittest source code 1817 | def print_source_code(source_code, line_numbers, context_lines=10): 1818 | for n in line_numbers: 1819 | logging.debug("##### Code around line %s #####" % n) 1820 | lines_to_print = set(range(n - context_lines, n + context_lines)) 1821 | for n2, line in enumerate(source_code.split("\n"), 1): 1822 | if n2 in lines_to_print: 1823 | logging.debug("%s %5d: %s" % ("**" if n2 == n else " ", n2, line)) 1824 | 1825 | # print_source_code(test_argparse_source_code, [4540, 4565]) 1826 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py35, py36, py37, py38, py39, py310, py311, pypy, pypy3 3 | 4 | [testenv] 5 | #setenv = PYTHONPATH = {toxinidir}:{toxinidir}/configmanager 6 | 7 | commands = python setup.py test 8 | # python -m unittest discover 9 | 10 | [testenv:py36] 11 | basepython=python3.6 12 | 13 | [testenv:py37] 14 | basepython=python3.7 15 | 16 | [testenv:py38] 17 | basepython=python3.8 18 | 19 | [testenv:py39] 20 | basepython=python3.9 21 | 22 | [testenv:py310] 23 | basepython=python3.10 24 | 25 | [testenv:py311] 26 | basepython=python3.11 27 | 28 | [testenv:py312] 29 | basepython=python3.12 30 | 31 | [testenv:py313] 32 | basepython=python3.13 33 | 34 | [testenv:pypy] 35 | basepython=pypy 36 | 37 | [testenv:pypy3] 38 | basepython=pypy3 39 | 40 | [testenv:apidocs] 41 | description = Build the API documentation 42 | 43 | 44 | deps = 45 | pydoctor>=22.3.0 46 | allowlist_externals = bash 47 | commands = 48 | bash ./apidocs.sh 49 | --------------------------------------------------------------------------------