├── .coveragerc ├── .github └── workflows │ └── build.yaml ├── .gitignore ├── .gitmodules ├── .readthedocs.yml ├── .travis.yml ├── LICENSE ├── README.rst ├── clean ├── doc ├── .static │ └── css │ │ └── custom.css ├── Makefile ├── alternatives.rst ├── basic_syntax.rst ├── basic_use.rst ├── changelog.rst ├── common_mistakes.rst ├── conf.py ├── examples.rst ├── file_format.rst ├── index.rst ├── minimal-nestedtext.rst ├── nestedtext.Location.rst ├── nestedtext.NestedTextError.rst ├── nestedtext.dumpers.rst ├── nestedtext.loaders.rst ├── nestedtext.utilities.rst ├── philosophy.rst ├── python_api.rst ├── related_projects.rst ├── releases.rst ├── requirements.in ├── requirements.txt ├── schemas.rst └── techniques.rst ├── examples ├── accumulation │ ├── example.in.nt │ ├── example.out.nt │ ├── example.py │ ├── settings.py │ ├── test_read_settings.nt │ └── test_read_settings.py ├── addresses │ ├── address │ ├── address.json │ ├── address.nt │ ├── address.orig │ ├── blue-address │ ├── fumiko.json │ └── fumiko.nt ├── conftest.py ├── conversion-utilities │ ├── csv-to-nestedtext │ ├── dmarc.nt │ ├── dmarc.xml │ ├── github-intent.nt │ ├── github-intent.yaml │ ├── github-orig.nt │ ├── github-orig.yaml │ ├── github-rt.yaml │ ├── json-to-nestedtext │ ├── nestedtext-to-json │ ├── nestedtext-to-yaml │ ├── percent_bachelors_degrees_women_usa.csv │ ├── percent_bachelors_degrees_women_usa.nt │ ├── rt-yaml │ ├── sparekeys.nt │ ├── sparekeys.toml │ ├── toml-to-nestedtext │ ├── xml-to-nestedtext │ └── yaml-to-nestedtext ├── cryptocurrency │ ├── cryptocurrency │ ├── cryptocurrency.nt │ └── voluptuous_errors.py ├── deduplication │ ├── michael_jordan │ ├── michael_jordan.nt │ └── michael_jordan.out ├── duplicate-keys.nt ├── groceries.nt ├── long_lines.py ├── long_lines │ ├── long_lines.out │ ├── long_lines_backslash │ └── long_lines_space ├── parametrize_from_file │ ├── test_misc.nt │ └── test_misc.py ├── postmortem │ ├── postmortem │ ├── postmortem.expanded.nt │ ├── postmortem.nt │ └── voluptuous_errors.py ├── references │ ├── diet │ └── diet.nt ├── test_examples.py └── validation │ ├── deploy.nt │ ├── deploy_pydantic.out │ ├── deploy_pydantic.py │ ├── deploy_voluptuous.out │ ├── deploy_voluptuous.py │ └── voluptuous_errors.py ├── nestedtext ├── __init__.py ├── __init__.pyi ├── nestedtext.py ├── nestedtext.pyi └── py.typed ├── proposed_tests ├── Makefile ├── README.rst ├── clean ├── convert ├── test_nt.py ├── tests.json └── tests.nt ├── pyproject.toml ├── pytest.ini ├── tests ├── test_nestedtext.py └── test_random.py └── tox.ini /.coveragerc: -------------------------------------------------------------------------------- 1 | [report] 2 | exclude_lines = 3 | pragma: no cover 4 | raise NotImplementedError 5 | precision=2 6 | show_missing=yes 7 | 8 | [run] 9 | source = nestedtext 10 | omit = */.tox/*,*/.local/* 11 | # the doctests use the installed version of NestedText, which suppresses 12 | # the coverage percentage 13 | -------------------------------------------------------------------------------- /.github/workflows/build.yaml: -------------------------------------------------------------------------------- 1 | name: NestedText 2 | on: [push, pull_request] 3 | jobs: 4 | check-bats-version: 5 | runs-on: ${{ matrix.os }} 6 | strategy: 7 | matrix: 8 | os: [ubuntu-latest] 9 | python-version: ["3.8", "3.x"] 10 | max-parallel: 6 11 | 12 | steps: 13 | - name: Checkout repository 14 | uses: actions/checkout@v4 15 | with: 16 | submodules: true 17 | - name: Set up Python ${{ matrix.python-version }} 18 | uses: actions/setup-node@v2 19 | with: 20 | python-version: ${{ matrix.python-version }} 21 | - name: Display Python version 22 | run: python -c "import sys; print(sys.version)" 23 | - name: Install packages 24 | run: | 25 | python -m pip install --upgrade pip 26 | pip install . 27 | pip install tox 28 | pip install coveralls 29 | - name: Run tests 30 | run: tox 31 | - name: Report test coverage 32 | env: 33 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 34 | COVERALLS_SERVICE_NAME: github 35 | run: coveralls 36 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | examples/access.nt 2 | .github/workflows/build.yaml.save 3 | .github/workflows/build.yaml.241014 4 | 5 | 6 | # version info 7 | .bump.cfg.nt 8 | data.nt 9 | 10 | # Byte-compiled / optimized / DLL files 11 | __pycache__/ 12 | *.py[cod] 13 | *$py.class 14 | 15 | # Distribution / packaging 16 | .Python 17 | build/ 18 | develop-eggs/ 19 | dist/ 20 | downloads/ 21 | eggs/ 22 | .eggs/ 23 | lib/ 24 | lib64/ 25 | parts/ 26 | sdist/ 27 | var/ 28 | wheels/ 29 | *.egg-info/ 30 | .installed.cfg 31 | *.egg 32 | MANIFEST 33 | .*.swp 34 | doc/.build 35 | 36 | # PyInstaller 37 | # Usually these files are written by a python script from a template 38 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 39 | *.manifest 40 | *.spec 41 | 42 | # Installer logs 43 | pip-log.txt 44 | pip-delete-this-directory.txt 45 | 46 | # Unit test / coverage reports 47 | htmlcov/ 48 | .tox/ 49 | .coverage 50 | .coverage.* 51 | .cache 52 | nosetests.xml 53 | coverage.xml 54 | *.cover 55 | .hypothesis/ 56 | .pytest_cache/ 57 | 58 | # Translations 59 | *.mo 60 | *.pot 61 | 62 | # Django stuff: 63 | *.log 64 | local_settings.py 65 | db.sqlite3 66 | 67 | # Flask stuff: 68 | instance/ 69 | .webassets-cache 70 | 71 | # Scrapy stuff: 72 | .scrapy 73 | 74 | # Sphinx documentation 75 | docs/_build/ 76 | 77 | # PyBuilder 78 | target/ 79 | 80 | # Jupyter Notebook 81 | .ipynb_checkpoints 82 | 83 | # pyenv 84 | .python-version 85 | 86 | # celery beat schedule file 87 | celerybeat-schedule 88 | 89 | # SageMath parsed files 90 | *.sage.py 91 | 92 | # Environments 93 | .env 94 | .venv 95 | env/ 96 | venv/ 97 | ENV/ 98 | env.bak/ 99 | venv.bak/ 100 | 101 | # Spyder project settings 102 | .spyderproject 103 | .spyproject 104 | 105 | # Rope project settings 106 | .ropeproject 107 | 108 | # mkdocs documentation 109 | /site 110 | 111 | # mypy 112 | .mypy_cache/ 113 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "tests/official_tests"] 2 | path = tests/official_tests 3 | url = https://github.com/KenKundert/nestedtext_tests.git 4 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | # .readthedocs.yml 2 | # Read the Docs configuration file 3 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 4 | 5 | # Required 6 | version: 2 7 | 8 | build: 9 | os: ubuntu-22.04 10 | tools: 11 | python: "3.11" 12 | 13 | # Build documentation in the doc/ directory with Sphinx 14 | sphinx: 15 | configuration: doc/conf.py 16 | 17 | # Optionally build your docs in additional formats such as PDF and ePub 18 | formats: all 19 | 20 | # Optionally set the version of Python and requirements required to build your docs 21 | python: 22 | install: 23 | - requirements: doc/requirements.txt 24 | - method: pip 25 | path: . 26 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | os: 3 | - linux 4 | install: 5 | - pip install virtualenv tox coveralls inform 6 | script: 7 | - tox 8 | after_success: 9 | - coveralls 10 | jobs: 11 | include: 12 | - python: 3.6 13 | - python: 3.7 14 | - python: 3.8 15 | - python: 3.9 16 | - python: 3.10-dev 17 | - python: nightly 18 | allow_failures: 19 | - python: 3.10-dev 20 | - python: nightly 21 | fast_finish: true 22 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020-2023 Kenneth S. Kundert and Kale Kundert 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | NestedText — A Human Friendly Data Format 2 | ========================================= 3 | 4 | |downloads| |build status| |coverage| |rtd status| |pypi version| |anaconda version| |python version| 5 | 6 | 7 | | Authors: Ken & Kale Kundert 8 | | Version: 3.8.dev2 9 | | Released: 2025-04-05 10 | | Documentation: nestedtext.org_ 11 | | Please post all questions, suggestions, and bug reports to GitHub_. 12 | | 13 | 14 | *NestedText* is a file format for holding structured data. It is similar in 15 | concept to JSON_, except that *NestedText* is designed to make it easy for 16 | people to enter, edit, or view the data directly. It organizes the data into 17 | a nested collection of name-value pairs, lists, and strings. The syntax is 18 | intended to be very simple and intuitive for most people. 19 | 20 | A unique feature of this file format is that it only supports one scalar type: 21 | strings.  As such, quoting strings is unnecessary, and without quoting there is 22 | no need for escaping. While the decision to forego other types (integers, 23 | reals, Booleans, etc.) may seem counter productive, it leads to simpler data 24 | files and applications that are more robust. 25 | 26 | *NestedText* is convenient for configuration files, data journals, address 27 | books, account information, and the like. Here is an example of a file that 28 | contains a few addresses: 29 | 30 | .. code-block:: nestedtext 31 | 32 | # Contact information for our officers 33 | 34 | Katheryn McDaniel: 35 | position: president 36 | address: 37 | > 138 Almond Street 38 | > Topeka, Kansas 20697 39 | phone: 40 | cell: 1-210-555-5297 41 | home: 1-210-555-8470 42 | # Katheryn prefers that we always call her on her cell phone. 43 | email: KateMcD@aol.com 44 | additional roles: 45 | - board member 46 | 47 | Margaret Hodge: 48 | position: vice president 49 | address: 50 | > 2586 Marigold Lane 51 | > Topeka, Kansas 20682 52 | phone: 1-470-555-0398 53 | email: margaret.hodge@ku.edu 54 | additional roles: 55 | - new membership task force 56 | - accounting task force 57 | 58 | Typical Applications 59 | -------------------- 60 | 61 | Configuration 62 | """"""""""""" 63 | 64 | Configuration files are an attractive application for *NestedText*. 65 | *NestedText* configuration files tend to be simple, clean and unambiguous. 66 | Plus, they handle hierarchy much better than alternatives such as Ini_ and 67 | TOML_. 68 | 69 | 70 | Structured Code 71 | """"""""""""""" 72 | 73 | One way to build tools to tackle difficult and complex tasks is to provide an 74 | application specific language. That can be a daunting challenge. However, in 75 | certain cases, such as specifying complex configurations, *NestedText* can help 76 | make the task much easier. *NestedText* conveys the structure of data leaving 77 | the end application to interpret the data itself. It can do so with 78 | a collection of small parsers that are tailored to the specific piece of data to 79 | which they are applied. This generally results in a simpler specification since 80 | each piece of data can be given in its natural format, which might otherwise 81 | confuse a shared parser. In this way, rather than building one large very 82 | general language and parser, a series of much smaller and simpler parsers are 83 | needed. These smaller parsers can be as simple as splitters or partitioners, 84 | value checkers, or converters for numbers in special forms (numbers with units, 85 | times or dates, GPS coordinates, etc.). Or they could be full-blown expression 86 | evaluators or mini-languages. Structured code provides a nice middle ground 87 | between data and code and its use is growing in popularity. 88 | 89 | An example of structured code is provided by GitHub with its workflow 90 | specification files. They use YAML_. Unfortunately, the syntax of the code 91 | snippets held in the various fields can be confused with *YAML* syntax, which 92 | leads to unnecessary errors, confusion, and complexity (see *YAML issues*). 93 | JSON_ suffers from similar problems. *NestedText* excels for these applications 94 | as it holds code snippets without any need for quoting or escaping. 95 | *NestedText* provides simple unambiguous rules for defining the structure of 96 | your data and when these rules are followed there is no way for any syntax or 97 | special characters in the values of your data to be confused with *NestedText* 98 | syntax. In fact, it is possible for *NestedText* to hold *NestedText* snippets 99 | without conflict. 100 | 101 | Another example of structured code is provided by the files that contain the 102 | test cases used by `Parametrize From File`_, a PyTest_ plugin. 103 | *Parametrize From File* simplifies the task of specifying test cases for 104 | *PyTest* by separating the test cases from the test code. Here it is being 105 | applied to test a command line program. Its response is checked using regular 106 | expressions. Each entry includes a shell command to run the program and 107 | a regular expression that must match the output for the test to pass:: 108 | 109 | - 110 | cmd: emborg version 111 | expected: emborg version: \d+\.\d+(\.\d+(\.?\w+\d+)?)? \(\d\d\d\d-\d\d-\d\d\) 112 | expected type: regex 113 | - 114 | cmd: emborg --quiet files -D 115 | expected: 116 | > Archive: home-\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d 117 | > \d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d.\d\d\d\d\d\d configs/subdir/(file|) 118 | > \d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d.\d\d\d\d\d\d configs/subdir/(file|) 119 | # Unfortunately, we cannot check the order as they were both 120 | # created at the same time. 121 | expected type: regex 122 | - 123 | cmd: emborg due --backup-days 1 --message "{elapsed} since last {action}" 124 | expected: home: (\d+(\.\d)? (seconds|minutes)) since last backup\. 125 | expected type: regex 126 | 127 | Notice that the regular expressions are given clean, without any quoting or 128 | escaping. 129 | 130 | 131 | Composable Utilities 132 | """""""""""""""""""" 133 | 134 | Another attractive use-case for *NestedText* is command line programs whose 135 | output is meant to be consumed by either people or other programs. This is 136 | another growing trend. Many programs do this by supporting a ``--json`` 137 | command-line flag that indicates the output should be computer readable rather 138 | than human readable. But, with *NestedText* it is not necessary to make people 139 | choose. Just output the result in *NestedText* and it can be read by people or 140 | computers. For example, consider a program that reads your address list and 141 | output particular fields on demand:: 142 | 143 | > address --email 144 | Katheryn McDaniel: KateMcD@aol.com 145 | Margaret Hodge: margaret.hodge@ku.edu 146 | 147 | This output could be fed directly into another program that accepts *NestedText* 148 | as input:: 149 | 150 | > address --email | mail-to-list 151 | 152 | 153 | Contributing 154 | ------------ 155 | 156 | This package contains a Python reference implementation of *NestedText* and 157 | a test suite. Implementation in many languages is required for *NestedText* to 158 | catch on widely. If you like the format, please consider contributing 159 | additional implementations. 160 | 161 | Also, please consider using *NestedText* for any applications you create. 162 | 163 | 164 | .. _json: https://www.json.org/json-en.html 165 | .. _yaml: https://yaml.org/ 166 | .. _toml: https://toml.io/en/ 167 | .. _ini: https://en.wikipedia.org/wiki/INI_file 168 | .. _parametrize from file: https://parametrize-from-file.readthedocs.io 169 | .. _pytest: https://docs.pytest.org 170 | .. _github: https://github.com/KenKundert/nestedtext/issues 171 | .. _nestedtext.org: https://nestedtext.org 172 | 173 | .. |downloads| image:: https://pepy.tech/badge/nestedtext/month 174 | :target: https://pepy.tech/project/nestedtext 175 | 176 | .. |rtd status| image:: https://img.shields.io/readthedocs/nestedtext.svg 177 | :target: https://nestedtext.readthedocs.io/en/latest/?badge=latest 178 | 179 | .. |build status| image:: https://github.com/KenKundert/nestedtext/actions/workflows/build.yaml/badge.svg 180 | :target: https://github.com/KenKundert/nestedtext/actions/workflows/build.yaml 181 | 182 | .. |coverage| image:: https://coveralls.io/repos/github/KenKundert/nestedtext/badge.svg?branch=master 183 | :target: https://coveralls.io/github/KenKundert/nestedtext?branch=master 184 | 185 | .. |pypi version| image:: https://img.shields.io/pypi/v/nestedtext.svg 186 | :target: https://pypi.python.org/pypi/nestedtext 187 | 188 | .. |anaconda version| image:: https://anaconda.org/conda-forge/nestedtext/badges/version.svg 189 | :target: https://anaconda.org/conda-forge/nestedtext 190 | 191 | .. |python version| image:: https://img.shields.io/pypi/pyversions/nestedtext.svg 192 | :target: https://pypi.python.org/pypi/nestedtext 193 | 194 | -------------------------------------------------------------------------------- /clean: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | shopt -s globstar 4 | rm -rf generated_settings .tox 5 | rm -rf .cache 6 | rm -rf data.nt tests/data.nt 7 | rm -rf /tmp/pytest-of-$USER 8 | 9 | # the rest is common to all python directories 10 | rm -f *.pyc *.pyo 11 | rm -f .test*.sum expected result install.out .*.log 12 | rm -rf build *.egg-info dist __pycache__ .eggs **/{__pycache__,*.pyc,*.pyo} 13 | rm -rf .coverage .coverage-html htmlcov .tox 14 | rm -rf .pytest_cache .cache dist .build quantiphy.egg.info 15 | rm -rf tests/{htmlcov,.cache,.coverage,.pytest_cache} 16 | rm -rf doc/.build 17 | -------------------------------------------------------------------------------- /doc/.static/css/custom.css: -------------------------------------------------------------------------------- 1 | /* 2 | * the following is used to make the highlighting that introduces classes, 3 | * methods, functions, etc extend across the width of the page. 4 | */ 5 | dt:first-child { 6 | width: 100%; 7 | } 8 | -------------------------------------------------------------------------------- /doc/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = .build 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 14 | # the i18n builder cannot share the environment and doctrees with the others 15 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 16 | 17 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 18 | 19 | default: html 20 | 21 | again: clean html 22 | 23 | help: 24 | @echo "Please use \`make ' where is one of" 25 | @echo " html to make standalone HTML files" 26 | @echo " dirhtml to make HTML files named index.html in directories" 27 | @echo " singlehtml to make a single large HTML file" 28 | @echo " pickle to make pickle files" 29 | @echo " json to make JSON files" 30 | @echo " htmlhelp to make HTML files and a HTML help project" 31 | @echo " qthelp to make HTML files and a qthelp project" 32 | @echo " devhelp to make HTML files and a Devhelp project" 33 | @echo " epub to make an epub" 34 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 35 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 36 | @echo " text to make text files" 37 | @echo " man to make manual pages" 38 | @echo " texinfo to make Texinfo files" 39 | @echo " info to make Texinfo files and run them through makeinfo" 40 | @echo " gettext to make PO message catalogs" 41 | @echo " changes to make an overview of all changed/added/deprecated items" 42 | @echo " linkcheck to check all external links for integrity" 43 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 44 | 45 | clean: 46 | -rm -rf $(BUILDDIR)/* 47 | 48 | html: 49 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 50 | @echo 51 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 52 | 53 | show: html 54 | firefox .build/html/index.html 55 | 56 | dirhtml: 57 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 58 | @echo 59 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 60 | 61 | singlehtml: 62 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 63 | @echo 64 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 65 | 66 | pickle: 67 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 68 | @echo 69 | @echo "Build finished; now you can process the pickle files." 70 | 71 | json: 72 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 73 | @echo 74 | @echo "Build finished; now you can process the JSON files." 75 | 76 | htmlhelp: 77 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 78 | @echo 79 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 80 | ".hhp project file in $(BUILDDIR)/htmlhelp." 81 | 82 | qthelp: 83 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 84 | @echo 85 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 86 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 87 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/avendesora.qhcp" 88 | @echo "To view the help file:" 89 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/avendesora.qhc" 90 | 91 | devhelp: 92 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 93 | @echo 94 | @echo "Build finished." 95 | @echo "To view the help file:" 96 | @echo "# mkdir -p $$HOME/.local/share/devhelp/avendesora" 97 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/avendesora" 98 | @echo "# devhelp" 99 | 100 | epub: 101 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 102 | @echo 103 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 104 | 105 | latex: 106 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 107 | @echo 108 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 109 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 110 | "(use \`make latexpdf' here to do that automatically)." 111 | 112 | latexpdf: 113 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 114 | @echo "Running LaTeX files through pdflatex..." 115 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 116 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 117 | 118 | text: 119 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 120 | @echo 121 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 122 | 123 | man: 124 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 125 | @echo 126 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 127 | 128 | texinfo: 129 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 130 | @echo 131 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 132 | @echo "Run \`make' in that directory to run these through makeinfo" \ 133 | "(use \`make info' here to do that automatically)." 134 | 135 | info: 136 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 137 | @echo "Running Texinfo files through makeinfo..." 138 | make -C $(BUILDDIR)/texinfo info 139 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 140 | 141 | gettext: 142 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 143 | @echo 144 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 145 | 146 | changes: 147 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 148 | @echo 149 | @echo "The overview file is in $(BUILDDIR)/changes." 150 | 151 | linkcheck: 152 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 153 | @echo 154 | @echo "Link check complete; look for any errors in the above output " \ 155 | "or in $(BUILDDIR)/linkcheck/output.txt." 156 | 157 | doctest: 158 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 159 | @echo "Testing of doctests in the sources finished, look at the " \ 160 | "results in $(BUILDDIR)/doctest/output.txt." 161 | -------------------------------------------------------------------------------- /doc/basic_syntax.rst: -------------------------------------------------------------------------------- 1 | ********************* 2 | Language introduction 3 | ********************* 4 | 5 | This is a overview of the syntax of a *NestedText* document, which consists of 6 | a :ref:`nested collection ` of :ref:`dictionaries `, 7 | :ref:`lists `, and :ref:`strings ` where indentation is used to 8 | indicate nesting. All leaf values must be simple text or empty. You can find 9 | more specifics :ref:`in the next section `. 10 | 11 | 12 | .. _dictionaries: 13 | 14 | Dictionaries 15 | ============ 16 | 17 | A dictionary is an ordered collection of key value pairs: 18 | 19 | .. code-block:: nestedtext 20 | 21 | key 1: value 1 22 | key 2: value 2 23 | key 3: value 3 24 | 25 | A dictionary item is a single key value pair. A dictionary is all adjacent 26 | dictionary items in which the keys all begin at the same level of indentation. 27 | There are several different ways to specify dictionaries. 28 | 29 | In the first form, the key and value are separated by a dictionary tag, which is 30 | a colon followed by a space or newline (``:␣`` or ``:↵``). The key must be 31 | a string and must not start with a ``-␣``, ``>␣``, ``:␣``, ``[``, ``{``, ``#``, 32 | or white space character; or contain newline characters or the ``:␣`` character 33 | sequence. Any spaces between the key and the tag are ignored. 34 | 35 | The value of this dictionary item may be a rest-of-line string, a multiline 36 | string, a list, or a dictionary. If it is a rest-of-line string, it contains all 37 | characters following the tag that separates the key from the value (``:␣``). 38 | For all other values, the rest of the line must be empty, with the value given 39 | on the next line, which must be further indented. 40 | 41 | 42 | .. code-block:: nestedtext 43 | 44 | key 1: value 1 45 | key 2: 46 | key 3: 47 | - value 3a 48 | - value 3b 49 | key 4: 50 | key 4a: value 4a 51 | key 4b: value 4b 52 | key 5: 53 | > first line of value 5 54 | > second line of value 5 55 | 56 | This is equivalent to the following JSON code: 57 | 58 | .. code-block:: json 59 | 60 | { 61 | "key 1": "value 1", 62 | "key 2": "", 63 | "key 3": [ 64 | "value 3a", 65 | "value 3b" 66 | ], 67 | "key 4": { 68 | "key 4a": "value 4a", 69 | "key 4b": "value 4b" 70 | }, 71 | "key 5": "first line of value 5\nsecond line of value 5" 72 | } 73 | 74 | A second less common form of a dictionary item employs multiline keys. In this 75 | case there are no limitations on the key other than it being a string. Each 76 | line of a multiline key is introduced with a colon (``:``) followed by a space 77 | or newline. The key is all adjacent lines at the same level that start with 78 | a colon tag with the tags removed but leading and trailing white space retained, 79 | including all newlines except the last. 80 | 81 | This form of dictionary does not allow rest-of-line string values; you would use 82 | a multiline string value instead: 83 | 84 | .. code-block:: nestedtext 85 | 86 | : key 1 87 | : the first key 88 | > value 1 89 | : key 2: the second key 90 | - value 2a 91 | - value 2b 92 | 93 | A dictionary may consist of dictionary items of either form. 94 | 95 | The final form of a dictionary is the inline dictionary. This is a compact form 96 | where all the dictionary items are given on the same line. There is a bit of 97 | syntax that defines inline dictionaries, so the keys and values are constrained 98 | to avoid ambiguities in the syntax. An inline dictionary starts with an opening 99 | brace (``{``), ends with a matching closing brace (``}``), and contains inline 100 | dictionary items separated by commas (``,``). An inline dictionary item is a key 101 | and value separated by a colon (``:``). A space need not follow the colon. The 102 | keys are inline strings and the values may be inline strings, inline lists, and 103 | inline dictionaries. An empty dictionary is represented with ``{}`` (there can 104 | be no space between the opening and closing braces). Leading and trailing 105 | spaces are stripped from keys and string values within inline dictionaries. 106 | 107 | For example: 108 | 109 | .. code-block:: nestedtext 110 | 111 | {key 1: value 1, key 2: value 2, key 3: value 3} 112 | 113 | .. code-block:: nestedtext 114 | 115 | {key 1: value 1, key 2: [value 2a, value 2b], key 3: {key 3a: value 3a, key 3b: value 3b}} 116 | 117 | 118 | .. _lists: 119 | 120 | Lists 121 | ===== 122 | 123 | A list is an ordered collection of values: 124 | 125 | .. code-block:: nestedtext 126 | 127 | - value 1 128 | - value 2 129 | - value 3 130 | 131 | A list item is introduced with a list tag: a dash followed by a space or 132 | a newline at the start of a line (``-␣`` or ``-↵``). All adjacent list items at 133 | the same level of indentation form the list. 134 | 135 | The value of a list item may be a rest-of-line string, a multiline string, 136 | a list, or a dictionary. If it is a rest-of-line string, it contains all 137 | characters that follow the tag that introduces the list item. For all other 138 | values, the rest of the line must be empty, with the value given on the next 139 | line, which must be further indented. 140 | 141 | .. code-block:: nestedtext 142 | 143 | - value 1 144 | - 145 | - 146 | - value 3a 147 | - value 3b 148 | - 149 | key 4a: value 4a 150 | key 4b: value 4b 151 | - 152 | > first line of value 5 153 | > second line of value 5 154 | 155 | Which is equivalent to the following JSON code: 156 | 157 | .. code-block:: json 158 | 159 | [ 160 | "value 1", 161 | "", 162 | [ 163 | "value 3a", 164 | "value 3b" 165 | ], 166 | { 167 | "key 4a": "value 4a", 168 | "key 4b": "value 4b" 169 | }, 170 | "first line of value 5\nsecond line of value 5" 171 | ] 172 | 173 | Another form of a list is the inline list. This is a compact form where all the 174 | list items are given on the same line. There is a bit of syntax that defines 175 | the list, so the values are constrained to avoid ambiguities in the syntax. An 176 | inline list starts with an opening bracket (``[``), ends with a matching closing 177 | bracket (``]``), and contains inline values separated by commas. The values may 178 | be inline strings, inline lists, and inline dictionaries. An empty list is 179 | represented by ``[]`` (there should be no space between the opening and closing 180 | brackets). Leading and trailing spaces are stripped from string values within 181 | inline lists. 182 | 183 | For example: 184 | 185 | .. code-block:: nestedtext 186 | 187 | [value 1, value 2, value 3] 188 | 189 | .. code-block:: nestedtext 190 | 191 | [value 1, [value 2a, value 2b], {key 3a: value 3a, key 3b: value 3b}] 192 | 193 | ``[ ]`` is not treated as an empty list as there is space between the brackets, 194 | rather this represents a list with a single empty string value. The contents of 195 | the brackets, which consists only of white space, is stripped of its padding, 196 | leaving an empty string. 197 | 198 | 199 | .. _strings: 200 | 201 | Strings 202 | ======= 203 | 204 | There are three types of strings: rest-of-line strings, multiline strings, and 205 | inline strings. Rest-of-line strings are simply all the characters on a line 206 | that follow a list tag (``-␣``) or dictionary tag (``:␣``), including any 207 | leading or trailing white space. They can contain any character other than 208 | a newline. The content of the rest-of-line string starts after the first space 209 | that follows the dash or colon of the tag: 210 | 211 | .. code-block:: nestedtext 212 | 213 | code : input signed [7:0] level 214 | regex : [+-]?([0-9]*[.])?[0-9]+\s*\w* 215 | math : $x = \frac{{-b \pm \sqrt {b^2 - 4ac}}}{2a}$ 216 | unicode: José and François 217 | 218 | Multi-line strings are all adjacent lines that are prefixed with a string tag; 219 | the greater-than symbol followed by a space or a newline (``>␣`` or ``>↵``). 220 | The content of each line starts after the first space that follows the 221 | greater-than symbol: 222 | 223 | .. code-block:: nestedtext 224 | 225 | > This is the first line of a multiline string, it is indented. 226 | > This is the second line, it is not indented. 227 | 228 | You can include empty lines in the string simply by specifying the greater-than 229 | symbol alone on a line: 230 | 231 | .. code-block:: nestedtext 232 | 233 | > 234 | > “The worth of a man to his society can be measured by the contribution he 235 | > makes to it — less the cost of sustaining himself and his mistakes in it.” 236 | > 237 | > — Erik Jonsson 238 | > 239 | 240 | The multiline string is all adjacent lines that start with a string tag with the 241 | tags removed and the lines joined together with newline characters inserted 242 | between each line. Except for the space that follows the ``>`` in the tag, 243 | white space from both the beginning and the end of each line is retained, along 244 | with all newlines except the last. 245 | 246 | Inline strings are the string values specified in inline dictionaries and lists. 247 | They are somewhat constrained in the characters that they may contain; nothing 248 | that might be confused with the syntax characters used by the inline list or 249 | dictionary that contains it. Specifically, inline strings may not contain 250 | newlines or any of the following characters: ``[``, ``]``, ``{``, ``}``, or 251 | ``,``. In addition, inline strings that are contained in inline dictionaries 252 | may not contain ``:``. Leading and trailing white space are ignored with inline 253 | strings. 254 | 255 | 256 | .. _comments: 257 | 258 | Comments 259 | ======== 260 | 261 | Lines that begin with a hash as the first non-white-space character, or lines 262 | that are empty or consist only of white space are comment lines and are ignored. 263 | Indentation is not significant on comment lines. 264 | 265 | .. code-block:: nestedtext 266 | 267 | # this line is ignored 268 | 269 | # this line is also ignored, as is the blank line above. 270 | 271 | Comment lines are ignored when determining whether adjacent lines belong to the 272 | same dictionary, list, or string. For example, the following represents one 273 | multiline string: 274 | 275 | .. code-block:: nestedtext 276 | 277 | > this is the first line of a multiline string 278 | # this line is ignored 279 | > this is the second line of the multiline string 280 | 281 | 282 | .. _nesting: 283 | 284 | Nesting 285 | ======= 286 | 287 | A value for a dictionary or list item may be a rest-of-line string or it may be 288 | a nested dictionary, list, multiline string, or inline dictionary or list. 289 | Indentation is used to indicate nesting. Indentation increases to indicate the 290 | beginning of a new nested object, and indentation returns to a prior level to 291 | indicate its end. In this way, data can be nested to an arbitrary depth: 292 | 293 | .. code-block:: nestedtext 294 | 295 | # Contact information for our officers 296 | 297 | Katheryn McDaniel: 298 | position: president 299 | address: 300 | > 138 Almond Street 301 | > Topeka, Kansas 20697 302 | phone: 303 | cell: 1-210-555-5297 304 | work: 1-210-555-3423 305 | home: 1-210-555-8470 306 | # Katheryn prefers that we always call her on her cell phone. 307 | email: KateMcD@aol.com 308 | kids: 309 | - Joanie 310 | - Terrance 311 | 312 | Margaret Hodge: 313 | position: vice president 314 | address: 315 | > 2586 Marigold Lane 316 | > Topeka, Kansas 20697 317 | phone: 318 | {cell: 1-470-555-0398, home: 1-470-555-7570} 319 | email: margaret.hodge@ku.edu 320 | kids: 321 | [Arnie, Zach, Maggie] 322 | 323 | It is recommended that each level of indentation be represented by a consistent 324 | number of spaces (with the suggested number being 2 or 4). However, it is not 325 | required. Any increase in the number of spaces in the indentation represents an 326 | indent and the number of spaces need only be consistent over the length of the 327 | nested object. 328 | 329 | The data can be nested arbitrarily deeply. 330 | 331 | 332 | .. _nestedtext_files: 333 | 334 | NestedText Files 335 | ================ 336 | 337 | *NestedText* files should be encoded with `UTF-8 338 | `_ and should end with a newline. The 339 | top-level object must not be indented. 340 | 341 | The name used for the file is arbitrary but it is tradition to use a 342 | .nt suffix. If you also wish to further distinguish the file type 343 | by giving the schema, it is recommended that you use two suffixes, 344 | with the suffix that specifies the schema given first and .nt given 345 | last. For example: officers.addr.nt. 346 | -------------------------------------------------------------------------------- /doc/basic_use.rst: -------------------------------------------------------------------------------- 1 | .. currentmodule:: nestedtext 2 | 3 | ********* 4 | Basic use 5 | ********* 6 | 7 | The *NestedText* Python API is similar to that of *JSON*, *YAML*, *TOML*, etc. 8 | 9 | 10 | Installation 11 | ------------ 12 | 13 | *NestedText* is also available from *pip*. Install it with: 14 | 15 | .. code-block:: text 16 | 17 | pip3 install nestedtext 18 | 19 | Alternately, *Inform* is also available in *Conda*. Install it with: 20 | 21 | .. code-block:: text 22 | 23 | conda install nestedtext --channel conda-forge 24 | 25 | 26 | NestedText Reader 27 | ----------------- 28 | 29 | The :func:`loads` function is used to convert *NestedText* held in a string into 30 | a Python data structure. If there is a problem interpreting the input text, 31 | a :exc:`NestedTextError` exception is raised. 32 | 33 | .. code-block:: python 34 | 35 | >>> import nestedtext as nt 36 | 37 | >>> content = """ 38 | ... access key id: 8N029N81 39 | ... secret access key: 9s83109d3+583493190 40 | ... """ 41 | 42 | >>> try: 43 | ... data = nt.loads(content, top='dict') 44 | ... except nt.NestedTextError as e: 45 | ... e.terminate() 46 | 47 | >>> print(data) 48 | {'access key id': '8N029N81', 'secret access key': '9s83109d3+583493190'} 49 | 50 | You can also read directly from a file or stream using the :func:`load` 51 | function. 52 | 53 | .. code-block:: python 54 | 55 | >>> from inform import fatal, os_error 56 | 57 | >>> try: 58 | ... groceries = nt.load('examples/groceries.nt', top='dict') 59 | ... except nt.NestedTextError as e: 60 | ... e.terminate() 61 | ... except OSError as e: 62 | ... fatal(os_error(e)) 63 | 64 | >>> print(groceries) 65 | {'groceries': ['Bread', 'Peanut butter', 'Jam']} 66 | 67 | Notice that the type of the return value is specified to be 'dict'. This is the 68 | default. You can also specify 'list', 'str', or 'any' (or *dict*, *list*, *str*, 69 | or *any*). All but 'any' constrain the data type of the top-level of the 70 | *NestedText* content. 71 | 72 | The *load* functions provide a *keymap* argument that is useful for adding line 73 | numbers to error message. This feature is demonstrated in :ref:`voluptuous 74 | example`. They also provide a *normalize_key* argument that can be used to 75 | ignore insignificant variation in keys, such as character case, or to convert 76 | keys to a desired form, such as to identifiers. These features are described in 77 | :meth:`loads`. 78 | 79 | 80 | NestedText Writer 81 | ----------------- 82 | 83 | The :func:`dumps` function is used to convert a Python data structure into 84 | a *NestedText* string. As before, if there is a problem converting the input 85 | data, a :exc:`NestedTextError` exception is raised. 86 | 87 | .. code-block:: python 88 | 89 | >>> try: 90 | ... content = nt.dumps(data) 91 | ... except nt.NestedTextError as e: 92 | ... e.terminate() 93 | 94 | >>> print(content) 95 | access key id: 8N029N81 96 | secret access key: 9s83109d3+583493190 97 | 98 | The :func:`dump` function writes *NestedText* to a file or stream. 99 | 100 | .. code-block:: python 101 | 102 | >>> try: 103 | ... nt.dump(data, 'examples/access.nt') 104 | ... except nt.NestedTextError as e: 105 | ... e.terminate() 106 | ... except OSError as e: 107 | ... fatal(os_error(e)) 108 | 109 | The *dump* functions provide arguments that can control the output format and 110 | can control the conversion of data types into forms that can be dumped. These 111 | features are described in :meth:`dumps`. 112 | -------------------------------------------------------------------------------- /doc/changelog.rst: -------------------------------------------------------------------------------- 1 | .. _language changes: 2 | 3 | **************** 4 | Language changes 5 | **************** 6 | 7 | .. currentmodule:: nestedtext 8 | 9 | Currently the language and the :ref:`Python implementation ` share version numbers. Since the language is more stable than the 11 | implementation, you will see versions that include no changes to the language. 12 | 13 | 14 | Latest development version 15 | -------------------------- 16 | 17 | | Version: 3.8.dev2 18 | | Released: 2025-04-05 19 | 20 | 21 | .. _v3.7: 22 | 23 | v3.7 (2024-04-27) 24 | ----------------- 25 | 26 | - Clarified policy on white space in inline strings. 27 | 28 | 29 | .. _v3.6: 30 | 31 | v3.6 (2023-05-30) 32 | ----------------- 33 | 34 | - No changes. 35 | 36 | 37 | .. _v3.5: 38 | 39 | v3.5 (2022-11-04) 40 | ----------------- 41 | 42 | - No changes. 43 | 44 | 45 | .. _v3.4: 46 | 47 | v3.4 (2022-06-15) 48 | ----------------- 49 | 50 | - No changes. 51 | 52 | 53 | .. _v3.3: 54 | 55 | v3.3 (2022-06-07) 56 | ----------------- 57 | 58 | - Defined *Minimal NestedText*, a subset of *NestedText*. 59 | - *NestedText* document files should end with a newline. 60 | 61 | 62 | .. _v3.2: 63 | 64 | v3.2 (2022-01-17) 65 | ----------------- 66 | 67 | - No changes. 68 | 69 | 70 | .. _v3.1: 71 | 72 | v3.1 (2021-07-23) 73 | ----------------- 74 | 75 | - No changes. 76 | 77 | 78 | .. _v3.0: 79 | 80 | v3.0 (2021-07-17) 81 | ----------------- 82 | 83 | - Deprecate trailing commas in inline lists and dictionaries. 84 | 85 | .. warning:: 86 | 87 | Be aware that aspects of this version are not backward compatible. 88 | Specifically, trailing commas are no longer supported in inline 89 | dictionaries and lists. In addition, ``[ ]`` now represents a list 90 | that contains an empty string, whereas previously it represented an 91 | empty list. 92 | 93 | 94 | .. _v2.0: 95 | 96 | v2.0 (2021-05-28) 97 | ----------------- 98 | 99 | - Deprecate quoted dictionary keys. 100 | - Add multiline dictionary keys to replace quoted keys. 101 | - Add single-line lists and dictionaries. 102 | 103 | .. warning:: 104 | 105 | Be aware that this version is not backward compatible because it no 106 | longer supports quoted dictionary keys. 107 | 108 | 109 | v1.3 (2021-01-02) 110 | ----------------- 111 | 112 | - No changes. 113 | 114 | 115 | v1.2 (2020-10-31) 116 | ----------------- 117 | 118 | - Treat CR LF, CR, or LF as a line break. 119 | 120 | 121 | v1.1 (2020-10-13) 122 | ----------------- 123 | 124 | - No changes. 125 | 126 | 127 | .. _v1.0: 128 | 129 | v1.0 (2020-10-03) 130 | ----------------- 131 | 132 | - Initial release. 133 | -------------------------------------------------------------------------------- /doc/common_mistakes.rst: -------------------------------------------------------------------------------- 1 | .. hide: 2 | 3 | >>> from inform import Inform 4 | >>> ignore = Inform(prog_name=False) 5 | 6 | 7 | *************** 8 | Common mistakes 9 | *************** 10 | 11 | .. currentmodule:: nestedtext 12 | 13 | Two values for one key 14 | ---------------------- 15 | 16 | When :func:`load()` or :func:`loads()` complains of errors it is important to 17 | look both at the line fingered by the error message and the one above it. The 18 | line that is the target of the error message might by an otherwise valid 19 | *NestedText* line if it were not for the line above it. For example, consider 20 | the following example: 21 | 22 | **Example:** 23 | 24 | .. code-block:: python 25 | 26 | >>> import nestedtext as nt 27 | 28 | >>> content = """ 29 | ... treasurer: 30 | ... name: Fumiko Purvis 31 | ... address: Home 32 | ... > 3636 Buffalo Ave 33 | ... > Topeka, Kansas 20692 34 | ... """ 35 | 36 | >>> try: 37 | ... data = nt.loads(content) 38 | ... except nt.NestedTextError as e: 39 | ... print(e.get_message()) 40 | ... print(e.get_codicil()[0]) 41 | invalid indentation. 42 | An indent may only follow a dictionary or list item that does not 43 | already have a value. 44 | 4 ❬ address: Home❭ 45 | 5 ❬ > 3636 Buffalo Ave❭ 46 | ▲ 47 | 48 | Notice that the complaint is about line 5, but problem stems from line 4 where 49 | *Home* gave a value to *address*. With a value specified for *address*, any 50 | further indentation on line 5 indicates a second value is being specified for 51 | *address*, which is illegal. 52 | 53 | A more subtle version of this same error follows: 54 | 55 | **Example:** 56 | 57 | .. code-block:: python 58 | 59 | >>> content = """ 60 | ... treasurer: 61 | ... name: Fumiko Purvis 62 | ... address:␣␣ 63 | ... > 3636 Buffalo Ave 64 | ... > Topeka, Kansas 20692 65 | ... """ 66 | 67 | >>> try: 68 | ... data = nt.loads(content.replace('␣␣', ' ')) 69 | ... except nt.NestedTextError as e: 70 | ... print(e.get_message()) 71 | ... print(e.get_codicil()[0]) 72 | invalid indentation. 73 | An indent may only follow a dictionary or list item that does not 74 | already have a value, which in this case consists only of whitespace. 75 | 4 ❬ address: ❭ 76 | 5 ❬ > 3636 Buffalo Ave❭ 77 | ▲ 78 | 79 | Notice the ␣␣ that follows *address* in *content*. These are replaced by 80 | 2 spaces before *content* is processed by *loads*. Thus, in this case there is 81 | an extra space at the end of line 4. Anything beyond the: ``:␣`` is considered 82 | the value for *address*, and in this case that is the single extra space 83 | specified at the end of the line. This extra space is taken to be the value of 84 | *address*, making the multiline string in lines 5 and 6 a value too many. 85 | 86 | This mistake is easier to see in advance if you configure your editor to show 87 | trailing whitespace. To do so in Vim, add:: 88 | 89 | set listchars=trail:␣ 90 | 91 | to your ~/.vimrc file. 92 | 93 | 94 | Lists or strings at the top level 95 | --------------------------------- 96 | 97 | Most *NestedText* files start with key-value pairs at the top-level and we 98 | noticed that many developers would simply assume this in their code, which would 99 | result in unexpected crashes when their programs read legal *NestedText* files 100 | that had either a list or a string at the top level. To avoid this, the 101 | :func:`load` and :func:`loads` functions are configured to expect a dictionary 102 | at the top level by default, which results in an error being reported if 103 | a dictionary key is not the first token found: 104 | 105 | .. code-block:: python 106 | 107 | >>> import nestedtext as nt 108 | 109 | >>> content = """ 110 | ... - a 111 | ... - b 112 | ... """ 113 | 114 | >>> try: 115 | ... print(nt.loads(content)) 116 | ... except nt.NestedTextError as e: 117 | ... e.report() 118 | error: 2: content must start with key or brace ({). 119 | 2 ❬- a❭ 120 | 121 | This restriction is easily removed using *top*: 122 | 123 | .. code-block:: python 124 | 125 | >>> try: 126 | ... print(nt.loads(content, top=list)) 127 | ... except nt.NestedTextError as e: 128 | ... e.report() 129 | ['a', 'b'] 130 | 131 | The *top* argument can take any of the values shown in the table below. The 132 | default value is *dict*. The value given for *top* also determines the value 133 | returned by :func:`load` and :func:`loads` if the *NestedText* document is 134 | empty. 135 | 136 | ================ ================================= 137 | *top* value returned for empty document 138 | ---------------- --------------------------------- 139 | *"dict"*, *dict* ``{}`` 140 | *"list"*, *list* ``[]`` 141 | *"str"*, *str* ``""`` 142 | *"any"*, *any* *None* 143 | ================ ================================= 144 | -------------------------------------------------------------------------------- /doc/conf.py: -------------------------------------------------------------------------------- 1 | import sys, os 2 | 3 | # General 4 | 5 | project = u'NestedText' 6 | copyright = u'2020-2025, Ken and Kale Kundert' 7 | release = '3.8.dev2' 8 | version = '.'.join(release.split('.')) 9 | 10 | master_doc = 'index' 11 | source_suffix = '.rst' 12 | templates_path = ['.templates'] 13 | exclude_patterns = ['.build'] 14 | 15 | # Extensions 16 | 17 | extensions = '''\ 18 | sphinx.ext.autodoc 19 | sphinx.ext.autosummary 20 | sphinx.ext.coverage 21 | sphinx.ext.doctest 22 | sphinx.ext.napoleon 23 | sphinx.ext.viewcode 24 | sphinx_rtd_theme 25 | sphinx_toolbox.collapse 26 | '''.split() 27 | 28 | autosummary_generate = True 29 | html_theme = 'sphinx_rtd_theme' 30 | pygments_style = 'sphinx' 31 | html_static_path = ['.static'] 32 | 33 | def setup(app): 34 | import os 35 | if os.path.exists('.static/css/custom.css'): 36 | app.add_css_file('css/custom.css') 37 | 38 | # The following are needed by the ReadTheDocs website. KSK 240726 39 | 40 | # Define the canonical URL if you are using a custom domain on Read the Docs 41 | html_baseurl = os.environ.get("READTHEDOCS_CANONICAL_URL", "") 42 | 43 | # Tell Jinja2 templates the build is running on Read the Docs 44 | if os.environ.get("READTHEDOCS", "") == "True": 45 | if "html_context" not in globals(): 46 | html_context = {} 47 | html_context["READTHEDOCS"] = True 48 | -------------------------------------------------------------------------------- /doc/examples.rst: -------------------------------------------------------------------------------- 1 | ******** 2 | Examples 3 | ******** 4 | 5 | .. doctests: 6 | 7 | >>> import nestedtext as nt 8 | 9 | .. currentmodule:: nestedtext 10 | 11 | .. _json-to-nestedtext example: 12 | 13 | JSON to NestedText 14 | ================== 15 | 16 | This example implements a command-line utility that converts a *JSON* file to 17 | *NestedText*. It demonstrates the use of :func:`dumps()` and 18 | :exc:`NestedTextError`. 19 | 20 | .. literalinclude:: ../examples/conversion-utilities/json-to-nestedtext 21 | :language: python 22 | 23 | Be aware that not all *JSON* data can be converted to *NestedText*, and in the 24 | conversion much of the type information is lost. 25 | 26 | *json-to-nestedtext* can be used as a JSON pretty printer: 27 | 28 | .. code-block:: text 29 | 30 | > json-to-nestedtext < fumiko.json 31 | treasurer: 32 | name: Fumiko Purvis 33 | address: 34 | > 3636 Buffalo Ave 35 | > Topeka, Kansas 20692 36 | phone: 1-268-555-0280 37 | email: fumiko.purvis@hotmail.com 38 | additional roles: 39 | - accounting task force 40 | 41 | 42 | .. _nestedtext-to-json example: 43 | 44 | NestedText to JSON 45 | ================== 46 | 47 | This example implements a command-line utility that converts a *NestedText* file 48 | to *JSON*. It demonstrates the use of :func:`load()` and 49 | :exc:`NestedTextError`. 50 | 51 | .. literalinclude:: ../examples/conversion-utilities/nestedtext-to-json 52 | :language: python 53 | 54 | 55 | .. ignore: 56 | .. _csv-to-nestedtext example: 57 | 58 | CSV to NestedText 59 | ================= 60 | 61 | This example implements a command-line utility that converts a *CSV* file to 62 | *NestedText*. It demonstrates the use of the *converters* argument to 63 | :func:`dumps()`, which is used to cull empty dictionary fields. 64 | 65 | .. literalinclude:: ../examples/conversion-utilities/csv-to-nestedtext 66 | :language: python 67 | 68 | 69 | .. _parametrize-from-file example: 70 | 71 | PyTest 72 | ====== 73 | 74 | This example highlights a PyTest_ package parametrize_from_file_ that allows you 75 | to neatly separate your test code from your test cases; the test cases being 76 | held in a *NestedText* file. Since test cases often contain code snippets, the 77 | ability of *NestedText* to hold arbitrary strings without the need for quoting 78 | or escaping results in very clean and simple test case specifications. Also, 79 | use of the *eval* function in the test code allows the fields in the test cases 80 | to be literal Python code. 81 | 82 | The test cases: 83 | 84 | .. literalinclude:: ../examples/parametrize_from_file/test_misc.nt 85 | :language: nestedtext 86 | 87 | And the corresponding test code: 88 | 89 | .. literalinclude:: ../examples/parametrize_from_file/test_misc.py 90 | :language: python 91 | 92 | 93 | .. _postmortem example: 94 | 95 | PostMortem 96 | ========== 97 | 98 | PostMortem_ is a program that generates a packet of information that is securely 99 | shared with your dependents in case of your death. Only the settings processing 100 | part of the package is shown here. 101 | 102 | This example includes :ref:`references `, :ref:`key normalization 103 | `, and different way to implement validation and conversion on 104 | a per field basis with voluptuous_. References allow you to define some content 105 | once and insert that content multiple places in the document. Key normalization 106 | allows the keys to be case insensitive and contain white space even though the 107 | program that uses the data prefers the keys to be lower case identifiers. 108 | 109 | Here is a configuration file that Odin might use to generate packets for his 110 | wife and kids: 111 | 112 | .. literalinclude:: ../examples/postmortem/postmortem.nt 113 | :language: nestedtext 114 | 115 | Notice that *estate docs* is defined at the top level. It is not a *PostMortem* 116 | setting; it simply defines a value that will be interpolated into settings 117 | later. The interpolation is done by specifying ``@`` along with the name of the 118 | reference as a value. So for example, in *recipients* *attach* is specified as 119 | ``@ estate docs``. This causes the list of estate documents to be used as 120 | attachments. The same thing is done in *sign with*, which interpolates *my gpg 121 | ids*. 122 | 123 | Here is the code for validating and transforming the *PostMortem* settings. For 124 | more on *report_voluptuous_errors*, see :ref:`voluptuous example`. 125 | 126 | .. literalinclude:: ../examples/postmortem/postmortem 127 | :language: python 128 | 129 | This code uses *expand_settings* to implement references, and it uses the 130 | *Voluptuous* schema to clean and validate the settings and convert them to 131 | convenient forms. For example, the user could specify *attach* as a string or 132 | a list, and the members could use a leading ``~`` to signify a home directory. 133 | Applying *to_paths* in the schema converts whatever is specified to a list and 134 | converts each member to a pathlib_ path with the ``~`` properly expanded. 135 | 136 | Notice that the schema is defined in a different manner than in the :ref:` 137 | Volupuous example `. In that example, you simply state 138 | which type you are expecting for the value and you use the *Coerce* function to 139 | indicate that the value should be cast to that type if needed. In this example, 140 | simple functions are passed in that perform validation and coercion as needed. 141 | This is a more flexible approach that allows better control of the conversions 142 | and the error messages. 143 | 144 | This code does not do any thing useful, it just reads in and expands the 145 | information contained in the input file. It simply represents the beginnings of 146 | a program that would use the specified information to generate the postmortem 147 | reports. In this case it simply prints the expanded information in the form of 148 | a *NestedText* document, which is easier to read that if it were pretty-printed 149 | as *Python* or *JSON*. 150 | 151 | Here are the processed settings: 152 | 153 | .. literalinclude:: ../examples/postmortem/postmortem.expanded.nt 154 | :language: nestedtext 155 | 156 | 157 | .. _voluptuous: https://github.com/alecthomas/voluptuous 158 | .. _PyTest: https://docs.pytest.org 159 | .. _parametrize_from_file: https://parametrize-from-file.readthedocs.io 160 | .. _pathlib: https://docs.python.org/3/library/pathlib.html 161 | .. _PostMortem: https://github.com/kenkundert/postmortem 162 | -------------------------------------------------------------------------------- /doc/file_format.rst: -------------------------------------------------------------------------------- 1 | .. _nestedtext file format: 2 | 3 | ****************** 4 | Language reference 5 | ****************** 6 | 7 | The *NestedText* format follows a small number of simple rules. Here they are. 8 | 9 | 10 | **Encoding**: 11 | 12 | A *NestedText* document is encoded in UTF-8 and may contain any printing 13 | UTF-8 character. 14 | 15 | 16 | **Line breaks**: 17 | 18 | A *NestedText* document is partitioned into lines where the lines are split 19 | by CR LF, CR, or LF where CR and LF are the ASCII carriage return and line 20 | feed characters. A single document may employ any or all of these ways of 21 | splitting lines. 22 | 23 | 24 | **Line types**: 25 | 26 | Each line in a *NestedText* document is assigned one of the following types: 27 | *comment*, *blank*, *list item*, *dictionary item*, *string item*, *key 28 | item* or *inline*. Any line that does not fit one of these types is an 29 | error. 30 | 31 | 32 | **Blank lines**: 33 | 34 | Blank lines are lines that are empty or consist only of ASCII space 35 | characters. Blank lines are ignored. 36 | 37 | 38 | **Line-type tags**: 39 | 40 | Most remaining lines are identified by the presence of tags, where a tag is: 41 | 42 | #. the first dash (``-``), colon (``:``), or greater-than symbol (``>``) on 43 | a line when followed immediately by an ASCII space or line break; 44 | #. or a hash {``#``), left bracket (``[``), or left brace (``{``) as the 45 | first non-ASCII-space character on a line. 46 | 47 | These symbols only introduce tags when they are the first non-ASCII-space 48 | character on a line, except for the colon (``:``) which introduces 49 | a dictionary item with an inline key midway through a line. 50 | 51 | The first (left-most) tag on a line determines the line type. Once the 52 | first tag has been found on the line, any subsequent occurrences of any of 53 | the line-type tags are treated as simple text. For example: 54 | 55 | .. code-block:: nestedtext 56 | 57 | - And the winner is: {winner} 58 | 59 | In this case the leading ``-␣`` determines the type of the line and the 60 | ``:␣`` is simply treated as part of the remaining text on the line. 61 | 62 | 63 | **Comments**: 64 | 65 | Comments are lines that have ``#`` as the first non-ASCII-space character on 66 | the line. Comments are ignored. 67 | 68 | 69 | **String items**: 70 | 71 | If the first non-space character on a line is a greater-than symbol followed 72 | immediately by an ASCII space (``>␣``) or a line break, the line is a *string 73 | item*. After comments and blank lines have been removed, adjacent string 74 | items with the same indentation level are combined in order into 75 | a multiline string. The string value is the multiline string with the 76 | tags removed. Any leading white space that follows the tag is retained, as 77 | is any trailing white space. The last newline is removed and all other 78 | newlines are converted to the default line terminator for the current 79 | operating system. 80 | 81 | String values may contain any printing UTF-8 character. 82 | 83 | 84 | **List items**: 85 | 86 | If the first non-space character on a line is a dash followed immediately by 87 | an ASCII space (``-␣``) or a line break, the line is a *list item*. 88 | Adjacent list items with the same indentation level are combined in order 89 | into a list. Each list item has a tag and a value. The tag is only used to 90 | determine the type of the line and is discarded leaving the value. The 91 | value takes one of three forms. 92 | 93 | #. If the line contains further text (characters after the dash-space), then 94 | the value is that text. The text ends at the line break and may contain 95 | any other printing UTF-8 character. 96 | 97 | #. If there is no further text on the line and the next line has greater 98 | indentation, then the next line holds the value, which may be a list, 99 | a dictionary, or a multiline string. 100 | 101 | #. Otherwise the value is empty; it is taken to be an empty string. 102 | 103 | 104 | **Key items**: 105 | 106 | If the first non-ASCII-space character on a line is a colon followed 107 | immediately by an ASCII space (``:␣``) or a line break, the line is a *key 108 | item*. After comments and blank lines have been removed, adjacent key items 109 | with the same indentation level are combined in order into a multiline key. 110 | The key itself is the multiline string with the tags removed. Any leading 111 | white space that follows the tag is retained, as is any trailing white 112 | space. The last newline is removed and all other newlines are converted to 113 | the default line terminator for the current operating system. 114 | 115 | Key values may contain any printing UTF-8 character. 116 | 117 | An indented value must follow a multiline key. The indented value may be 118 | either a multiline string, a list or a dictionary. The combination of the 119 | key item and its value forms a *dictionary item*. 120 | 121 | 122 | **Dictionary items**: 123 | 124 | Dictionary items take two possible forms. 125 | 126 | The first is a *dictionary item with inline key*. In this case the line 127 | starts with a key followed by a dictionary tag: a colon followed by either 128 | an ASCII space (``:␣``) or a newline. The dictionary item consists of the 129 | key, the tag, and the trailing value. Any white space between the key and 130 | the tag is ignored. 131 | 132 | The inline key precedes the tag. It must be a non-empty string and must not: 133 | 134 | #. contain a line break character. 135 | #. start with a list item, string item or key item tag, 136 | #. start with ``[`` or ``{``, 137 | #. contain a dictionary item tag, or 138 | #. contain Unicode leading spaces 139 | (any Unicode spaces that follow the key are ignored). 140 | 141 | The tag is only used to determine the type of the line and is discarded 142 | leaving the key and the value, which follows the tag. The value takes one 143 | of three forms. 144 | 145 | #. If the line contains further text (characters after the colon-space), 146 | then the value is that text. The text ends at the line break and may 147 | contain any other printing UTF-8 character. 148 | 149 | #. If there is no further text on the line and the next line has greater 150 | indentation, then the next line holds the value, which may be a list, 151 | a dictionary, or a multiline string. 152 | 153 | #. Otherwise the value is empty; it is taken to be an empty string. 154 | 155 | The second form of *dictionary item* is the *dictionary item with multiline 156 | key*. It consists of a multiline key value followed by an indented value. 157 | The value may be a multiline string, list, or dictionary; or an inline list 158 | or dictionary. 159 | 160 | Adjacent dictionary items of either form with the same indentation level are 161 | combined in order into a dictionary. 162 | 163 | 164 | **Inline Lists and Dictionaries**: 165 | 166 | If the first non-ASCII-space character on a line is either a left bracket 167 | (``[``) or a left brace (``{``) the line is an *inline structure*. 168 | A bracket introduces an inline list and a brace introduces an inline 169 | dictionary. 170 | 171 | Inlines are confined to a single line, and so must not contain any 172 | line-break white space, such as newlines. 173 | 174 | An *inline list* starts with an open bracket (``[``), ends with a matching 175 | closed bracket (``]``), contains inline values separated by commas (``,``), 176 | and is contained on a single line. The values may be inline strings, inline 177 | lists, and inline dictionaries. 178 | 179 | An *inline dictionary* starts with an open brace (``{``), ends with 180 | a matching closed brace (``}``), contains inline dictionary items separated 181 | by commas (``,``), and is contained on a single line. An inline dictionary 182 | item is a key and value separated by a colon (``:``). A space need not 183 | follow the colon and any white space that does follow the colon is ignored. 184 | The keys are inline strings and the values may be inline strings, inline 185 | lists, and inline dictionaries. 186 | 187 | *Inline strings* are the string values specified in inline dictionaries and 188 | lists. They are somewhat constrained in the characters that they may 189 | contain; nothing is allowed that might be confused with the syntax 190 | characters used by the inline list or dictionary that contains it. 191 | Specifically, inline strings may not include line-break white space 192 | characters such as newlines or any of the following characters: ``[``, 193 | ``]``, ``{``, ``}``, or ``,``. In addition, inline strings that are 194 | contained in inline dictionaries may not contain ``:``. Both leading and 195 | trailing white space is ignored with inline strings. This includes all 196 | non-line-break white space characters such as ASCII spaces and tabs, as well 197 | as the various Unicode white space characters. 198 | 199 | Both inline lists and dictionaries may be empty, and represent the only way 200 | to represent empty lists or empty dictionaries in *NestedText*. An empty 201 | dictionary is represented with ``{}`` and an empty list with ``[]``. In 202 | both cases there must be no space between the opening and closing 203 | delimiters. An inline list that contains only white spaces, such as ``[ 204 | ]``, is treated as a list with a single empty string (the whitespace is 205 | considered a string value, and string values have leading and trailing 206 | spaces removed, resulting in an empty string value). If a list contains 207 | multiple values, no white space is required to represent an empty string 208 | value. Thus, ``[]`` represents an empty list, ``[ ]`` a list with a single 209 | empty string value, and ``[,]`` a list with two empty string values. 210 | 211 | 212 | **Indentation**: 213 | 214 | Leading spaces on a line represents indentation. Only ASCII spaces are 215 | allowed in the indentation. Specifically, tabs and the various Unicode 216 | spaces are not allowed. 217 | 218 | There is no indentation on the top-level object. 219 | 220 | An increase in the number of spaces in the indentation signifies the start 221 | of a nested object. Indentation must return to a prior level when the 222 | nested object ends. 223 | 224 | Each level of indentation need not employ the same number of additional 225 | spaces, though it is recommended that you choose either 2 or 4 spaces to 226 | represent a level of nesting and you use that consistently throughout the 227 | document. However, this is not required. Any increase in the number of 228 | spaces in the indentation represents an indent and a decrease to return to 229 | a prior indentation represents a dedent. 230 | 231 | An indented value may only follow a list item or dictionary item that does 232 | not have a value on the same line. An indented value must follow a key 233 | item. 234 | 235 | 236 | **Escaping and Quoting**: 237 | 238 | There is no escaping or quoting in *NestedText*. Once the line has been 239 | identified by its tag, and the tag is removed, the remaining text is taken 240 | literally. 241 | 242 | 243 | **Empty document**: 244 | 245 | A document may be empty. A document is empty if it consists only of 246 | comments and blank lines. An empty document corresponds to an empty value 247 | of unknown type. Implementations may allow a default top-level type of 248 | dictionary, list, or string to be specified. 249 | 250 | 251 | **End of file**: 252 | 253 | The last character in a *NestedText* document file is a newline. 254 | 255 | 256 | **Result**: 257 | 258 | When a document is converted from *NestedText* the result is a hierarchical 259 | collection of dictionaries, lists and strings. All dictionary keys are 260 | strings. 261 | -------------------------------------------------------------------------------- /doc/index.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../README.rst 2 | 3 | .. toctree:: 4 | :caption: Language 5 | :maxdepth: 1 6 | 7 | Philosophy 8 | alternatives 9 | basic_syntax 10 | file_format 11 | minimal-nestedtext 12 | related_projects 13 | changelog 14 | 15 | .. toctree:: 16 | :caption: Python Implementation 17 | :maxdepth: 1 18 | 19 | basic_use 20 | schemas 21 | techniques 22 | examples 23 | common_mistakes 24 | python_api 25 | releases 26 | -------------------------------------------------------------------------------- /doc/minimal-nestedtext.rst: -------------------------------------------------------------------------------- 1 | .. _minimal nestedtext: 2 | 3 | ****************** 4 | Minimal NestedText 5 | ****************** 6 | 7 | *Minimal NestedText* is *NestedText* without support for multi-line keys and 8 | inline dictionaries and lists. 9 | 10 | *Minimal NestedText* is a subset of *NestedText* that foregoes some of the 11 | complications of *NestedText*. It sacrifices the completeness of *NestedText* 12 | for an even simpler data file format that is still appropriate for 13 | a surprisingly wide variety of applications, such as most configuration files. 14 | The simplicity of *Minimal NestedText* makes it very easy to create readers and 15 | writers. Indeed, writing such functions is good programming exercise for people 16 | new to recursion. 17 | 18 | If you choose to create a *Minimal NestedText* reader or writer it is important 19 | to code it in such a way as to discourage the creation *Minimal NestedText* 20 | documents that are invalid *NestedText*. Thus, your implementation should 21 | disallow keys that start with ``:␣``, ``[`` or ``{``. Also, please clearly 22 | indicate that your implementation only supports *Minimal NestedText* to avoid 23 | any confusion. 24 | 25 | Many of the examples given in this document conform to the *Minimal NestedText* 26 | subset. For convenience, here is another. It is a configuration file: 27 | 28 | .. code-block:: nestedtext 29 | 30 | default repository: home 31 | report style: tree 32 | compact format: {repo}: {size:{fmt}}. Last back up: {last_create:ddd, MMM DD}. 33 | normal format: {host:<8} {user:<5} {config:<9} {size:<8.2b} {last_create:ddd, MMM DD} 34 | date format: D MMMM YYYY 35 | size format: .2b 36 | 37 | repositories: 38 | # only the composite repositories need be included 39 | home: 40 | children: rsync borgbase 41 | caches: 42 | children: cache cache@media cache@files 43 | servers: 44 | children: 45 | - root@dev~root 46 | - root@mail~root 47 | - root@media~root 48 | - root@web~root 49 | all: 50 | children: home caches servers 51 | 52 | Finally, here is a short description of *Minimal NestedText* that you can use to 53 | describe to your users if you decide to use it for your application. 54 | 55 | *Minimal NestedText*: 56 | `NestedText `_ is a file format for holding 57 | structured data. It is intended to be easily entered, edited, or viewed by 58 | people. As such, the syntax is very simple and intuitive. 59 | 60 | It organizes the data into a nested collection of lists and name-value pairs 61 | where the leaf-level values are all simple text. For example, a simple 62 | collection of name-value pairs is represented using: 63 | 64 | .. code-block:: nestedtext 65 | 66 | Name 1: Value 1 67 | Name 2: Value 2 68 | 69 | The name and value are separated by a colon followed immediately by a space. 70 | The characters that follow the space are the value. 71 | 72 | A simple list is represented with: 73 | 74 | .. code-block:: nestedtext 75 | 76 | - Value 1 77 | - Value 2 78 | 79 | A list item is introduced by dash as the first non-blank character on a line 80 | followed by a space. The characters that follow the space are the value. 81 | 82 | Indentation is used to denote nesting. In this case the colon or dash is 83 | the last character on the line and is followed by an indented value. The 84 | value may be a collection of name-value pairs, a list, or a multi-line 85 | string. Every line of a multi-line string is introduced by a greater-than 86 | symbol followed by a space or newline. 87 | 88 | .. code-block:: nestedtext 89 | 90 | Name 1: Value 1 91 | Name 2: 92 | Name 2a: Value 2a 93 | Name 2b: Value 2b 94 | Name 3: 95 | - Value 3a 96 | - Value 3b 97 | Name 4: 98 | > Value 4 line 1 99 | > Value 4 line 2 100 | 101 | Any line that starts with pound sign (`#`) as the first non-blank character 102 | is ignored and so can be used to add comments. 103 | 104 | .. code-block:: nestedtext 105 | 106 | # this line is a comment 107 | Name: Value 108 | 109 | The name in a name-value pair is referred to as a key. In *Minimal 110 | NestedText* keys cannot start with a space, an opening bracket (``[``) or 111 | brace (``{``), or a dash followed by a space. Nor can it contain a colon 112 | followed by a space. Other that that, there are no restrictions on the 113 | characters that make up a key or value, and any characters given are taken 114 | literally. 115 | -------------------------------------------------------------------------------- /doc/nestedtext.Location.rst: -------------------------------------------------------------------------------- 1 | Location 2 | ======== 3 | 4 | .. currentmodule:: nestedtext 5 | 6 | Location objects are returned from :func:`load` and :func:`loads` as the values 7 | in a *keymap*. They are also returned by :func:`get_location()`. Objects of 8 | this class holds the line and column numbers of the key and value tokens. 9 | 10 | .. autoclass:: Location 11 | :members: 12 | -------------------------------------------------------------------------------- /doc/nestedtext.NestedTextError.rst: -------------------------------------------------------------------------------- 1 | NestedTextError 2 | =============== 3 | 4 | .. currentmodule:: nestedtext 5 | 6 | .. autoexception:: NestedTextError 7 | :members: 8 | 9 | .. automethod:: NestedTextError.get_message 10 | .. automethod:: NestedTextError.get_culprit 11 | .. automethod:: NestedTextError.get_codicil 12 | .. automethod:: NestedTextError.report 13 | .. automethod:: NestedTextError.terminate 14 | .. automethod:: NestedTextError.reraise 15 | .. automethod:: NestedTextError.render 16 | -------------------------------------------------------------------------------- /doc/nestedtext.dumpers.rst: -------------------------------------------------------------------------------- 1 | Convert Data to *NestedText* 2 | ============================ 3 | 4 | .. currentmodule:: nestedtext 5 | 6 | .. autofunction:: dumps 7 | .. autofunction:: dump 8 | -------------------------------------------------------------------------------- /doc/nestedtext.loaders.rst: -------------------------------------------------------------------------------- 1 | Convert *NestedText* to Data 2 | ============================ 3 | 4 | .. currentmodule:: nestedtext 5 | 6 | .. autofunction:: loads 7 | .. autofunction:: load 8 | -------------------------------------------------------------------------------- /doc/nestedtext.utilities.rst: -------------------------------------------------------------------------------- 1 | .. currentmodule:: nestedtext 2 | 3 | Utilities 4 | ========= 5 | 6 | Extras that are useful when using *NestedText*. 7 | 8 | .. autofunction:: get_keys 9 | 10 | .. autofunction:: get_value 11 | 12 | .. autofunction:: get_line_numbers 13 | 14 | .. autofunction:: get_location 15 | 16 | 17 | Deprecated functions 18 | -------------------- 19 | 20 | These functions are to be removed in future versions. 21 | 22 | .. autofunction:: get_value_from_keys 23 | 24 | .. autofunction:: get_lines_from_keys 25 | 26 | .. autofunction:: get_original_keys 27 | 28 | .. autofunction:: join_keys 29 | -------------------------------------------------------------------------------- /doc/philosophy.rst: -------------------------------------------------------------------------------- 1 | *********************** 2 | The Zen of *NestedText* 3 | *********************** 4 | 5 | *NestedText* aspires to be a simple dumb receptacle that holds peoples' 6 | structured data and does so in a way that allows people to easily interact with 7 | that data. 8 | 9 | The desire to be simple is an attempt to minimize the effort required to learn 10 | and use the language. Ideally, people can understand it by looking at a few 11 | examples. And ideally, they can use it without needing to remember any arcane 12 | rules or relying on any knowledge that programmers accumulate through years of 13 | experience. One source of simplicity is consistency. As such, *NestedText* 14 | uses a small number of rules that it applies with few exceptions. 15 | 16 | The desire to be dumb means that *NestedText* tries not to transform the data in 17 | any meaningful way to avoid creating unpleasant surprises. It parses the 18 | structure of the data without doing anything that might change how the data is 19 | interpreted. Instead, it aims to make it easy for you to interpret the data 20 | yourself. After all, you understand what the data is supposed to mean, so you 21 | are in the best position to interpret it. There are also many powerful tools 22 | available to help with :doc:`this exact task `. 23 | -------------------------------------------------------------------------------- /doc/python_api.rst: -------------------------------------------------------------------------------- 1 | ********** 2 | Python API 3 | ********** 4 | 5 | .. get rid of autosummary, it keeps overriding Location 6 | .. autosummary:: 7 | :toctree: 8 | 9 | .. toctree:: 10 | 11 | nestedtext.dumpers 12 | nestedtext.loaders 13 | nestedtext.Location 14 | nestedtext.utilities 15 | nestedtext.NestedTextError 16 | -------------------------------------------------------------------------------- /doc/related_projects.rst: -------------------------------------------------------------------------------- 1 | Related projects 2 | ================ 3 | 4 | Reference Material 5 | ------------------ 6 | 7 | `NestedText Documentation `_ 8 | """""""""""""""""""""""""""""""""""""""""""""""""""" 9 | *NestedText* documentation and language specification. 10 | 11 | 12 | `NestedText Source `_ 13 | """"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" 14 | Source code repository for language documentation and Python implementation. 15 | Report any issues here. 16 | 17 | 18 | `NestedText Tests `_ 19 | """""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" 20 | Official *NestedText* test suite. Also included as submodule in 21 | `nestedtext `_. 22 | 23 | 24 | Implementations 25 | --------------- 26 | 27 | Go 28 | "" 29 | 30 | `nestex `_ 31 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 32 | 33 | `Go `_ implementation of *NestedText*. 34 | Supports :ref:`NestedText v3.0 `. 35 | 36 | 37 | Janet 38 | """"" 39 | 40 | `janet-nested-text `_ 41 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 42 | 43 | `Janet `_ implementation of *NestedText*. 44 | Supports :ref:`NestedText v3.0 `. 45 | 46 | 47 | JavaScript 48 | """""""""" 49 | 50 | `NestedText `_ 51 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 52 | 53 | Native JavaScript implementation of *NestedText*. It's not a WASM build of 54 | a compiled project, so it is significantly lighter in weight than 55 | *@rmw/nestedtext*. Also available from `GitHub 56 | `_. Supports :ref:`NestedText v3.0 57 | `. 58 | 59 | `@rmw/nestedtext `_ 60 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 61 | 62 | NodeJS (es module) implementation of *NestedText*. Uses `WASM 63 | `_ for *NestedText* decode. Supports 64 | :ref:`NestedText v3.0 `. 65 | 66 | 67 | .NET 68 | """" 69 | 70 | `NestedText `_ 71 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 72 | 73 | `.NET `_ implementation of *NestedText*. 74 | Supports :ref:`NestedText v3.7 `. 75 | 76 | 77 | Ruby 78 | """" 79 | 80 | `nestedtext-ruby `_ 81 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 82 | 83 | `Ruby `_ implementation of *NestedText*. 84 | Supports :ref:`NestedText v3.0 `. 85 | 86 | 87 | Zig 88 | """ 89 | 90 | `zig-nestedtext `_ 91 | """"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" 92 | 93 | `Zig `_ implementation of *NestedText* 94 | (slight subset of :ref:`NestedText v2.0 `). Also contains *nt-cli*, an 95 | efficient command line utility for converting between *NestedText* and *JSON*. 96 | 97 | 98 | Utilities 99 | --------- 100 | 101 | `NestedTextTo `_ 102 | """""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" 103 | Command line utilities for converting between *NestedText*, *JSON*, *YAML*, and 104 | *TOML*. 105 | 106 | 107 | `ntLog `_ 108 | """""""""""""""""""""""""""""""""""""""""""""" 109 | *ntlog* is a *NestedText* logfile aggregation utility. 110 | 111 | 112 | `parametrize from file `_ 113 | """"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" 114 | Separate your test cases, held in *NestedText*, 115 | from your `PyTest `_ test code. 116 | 117 | 118 | `pygments `_ 119 | """""""""""""""""""""""""""""""""""""""""""""""""""" 120 | Version of the popular *pygments* Python library that supports :ref:`NestedText 121 | v3.0 `. 122 | 123 | 124 | `vim-nestedtext `_ 125 | """""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" 126 | Vim syntax files for *NestedText* (supports :ref:`NestedText v3.0 `). 127 | 128 | 129 | `visual studio code `_ 130 | """""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" 131 | Syntax files for *Visual Studio Code* (supports :ref:`NestedText v3.0 `). 132 | -------------------------------------------------------------------------------- /doc/releases.rst: -------------------------------------------------------------------------------- 1 | .. _implementation changes: 2 | 3 | ******** 4 | Releases 5 | ******** 6 | 7 | .. currentmodule:: nestedtext 8 | 9 | This page documents the changes in the Python implementation of *NestedText*. 10 | Changes to the *NestedText* language are shown in :ref:`language changes`. 11 | 12 | 13 | Latest development version 14 | -------------------------- 15 | 16 | | Version: 3.8.dev2 17 | | Released: 2025-04-05 18 | 19 | **Bug Fixes**: 20 | 21 | - Catch multi-line key not followed by indented value. 22 | - Make line-ending recognition consistent for files and strings. 23 | - Fix bug in :func:`get_value()`. 24 | - Report error if there is content that follows top-level inline dictionary. 25 | 26 | **Enhancements**: 27 | 28 | - Support files that have utf8 byte-order marker (BOM). 29 | - Allow binary files to be passed to :func:`load()` and :func:`dump()`. 30 | - Allow byte strings to be passed to :func:`loads()`. 31 | 32 | **Tests**: 33 | 34 | In version 3.8 a new implementation independent set of tests is being developed. 35 | These tests should be more comprehensive and more independent of the Python 36 | implementation of *NestedText*. 37 | 38 | 39 | v3.7 (2024-04-27) 40 | ----------------- 41 | - Added ability to disable support for inlines using *dialect* argument to 42 | :func:`load()` and :func:`loads()`. 43 | - Added :func:`get_keys()`, :func:`get_value()`, :func:`get_line_numbers()`, and 44 | :func:`get_location()`. 45 | - Deprecated :func:`get_value_from_keys()`, :func:`get_lines_from_keys()`, 46 | :func:`get_original_keys()`, and :func:`join_keys()`. 47 | - Added *offset* argument to :meth:`Location.as_line()`. 48 | - Add ability to specify *source* to :func:`load()`. 49 | - Clarified policy on white space in inline strings. 50 | 51 | 52 | v3.6 (2023-05-30) 53 | ----------------- 54 | 55 | - De-duplicating with the *on_dup* argument to :func:`loads` now works well for 56 | error reporting with keymaps. 57 | - The *map_keys* argument has been added to :func:`dump` and :func:`dumps`. 58 | 59 | .. warning:: 60 | 61 | The *sort_keys* argument to :func:`dump` and :func:`dumps` has changed. 62 | When passing a call-back function to *sort_keys*, that call-back function 63 | now has a second argument, *parent_keys*. In addition, the first argument 64 | has changed. It is now a tuple with three members rather than two, with the 65 | new and leading member being the mapped key rather than the original key. 66 | 67 | .. warning:: 68 | 69 | The state passed to the *on_dup* functions of :func:`dump` and :func:`dumps` 70 | no longer contains the value associated with the duplicate key. 71 | 72 | 73 | v3.5 (2022-11-04) 74 | ----------------- 75 | 76 | - Minor refinements and fixes. 77 | 78 | 79 | v3.4 (2022-06-15) 80 | ----------------- 81 | 82 | - Improved the *on_dup* parameter to :meth:`load` and :meth:`loads`. 83 | - Added *strict* argument to :func:`join_keys`. 84 | 85 | .. warning:: 86 | 87 | Be aware that the new version of the *on_dup* parameter is not compatible 88 | with previous versions. 89 | 90 | 91 | v3.3 (2022-06-07) 92 | ----------------- 93 | 94 | - Add *normalize_key* argument to :meth:`load` and :meth:`loads`. 95 | - Added utility functions for operating on keys and keymaps: 96 | 97 | - :func:`get_value_from_keys` 98 | - :func:`get_lines_from_keys` 99 | - :func:`get_original_keys` 100 | - :func:`join_keys` 101 | 102 | - None passed as key is now converted to an empty string rather than "None". 103 | 104 | 105 | v3.2 (2022-01-17) 106 | ----------------- 107 | 108 | - Add circular reference detection and reporting. 109 | 110 | 111 | v3.1 (2021-07-23) 112 | ----------------- 113 | 114 | - Change error reporting for :func:`dumps` and :func:`dump` functions; 115 | culprit is now the keys rather than the value. 116 | 117 | 118 | v3.0 (2021-07-17) 119 | ----------------- 120 | 121 | - Deprecate trailing commas in inline lists and dictionaries. 122 | - Adds *keymap* argument to :func:`load` and :func:`loads`. 123 | - Adds *inline_level* argument to :func:`dump` and :func:`dumps`. 124 | - Implement *on_dup* argument to :func:`load` and :func:`loads` in inline 125 | dictionaries. 126 | - Apply *convert* and *default* arguments of :func:`dump` and :func:`dumps` to 127 | dictionary keys. 128 | 129 | .. warning:: 130 | 131 | Be aware that aspects of this version are not backward compatible. 132 | Specifically, trailing commas are no longer supported in inline dictionaries 133 | and lists. In addition, ``[ ]`` now represents a list that contains an 134 | empty string, whereas previously it represented an empty list. 135 | 136 | 137 | v2.0 (2021-05-28) 138 | ----------------- 139 | 140 | - Deprecate quoted keys. 141 | - Add multiline keys to replace quoted keys. 142 | - Add inline lists and dictionaries. 143 | - Move from *renderers* to *converters* in :func:`dump` and :func:`dumps`. 144 | Both allow you to support arbitrary data types. With *renderers* you 145 | provide functions that are responsible for directly creating the text to 146 | be inserted in the *NestedText* output. This can be complicated and error 147 | prone. With *converters* you instead convert the object to a known 148 | *NestedText* data type (dict, list, string, ...) and the *dump* function 149 | automatically formats it appropriately. 150 | - Restructure documentation. 151 | 152 | .. warning:: 153 | 154 | Be aware that aspects of this version are not backward compatible. 155 | 156 | 1. It no longer supports quoted dictionary keys. 157 | 158 | 2. The *renderers* argument to :func:`dump` and :func:`dumps` has been replaced by *converters*. 159 | 160 | 3. It no longer allows one to specify *level* in :func:`dump` and :func:`dumps`. 161 | 162 | 163 | v1.3 (2021-01-02) 164 | ----------------- 165 | 166 | - Move the test cases to a submodule. 167 | 168 | .. note:: 169 | 170 | When cloning the *NestedText* repository you should use the --recursive 171 | flag to get the *official_tests* submodule:: 172 | 173 | git clone --recursive https://github.com/KenKundert/nestedtext.git 174 | 175 | When updating an existing repository, you need to initialize the 176 | submodule after doing a pull:: 177 | 178 | git submodule update --init --remote tests/official_tests 179 | 180 | This only need be done once. 181 | 182 | 183 | v1.2 (2020-10-31) 184 | ----------------- 185 | 186 | - Treat CR LF, CR, or LF as a line break. 187 | - Always quote keys that start with a quote. 188 | 189 | 190 | v1.1 (2020-10-13) 191 | ----------------- 192 | 193 | - Add ability to specify return type of :func:`load` and :func:`loads`. 194 | - Quoted keys are now less restricted. 195 | - Empty dictionaries and lists are rejected by :func:`dump` and 196 | :func:`dumps` except as top-level object if *default* argument is 197 | specified as 'strict'. 198 | 199 | .. warning:: 200 | 201 | Be aware that this version is not fully backward compatible. Unlike 202 | previous versions, this version allows you to restrict the type of the 203 | return value of the :func:`load` and :func:`loads` functions, and the 204 | default is 'dict'. The previous behavior is still supported, but you 205 | must explicitly specify `top='any'` as an argument. 206 | 207 | This change results in a simpler return value from :func:`load` and 208 | :func:`loads` in most cases. This substantially reduces the chance of 209 | coding errors. It was noticed that it was common to simply assume that 210 | the top-level was a dictionary when writing code that used these 211 | functions, which could result in unexpected errors when users 212 | hand-create the input data. Specifying the return value eliminates this 213 | type of error. 214 | 215 | There is another small change that is not backward compatible. The 216 | source argument to these functions is now a keyword only argument. 217 | 218 | 219 | v1.0 (2020-10-03) 220 | ----------------- 221 | - Production release. 222 | 223 | 224 | .. ignore earlier releases: 225 | 226 | v0.6 (2020-09-26) 227 | ----------------- 228 | - Added :func:`load` and :func:`dump`. 229 | - Eliminated *NestedTextError.get_extended_codicil*. 230 | 231 | 232 | v0.5 (2020-09-11) 233 | ----------------- 234 | - allow user to manage duplicate keys detected by :func:`loads`. 235 | 236 | 237 | v0.4 (2020-09-07) 238 | ----------------- 239 | - Change rest-of-line strings to include all characters given, including 240 | leading and trailing quotes and spaces. 241 | - The *NestedText* top-level is no longer restricted to only dictionaries 242 | and lists. The top-level can now also be a single string. 243 | - :func:`loads` now returns *None* when given an empty *NestedText* document. 244 | - Change :exc:`NestedTextError` attribute names to make them more consistent 245 | with those used by JSON package. 246 | - Added *NestedTextError.get_extended_codicil*. 247 | 248 | 249 | v0.3 (2020-09-03) 250 | ----------------- 251 | - Allow comments to be indented. 252 | 253 | 254 | v0.2 (2020-09-02) 255 | ----------------- 256 | - Minor enhancements and bug fixes. 257 | 258 | 259 | v0.1 (2020-08-30) 260 | ----------------- 261 | - Initial release. 262 | -------------------------------------------------------------------------------- /doc/requirements.in: -------------------------------------------------------------------------------- 1 | autoclasstoc >= 1.6 2 | docutils==0.16 3 | # There is an incompatibility between Sphinx and docutils version 0.18. 4 | # Also, newer versions than 0.16 or perhaps 1.17 do not render bullets in bullet lists 5 | sphinx_toolbox 6 | sphinx_rtd_theme 7 | -------------------------------------------------------------------------------- /doc/requirements.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile with Python 3.12 3 | # by the following command: 4 | # 5 | # pip-compile ./requirements.in 6 | # 7 | alabaster==0.7.13 8 | # via sphinx 9 | apeye==1.4.1 10 | # via sphinx-toolbox 11 | apeye-core==1.1.4 12 | # via apeye 13 | autoclasstoc==1.6.0 14 | # via -r ./requirements.in 15 | autodocsumm==0.2.11 16 | # via sphinx-toolbox 17 | babel==2.13.1 18 | # via sphinx 19 | beautifulsoup4==4.12.2 20 | # via sphinx-toolbox 21 | cachecontrol[filecache]==0.13.1 22 | # via sphinx-toolbox 23 | certifi==2023.11.17 24 | # via requests 25 | charset-normalizer==3.3.2 26 | # via requests 27 | cssutils==2.9.0 28 | # via dict2css 29 | dict2css==0.3.0.post1 30 | # via sphinx-toolbox 31 | docutils==0.16 32 | # via 33 | # -r ./requirements.in 34 | # autoclasstoc 35 | # sphinx 36 | # sphinx-rtd-theme 37 | # sphinx-tabs 38 | # sphinx-toolbox 39 | domdf-python-tools==3.7.0 40 | # via 41 | # apeye 42 | # apeye-core 43 | # dict2css 44 | # sphinx-toolbox 45 | filelock==3.13.1 46 | # via 47 | # cachecontrol 48 | # sphinx-toolbox 49 | html5lib==1.1 50 | # via sphinx-toolbox 51 | idna==3.6 52 | # via 53 | # apeye-core 54 | # requests 55 | imagesize==1.4.1 56 | # via sphinx 57 | jinja2==3.1.2 58 | # via 59 | # sphinx 60 | # sphinx-jinja2-compat 61 | markupsafe==2.1.3 62 | # via 63 | # jinja2 64 | # sphinx-jinja2-compat 65 | more-itertools==10.1.0 66 | # via autoclasstoc 67 | msgpack==1.0.7 68 | # via cachecontrol 69 | natsort==8.4.0 70 | # via domdf-python-tools 71 | packaging==23.2 72 | # via sphinx 73 | platformdirs==4.1.0 74 | # via apeye 75 | pygments==2.17.2 76 | # via 77 | # sphinx 78 | # sphinx-prompt 79 | # sphinx-tabs 80 | requests==2.31.0 81 | # via 82 | # apeye 83 | # cachecontrol 84 | # sphinx 85 | ruamel-yaml==0.18.5 86 | # via sphinx-toolbox 87 | ruamel-yaml-clib==0.2.8 88 | # via ruamel-yaml 89 | six==1.16.0 90 | # via html5lib 91 | snowballstemmer==2.2.0 92 | # via sphinx 93 | soupsieve==2.5 94 | # via beautifulsoup4 95 | sphinx==5.3.0 96 | # via 97 | # autoclasstoc 98 | # autodocsumm 99 | # sphinx-autodoc-typehints 100 | # sphinx-prompt 101 | # sphinx-rtd-theme 102 | # sphinx-tabs 103 | # sphinx-toolbox 104 | # sphinxcontrib-jquery 105 | sphinx-autodoc-typehints==1.23.0 106 | # via sphinx-toolbox 107 | sphinx-jinja2-compat==0.2.0.post1 108 | # via sphinx-toolbox 109 | sphinx-prompt==1.5.0 110 | # via sphinx-toolbox 111 | sphinx-rtd-theme==2.0.0 112 | # via -r ./requirements.in 113 | sphinx-tabs==3.4.5 114 | # via sphinx-toolbox 115 | sphinx-toolbox==3.5.0 116 | # via -r ./requirements.in 117 | sphinxcontrib-applehelp==1.0.8 118 | # via sphinx 119 | sphinxcontrib-devhelp==1.0.6 120 | # via sphinx 121 | sphinxcontrib-htmlhelp==2.0.5 122 | # via sphinx 123 | sphinxcontrib-jquery==4.1 124 | # via sphinx-rtd-theme 125 | sphinxcontrib-jsmath==1.0.1 126 | # via sphinx 127 | sphinxcontrib-qthelp==1.0.7 128 | # via sphinx 129 | sphinxcontrib-serializinghtml==1.1.10 130 | # via sphinx 131 | tabulate==0.9.0 132 | # via sphinx-toolbox 133 | typing-extensions==4.9.0 134 | # via 135 | # domdf-python-tools 136 | # sphinx-toolbox 137 | urllib3==2.1.0 138 | # via requests 139 | webencodings==0.5.1 140 | # via html5lib 141 | 142 | # The following packages are considered to be unsafe in a requirements file: 143 | # setuptools 144 | -------------------------------------------------------------------------------- /doc/schemas.rst: -------------------------------------------------------------------------------- 1 | .. _schemas: 2 | 3 | ******* 4 | Schemas 5 | ******* 6 | 7 | Because *NestedText* explicitly does not attempt to interpret the data it 8 | parses, it is meant to be paired with a tool that can both validate the data 9 | and convert them to the expected types. For example, if you are expecting a 10 | date for a particular field, you would want to validate that the input looks 11 | like a date (e.g. ``YYYY/MM/DD``) and then convert it to a useful type (e.g. 12 | :class:`arrow.Arrow`). You can do this on an ad hoc basis, or you can apply 13 | a schema. 14 | 15 | A schema is the specification of what fields are expected (e.g. "birthday"), 16 | what types they should be (e.g. a date), and what values are legal (e.g. must 17 | be in the past). There are many libraries available for applying a schema to 18 | data such as those parsed by *NestedText*. Because different libraries may be 19 | more or less appropriate in different scenarios, *NestedText* avoids favoring 20 | any one library specifically: 21 | 22 | - voluptuous_: Define schema using objects 23 | - pydantic_: Define schema using type annotations 24 | - schema_: Define schema using objects 25 | - colander_: Define schema using classes 26 | - schematics_: Define schema using classes 27 | - cerebus_ : Define schema using strings 28 | - valideer_: Define schema using strings 29 | - jsonschema_: Define schema using JSON 30 | 31 | See the :doc:`techniques` page for examples of how to use some of these 32 | libraries with *NestedText*. 33 | 34 | The approach of using separate tools for parsing and interpreting the data has 35 | two significant advantages that are worth briefly highlighting. First is that 36 | the validation tool understands the context and meaning of the data in a way 37 | that the parsing tool cannot. For example, "12" can be an integer if it 38 | represents a day of a month, a float if it represents the output voltage of a 39 | power brick, or a string if represents the version of a software package. 40 | Attempting to interpret "12" without this context is inherently unreliable. 41 | Second is that when data is interpreted by the parser, it puts the onus on the 42 | user to specify the correct types. Going back to the previous example, the 43 | user would be required to know whether ``12``, ``12.0``, or ``"12"`` should be 44 | entered. It does not make sense for this decision to be made by the user 45 | instead of the application. 46 | 47 | 48 | .. _voluptuous: https://github.com/alecthomas/voluptuous 49 | .. _pydantic: https://pydantic-docs.helpmanual.io/ 50 | .. _cerebus: https://docs.python-cerberus.org/en/stable/ 51 | .. _colander: https://docs.pylonsproject.org/projects/colander/en/latest/ 52 | .. _jsonschema: https://python-jsonschema.readthedocs.io/en/latest/ 53 | .. _schema: https://github.com/keleshev/schema 54 | .. _schematics: http://schematics.readthedocs.io/en/latest/ 55 | .. _valideer: https://github.com/podio/valideer 56 | -------------------------------------------------------------------------------- /examples/accumulation/example.in.nt: -------------------------------------------------------------------------------- 1 | name: trantor 2 | actions: 3 | default: clean 4 | patterns: .. 5 | limit: 60 6 | 7 | name: terminus 8 | +patterns: ../**/{name}.nt 9 | +patterns: ../**/*.{name}:*.nt 10 | 11 | + actions: 12 | final: archive 13 | -------------------------------------------------------------------------------- /examples/accumulation/example.out.nt: -------------------------------------------------------------------------------- 1 | name: terminus 2 | actions: 3 | default: clean 4 | final: archive 5 | patterns: 6 | - .. 7 | - ../**/{name}.nt 8 | - ../**/*.{name}:*.nt 9 | limit: 60.0 10 | -------------------------------------------------------------------------------- /examples/accumulation/example.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from settings import read_settings 4 | from inform import Error, fatal, os_error 5 | import nestedtext as nt 6 | 7 | try: 8 | settings = read_settings('example.in.nt') 9 | nt.dump(settings, 'example.out.nt') 10 | except Error as e: 11 | e.terminate() 12 | except OSError as e: 13 | fatal(os_error(e)) 14 | -------------------------------------------------------------------------------- /examples/accumulation/settings.py: -------------------------------------------------------------------------------- 1 | from inform import Error, full_stop, os_error 2 | import nestedtext as nt 3 | from pathlib import Path 4 | 5 | schema = dict( 6 | name = str, 7 | limit = float, 8 | actions = dict, 9 | patterns = list, 10 | ) 11 | list_settings = set(k for k, v in schema.items() if v == list) 12 | dict_settings = set(k for k, v in schema.items() if v == dict) 13 | 14 | def de_dup(key, state): 15 | if key not in state: 16 | state[key] = 1 17 | state[key] += 1 18 | return f"{key}#{state[key]}" 19 | 20 | def normalize_key(key, parent_keys): 21 | return '_'.join(key.lower().split()) # convert key to snake case 22 | 23 | def read_settings(path, processed=None): 24 | if processed is None: 25 | processed = {} 26 | 27 | try: 28 | keymap = {} 29 | settings = nt.load( 30 | path, 31 | top = dict, 32 | normalize_key = normalize_key, 33 | on_dup = de_dup, 34 | keymap = keymap 35 | ) 36 | except OSError as e: 37 | raise Error(os_error(e)) 38 | 39 | def report_error(msg): 40 | keys = key_as_given, 41 | offset = key_as_given.index(key) 42 | raise Error( 43 | full_stop(msg), 44 | culprit = path, 45 | codicil = nt.get_location(keys, keymap=keymap).as_line('key', offset=offset) 46 | ) 47 | 48 | # process settings 49 | for key_as_given, value in settings.items(): 50 | 51 | # remove any decorations on the key 52 | key = key_as_given 53 | 54 | accumulate = '+' in key_as_given 55 | if accumulate: 56 | cruft, _, key = key_as_given.partition('+') 57 | if cruft: 58 | report_error("‘+’ must precede setting name") 59 | 60 | if '#' in key: # get original name for duplicate key 61 | key, _, _ = key.partition('#') 62 | 63 | key = key.strip('_') 64 | if not key.isidentifier(): 65 | report_error("expected identifier") 66 | 67 | # check the type of the value 68 | if key in list_settings: 69 | if isinstance(value, str): 70 | value = value.split() 71 | if not isinstance(value, list): 72 | report_error(f"expected list, found {value.__class__.__name__}") 73 | if accumulate: 74 | base = processed.get(key, []) 75 | value = base + value 76 | elif key in dict_settings: 77 | if value == "": 78 | value = {} 79 | if not isinstance(value, dict): 80 | report_error(f"expected dict, found {value.__class__.__name__}") 81 | if accumulate: 82 | base = processed.get(key, {}) 83 | base.update(value) 84 | value = base 85 | elif key in schema: 86 | if accumulate: 87 | report_error("setting is unsuitable for accumulation") 88 | value = schema[key](value) # cast to desired type 89 | else: 90 | report_error("unknown setting") 91 | processed[key] = value 92 | 93 | return processed 94 | -------------------------------------------------------------------------------- /examples/accumulation/test_read_settings.nt: -------------------------------------------------------------------------------- 1 | basic_tests: 2 | no_accumulation: 3 | given: 4 | > name: terminus 5 | > actions: 6 | > default: clean 7 | > patterns: .. 8 | expected: 9 | name: terminus 10 | actions: 11 | default: clean 12 | patterns: 13 | [..] 14 | 15 | basic_accumulation: 16 | given: 17 | > name: terminus 18 | > actions: 19 | > default: clean 20 | > patterns: .. 21 | > +patterns: ../**/{name}.nt 22 | > + patterns: ../**/*.{name}:*.nt 23 | > + actions: 24 | > final: archive 25 | expected: 26 | name: terminus 27 | actions: 28 | default: clean 29 | final: archive 30 | patterns: 31 | - .. 32 | - ../**/{name}.nt 33 | - ../**/*.{name}:*.nt 34 | 35 | accumulation_from_nothing: 36 | given: 37 | > + patterns: .. 38 | > + patterns: ../**/{name}.nt 39 | > + patterns: ../**/*.{name}:*.nt 40 | > + actions: 41 | > default: clean 42 | > + actions: 43 | > final: archive 44 | expected: 45 | actions: 46 | default: clean 47 | final: archive 48 | patterns: 49 | - .. 50 | - ../**/{name}.nt 51 | - ../**/*.{name}:*.nt 52 | 53 | accumulation_with_multiline_keys: 54 | given: 55 | > : name 56 | > > terminus 57 | > : actions 58 | > default: clean 59 | > : + 60 | > : actions 61 | > final: archive 62 | > : patterns 63 | > - .. 64 | > : + patterns 65 | > - ../**/{name}.nt 66 | > : + 67 | > : patterns 68 | > > ../**/*.{name}:*.nt 69 | expected: 70 | name: terminus 71 | actions: 72 | default: clean 73 | final: archive 74 | patterns: 75 | - .. 76 | - ../**/{name}.nt 77 | - ../**/*.{name}:*.nt 78 | 79 | empty_dict: 80 | given: 81 | > actions: 82 | > + actions: 83 | > final: archive 84 | expected: 85 | actions: 86 | final: archive 87 | 88 | empty_list: 89 | given: 90 | > patterns: 91 | > + patterns: ../**/*.{name}:*.nt 92 | expected: 93 | patterns: 94 | - ../**/*.{name}:*.nt 95 | 96 | duplicate_setting: 97 | given: 98 | > name: terminus 99 | > name: trantor 100 | expected: 101 | name: trantor 102 | 103 | cast_to_float: 104 | given: 105 | > limit: 60 106 | expected: 107 | > !dict(limit=60.0) 108 | 109 | adding_dict_to_list: 110 | given: 111 | > patterns: .. 112 | > + patterns: 113 | > final: archive 114 | error: expected list, found dict. 115 | 116 | adding_list_to_dict: 117 | given: 118 | > actions: 119 | > default: clean 120 | > + actions: 121 | > - ../**/{name}.nt 122 | error: expected dict, found list. 123 | 124 | adding_str_to_dict: 125 | given: 126 | > actions: 127 | > default: clean 128 | > + actions: ../**/{name}.nt 129 | error: expected dict, found str. 130 | 131 | unknown_setting: 132 | given: 133 | > action: 134 | > default: clean 135 | error: unknown setting. 136 | 137 | expected_list: 138 | given: 139 | > patterns: 140 | > default: clean 141 | error: expected list, found dict. 142 | 143 | expected_dict_found_str: 144 | given: 145 | > actions: Hari Seldon 146 | error: expected dict, found str. 147 | 148 | expected_dict_found_list: 149 | given: 150 | > actions: 151 | > - .. 152 | error: expected dict, found list. 153 | 154 | plus_must_precede_setting_name: 155 | given: 156 | > left + right: 157 | error: ‘+’ must precede setting name. 158 | 159 | expected_identifier: 160 | given: 161 | > top & bottom: middle 162 | error: expected identifier. 163 | 164 | not_suitable_for_accumulation: 165 | given: 166 | > name: terminus 167 | > +name: trantor 168 | error: setting is unsuitable for accumulation. 169 | -------------------------------------------------------------------------------- /examples/accumulation/test_read_settings.py: -------------------------------------------------------------------------------- 1 | # imports {{{1 2 | from flatten_dict import flatten, unflatten 3 | from functools import partial 4 | from inform import Error 5 | from parametrize_from_file import parametrize 6 | from voluptuous import Schema, Required, Optional, Any 7 | from settings import read_settings 8 | import pytest 9 | 10 | 11 | # parameterization {{{1 12 | # Adapt parametrize_for_file to read dictionary rather than list {{{2 13 | def name_from_dict_keys(cases): 14 | return [{**v, 'scenario': k} for k,v in cases.items()] 15 | 16 | parametrize = partial(parametrize, preprocess=name_from_dict_keys) 17 | 18 | 19 | class Checker: 20 | def __init__(self, scenario): 21 | self.scenario = scenario 22 | 23 | def check_dicts(self, expected, results): 24 | try: 25 | if expected[0] == '!': 26 | expected = eval(expected[1:]) 27 | except KeyError: 28 | pass 29 | self.expected = expected 30 | self.results = results 31 | assert expected == results, self.cmp_dicts(expected, results) 32 | 33 | def check_text(self, expected, results): 34 | self.expected = expected 35 | self.results = results 36 | assert expected == results, self.cmp_text(expected, results) 37 | 38 | def cmp_dicts(self, expected, results): 39 | expected = flatten(expected) 40 | results = flatten(results) 41 | message = [f"scenario: {self.scenario}"] 42 | missing = expected.keys() - results.keys() 43 | if missing: 44 | message.append(f"missing from results: {', '.join('.'.join(k) for k in missing)}") 45 | extra = results.keys() - expected.keys() 46 | if extra: 47 | message.append(f"extra in results: {', '.join('.'.join(k) for k in extra)}") 48 | for key in expected.keys() & results.keys(): 49 | if expected[key] != results[key]: 50 | message.append(f"{key} differs:") 51 | message.append(f" e: {expected[key]}") 52 | message.append(f" r: {results[key]}") 53 | return '\n'.join(message) 54 | 55 | def cmp_text(self, expected, results): 56 | message = [f"scenario: {self.scenario}"] 57 | message.append(f" expected: {expected}") 58 | message.append(f" result: {results}") 59 | return '\n'.join(message) 60 | 61 | # schema for test cases {{{1 62 | scenario_schema = Schema({ 63 | Required("scenario"): str, 64 | Required("given"): str, 65 | Optional("expected", default=""): Any(str, list, dict), 66 | Optional("error", default=""): str, 67 | }, required=True) 68 | 69 | @parametrize( 70 | key = "basic_tests", 71 | schema = scenario_schema, 72 | ) 73 | def test_basic(tmp_path, scenario, given, expected, error): 74 | checker = Checker(scenario) 75 | path = tmp_path / 'settings.nt' 76 | path.write_text(given) 77 | if expected: 78 | settings = read_settings(path) 79 | checker.check_dicts(expected, settings) 80 | return 81 | with pytest.raises(Error) as exception: 82 | read_settings(path) 83 | result = exception.value.get_message() 84 | # _, _, message = message.rpartition('/') 85 | checker.check_text(error, result) 86 | -------------------------------------------------------------------------------- /examples/addresses/address: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | Display Contact Information 4 | 5 | Usage: 6 | contact ... 7 | """ 8 | 9 | from docopt import docopt 10 | from inform import codicil, display, error, full_stop, os_error, terminate 11 | import nestedtext as nt 12 | from voluptuous import Schema, Any, MultipleInvalid 13 | import re 14 | 15 | contacts_file = "address.nt" 16 | 17 | def normalize_key(key, parent_keys): 18 | if len(parent_keys) == 0: 19 | return key 20 | return '_'.join(key.lower().split()) 21 | 22 | def render_contact(data, keymap=None): 23 | text = nt.dumps(data, map_keys=keymap) 24 | return re.sub(r'^(\s*)[>:][ ]?(.*)$', r'\1\2', text, flags=re.M) 25 | 26 | cmdline = docopt(__doc__) 27 | names = cmdline[''] 28 | 29 | try: 30 | # define structure of contacts database 31 | contacts_schema = Schema({ 32 | str: { 33 | 'position': str, 34 | 'address': str, 35 | 'phone': Any({str:str},str), 36 | 'email': Any({str:str},str), 37 | 'additional_roles': Any(list,str), 38 | } 39 | }) 40 | 41 | # read contacts database 42 | contacts = contacts_schema( 43 | nt.load( 44 | contacts_file, 45 | top = 'dict', 46 | normalize_key = normalize_key, 47 | keymap = (keymap:={}) 48 | ) 49 | ) 50 | 51 | # display requested contact information, excluding additional_roles 52 | filtered = {} 53 | for fullname, contact_info in contacts.items(): 54 | for name in names: 55 | if name in fullname.lower(): 56 | filtered[fullname] = contact_info 57 | if 'additional_roles' in contact_info: 58 | del contact_info['additional_roles'] 59 | 60 | # display contact using normalized keys 61 | # display(render_contact(filtered)) 62 | 63 | # display contact using original keys 64 | display(render_contact(filtered, keymap)) 65 | 66 | except nt.NestedTextError as e: 67 | e.report() 68 | except MultipleInvalid as exception: 69 | for e in exception.errors: 70 | kind = 'key' if 'key' in e.msg else 'value' 71 | keys = tuple(e.path) 72 | codicil = keymap[keys].as_line(kind) if keys in keymap else None 73 | line_num, col_num = keymap[keys].as_tuple() 74 | file_and_lineno = f"{contacts_file!s}@{line_num}" 75 | key_path = nt.join_keys(keys, keymap=keymap, sep="›") 76 | error( 77 | full_stop(e.msg), 78 | culprit = (file_and_lineno, key_path), 79 | codicil = codicil 80 | ) 81 | except OSError as e: 82 | error(os_error(e)) 83 | terminate() 84 | -------------------------------------------------------------------------------- /examples/addresses/address.json: -------------------------------------------------------------------------------- 1 | { 2 | "Katheryn McDaniel": { 3 | "position": "president", 4 | "address": "138 Almond Street\nTopeka, Kansas 20697", 5 | "phone": { 6 | "cell": "1-210-555-5297", 7 | "work": "1-210-555-8470" 8 | }, 9 | "email": "KateMcD@aol.com", 10 | "additional roles": [ 11 | "board member" 12 | ] 13 | }, 14 | "Margaret Hodge": { 15 | "position": "vice president", 16 | "address": "2586 Marigold Lane\nTopeka, Kansas 20682", 17 | "phone": "1-470-555-0398", 18 | "email": "margaret.hodge@ku.edu", 19 | "additional roles": [ 20 | "new membership task force", 21 | "accounting task force" 22 | ] 23 | }, 24 | "Fumiko Purvis": { 25 | "Position": "Treasurer", 26 | "Address": "3636 Buffalo Ave\nTopeka, Kansas 20692", 27 | "Phone": "1-268-555-0280", 28 | "EMail": "fumiko.purvis@hotmail.com", 29 | "Additional Roles": [ 30 | "accounting task force" 31 | ] 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /examples/addresses/address.nt: -------------------------------------------------------------------------------- 1 | # Contact information for our officers 2 | 3 | Katheryn McDaniel: 4 | position: president 5 | address: 6 | > 138 Almond Street 7 | > Topeka, Kansas 20697 8 | phone: 9 | cell: 1-210-555-5297 10 | # Katheryn prefers that we call her on her cell phone 11 | work: 1-210-555-8470 12 | email: KateMcD@aol.com 13 | additional roles: 14 | - board member 15 | 16 | Margaret Hodge: 17 | position: vice president 18 | address: 19 | > 2586 Marigold Lane 20 | > Topeka, Kansas 20682 21 | phone: 1-470-555-0398 22 | email: margaret.hodge@ku.edu 23 | additional roles: 24 | - new membership task force 25 | - accounting task force 26 | 27 | Fumiko Purvis: 28 | Position: Treasurer 29 | # Fumiko's term is ending at the end of the year. 30 | Address: 31 | > 3636 Buffalo Ave 32 | > Topeka, Kansas 20692 33 | Phone: 1-268-555-0280 34 | EMail: fumiko.purvis@hotmail.com 35 | Additional Roles: 36 | - accounting task force 37 | -------------------------------------------------------------------------------- /examples/addresses/address.orig: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | Display Contact Information 4 | 5 | Usage: 6 | contact ... 7 | """ 8 | 9 | from docopt import docopt 10 | from inform import codicil, display, fatal, full_stop, os_error 11 | import nestedtext as nt 12 | from voluptuous import Schema, Required, Any, MultipleInvalid 13 | import re 14 | 15 | contacts_file = "address.nt" 16 | 17 | def normalize_key(key, parent_keys): 18 | if len(parent_keys) == 0: 19 | return key 20 | return '_'.join(key.lower().split()) 21 | 22 | def render_contact(data, keymap=None): 23 | text = nt.dumps(data, map_keys=keymap) 24 | return re.sub(r'^(\s*)[>:][ ]?(.*)$', r'\1\2', text, flags=re.M) 25 | 26 | def last_name_first(key, parent_keys): 27 | if len(parent_keys) == 0: 28 | # rearrange names so that last name is given first 29 | names = key.split() 30 | return f"{names[-1]}, {' '.join(names[:-1])}" 31 | 32 | def sort_key(key, parent_keys): 33 | debug(key) 34 | return key[1] if len(parent_keys) == 0 else '' # only sort first level keys 35 | 36 | cmdline = docopt(__doc__) 37 | names = cmdline[''] 38 | 39 | try: 40 | # define structure of contacts database 41 | contacts_schema = Schema({ 42 | str: { 43 | 'position': str, 44 | 'address': str, 45 | 'phone': Required(Any({str:str},str)), 46 | 'email': Required(Any({str:str},str)), 47 | 'additional_roles': Any(list,str), 48 | } 49 | }) 50 | 51 | # read contacts database 52 | contacts = contacts_schema( 53 | nt.load( 54 | contacts_file, 55 | top = 'dict', 56 | normalize_key = normalize_key, 57 | keymap = (keymap:={}) 58 | ) 59 | ) 60 | print(nt.dumps(contacts, map_keys=last_name_first, sort_keys=sort_key)) 61 | 62 | # display requested contact information, excluding additional_roles 63 | filtered = {} 64 | for fullname, contact_info in contacts.items(): 65 | for name in names: 66 | if name in fullname.lower(): 67 | filtered[fullname] = contact_info 68 | if 'additional_roles' in contact_info: 69 | del contact_info['additional_roles'] 70 | 71 | # display contact using normalized keys 72 | # display(render_contact(filtered)) 73 | 74 | # display contact using original keys 75 | display(render_contact(filtered, keymap)) 76 | 77 | except nt.NestedTextError as e: 78 | e.report() 79 | except MultipleInvalid as exception: 80 | for e in exception.errors: 81 | kind = 'key' if 'key' in e.msg else 'value' 82 | keys = tuple(e.path) 83 | codicil = keymap[keys].as_line(kind) if keys in keymap else None 84 | fatal( 85 | full_stop(e.msg), 86 | culprit = (contacts_file, nt.join_keys(keys, keymap=keymap)), 87 | codicil = codicil 88 | ) 89 | except OSError as e: 90 | fatal(os_error(e)) 91 | -------------------------------------------------------------------------------- /examples/addresses/blue-address: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | Display Contact Information 4 | 5 | Usage: 6 | contact ... 7 | """ 8 | 9 | from docopt import docopt 10 | from inform import codicil, display, error, full_stop, os_error, terminate, Color 11 | import nestedtext as nt 12 | from voluptuous import Schema, Required, Any, MultipleInvalid 13 | from voluptuous_errors import report_voluptuous_errors 14 | import re 15 | 16 | blue = Color('blue', enable=Color.isTTY()) 17 | 18 | contacts_file = "address.nt" 19 | 20 | def normalize_key(key, parent_keys): 21 | if len(parent_keys) == 0: 22 | return key 23 | return '_'.join(key.lower().split()) 24 | 25 | def format_key(key, parent_keys): 26 | orig_keys = nt.get_keys(parent_keys + (key,), keymap) 27 | return blue(orig_keys[-1]) 28 | 29 | def render_contact(data, keymap=None): 30 | text = nt.dumps(data, map_keys=format_key) 31 | return re.sub(r'^(\s*)[>:][ ]?(.*)$', r'\1\2', text, flags=re.M) 32 | 33 | cmdline = docopt(__doc__) 34 | names = cmdline[''] 35 | 36 | try: 37 | # define structure of contacts database 38 | contacts_schema = Schema({ 39 | str: { 40 | 'position': str, 41 | 'address': str, 42 | 'phone': Required(Any({str:str},str)), 43 | 'email': Required(Any({str:str},str)), 44 | 'additional_roles': Any(list,str), 45 | } 46 | }) 47 | 48 | # read contacts database 49 | contacts = contacts_schema( 50 | nt.load( 51 | contacts_file, 52 | top = 'dict', 53 | normalize_key = normalize_key, 54 | keymap = (keymap:={}) 55 | ) 56 | ) 57 | 58 | # display requested contact information, excluding additional_roles 59 | filtered = {} 60 | for fullname, contact_info in contacts.items(): 61 | for name in names: 62 | if name in fullname.lower(): 63 | filtered[fullname] = contact_info 64 | if 'additional_roles' in contact_info: 65 | del contact_info['additional_roles'] 66 | 67 | # display contact using normalized keys 68 | # display(render_contact(filtered)) 69 | 70 | # display contact using original keys 71 | display(render_contact(filtered, keymap)) 72 | 73 | except nt.NestedTextError as e: 74 | e.report() 75 | except MultipleInvalid as e: 76 | report_voluptuous_errors(e, keymap, contacts_file) 77 | except OSError as e: 78 | error(os_error(e)) 79 | terminate() 80 | -------------------------------------------------------------------------------- /examples/addresses/fumiko.json: -------------------------------------------------------------------------------- 1 | { 2 | "treasurer": { 3 | "name": "Fumiko Purvis", 4 | "address": "3636 Buffalo Ave\nTopeka, Kansas 20692", 5 | "phone": "1-268-555-0280", 6 | "email": "fumiko.purvis@hotmail.com", 7 | "additional roles": [ 8 | "accounting task force" 9 | ] 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /examples/addresses/fumiko.nt: -------------------------------------------------------------------------------- 1 | treasurer: 2 | name: Fumiko Purvis 3 | # Fumiko's term is ending at the end of the year. 4 | # She will be replaced by Merrill Eldridge. 5 | address: 6 | > 3636 Buffalo Ave 7 | > Topeka, Kansas 20692 8 | phone: 1-268-555-0280 9 | email: fumiko.purvis@hotmail.com 10 | additional roles: 11 | - accounting task force 12 | -------------------------------------------------------------------------------- /examples/conftest.py: -------------------------------------------------------------------------------- 1 | def pytest_ignore_collect(collection_path): 2 | components = collection_path.parts 3 | name = components[-1] 4 | if name.startswith('deploy_') and name.endswith('.py'): 5 | return True 6 | if name == 'example.py': 7 | return True 8 | -------------------------------------------------------------------------------- /examples/conversion-utilities/csv-to-nestedtext: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | Read a CSV file and convert it to NestedText. 4 | 5 | usage: 6 | csv-to-nestedtext [options] [] 7 | 8 | options: 9 | -n, --names first row contains column names 10 | -c, --cull remove empty fields (only for --names) 11 | -f, --force force overwrite of output file 12 | -i , --indent number of spaces per indent [default: 4] 13 | 14 | If is not given, csv input is taken from stdin and NestedText output 15 | is written to stdout. 16 | 17 | If --names is specified, then the first line is assumed to hold the column/field 18 | names with the remaining lines containing the data. In this case the output is 19 | a list of dictionaries. Otherwise every line contains data and that data is 20 | output as a list of lists. 21 | """ 22 | 23 | from docopt import docopt 24 | from inform import cull, done, fatal, full_stop, os_error, warn 25 | from pathlib import Path 26 | import csv 27 | import nestedtext as nt 28 | import sys 29 | sys.stdin.reconfigure(encoding='utf-8') 30 | sys.stdout.reconfigure(encoding='utf-8') 31 | 32 | cmdline = docopt(__doc__) 33 | input_filename = cmdline[''] 34 | try: 35 | indent = int(cmdline['--indent']) 36 | except Exception: 37 | warn('expected positive integer for indent.', culprit=cmdline['--indent']) 38 | indent = 4 39 | 40 | # strip dictionaries of empty fields if requested 41 | converters = {dict: cull} if cmdline['--cull'] else {} 42 | 43 | try: 44 | # read CSV content; from file or from stdin 45 | if input_filename: 46 | input_path = Path(input_filename) 47 | csv_content = input_path.read_text(encoding='utf-8') 48 | else: 49 | csv_content = sys.stdin.read() 50 | if cmdline['--names']: 51 | data = csv.DictReader(csv_content.splitlines()) 52 | else: 53 | data = csv.reader(csv_content.splitlines()) 54 | 55 | # convert to NestedText 56 | nt_content = nt.dumps(data, indent=indent, converters=converters) + "\n" 57 | 58 | # output NestedText content; to file or to stdout 59 | if input_filename: 60 | output_path = input_path.with_suffix('.nt') 61 | if output_path.exists(): 62 | if not cmdline['--force']: 63 | fatal('file exists, use -f to force over-write.', culprit=output_path) 64 | output_path.write_text(nt_content, encoding='utf-8') 65 | else: 66 | sys.stdout.write(nt_content) 67 | 68 | except OSError as e: 69 | fatal(os_error(e)) 70 | except nt.NestedTextError as e: 71 | e.terminate() 72 | except csv.Error as e: 73 | fatal(full_stop(e), culprit=(input_filename, data.line_num)) 74 | except KeyboardInterrupt: 75 | done() 76 | -------------------------------------------------------------------------------- /examples/conversion-utilities/dmarc.nt: -------------------------------------------------------------------------------- 1 | feedback: 2 | @xmlns:xsd: http://www.w3.org/2001/XMLSchema 3 | @xmlns:xsi: http://www.w3.org/2001/XMLSchema-instance 4 | version: 1.0 5 | report_metadata: 6 | org_name: Enterprise Outlook 7 | email: dmarcreport@microsoft.com 8 | report_id: 2c9ff66e0868713ebdff30x2e3f9121c 9 | date_range: 10 | begin: 1709596800 11 | end: 1709683200 12 | policy_published: 13 | domain: tenaya-networks.net 14 | adkim: r 15 | aspf: r 16 | p: none 17 | sp: none 18 | pct: 100 19 | fo: 0 20 | record: 21 | row: 22 | source_ip: 187.27.98.108 23 | count: 2 24 | policy_evaluated: 25 | disposition: none 26 | dkim: pass 27 | spf: pass 28 | identifiers: 29 | envelope_to: westeros.com 30 | envelope_from: tenaya-networks.net 31 | header_from: tenaya-networks.net 32 | auth_results: 33 | dkim: 34 | domain: tenaya-networks.net 35 | selector: default 36 | result: pass 37 | spf: 38 | domain: tenaya-networks.net 39 | scope: mfrom 40 | result: pass -------------------------------------------------------------------------------- /examples/conversion-utilities/dmarc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 1.0 4 | 5 | Enterprise Outlook 6 | dmarcreport@microsoft.com 7 | 2c9ff66e0868713ebdff30x2e3f9121c 8 | 9 | 1709596800 10 | 1709683200 11 | 12 | 13 | 14 | tenaya-networks.net 15 | r 16 | r 17 |

none

18 | none 19 | 100 20 | 0 21 | 22 | 23 | 24 | 187.27.98.108 25 | 2 26 | 27 | none 28 | pass 29 | pass 30 | 31 | 32 | 33 | westeros.com 34 | tenaya-networks.net 35 | tenaya-networks.net 36 | 37 | 38 | 39 | tenaya-networks.net 40 | default 41 | pass 42 | 43 | 44 | tenaya-networks.net 45 | mfrom 46 | pass 47 | 48 | 49 | 50 | 51 | -------------------------------------------------------------------------------- /examples/conversion-utilities/github-intent.nt: -------------------------------------------------------------------------------- 1 | name: Python package 2 | on: 3 | - push 4 | build: 5 | python-version: 6 | - 3.6 7 | - 3.7 8 | - 3.8 9 | - 3.9 10 | - 3.10 11 | steps: 12 | - 13 | name: Install dependencies 14 | run: 15 | > python -m pip install --upgrade pip 16 | > pip install pytest 17 | > if [ -f 'requirements.txt' ]; then pip install -r requirements.txt; fi 18 | - 19 | name: Test with pytest 20 | run: pytest 21 | -------------------------------------------------------------------------------- /examples/conversion-utilities/github-intent.yaml: -------------------------------------------------------------------------------- 1 | build: 2 | python-version: 3 | - '3.6' 4 | - '3.7' 5 | - '3.8' 6 | - '3.9' 7 | - '3.10' 8 | steps: 9 | - name: Install dependencies 10 | run: 'python -m pip install --upgrade pip 11 | 12 | pip install pytest 13 | 14 | if [ -f ''requirements.txt'' ]; then pip install -r requirements.txt; fi' 15 | - name: Test with pytest 16 | run: pytest 17 | name: Python package 18 | 'on': 19 | - push 20 | 21 | -------------------------------------------------------------------------------- /examples/conversion-utilities/github-orig.nt: -------------------------------------------------------------------------------- 1 | name: Python package 2 | True: 3 | - push 4 | build: 5 | python-version: 6 | - 3.6 7 | - 3.7 8 | - 3.8 9 | - 3.9 10 | - 3.1 11 | steps: 12 | - 13 | name: Install dependencies 14 | run: 15 | > python -m pip install --upgrade pip 16 | > pip install pytest 17 | > if [ -f 'requirements.txt' ]; then pip install -r requirements.txt; fi 18 | > 19 | - 20 | name: Test with pytest 21 | run: 22 | > pytest 23 | > 24 | -------------------------------------------------------------------------------- /examples/conversion-utilities/github-orig.yaml: -------------------------------------------------------------------------------- 1 | name: Python package 2 | on: [push] 3 | build: 4 | python-version: [3.6, 3.7, 3.8, 3.9, 3.10] 5 | steps: 6 | - name: Install dependencies 7 | run: | 8 | python -m pip install --upgrade pip 9 | pip install pytest 10 | if [ -f 'requirements.txt' ]; then pip install -r requirements.txt; fi 11 | - name: Test with pytest 12 | run: | 13 | pytest 14 | 15 | -------------------------------------------------------------------------------- /examples/conversion-utilities/github-rt.yaml: -------------------------------------------------------------------------------- 1 | name: Python package 2 | true: 3 | - push 4 | build: 5 | python-version: 6 | - 3.6 7 | - 3.7 8 | - 3.8 9 | - 3.9 10 | - 3.1 11 | steps: 12 | - name: Install dependencies 13 | run: 'python -m pip install --upgrade pip 14 | 15 | pip install pytest 16 | 17 | if [ -f ''requirements.txt'' ]; then pip install -r requirements.txt; fi 18 | 19 | ' 20 | - name: Test with pytest 21 | run: 'pytest 22 | 23 | ' 24 | -------------------------------------------------------------------------------- /examples/conversion-utilities/json-to-nestedtext: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | Read a JSON file and convert it to NestedText. 4 | 5 | usage: 6 | json-to-nestedtext [options] [] 7 | 8 | options: 9 | -f, --force force overwrite of output file 10 | -i , --indent number of spaces per indent [default: 4] 11 | -s, --sort sort the keys 12 | -w , --width desired maximum line width; specifying enables 13 | use of single-line lists and dictionaries as long 14 | as the fit in given width [default: 0] 15 | 16 | If is not given, JSON input is taken from stdin and NestedText output 17 | is written to stdout. 18 | """ 19 | 20 | from docopt import docopt 21 | from inform import done, fatal, full_stop, os_error, warn 22 | from pathlib import Path 23 | import json 24 | import nestedtext as nt 25 | import sys 26 | sys.stdin.reconfigure(encoding='utf-8') 27 | sys.stdout.reconfigure(encoding='utf-8') 28 | 29 | cmdline = docopt(__doc__) 30 | input_filename = cmdline[''] 31 | try: 32 | indent = int(cmdline['--indent']) 33 | except Exception: 34 | warn('expected positive integer for indent.', culprit=cmdline['--indent']) 35 | indent = 4 36 | try: 37 | width = int(cmdline['--width']) 38 | except Exception: 39 | warn('expected non-negative integer for width.', culprit=cmdline['--width']) 40 | width = 0 41 | 42 | try: 43 | # read JSON content; from file or from stdin 44 | if input_filename: 45 | input_path = Path(input_filename) 46 | json_content = input_path.read_text(encoding='utf-8') 47 | else: 48 | json_content = sys.stdin.read() 49 | data = json.loads(json_content) 50 | 51 | # convert to NestedText 52 | nestedtext_content = nt.dumps( 53 | data, 54 | indent = indent, 55 | width = width, 56 | sort_keys = cmdline['--sort'] 57 | ) 58 | 59 | # output NestedText content; to file or to stdout 60 | if input_filename: 61 | output_path = input_path.with_suffix('.nt') 62 | if output_path.exists(): 63 | if not cmdline['--force']: 64 | fatal('file exists, use -f to force over-write.', culprit=output_path) 65 | output_path.write_text(nestedtext_content, encoding='utf-8') 66 | else: 67 | sys.stdout.write(nestedtext_content + "\n") 68 | 69 | except OSError as e: 70 | fatal(os_error(e)) 71 | except nt.NestedTextError as e: 72 | e.terminate() 73 | except UnicodeError as e: 74 | fatal(full_stop(e)) 75 | except KeyboardInterrupt: 76 | done() 77 | except json.JSONDecodeError as e: 78 | # create a nice error message with surrounding context 79 | msg = e.msg 80 | culprit = input_filename 81 | codicil = None 82 | try: 83 | lineno = e.lineno 84 | culprit = (culprit, lineno) 85 | colno = e.colno 86 | lines_before = e.doc.split('\n')[max(lineno-2, 0):lineno] 87 | lines = [] 88 | for i, l in zip(range(lineno-len(lines_before), lineno), lines_before): 89 | lines.append(f'{i+1:>4}> {l}') 90 | lines_before = '\n'.join(lines) 91 | lines_after = e.doc.split('\n')[lineno:lineno+1] 92 | lines = [] 93 | for i, l in zip(range(lineno, lineno + len(lines_after)), lines_after): 94 | lines.append(f'{i+1:>4}> {l}') 95 | lines_after = '\n'.join(lines) 96 | codicil = f"{lines_before}\n {colno*' '}▲\n{lines_after}" 97 | except Exception: 98 | pass 99 | fatal(full_stop(msg), culprit=culprit, codicil=codicil) 100 | -------------------------------------------------------------------------------- /examples/conversion-utilities/nestedtext-to-json: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | Read a NestedText file and convert it to JSON. 4 | 5 | usage: 6 | nestedtext-to-json [options] [] 7 | 8 | options: 9 | -f, --force force overwrite of output file 10 | -d, --dedup de-duplicate keys in dictionaries 11 | 12 | If is not given, NestedText input is taken from stdin and JSON output 13 | is written to stdout. 14 | """ 15 | 16 | from docopt import docopt 17 | from inform import done, fatal, os_error, full_stop 18 | from pathlib import Path 19 | import json 20 | import nestedtext as nt 21 | import sys 22 | sys.stdin.reconfigure(encoding='utf-8') 23 | sys.stdout.reconfigure(encoding='utf-8') 24 | 25 | 26 | def de_dup(key, state): 27 | if key not in state: 28 | state[key] = 1 29 | state[key] += 1 30 | return f"{key} — #{state[key]}" 31 | 32 | 33 | cmdline = docopt(__doc__) 34 | input_filename = cmdline[''] 35 | on_dup = de_dup if cmdline['--dedup'] else None 36 | 37 | try: 38 | if input_filename: 39 | input_path = Path(input_filename) 40 | data = nt.load(input_path, top='any', on_dup=on_dup) 41 | json_content = json.dumps(data, indent=4, ensure_ascii=False) 42 | output_path = input_path.with_suffix('.json') 43 | if output_path.exists(): 44 | if not cmdline['--force']: 45 | fatal('file exists, use -f to force over-write.', culprit=output_path) 46 | output_path.write_text(json_content, encoding='utf-8') 47 | else: 48 | data = nt.load(sys.stdin, top='any', on_dup=on_dup) 49 | json_content = json.dumps(data, indent=4, ensure_ascii=False) 50 | sys.stdout.write(json_content + '\n') 51 | except OSError as e: 52 | fatal(os_error(e)) 53 | except nt.NestedTextError as e: 54 | e.terminate() 55 | except UnicodeError as e: 56 | fatal(full_stop(e)) 57 | except KeyboardInterrupt: 58 | done() 59 | -------------------------------------------------------------------------------- /examples/conversion-utilities/nestedtext-to-yaml: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | Read a NestedText file and convert it to YAML. 4 | 5 | usage: 6 | nestedtext-to-yaml [options] [] 7 | 8 | options: 9 | -f, --force force overwrite of output file 10 | -d, --dedup de-duplicate keys in dictionaries 11 | 12 | If is not given, NestedText input is taken from stdin and YAML output 13 | is written to stdout. 14 | """ 15 | 16 | from docopt import docopt 17 | from inform import done, fatal, os_error 18 | from pathlib import Path 19 | try: 20 | import yaml 21 | except ImportError: 22 | fatal("must install PyYAML using: 'pip install pyyaml'.") 23 | import nestedtext as nt 24 | import sys 25 | sys.stdin.reconfigure(encoding='utf-8') 26 | sys.stdout.reconfigure(encoding='utf-8') 27 | 28 | 29 | def de_dup(key, state): 30 | if key not in state: 31 | state[key] = 1 32 | state[key] += 1 33 | return f"{key} — {state[key]}" 34 | 35 | 36 | cmdline = docopt(__doc__) 37 | input_filename = cmdline[''] 38 | on_dup = de_dup if cmdline['--dedup'] else None 39 | 40 | try: 41 | if input_filename: 42 | input_path = Path(input_filename) 43 | data = nt.load(input_path, top='any', on_dup=on_dup) 44 | yaml_content = yaml.dump(data, allow_unicode=True) 45 | output_path = input_path.with_suffix('.yaml') 46 | if output_path.exists(): 47 | if not cmdline['--force']: 48 | fatal('file exists, use -f to force over-write.', culprit=output_path) 49 | output_path.write_text(yaml_content, encoding='utf-8') 50 | else: 51 | data = nt.load(sys.stdin, top='any', on_dup=on_dup) 52 | yaml_content = yaml.dump(data, allow_unicode=True) 53 | sys.stdout.write(yaml_content + '\n') 54 | except OSError as e: 55 | fatal(os_error(e)) 56 | except nt.NestedTextError as e: 57 | e.terminate() 58 | except KeyboardInterrupt: 59 | done() 60 | -------------------------------------------------------------------------------- /examples/conversion-utilities/percent_bachelors_degrees_women_usa.csv: -------------------------------------------------------------------------------- 1 | Year,Agriculture,Architecture,Art and Performance,Biology,Business,Communications and Journalism,Computer Science,Education,Engineering,English,Foreign Languages,Health Professions,Math and Statistics,Physical Sciences,Psychology,Public Administration,Social Sciences and History 2 | 1970,4.22979798,11.92100539,59.7,29.08836297,9.064438975,35.3,13.6,74.53532758,0.8,65.57092343,73.8,77.1,38,13.8,44.4,68.4,36.8 3 | 1980,30.75938956,28.08038075,63.4,43.99925716,36.76572529,54.7,32.5,74.98103152,10.3,65.28413007,74.1,83.5,42.8,24.6,65.1,74.6,44.2 4 | 1990,32.70344407,40.82404662,62.6,50.81809432,47.20085084,60.8,29.4,78.86685859,14.1,66.92190193,71.2,83.9,47.3,31.6,72.6,77.6,45.1 5 | 2000,45.05776637,40.02358491,59.2,59.38985737,49.80361649,61.9,27.7,76.69214284,18.4,68.36599498,70.9,83.5,48.2,41,77.5,81.1,51.8 6 | 2010,48.73004227,42.06672091,61.3,59.01025521,48.75798769,62.5,17.6,79.61862451,17.2,67.92810557,69,85,43.1,40.2,77,81.7,49.3 7 | -------------------------------------------------------------------------------- /examples/conversion-utilities/percent_bachelors_degrees_women_usa.nt: -------------------------------------------------------------------------------- 1 | - 2 | Year: 1970 3 | Agriculture: 4.22979798 4 | Architecture: 11.92100539 5 | Art and Performance: 59.7 6 | Biology: 29.08836297 7 | Business: 9.064438975 8 | Communications and Journalism: 35.3 9 | Computer Science: 13.6 10 | Education: 74.53532758 11 | Engineering: 0.8 12 | English: 65.57092343 13 | Foreign Languages: 73.8 14 | Health Professions: 77.1 15 | Math and Statistics: 38 16 | Physical Sciences: 13.8 17 | Psychology: 44.4 18 | Public Administration: 68.4 19 | Social Sciences and History: 36.8 20 | - 21 | Year: 1980 22 | Agriculture: 30.75938956 23 | Architecture: 28.08038075 24 | Art and Performance: 63.4 25 | Biology: 43.99925716 26 | Business: 36.76572529 27 | Communications and Journalism: 54.7 28 | Computer Science: 32.5 29 | Education: 74.98103152 30 | Engineering: 10.3 31 | English: 65.28413007 32 | Foreign Languages: 74.1 33 | Health Professions: 83.5 34 | Math and Statistics: 42.8 35 | Physical Sciences: 24.6 36 | Psychology: 65.1 37 | Public Administration: 74.6 38 | Social Sciences and History: 44.2 39 | - 40 | Year: 1990 41 | Agriculture: 32.70344407 42 | Architecture: 40.82404662 43 | Art and Performance: 62.6 44 | Biology: 50.81809432 45 | Business: 47.20085084 46 | Communications and Journalism: 60.8 47 | Computer Science: 29.4 48 | Education: 78.86685859 49 | Engineering: 14.1 50 | English: 66.92190193 51 | Foreign Languages: 71.2 52 | Health Professions: 83.9 53 | Math and Statistics: 47.3 54 | Physical Sciences: 31.6 55 | Psychology: 72.6 56 | Public Administration: 77.6 57 | Social Sciences and History: 45.1 58 | - 59 | Year: 2000 60 | Agriculture: 45.05776637 61 | Architecture: 40.02358491 62 | Art and Performance: 59.2 63 | Biology: 59.38985737 64 | Business: 49.80361649 65 | Communications and Journalism: 61.9 66 | Computer Science: 27.7 67 | Education: 76.69214284 68 | Engineering: 18.4 69 | English: 68.36599498 70 | Foreign Languages: 70.9 71 | Health Professions: 83.5 72 | Math and Statistics: 48.2 73 | Physical Sciences: 41 74 | Psychology: 77.5 75 | Public Administration: 81.1 76 | Social Sciences and History: 51.8 77 | - 78 | Year: 2010 79 | Agriculture: 48.73004227 80 | Architecture: 42.06672091 81 | Art and Performance: 61.3 82 | Biology: 59.01025521 83 | Business: 48.75798769 84 | Communications and Journalism: 62.5 85 | Computer Science: 17.6 86 | Education: 79.61862451 87 | Engineering: 17.2 88 | English: 67.92810557 89 | Foreign Languages: 69 90 | Health Professions: 85 91 | Math and Statistics: 43.1 92 | Physical Sciences: 40.2 93 | Psychology: 77 94 | Public Administration: 81.7 95 | Social Sciences and History: 49.3 96 | -------------------------------------------------------------------------------- /examples/conversion-utilities/rt-yaml: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Round-trip YAML file 3 | # 4 | # reads: github-orig.yaml 5 | # writes: github-rt.yaml 6 | 7 | from inform import fatal, full_stop, os_error 8 | from pathlib import Path 9 | import yaml 10 | from yaml.loader import SafeLoader 11 | 12 | try: 13 | # read YAML file 14 | path = Path("github-orig.yaml") 15 | orig_content = path.read_text() 16 | data = yaml.load(orig_content, Loader=SafeLoader) 17 | path = Path("github-rt.yaml") 18 | rt_content = yaml.dump(data) 19 | path.write_text(rt_content) 20 | except OSError as e: 21 | fatal(os_error(e)) 22 | except nt.NestedTextError as e: 23 | e.terminate() 24 | except yaml.YAMLError as e: 25 | # create a nice error message with surrounding context 26 | fatal(full_stop(e)) 27 | -------------------------------------------------------------------------------- /examples/conversion-utilities/sparekeys.nt: -------------------------------------------------------------------------------- 1 | plugins: 2 | auth: 3 | - avendesora 4 | archive: 5 | - ssh 6 | - gpg 7 | - avendesora 8 | - emborg 9 | - file 10 | publish: 11 | - scp 12 | - mount 13 | auth: 14 | avendesora: 15 | account: login 16 | field: passcode 17 | archive: 18 | file: 19 | src: 20 | - ~/src/nfo/contacts 21 | avendesora: 22 | {} 23 | emborg: 24 | config: rsync 25 | publish: 26 | scp: 27 | host: 28 | - backups 29 | remote_dir: archives/{date:YYMMDD} 30 | mount: 31 | drive: /mnt/secrets 32 | remote_dir: sparekeys/{date:YYMMDD} 33 | -------------------------------------------------------------------------------- /examples/conversion-utilities/sparekeys.toml: -------------------------------------------------------------------------------- 1 | [plugins] 2 | auth = ['avendesora'] 3 | archive = ['ssh', 'gpg', 'avendesora', 'emborg', 'file'] 4 | publish = ['scp', 'mount'] 5 | 6 | [auth.avendesora] 7 | account = 'login' 8 | field = 'passcode' 9 | 10 | [archive.file] 11 | src = ['~/src/nfo/contacts'] 12 | [archive.avendesora] 13 | [archive.emborg] 14 | config = 'rsync' 15 | 16 | [publish.scp] 17 | host = ['backups'] 18 | remote_dir = 'archives/{date:YYMMDD}' 19 | 20 | [publish.mount] 21 | drive = '/mnt/secrets' 22 | remote_dir = 'sparekeys/{date:YYMMDD}' 23 | -------------------------------------------------------------------------------- /examples/conversion-utilities/toml-to-nestedtext: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | Read a TOML file and convert it to NestedText. 4 | 5 | usage: 6 | toml-to-nestedtext [options] [] 7 | 8 | options: 9 | -f, --force force overwrite of output file 10 | -i , --indent number of spaces per indent [default: 4] 11 | -w , --width desired maximum line width; specifying enables 12 | use of single-line lists and dictionaries as long 13 | as the fit in given width [default: 0] 14 | 15 | If is not given, TOML input is taken from stdin and NestedText output 16 | is written to stdout. 17 | """ 18 | 19 | from docopt import docopt 20 | from inform import done, fatal, full_stop, os_error, warn 21 | from pathlib import Path 22 | import toml 23 | import nestedtext as nt 24 | import sys 25 | sys.stdin.reconfigure(encoding='utf-8') 26 | sys.stdout.reconfigure(encoding='utf-8') 27 | 28 | cmdline = docopt(__doc__) 29 | input_filename = cmdline[''] 30 | try: 31 | indent = int(cmdline['--indent']) 32 | except Exception: 33 | warn('expected positive integer for indent.', culprit=cmdline['--indent']) 34 | indent = 4 35 | try: 36 | width = int(cmdline['--width']) 37 | except Exception: 38 | warn('expected non-negative integer for width.', culprit=cmdline['--width']) 39 | width = 0 40 | 41 | try: 42 | # read TOML content; from file or from stdin 43 | if input_filename: 44 | input_path = Path(input_filename) 45 | toml_content = input_path.read_text(encoding='utf-8') 46 | else: 47 | toml_content = sys.stdin.read() 48 | data = toml.loads(toml_content) 49 | 50 | # convert to NestedText 51 | nestedtext_content = nt.dumps(data, indent=indent, width=width) + "\n" 52 | 53 | # output NestedText content; to file or to stdout 54 | if input_filename: 55 | output_path = input_path.with_suffix('.nt') 56 | if output_path.exists(): 57 | if not cmdline['--force']: 58 | fatal('file exists, use -f to force over-write.', culprit=output_path) 59 | output_path.write_text(nestedtext_content, encoding='utf-8') 60 | else: 61 | sys.stdout.write(nestedtext_content) 62 | 63 | except OSError as e: 64 | fatal(os_error(e)) 65 | except nt.NestedTextError as e: 66 | e.terminate() 67 | except KeyboardInterrupt: 68 | done() 69 | except ValueError as e: 70 | # create a nice error message with surrounding context 71 | msg = e.msg 72 | culprit = input_filename 73 | codicil = None 74 | try: 75 | lineno = e.lineno 76 | culprit = (culprit, lineno) 77 | colno = e.colno 78 | lines_before = e.doc.split('\n')[lineno-2:lineno] 79 | lines = [] 80 | for i, l in zip(range(lineno-len(lines_before), lineno), lines_before): 81 | lines.append(f'{i+1:>4}> {l}') 82 | lines_before = '\n'.join(lines) 83 | lines_after = e.doc.split('\n')[lineno:lineno+1] 84 | lines = [] 85 | for i, l in zip(range(lineno, lineno + len(lines_after)), lines_after): 86 | lines.append(f'{i+1:>4}> {l}') 87 | lines_after = '\n'.join(lines) 88 | codicil = f"{lines_before}\n {colno*' '}▲\n{lines_after}" 89 | except Exception: 90 | pass 91 | fatal(full_stop(msg), culprit=culprit, codicil=codicil) 92 | -------------------------------------------------------------------------------- /examples/conversion-utilities/xml-to-nestedtext: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # USAGE {{{1 3 | """ 4 | Read an XML file and convert it to NestedText. 5 | 6 | usage: 7 | xml-to-nestedtext [options] [] 8 | 9 | options: 10 | -f, --force force overwrite of output file 11 | -i , --indent number of spaces per indent [default: 4] 12 | -s, --sort sort the keys 13 | -w , --width desired maximum line width; specifying enables 14 | use of single-line lists and dictionaries as long 15 | as the fit in given width [default: 0] 16 | 17 | If is not given, XML input is taken from stdin and NestedText output␣ 18 | is written to stdout. 19 | """ 20 | 21 | # IMPORTS {{{1 22 | from docopt import docopt 23 | import xmltodict 24 | import nestedtext as nt 25 | from inform import done, fatal, full_stop, os_error, warn 26 | from pathlib import Path 27 | from xml.parsers.expat import ExpatError, errors 28 | import sys 29 | sys.stdin.reconfigure(encoding='utf-8') 30 | sys.stdout.reconfigure(encoding='utf-8') 31 | 32 | 33 | # COMMAND LINE {{{1 34 | cmdline = docopt(__doc__) 35 | input_filename = cmdline[''] 36 | try: 37 | indent = int(cmdline['--indent']) 38 | except Exception: 39 | warn('expected positive integer for indent.', culprit=cmdline['--indent']) 40 | indent = 4 41 | try: 42 | width = int(cmdline['--width']) 43 | except Exception: 44 | warn('expected non-negative integer for width.', culprit=cmdline['--width']) 45 | width = 0 46 | 47 | 48 | # READ XML {{{1 49 | try: 50 | # read XML content; from file or from stdin 51 | if input_filename: 52 | input_path = Path(input_filename) 53 | xml_content = input_path.read_text(encoding='utf-8') 54 | else: 55 | xml_content = sys.stdin.read() 56 | data = xmltodict.parse(xml_content) 57 | 58 | # CONVERT TO NESTEDTEXT {{{1 59 | # convert to NestedText 60 | nestedtext_content = nt.dumps( 61 | data, 62 | indent = indent, 63 | width = width, 64 | sort_keys = cmdline['--sort'] 65 | ) 66 | 67 | # WRITE NESTEDTEXT {{{1 68 | # output NestedText content; to file or to stdout 69 | if input_filename: 70 | output_path = input_path.with_suffix('.nt') 71 | if output_path.exists(): 72 | if not cmdline['--force']: 73 | fatal('file exists, use -f to force over-write.', culprit=output_path) 74 | output_path.write_text(nestedtext_content, encoding='utf-8') 75 | else: 76 | sys.stdout.write(nestedtext_content + "\n") 77 | 78 | # EXCEPTION HANDLING {{{1 79 | except OSError as e: 80 | fatal(os_error(e)) 81 | except nt.NestedTextError as e: 82 | e.terminate() 83 | except UnicodeError as e: 84 | fatal(full_stop(e)) 85 | except KeyboardInterrupt: 86 | done() 87 | except ExpatError as e: 88 | msg = errors.messages[e.code] 89 | if input_filename: 90 | culprit = f"{filename}@{e.lineno}" 91 | else: 92 | culprit = e.lineno 93 | if xml_content: 94 | lines = xml_content.splitlines() 95 | line = lines[e.lineno-1] 96 | pointer = e.offset*" " + "▲" 97 | codicil = (line, pointer) 98 | else: 99 | codicil = None 100 | fatal(full_stop(msg), culprit=culprit, codicil=codicil) 101 | -------------------------------------------------------------------------------- /examples/conversion-utilities/yaml-to-nestedtext: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | Read a YAML file and convert it to NestedText. 4 | 5 | usage: 6 | yaml-to-nestedtext [options] [] 7 | 8 | options: 9 | -f, --force force overwrite of output file 10 | -i , --indent number of spaces per indent [default: 4] 11 | -w , --width desired maximum line width; specifying enables 12 | use of single-line lists and dictionaries as long 13 | as the fit in given width [default: 0] 14 | 15 | If is not given, YAML input is taken from stdin and NestedText output 16 | is written to stdout. 17 | """ 18 | 19 | from docopt import docopt 20 | from inform import done, fatal, full_stop, os_error, warn 21 | from pathlib import Path 22 | try: 23 | import yaml 24 | from yaml.loader import SafeLoader 25 | except ImportError: 26 | fatal("must install PyYAML using: 'pip install pyyaml'.") 27 | import nestedtext as nt 28 | import sys 29 | sys.stdin.reconfigure(encoding='utf-8') 30 | sys.stdout.reconfigure(encoding='utf-8') 31 | 32 | cmdline = docopt(__doc__) 33 | input_filename = cmdline[''] 34 | try: 35 | indent = int(cmdline['--indent']) 36 | except Exception: 37 | warn('expected positive integer for indent.', culprit=cmdline['--indent']) 38 | indent = 4 39 | try: 40 | width = int(cmdline['--width']) 41 | except Exception: 42 | warn('expected non-negative integer for width.', culprit=cmdline['--width']) 43 | width = 0 44 | 45 | try: 46 | # read YAML content; from file or from stdin 47 | if input_filename: 48 | data = yaml.load(input_filename, Loader=SafeLoader) 49 | 50 | else: 51 | data = yaml.load(sys.stdin, Loader=SafeLoader) 52 | 53 | # convert to NestedText 54 | nestedtext_content = nt.dumps(data, indent=indent, width=width) + "\n" 55 | 56 | # output NestedText content; to file or to stdout 57 | if input_filename: 58 | output_path = Path(input_filename).with_suffix('.nt') 59 | if output_path.exists(): 60 | if not cmdline['--force']: 61 | fatal('file exists, use -f to force over-write.', culprit=output_path) 62 | output_path.write_text(nestedtext_content, encoding='utf-8') 63 | else: 64 | sys.stdout.write(nestedtext_content) 65 | 66 | except OSError as e: 67 | fatal(os_error(e)) 68 | except nt.NestedTextError as e: 69 | e.terminate() 70 | except KeyboardInterrupt: 71 | done() 72 | except yaml.YAMLError as e: 73 | fatal(full_stop(e), culprit=input_filename) 74 | -------------------------------------------------------------------------------- /examples/cryptocurrency/cryptocurrency: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import nestedtext as nt 4 | from voluptuous import Schema, Required, All, Length, MultipleInvalid, Coerce 5 | from voluptuous_errors import report_voluptuous_errors 6 | from inform import ( 7 | codicil, display, error, is_collection, os_error, render_bar, full_stop, 8 | terminate 9 | ) 10 | import arrow 11 | import requests 12 | from quantiphy import Quantity 13 | from pathlib import Path 14 | 15 | Quantity.set_prefs(prec=2, ignore_sf = True) 16 | currency_symbols = dict(USD='$', EUR='€', JPY='¥', GBP='£') 17 | 18 | def normalize_key(key, parent_keys): 19 | return ' '.join(key.lower().split()) 20 | 21 | try: 22 | # read settings 23 | settings_file = 'cryptocurrency.nt' 24 | settings_schema = Schema({ 25 | Required('holdings'): All([Coerce(Quantity)], Length(min=1)), 26 | 'currency': str, 27 | 'date format': str, 28 | 'screen width': Coerce(int) 29 | }) 30 | settings = settings_schema( 31 | nt.load( 32 | settings_file, 33 | top = 'dict', 34 | keymap = (keymap:={}), 35 | normalize_key = normalize_key 36 | ) 37 | ) 38 | currency = settings.get('currency', 'USD') 39 | currency_symbol = currency_symbols.get(currency, currency) 40 | screen_width = settings.get('screen width', 80) 41 | 42 | # download latest asset prices from cryptocompare.com 43 | params = dict( 44 | fsyms = ','.join(coin.units for coin in settings['holdings']), 45 | tsyms = currency, 46 | ) 47 | url = 'https://min-api.cryptocompare.com/data/pricemulti' 48 | try: 49 | r = requests.get(url, params=params) 50 | if r.status_code != requests.codes.ok: 51 | r.raise_for_status() 52 | except Exception as e: 53 | raise Error('cannot access cryptocurrency prices:', codicil=str(e)) 54 | prices = {k: Quantity(v[currency], currency_symbol) for k, v in r.json().items()} 55 | 56 | # compute total 57 | total = Quantity(0, currency_symbol) 58 | for coin in settings['holdings']: 59 | price = prices[coin.units] 60 | value = price.scale(coin) 61 | total = total.add(value) 62 | 63 | # display holdings 64 | now = arrow.now().format(settings.get('date format', 'h:mm A, dddd MMMM D, YYYY')) 65 | print(f'Holdings as of {now}.') 66 | bar_width = screen_width - 37 67 | for coin in settings['holdings']: 68 | price = prices[coin.units] 69 | value = price.scale(coin) 70 | portion = value/total 71 | summary = f'{coin} = {value} @ {price}/{coin.units}' 72 | print(f'{summary:<30} {portion:<5.1%} {render_bar(portion, bar_width)}') 73 | print(f'Total value = {total}.') 74 | 75 | except nt.NestedTextError as e: 76 | e.terminate() 77 | except MultipleInvalid as e: 78 | report_voluptuous_errors(e, keymap, settings_file) 79 | except OSError as e: 80 | error(os_error(e)) 81 | except KeyboardInterrupt: 82 | pass 83 | terminate() 84 | -------------------------------------------------------------------------------- /examples/cryptocurrency/cryptocurrency.nt: -------------------------------------------------------------------------------- 1 | Holdings : 2 | - 5 BTC 3 | - 50 ETH 4 | - 50,000 XLM 5 | Screen Width: 90 6 | Date Format: h:mm A, dddd MMMM D 7 | Currency : USD 8 | -------------------------------------------------------------------------------- /examples/cryptocurrency/voluptuous_errors.py: -------------------------------------------------------------------------------- 1 | ../validation/voluptuous_errors.py -------------------------------------------------------------------------------- /examples/deduplication/michael_jordan: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | from inform import codicil, display, fatal, full_stop, os_error 3 | import nestedtext as nt 4 | 5 | filename = "michael_jordan.nt" 6 | 7 | def de_dup(key, state): 8 | if key not in state: 9 | state[key] = 1 10 | state[key] += 1 11 | return f"{key} #{state[key]}" 12 | 13 | try: 14 | # read contacts database 15 | data = nt.load(filename, 'dict', on_dup=de_dup, keymap=(keymap:={})) 16 | 17 | # display contact using deduplicated keys 18 | display("DE-DUPLICATED KEYS:") 19 | display(nt.dumps(data)) 20 | 21 | # display contact using original keys 22 | display() 23 | display("ORIGINAL KEYS:") 24 | display(nt.dumps(data, map_keys=keymap)) 25 | 26 | except nt.NestedTextError as e: 27 | e.terminate() 28 | except OSError as e: 29 | fatal(os_error(e)) 30 | -------------------------------------------------------------------------------- /examples/deduplication/michael_jordan.nt: -------------------------------------------------------------------------------- 1 | Michael Jordan: 2 | occupation: basketball player 3 | 4 | Michael Jordan: 5 | occupation: actor 6 | 7 | Michael Jordan: 8 | occupation: football player 9 | -------------------------------------------------------------------------------- /examples/deduplication/michael_jordan.out: -------------------------------------------------------------------------------- 1 | DE-DUPLICATED KEYS: 2 | Michael Jordan: 3 | occupation: basketball player 4 | Michael Jordan #2: 5 | occupation: actor 6 | Michael Jordan #3: 7 | occupation: football player 8 | 9 | ORIGINAL KEYS: 10 | Michael Jordan: 11 | occupation: basketball player 12 | Michael Jordan: 13 | occupation: actor 14 | Michael Jordan: 15 | occupation: football player 16 | -------------------------------------------------------------------------------- /examples/duplicate-keys.nt: -------------------------------------------------------------------------------- 1 | # this is an invalid NestedText file because it contains a dictionary with 2 | # duplicate keys 3 | 4 | name: 5 | name: 6 | -------------------------------------------------------------------------------- /examples/groceries.nt: -------------------------------------------------------------------------------- 1 | groceries: 2 | - Bread 3 | - Peanut butter 4 | - Jam 5 | -------------------------------------------------------------------------------- /examples/long_lines.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import re 4 | from textwrap import dedent 5 | 6 | from voluptuous import Schema 7 | 8 | import nestedtext as nt 9 | 10 | document = dedent(r""" 11 | lorum ipsum: 12 | > Lorem ipsum dolor sit amet, \ 13 | > consectetur adipiscing elit. 14 | > Sed do eiusmod tempor incididunt \ 15 | > ut labore et dolore magna aliqua. 16 | """) 17 | 18 | def fix_newlines(text): 19 | return text.replace("\\\n", "") 20 | 21 | schema = Schema({str: fix_newlines}) 22 | 23 | def pp(data): 24 | try: 25 | text = nt.dumps(data, default=repr) 26 | print(re.sub(r'^(\s*)[>:][ ]?(.*)$', r'\1\2', text, flags=re.M)) 27 | except nt.NestedTextError as e: 28 | e.report() 29 | 30 | data = schema(nt.loads(document)) 31 | 32 | print(nt.dumps(data)) 33 | pp(data) 34 | -------------------------------------------------------------------------------- /examples/long_lines/long_lines.out: -------------------------------------------------------------------------------- 1 | lorum ipsum: 2 | > Lorem ipsum dolor sit amet, consectetur adipiscing elit. 3 | > Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. 4 | lorum ipsum: 5 | Lorem ipsum dolor sit amet, consectetur adipiscing elit. 6 | Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. 7 | -------------------------------------------------------------------------------- /examples/long_lines/long_lines_backslash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import re 4 | from textwrap import dedent 5 | 6 | from voluptuous import Schema 7 | 8 | import nestedtext as nt 9 | 10 | document = dedent(r""" 11 | lorum ipsum: 12 | > Lorem ipsum dolor sit amet, \ 13 | > consectetur adipiscing elit. 14 | > Sed do eiusmod tempor incididunt \ 15 | > ut labore et dolore magna aliqua. 16 | """) 17 | 18 | def fix_newlines(text): 19 | return text.replace("\\\n", "") 20 | 21 | schema = Schema({str: fix_newlines}) 22 | 23 | def pp(data): 24 | try: 25 | text = nt.dumps(data, default=repr) 26 | print(re.sub(r'^(\s*)[>:][ ]?(.*)$', r'\1\2', text, flags=re.M)) 27 | except nt.NestedTextError as e: 28 | e.report() 29 | 30 | data = schema(nt.loads(document)) 31 | 32 | print(nt.dumps(data)) 33 | pp(data) 34 | -------------------------------------------------------------------------------- /examples/long_lines/long_lines_space: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import re 4 | from textwrap import dedent 5 | 6 | from voluptuous import Schema 7 | 8 | import nestedtext as nt 9 | 10 | document = dedent(r""" 11 | lorum ipsum: 12 | > Lorem ipsum dolor sit amet,␣ 13 | > consectetur adipiscing elit. 14 | > Sed do eiusmod tempor incididunt␣ 15 | > ut labore et dolore magna aliqua. 16 | """).replace('␣', ' ') 17 | 18 | def fix_newlines(text): 19 | return text.replace(" \n", " ") 20 | 21 | schema = Schema({str: fix_newlines}) 22 | 23 | def pp(data): 24 | try: 25 | text = nt.dumps(data, default=repr) 26 | print(re.sub(r'^(\s*)[>:][ ]?(.*)$', r'\1\2', text, flags=re.M)) 27 | except nt.NestedTextError as e: 28 | e.report() 29 | 30 | data = schema(nt.loads(document)) 31 | 32 | print(nt.dumps(data)) 33 | pp(data) 34 | -------------------------------------------------------------------------------- /examples/parametrize_from_file/test_misc.nt: -------------------------------------------------------------------------------- 1 | # test_expr.nt 2 | test_substitution: 3 | - 4 | given: first second 5 | search: ^\s*(\w+)\s*(\w+)\s*$ 6 | replace: \2 \1 7 | expected: second first 8 | - 9 | given: 4 * 7 10 | search: ^\s*(\d+)\s*([-+*/])\s*(\d+)\s*$ 11 | replace: \1 \3 \2 12 | expected: 4 7 * 13 | 14 | test_expression: 15 | - 16 | given: 1 + 2 17 | expected: 3 18 | - 19 | given: "1" + "2" 20 | expected: "12" 21 | - 22 | given: pathlib.Path("/") / "tmp" 23 | expected: pathlib.Path("/tmp") 24 | -------------------------------------------------------------------------------- /examples/parametrize_from_file/test_misc.py: -------------------------------------------------------------------------------- 1 | # test_misc.py 2 | import parametrize_from_file 3 | import re 4 | import pathlib 5 | 6 | @parametrize_from_file 7 | def test_substitution(given, search, replace, expected): 8 | assert re.sub(search, replace, given) == expected 9 | 10 | @parametrize_from_file 11 | def test_expression(given, expected): 12 | assert eval(given) == eval(expected) 13 | 14 | -------------------------------------------------------------------------------- /examples/postmortem/postmortem: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from inform import error, os_error, terminate 4 | import nestedtext as nt 5 | from pathlib import Path 6 | from voluptuous import ( 7 | Schema, Invalid, MultipleInvalid, Extra, Required, REMOVE_EXTRA 8 | ) 9 | from voluptuous_errors import report_voluptuous_errors 10 | from pprint import pprint 11 | 12 | # Settings schema 13 | # First define some functions that are used for validation and coercion 14 | def to_str(arg): 15 | if isinstance(arg, str): 16 | return arg 17 | raise Invalid(f"expected text, found {arg.__class__.__name__}") 18 | 19 | def to_ident(arg): 20 | arg = to_str(arg) 21 | if arg.isidentifier(): 22 | return arg 23 | raise Invalid('expected simple identifier') 24 | 25 | def to_list(arg): 26 | if isinstance(arg, str): 27 | return arg.split() 28 | if isinstance(arg, dict): 29 | raise Invalid(f"expected list, found {arg.__class__.__name__}") 30 | return arg 31 | 32 | def to_paths(arg): 33 | return [Path(p).expanduser() for p in to_list(arg)] 34 | 35 | def to_email(arg): 36 | email = to_str(arg) 37 | user, _, host = email.partition('@') 38 | if '.' in host and '@' not in host: 39 | return arg 40 | raise Invalid('expected email address') 41 | 42 | def to_emails(arg): 43 | return [to_email(e) for e in to_list(arg)] 44 | 45 | def to_gpg_id(arg): 46 | try: 47 | return to_email(arg) # gpg ID may be an email address 48 | except Invalid: 49 | try: 50 | int(arg, base=16) # if not an email, it must be a hex key 51 | assert len(arg) >= 8 # at least 8 characters long 52 | return arg 53 | except (ValueError, TypeError, AssertionError): 54 | raise Invalid('expected GPG id') 55 | 56 | def to_gpg_ids(arg): 57 | return [to_gpg_id(i) for i in to_list(arg)] 58 | 59 | def to_snake_case(key): 60 | return '_'.join(key.strip().lower().split()) 61 | 62 | # provide user-friendly error messages 63 | voluptuous_error_msg_mappings = { 64 | "extra keys not allowed": ("unknown key", "key"), 65 | "expected a dictionary": ("expected key-value pairs", "value"), 66 | } 67 | 68 | # define the schema for the settings file 69 | schema = Schema( 70 | { 71 | Required('my_gpg_ids'): to_gpg_ids, 72 | 'sign_with': to_gpg_id, 73 | 'avendesora_gpg_passphrase_account': to_str, 74 | 'avendesora_gpg_passphrase_field': to_str, 75 | 'name_template': to_str, 76 | Required('recipients'): { 77 | Extra: { 78 | Required('category'): to_ident, 79 | Required('email'): to_emails, 80 | 'gpg_id': to_gpg_id, 81 | 'attach': to_paths, 82 | 'networth': to_ident, 83 | } 84 | }, 85 | }, 86 | extra = REMOVE_EXTRA 87 | ) 88 | 89 | # this function implements references 90 | def expand_settings(value): 91 | # allows macro values to be defined as a top-level setting. 92 | # allows macro reference to be found anywhere. 93 | if isinstance(value, str): 94 | value = value.strip() 95 | if value[:1] == '@': 96 | value = settings.get(to_snake_case(value[1:])) 97 | return value 98 | if isinstance(value, dict): 99 | return {k:expand_settings(v) for k, v in value.items()} 100 | if isinstance(value, list): 101 | return [expand_settings(v) for v in value] 102 | raise NotImplementedError(value) 103 | 104 | def normalize_key(key, parent_keys): 105 | if parent_keys != ('recipients',): 106 | # normalize all keys except the recipient names 107 | return to_snake_case(key) 108 | return key 109 | 110 | try: 111 | # Read settings 112 | config_filepath = Path('postmortem.nt') 113 | if config_filepath.exists(): 114 | 115 | # load from file 116 | settings = nt.load( 117 | config_filepath, 118 | keymap = (keymap:={}), 119 | normalize_key = normalize_key 120 | ) 121 | 122 | # expand references 123 | settings = expand_settings(settings) 124 | 125 | # check settings and transform to desired types 126 | settings = schema(settings) 127 | 128 | # show the resulting settings 129 | print(nt.dumps(settings, default=str)) 130 | 131 | except nt.NestedTextError as e: 132 | e.report() 133 | except MultipleInvalid as e: 134 | report_voluptuous_errors(e, keymap, config_filepath) 135 | except OSError as e: 136 | error(os_error(e)) 137 | terminate() 138 | -------------------------------------------------------------------------------- /examples/postmortem/postmortem.expanded.nt: -------------------------------------------------------------------------------- 1 | my_gpg_ids: 2 | - odin@norse-gods.com 3 | sign_with: odin@norse-gods.com 4 | name_template: {name}-{now:YYMMDD} 5 | recipients: 6 | Frigg: 7 | email: 8 | - frigg@norse-gods.com 9 | category: wife 10 | attach: 11 | - ~/home/estate/trust.pdf 12 | - ~/home/estate/will.pdf 13 | - ~/home/estate/deed-valhalla.pdf 14 | networth: odin 15 | Thor: 16 | email: 17 | - thor@norse-gods.com 18 | category: kids 19 | attach: 20 | - ~/home/estate/trust.pdf 21 | - ~/home/estate/will.pdf 22 | - ~/home/estate/deed-valhalla.pdf 23 | Loki: 24 | email: 25 | - loki@norse-gods.com 26 | category: kids 27 | attach: 28 | - ~/home/estate/trust.pdf 29 | - ~/home/estate/will.pdf 30 | - ~/home/estate/deed-valhalla.pdf 31 | -------------------------------------------------------------------------------- /examples/postmortem/postmortem.nt: -------------------------------------------------------------------------------- 1 | my GPG ids: odin@norse-gods.com 2 | sign with: @ my gpg ids 3 | name template: {name}-{now:YYMMDD} 4 | estate docs: 5 | - ~/home/estate/trust.pdf 6 | - ~/home/estate/will.pdf 7 | - ~/home/estate/deed-valhalla.pdf 8 | 9 | recipients: 10 | Frigg: 11 | email: frigg@norse-gods.com 12 | category: wife 13 | attach: @ estate docs 14 | networth: odin 15 | Thor: 16 | email: thor@norse-gods.com 17 | category: kids 18 | attach: @ estate docs 19 | Loki: 20 | email: loki@norse-gods.com 21 | category: kids 22 | attach: @ estate docs 23 | -------------------------------------------------------------------------------- /examples/postmortem/voluptuous_errors.py: -------------------------------------------------------------------------------- 1 | ../validation/voluptuous_errors.py -------------------------------------------------------------------------------- /examples/references/diet: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from inform import Error, display, dedent 4 | import nestedtext as nt 5 | import re 6 | 7 | foods = nt.loads(dedent(""" 8 | oatmeal: 9 | steel cut oats: 1/4 cup 10 | tangerines: 1 each 11 | whole milk: 1/4 cup 12 | steel cut oats: 13 | calories by weight: 150/40 cals/gram 14 | tangerines: 15 | calories each: 40 cals 16 | calories by weight: 53/100 cals/gram 17 | whole milk: 18 | calories by weight: 149/255 cals/gram 19 | calories by volume: 149 cals/cup 20 | almonds: 21 | calories each: 40 cals 22 | calories by weight: 822/143 cals/gram 23 | calories by volume: 822 cals/cup 24 | """), dict) 25 | 26 | meals = nt.loads(dedent(""" 27 | 21 March 2023: 28 | breakfast: oatmeal 29 | 22 March 2023: 30 | breakfast: @oatmeal 31 | 23 March 2023: 32 | breakfast: @oatmeal(tangerines: 0 each, almonds: 10 each) 33 | """), dict) 34 | 35 | def expand_foods(value): 36 | # allows macro values to be defined as a top-level food. 37 | # allows macro reference to be found anywhere. 38 | if isinstance(value, str): 39 | value = value.strip() 40 | if value[:1] == '@': 41 | value = parse_macro(value[1:].strip()) 42 | return value 43 | if isinstance(value, dict): 44 | return {k:expand_foods(v) for k, v in value.items()} 45 | if isinstance(value, list): 46 | return [expand_foods(v) for v in value] 47 | raise NotImplementedError(value) 48 | 49 | def parse_macro(macro): 50 | match = re.match(r'(\w+)(?:\((.*)\))?', macro) 51 | if match: 52 | name, args = match.groups() 53 | try: 54 | food = foods[name].copy() 55 | except KeyError: 56 | raise Error("unknown food.", culprit=name) 57 | if args: 58 | args = nt.loads('{' + args + '}', dict) 59 | food.update(args) 60 | return food 61 | raise Error("unknown macro.", culprit=macro) 62 | 63 | 64 | try: 65 | meals = expand_foods(meals) 66 | display(nt.dumps(meals)) 67 | except Error as e: 68 | e.terminate() 69 | -------------------------------------------------------------------------------- /examples/references/diet.nt: -------------------------------------------------------------------------------- 1 | 21 March 2023: 2 | breakfast: oatmeal 3 | 22 March 2023: 4 | breakfast: 5 | steel cut oats: 1/4 cup 6 | tangerines: 1 each 7 | whole milk: 1/4 cup 8 | 23 March 2023: 9 | breakfast: 10 | steel cut oats: 1/4 cup 11 | tangerines: 0 each 12 | whole milk: 1/4 cup 13 | almonds: 10 each 14 | -------------------------------------------------------------------------------- /examples/test_examples.py: -------------------------------------------------------------------------------- 1 | # encoding: utf8 2 | 3 | from pathlib import Path 4 | from shlib import Run, cd, cwd, to_path 5 | from textwrap import dedent 6 | import pytest 7 | import sys 8 | 9 | tests_dir = Path(__file__).parent 10 | 11 | def strip_comments(text): 12 | return '\n'.join( 13 | l for l in text.splitlines() 14 | if l.strip() and not l.strip().startswith('#') 15 | ) 16 | 17 | def test_nestedtext_to_json(): 18 | with cd(tests_dir / "conversion-utilities"): 19 | stimulus = Path('../addresses/address.nt').read_text() 20 | expected = Path('../addresses/address.json').read_text() 21 | nt2json = Run('./nestedtext-to-json', stdin=stimulus, modes='sOEW') 22 | assert nt2json.stdout.strip() == expected.strip() 23 | 24 | def test_nestedtext_to_json_fumiko(): 25 | with cd(tests_dir / "conversion-utilities"): 26 | stimulus = Path('../addresses/fumiko.nt').read_text() 27 | expected = Path('../addresses/fumiko.json').read_text() 28 | nt2json = Run('./nestedtext-to-json', stdin=stimulus, modes='sOEW') 29 | assert nt2json.stdout.strip() == expected.strip() 30 | 31 | def test_json_to_nestedtext(): 32 | with cd(tests_dir / "conversion-utilities"): 33 | stimulus = Path('../addresses/address.json').read_text() 34 | expected = strip_comments(Path('../addresses/address.nt').read_text()) 35 | json2nt = Run('./json-to-nestedtext', stdin=stimulus, modes='sOEW') 36 | assert json2nt.stdout.strip() == expected.strip() 37 | 38 | def test_json_to_nestedtext_fumiko(): 39 | with cd(tests_dir / "conversion-utilities"): 40 | stimulus = Path('../addresses/fumiko.json').read_text() 41 | expected = strip_comments(Path('../addresses/fumiko.nt').read_text()) 42 | json2nt = Run('./json-to-nestedtext', stdin=stimulus, modes='sOEW') 43 | assert json2nt.stdout.strip() == expected.strip() 44 | 45 | def test_yaml_to_nestedtext(): 46 | with cd(tests_dir / "conversion-utilities"): 47 | stimulus = Path('github-orig.yaml').read_text() 48 | expected = strip_comments(Path('github-orig.nt').read_text()) 49 | yaml2nt = Run('./yaml-to-nestedtext', stdin=stimulus, modes='sOEW') 50 | assert yaml2nt.stdout.strip() == expected.strip() 51 | 52 | def test_nestedtext_to_yaml(): 53 | with cd(tests_dir / "conversion-utilities"): 54 | stimulus = Path('github-intent.nt').read_text() 55 | expected = Path('github-intent.yaml').read_text() 56 | nt2yaml = Run('./nestedtext-to-yaml', stdin=stimulus, modes='sOEW') 57 | assert nt2yaml.stdout.strip() == expected.strip() 58 | 59 | def test_toml_to_nestedtext(): 60 | with cd(tests_dir / "conversion-utilities"): 61 | stimulus = Path('sparekeys.toml').read_text() 62 | expected = Path('sparekeys.nt').read_text() 63 | toml2nt = Run('./toml-to-nestedtext', stdin=stimulus, modes='sOEW') 64 | assert toml2nt.stdout.strip() == expected.strip() 65 | 66 | def test_csv_to_nestedtext(): 67 | with cd(tests_dir / "conversion-utilities"): 68 | stimulus = Path('percent_bachelors_degrees_women_usa.csv').read_text() 69 | expected = Path('percent_bachelors_degrees_women_usa.nt').read_text() 70 | csv2nt = Run('./csv-to-nestedtext -n', stdin=stimulus, modes='sOEW') 71 | assert csv2nt.stdout.strip() == expected.strip() 72 | 73 | def test_xml_to_nestedtext(): 74 | with cd(tests_dir / "conversion-utilities"): 75 | stimulus = Path('dmarc.xml').read_text() 76 | expected = Path('dmarc.nt').read_text() 77 | xml2nt = Run('./xml-to-nestedtext', stdin=stimulus, modes='sOEW') 78 | assert xml2nt.stdout.strip() == expected.strip() 79 | 80 | def test_deploy_pydantic(): 81 | with cd(tests_dir / "validation"): 82 | expected = Path('deploy_pydantic.out').read_text() 83 | dp = Run('python3 deploy_pydantic.py', modes='sOEW') 84 | assert dp.stdout.strip() == expected.strip() 85 | 86 | def test_deploy_voluptuous(): 87 | with cd(tests_dir / "validation"): 88 | expected = Path('deploy_voluptuous.out').read_text() 89 | dv = Run('python3 deploy_voluptuous.py', modes='sOEW') 90 | assert dv.stdout.strip() == expected.strip() 91 | 92 | def test_address(): 93 | if sys.version_info < (3, 8): 94 | return # address example uses walrus operator 95 | with cd(tests_dir / "addresses"): 96 | fumiko = Run('./address fumiko', modes='sOEW') 97 | assert fumiko.stdout.strip() == dedent(""" 98 | Fumiko Purvis: 99 | Position: Treasurer 100 | Address: 101 | 3636 Buffalo Ave 102 | Topeka, Kansas 20692 103 | Phone: 1-268-555-0280 104 | EMail: fumiko.purvis@hotmail.com 105 | """).strip() 106 | 107 | def test_michael_jordan(): 108 | if sys.version_info < (3, 8): 109 | return # address example uses walrus operator 110 | with cd(tests_dir / "deduplication"): 111 | mj = Run('./michael_jordan', modes='sOEW') 112 | expected = Path('./michael_jordan.out').read_text() 113 | assert mj.stdout.strip() == expected.strip() 114 | 115 | def test_cryptocurrency(): 116 | if sys.version_info < (3, 8): 117 | return # cryptocurrency example uses walrus operator 118 | with cd(tests_dir / "cryptocurrency"): 119 | # the cryptocurrency example outputs the current prices, so just do some 120 | # simple sanity checking on the output. 121 | cc = Run('./cryptocurrency', modes='sOEW') 122 | assert '5 BTC =' in cc.stdout 123 | assert '50 ETH =' in cc.stdout 124 | assert '50 kXLM =' in cc.stdout 125 | 126 | def test_postmortem(): 127 | if sys.version_info < (3, 8): 128 | return # postmortem example uses walrus operator 129 | with cd(tests_dir / "postmortem"): 130 | expected = Path('postmortem.expanded.nt').read_text() 131 | user_home_dir = str(to_path('~')) 132 | expected = expected.replace('~', user_home_dir) 133 | 134 | pm = Run('./postmortem', modes='sOEW') 135 | assert pm.stdout.strip() == expected.strip() 136 | 137 | def test_diet(): 138 | with cd(tests_dir / "references"): 139 | mj = Run('./diet', modes='sOEW') 140 | expected = Path('./diet.nt').read_text() 141 | assert mj.stdout.strip() == expected.strip() 142 | 143 | def test_long_lines(): 144 | with cd(tests_dir / "long_lines"): 145 | ll = Run('./long_lines_backslash', modes='sOEW') 146 | expected = Path('./long_lines.out').read_text() 147 | assert ll.stdout.strip() == expected.strip() 148 | 149 | ll = Run('./long_lines_space', modes='sOEW') 150 | expected = Path('./long_lines.out').read_text() 151 | assert ll.stdout.strip() == expected.strip() 152 | -------------------------------------------------------------------------------- /examples/validation/deploy.nt: -------------------------------------------------------------------------------- 1 | debug: false 2 | secret key: t=)40**y&883y9gdpuw%aiig+wtc033(ui@^1ur72w#zhw3_ch 3 | 4 | allowed hosts: 5 | - www.example.com 6 | 7 | database: 8 | engine: django.db.backends.mysql 9 | host: db.example.com 10 | port: 3306 11 | user: www 12 | 13 | webmaster email: admin@example.com 14 | -------------------------------------------------------------------------------- /examples/validation/deploy_pydantic.out: -------------------------------------------------------------------------------- 1 | {'allowed_hosts': ['www.example.com'], 2 | 'database': {'engine': 'django.db.backends.mysql', 3 | 'host': 'db.example.com', 4 | 'port': 3306, 5 | 'user': 'www'}, 6 | 'debug': False, 7 | 'secret_key': 't=)40**y&883y9gdpuw%aiig+wtc033(ui@^1ur72w#zhw3_ch', 8 | 'webmaster_email': 'admin@example.com'} 9 | -------------------------------------------------------------------------------- /examples/validation/deploy_pydantic.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import nestedtext as nt 4 | from pydantic import BaseModel, EmailStr 5 | from typing import List 6 | from pprint import pprint 7 | 8 | class Database(BaseModel): 9 | engine: str 10 | host: str 11 | port: int 12 | user: str 13 | 14 | class Config(BaseModel): 15 | debug: bool 16 | secret_key: str 17 | allowed_hosts: List[str] 18 | database: Database 19 | webmaster_email: EmailStr 20 | 21 | def normalize_key(key, parent_keys): 22 | return '_'.join(key.lower().split()) 23 | 24 | obj = nt.load('deploy.nt', normalize_key=normalize_key) 25 | config = Config.parse_obj(obj) 26 | 27 | pprint(config.dict()) 28 | -------------------------------------------------------------------------------- /examples/validation/deploy_voluptuous.out: -------------------------------------------------------------------------------- 1 | {'allowed_hosts': ['www.example.com'], 2 | 'database': {'engine': 'django.db.backends.mysql', 3 | 'host': 'db.example.com', 4 | 'port': 3306, 5 | 'user': 'www'}, 6 | 'debug': True, 7 | 'secret_key': 't=)40**y&883y9gdpuw%aiig+wtc033(ui@^1ur72w#zhw3_ch', 8 | 'webmaster_email': 'admin@example.com'} 9 | -------------------------------------------------------------------------------- /examples/validation/deploy_voluptuous.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import nestedtext as nt 4 | from voluptuous import Schema, Coerce, MultipleInvalid 5 | from voluptuous_errors import report_voluptuous_errors 6 | from inform import error, full_stop, terminate 7 | from pprint import pprint 8 | 9 | schema = Schema({ 10 | 'debug': Coerce(bool), 11 | 'secret_key': str, 12 | 'allowed_hosts': [str], 13 | 'database': { 14 | 'engine': str, 15 | 'host': str, 16 | 'port': Coerce(int), 17 | 'user': str, 18 | }, 19 | 'webmaster_email': str, 20 | }) 21 | 22 | def normalize_key(key, parent_keys): 23 | return '_'.join(key.lower().split()) 24 | 25 | filename = "deploy.nt" 26 | try: 27 | keymap = {} 28 | raw = nt.load(filename, keymap=keymap, normalize_key=normalize_key) 29 | config = schema(raw) 30 | except nt.NestedTextError as e: 31 | e.terminate() 32 | except MultipleInvalid as e: 33 | report_voluptuous_errors(e, keymap, filename) 34 | terminate() 35 | 36 | pprint(config) 37 | -------------------------------------------------------------------------------- /examples/validation/voluptuous_errors.py: -------------------------------------------------------------------------------- 1 | from inform import cull, error, full_stop 2 | import nestedtext as nt 3 | 4 | voluptuous_error_msg_mappings = { 5 | "extra keys not allowed": ("unknown key", "key"), 6 | "expected a dict": ("expected a key-value pair", "value"), 7 | "required key not provided": ("required key is missing", "value"), 8 | } 9 | 10 | def report_voluptuous_errors(multiple_invalid, keymap, source=None, sep="›"): 11 | source = str(source) if source else "" 12 | 13 | for err in multiple_invalid.errors: 14 | 15 | # convert message to something easier for non-savvy user to understand 16 | msg, kind = voluptuous_error_msg_mappings.get( 17 | err.msg, (err.msg, 'value') 18 | ) 19 | 20 | # get metadata about error 21 | if keymap: 22 | culprit = nt.get_keys(err.path, keymap=keymap, strict="found", sep=sep) 23 | line_nums = nt.get_line_numbers(err.path, keymap, kind=kind, sep="-", strict=False) 24 | loc = nt.get_location(err.path, keymap) 25 | if loc: 26 | codicil = loc.as_line(kind) 27 | else: # required key is missing 28 | missing = nt.get_keys(err.path, keymap, strict="missing", sep=sep) 29 | codicil = f"‘{missing}’ was not found." 30 | 31 | file_and_lineno = f"{source!s}@{line_nums}" 32 | culprit = cull((file_and_lineno, culprit)) 33 | else: 34 | keys = sep.join(str(c) for c in err.path) 35 | culprit = cull([source, keys]) 36 | codicil = None 37 | 38 | # report error 39 | error(full_stop(msg), culprit=culprit, codicil=codicil) 40 | -------------------------------------------------------------------------------- /nestedtext/__init__.py: -------------------------------------------------------------------------------- 1 | # NestedText 2 | __version__ = "3.8.dev2" 3 | __released__ = "2025-04-05" 4 | 5 | from .nestedtext import ( 6 | load, loads, dump, dumps, NestedTextError, 7 | 8 | # utitilies 9 | get_keys, get_value, get_location, get_line_numbers, 10 | 11 | # deprecated utitilies 12 | get_value_from_keys, get_lines_from_keys, get_original_keys, join_keys, 13 | 14 | # the following is only needed in order to generate the documentation 15 | Location 16 | ) 17 | -------------------------------------------------------------------------------- /nestedtext/__init__.pyi: -------------------------------------------------------------------------------- 1 | from .nestedtext import ( 2 | # reader 3 | load as load, 4 | loads as loads, 5 | get_lines_from_keys as get_lines_from_keys, 6 | get_original_keys as get_original_keys, 7 | get_value_from_keys as get_value_from_keys, 8 | join_keys as join_keys, 9 | 10 | # writer 11 | dump as dump, 12 | dumps as dumps, 13 | 14 | # exceptions 15 | NestedTextError as NestedTextError, 16 | 17 | # internals 18 | Location as Location, 19 | Line as Line, 20 | ) 21 | 22 | __released__: str 23 | __version__: str 24 | -------------------------------------------------------------------------------- /nestedtext/nestedtext.pyi: -------------------------------------------------------------------------------- 1 | from inform import Error 2 | from pathlib import Path 3 | from typing import Any, Callable, TextIO, Type 4 | 5 | class NestedTextError(Error, ValueError): ... 6 | 7 | class Line: 8 | text: str 9 | lineno: int 10 | kind: str 11 | depth: int 12 | key: str 13 | value: str | None 14 | prev_line: Line 15 | 16 | def render( 17 | self, 18 | col: int = ... 19 | ) -> str: 20 | ... 21 | 22 | class Location: 23 | line: Line 24 | key_line: Line 25 | col: int 26 | key_col: int 27 | 28 | def __init__( 29 | self, 30 | line: Line = ..., 31 | col: int = ..., 32 | key_line: Line = ..., 33 | key_col: int = ... 34 | ) -> None: 35 | ... 36 | 37 | def as_tuple( 38 | self, 39 | kind: str = ... 40 | ) -> tuple[Line, int]: 41 | ... 42 | 43 | def as_line( 44 | self, 45 | kind: str = ..., 46 | offset: int | (int, int) | None = ... 47 | ) -> str: 48 | ... 49 | 50 | def get_line_numbers( 51 | self, 52 | kind: str = ..., 53 | sep: str = ..., 54 | ) -> tuple[int, int] | str: 55 | ... 56 | def loads( 57 | content : str, 58 | top: str | Callable | Type[dict] | Type[list] | Type[str] = ..., 59 | *, 60 | source: str | Path = ..., 61 | on_dup: str | Callable = ..., 62 | # keymap: dict[tuple[str, ...], Location] = ..., 63 | keymap: dict[tuple[str, ...], Any] = ..., 64 | normalize_key: Callable = ..., 65 | ) -> str | list | dict | None: 66 | ... 67 | 68 | def load( 69 | f: str | Path | TextIO, 70 | top: str | Callable | Type[dict] | Type[list] | Type[str] = ..., 71 | *, 72 | on_dup: str | Callable = ..., 73 | # keymap: dict[tuple[str, ...], Location] = ..., 74 | keymap: dict[tuple[str, ...], Any] = ..., 75 | normalize_key: Callable = ..., 76 | ) -> str | list | dict | None: 77 | ... 78 | 79 | def dumps( 80 | obj: Any, 81 | *, 82 | width: int = ..., 83 | inline_level: int = ..., 84 | sort_keys: bool | Callable = ..., 85 | indent: int = ..., 86 | converters: dict[Type, Callable] | None = ..., 87 | default: str | Callable | None = ... 88 | ) -> str: 89 | ... 90 | 91 | def dump( 92 | obj: Any, 93 | dest, 94 | **kwargs 95 | ) -> None: ... 96 | 97 | def get_value_from_keys( 98 | obj: str | list | dict | None, 99 | keys: tuple[str, ...], 100 | ) -> str | list | dict: 101 | ... 102 | 103 | def get_lines_from_keys( 104 | obj: str | list | dict | None, 105 | keys: tuple[str, ...], 106 | # keymap: dict[tuple[str, ...], Location], 107 | keymap: dict[tuple[str, ...], Any] = ..., 108 | kind: str = ..., 109 | sep: str = ... 110 | ) -> str | tuple[int, int]: 111 | ... 112 | 113 | def get_original_keys( 114 | keys: tuple[str | int, ...], 115 | # keymap: dict[tuple[str, ...], Location], 116 | keymap: dict[tuple[str | int, ...], Any] = ..., 117 | strict: bool | str = ... 118 | ) -> tuple[int]: 119 | ... 120 | 121 | def join_keys( 122 | keys: tuple[str | int, ...], 123 | sep: str = ..., 124 | # keymap: dict[tuple[str], Location] = ..., 125 | keymap: dict[tuple[str | int, ...], Any] = ..., 126 | strict: bool | str = ... 127 | ) -> str: 128 | ... 129 | 130 | def get_location( 131 | keys: tuple[str | int, ...], 132 | keymap: dict[tuple[str | int, ...], Location], 133 | ) -> Location: 134 | ... 135 | 136 | def get_line_numbers( 137 | keys: tuple[str | int, ...], 138 | keymap: dict[tuple[str | int, ...], Location], 139 | kind: str = ..., 140 | base: int = ..., 141 | strict: bool = ..., 142 | sep: str = ..., 143 | ) -> tuple[int, int] | str: 144 | ... 145 | 146 | def get_keys( 147 | keys: tuple[str | int, ...], 148 | keymap: dict[tuple[str | int, ...], Location], 149 | original: bool = ..., 150 | strict: bool | string = ..., 151 | sep: str = ..., 152 | ) -> tuple[str | int, ...] | str: 153 | ... 154 | -------------------------------------------------------------------------------- /nestedtext/py.typed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KenKundert/nestedtext/ff95310eb73e513b0f18f06517796507248aaa4c/nestedtext/py.typed -------------------------------------------------------------------------------- /proposed_tests/Makefile: -------------------------------------------------------------------------------- 1 | tests.json: tests.nt convert 2 | ./convert 3 | -------------------------------------------------------------------------------- /proposed_tests/README.rst: -------------------------------------------------------------------------------- 1 | Proposed Replacement Official NestedText Test Suite 2 | =================================================== 3 | 4 | 5 | Issues with Previous Test Suite 6 | ------------------------------- 7 | 8 | The previous test suite had the following issues. 9 | 10 | 1. It contained symbolic links, which caused problems on Windows. 11 | 12 | 2. The tests contain invisible characters in the form of tabs, end-of-line 13 | spaces, unicode white spaces, control characters, and line termination 14 | characters. These invisible characters were often confusing and could easily 15 | be lost. 16 | 17 | 3. The tests were too tailored to the specific behavior of the Python 18 | *NestedText* implementation. 19 | 20 | 4. The tests were categorized and numbered. Both the categorization and 21 | numbering were problematic. Each test often fit into many categories but 22 | could only be placed in one. The numbering implied an order when no natural 23 | order exists. 24 | 25 | 5. Important tests were missing from the test suite. 26 | 27 | 28 | The New Test Suite 29 | ------------------ 30 | 31 | These test cases are focused on assuring that valid *NestedText* input is 32 | properly read and invalid NestedText is properly identified by an implementation 33 | of *NestedText*. No attempt is made to assure that the implementation produces 34 | valid *NestedText*. There is considerably flexibility in the way that 35 | *NestedText* may be generated. In light of this flexibility the best way to 36 | test a *NestedText* writer is to couple it with a *NestedText* reader and 37 | perform a round trip through both, which can be performed with these test cases. 38 | 39 | The test cases are contained in *tests.nt*. The convert command converts these 40 | test cases into JSON (*tests.json*). It is expected this JSON file is what is 41 | used to test *NestedText* implementations. The *NestedText* file of test cases, 42 | *tests.nt*, is used to generate *tests.json*, and is only needed if you plan to 43 | add or modify tests. Do not modify the JSON file directly, as any changes will 44 | be overridden whenever *convert* is run. 45 | 46 | Each test case in *tests.nt* is a dictionary entry. The key is used as the name 47 | of the test. The keys must be unique and are largely chosen at random, but any 48 | words that are expected to be found within the test cases are avoided. This 49 | allows test cases to be quickly found by searching for their name. 50 | 51 | The fields that may be specified are: 52 | 53 | description (str): 54 | Simply describes the test. Is optional and unused. 55 | 56 | string_in (str): 57 | This is a string that will be fed into a *NestedText* load function. This 58 | string contains a *NestedText* document, though that document may contain 59 | errors. This string may contain Unicode characters and Unicode escape 60 | sequences. It is generally recommended that Unicode white space be given 61 | using escape sequences so that there presence is obvious. 62 | 63 | bytes_in (str): 64 | This is an alternate to *string_in*. In this case the *NestedText* document 65 | is constrained to consist of ASCII characters and escape sequences such as 66 | \t, \r, \n, etc. Also supported are binary escape sequences, \x00 to \xFF. 67 | 68 | encoding (str): 69 | The desired encoding for the *NestedText* document. The default encoding is 70 | UTF-8. If you specify *bytes* as the encoding, no encoding is used. 71 | 72 | load_out (None | str | list | dict): 73 | The expected output from the *NestedText* load function if no errors are 74 | expected. If a string is given and if the first character in the string is 75 | ‘!’, then the mark is removed and what remains is evaluated by Python, with 76 | the result becoming the expected output. 77 | 78 | load_err (dict): 79 | Details about the error that is expected to be found and reported by the 80 | *NestedText* load function. It consists of 4 fields: 81 | 82 | message (str): 83 | The error message that is emitted by the Python implementation of 84 | *NestedText*. 85 | 86 | line (str): 87 | The text of the line in the *NestedText* input from *load_in* or 88 | *bytes_in* that contains the error. 89 | 90 | lineno (None, int): 91 | The line number of the line that contains the error. The first line 92 | is line 0. 93 | 94 | colno (None, int): 95 | The number of the column where the error is likely to be. The first 96 | column is column 0. 97 | 98 | Here is an example of a test with a valid *NestedText* document:: 99 | 100 | pundit: 101 | description: a single-level dictionary 102 | load_in: 103 | > key 1: value 1 104 | > key 2: value 2 105 | > 106 | load_out: 107 | key 1: value 1 108 | key 2: value 2 109 | 110 | And here is an example of a test with an invalid *NestedText* document:: 111 | 112 | foundling: 113 | load_in: 114 | > ingredients: 115 | > - 3 green chilies 116 | > - 3 red chilies 117 | load_err: 118 | message: invalid indentation 119 | line: - 3 red chilies 120 | lineno: 2 121 | colno: 2 122 | 123 | The test keys (*pundit* and *foundling*) are arbitrary. 124 | 125 | Control characters and Unicode white space characters are expected to be 126 | backslash escaped in *load_in*, *bytes_in*, *load_out*, and *load_err.lines*. 127 | Here are some specific cases where backslash escapes should be used: 128 | 129 | **Line Terminations** Newlines are replaced by line feed (LF) characters unless 130 | the newline is preceded by either \\r or \\n or both. The \\r is replaced by 131 | a carriage return (CR) and the \\n is replaced by a line feed (LF). In this way 132 | the line termination characters can be specified explicitly on a per line basis. 133 | For example:: 134 | 135 | key 1: this line ends with CR & LF\r\n 136 | key 2: this line ends with CR\r 137 | key 3: this line ends with LF\n 138 | key 4: this line also ends with LF 139 | key 5: this line, being the last, has no line termination character 140 | 141 | **White Space** All white space other than ASCII spaces and newlines should be 142 | made explicit by using backslash escape sequences. Specifically tabs should be 143 | specified as \\t and the Unicode white spaces should be specified using their 144 | \\x or \\u code (ex. \\xa0 or \\u00a0 for the no-break space). In addition, end 145 | of line spaces are optionally made explicit by replacing them with \\x20 if they 146 | are important and there is concern that they may be accidentally lost. 147 | 148 | **Other Special Characters** Backslash escape codes should also be used for 149 | control codes (\\a for bell, \\b for backspace, \\x7f for delete, \\x1b for 150 | escape, etc) and for backslash itself (\\\\). 151 | 152 | 153 | tests.json 154 | ---------- 155 | 156 | The *convert* command creates *tests.json*, but if you do not wish to add or 157 | modify the tests, you can simply use *tests.json* from the GitHub repository. 158 | 159 | *tests.json* is a file suitable for use with `parametrize_from_file 160 | `_, 161 | which is a *pytest* plugin suitable for testing Python projects (*test_nt.py* 162 | uses *parametrize_from_file* to apply *tests.json* to the Python implementation 163 | of *NestedText*). However, you can use *tests.json* directly to implement tests 164 | for any for any *NestedText* implementation in any language. 165 | 166 | It contains dictionary with a single key, *load_tests*. The value of this key 167 | is a nested dictionary where each key-value pair is one test. The key is the 168 | name of the test and the value is the test. The test consists of the following 169 | fields: 170 | 171 | load_in: 172 | This is a string that contains the *NestedText* document to be loaded for 173 | the test. The string is a base64 encoded string of bytes. 174 | 175 | load_out: 176 | The expected output from the *NestedText* loader if no error is expected. 177 | The structure of this value is dependent on the *NestedText* document 178 | encoded in *load_in*. It may be a nested collection of lists, dictionaries 179 | and strings, or it may be *null*. 180 | 181 | load_err: 182 | Details about an expected error. *load_err* supports the following 183 | subfields: 184 | 185 | message: 186 | The message generated by the Python implementation of *NestedText* for 187 | the expected error. 188 | 189 | line: 190 | The line in the input document where the error occurs. 191 | 192 | lineno: 193 | The line number of the line where the error occurs. 0 represents the 194 | first line in the document. Is *null* or missing if the line number is 195 | unknown. 196 | 197 | colno: 198 | The column number where the error occurs. 0 represents the first 199 | column. Is *null* or missing if the column number is unknown. 200 | 201 | types: 202 | A dictionary of line-type counts. It contains the count of each type of 203 | line contained in the input document. These counts can be used to filter 204 | the tests if desired. 205 | 206 | The line types are:: 207 | 208 | blank 209 | comment 210 | dict item 211 | inline dict 212 | inline list 213 | key item 214 | list item 215 | string item 216 | unrecognized 217 | 218 | 219 | Caveats 220 | ------- 221 | 222 | Be aware that this is a trial version of the official *NestedText* tests, and so 223 | is subject to change. 224 | 225 | This is the second trial version of this new test suite. It was uploaded on 23 226 | March 2025 and again on 24 March with more tests (there are now 143 tests). 227 | 228 | Version 3.7 of the Python implementation of *NestedText* does not yet pass all 229 | of these tests. 230 | -------------------------------------------------------------------------------- /proposed_tests/clean: -------------------------------------------------------------------------------- 1 | #!/bin/csh -f 2 | 3 | set nonomatch 4 | 5 | foreach path (*/{load,dump}_{in,out,err}.{nt,json,yaml}) 6 | if (-e $path) then 7 | set dir=$path:h 8 | /bin/rm -rf $dir 9 | endif 10 | end 11 | -------------------------------------------------------------------------------- /proposed_tests/convert: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # IMPORTS {{{1 4 | import nestedtext as nt 5 | from inform import ( 6 | Error, fatal, indent, is_str, is_mapping, is_collection, os_error, 7 | terminate, warn 8 | ) 9 | from base64 import b64encode 10 | import codecs 11 | try: 12 | from nestedtext.nestedtext import Lines 13 | except ImportError: 14 | # To include line types in the generated json file we need a direct link to 15 | # NestedText Python implementation. Create a symbolic link from this 16 | # directory to the source directory: 17 | # ln -s .../nestedtext/nestedtext/nestedtext.py . 18 | warn( 19 | "could not import internal NestedText Lines class,", 20 | "line types are not available." 21 | ) 22 | Lines = None 23 | from collections import defaultdict 24 | from itertools import batched 25 | from voluptuous import ( 26 | Schema, Optional, Required, Any, Self, Invalid, MultipleInvalid 27 | ) 28 | from voluptuous_errors import report_voluptuous_errors 29 | import re 30 | import json 31 | 32 | 33 | # SCHEMA {{{1 34 | def as_string_with_escapes(arg): 35 | if not is_str(arg): 36 | raise Invalid("expected string.") 37 | return arg.encode('ascii', errors='backslashreplace').decode('unicode-escape') 38 | 39 | hierarchy_with_escapes = Schema( 40 | Any(as_string_with_escapes, [Self], {as_string_with_escapes: Self}) 41 | ) 42 | 43 | def evaluate(arg): 44 | # use Python to evaluate argument if it begins with ! 45 | if is_str(arg): 46 | arg = arg.strip() 47 | if arg == 'None': 48 | return None 49 | if arg[0:1].strip() == '!': 50 | return eval(arg[1:]) 51 | raise Invalid("expected ‘None’ or string that starts with ‘!’.") 52 | 53 | def as_index(arg): 54 | if arg == 'None': 55 | return None 56 | return int(arg) 57 | 58 | tests_validator = Schema({ 59 | str: dict( 60 | description = str, 61 | string_in = str, 62 | bytes_in = str, 63 | encoding = str, 64 | load_out = Any(evaluate, hierarchy_with_escapes), 65 | load_err = dict( 66 | message = str, 67 | line = as_string_with_escapes, 68 | lineno = as_index, 69 | colno = as_index, 70 | ), 71 | ) 72 | }) 73 | 74 | # UTILITIES {{{1 75 | # process backslash escapes {{{2 76 | nl_ptn = re.compile(rb'((?:\\n|\\r){1,2})\n') 77 | 78 | def fix_eol(match): 79 | # remove implicitly added LF 80 | stripped_of_lf = match.group(0).replace(b'\n', b'') 81 | 82 | # replace explicitly specified \r and \n with CR and LF 83 | return stripped_of_lf.replace(rb'\r', b'\r').replace(rb'\n', b'\n') 84 | 85 | # extract line types {{{1 86 | if Lines: 87 | def extract_line_types(text): 88 | types = defaultdict(int) 89 | lines = Lines(text.splitlines(), True) 90 | 91 | for line in lines.read_lines(): 92 | types[line.kind] += 1 93 | 94 | return types 95 | else: 96 | def extract_line_types(text): 97 | return {} 98 | 99 | # encode() {{{2 100 | def utf8_encode(given, encoding): 101 | # interpret escape sequences, convert to base64 102 | 103 | # oddly, unicode-escape cannot handle unicode, so convert to ascii 104 | bytes = given.encode('ascii', errors='backslashreplace') # convert to bytes 105 | bytes = nl_ptn.sub(fix_eol, bytes) # remove excess newlines 106 | text = bytes.decode('unicode-escape') # expand escape sequences 107 | bytes = text.encode(encoding) # encode with desired encoding 108 | return b64encode(bytes).decode('ascii') # encode in base64 109 | 110 | def bytes_encode(given, encoding): 111 | bytes = given.encode('ascii', errors='strict') # convert to bytes 112 | bytes = nl_ptn.sub(fix_eol, bytes) # remove excess newlines 113 | bytes, _ = codecs.escape_decode(bytes) 114 | # unfortunately, escape_decode is an undocumented private function 115 | # if this becomes a problem this line should probably be replaced with a 116 | # regular expression substitution that matches and maps the normal ASCII 117 | # escape sequences and the binary escape sequences (\x00 to \xFF). 118 | return b64encode(bytes).decode('ascii') # encode in base64 119 | 120 | 121 | # CONVERT {{{1 122 | # read tests.nt {{{1 123 | try: 124 | keymap = {} 125 | tests = nt.load("tests.nt", keymap=keymap) 126 | tests = tests_validator(tests) 127 | except OSError as e: 128 | fatal(os_error(e)) 129 | except nt.NestedTextError as e: 130 | e.terminate() 131 | except MultipleInvalid as e: 132 | report_voluptuous_errors(e, keymap, source='tests.nt') 133 | terminate() 134 | 135 | 136 | # process tests {{{1 137 | processed = {} 138 | accumulated_line_types = defaultdict(int) 139 | try: 140 | for key, fields in tests.items(): 141 | if 'string_in' not in fields and 'bytes_in' not in fields: 142 | warn("‘string_in’ is missing.", culprit=key) 143 | continue 144 | 145 | if 'string_in' in fields and 'bytes_in' in fields: 146 | warn("must not have both ‘string_in’ and ‘bytes_in’ fields.", culprit=key) 147 | continue 148 | 149 | # process string_in or bytes_in 150 | encoding = fields.get('encoding', 'utf-8') 151 | try: 152 | if 'string_in' in fields: 153 | load_in = fields.get('string_in') 154 | load_in_encoded = utf8_encode(load_in, encoding) 155 | else: 156 | load_in = fields.get('bytes_in') 157 | load_in_encoded = bytes_encode(load_in, encoding) 158 | except (UnicodeEncodeError, UnicodeDecodeError) as e: 159 | raise Error(e, culprit=key) 160 | 161 | load_out = fields.get('load_out') 162 | load_err = fields.get("load_err", {}) 163 | if load_out and load_err: 164 | raise Error("must not specify both ‘load_out’ and ‘load_err’.") 165 | 166 | processed_test = dict( 167 | load_in = load_in_encoded, 168 | load_out = load_out, 169 | load_err = load_err, 170 | encoding = encoding, 171 | types = extract_line_types(load_in), 172 | ) 173 | for line_type, count in processed_test['types'].items(): 174 | accumulated_line_types[line_type] += count 175 | 176 | processed[key] = processed_test 177 | 178 | # write tests.json {{{1 179 | with open('tests.json', 'w') as f: 180 | json.dump(dict(load_tests=processed), f, indent=4, ensure_ascii=False) 181 | f.write('\n') 182 | 183 | except OSError as e: 184 | fatal(os_error(e)) 185 | except Error as e: 186 | e.terminate(culprit=e.get_culprit(key)) 187 | 188 | if accumulated_line_types: 189 | print("Count of line types found:") 190 | print(indent(nt.dumps(accumulated_line_types, sort_keys=True))) 191 | print() 192 | print(f"Number of tests: {len(tests)}") 193 | 194 | -------------------------------------------------------------------------------- /proposed_tests/test_nt.py: -------------------------------------------------------------------------------- 1 | # IMPORTS {{{1 2 | from functools import partial 3 | from inform import cull, indent, render 4 | from parametrize_from_file import parametrize 5 | from pathlib import Path 6 | from voluptuous import Schema, Optional, Required, Any, Invalid 7 | from base64 import b64decode 8 | import nestedtext as nt 9 | import json 10 | 11 | # GLOBALS {{{1 12 | TEST_SUITE = Path('tests.json') 13 | TEST_DIR = Path(__file__).parent 14 | 15 | # PARAMETERIZATION {{{1 16 | # Adapt parametrize_for_file to read dictionary rather than list {{{2 17 | def name_from_dict_keys(cases): 18 | return [{**v, 'id': k} for k,v in cases.items()] 19 | # the name 'id' is special, do not change it 20 | parametrize = partial(parametrize, preprocess=name_from_dict_keys) 21 | 22 | 23 | # SCHEMA {{{1 24 | def as_int(arg): 25 | return int(arg) 26 | 27 | schema = Schema({ 28 | Required("id", default='❬not given❭'): str, 29 | Required("load_in"): str, 30 | Required("load_out", default=None): Any(dict, list, str, None), 31 | Required("load_err", default={}): dict( 32 | message = str, 33 | line = str, 34 | lineno = Any(None, as_int), 35 | colno = Any(None, as_int) 36 | ), 37 | Required("encoding", default='utf-8'): str, 38 | Required("types"): {str:int}, 39 | }) 40 | 41 | 42 | # Checker {{{1 43 | class Checker: 44 | def __init__(self, test_name): 45 | self.test_name = test_name 46 | 47 | def check(self, expected, result, phase): 48 | self.expected = expected 49 | self.result = result 50 | self.phase = phase 51 | assert expected == result, self.fail_message() 52 | 53 | def fail_message(self): 54 | expected = list(render(self.expected).splitlines()) 55 | result = list(render(self.result).splitlines()) 56 | 57 | for i, lines in enumerate(zip(expected, result)): 58 | eline, rline = lines 59 | if eline != rline: 60 | break 61 | else: 62 | elen = len(expected) 63 | rlen = len(result) 64 | i += 1 65 | if elen > rlen: 66 | eline = expected[i] 67 | rline = '❬not available❭' 68 | else: 69 | eline = '❬not available❭' 70 | rline = result[i] 71 | 72 | expected = f"expected[{i}]: {eline}" 73 | result = f" result[{i}]: {rline}" 74 | desc = f"{self.test_name} while {self.phase}" 75 | 76 | return '\n'.join([desc, expected, result]) 77 | 78 | # TESTS {{{1 79 | @parametrize( 80 | path = TEST_DIR / TEST_SUITE, 81 | key = "load_tests", 82 | schema = schema, 83 | ) 84 | def test_nt(tmp_path, load_in, load_out, load_err, encoding, types, request): 85 | checker = Checker(request.node.callspec.id) 86 | 87 | # check load 88 | content = b64decode(load_in.encode('ascii')) 89 | try: 90 | result = nt.loads(content, top=any) 91 | if load_err: 92 | checker.check("@@@ an error @@@", result, "loading") 93 | return 94 | else: 95 | checker.check(load_out, result, "loading") 96 | except nt.NestedTextError as e: 97 | result = dict( 98 | message = e.get_message(), 99 | line = e.line, 100 | lineno = e.lineno, 101 | colno = e.colno 102 | ) 103 | checker.check(cull(load_err), cull(result), "loading") 104 | return 105 | except UnicodeDecodeError as e: 106 | problematic = e.object[e.start:e.end] 107 | prefix = e.object[:e.start] 108 | suffix = e.object[e.start:] 109 | lineno = prefix.count(b'\n') 110 | _, _, bol = prefix.rpartition(b'\n') 111 | eol, _, _ = e.object[e.start:].partition(b'\n') 112 | line = bol + eol 113 | colno = line.index(problematic) 114 | 115 | if encoding != 'bytes': 116 | line = line.decode(encoding) 117 | else: 118 | line = line.decode('ascii', errors='backslashreplace') 119 | load_err['line'] = load_err['line'].encode( 120 | 'ascii', errors='backslashreplace' 121 | ).decode('ascii') 122 | 123 | result = dict( 124 | message = e.reason, 125 | line = line, 126 | lineno = lineno, 127 | colno = colno, 128 | ) 129 | checker.check(load_err, result, "loading") 130 | return 131 | 132 | # check dump by doing a round-trip through load 133 | # the stimulus file does not have expected dump results because they can 134 | # vary between implementations and with dump options. 135 | try: 136 | dumped = nt.dumps(result) 137 | except nt.NestedTextError as e: 138 | checker.check(None, result, "dumping") 139 | 140 | try: 141 | result = nt.loads(dumped, top=any) 142 | checker.check(load_out, result, "re-loading") 143 | except nt.NestedTextError as e: 144 | checker.check(None, result, "re-loading") 145 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "nestedtext" 3 | version = "3.8.dev2" 4 | description = "human readable and writable data interchange format" 5 | readme = "README.rst" 6 | keywords = ["data", "serialization", "json", "yaml"] 7 | authors = [ 8 | {name = "Ken Kundert"}, 9 | {name = "Kale Kundert"}, 10 | {email = "nestedtext@nurdletech.com"} 11 | ] 12 | classifiers = [ 13 | "Development Status :: 5 - Production/Stable", 14 | "Intended Audience :: Developers", 15 | "License :: OSI Approved :: MIT License", 16 | "Natural Language :: English", 17 | "Operating System :: POSIX :: Linux", 18 | "Programming Language :: Python :: 3", 19 | "Topic :: Text Processing :: Markup", 20 | "Topic :: Utilities", 21 | ] 22 | requires-python = ">=3.6" 23 | dependencies = [ 24 | "inform>=1.28", 25 | ] 26 | 27 | [project.urls] 28 | homepage = "https://nestedtext.org" 29 | documentation = "https://nestedtext.org" 30 | repository = "https://github.com/kenkundert/nestedtext" 31 | changelog = "https://github.com/KenKundert/nestedtext/blob/master/doc/releases.rst" 32 | 33 | [build-system] 34 | requires = ["flit_core >=2,<4"] 35 | build-backend = "flit_core.buildapi" 36 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | pythonpath = . 3 | python_files = test_*.py 4 | addopts = 5 | --doctest-glob='*.rst' 6 | --ignore=test.clones.py 7 | --ignore=clones.py 8 | --ignore=tests/official_tests 9 | --ignore=Diffs 10 | --doctest-modules 11 | --tb=short 12 | -------------------------------------------------------------------------------- /tests/test_random.py: -------------------------------------------------------------------------------- 1 | # encoding: utf8 2 | # This test suite runs random text strings on a round trip through NestedText to 3 | # assure that what comes out is what was sent in. Non-printing characters can 4 | # confuse NestedText (often treated as white space by strip(), which can result 5 | # in invalid indentation errors), so they are filtered out. Also, NestedText 6 | # maps CR/LF and CR to LF, so the same is done to the input before comparing it 7 | # to the output. 8 | 9 | from hypothesis import assume, given, settings, strategies as st 10 | import nestedtext as nt 11 | from random import randint 12 | import re 13 | import sys 14 | 15 | max_examples = 1000 # takes several minutes 16 | max_examples = 100 17 | 18 | def normalize_line_breaks(s): 19 | return s.replace('\r\n', '\n').replace('\r', '\n') 20 | 21 | non_printing_chars = set( 22 | chr(n) for n in range(sys.maxunicode+1) if not chr(n).isprintable() 23 | ) 24 | def has_invalid_chars_for_strs(s): 25 | return non_printing_chars & set(s) 26 | 27 | def has_invalid_chars_for_dicts(s): 28 | return (non_printing_chars | set('{}[],:')) & set(s) 29 | 30 | def has_invalid_chars_for_lists(s): 31 | return (non_printing_chars | set('{}[],')) & set(s) 32 | 33 | def pad_randomly(match): 34 | # pad with 0-2 spaces before and after text 35 | text = match.group(0) 36 | leading_padding = randint(0,1)*' ' 37 | trailing_padding = randint(0,1)*' ' 38 | return leading_padding + text + trailing_padding 39 | 40 | def add_spaces(content, targets): 41 | if content[0:] not in ['[', '{']: 42 | return content # ignore content that is not inline list or dict 43 | if content.strip(' ') in ['[]', '{}']: 44 | return content # ignore an empty list or dict 45 | return re.sub(f'[{re.escape(targets)}]', pad_randomly, content).lstrip(' ') 46 | 47 | 48 | if sys.version_info[:3] > (3,10): 49 | @settings(max_examples=max_examples) 50 | @given(st.from_type(bool | None | int | float)) 51 | def test_types(v): 52 | expected = None if v is None else str(v) 53 | assert nt.loads(nt.dumps(v), top=any) == expected 54 | 55 | @settings(max_examples=max_examples) 56 | @given(st.text()) 57 | def test_strings(s): 58 | assert nt.loads(nt.dumps(s), top=str) == normalize_line_breaks(s) 59 | 60 | 61 | @settings(max_examples=max_examples) 62 | @given( 63 | st.dictionaries( 64 | keys = st.text( 65 | alphabet = st.characters(blacklist_categories='C') 66 | ), 67 | values = st.text( 68 | alphabet = st.characters(blacklist_categories='C') 69 | ) 70 | ) 71 | ) 72 | def test_dicts(data): 73 | expected = { 74 | normalize_line_breaks(k): normalize_line_breaks(v) 75 | for k, v in data.items() 76 | } 77 | 78 | # test normal dump 79 | result = nt.loads(nt.dumps(data), top=dict) 80 | assert nt.loads(nt.dumps(data), top=dict) == expected 81 | 82 | # test dump with inlines 83 | content = nt.dumps(data, width=999) 84 | assert nt.loads(content, top=dict) == expected 85 | 86 | # test dump with inlines and random spaces 87 | if content[0:] in ['[', '{']: 88 | spacey_content = add_spaces(content, '{}[],:') 89 | assert nt.loads(spacey_content, top=dict) == expected 90 | 91 | 92 | @settings(max_examples=max_examples) 93 | @given( 94 | st.lists( 95 | st.text( 96 | alphabet = st.characters(blacklist_categories='C') 97 | ) 98 | ) 99 | ) 100 | def test_lists(values): 101 | expected = [normalize_line_breaks(v) for v in values] 102 | 103 | # test normal dump 104 | assert nt.loads(nt.dumps(values), top=list) == expected 105 | 106 | # test dump with inlines 107 | content = nt.dumps(values, width=999) 108 | assert nt.loads(content, top=list) == expected 109 | 110 | # test dump with inlines and random spaces 111 | if content[0:] in ['[', '{']: 112 | content = add_spaces(content, '{}[],') 113 | assert nt.loads(content, top=list) == expected 114 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = lint, pytest, mypy 3 | isolated_build = True 4 | 5 | [testenv:lint] 6 | deps = 7 | setuptools 8 | pylama 9 | skip_install = true 10 | commands = 11 | pylama --ignore C901,E116,E226,E251,E203,E501,E741,E731 src/nestedtext/*.py 12 | 13 | [testenv] 14 | deps = 15 | arrow 16 | docopt 17 | flatten_dict 18 | hypothesis 19 | inform>=1.29 20 | natsort 21 | parametrize_from_file 22 | pydantic 23 | pydantic[email] 24 | pytest 25 | pytest-cov 26 | pyyaml 27 | quantiphy 28 | requests 29 | shlib 30 | toml 31 | voluptuous 32 | xmltodict 33 | 34 | [testenv:pytest] 35 | commands = py.test --cov {posargs} --cov-branch --cov-report term-missing 36 | 37 | [testenv:mypy] 38 | description = Run mypy 39 | deps = 40 | mypy 41 | {[testenv]deps} 42 | commands = 43 | # mypy --install-types --non-interactive {toxinidir}/nestedtext 44 | # mypy \ 45 | # --install-types \ 46 | # --non-interactive \ 47 | # --disable-error-code import \ 48 | # {toxinidir}/tests 49 | --------------------------------------------------------------------------------