├── test ├── __init__.py ├── wcfg-test-06.conf ├── wcfg-test-02.conf ├── wcfg-test-03.conf ├── wcfg-test-07.conf ├── wcfg-test-01.conf ├── drove-02.conf ├── wcfg-test-08.conf ├── heroes.conf ├── wcfg-test-09.conf ├── wcfg-test-05.conf ├── wcfg-test-00.conf ├── wcfg-test-04.conf ├── drove-01.conf ├── omni-01.conf ├── hep-001.conf ├── util.py ├── test_files.py ├── test_cast.py └── test_parser.py ├── .gitignore ├── MANIFEST.in ├── requirements.txt ├── doc ├── requirements.in ├── apiref.rst ├── index.rst ├── requirements.txt ├── quickstart.rst ├── advanced.rst ├── make.bat ├── Makefile └── conf.py ├── tox.ini ├── .readthedocs.yaml ├── .travis.yml ├── .github └── workflows │ └── test.yml ├── examples ├── hipack-webservice.html └── hipack-webservice ├── setup.py ├── CHANGELOG.md ├── hipack ├── README.rst └── hipack.py /test/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[co] 2 | -------------------------------------------------------------------------------- /test/wcfg-test-06.conf: -------------------------------------------------------------------------------- 1 | a { } 2 | -------------------------------------------------------------------------------- /test/wcfg-test-02.conf: -------------------------------------------------------------------------------- 1 | a 1 2 | 3 | -------------------------------------------------------------------------------- /test/wcfg-test-03.conf: -------------------------------------------------------------------------------- 1 | 2 | a 1 3 | b 2 4 | 5 | -------------------------------------------------------------------------------- /test/wcfg-test-07.conf: -------------------------------------------------------------------------------- 1 | a {} 2 | b {} 3 | c {} 4 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.rst 2 | include test/*.conf 3 | -------------------------------------------------------------------------------- /test/wcfg-test-01.conf: -------------------------------------------------------------------------------- 1 | # Empty file with just a comment 2 | 3 | -------------------------------------------------------------------------------- /test/drove-02.conf: -------------------------------------------------------------------------------- 1 | multiline: "this is 2 | a multiline string" 3 | 4 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # Recipe from: https://caremad.io/blog/setup-vs-requirement/ 2 | -e . 3 | -------------------------------------------------------------------------------- /doc/requirements.in: -------------------------------------------------------------------------------- 1 | sphinx==7.2 2 | sphinx_rtd_theme==2.0 3 | readthedocs-sphinx-search==0.3.2 4 | -------------------------------------------------------------------------------- /test/wcfg-test-08.conf: -------------------------------------------------------------------------------- 1 | a { 2 | b { 3 | c { 4 | d { 5 | } 6 | } 7 | } 8 | } 9 | 10 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py35,py36,py311,py3 3 | 4 | [testenv] 5 | commands = python -m unittest [] 6 | -------------------------------------------------------------------------------- /test/heroes.conf: -------------------------------------------------------------------------------- 1 | { name: "Spiderman", alter-ego: "Peter Parker" } 2 | { name: "Superman", alter-ego: "Clark Kent" } 3 | { name: "Batman", alter-ego: "Bruce Wayne" } 4 | -------------------------------------------------------------------------------- /test/wcfg-test-09.conf: -------------------------------------------------------------------------------- 1 | 2 | a { 3 | a 1 4 | b "Hiz!" 5 | } 6 | 7 | b { 8 | a 2 9 | b "Hiz!!" 10 | l 2 11 | } 12 | 13 | c { 14 | d { 15 | e "c.d.e" 16 | l 3 17 | } 18 | l 2 19 | } 20 | 21 | l 1 22 | 23 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | build: 3 | os: "ubuntu-22.04" 4 | tools: 5 | python: "3.11" 6 | sphinx: 7 | configuration: doc/conf.py 8 | python: 9 | install: 10 | - requirements: doc/requirements.txt 11 | -------------------------------------------------------------------------------- /test/wcfg-test-05.conf: -------------------------------------------------------------------------------- 1 | 2 | s01 "" 3 | s02 "this is used to test strings in props" 4 | s03 "even with embedded (escaped) stuff" 5 | s04 "\n\r\b\e\a\t\v" 6 | s05 "now test embedded hex numbers" 7 | s06 "\x65\x66\x67\X68\X69\X70" 8 | 9 | -------------------------------------------------------------------------------- /test/wcfg-test-00.conf: -------------------------------------------------------------------------------- 1 | 2 | number 123 3 | nested { 4 | num 23 5 | #octal 0664 6 | #hex 0xFF 7 | 8 | nested2 { 9 | string "this a text-only string" 10 | } 11 | 12 | string "this is a test string with embedded \nnewline" 13 | } 14 | 15 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | sudo: false 3 | python: 4 | - "2.6" 5 | - "2.7" 6 | - "3.3" 7 | - "3.4" 8 | - "pypy" 9 | install: 10 | - pip install coverage coveralls 11 | - pip install -r requirements.txt 12 | script: 13 | - coverage run --include=hipack.py setup.py test 14 | after_success: coveralls 15 | -------------------------------------------------------------------------------- /doc/apiref.rst: -------------------------------------------------------------------------------- 1 | ============= 2 | API Reference 3 | ============= 4 | 5 | :mod:`hipack` 6 | ============= 7 | 8 | .. automodule:: hipack 9 | :members: cast, dump, dumps, load, loads, value, ParseError 10 | 11 | :class:`hipack.Parser` 12 | ====================== 13 | 14 | .. autoclass:: hipack.Parser 15 | :members: 16 | -------------------------------------------------------------------------------- /test/wcfg-test-04.conf: -------------------------------------------------------------------------------- 1 | 2 | num00 0644 3 | num01 1 4 | num02 1.2 5 | num03 +1 6 | num04 -1 7 | num05 +1.2 8 | num06 -1.2 9 | num07 1e1 10 | num08 1e-1 11 | num09 1e+1 12 | num10 +1e1 13 | num11 -1e1 14 | num12 1.2e1 15 | num13 -1.2e1 16 | num14 +1.2e1 17 | num15 1e-1 18 | num16 1e+1 19 | num17 +1e+1 20 | num18 +1e-1 21 | num19 1.2E+1 22 | num20 -1.2E-3 23 | 24 | -------------------------------------------------------------------------------- /test/drove-01.conf: -------------------------------------------------------------------------------- 1 | # some comment 2 | parameter: true 3 | false_parameter: false 4 | 5 | string: "hello world" 6 | number: 1 7 | 8 | list: [ 1 2 3 ] 9 | 10 | hex: 0x10 11 | 12 | oct: 010 13 | 14 | dict { 15 | key: "value" 16 | key2: "value2" 17 | } 18 | 19 | plugin.one.value: "value1" 20 | plugin.one.othervalue: "othervalue1" 21 | 22 | plugin.two.value: "value2" 23 | plugin.two.othervalue: "othervalue2" 24 | 25 | -------------------------------------------------------------------------------- /test/omni-01.conf: -------------------------------------------------------------------------------- 1 | # User stores that can be used to authenticate services against. 2 | stores { 3 | ldap.office { 4 | url: "ldap://ldap.local.office.com" 5 | base: "dc=com,dc=office" 6 | } 7 | 8 | sqlite.issuetracker { 9 | path: "/var/lib/omni/issuetracker.sqlite" 10 | } 11 | } 12 | 13 | # Each "realm" is a set of rules which determine how to authenticate users. 14 | realms { 15 | issuetracker { 16 | # The order matters. 17 | methods: ["sqlite.issuetracker" "ldap.office"] 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /doc/index.rst: -------------------------------------------------------------------------------- 1 | .. HiPack documentation master file, created by 2 | sphinx-quickstart on Fri Nov 27 21:07:08 2015. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | HiPack's documentation 7 | ====================== 8 | 9 | Contents: 10 | 11 | .. toctree:: 12 | :maxdepth: 2 13 | 14 | quickstart 15 | advanced 16 | apiref 17 | 18 | 19 | Indices and tables 20 | ================== 21 | 22 | * :ref:`genindex` 23 | * :ref:`modindex` 24 | * :ref:`search` 25 | 26 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: test 3 | 4 | on: 5 | push: 6 | branches: 7 | - master 8 | pull_request: {} 9 | 10 | jobs: 11 | test: 12 | runs-on: ubuntu-20.04 13 | strategy: 14 | matrix: 15 | version: 16 | - { py: '3.6', tox: 'py36' } 17 | - { py: '3.x', tox: 'py3' } 18 | steps: 19 | - name: Checkout 20 | uses: actions/checkout@v4 21 | - name: Setup Python 22 | uses: actions/setup-python@v5 23 | with: 24 | python-version: "${{ matrix.version.py }}" 25 | - name: Install Tox 26 | run: pip install tox 27 | - name: Test 28 | run: tox -e ${{ matrix.version.tox }} -- -v 29 | -------------------------------------------------------------------------------- /test/hep-001.conf: -------------------------------------------------------------------------------- 1 | single-annotations { 2 | integer-value :annot 0 3 | float-value :annot 1.1 4 | string-value :annot "Hello" 5 | bool-value :annot True 6 | list-value :annot [] 7 | dict-value :annot {} 8 | } 9 | multiple-annotations { 10 | integer-value :annot1 :annot2 0 11 | float-value :annot1 :annot2 1.1 12 | string-value :annot1 :annot2 :annot3 "Three" 13 | bool-value :has :two :annots False 14 | list-value :annot1 :annot2 [] 15 | dict-value :annot1 :annot2 :annot3 :annot4 {} 16 | } 17 | annotation-delimiter-variations { 18 | double-colon-no-space::annot 0 19 | double-colon-space: :annot 0 20 | multi-double-colon-no-space::annot1:annot2 0 21 | multi-double-colon-space: :anot1:annot2 0 22 | multi-spaced-double-colon-nospace::annot1 :annot2 0 23 | multi-spaced-double-colon-space: :annot1 :annot2 0 24 | } 25 | intrinsic-annotations { 26 | integer-value .int 0 27 | float-value .float 1.1 28 | string-value .string "Hello" 29 | bool-value .bool True 30 | list-value .list [] 31 | dict-value .dict {} 32 | } 33 | list-item-annotations [ 34 | spaced-items [:annot1 "item1" :annot2 "item2"] 35 | comma-items [:annot1 "item1",:annot2 "item2"] 36 | commaspaced-items [:annot1 "item1", :annot2 "item2"] 37 | leading-space [ :annot1 "item1" :annot2 "item2"] 38 | trailing-space [:annot1 "item1" :annot2 "item2" ] 39 | leadtrail-space [ :annot1 "item1" :annot2 "item2" ] 40 | } 41 | -------------------------------------------------------------------------------- /doc/requirements.txt: -------------------------------------------------------------------------------- 1 | # This file was autogenerated by uv via the following command: 2 | # uv pip compile requirements.in 3 | alabaster==0.7.16 4 | # via sphinx 5 | babel==2.15.0 6 | # via sphinx 7 | certifi==2024.7.4 8 | # via requests 9 | charset-normalizer==3.3.2 10 | # via requests 11 | docutils==0.20.1 12 | # via 13 | # sphinx 14 | # sphinx-rtd-theme 15 | idna==3.7 16 | # via requests 17 | imagesize==1.4.1 18 | # via sphinx 19 | jinja2==3.1.4 20 | # via sphinx 21 | markupsafe==2.1.5 22 | # via jinja2 23 | packaging==24.1 24 | # via sphinx 25 | pygments==2.18.0 26 | # via sphinx 27 | readthedocs-sphinx-search==0.3.2 28 | # via -r requirements.in 29 | requests==2.32.3 30 | # via sphinx 31 | snowballstemmer==2.2.0 32 | # via sphinx 33 | sphinx==7.2.0 34 | # via 35 | # -r requirements.in 36 | # sphinx-rtd-theme 37 | # sphinxcontrib-jquery 38 | sphinx-rtd-theme==2.0.0 39 | # via -r requirements.in 40 | sphinxcontrib-applehelp==1.0.8 41 | # via sphinx 42 | sphinxcontrib-devhelp==1.0.6 43 | # via sphinx 44 | sphinxcontrib-htmlhelp==2.0.5 45 | # via sphinx 46 | sphinxcontrib-jquery==4.1 47 | # via sphinx-rtd-theme 48 | sphinxcontrib-jsmath==1.0.1 49 | # via sphinx 50 | sphinxcontrib-qthelp==1.0.7 51 | # via sphinx 52 | sphinxcontrib-serializinghtml==1.1.10 53 | # via sphinx 54 | urllib3==2.2.2 55 | # via requests 56 | -------------------------------------------------------------------------------- /test/util.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # vim:fenc=utf-8 4 | # 5 | # Copyright © 2015, 2022 Adrian Perez 6 | # 7 | # Distributed under terms of the MIT license. 8 | 9 | import re 10 | import sys 11 | 12 | 13 | def data(seq): 14 | def wrapper(f): 15 | f.test_data_iter = iter(seq) 16 | return f 17 | return wrapper 18 | 19 | _non_id_chars_re = re.compile(r"[^a-zA-Z0-9_]") 20 | _collapse_underscore_re = re.compile(r"_+") 21 | def _make_id(s): 22 | if isinstance(s, str): 23 | s = str(s.encode("ascii", "replace")) 24 | else: 25 | s = str(s) 26 | s = _non_id_chars_re.subn("_", s)[0] 27 | return _collapse_underscore_re.subn("_", s)[0] 28 | 29 | def _make_call_method(func, data_item): 30 | return lambda self: func(self, data_item) 31 | 32 | def unpack_data(cls): 33 | add_funcs = {} 34 | del_keys = set() 35 | for name, value in cls.__dict__.items(): 36 | counter = 0 37 | if name.startswith("test_") and callable(value) \ 38 | and hasattr(value, "test_data_iter"): 39 | del_keys.add(name) 40 | for data_item in value.test_data_iter: 41 | key = name + "_" + str(counter) + "_" + _make_id(data_item) 42 | add_funcs[key] = _make_call_method(value, data_item) 43 | counter += 1 44 | [setattr(cls, name, f) for name, f in add_funcs.items()] 45 | [delattr(cls, name) for name in del_keys] 46 | 47 | 48 | def also_annotations(sequence): 49 | for item in iter(sequence): 50 | yield item 51 | yield (u":ann1 " + item[0], item[1]) 52 | yield (u":ann1:ann2 " + item[0], item[1]) 53 | yield (u":ann1 :ann2 " + item[0], item[1]) 54 | yield (u":ann1:ann2:ann3 " + item[0], item[1]) 55 | yield (u":ann1 :ann2:ann3 " + item[0], item[1]) 56 | yield (u":ann1:ann2 :ann3 " + item[0], item[1]) 57 | yield (u":ann1 :ann2 :ann3 " + item[0], item[1]) 58 | -------------------------------------------------------------------------------- /examples/hipack-webservice.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | HiPack Web Service Test 6 | 17 | 52 | 53 | 54 |
55 | 56 | 61 | 62 | 63 |
64 | 65 | 66 |
67 | 68 |
69 | 70 | 71 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # vim:fenc=utf-8 4 | # 5 | # Copyright © 2014-2015, 2022 Adrian Perez 6 | # 7 | # Distributed under terms of the GPL3 license or, if that suits you 8 | # better the MIT/X11 license. 9 | 10 | from setuptools import setup 11 | from codecs import open 12 | from os import path 13 | 14 | 15 | def distrib_file(*relpath): 16 | try: 17 | return open(path.join(path.dirname(__file__), *relpath), "r", \ 18 | encoding="utf-8") 19 | except IOError: 20 | class DummyFile(object): 21 | def read(self): 22 | return "" 23 | return DummyFile() 24 | 25 | 26 | def get_version(): 27 | for line in distrib_file("hipack.py"): 28 | if line.startswith("__version__"): 29 | line = line.split() 30 | if line[0] == "__version__": 31 | return line[2] 32 | return None 33 | 34 | 35 | def get_readme(): 36 | return distrib_file("README.rst").read() 37 | 38 | 39 | setup( 40 | name="hipack", 41 | version=get_version(), 42 | description="Serialization library or the HiPack interchange format", 43 | long_description=get_readme(), 44 | author="Adrian Perez de Castro", 45 | author_email="aperez@igalia.com", 46 | url="https://github.com/aperezdc/hipack-python", 47 | py_modules=["hipack"], 48 | scripts=["hipack"], 49 | license="Dual GPL3 / MIT", 50 | classifiers=[ 51 | "Development Status :: 4 - Beta", 52 | "Intended Audience :: Developers", 53 | "Natural Language :: English", 54 | "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", 55 | "License :: OSI Approved :: MIT License", 56 | "Programming Language :: Python :: 3", 57 | "Programming Language :: Python :: 3.3", 58 | "Programming Language :: Python :: 3.4", 59 | "Programming Language :: Python :: 3.5", 60 | "Programming Language :: Python :: 3.11", 61 | "Programming Language :: Python", 62 | "Operating System :: OS Independent", 63 | ], 64 | tests_require=["tox"], 65 | ) 66 | -------------------------------------------------------------------------------- /test/test_files.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # vim:fenc=utf-8 4 | # 5 | # Copyright © 2014, 2022 Adrian Perez 6 | # 7 | # Distributed under terms of the GPL3 license or, if that suits you 8 | # better the MIT/X11 license. 9 | 10 | import unittest 11 | import hipack 12 | from os import path 13 | from os import listdir 14 | 15 | 16 | class TestConfigFiles(unittest.TestCase): 17 | def check_file(self, filepath): 18 | f = open(filepath, "rb") 19 | value = hipack.load(f) 20 | f.close() 21 | self.assertTrue(isinstance(value, dict)) 22 | 23 | @classmethod 24 | def setup_tests(cls): 25 | dirpath = path.abspath(path.dirname(__file__)) 26 | for filename in listdir(dirpath): 27 | if filename.endswith(".conf"): 28 | sanitized_name = "test_" + filename[:-5].replace("-", "_") 29 | filepath = path.join(dirpath, filename) 30 | setattr(cls, sanitized_name, 31 | lambda self: self.check_file(filepath)) 32 | 33 | heroes = ( 34 | { u"name": u"Spiderman", u"alter-ego": "Peter Parker" }, 35 | { u"name": u"Superman", u"alter-ego": "Clark Kent" }, 36 | { u"name": u"Batman", u"alter-ego": "Bruce Wayne" }, 37 | ) 38 | def test_framed_input(self): 39 | with open(path.join(path.dirname(__file__), "heroes.conf"), "rb") as f: 40 | parser = hipack.Parser(f) 41 | for hero in self.heroes: 42 | self.assertEqual(hero, parser.parse_message()) 43 | 44 | def test_framed_input_stop_on_eof(self): 45 | with open(path.join(path.dirname(__file__), "heroes.conf"), "rb") as f: 46 | parser = hipack.Parser(f) 47 | i = 0 48 | while True: 49 | hero = parser.parse_message() 50 | if hero is None: 51 | break 52 | self.assertEqual(self.heroes[i], hero) 53 | i += 1 54 | 55 | def test_framed_input_generator(self): 56 | with open(path.join(path.dirname(__file__), "heroes.conf"), "rb") as f: 57 | i = 0 58 | for hero in hipack.Parser(f).messages(): 59 | self.assertEqual(self.heroes[i], hero) 60 | i += 1 61 | 62 | TestConfigFiles.setup_tests() 63 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change Log 2 | All notable changes to this project will be documented in this file. 3 | 4 | ## [Unreleased] 5 | 6 | ## [v15] - 2024-04-30 7 | ### Changed 8 | - The `hipack-webservice` example program no longer requires `six`. 9 | 10 | ### Fixed 11 | - Fix usage of the `--to` and `--from` command line options with the 12 | `hipack` tool. 13 | - The `hipack-webservice` example program will now correctly load stats 14 | from disk during startup. 15 | 16 | ## [v14] - 2024-02-22 17 | ### Removed 18 | - Python 2.x is no longer supported. 19 | 20 | ### Fixed 21 | - Fix the build with Python 3.11 22 | 23 | ### Added 24 | - Configuration files for ReadTheDocs. 25 | 26 | ## [v13] - 2016-03-21 27 | ### Added 28 | - Allow serializing Python values of type `set` and `frozenset` as lists. 29 | 30 | ## [v12] - 2015-12-03 31 | ### Added 32 | - API reference documentation now includes the `Parser` class. 33 | - New `Parser.messages()` generator method, to easily iterate over multiple framed messages 34 | 35 | ### Fixed 36 | - Frames messages in an input stream are now handled correctly when 37 | `Parser.parse_message()` is called repeatedly. 38 | - Fixed typos in the documentation. (Patch by Óscar García Amor, .) 39 | 40 | ## [v11] - 2015-12-02 41 | ### Fixed 42 | - Fixed parsing of annotations after a dictionary key when whitespace is used 43 | as key separator. 44 | 45 | ## [v10] - 2015-11-27 46 | ### Added 47 | - Documentation (using [Sphinx](http://sphinx-doc.org/). The generated documentation is available online [at Read The Docs](http://hipack-python.readthedocs.io/en/latest/). 48 | - Support for [HEP-1: Value Annotations](https://github.com/aperezdc/hipack/blob/gh-pages/heps/hep-001.rst) 49 | 50 | ### Fixed 51 | - Hex escape sequences in string literals no longer cause an error. 52 | 53 | ## v9 - 2015-07-26 54 | - Added this changelog. 55 | 56 | [Unreleased]: https://github.com/aperezdc/hipack-python/compare/v14...HEAD 57 | [v15]: https://github.com/aperezdc/hipack-python/compare/v14...v15 58 | [v14]: https://github.com/aperezdc/hipack-python/compare/v13...v14 59 | [v13]: https://github.com/aperezdc/hipack-python/compare/v12...v13 60 | [v12]: https://github.com/aperezdc/hipack-python/compare/v11...v12 61 | [v11]: https://github.com/aperezdc/hipack-python/compare/v10...v11 62 | [v10]: https://github.com/aperezdc/hipack-python/compare/v9...v10 63 | -------------------------------------------------------------------------------- /examples/hipack-webservice: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # vim:fenc=utf-8 4 | # 5 | # Copyright © 2015 Adrian Perez 6 | # 7 | # Distributed under terms of the MIT license. 8 | 9 | import bottle 10 | import hipack 11 | import json 12 | import sys 13 | import yaml 14 | from os import path 15 | 16 | dir_path = path.dirname(__file__) 17 | stats_path = path.join(dir_path, "stats.hi") 18 | 19 | 20 | loaders = { 21 | "hipack": hipack.load, 22 | "yaml" : yaml.safe_load, 23 | "json" : json.load, 24 | } 25 | 26 | 27 | stats = { 28 | "requests": 0, 29 | "requests_failed": 0, 30 | "requests_with_nulls": 0, 31 | "cors_wrong_origin": 0, 32 | } 33 | 34 | 35 | def load_stats(): 36 | global stats 37 | try: 38 | with open(stats_path, "rb") as fd: 39 | stats = hipack.load(fd) 40 | except Exception as e: 41 | print("Could not load stats:", e, file=sys.stderr) 42 | 43 | def save_stats(): 44 | try: 45 | with open(stats_path, "wb") as fd: 46 | hipack.dump(stats, fd) 47 | except Exception as e: 48 | print("Could not save stats:", e, file=sys.stderr) 49 | 50 | 51 | @bottle.route("/") 52 | def index_html(): 53 | return bottle.static_file("hipack-webservice.html", root=dir_path) 54 | 55 | 56 | allowed_origins = ( 57 | "http://hipack.org", 58 | "http://hipack.org/", 59 | "http://www.hipack.org", 60 | "http://www.hipack.org/", 61 | ) 62 | 63 | 64 | @bottle.route("/", method="POST") 65 | def hipack_from(format): 66 | global stats 67 | stats["requests"] += 1 68 | 69 | # Handle CORS. 70 | origin = bottle.request.get_header("Origin", None) 71 | if origin is not None: 72 | if origin not in allowed_origins: 73 | stats["cors_wrong_origin"] += 1 74 | bottle.response.set_header("Access-Control-Allow-Origin", "null") 75 | bottle.abort(401, "Invalid CORS origin.") 76 | else: 77 | bottle.response.set_header("Access-Control-Allow-Origin", origin) 78 | 79 | try: 80 | compact = bool(bottle.request.query.compact or False) 81 | except ValueError: 82 | compact = False 83 | 84 | bottle.response.set_header("Content-Type", "text/plain") 85 | try: 86 | data = loaders[format](bottle.request.body) 87 | except Exception as e: 88 | bottle.response.status = 500 89 | stats["requests_failed"] += 1 90 | return e.message 91 | 92 | try: 93 | return hipack.dumps(data, not compact) 94 | except TypeError as e: 95 | if e.message == "Values of type cannot be dumped": 96 | stats["requests_with_nulls"] += 1 97 | stats["requests_failed"] += 1 98 | raise 99 | except Exception: 100 | stats["requests_failed"] += 1 101 | raise 102 | 103 | 104 | @bottle.route("/stats") 105 | def get_stats(): 106 | bottle.response.set_header("Content-Type", "text/plain") 107 | return hipack.dumps(stats) 108 | 109 | 110 | if __name__ == "__main__": 111 | import atexit 112 | atexit.register(save_stats) 113 | load_stats() 114 | bottle.run(host="localhost", port=10080) 115 | -------------------------------------------------------------------------------- /hipack: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # vim:fenc=utf-8 4 | # 5 | # Copyright © 2015, 2022 Adrian Perez 6 | # 7 | # Distributed under terms of the MIT license. 8 | 9 | import hipack 10 | import argparse 11 | import sys 12 | 13 | def import_format(formatname, modulename=None): 14 | from importlib import import_module 15 | if modulename is None: 16 | modulename = formatname 17 | try: 18 | return import_module(modulename) 19 | except ImportError: 20 | raise SystemExit("Format '" + formatname + "' requires the '" 21 | + modulename + "' module, which is not installed") 22 | 23 | def load_hipack(fd): 24 | return hipack.load(fd) 25 | 26 | def save_hipack(data, fd): 27 | hipack.dump(data, fd) 28 | 29 | def save_hipack_compact(data, fd): 30 | hipack.dump(data, fd, indent=False) 31 | 32 | def load_json(fd): 33 | return import_format("json").load(fd) 34 | 35 | def save_json(data, fd): 36 | import_format("json").dump(data, fd) 37 | 38 | def load_yaml(fd): 39 | return import_format("yaml").safe_load(fd) 40 | 41 | def save_yaml(data, fd): 42 | import_format("yaml").safe_dump(data, fd) 43 | 44 | def load_msgpack(fd): 45 | return import_format("msgpack").load(fd) 46 | 47 | def save_msgpack(data, fd): 48 | import_format("msgpack").dump(data, fd) 49 | 50 | 51 | formats = { 52 | "hipack": 53 | (load_hipack, save_hipack), 54 | "hipack-compact": 55 | (load_hipack, save_hipack_compact), 56 | "json": 57 | (load_json, save_json), 58 | "yaml": 59 | (load_yaml, save_yaml), 60 | "msgpack": 61 | (load_msgpack, save_msgpack), 62 | } 63 | 64 | 65 | parser = argparse.ArgumentParser() 66 | parser.add_argument('input', nargs='?', type=argparse.FileType('r'), 67 | help='Input file or - [default: stdin]', default=sys.stdin) 68 | parser.add_argument('output', nargs='?', type=argparse.FileType('w'), 69 | help='Output file [default: stdout]', default=sys.stdout) 70 | parser.add_argument('-f', '--from', dest='from_format', default='json', 71 | help='Read input in the specified FORMAT [default: %(default)s]', 72 | metavar='FORMAT') 73 | parser.add_argument('-t', '--to', dest='to_format', default='hipack', 74 | help='Write output in the specified FORMAT [default: %(default)s]', 75 | metavar='FORMAT') 76 | parser.add_argument('--formats', default=False, action='store_true', 77 | help='Print list of supported formats and exit') 78 | parser.add_argument('--hipack-module-version', default=False, 79 | action='store_true', dest='hipack_module_version', 80 | help="Show the version of the hipack Python module") 81 | 82 | if __name__ == "__main__": 83 | args = parser.parse_args() 84 | if args.formats: 85 | [print(name) for name in sorted(formats.keys())] 86 | elif args.hipack_module_version: 87 | print(hipack.__version__) 88 | else: 89 | try: 90 | load = formats[args.from_format][0] 91 | except KeyError: 92 | raise SystemExit("No such format: " + args.from_format) 93 | try: 94 | save = formats[args.to_format][1] 95 | except KeyError: 96 | raise SystemExit("No such format: " + args.to_format) 97 | 98 | save(load(args.input), args.output) 99 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | =============== 2 | hipack-python 3 | =============== 4 | 5 | .. image:: https://readthedocs.org/projects/hipack-python/badge/?version=latest 6 | :target: https://hipack-python.readthedocs.io/en/latest 7 | :alt: Documentation Status 8 | 9 | .. image:: https://github.com/aperezdc/hipack-python/actions/workflows/test.yml/badge.svg 10 | :target: https://github.com/aperezdc/hipack-python/actions/workflows/test.yml 11 | :alt: Build Status 12 | 13 | .. image:: https://img.shields.io/coveralls/aperezdc/hipack-python/master.svg?style=flat 14 | :target: https://coveralls.io/r/aperezdc/hipack-python?branch=master 15 | :alt: Code Coverage 16 | 17 | 18 | ``hipack`` is a Python module to work with the `HiPack `_ 19 | serialization format. The API is intentionally similar to that of the standard 20 | ``json`` and ``pickle`` modules. 21 | 22 | Features: 23 | 24 | * Both reading, and writing `HiPack version 1 25 | `__ is supported. The 26 | following extensions are implemented as well: 27 | 28 | - `HEP-1: Value Annotations 29 | `__. 30 | 31 | (Note that extensions defined in HEPs are subject to change while they are 32 | being discussed as proposals.) 33 | 34 | * Small, self-contained, pure Python implementation. 35 | 36 | * Compatible with both Python 2.6 (or newer), and 3.2 (or newer). 37 | 38 | 39 | Usage 40 | ===== 41 | 42 | Given the following input file: 43 | 44 | .. code-block:: 45 | 46 | # Configuration file for SuperFooBar v3000 47 | interface { 48 | language: "en_US" 49 | panes { 50 | top: ["menu", "toolbar"] # Optional commas in lists 51 | # The colon separating keys and values is optional 52 | bottom 53 | ["statusbar"] 54 | } 55 | ☺ : True # Enables emoji 56 | Unicode→Suþþorteð? : "Indeed, Jürgen!" 57 | } 58 | 59 | # Configure plug-ins 60 | plugin: { 61 | preview # Whitespace is mostly ignored 62 | { 63 | enabled: true 64 | timeout: 500 # Update every 500ms 65 | } 66 | } 67 | 68 | Note that the ``:`` separator in between keys and values is optional, and 69 | can be omitted. Also, notice how white space —including new lines— are 70 | completely meaningless and the structure is determined using only braces 71 | and brackets. Last but not least, a valid key is any Unicode character 72 | sequence which *does not* include white space or a colon. 73 | 74 | The following code can be used to read it into a Python dictionary: 75 | 76 | .. code-block:: python 77 | 78 | import hipack 79 | with open("superfoobar3000.conf", "rb") as f: 80 | config = hipack.load(f) 81 | 82 | Conversions work as expected: 83 | 84 | * Sections are converted into dictionaries. 85 | * Keys are converted conveted to strings. 86 | * Text in double quotes are converted to strings. 87 | * Sections enclosed into ``{ }`` are converted to dictionaries. 88 | * Arrays enclosed into ``[ ]`` are converted to lists. 89 | * Numbers are converted either to ``int`` or ``float``, whichever is more 90 | appropriate. 91 | * Boolean values are converted to ``bool``. 92 | 93 | The following can be used to convert a Python dictionary into its textual 94 | representation: 95 | 96 | .. code-block:: python 97 | 98 | users = { 99 | "peter": { 100 | "uid": 1000, 101 | "name": "Peter Jøglund", 102 | "groups": ["wheel", "peter"], 103 | }, 104 | "root": { 105 | "uid": 0, 106 | "groups": ["root"], 107 | } 108 | } 109 | 110 | import hipack 111 | text = hipack.dumps(users) 112 | 113 | When generating a textual representation, the keys of each dictionary will 114 | be sorted, to guarantee that the generated output is stable. The dictionary 115 | from the previous snippet would be written in text form as follows: 116 | 117 | .. code-block:: 118 | 119 | peter: { 120 | name: "Peter Jøglund" 121 | groups: ["wheel" "peter"] 122 | uid: 1000 123 | } 124 | root: { 125 | groups: ["root"] 126 | uid: 0 127 | } 128 | 129 | 130 | Installation 131 | ============ 132 | 133 | The stable releases are uploaded to `PyPI `_, so you 134 | can install them and upgrade using ``pip``:: 135 | 136 | pip install hipack 137 | 138 | Alternatively, you can install development versions —at your own risk— 139 | directly from the Git repository:: 140 | 141 | pip install -e git://github.com/aperezdc/hipack-python 142 | 143 | 144 | Development 145 | =========== 146 | 147 | If you want to contribute, please use the usual GitHub workflow: 148 | 149 | 1. Clone the repository. 150 | 2. Hack on your clone. 151 | 3. Send a pull request for review. 152 | 153 | If you do not have programming skills, you can still contribute by `reporting 154 | issues `_ that you may 155 | encounter. 156 | 157 | 158 | -------------------------------------------------------------------------------- /doc/quickstart.rst: -------------------------------------------------------------------------------- 1 | ========== 2 | Quickstart 3 | ========== 4 | 5 | This guide will walk you through the usage of the Python :mod:`hipack` module. 6 | 7 | 8 | Basic Usage 9 | =========== 10 | 11 | The main use cases of the :mod:`hipack` module are to read and write messages 12 | in `HiPack format `__, and convert between the text-based 13 | representation used in HiPack and the corresponding Python types. 14 | 15 | The module defines functions to read (and write) HiPack-formatted messages 16 | from strings and files: 17 | 18 | * :func:`hipack.load()` parses a message from a file-like object. 19 | * :func:`hipack.loads()` does the same, using a string as input. 20 | * :func:`hipack.dump()` writes a message to a file-like object. 21 | * :func:`hipack.dumps()` writes a message and returns it as a string. 22 | 23 | Loading (deserialization) 24 | ------------------------- 25 | 26 | Let's start parsing some data from a string which contains a HiPack message: 27 | 28 | >>> text = """\ 29 | ... title: "Quickstart" 30 | ... is-documentation? True 31 | ... """ 32 | >>> import hipack 33 | >>> value = hipack.loads(text) 34 | 35 | HiPack messages are converted to dicionaries when parsing them, so now ``value`` 36 | contains a Python dictionary, and their values properly converted: 37 | 38 | >>> isinstance(value, dict) 39 | True 40 | >>> value[u"title"], value[u"is-documentation?"] 41 | (u'Quickstart", True) 42 | 43 | 44 | Dumping (serialization) 45 | ----------------------- 46 | 47 | The inverse operation, converting a Python dictionary to a HiPack message, 48 | works in the expected way: 49 | 50 | >>> print(hipack.dumps(value)) 51 | is-documentation?: True 52 | title: "Quickstart"' 53 | 54 | Note how the output is nicely formatted. While that is desirable for human 55 | consumption, some applications may want to make messages as small as possible, 56 | and the module supports this by providing a setting to disable indentation and 57 | unneeded spacing. Let's try now passing ``indent=False``: 58 | 59 | >>> print(hipack.dumps(value, indent=False)) 60 | is-documentation?:False title:"Quickstart" 61 | 62 | 63 | Framed messages 64 | =============== 65 | 66 | Reading 67 | ------- 68 | 69 | Using :meth:`hipack.Parser.parse_message()` to parse one message at a time, 70 | it is possible to provide multiple messages chained, enclosed into braces, 71 | in the same input. For example, consider the following input file, named 72 | ``heroes.hipack``:: 73 | 74 | { name: "Spiderman", alter-ego: "Peter Parker" } 75 | { name: "Superman", alter-ego: "Clark Kent" } 76 | { name: "Batman", alter-ego: "Bruce Wayne" } 77 | 78 | The following loop will iterate over the file, parsing and converting the 79 | HiPack messages containing information about superheroes, one at a time, 80 | and printing only their names (but not who is the person behind the mask): 81 | 82 | >>> with open("heroes.hipack", "r") as stream: 83 | ... parser = hipack.Parser(stream) 84 | ... while True: 85 | ... hero = parser.parse_message() 86 | ... if not hero: 87 | ... break 88 | ... print(hero[u"name"]) 89 | ... 90 | Spiderman 91 | Superman 92 | Batman 93 | 94 | Alternatively, if the input stream is known to have a fixed size (e.g. it is 95 | a plain file and not a socket or a pipe in asynchronous mode), the generator 96 | :meth:`hipack.Parser.messages()` can be used instead: 97 | 98 | >>> with open("heroes.hipack", "r") as stream: 99 | ... parser = hipack.Parser(stream) 100 | ... names = [hero[u"name"] for hero in parser.messages()] 101 | ... 102 | >>> names 103 | [u'Spiderman', u'Superman', u'Batman'] 104 | 105 | 106 | Writing 107 | ------- 108 | 109 | At the moment, there is no support to write messages *and* automatically add 110 | the frame markers automatically in the module. Nevertheless, it is trivial to 111 | write a loop which calls :func:`hipack.dump()` repeatedly and adds the braces 112 | enclosing each message: 113 | 114 | >>> philosophers = ( 115 | ... {"name": "Karl Marx", "book": "The Capital"}, 116 | ... {"name": "Nietzsche", "book": "Thus Spoke Zarathustra"}, 117 | ... {"name": "Wittgenstein", "book": "Tractatus"}, 118 | ... ) 119 | >>> with open("philosophers.hipack", "w") as stream: 120 | ... for item in philosophers: 121 | ... stream.write("{\n") 122 | ... hipack.dump(item, stream, indent=1) 123 | ... stream.write("}\n") 124 | ... 125 | 126 | Note how we pass ``indent=1`` above to indicate the initial level of 127 | indentation, which gets us a nicely formatted ``philosophers.hipack`` file 128 | with the following contents:: 129 | 130 | { 131 | book: "The Capital" 132 | name: "Karl Marx" 133 | } 134 | { 135 | book: "Thus Spoke Zarathustra" 136 | name: "Nietzsche" 137 | } 138 | { 139 | book: "Tractatus" 140 | name: "Wittgenstein" 141 | } 142 | -------------------------------------------------------------------------------- /doc/advanced.rst: -------------------------------------------------------------------------------- 1 | ============== 2 | Advanced Usage 3 | ============== 4 | 5 | Annotations 6 | =========== 7 | 8 | Annotations are supported as defined in `HEP-1: Value Annotations 9 | `__. By 10 | default, the parser accepts and validate annotations, but they are *not* 11 | passed to the user. Conversely, dumping values does *not* write any 12 | annotations by default. In order to make use of parsed annotations a 13 | “cast callback” must be provided, and to write annotations a “value 14 | callback”. 15 | 16 | Cast Callbacks 17 | -------------- 18 | 19 | A “cast callback” is any callable object which will be called by the parser 20 | every time it has ended parsing a value, and accepts the following arguments: 21 | 22 | * A set of annotations. 23 | * The text representation of a parsed value. 24 | * The parsed value, converted to irs corresponding Python type. 25 | 26 | The function must return a value, which can be either the same provided by the 27 | parser or any other value. The result returned from :func:`hipack.load()` (or 28 | :func:`hipack.loads()` contains the values returned by the cast function 29 | *instead of the ones originally seen by the parser*. The main use case for 30 | a cast callback is to convert HiPack values on-the-fly into objects of the 31 | application domain, optinally making use of annotations to help the cast 32 | function determine which kind of conversion to perform. 33 | 34 | As en example, consider a simple contacts application which stores the 35 | information of each contact in a file like the following:: 36 | 37 | name "Peter" 38 | surname "Parker" 39 | instant-messaging [ 40 | :xmpp "spiderman@marvel.com" 41 | :skype "spideysenses" 42 | ] 43 | ... 44 | 45 | Note how the items inside the ``instant-messaging`` list are annotated with 46 | the kind of instant messaging service they correspond to. When parsing a 47 | file like this, we probably want to use the following classes to represent the 48 | data above: 49 | 50 | .. code-block:: python 51 | 52 | class Contact(object): 53 | def __init__(self, name, surname=u"", im=None): 54 | self.name = name 55 | self.surname = surname 56 | self.im = [] if im is None else im 57 | 58 | class IMAccount(object): 59 | kind = None 60 | def __init__(self, address): 61 | self.address = address 62 | 63 | class XMPPAccount(IMAccount): 64 | kind = "XMPP" 65 | 66 | class SkypeAccount(IMAccount): 67 | kind = "Skype" 68 | 69 | Now, with those classes in place, we can write our cast function as follows: 70 | 71 | .. code-block:: python 72 | 73 | def contacts_cast(annotations, stringvalue, value): 74 | if u"xmpp" in annotations: 75 | return XMPPAccount(value) 76 | elif u"skype" in annotations: 77 | return SkypeAccount(value) 78 | else: 79 | return value 80 | 81 | Finally, we can load a contact file as follows: 82 | 83 | >>> with open("contact.hipack", "r") as f: 84 | ... contact = Contact(**hipack.load(f, cast=contacts_cast)) 85 | ... 86 | >>> isinstance(contact, Contact) 87 | True 88 | >>> isinstance(contact.im[0], XMPPAccount) 89 | True 90 | >>> contact.name, contact.im[0].address 91 | (u'Peter', u'spiderman@marvel.com') 92 | 93 | Note that cast callbacks receive a set containing *all* the annotations 94 | attached to a value, *including intrinsic implicit annotations*. This means 95 | that every time the callback is invoked, there will be at least always the 96 | intrinsic annotation which informs of the type of the value (``.int``, 97 | ``.float``, ``.list``, etc). 98 | 99 | 100 | Value Callbacks 101 | --------------- 102 | 103 | A “value callback” performs the opposite operation to `cast callbacks`_: it is 104 | called when before serializing a value into its HiPack representation to give 105 | the application an opportunity to convert arbitrary Python objects, and attach 106 | annotations to the serialized value. Value callbacks must accept a Python 107 | object as its first argument, and return two values: 108 | 109 | * A basic value for which HiPack specifies a representation. 110 | * An iterable which yields the annotations to attach to the value, or ``None`` 111 | if the value has no annotations associated to it. 112 | 113 | Continuing with the contacts example above, we can define a value callback 114 | like the following to allow direct serialization of ``IMAccount`` objects: 115 | 116 | .. code-block:: python 117 | 118 | def contact_value(obj): 119 | if isinstance(obj, IMAccount): 120 | return obj.address, (obj.kind.lower(),) 121 | else: 122 | return obj, None 123 | 124 | Value callbacks are used in a way similar to cast callbacks, passing them to 125 | the :func:`hipack.dump()` function. For example: 126 | 127 | >>> print(hipack.dumps({ 128 | ... "work-im": XMPPAccount("spiderman@marvel.com"), 129 | ... "home-im": SkypeAccount("spideysenses"), 130 | ... }, value=contact_value) 131 | ... 132 | home-im::skype "spideysenses" 133 | work-im::xmpp "spiderman@marvel.com" 134 | 135 | -------------------------------------------------------------------------------- /test/test_cast.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # vim:fenc=utf-8 4 | # 5 | # Copyright © 2015, 2022 Adrian Perez 6 | # 7 | # Distributed under terms of the MIT license. 8 | 9 | from test.util import * 10 | import unittest 11 | import hipack 12 | from textwrap import dedent 13 | from io import BytesIO 14 | 15 | 16 | basic_values = ( 17 | (u"0", hipack.ANNOT_INT), 18 | (u"1.1", hipack.ANNOT_FLOAT), 19 | (u"True", hipack.ANNOT_BOOL), 20 | (u"\"\"", hipack.ANNOT_STRING), 21 | (u"[]", hipack.ANNOT_LIST), 22 | (u"{}", hipack.ANNOT_DICT), 23 | ) 24 | annot_values = ( 25 | (u"0", (".int",)), 26 | (u":one 0", (".int", "one")), 27 | (u":one:two 0", (".int", "one", "two")), 28 | (u":one :two 0", (".int", "one", "two")), 29 | (u":☺ 0", (".int", u"☺")), # Unicode 30 | (u":;%&@ 0", (".int", ";%&@")), # ASCII symbols 31 | ) 32 | 33 | 34 | class TestCast(unittest.TestCase): 35 | 36 | @staticmethod 37 | def parser(string, cast): 38 | return hipack.Parser(BytesIO(string.encode("utf-8")), cast) 39 | 40 | @data(also_annotations(basic_values)) 41 | def test_intrinsic_annots(self, data): 42 | value, intrinsic_annot = data 43 | def check_cast(annotations, bytestring, value): 44 | self.assertIn(intrinsic_annot, annotations) 45 | return value 46 | value = self.parser(value, check_cast).parse_value() 47 | self.assertIsNotNone(value) 48 | 49 | @data(annot_values) 50 | def test_annots(self, data): 51 | value, expected = data[0], frozenset(data[1]) 52 | def check_cast(annotations, bytestring, value): 53 | self.assertEqual(expected, annotations) 54 | return value 55 | value = self.parser(value, check_cast).parse_value() 56 | self.assertIsNotNone(value) 57 | 58 | person_input = dedent("""\ 59 | :person { 60 | name: "Peter" 61 | surname: "Parker" 62 | } 63 | """) 64 | def test_create_obj(self): 65 | class Person(object): 66 | def __init__(self, name, surname): 67 | self.name = name 68 | self.surname = surname 69 | def obj_cast(annotations, bytestring, value): 70 | if u"person" in annotations: 71 | return Person(**value) 72 | else: 73 | return value 74 | value = self.parser(self.person_input, obj_cast).parse_value() 75 | self.assertIsInstance(value, Person) 76 | self.assertEqual(value.name, u"Peter") 77 | self.assertEqual(value.surname, u"Parker") 78 | 79 | http_config = dedent("""\ 80 | myserver::http { 81 | port 80 82 | cgi-bin::alias { 83 | path: "/srv/cgi-scripts/" 84 | } 85 | icons::alias { 86 | path: "/usr/share/icons" 87 | } 88 | } 89 | """) 90 | def test_create_obj_nested(self): 91 | class Record(object): 92 | def __init__(self, **kw): 93 | self.__dict__.update(kw) 94 | class HttpServer(Record): pass 95 | class HttpAlias(Record): pass 96 | def obj_cast(annotations, bytestring, value): 97 | if u"alias" in annotations: 98 | return HttpAlias(**value) 99 | if u"http" in annotations: 100 | return HttpServer(**value) 101 | return value 102 | value = self.parser(self.http_config, obj_cast).parse_message() 103 | self.assertIsInstance(value[u"myserver"], HttpServer) 104 | self.assertIsInstance(value[u"myserver"].icons, HttpAlias) 105 | self.assertEqual(80, value[u"myserver"].port) 106 | 107 | unpack_data(TestCast) 108 | 109 | 110 | class TestValue(unittest.TestCase): 111 | 112 | @data(( 113 | ((), "x: 0"), 114 | (("a",), "x::a 0"), 115 | (("a", "b"), "x::a:b 0"), 116 | (("b", "a"), "x::b:a 0"), # Order matters 117 | )) 118 | def test_dump_annot(self, data): 119 | annotations, expected = data 120 | text = hipack.dumps({"x": 0}, value=lambda x: (x, annotations)) 121 | self.assertEqual(expected + "\n", text.decode("utf-8")) 122 | 123 | @data(( 124 | "col:on", "com,ma", 125 | "sp ace", "new\nline", 126 | "carriage\rreturn", "\tab", 127 | "lbr[acket", "rbr]acket", 128 | "l{brace", "r}brace", 129 | )) 130 | def test_invalid_annots(self, annot): 131 | with self.assertRaises(ValueError): 132 | text = hipack.dumps({"x": 0}, value=lambda x: (x, (annot,))) 133 | 134 | def test_serialize_object(self): 135 | class Person(object): 136 | def __init__(self, name, surname): 137 | self.name = name 138 | self.surname = surname 139 | def as_dict(self): 140 | return { "name": self.name, "surname": self.surname } 141 | 142 | def obj_value(obj): 143 | if isinstance(obj, Person): 144 | return obj.as_dict(), ("person",) 145 | return obj, None 146 | 147 | text = hipack.dumps({"spiderman": Person("Peter", "Parker")}, 148 | value=obj_value) 149 | self.assertEqual(dedent(u"""\ 150 | spiderman::person { 151 | name: "Peter" 152 | surname: "Parker" 153 | } 154 | """), text.decode("utf-8")) 155 | 156 | def test_serialize_object_nested(self): 157 | class Person(object): 158 | def __init__(self, name, surname): 159 | self.name = name 160 | self.surname = surname 161 | def as_dict(self): 162 | return { "name": self.name, "surname": self.surname } 163 | class Hero(object): 164 | def __init__(self, nick, alterego): 165 | self.nick = nick 166 | self.alterego = alterego 167 | def as_dict(self): 168 | return { "nick": self.nick, "alter-ego": self.alterego } 169 | 170 | def obj_value(obj): 171 | if isinstance(obj, (Person, Hero)): 172 | return obj.as_dict(), (obj.__class__.__name__,) 173 | return obj, None 174 | 175 | h = Hero("Spider-Man", Person("Peter", "Parker")) 176 | expected = dedent(u"""\ 177 | item::Hero { 178 | alter-ego::Person { 179 | name: "Peter" 180 | surname: "Parker" 181 | } 182 | nick: "Spider-Man" 183 | } 184 | """) 185 | text = hipack.dumps({"item": h}, value=obj_value) 186 | self.assertEqual(expected, text.decode("utf-8")) 187 | 188 | def test_serialize_duplicate_annotation(self): 189 | with self.assertRaises(ValueError): 190 | hipack.dumps({"x":0}, value=lambda x: (x, ("a", "a"))) 191 | 192 | unpack_data(TestValue) 193 | -------------------------------------------------------------------------------- /doc/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=_build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 10 | set I18NSPHINXOPTS=%SPHINXOPTS% . 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 31 | echo. text to make text files 32 | echo. man to make manual pages 33 | echo. texinfo to make Texinfo files 34 | echo. gettext to make PO message catalogs 35 | echo. changes to make an overview over all changed/added/deprecated items 36 | echo. xml to make Docutils-native XML files 37 | echo. pseudoxml to make pseudoxml-XML files for display purposes 38 | echo. linkcheck to check all external links for integrity 39 | echo. doctest to run all doctests embedded in the documentation if enabled 40 | echo. coverage to run coverage check of the documentation if enabled 41 | goto end 42 | ) 43 | 44 | if "%1" == "clean" ( 45 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 46 | del /q /s %BUILDDIR%\* 47 | goto end 48 | ) 49 | 50 | 51 | REM Check if sphinx-build is available and fallback to Python version if any 52 | %SPHINXBUILD% 2> nul 53 | if errorlevel 9009 goto sphinx_python 54 | goto sphinx_ok 55 | 56 | :sphinx_python 57 | 58 | set SPHINXBUILD=python -m sphinx.__init__ 59 | %SPHINXBUILD% 2> nul 60 | if errorlevel 9009 ( 61 | echo. 62 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 63 | echo.installed, then set the SPHINXBUILD environment variable to point 64 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 65 | echo.may add the Sphinx directory to PATH. 66 | echo. 67 | echo.If you don't have Sphinx installed, grab it from 68 | echo.http://sphinx-doc.org/ 69 | exit /b 1 70 | ) 71 | 72 | :sphinx_ok 73 | 74 | 75 | if "%1" == "html" ( 76 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 77 | if errorlevel 1 exit /b 1 78 | echo. 79 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 80 | goto end 81 | ) 82 | 83 | if "%1" == "dirhtml" ( 84 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 85 | if errorlevel 1 exit /b 1 86 | echo. 87 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 88 | goto end 89 | ) 90 | 91 | if "%1" == "singlehtml" ( 92 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 93 | if errorlevel 1 exit /b 1 94 | echo. 95 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 96 | goto end 97 | ) 98 | 99 | if "%1" == "pickle" ( 100 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 101 | if errorlevel 1 exit /b 1 102 | echo. 103 | echo.Build finished; now you can process the pickle files. 104 | goto end 105 | ) 106 | 107 | if "%1" == "json" ( 108 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 109 | if errorlevel 1 exit /b 1 110 | echo. 111 | echo.Build finished; now you can process the JSON files. 112 | goto end 113 | ) 114 | 115 | if "%1" == "htmlhelp" ( 116 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 117 | if errorlevel 1 exit /b 1 118 | echo. 119 | echo.Build finished; now you can run HTML Help Workshop with the ^ 120 | .hhp project file in %BUILDDIR%/htmlhelp. 121 | goto end 122 | ) 123 | 124 | if "%1" == "qthelp" ( 125 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 126 | if errorlevel 1 exit /b 1 127 | echo. 128 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 129 | .qhcp project file in %BUILDDIR%/qthelp, like this: 130 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\HiPack.qhcp 131 | echo.To view the help file: 132 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\HiPack.ghc 133 | goto end 134 | ) 135 | 136 | if "%1" == "devhelp" ( 137 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 138 | if errorlevel 1 exit /b 1 139 | echo. 140 | echo.Build finished. 141 | goto end 142 | ) 143 | 144 | if "%1" == "epub" ( 145 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 146 | if errorlevel 1 exit /b 1 147 | echo. 148 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 149 | goto end 150 | ) 151 | 152 | if "%1" == "latex" ( 153 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 154 | if errorlevel 1 exit /b 1 155 | echo. 156 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 157 | goto end 158 | ) 159 | 160 | if "%1" == "latexpdf" ( 161 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 162 | cd %BUILDDIR%/latex 163 | make all-pdf 164 | cd %~dp0 165 | echo. 166 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 167 | goto end 168 | ) 169 | 170 | if "%1" == "latexpdfja" ( 171 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 172 | cd %BUILDDIR%/latex 173 | make all-pdf-ja 174 | cd %~dp0 175 | echo. 176 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 177 | goto end 178 | ) 179 | 180 | if "%1" == "text" ( 181 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 182 | if errorlevel 1 exit /b 1 183 | echo. 184 | echo.Build finished. The text files are in %BUILDDIR%/text. 185 | goto end 186 | ) 187 | 188 | if "%1" == "man" ( 189 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 190 | if errorlevel 1 exit /b 1 191 | echo. 192 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 193 | goto end 194 | ) 195 | 196 | if "%1" == "texinfo" ( 197 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 198 | if errorlevel 1 exit /b 1 199 | echo. 200 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 201 | goto end 202 | ) 203 | 204 | if "%1" == "gettext" ( 205 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 206 | if errorlevel 1 exit /b 1 207 | echo. 208 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 209 | goto end 210 | ) 211 | 212 | if "%1" == "changes" ( 213 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 214 | if errorlevel 1 exit /b 1 215 | echo. 216 | echo.The overview file is in %BUILDDIR%/changes. 217 | goto end 218 | ) 219 | 220 | if "%1" == "linkcheck" ( 221 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 222 | if errorlevel 1 exit /b 1 223 | echo. 224 | echo.Link check complete; look for any errors in the above output ^ 225 | or in %BUILDDIR%/linkcheck/output.txt. 226 | goto end 227 | ) 228 | 229 | if "%1" == "doctest" ( 230 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 231 | if errorlevel 1 exit /b 1 232 | echo. 233 | echo.Testing of doctests in the sources finished, look at the ^ 234 | results in %BUILDDIR%/doctest/output.txt. 235 | goto end 236 | ) 237 | 238 | if "%1" == "coverage" ( 239 | %SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage 240 | if errorlevel 1 exit /b 1 241 | echo. 242 | echo.Testing of coverage in the sources finished, look at the ^ 243 | results in %BUILDDIR%/coverage/python.txt. 244 | goto end 245 | ) 246 | 247 | if "%1" == "xml" ( 248 | %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml 249 | if errorlevel 1 exit /b 1 250 | echo. 251 | echo.Build finished. The XML files are in %BUILDDIR%/xml. 252 | goto end 253 | ) 254 | 255 | if "%1" == "pseudoxml" ( 256 | %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml 257 | if errorlevel 1 exit /b 1 258 | echo. 259 | echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. 260 | goto end 261 | ) 262 | 263 | :end 264 | -------------------------------------------------------------------------------- /doc/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " applehelp to make an Apple Help Book" 34 | @echo " devhelp to make HTML files and a Devhelp project" 35 | @echo " epub to make an epub" 36 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 37 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 38 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 39 | @echo " text to make text files" 40 | @echo " man to make manual pages" 41 | @echo " texinfo to make Texinfo files" 42 | @echo " info to make Texinfo files and run them through makeinfo" 43 | @echo " gettext to make PO message catalogs" 44 | @echo " changes to make an overview of all changed/added/deprecated items" 45 | @echo " xml to make Docutils-native XML files" 46 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 47 | @echo " linkcheck to check all external links for integrity" 48 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 49 | @echo " coverage to run coverage check of the documentation (if enabled)" 50 | 51 | clean: 52 | rm -rf $(BUILDDIR)/* 53 | 54 | html: 55 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 56 | @echo 57 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 58 | 59 | dirhtml: 60 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 61 | @echo 62 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 63 | 64 | singlehtml: 65 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 66 | @echo 67 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 68 | 69 | pickle: 70 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 71 | @echo 72 | @echo "Build finished; now you can process the pickle files." 73 | 74 | json: 75 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 76 | @echo 77 | @echo "Build finished; now you can process the JSON files." 78 | 79 | htmlhelp: 80 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 81 | @echo 82 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 83 | ".hhp project file in $(BUILDDIR)/htmlhelp." 84 | 85 | qthelp: 86 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 87 | @echo 88 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 89 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 90 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/HiPack.qhcp" 91 | @echo "To view the help file:" 92 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/HiPack.qhc" 93 | 94 | applehelp: 95 | $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp 96 | @echo 97 | @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." 98 | @echo "N.B. You won't be able to view it unless you put it in" \ 99 | "~/Library/Documentation/Help or install it in your application" \ 100 | "bundle." 101 | 102 | devhelp: 103 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 104 | @echo 105 | @echo "Build finished." 106 | @echo "To view the help file:" 107 | @echo "# mkdir -p $$HOME/.local/share/devhelp/HiPack" 108 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/HiPack" 109 | @echo "# devhelp" 110 | 111 | epub: 112 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 113 | @echo 114 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 115 | 116 | latex: 117 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 118 | @echo 119 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 120 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 121 | "(use \`make latexpdf' here to do that automatically)." 122 | 123 | latexpdf: 124 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 125 | @echo "Running LaTeX files through pdflatex..." 126 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 127 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 128 | 129 | latexpdfja: 130 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 131 | @echo "Running LaTeX files through platex and dvipdfmx..." 132 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 133 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 134 | 135 | text: 136 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 137 | @echo 138 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 139 | 140 | man: 141 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 142 | @echo 143 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 144 | 145 | texinfo: 146 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 147 | @echo 148 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 149 | @echo "Run \`make' in that directory to run these through makeinfo" \ 150 | "(use \`make info' here to do that automatically)." 151 | 152 | info: 153 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 154 | @echo "Running Texinfo files through makeinfo..." 155 | make -C $(BUILDDIR)/texinfo info 156 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 157 | 158 | gettext: 159 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 160 | @echo 161 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 162 | 163 | changes: 164 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 165 | @echo 166 | @echo "The overview file is in $(BUILDDIR)/changes." 167 | 168 | linkcheck: 169 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 170 | @echo 171 | @echo "Link check complete; look for any errors in the above output " \ 172 | "or in $(BUILDDIR)/linkcheck/output.txt." 173 | 174 | doctest: 175 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 176 | @echo "Testing of doctests in the sources finished, look at the " \ 177 | "results in $(BUILDDIR)/doctest/output.txt." 178 | 179 | coverage: 180 | $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage 181 | @echo "Testing of coverage in the sources finished, look at the " \ 182 | "results in $(BUILDDIR)/coverage/python.txt." 183 | 184 | xml: 185 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 186 | @echo 187 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 188 | 189 | pseudoxml: 190 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 191 | @echo 192 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 193 | -------------------------------------------------------------------------------- /doc/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # HiPack documentation build configuration file, created by 5 | # sphinx-quickstart on Fri Nov 27 21:07:08 2015. 6 | # 7 | # This file is execfile()d with the current directory set to its 8 | # containing dir. 9 | # 10 | # Note that not all possible configuration values are present in this 11 | # autogenerated file. 12 | # 13 | # All configuration values have a default; values that are commented out 14 | # serve to show the default. 15 | 16 | import sys 17 | import os 18 | 19 | # If extensions (or modules to document with autodoc) are in another directory, 20 | # add these directories to sys.path here. If the directory is relative to the 21 | # documentation root, use os.path.abspath to make it absolute, like shown here. 22 | sys.path.insert(0, os.path.abspath(os.path.dirname(os.path.dirname(__file__)))) 23 | 24 | # -- General configuration ------------------------------------------------ 25 | 26 | # If your documentation needs a minimal Sphinx version, state it here. 27 | #needs_sphinx = '1.0' 28 | 29 | # Add any Sphinx extension module names here, as strings. They can be 30 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 31 | # ones. 32 | extensions = [ 33 | 'sphinx.ext.autodoc', 34 | 'sphinx.ext.coverage', 35 | ] 36 | 37 | # Add any paths that contain templates here, relative to this directory. 38 | templates_path = ['_templates'] 39 | 40 | # The suffix(es) of source filenames. 41 | # You can specify multiple suffix as a list of string: 42 | # source_suffix = ['.rst', '.md'] 43 | source_suffix = '.rst' 44 | 45 | # The encoding of source files. 46 | #source_encoding = 'utf-8-sig' 47 | 48 | # The master toctree document. 49 | master_doc = 'index' 50 | 51 | # General information about the project. 52 | project = 'HiPack' 53 | copyright = '2015, Adrian Perez de Castro' 54 | author = 'Adrian Perez de Castro' 55 | 56 | # The version info for the project you're documenting, acts as replacement for 57 | # |version| and |release|, also used in various other places throughout the 58 | # built documents. 59 | # 60 | # The short X.Y version. 61 | 62 | from hipack import __version__ as hipack_version 63 | 64 | version = str(hipack_version) 65 | # The full version, including alpha/beta/rc tags. 66 | release = version 67 | 68 | # The language for content autogenerated by Sphinx. Refer to documentation 69 | # for a list of supported languages. 70 | # 71 | # This is also used if you do content translation via gettext catalogs. 72 | # Usually you set "language" from the command line for these cases. 73 | language = 'en' 74 | 75 | # There are two options for replacing |today|: either, you set today to some 76 | # non-false value, then it is used: 77 | #today = '' 78 | # Else, today_fmt is used as the format for a strftime call. 79 | #today_fmt = '%B %d, %Y' 80 | 81 | # List of patterns, relative to source directory, that match files and 82 | # directories to ignore when looking for source files. 83 | exclude_patterns = ['_build'] 84 | 85 | # The reST default role (used for this markup: `text`) to use for all 86 | # documents. 87 | #default_role = None 88 | 89 | # If true, '()' will be appended to :func: etc. cross-reference text. 90 | #add_function_parentheses = True 91 | 92 | # If true, the current module name will be prepended to all description 93 | # unit titles (such as .. function::). 94 | #add_module_names = True 95 | 96 | # If true, sectionauthor and moduleauthor directives will be shown in the 97 | # output. They are ignored by default. 98 | #show_authors = False 99 | 100 | # The name of the Pygments (syntax highlighting) style to use. 101 | pygments_style = 'sphinx' 102 | 103 | # A list of ignored prefixes for module index sorting. 104 | #modindex_common_prefix = [] 105 | 106 | # If true, keep warnings as "system message" paragraphs in the built documents. 107 | #keep_warnings = False 108 | 109 | # If true, `todo` and `todoList` produce output, else they produce nothing. 110 | todo_include_todos = False 111 | 112 | 113 | # -- Options for HTML output ---------------------------------------------- 114 | 115 | # The theme to use for HTML and HTML Help pages. See the documentation for 116 | # a list of builtin themes. 117 | html_theme = 'alabaster' 118 | 119 | # Theme options are theme-specific and customize the look and feel of a theme 120 | # further. For a list of options available for each theme, see the 121 | # documentation. 122 | #html_theme_options = {} 123 | 124 | # Add any paths that contain custom themes here, relative to this directory. 125 | #html_theme_path = [] 126 | 127 | # The name for this set of Sphinx documents. If None, it defaults to 128 | # " v documentation". 129 | #html_title = None 130 | 131 | # A shorter title for the navigation bar. Default is the same as html_title. 132 | #html_short_title = None 133 | 134 | # The name of an image file (relative to this directory) to place at the top 135 | # of the sidebar. 136 | #html_logo = None 137 | 138 | # The name of an image file (within the static path) to use as favicon of the 139 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 140 | # pixels large. 141 | #html_favicon = None 142 | 143 | # Add any paths that contain custom static files (such as style sheets) here, 144 | # relative to this directory. They are copied after the builtin static files, 145 | # so a file named "default.css" will overwrite the builtin "default.css". 146 | html_static_path = [] 147 | 148 | # Add any extra paths that contain custom files (such as robots.txt or 149 | # .htaccess) here, relative to this directory. These files are copied 150 | # directly to the root of the documentation. 151 | #html_extra_path = [] 152 | 153 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 154 | # using the given strftime format. 155 | #html_last_updated_fmt = '%b %d, %Y' 156 | 157 | # If true, SmartyPants will be used to convert quotes and dashes to 158 | # typographically correct entities. 159 | #html_use_smartypants = True 160 | 161 | # Custom sidebar templates, maps document names to template names. 162 | #html_sidebars = {} 163 | 164 | # Additional templates that should be rendered to pages, maps page names to 165 | # template names. 166 | #html_additional_pages = {} 167 | 168 | # If false, no module index is generated. 169 | #html_domain_indices = True 170 | 171 | # If false, no index is generated. 172 | #html_use_index = True 173 | 174 | # If true, the index is split into individual pages for each letter. 175 | #html_split_index = False 176 | 177 | # If true, links to the reST sources are added to the pages. 178 | #html_show_sourcelink = True 179 | 180 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 181 | #html_show_sphinx = True 182 | 183 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 184 | #html_show_copyright = True 185 | 186 | # If true, an OpenSearch description file will be output, and all pages will 187 | # contain a tag referring to it. The value of this option must be the 188 | # base URL from which the finished HTML is served. 189 | #html_use_opensearch = '' 190 | 191 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 192 | #html_file_suffix = None 193 | 194 | # Language to be used for generating the HTML full-text search index. 195 | # Sphinx supports the following languages: 196 | # 'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja' 197 | # 'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr' 198 | #html_search_language = 'en' 199 | 200 | # A dictionary with options for the search language support, empty by default. 201 | # Now only 'ja' uses this config value 202 | #html_search_options = {'type': 'default'} 203 | 204 | # The name of a javascript file (relative to the configuration directory) that 205 | # implements a search results scorer. If empty, the default will be used. 206 | #html_search_scorer = 'scorer.js' 207 | 208 | # Output file base name for HTML help builder. 209 | htmlhelp_basename = 'HiPackdoc' 210 | 211 | # -- Options for LaTeX output --------------------------------------------- 212 | 213 | latex_elements = { 214 | # The paper size ('letterpaper' or 'a4paper'). 215 | #'papersize': 'letterpaper', 216 | 217 | # The font size ('10pt', '11pt' or '12pt'). 218 | #'pointsize': '10pt', 219 | 220 | # Additional stuff for the LaTeX preamble. 221 | #'preamble': '', 222 | 223 | # Latex figure (float) alignment 224 | #'figure_align': 'htbp', 225 | } 226 | 227 | # Grouping the document tree into LaTeX files. List of tuples 228 | # (source start file, target name, title, 229 | # author, documentclass [howto, manual, or own class]). 230 | latex_documents = [ 231 | (master_doc, 'HiPack.tex', 'HiPack Documentation', 232 | 'Adrián Pérez de Castro', 'manual'), 233 | ] 234 | 235 | # The name of an image file (relative to this directory) to place at the top of 236 | # the title page. 237 | #latex_logo = None 238 | 239 | # For "manual" documents, if this is true, then toplevel headings are parts, 240 | # not chapters. 241 | #latex_use_parts = False 242 | 243 | # If true, show page references after internal links. 244 | #latex_show_pagerefs = False 245 | 246 | # If true, show URL addresses after external links. 247 | #latex_show_urls = False 248 | 249 | # Documents to append as an appendix to all manuals. 250 | #latex_appendices = [] 251 | 252 | # If false, no module index is generated. 253 | #latex_domain_indices = True 254 | 255 | 256 | # -- Options for manual page output --------------------------------------- 257 | 258 | # One entry per manual page. List of tuples 259 | # (source start file, name, description, authors, manual section). 260 | man_pages = [ 261 | (master_doc, 'hipack', 'HiPack Documentation', 262 | [author], 1) 263 | ] 264 | 265 | # If true, show URL addresses after external links. 266 | #man_show_urls = False 267 | 268 | 269 | # -- Options for Texinfo output ------------------------------------------- 270 | 271 | # Grouping the document tree into Texinfo files. List of tuples 272 | # (source start file, target name, title, author, 273 | # dir menu entry, description, category) 274 | texinfo_documents = [ 275 | (master_doc, 'HiPack', 'HiPack Documentation', 276 | author, 'HiPack', 'One line description of project.', 277 | 'Miscellaneous'), 278 | ] 279 | 280 | # Documents to append as an appendix to all manuals. 281 | #texinfo_appendices = [] 282 | 283 | # If false, no module index is generated. 284 | #texinfo_domain_indices = True 285 | 286 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 287 | #texinfo_show_urls = 'footnote' 288 | 289 | # If true, do not generate a @detailmenu in the "Top" node's menu. 290 | #texinfo_no_detailmenu = False 291 | -------------------------------------------------------------------------------- /test/test_parser.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # vim:fenc=utf-8 4 | # 5 | # Copyright © 2014, 2022 Adrian Perez 6 | # 7 | # Distributed under terms of the GPL3 license or, if that suits you 8 | # better the MIT/X11 license. 9 | 10 | from test.util import * 11 | import unittest 12 | import hipack 13 | from textwrap import dedent 14 | from io import BytesIO 15 | 16 | 17 | def make_tuples(sequence): 18 | for item in iter(sequence): 19 | if isinstance(item, tuple): 20 | yield item 21 | else: 22 | yield (item, item) 23 | 24 | def wrap_strings(sequence): 25 | return ((u"\"" + x + u"\"", y) for (x, y) in iter(sequence)) 26 | 27 | 28 | class TestParser(unittest.TestCase): 29 | 30 | @staticmethod 31 | def parser(string): 32 | return hipack.Parser(BytesIO(string.encode("utf-8"))) 33 | 34 | def test_parse_valid_booleans(self): 35 | booleans = ( 36 | (u"True", True), 37 | (u"true", True), 38 | (u"False", False), 39 | (u"false", False), 40 | ) 41 | for item, expected in also_annotations(booleans): 42 | value = self.parser(item).parse_value() 43 | self.assertEqual(expected, value) 44 | self.assertTrue(isinstance(value, bool)) 45 | 46 | def check_numbers(self, numbers, type_): 47 | for item, result in also_annotations(numbers): 48 | value = self.parser(item).parse_value() 49 | self.assertEqual(result, value) 50 | self.assertTrue(isinstance(value, type_)) 51 | # Trailing whitespace must not alter the result. 52 | value = self.parser(item + " ").parse_value() 53 | self.assertEqual(result, value) 54 | self.assertTrue(isinstance(value, type_)) 55 | 56 | def test_parse_valid_integer_numbers(self): 57 | numbers = ( 58 | (u"1", 1), 59 | (u"+1", 1), 60 | (u"-2", -2), 61 | (u"123", 123), 62 | (u"+123", 123), 63 | (u"-123", -123), 64 | ) 65 | self.check_numbers(numbers, int) 66 | 67 | def test_parse_valid_hex_numbers(self): 68 | numbers = ( 69 | (u"0x1", 1), 70 | (u"0X1", 1), 71 | (u"-0x3", -0x3), 72 | (u"0xcafe", 0xCAFE), 73 | (u"0XCAFE", 0xCAFE), 74 | (u"0xCAFE", 0xCAFE), 75 | (u"0xCaFe", 0xCAFE), 76 | (u"0x1234", 0x1234), 77 | (u"0xC00FFEE", 0xC00FFEE), 78 | (u"0xdeadbeef", 0xdeadbeef), 79 | ) 80 | self.check_numbers(numbers, int) 81 | 82 | def test_parse_valid_octal_numbers(self): 83 | numbers = ( 84 | (u"01", 0o1), 85 | (u"01234567", 0o1234567), 86 | (u"042", 0o42), 87 | (u"-032", -0o32), 88 | ) 89 | self.check_numbers(numbers, int) 90 | 91 | def test_parse_valid_float_numbers(self): 92 | numbers = ( 93 | (u"1.", 1.0), 94 | (u".5", 0.5), 95 | (u"0.1", 0.1), 96 | (u"1.5", 1.5), 97 | (u"1e3", 1e3), 98 | (u"1e-3", 1e-3), 99 | (u"1.e3", 1e3), 100 | (u"1.e-3", 1e-3), 101 | (u".5e3", 0.5e3), 102 | (u".5e-3", 0.5e-3), 103 | (u"1.5e3", 1.5e3), 104 | (u"1.5e-3", 1.5e-3), 105 | ) 106 | def gen_signs(): 107 | for item, value in numbers: 108 | yield (item, value) 109 | yield (u"+" + item, value) 110 | yield (u"-" + item, -value) 111 | 112 | self.check_numbers(gen_signs(), float) 113 | 114 | def test_parse_valid_keys(self): 115 | keys = ( 116 | # Typical definition of identifiers. 117 | u"foo", 118 | u"ident-with-dashes", 119 | u"-leading-dash", 120 | u"trailing-dash-", 121 | u"ident_with_underscores", 122 | u"_leading_underscore", 123 | u"trailing_underscore_", 124 | u"ident.with.dots", 125 | # Unicode identifiers must work. 126 | u"☺", 127 | u"武術", 128 | u"空手", 129 | u"→", 130 | # Mixed. 131 | u"arrow→", 132 | u"Trømso", 133 | u"Güedángaños", 134 | ) 135 | for item in keys: 136 | key = self.parser(item).parse_key() 137 | self.assertEqual(item, key) 138 | self.assertTrue(isinstance(key, str)) 139 | 140 | def check_strings(self, strings, type_): 141 | for item, expected in also_annotations(wrap_strings(make_tuples(strings))): 142 | value = self.parser(item).parse_value() 143 | self.assertEqual(expected, value) 144 | self.assertTrue(isinstance(value, type_)) 145 | 146 | def test_parse_valid_strings(self): 147 | strings = ( 148 | u"", 149 | u"this is a string", 150 | u" another with leading space", 151 | u"yet one more with trailing space ", 152 | u"unicode: this → that, Trømso, Java™, ☺", 153 | (u"numeric: \\65\\5d\\5F", u"numeric: e]_"), 154 | (u"new\\nline", u"new\nline"), 155 | (u"horizontal\\tab", u"horizontal\tab"), 156 | (u"carriage\\return", u"carriage\return"), 157 | (u"escaped backslash: \\\\", u"escaped backslash: \\"), 158 | (u"escaped double quote: \\\"", u"escaped double quote: \""), 159 | ) 160 | self.check_strings(strings, str) 161 | 162 | def test_parse_valid_arrays(self): 163 | arrays = ( 164 | (u"[]", []), 165 | (u"[ ]", []), 166 | (u"[1]", [1]), 167 | (u"[ 1]", [1]), 168 | (u"[1 ]", [1]), 169 | (u"[ 1 ]", [1]), 170 | (u"[1 2]", [1, 2]), 171 | (u"[ 1 2]", [1, 2]), 172 | (u"[1 2 ]", [1, 2]), 173 | (u"[ 1 2 ]", [1, 2]), 174 | ) 175 | for item, expected in also_annotations(arrays): 176 | value = self.parser(item).parse_value() 177 | self.assertTrue(isinstance(value, list)) 178 | self.assertListEqual(expected, value) 179 | # Replacing the spaces with newlines should work as well. 180 | value = self.parser(item.replace(" ", "\n")).parse_value() 181 | self.assertTrue(isinstance(value, list)) 182 | self.assertListEqual(expected, value) 183 | 184 | def test_parse_valid_arrays_with_commas(self): 185 | arrays = ( 186 | (u"[1,]", [1]), # Dangling commas are allowed. 187 | (u"[1,2]", [1, 2]), # Spaces around commas are optional. 188 | (u"[1, 2]", [1, 2]), # Space can be provided, of course... 189 | (u"[1 , 2]", [1, 2]), # ...even on both sides 190 | (u"[1\n,2]", [1, 2]), # Newlines and commas. 191 | (u"[1,\n2]", [1, 2]), # Ditto. 192 | (u"[1\t,2]", [1, 2]), # Tabs around commas. 193 | (u"[1,2,]", [1, 2]), # More than one item and dangling comma. 194 | (u"[1, 2, ]", [1, 2]), # Spaces after dangling comma. 195 | (u"[1 2,3]", [1,2,3]), # Mixed spaces and commas. 196 | ) 197 | for item, expected in also_annotations(arrays): 198 | value = self.parser(item).parse_value() 199 | self.assertTrue(isinstance(value, list)) 200 | self.assertListEqual(expected, value) 201 | 202 | def test_parse_invalid_arrays(self): 203 | invalid_arrays = ( 204 | u"[", # Unterminated array. 205 | u"[1", # Ditto. 206 | u"[1 2", # Ditto. 207 | u"[[]", # Unterminated inner array. 208 | u"(1)", # Not an array. 209 | u"[:]", # Invalid value inside array. 210 | u"[\"]", # Unterminated string inside array. 211 | u"[\"]\"", # Unbalanced double-quote and brackets. 212 | ) 213 | for item, _ in also_annotations(make_tuples(invalid_arrays)): 214 | with self.assertRaises(hipack.ParseError): 215 | self.parser(item).parse_value() 216 | 217 | @data(also_annotations(( 218 | u"{ 0 }", 219 | u"{ foo: }", 220 | u"{ foo:foo }", 221 | u"{foo:0]", u"{ foo: 0 ]", 222 | u"{a{b{c}}}", 223 | u"{a,}", 224 | u"{a()}", 225 | ))) 226 | def test_parse_invalid_dict(self, text): 227 | with self.assertRaises(hipack.ParseError): 228 | self.parser(text[0]).parse_value() 229 | 230 | @data(also_annotations(( 231 | (u"{}", {}), 232 | (u"{a:1}", {"a":1}), 233 | (u"{a:1,b:2}", {"a":1, "b":2}), 234 | (u"{a{b{c{}}}}", {"a":{"b":{"c":{}}}}), 235 | ))) 236 | def test_parse_valid_dict(self, data): 237 | text, expected = data 238 | value = self.parser(text).parse_value() 239 | self.assertEqual(expected, value) 240 | 241 | @data(( 242 | u",", u"{", u"}", u"[", u"]", u"{]", u"[]", u"[}", 243 | u"{ a: 1 ]", u"{ a { foo: 1, ], ]", u"{a:1,,}", 244 | )) 245 | def test_parse_invalid_message(self, text): 246 | with self.assertRaises(hipack.ParseError): 247 | self.parser(text).parse_message() 248 | 249 | @data(( 250 | u"value :annot 0", 251 | u"value :annot 1.1", 252 | u"value :annot True", 253 | u"value :annot []", 254 | u"value :annot {}", 255 | )) 256 | def test_parse_valid_message_with_annots(self, text): 257 | def check_annot(annotations, text, value): 258 | self.assertIn(u"annot", annotations) 259 | return value 260 | value = hipack.loads(text.encode("utf-8"), check_annot) 261 | self.assertIsInstance(value, dict) 262 | 263 | def test_parse_invalid_arrays_with_commas(self): 264 | invalid_arrays = ( 265 | u"[,]", # Array with holes. 266 | u"[ ,]", # Ditto. 267 | u"[, ]", # Ditto. 268 | u"[,,]", # Multiple holes. 269 | u"[\v,\t]", # Ditto. 270 | u"[,1]", # Leading comma. 271 | u"[1,,]", # Double trailing comma. 272 | ) 273 | for item, _ in also_annotations(make_tuples(invalid_arrays)): 274 | with self.assertRaises(hipack.ParseError): 275 | self.parser(item).parse_value() 276 | 277 | def test_parse_invalid_numbers(self): 278 | invalid_numbers = ( 279 | u"", u"-", u"+", u"-+", u"a", u"☺", u"-.", u".", u"e", u".e", 280 | u"+e", u"-e", u"-.e", u"+.e", u"e+", u"e-", u".-e", u".+e", 281 | u"--", u"++", u"+1e3.", u"..1", u"1.2.", u"1..2", u"\"foo\"", 282 | u"True", u"False", u"{}", u"[]", u"()", u"0xx00", "0.1AeA3", 283 | u"ee", u"1ee", u"1e1e1", u"0.1x2", u"1x.0", u"01.0", u"01e1", 284 | u"0x10.20", 285 | ) 286 | for item, _ in also_annotations(make_tuples(invalid_numbers)): 287 | with self.assertRaises(hipack.ParseError): 288 | parser = self.parser(item) 289 | parser.parse_annotations() 290 | parser.parse_number(set()) 291 | 292 | def test_parse_invalid_booleans(self): 293 | invalid_booleans = ( 294 | u"Tr", u"TRUE", u"TrUe", u"TruE", u"TrUE", 295 | u"Fa", u"FALSE", u"FaLSE", u"FaLsE", u"FalsE", 296 | u"1", u"0", u"\"True\"", u"\"False\"", 297 | ) 298 | for item, _ in also_annotations(make_tuples(invalid_booleans)): 299 | with self.assertRaises(hipack.ParseError): 300 | parser = self.parser(item) 301 | parser.parse_annotations() 302 | parser.parse_bool(set()) 303 | 304 | def test_parse_invalid_strings(self): 305 | invalid_strings = ( 306 | u"\"", # Unterminated. 307 | u"\"a", # Ditto. 308 | u"\"\\\"", # Ditto. 309 | u"\"\\gg\"", # On-hex escape sequence. 310 | ) 311 | for item, _ in also_annotations(make_tuples(invalid_strings)): 312 | with self.assertRaises(hipack.ParseError): 313 | parser = self.parser(item) 314 | parser.parse_annotations() 315 | parser.parse_string(set()) 316 | 317 | def test_parse_invalid_annotations(self): 318 | invalid_annotations = ( 319 | u":", u"::", u": :", u":a :", ":a:", u":a::", 320 | u":[", u":]", u":,", u":{", u":}", u".:foo", 321 | u":::", u":a::", u":a:b:", 322 | u":a :a", # Duplicate annotation 323 | ) 324 | for item in invalid_annotations: 325 | with self.assertRaises(hipack.ParseError): 326 | self.parser(item + u" 0").parse_value() 327 | 328 | def test_textwrap(self): 329 | from io import TextIOWrapper 330 | stream = BytesIO() 331 | wrapper = TextIOWrapper(stream) 332 | hipack.dump({"a": True}, wrapper) 333 | self.assertEqual(b"a: True\n", stream.getvalue()) 334 | 335 | 336 | class TestDump(unittest.TestCase): 337 | 338 | @staticmethod 339 | def dump_value(value): 340 | stream = BytesIO() 341 | hipack._dump_value(value, stream, 0, hipack.value) 342 | return stream.getvalue() 343 | 344 | def test_dump_values(self): 345 | values = ( 346 | (True, b"True"), 347 | (False, b"False"), 348 | (123, b"123"), 349 | (-123, b"-123"), 350 | (0xFF, b"255"), # An hex literal in Python is still an integer. 351 | (0o5, b"5"), # Ditto for octals in Python. 352 | (0.5, b"0.5"), 353 | (-0.5, b"-0.5"), 354 | (b"byte string", b'"byte string"'), 355 | (u"a string", b'"a string"'), 356 | (u"double quote: \"", b'"double quote: \\""'), 357 | ((1, 2, 3), b"[\n 1\n 2\n 3\n]"), 358 | ([1, 2, 3], b"[\n 1\n 2\n 3\n]"), 359 | ({"a": 1, "b": 2}, b"{\n a: 1\n b: 2\n}"), 360 | ) 361 | for value, expected in values: 362 | result = self.dump_value(value) 363 | self.assertEqual(expected, result) 364 | self.assertTrue(isinstance(result, bytes)) 365 | 366 | def test_invalid_key_types(self): 367 | invalid_keys = ( 368 | 42, 3.14, # Numeric. 369 | object(), # Objects. 370 | object, # Classes. 371 | True, False, # Booleans. 372 | [], (), {}, # Collections. 373 | ) 374 | for key in invalid_keys: 375 | with self.assertRaises(TypeError): 376 | hipack.dumps({ key: True }) 377 | 378 | def test_invalid_key_values(self): 379 | invalid_keys = ( 380 | "col:on", # Colons are not allowed. 381 | "sp ace", # Ditto for spaces. 382 | ) 383 | for key in invalid_keys: 384 | with self.assertRaises(ValueError): 385 | hipack.dumps({ key: True }) 386 | 387 | def test_dump_non_dict(self): 388 | invalid_values = ( 389 | "string", 390 | 42, 3.14, # Numbers. 391 | object(), # Objects. 392 | object, # Classes. 393 | True, False, # Booleans. 394 | [], (), # Non-dict collections. 395 | ) 396 | for value in invalid_values: 397 | with self.assertRaises(TypeError): 398 | hipack.dumps(value) 399 | 400 | def test_invalid_values(self): 401 | invalid_values = ( 402 | object(), # Objects. 403 | object, # Classes. 404 | ) 405 | for value in invalid_values: 406 | with self.assertRaises(TypeError): 407 | hipack.dumps({ "value": value }) 408 | 409 | unpack_data(TestParser) 410 | 411 | 412 | class TestAPI(unittest.TestCase): 413 | 414 | TEST_VALUES = ( 415 | ({}, "", ""), 416 | ({"a": 1, "b": 2}, 417 | "a:1 b:2 ", 418 | """\ 419 | a: 1 420 | b: 2 421 | """), 422 | ({"a": {"b": {}}}, 423 | "a:{b:{} } ", 424 | """\ 425 | a: { 426 | b: { 427 | } 428 | } 429 | """), 430 | ({"a": {"t": True}, "nums": [1e4, -2, 0xA], "f": False}, 431 | "a:{t:True } f:False nums:[10000.0,-2,10,] ", 432 | """\ 433 | a: { 434 | t: True 435 | } 436 | f: False 437 | nums: [ 438 | 10000.0 439 | -2 440 | 10 441 | ] 442 | """), 443 | ({"a": [{"b": 1}, [1, 2, 3]]}, 444 | "a:[{b:1 },[1,2,3,],] ", 445 | """\ 446 | a: [ 447 | { 448 | b: 1 449 | } 450 | [ 451 | 1 452 | 2 453 | 3 454 | ] 455 | ] 456 | """), 457 | ) 458 | 459 | TEST_VALUES_LOADS_ONLY = ( 460 | ({}, "# This only has a comment"), 461 | ({"list": [1, 2, 3]}, "list:[1 2 3]"), 462 | ({"list": [1, 2, 3]}, "list: [ 1 2 3 ]"), 463 | ({"list": [1, 2, 3]}, """\ 464 | # Leading comment 465 | list: [1 2 3] # Inline comment 466 | # Trailing comment 467 | """), 468 | ({"list": [1, 2, 3]}, """\ 469 | # This skips the optional colon 470 | list [ 471 | 1 2 3 472 | ]"""), 473 | ) 474 | 475 | def get_dumps_test_values(self): 476 | for item in self.TEST_VALUES: 477 | yield item 478 | 479 | def get_loads_test_values(self): 480 | for a, b, c in self.TEST_VALUES: 481 | yield a, b 482 | yield a, c 483 | for item in self.TEST_VALUES_LOADS_ONLY: 484 | yield item 485 | 486 | def test_dumps(self): 487 | for value, expected_noindent, expected \ 488 | in self.get_dumps_test_values(): 489 | result = hipack.dumps(value) 490 | expected = dedent(expected).encode("utf-8") 491 | self.assertEqual(expected, result) 492 | self.assertTrue(isinstance(result, bytes)) 493 | expected_noindent = dedent(expected_noindent).encode("utf-8") 494 | result = hipack.dumps(value, False) 495 | self.assertEqual(expected_noindent, result) 496 | self.assertTrue(isinstance(result, bytes)) 497 | 498 | def test_loads(self): 499 | for expected, value in self.get_loads_test_values(): 500 | result = hipack.loads(dedent(value).encode("utf-8")) 501 | self.assertTrue(isinstance(result, dict)) 502 | self.assertDictEqual(expected, result) 503 | # Passing Unicode text should work as well. 504 | result = hipack.loads(dedent(value)) 505 | self.assertTrue(isinstance(result, dict)) 506 | self.assertDictEqual(expected, result) 507 | -------------------------------------------------------------------------------- /hipack.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # vim:fenc=utf-8 4 | # 5 | # Copyright © 2014-2015, 2022 Adrian Perez 6 | # 7 | # Distributed under terms of the GPL3 license or, if that suits you 8 | # better the MIT/X11 license. 9 | 10 | """This module provide a pythonic way to handle HiPack messages. 11 | """ 12 | 13 | __version__ = 15 14 | __heps__ = (1,) 15 | 16 | import string 17 | from io import BytesIO, TextIOWrapper 18 | 19 | _SPACE = b" " 20 | _COMMA = b"," 21 | _COLON = b":" 22 | _LBRACE = b"{" 23 | _RBRACE = b"}" 24 | _TAB = b"\t" 25 | _RETURN = b"\r" 26 | _NEWLINE = b"\n" 27 | _LBRACKET = b"[" 28 | _RBRACKET = b"]" 29 | _DQUOTE = b"\"" 30 | _OCTOTHORPE = b"#" 31 | _SLASHDQUOTE = b"\\\"" 32 | _BACKSLASH = b"\\" 33 | _NUMBER_SIGNS = b"+-" 34 | _ZERO = b"0" 35 | _HEX_X = b"xX" 36 | _NUMBER_EXP = b"eE" 37 | _DOT = b"." 38 | _EOF = b"" 39 | _BOOL_LEADERS = b"tTfF" 40 | _TRUE_T = b"tT" 41 | _TRUE_RUE = (b"r", b"u", b"e") 42 | _FALSE_F = b"fF" 43 | _FALSE_ALSE = (b"a", b"l", b"s", b"e") 44 | _TRUE = b"True" 45 | _FALSE = b"False" 46 | _CHAR_n = b"n" 47 | _CHAR_r = b"r" 48 | _CHAR_t = b"t" 49 | 50 | 51 | whitespaces = string.whitespace.encode("ascii") 52 | _HEX_CHARS = b"abcdefABCDEF" 53 | _HEX_DIGITS = b"0123456789" + _HEX_CHARS 54 | _OCTAL_NONZERO_DIGITS = b"1234567" 55 | _NUMBER_CHARS = string.digits.encode("ascii") + \ 56 | _HEX_CHARS + _DOT + _NUMBER_EXP + _NUMBER_SIGNS + _HEX_X 57 | 58 | 59 | # Those are defined according to the spec. 60 | _WHITESPACE = b"\t\n\r " 61 | _NON_KEY_CHARS = _WHITESPACE + b"[]{}:," 62 | 63 | 64 | # Intrinsic type annotations 65 | ANNOT_INT = ".int" 66 | ANNOT_FLOAT = ".float" 67 | ANNOT_BOOL = ".bool" 68 | ANNOT_STRING = ".string" 69 | ANNOT_LIST = ".list" 70 | ANNOT_DICT = ".dict" 71 | 72 | 73 | def _is_hipack_key_character(ch): 74 | return ch not in _NON_KEY_CHARS 75 | 76 | 77 | def _is_hipack_whitespace(ch): 78 | return ch in _WHITESPACE 79 | 80 | 81 | def _dump_value(obj, stream, indent, value): 82 | if isinstance(obj, float): 83 | stream.write(str(obj).encode("ascii")) 84 | elif isinstance(obj, bool): 85 | stream.write(_TRUE if obj else _FALSE) 86 | elif isinstance(obj, int): 87 | stream.write(str(obj).encode("ascii")) 88 | elif isinstance(obj, str): 89 | stream.write(_DQUOTE) 90 | stream.write(obj.encode("utf-8").replace(_DQUOTE, _SLASHDQUOTE)) 91 | stream.write(_DQUOTE) 92 | elif isinstance(obj, str) or isinstance(obj, bytes): 93 | stream.write(_DQUOTE) 94 | stream.write(obj.replace(_DQUOTE, _SLASHDQUOTE)) 95 | stream.write(_DQUOTE) 96 | elif isinstance(obj, (tuple, list, set, frozenset)): 97 | stream.write(_LBRACKET) 98 | for item in obj: 99 | if indent >= 0: 100 | stream.write(_NEWLINE) 101 | stream.write(_SPACE * ((indent + 1) * 2)) 102 | _dump_value(item, stream, indent + 1, value) 103 | else: 104 | _dump_value(item, stream, indent, value) 105 | stream.write(_COMMA) 106 | if indent >= 0: 107 | stream.write(_NEWLINE) 108 | stream.write(_SPACE * (indent * 2)) 109 | stream.write(_RBRACKET) 110 | elif isinstance(obj, dict): 111 | stream.write(_LBRACE) 112 | if indent >= 0: 113 | stream.write(_NEWLINE) 114 | _dump_dict(obj, stream, indent + 1, value) 115 | stream.write(_SPACE * (indent * 2)) 116 | else: 117 | _dump_dict(obj, stream, indent, value) 118 | stream.write(_RBRACE) 119 | else: 120 | raise TypeError("Values of type " + str(type(obj)) + 121 | " cannot be dumped") 122 | 123 | 124 | def _check_key(k, thing="Key"): 125 | if isinstance(k, str): 126 | k = k.encode("utf-8") 127 | else: 128 | raise TypeError(thing + " is not a string: " + repr(k)) 129 | if _COLON in k: 130 | raise ValueError(thing + " contains a colon: " + repr(k)) 131 | if _COMMA in k: 132 | raise ValueError(thing + " contains a comma: " + repr(k)) 133 | if _SPACE in k or _NEWLINE in k or _RETURN in k or _TAB in k: 134 | raise ValueError(thing + " contains whitespace: " + repr(k)) 135 | if _LBRACE in k or _RBRACE in k: 136 | raise ValueError(thing + " contains a brace: " + repr(k)) 137 | if _LBRACKET in k or _RBRACKET in k: 138 | raise ValueError(thing + " contains a bracket: " + repr(k)) 139 | return k 140 | 141 | 142 | def _dump_dict(obj, stream, indent, value): 143 | # Dictionaries are always dumped with their keys sorted, 144 | # in order to produce a predictable output. 145 | for k in sorted(obj.keys()): 146 | v, annotations = value(obj[k]) 147 | k = _check_key(k) 148 | stream.write(_SPACE * (indent * 2)) 149 | stream.write(k) 150 | stream.write(_COLON) 151 | if annotations is not None and len(annotations) > 0: 152 | seen = set() 153 | for annot in iter(annotations): 154 | annot = _check_key(annot, "Annotation") 155 | if annot in seen: 156 | raise ValueError("Duplicated annotation: " + repr(annot)) 157 | seen.add(annot) 158 | stream.write(_COLON) 159 | stream.write(annot) 160 | stream.write(_SPACE) 161 | elif indent >= 0: 162 | stream.write(_SPACE) 163 | _dump_value(v, stream, indent, value) 164 | stream.write(_NEWLINE if indent >= 0 else _SPACE) 165 | 166 | 167 | def value(obj): 168 | """ 169 | Default “value” function. 170 | """ 171 | return obj, None 172 | 173 | 174 | def dump(obj, stream, indent=True, value=value): 175 | """ 176 | Writes Python objects to a writable stream as a HiPack message. 177 | 178 | :param obj: 179 | Object to be serialized and written. 180 | :param stream: 181 | A file-like object with a `.write(b)` method. 182 | :param bool indent: 183 | Whether to pretty-print and indent the written message, instead 184 | of writing the whole message in single line. (Default: `False`). 185 | :param callable value: 186 | Function called before writing each value, which can perform 187 | additional conversions to support direct serialization of values other 188 | than those supported by HiPack. The function is passed a Python 189 | object, and it must return an object that can be represented as a 190 | HiPack value. 191 | """ 192 | assert callable(value) 193 | obj, annotations = value(obj) 194 | if not isinstance(obj, dict): 195 | raise TypeError("Dictionary value expected") 196 | 197 | flush_after = False 198 | if isinstance(stream, TextIOWrapper): 199 | stream.flush() # Make sure there are no buffered leftovers 200 | stream = stream.buffer 201 | flush_after = True 202 | 203 | _dump_dict(obj, stream, 0 if indent else -1, value) 204 | 205 | if flush_after: 206 | stream.flush() 207 | 208 | 209 | def dumps(obj, indent=True, value=value): 210 | """ 211 | Serializes a Python object into a string in HiPack format. 212 | 213 | :param obj: 214 | Object to be serialized and written. 215 | :param bool indent: 216 | Whether to pretty-print and indent the written message, instead 217 | of writing the whole message in single line. (Default: `False`). 218 | :param callable value: 219 | A Python object conversion function, see :func:`dump()` for details. 220 | """ 221 | output = BytesIO() 222 | dump(obj, output, indent, value) 223 | return output.getvalue() 224 | 225 | 226 | class ParseError(ValueError): 227 | """ 228 | Use to signal an error when parsing a HiPack message. 229 | 230 | :attribute line: 231 | Input line where the error occurred. 232 | :attribute column: 233 | Input column where the error occured. 234 | :attribute message: 235 | Textual description of the error. 236 | """ 237 | 238 | def __init__(self, line, column, message): 239 | super(ParseError, self).__init__(str(line) + ":" + str(column) + 240 | ": " + message) 241 | self.line = line 242 | self.column = column 243 | self.message = message 244 | 245 | 246 | def cast(annotations, bytestring, value): 247 | """ 248 | Default “cast” function. 249 | """ 250 | return value 251 | 252 | 253 | class Parser(object): 254 | """ 255 | Parses HiPack messages and converts them to Python objects. 256 | 257 | :param stream: 258 | A file-like object with a `.read(n)` method. 259 | 260 | :param callable cast: 261 | Function called after each value has been parsed successfully, which 262 | can perform additional conversions on the data. The `cast` function 263 | is passed the set of annotations associated with the value, the 264 | `bytes` representation before converting a simple literal value (that 265 | is, all except lists and dictionaries, for which `None` is passed 266 | instead), and the converted value. 267 | """ 268 | 269 | def __init__(self, stream, cast=cast): 270 | assert callable(cast) 271 | self.cast = cast 272 | self.look = None 273 | self.line = 1 274 | self.column = 0 275 | self.stream = stream 276 | self.nextchar() 277 | self.skip_whitespace() 278 | self.framed = (self.look == _LBRACE) 279 | 280 | def error(self, message): 281 | raise ParseError(self.line, self.column, message) 282 | 283 | def _basic_match(self, char, expected_message): 284 | if self.look != char: 285 | if expected_message is None: # pragma: no cover 286 | expected_message = "character '" + str(char) + "'" 287 | self.error("Unexpected input '" + str(self.look) + "', " + 288 | str(expected_message) + " was expected") 289 | self.nextchar() 290 | 291 | def match(self, char, expected_message=None): 292 | self._basic_match(char, expected_message) 293 | 294 | def match_sequence(self, chars, expected_message=None): 295 | for char in chars: 296 | self._basic_match(char, expected_message) 297 | 298 | def getchar(self): 299 | ch = self.stream.read(1) 300 | if ch == _EOF: 301 | return _EOF 302 | elif ch == _NEWLINE: 303 | self.column = 0 304 | self.line += 1 305 | self.column += 1 306 | return ch 307 | 308 | def nextchar(self): 309 | self.look = _OCTOTHORPE # XXX Enter the loop at least once. 310 | while self.look == _OCTOTHORPE: 311 | self.look = self.getchar() 312 | if self.look == _OCTOTHORPE: 313 | while self.look not in (_EOF, _NEWLINE): 314 | self.look = self.getchar() 315 | 316 | def skip_whitespace(self): 317 | while self.look != _EOF and _is_hipack_whitespace(self.look): 318 | self.nextchar() 319 | 320 | def parse_key(self): 321 | key = BytesIO() 322 | while self.look != _EOF and _is_hipack_key_character(self.look): 323 | key.write(self.look) 324 | self.nextchar() 325 | key = key.getvalue().decode("utf-8") 326 | if len(key) == 0: 327 | self.error("key expected") 328 | return key 329 | 330 | def parse_bool(self, annotations): 331 | s = self.look 332 | if self.look in _TRUE_T: 333 | remaining = _TRUE_RUE 334 | ret = True 335 | elif self.look in _FALSE_F: 336 | remaining = _FALSE_ALSE 337 | ret = False 338 | else: 339 | self.error("True or False expected for boolean") 340 | self.nextchar() 341 | self.match_sequence(remaining, _TRUE if ret else _FALSE) 342 | annotations.add(ANNOT_BOOL) 343 | return self.cast(frozenset(annotations), s.decode("utf-8"), ret) 344 | 345 | def parse_string(self, annotations): 346 | value = BytesIO() 347 | self.match(_DQUOTE) 348 | value.write(_DQUOTE) 349 | 350 | while self.look != _EOF and self.look != _DQUOTE: 351 | if self.look == _BACKSLASH: 352 | self.look = self.getchar() 353 | if self.look in (_DQUOTE, _BACKSLASH): 354 | pass 355 | elif self.look == _CHAR_n: 356 | self.look = _NEWLINE 357 | elif self.look == _CHAR_r: 358 | self.look = _RETURN 359 | elif self.look == _CHAR_t: 360 | self.look = _TAB 361 | else: 362 | extra = self.getchar() 363 | if extra not in _HEX_DIGITS or \ 364 | self.look not in _HEX_DIGITS: 365 | self.error("invalid escape sequence") 366 | self.look = (chr(16 * int(self.look, 16) + int(extra, 16))).encode("ascii") 367 | 368 | value.write(self.look) 369 | self.look = self.getchar() 370 | self.match(_DQUOTE) 371 | value.write(_DQUOTE) 372 | 373 | annotations.add(ANNOT_STRING) 374 | value = value.getvalue() 375 | return self.cast(frozenset(annotations), value, value[1:-1].decode("utf-8")) 376 | 377 | def parse_number(self, annotations): 378 | number = BytesIO() 379 | 380 | # Optional sign. 381 | if self.look in _NUMBER_SIGNS: 382 | number.write(self.look) 383 | self.nextchar() 384 | 385 | # Detect octal and hexadecimal numbers. 386 | is_hex = False 387 | is_octal = False 388 | if self.look == _ZERO: 389 | number.write(self.look) 390 | self.nextchar() 391 | if self.look in _HEX_X: 392 | is_hex = True 393 | number.write(self.look) 394 | self.nextchar() 395 | elif self.look in _OCTAL_NONZERO_DIGITS: 396 | is_octal = True 397 | 398 | # Read the rest of the number. 399 | dot_seen = False 400 | exp_seen = False 401 | while self.look != _EOF and self.look in _NUMBER_CHARS: 402 | if self.look in _NUMBER_EXP and not is_hex: 403 | if exp_seen: 404 | self.error("Malformed number at '" + str(self.look) + "'") 405 | exp_seen = True 406 | # Handle the optional sign of the exponent. 407 | number.write(self.look) 408 | self.nextchar() 409 | if self.look in _NUMBER_SIGNS: 410 | number.write(self.look) 411 | self.nextchar() 412 | else: 413 | if self.look == _DOT: 414 | if dot_seen: 415 | self.error("Malformed number at '" + 416 | str(self.look) + "'") 417 | dot_seen = True 418 | number.write(self.look) 419 | self.nextchar() 420 | 421 | # Return number converted to the most appropriate type. 422 | number = number.getvalue().decode("ascii") 423 | value = None 424 | try: 425 | if is_hex: 426 | assert not is_octal 427 | if exp_seen or dot_seen: 428 | raise ValueError(str(number)) 429 | annotations.add(ANNOT_INT) 430 | value = int(number, 16) 431 | elif is_octal: 432 | assert not is_hex 433 | if dot_seen or exp_seen: 434 | raise ValueError(str(number)) 435 | annotations.add(ANNOT_INT) 436 | value = int(number, 8) 437 | elif dot_seen or exp_seen: 438 | assert not is_hex 439 | assert not is_octal 440 | annotations.add(ANNOT_FLOAT) 441 | value = float(number) 442 | else: 443 | assert not is_hex 444 | assert not is_octal 445 | assert not exp_seen 446 | assert not dot_seen 447 | annotations.add(ANNOT_INT) 448 | value = int(number, 10) 449 | except ValueError: 450 | self.error("Malformed number: '" + str(number) + "'") 451 | 452 | return self.cast(frozenset(annotations), number, value) 453 | 454 | def parse_dict(self, annotations): 455 | self.match(_LBRACE) 456 | self.skip_whitespace() 457 | result = self.parse_keyval_items(_RBRACE) 458 | self.match(_RBRACE) 459 | annotations.add(ANNOT_DICT) 460 | return self.cast(frozenset(annotations), None, result) 461 | 462 | def parse_list(self, annotations): 463 | self.match(_LBRACKET) 464 | self.skip_whitespace() 465 | 466 | result = [] 467 | while self.look != _RBRACKET and self.look != _EOF: 468 | result.append(self.parse_value()) 469 | 470 | got_whitespace = _is_hipack_whitespace(self.look) 471 | self.skip_whitespace() 472 | if self.look == _COMMA: 473 | self.nextchar() 474 | elif not got_whitespace and not _is_hipack_whitespace(self.look): 475 | break 476 | self.skip_whitespace() 477 | 478 | self.match(_RBRACKET) 479 | annotations.add(ANNOT_LIST) 480 | return self.cast(frozenset(annotations), None, result) 481 | 482 | def parse_annotations(self): 483 | annotations = set() 484 | while self.look == _COLON: 485 | self.nextchar() 486 | key = self.parse_key() 487 | if key in annotations: 488 | self.error("Duplicate annotation '" + str(key) + "'") 489 | annotations.add(key) 490 | self.skip_whitespace() 491 | return annotations 492 | 493 | def parse_value(self): 494 | value = None 495 | annotations = self.parse_annotations() 496 | # TODO: Unify annotation handling 497 | if self.look == _DQUOTE: 498 | value = self.parse_string(annotations) 499 | elif self.look == _LBRACE: 500 | value = self.parse_dict(annotations) 501 | elif self.look == _LBRACKET: 502 | value = self.parse_list(annotations) 503 | elif self.look in _BOOL_LEADERS: 504 | value = self.parse_bool(annotations) 505 | else: 506 | value = self.parse_number(annotations) 507 | return value 508 | 509 | def parse_keyval_items(self, eos): 510 | result = {} 511 | while self.look != eos and self.look != _EOF: 512 | key = self.parse_key() 513 | 514 | got_separator = False 515 | if _is_hipack_whitespace(self.look): 516 | got_separator = True 517 | self.skip_whitespace() 518 | elif self.look == _COLON: 519 | got_separator = True 520 | self.nextchar() 521 | self.skip_whitespace() 522 | elif self.look in (_LBRACE, _LBRACKET): 523 | got_separator = True 524 | 525 | if not got_separator: 526 | self.error("missing separator") 527 | 528 | result[key] = self.parse_value() 529 | 530 | # There must be either a comma or a whitespace character after the 531 | # value, or the end-of-sequence character. 532 | if self.look == _COMMA: 533 | self.nextchar() 534 | elif self.look != eos and not _is_hipack_whitespace(self.look): 535 | break 536 | self.skip_whitespace() 537 | 538 | return result 539 | 540 | def parse_message(self): 541 | """ 542 | Parses a single message from the input stream. If the stream contains 543 | multiple messages delimited by braces, each subsequent calls 544 | will return the following message. 545 | """ 546 | result = None 547 | if self.framed: 548 | if self.look != _EOF: 549 | self.match(_LBRACE) 550 | self.nextchar() 551 | self.skip_whitespace() 552 | result = self.parse_keyval_items(_RBRACE) 553 | self.match(_RBRACE) 554 | self.skip_whitespace() 555 | else: 556 | result = self.parse_keyval_items(_EOF) 557 | return result 558 | 559 | def messages(self): 560 | """ 561 | Parses and yields each message contained in the input stream. 562 | 563 | For an input stream which contains a single, unframed message, the 564 | method yields exactly once. For an input with multiple, framed 565 | messages, each message is yield in the same order as they are in 566 | the input stream. 567 | """ 568 | while True: 569 | message = self.parse_message() 570 | if message is None: 571 | break 572 | yield message 573 | 574 | 575 | def load(stream, cast=cast): 576 | """ 577 | Parses a single message from an input stream. 578 | 579 | :param stream: 580 | A file-like object with a `.read(n)` method. 581 | :param callable cast: 582 | A value conversion function, see :class:`Parser` for details. 583 | """ 584 | return Parser(stream, cast).parse_message() 585 | 586 | 587 | def loads(bytestring, cast=cast): 588 | """ 589 | Parses a single message contained in a string. 590 | 591 | :param bytestring: 592 | Input string. It is valid to pass any of `str`, `unicode`, and `bytes` 593 | objects as input. 594 | :param callable cast: 595 | A value conversion function, see :class:`Parser` for details. 596 | """ 597 | if isinstance(bytestring, str): 598 | bytestring = bytestring.encode("utf-8") 599 | return load(BytesIO(bytestring), cast) 600 | 601 | 602 | if __name__ == "__main__": ## pragma nocover 603 | import sys 604 | dump(load(sys.stdin), sys.stdout) 605 | --------------------------------------------------------------------------------