├── .gitignore ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── apidoc ├── Makefile ├── conf.py ├── index.rst └── transit.rst ├── bin ├── make-release ├── read-write ├── read-write-pipe ├── revision └── roundtrip ├── build └── release ├── setup.cfg ├── setup.py ├── setup.py.m4 ├── tests ├── __init__.py ├── exemplars_test.py ├── helpers.py ├── regression.py └── seattle_benchmark.py └── transit ├── __init__.py ├── class_hash.py ├── constants.py ├── decoder.py ├── helpers.py ├── pyversion.py ├── read_handlers.py ├── reader.py ├── rolling_cache.py ├── sosjson.py ├── transit_types.py ├── write_handlers.py └── writer.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.swp 3 | apidoc/_build 4 | dist/ 5 | build/* 6 | !build/release 7 | *.egg* 8 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## Contributing 2 | 3 | This library is open source, developed internally by Cognitect. We welcome discussions of potential problems and enhancement suggestions on the [transit-format mailing list](https://groups.google.com/forum/#!forum/transit-format). Issues can be filed using GitHub [issues](https://github.com/cognitect/transit-python/issues) for this project. Because transit is incorporated into products and client projects, we prefer to do development internally and are not accepting pull requests or patches. 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # This library is no longer maintained. An unaffiliated fork exists at https://github.com/3wnbr1/transit-python2. 2 | 3 | transit-python 4 | ============== 5 | 6 | Transit is a format and set of libraries for conveying values between 7 | applications written in different programming languages. The library provides 8 | support for marshalling data to/from Python. 9 | 10 | * [Rationale](http://blog.cognitect.com/blog/2014/7/22/transit) 11 | * [API docs](http://cognitect.github.io/transit-python/) 12 | * [Mirrored for PyPI](http://pythonhosted.org/transit-python/) 13 | * [Specification](http://github.com/cognitect/transit-format) 14 | 15 | This implementation's major.minor version number corresponds to the 16 | version of the Transit specification it supports. 17 | 18 | _NOTE: Transit is intended primarily as a wire protocol for transferring data between applications. If storing Transit data durably, readers and writers are expected to use the same version of Transit and you are responsible for migrating/transforming/re-storing that data when and if the transit format changes._ 19 | 20 | ## Releases and Dependency Information 21 | 22 | The [PYPI](https://pypi.python.org/pypi) package is 23 | [`transit-python`](https://pypi.python.org/pypi/transit-python) 24 | 25 | * Latest stable release: [0.8](https://pypi.python.org/pypi/transit-python) 26 | 27 | You can install with any of the following: 28 | 29 | * `easy_install transit-python` 30 | * `pip install --use-wheel --pre transit-python` 31 | 32 | You can uninstall with: 33 | 34 | * `pip uninstall transit-python` 35 | 36 | ## Usage 37 | 38 | ```python 39 | # io can be any Python file descriptor, 40 | # like you would typically use with JSON's load/dump 41 | 42 | from transit.writer import Writer 43 | from transit.reader import Reader 44 | 45 | writer = Writer(io, "json") # or "json-verbose", "msgpack" 46 | writer.write(value) 47 | 48 | reader = Reader("json") # or "msgpack" 49 | val = reader.read(io) 50 | ``` 51 | 52 | For example: 53 | 54 | ``` 55 | >>> from transit.writer import Writer 56 | >>> from transit.reader import Reader 57 | >>> from StringIO import StringIO 58 | >>> io = StringIO() 59 | >>> writer = Writer(io, "json") 60 | >>> writer.write(["abc", 1234567890]) 61 | >>> s = io.getvalue() 62 | >>> reader = Reader() 63 | >>> vals = reader.read(StringIO(s)) 64 | ``` 65 | 66 | 67 | ## Supported Python versions 68 | 69 | * 2.7.X 70 | * 3.5.X 71 | 72 | 73 | ## Type Mapping 74 | 75 | ### Typed arrays, lists, and chars 76 | 77 | The [transit spec](https://github.com/cognitect/transit-format) 78 | defines several semantic types that map to more general types in Python: 79 | 80 | * typed arrays (ints, longs, doubles, floats, bools) map to Python Tuples 81 | * lists map to Python Tuples 82 | * chars map to Strings/Unicode 83 | 84 | When the reader encounters an of these (e.g. {"ints" => 85 | [1,2,3]}) it delivers just the appropriate object to the app 86 | (e.g. (1,2,3)). 87 | 88 | Use a TaggedValue to write these out if it will benefit a consuming 89 | app e.g.: 90 | 91 | ```python 92 | writer.write(TaggedValue("ints", [1,2,3])) 93 | ``` 94 | 95 | ### Python's bool and int 96 | 97 | In Python, bools are subclasses of int (that is, `True` is actually `1`). 98 | 99 | ```python 100 | >>> hash(1) 101 | 1 102 | >>> hash(True) 103 | 1 104 | >>> True == 1 105 | True 106 | ``` 107 | 108 | This becomes problematic when decoding a map that contains bool and 109 | int keys. The bool keys may be overridden (ie: you'll only see the int key), 110 | and the value will be one of any possible bool/int keyed value. 111 | 112 | ```python 113 | >>> {1: "Hello", True: "World"} 114 | {1: 'World'} 115 | ``` 116 | 117 | To counter this problem, the latest version of Transit Python introduces a 118 | Boolean type with singleton (by convention of use) instances of "true" and 119 | "false." A Boolean can be converted to a native Python bool with bool(x) where 120 | x is the "true" or "false" instance. Logical evaluation works correctly with 121 | Booleans (that is, they override the __nonzero__ method and correctly evaluate 122 | as true and false in simple logical evaluation), but uses of a Boolean as an 123 | integer will fail. 124 | 125 | ### Default type mapping 126 | 127 | |Transit type|Write accepts|Read returns| 128 | |------------|-------------|------------| 129 | |null|None|None| 130 | |string|unicode, str|unicode| 131 | |boolean|bool|bool| 132 | |integer|int|int| 133 | |decimal|float|float| 134 | |keyword|transit\_types.Keyword|transit\_types.Keyword| 135 | |symbol|transit\_types.Symbol|transit\_types.Symbol| 136 | |big decimal|float|float| 137 | |big integer|long|long| 138 | |time|long, int, datetime|datetime| 139 | |uri|transit\_types.URI|transit\_types.URI| 140 | |uuid|uuid.UUID|uuid.UUID| 141 | |char|transit\_types.TaggedValue|unicode| 142 | |array|list, tuple|tuple| 143 | |list|list, tuple|tuple| 144 | |set|set|set| 145 | |map|dict|dict| 146 | |bytes|transit\_types.TaggedValue|tuple| 147 | |shorts|transit\_types.TaggedValue|tuple| 148 | |ints|transit\_types.TaggedValue|tuple| 149 | |longs|transit\_types.TaggedValue|tuple| 150 | |floats|transit\_types.TaggedValue|tuple| 151 | |doubles|transit\_types.TaggedValue|tuple| 152 | |chars|transit\_types.TaggedValue|tuple| 153 | |bools|transit\_types.TaggedValue|tuple| 154 | |link|transit\_types.Link|transit\_types.Link| 155 | 156 | 157 | ## Development 158 | 159 | ### Setup 160 | 161 | Transit Python requires [Transit](http://github.com/cognitect/transit-format) to be at the same directory level as 162 | transit-python for access to the exemplar files. You will also need 163 | to add transit-python to your PYTHONPATH. 164 | 165 | ```sh 166 | export PYTHONPATH=$(pwd) 167 | ``` 168 | 169 | Tests should be run from the transit-python directory. 170 | 171 | ### Benchmarks 172 | 173 | ```sh 174 | python tests/seattle_benchmark.py 175 | ``` 176 | 177 | ### Running the examples 178 | 179 | ```sh 180 | python tests/exemplars_test.py 181 | ``` 182 | 183 | ### Build 184 | 185 | ```sh 186 | pip install -e . 187 | ``` 188 | 189 | The version number is automatically incremented based on the number of commits. 190 | The command below shows what version number will be applied. 191 | 192 | ```sh 193 | bin/revision 194 | ``` 195 | 196 | 197 | ## Contributing 198 | 199 | This library is open source, developed internally by Cognitect. We welcome discussions of potential problems and enhancement suggestions on the [transit-format mailing list](https://groups.google.com/forum/#!forum/transit-format). Issues can be filed using GitHub [issues](https://github.com/cognitect/transit-python/issues) for this project. Because transit is incorporated into products and client projects, we prefer to do development internally and are not accepting pull requests or patches. 200 | 201 | ## Copyright and License 202 | 203 | Copyright © 2014-2016 Cognitect 204 | 205 | Licensed under the Apache License, Version 2.0 (the "License"); 206 | you may not use this file except in compliance with the License. 207 | You may obtain a copy of the License at 208 | 209 | http://www.apache.org/licenses/LICENSE-2.0 210 | 211 | Unless required by applicable law or agreed to in writing, software 212 | distributed under the License is distributed on an "AS IS" BASIS, 213 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 214 | See the License for the specific language governing permissions and 215 | limitations under the License. 216 | -------------------------------------------------------------------------------- /apidoc/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " devhelp to make HTML files and a Devhelp project" 34 | @echo " epub to make an epub" 35 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 36 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 37 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 38 | @echo " text to make text files" 39 | @echo " man to make manual pages" 40 | @echo " texinfo to make Texinfo files" 41 | @echo " info to make Texinfo files and run them through makeinfo" 42 | @echo " gettext to make PO message catalogs" 43 | @echo " changes to make an overview of all changed/added/deprecated items" 44 | @echo " xml to make Docutils-native XML files" 45 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 46 | @echo " linkcheck to check all external links for integrity" 47 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 48 | 49 | clean: 50 | rm -rf $(BUILDDIR)/* 51 | 52 | html: 53 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 54 | @echo 55 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 56 | 57 | dirhtml: 58 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 59 | @echo 60 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 61 | 62 | singlehtml: 63 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 64 | @echo 65 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 66 | 67 | pickle: 68 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 69 | @echo 70 | @echo "Build finished; now you can process the pickle files." 71 | 72 | json: 73 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 74 | @echo 75 | @echo "Build finished; now you can process the JSON files." 76 | 77 | htmlhelp: 78 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 79 | @echo 80 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 81 | ".hhp project file in $(BUILDDIR)/htmlhelp." 82 | 83 | qthelp: 84 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 85 | @echo 86 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 87 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 88 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/transit.qhcp" 89 | @echo "To view the help file:" 90 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/transit.qhc" 91 | 92 | devhelp: 93 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 94 | @echo 95 | @echo "Build finished." 96 | @echo "To view the help file:" 97 | @echo "# mkdir -p $$HOME/.local/share/devhelp/transit" 98 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/transit" 99 | @echo "# devhelp" 100 | 101 | epub: 102 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 103 | @echo 104 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 105 | 106 | latex: 107 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 108 | @echo 109 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 110 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 111 | "(use \`make latexpdf' here to do that automatically)." 112 | 113 | latexpdf: 114 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 115 | @echo "Running LaTeX files through pdflatex..." 116 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 117 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 118 | 119 | latexpdfja: 120 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 121 | @echo "Running LaTeX files through platex and dvipdfmx..." 122 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 123 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 124 | 125 | text: 126 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 127 | @echo 128 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 129 | 130 | man: 131 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 132 | @echo 133 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 134 | 135 | texinfo: 136 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 137 | @echo 138 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 139 | @echo "Run \`make' in that directory to run these through makeinfo" \ 140 | "(use \`make info' here to do that automatically)." 141 | 142 | info: 143 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 144 | @echo "Running Texinfo files through makeinfo..." 145 | make -C $(BUILDDIR)/texinfo info 146 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 147 | 148 | gettext: 149 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 150 | @echo 151 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 152 | 153 | changes: 154 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 155 | @echo 156 | @echo "The overview file is in $(BUILDDIR)/changes." 157 | 158 | linkcheck: 159 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 160 | @echo 161 | @echo "Link check complete; look for any errors in the above output " \ 162 | "or in $(BUILDDIR)/linkcheck/output.txt." 163 | 164 | doctest: 165 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 166 | @echo "Testing of doctests in the sources finished, look at the " \ 167 | "results in $(BUILDDIR)/doctest/output.txt." 168 | 169 | xml: 170 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 171 | @echo 172 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 173 | 174 | pseudoxml: 175 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 176 | @echo 177 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 178 | -------------------------------------------------------------------------------- /apidoc/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # transit documentation build configuration file, created by 4 | # sphinx-quickstart on Mon Jun 30 16:05:29 2014. 5 | # 6 | # This file is execfile()d with the current directory set to its 7 | # containing dir. 8 | # 9 | # Note that not all possible configuration values are present in this 10 | # autogenerated file. 11 | # 12 | # All configuration values have a default; values that are commented out 13 | # serve to show the default. 14 | 15 | import sys 16 | import os 17 | 18 | # If extensions (or modules to document with autodoc) are in another directory, 19 | # add these directories to sys.path here. If the directory is relative to the 20 | # documentation root, use os.path.abspath to make it absolute, like shown here. 21 | sys.path.insert(0, os.path.abspath('..')) 22 | 23 | # -- General configuration ------------------------------------------------ 24 | 25 | # If your documentation needs a minimal Sphinx version, state it here. 26 | #needs_sphinx = '1.0' 27 | 28 | # Add any Sphinx extension module names here, as strings. They can be 29 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 30 | # ones. 31 | extensions = [ 32 | 'sphinx.ext.autodoc', 33 | 'sphinx.ext.viewcode', 34 | ] 35 | 36 | # Add any paths that contain templates here, relative to this directory. 37 | templates_path = ['_templates'] 38 | 39 | # The suffix of source filenames. 40 | source_suffix = '.rst' 41 | 42 | # The encoding of source files. 43 | #source_encoding = 'utf-8-sig' 44 | 45 | # The master toctree document. 46 | master_doc = 'index' 47 | 48 | # General information about the project. 49 | project = u'transit' 50 | copyright = u'2014, Cognitect' 51 | 52 | # The version info for the project you're documenting, acts as replacement for 53 | # |version| and |release|, also used in various other places throughout the 54 | # built documents. 55 | # 56 | # The short X.Y version. 57 | version = '0.8' 58 | # The full version, including alpha/beta/rc tags. 59 | release = '0.8' 60 | 61 | # The language for content autogenerated by Sphinx. Refer to documentation 62 | # for a list of supported languages. 63 | #language = None 64 | 65 | # There are two options for replacing |today|: either, you set today to some 66 | # non-false value, then it is used: 67 | #today = '' 68 | # Else, today_fmt is used as the format for a strftime call. 69 | #today_fmt = '%B %d, %Y' 70 | 71 | # List of patterns, relative to source directory, that match files and 72 | # directories to ignore when looking for source files. 73 | exclude_patterns = ['_build'] 74 | 75 | # The reST default role (used for this markup: `text`) to use for all 76 | # documents. 77 | #default_role = None 78 | 79 | # If true, '()' will be appended to :func: etc. cross-reference text. 80 | #add_function_parentheses = True 81 | 82 | # If true, the current module name will be prepended to all description 83 | # unit titles (such as .. function::). 84 | #add_module_names = True 85 | 86 | # If true, sectionauthor and moduleauthor directives will be shown in the 87 | # output. They are ignored by default. 88 | #show_authors = False 89 | 90 | # The name of the Pygments (syntax highlighting) style to use. 91 | pygments_style = 'sphinx' 92 | 93 | # A list of ignored prefixes for module index sorting. 94 | #modindex_common_prefix = [] 95 | 96 | # If true, keep warnings as "system message" paragraphs in the built documents. 97 | #keep_warnings = False 98 | 99 | 100 | # -- Options for HTML output ---------------------------------------------- 101 | 102 | # The theme to use for HTML and HTML Help pages. See the documentation for 103 | # a list of builtin themes. 104 | html_theme = 'default' 105 | 106 | # Theme options are theme-specific and customize the look and feel of a theme 107 | # further. For a list of options available for each theme, see the 108 | # documentation. 109 | #html_theme_options = {} 110 | 111 | # Add any paths that contain custom themes here, relative to this directory. 112 | #html_theme_path = [] 113 | 114 | # The name for this set of Sphinx documents. If None, it defaults to 115 | # " v documentation". 116 | #html_title = None 117 | 118 | # A shorter title for the navigation bar. Default is the same as html_title. 119 | #html_short_title = None 120 | 121 | # The name of an image file (relative to this directory) to place at the top 122 | # of the sidebar. 123 | #html_logo = None 124 | 125 | # The name of an image file (within the static path) to use as favicon of the 126 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 127 | # pixels large. 128 | #html_favicon = None 129 | 130 | # Add any paths that contain custom static files (such as style sheets) here, 131 | # relative to this directory. They are copied after the builtin static files, 132 | # so a file named "default.css" will overwrite the builtin "default.css". 133 | html_static_path = ['_static'] 134 | 135 | # Add any extra paths that contain custom files (such as robots.txt or 136 | # .htaccess) here, relative to this directory. These files are copied 137 | # directly to the root of the documentation. 138 | #html_extra_path = [] 139 | 140 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 141 | # using the given strftime format. 142 | #html_last_updated_fmt = '%b %d, %Y' 143 | 144 | # If true, SmartyPants will be used to convert quotes and dashes to 145 | # typographically correct entities. 146 | #html_use_smartypants = True 147 | 148 | # Custom sidebar templates, maps document names to template names. 149 | #html_sidebars = {} 150 | 151 | # Additional templates that should be rendered to pages, maps page names to 152 | # template names. 153 | #html_additional_pages = {} 154 | 155 | # If false, no module index is generated. 156 | #html_domain_indices = True 157 | 158 | # If false, no index is generated. 159 | #html_use_index = True 160 | 161 | # If true, the index is split into individual pages for each letter. 162 | #html_split_index = False 163 | 164 | # If true, links to the reST sources are added to the pages. 165 | #html_show_sourcelink = True 166 | 167 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 168 | #html_show_sphinx = True 169 | 170 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 171 | #html_show_copyright = True 172 | 173 | # If true, an OpenSearch description file will be output, and all pages will 174 | # contain a tag referring to it. The value of this option must be the 175 | # base URL from which the finished HTML is served. 176 | #html_use_opensearch = '' 177 | 178 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 179 | #html_file_suffix = None 180 | 181 | # Output file base name for HTML help builder. 182 | htmlhelp_basename = 'transit-pythondoc' 183 | 184 | 185 | # -- Options for LaTeX output --------------------------------------------- 186 | 187 | latex_elements = { 188 | # The paper size ('letterpaper' or 'a4paper'). 189 | #'papersize': 'letterpaper', 190 | 191 | # The font size ('10pt', '11pt' or '12pt'). 192 | #'pointsize': '10pt', 193 | 194 | # Additional stuff for the LaTeX preamble. 195 | #'preamble': '', 196 | } 197 | 198 | # Grouping the document tree into LaTeX files. List of tuples 199 | # (source start file, target name, title, 200 | # author, documentclass [howto, manual, or own class]). 201 | latex_documents = [ 202 | ('index', 'transit-python.tex', u'transit-python Documentation', 203 | u'Cognitect', 'manual'), 204 | ] 205 | 206 | # The name of an image file (relative to this directory) to place at the top of 207 | # the title page. 208 | #latex_logo = None 209 | 210 | # For "manual" documents, if this is true, then toplevel headings are parts, 211 | # not chapters. 212 | #latex_use_parts = False 213 | 214 | # If true, show page references after internal links. 215 | #latex_show_pagerefs = False 216 | 217 | # If true, show URL addresses after external links. 218 | #latex_show_urls = False 219 | 220 | # Documents to append as an appendix to all manuals. 221 | #latex_appendices = [] 222 | 223 | # If false, no module index is generated. 224 | #latex_domain_indices = True 225 | 226 | 227 | # -- Options for manual page output --------------------------------------- 228 | 229 | # One entry per manual page. List of tuples 230 | # (source start file, name, description, authors, manual section). 231 | man_pages = [ 232 | ('index', 'transit-python', u'transit-python Documentation', 233 | [u'Cognitect'], 1) 234 | ] 235 | 236 | # If true, show URL addresses after external links. 237 | #man_show_urls = False 238 | 239 | 240 | # -- Options for Texinfo output ------------------------------------------- 241 | 242 | # Grouping the document tree into Texinfo files. List of tuples 243 | # (source start file, target name, title, author, 244 | # dir menu entry, description, category) 245 | texinfo_documents = [ 246 | ('index', 'transit-python', u'transit-python Documentation', 247 | u'Cognitect', 'transit-python', 'Transit format marshalling for Python', 248 | 'Miscellaneous'), 249 | ] 250 | 251 | # Documents to append as an appendix to all manuals. 252 | #texinfo_appendices = [] 253 | 254 | # If false, no module index is generated. 255 | #texinfo_domain_indices = True 256 | 257 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 258 | #texinfo_show_urls = 'footnote' 259 | 260 | # If true, do not generate a @detailmenu in the "Top" node's menu. 261 | #texinfo_no_detailmenu = False 262 | 263 | 264 | # -- Options for Epub output ---------------------------------------------- 265 | 266 | # Bibliographic Dublin Core info. 267 | epub_title = u'transit-python' 268 | epub_author = u'Cognitect' 269 | epub_publisher = u'Cognitect' 270 | epub_copyright = u'2014, Cognitect' 271 | 272 | # The basename for the epub file. It defaults to the project name. 273 | #epub_basename = u'transit' 274 | 275 | # The HTML theme for the epub output. Since the default themes are not optimized 276 | # for small screen space, using the same theme for HTML and epub output is 277 | # usually not wise. This defaults to 'epub', a theme designed to save visual 278 | # space. 279 | #epub_theme = 'epub' 280 | 281 | # The language of the text. It defaults to the language option 282 | # or en if the language is not set. 283 | #epub_language = '' 284 | 285 | # The scheme of the identifier. Typical schemes are ISBN or URL. 286 | #epub_scheme = '' 287 | 288 | # The unique identifier of the text. This can be a ISBN number 289 | # or the project homepage. 290 | #epub_identifier = '' 291 | 292 | # A unique identification for the text. 293 | #epub_uid = '' 294 | 295 | # A tuple containing the cover image and cover page html template filenames. 296 | #epub_cover = () 297 | 298 | # A sequence of (type, uri, title) tuples for the guide element of content.opf. 299 | #epub_guide = () 300 | 301 | # HTML files that should be inserted before the pages created by sphinx. 302 | # The format is a list of tuples containing the path and title. 303 | #epub_pre_files = [] 304 | 305 | # HTML files shat should be inserted after the pages created by sphinx. 306 | # The format is a list of tuples containing the path and title. 307 | #epub_post_files = [] 308 | 309 | # A list of files that should not be packed into the epub file. 310 | epub_exclude_files = ['search.html'] 311 | 312 | # The depth of the table of contents in toc.ncx. 313 | #epub_tocdepth = 3 314 | 315 | # Allow duplicate toc entries. 316 | #epub_tocdup = True 317 | 318 | # Choose between 'default' and 'includehidden'. 319 | #epub_tocscope = 'default' 320 | 321 | # Fix unsupported image types using the PIL. 322 | #epub_fix_images = False 323 | 324 | # Scale large images. 325 | #epub_max_image_width = 0 326 | 327 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 328 | #epub_show_urls = 'inline' 329 | 330 | # If false, no index is generated. 331 | #epub_use_index = True 332 | -------------------------------------------------------------------------------- /apidoc/index.rst: -------------------------------------------------------------------------------- 1 | .. transit documentation master file, created by 2 | sphinx-quickstart on Mon Jun 30 16:05:29 2014. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to transit's documentation! 7 | =================================== 8 | 9 | Contents: 10 | 11 | .. toctree:: 12 | :maxdepth: 4 13 | 14 | transit 15 | 16 | 17 | Indices and tables 18 | ================== 19 | 20 | * :ref:`genindex` 21 | * :ref:`modindex` 22 | * :ref:`search` 23 | 24 | -------------------------------------------------------------------------------- /apidoc/transit.rst: -------------------------------------------------------------------------------- 1 | transit package 2 | =============== 3 | 4 | Submodules 5 | ---------- 6 | 7 | transit.class_hash module 8 | ------------------------- 9 | 10 | .. automodule:: transit.class_hash 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | transit.constants module 16 | ------------------------ 17 | 18 | .. automodule:: transit.constants 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | transit.decoder module 24 | ---------------------- 25 | 26 | .. automodule:: transit.decoder 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | transit.helpers module 32 | ---------------------- 33 | 34 | .. automodule:: transit.helpers 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | transit.read_handlers module 40 | ---------------------------- 41 | 42 | .. automodule:: transit.read_handlers 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | transit.reader module 48 | --------------------- 49 | 50 | .. automodule:: transit.reader 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | transit.rolling_cache module 56 | ---------------------------- 57 | 58 | .. automodule:: transit.rolling_cache 59 | :members: 60 | :undoc-members: 61 | :show-inheritance: 62 | 63 | transit.transit_types module 64 | ---------------------------- 65 | 66 | .. automodule:: transit.transit_types 67 | :members: 68 | :undoc-members: 69 | :show-inheritance: 70 | 71 | transit.write_handlers module 72 | ----------------------------- 73 | 74 | .. automodule:: transit.write_handlers 75 | :members: 76 | :undoc-members: 77 | :show-inheritance: 78 | 79 | transit.writer module 80 | --------------------- 81 | 82 | .. automodule:: transit.writer 83 | :members: 84 | :undoc-members: 85 | :show-inheritance: 86 | 87 | 88 | Module contents 89 | --------------- 90 | 91 | .. automodule:: transit 92 | :members: 93 | :undoc-members: 94 | :show-inheritance: 95 | -------------------------------------------------------------------------------- /bin/make-release: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | m4 -DREVISION=`bin/revision` setup.py.m4 > setup.py 3 | -------------------------------------------------------------------------------- /bin/read-write: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import sys, os 3 | from io import BytesIO 4 | from StringIO import StringIO 5 | import codecs 6 | 7 | sys.path.append(os.path.abspath(os.path.dirname(__file__) + os.path.sep + os.path.pardir)) 8 | 9 | import transit.reader as treader 10 | import transit.writer as twriter 11 | 12 | transport = (len(sys.argv) > 1) and sys.argv[1].replace("-", "_") or "json" 13 | stdin = codecs.getreader("utf-8")(sys.stdin) 14 | stdout = codecs.getwriter("utf-8")(sys.stdout) 15 | 16 | 17 | def json_rw_loop(): 18 | r = treader.Reader(protocol=transport) 19 | while True: 20 | for o in r.readeach(stdin): 21 | s = StringIO() 22 | w = twriter.Writer(s, protocol=transport) 23 | w.write(o) 24 | result = s.getvalue().strip() 25 | # remove extra commas for multiple values 26 | if result[0] == ",": 27 | result = result[1:] 28 | stdout.write(result) 29 | stdout.flush() 30 | 31 | def msgpack_rw_loop(): 32 | r = treader.Reader(protocol=transport) 33 | w = twriter.Writer(sys.stdout, protocol=transport) 34 | u = r.unpacker 35 | while True: 36 | u.feed(sys.stdin.read(1)) 37 | for o in r.readeach(sys.stdin): 38 | w.write(o) 39 | 40 | if transport == "msgpack": 41 | msgpack_rw_loop() 42 | else: 43 | json_rw_loop() 44 | 45 | 46 | # If you want to use this with pipe, or interactively... 47 | #while True: 48 | # line = sys.stdin.readline() # `for line in stdin` won't play nice in all situations 49 | # if line == '': 50 | # break 51 | # s = StringIO(line) 52 | # o = r.read(s) 53 | # w.write(o) 54 | print "" 55 | 56 | -------------------------------------------------------------------------------- /bin/read-write-pipe: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys, os 4 | from StringIO import StringIO 5 | 6 | sys.path.append(os.path.abspath(os.path.dirname(__file__) + os.path.sep + os.path.pardir)) 7 | 8 | import transit.reader as treader 9 | import transit.writer as twriter 10 | 11 | transport = (len(sys.argv) > 1) and sys.argv[1].replace("-", "_") or "json" 12 | 13 | r = treader.Reader(protocol=transport) 14 | w = twriter.Writer(sys.stdout, protocol=transport) 15 | 16 | while True: 17 | line = sys.stdin.readline() # `for line in stdin` won't play nice in all situations 18 | if line == '': 19 | break 20 | s = StringIO(line) 21 | o = r.read(s) 22 | w.write(o) 23 | print "" 24 | 25 | -------------------------------------------------------------------------------- /bin/revision: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Returns the revision number used for deployment. 4 | 5 | set -e 6 | 7 | REVISION=`git --no-replace-objects describe --tags --match v0.0` 8 | 9 | # Extract the version number from the string. Do this in two steps so 10 | # it is a little easier to understand. 11 | REVISION=${REVISION:5} # drop the first 5 characters 12 | REVISION=${REVISION:0:${#REVISION}-9} # drop the last 9 characters 13 | 14 | echo $REVISION 15 | 16 | -------------------------------------------------------------------------------- /bin/roundtrip: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Copyright (c) Cognitect, Inc. 3 | # All rights reserved. 4 | 5 | cd `dirname $0`/.. 6 | 7 | exec bin/read-write "$@" 8 | 9 | -------------------------------------------------------------------------------- /build/release: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | echo "Cleaning..." 6 | rm -rf ./target 7 | 8 | echo "Calculating version..." 9 | prefix=`cat VERSION_PREFIX` 10 | suffix=`build/revision` 11 | version=$prefix.$suffix 12 | echo $version 13 | 14 | echo "Releasing..." 15 | # tar? 16 | # upload? 17 | 18 | echo "Tagging..." 19 | git tag -a v${version} -m "Release ${version}" 20 | git push origin v${version} 21 | 22 | echo "Updating README.md versions" 23 | sed -i '' "s/[[:digit:]]\{1,2\}\.[[:digit:]]\{1,2\}\.[[:digit:]]\{2,4\}/${version}/g" README.md 24 | git ci -m "Update README.md with ${version}" README.md 25 | git push 26 | 27 | echo "Release done!" 28 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.md 3 | 4 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | ## Copyright 2014 Cognitect. All Rights Reserved. 4 | ## 5 | ## Licensed under the Apache License, Version 2.0 (the "License"); 6 | ## you may not use this file except in compliance with the License. 7 | ## You may obtain a copy of the License at 8 | ## 9 | ## http://www.apache.org/licenses/LICENSE-2.0 10 | ## 11 | ## Unless required by applicable law or agreed to in writing, software 12 | ## distributed under the License is distributed on an "AS-IS" BASIS, 13 | ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | ## See the License for the specific language governing permissions and 15 | ## limitations under the License. 16 | 17 | from setuptools import setup, find_packages 18 | import subprocess 19 | 20 | setup(name="transit-python", 21 | version="0.8.284", 22 | description="Transit marshalling for Python", 23 | author="Cognitect", 24 | url="https://github.com/cognitect/transit-python", 25 | packages=find_packages(), 26 | install_requires=["python-dateutil", "msgpack-python"]) 27 | 28 | -------------------------------------------------------------------------------- /setup.py.m4: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | ## Copyright 2014 Cognitect. All Rights Reserved. 4 | ## 5 | ## Licensed under the Apache License, Version 2.0 (the "License"); 6 | ## you may not use this file except in compliance with the License. 7 | ## You may obtain a copy of the License at 8 | ## 9 | ## http://www.apache.org/licenses/LICENSE-2.0 10 | ## 11 | ## Unless required by applicable law or agreed to in writing, software 12 | ## distributed under the License is distributed on an "AS-IS" BASIS, 13 | ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | ## See the License for the specific language governing permissions and 15 | ## limitations under the License. 16 | 17 | from setuptools import setup, find_packages 18 | import subprocess 19 | 20 | setup(name="transit-python", 21 | version="0.8.REVISION", 22 | description="Transit marshalling for Python", 23 | author="Cognitect", 24 | url="https://github.com/cognitect/transit-python", 25 | packages=find_packages(), 26 | install_requires=["python-dateutil", "msgpack-python"]) 27 | 28 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | ## Copyright 2014 Cognitect. All Rights Reserved. 2 | ## 3 | ## Licensed under the Apache License, Version 2.0 (the "License"); 4 | ## you may not use this file except in compliance with the License. 5 | ## You may obtain a copy of the License at 6 | ## 7 | ## http://www.apache.org/licenses/LICENSE-2.0 8 | ## 9 | ## Unless required by applicable law or agreed to in writing, software 10 | ## distributed under the License is distributed on an "AS-IS" BASIS, 11 | ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | ## See the License for the specific language governing permissions and 13 | ## limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/exemplars_test.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ## Copyright 2014 Cognitect. All Rights Reserved. 3 | ## 4 | ## Licensed under the Apache License, Version 2.0 (the "License"); 5 | ## you may not use this file except in compliance with the License. 6 | ## You may obtain a copy of the License at 7 | ## 8 | ## http://www.apache.org/licenses/LICENSE-2.0 9 | ## 10 | ## Unless required by applicable law or agreed to in writing, software 11 | ## distributed under the License is distributed on an "AS-IS" BASIS, 12 | ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | ## See the License for the specific language governing permissions and 14 | ## limitations under the License. 15 | import unittest 16 | 17 | # then import transit stuff 18 | from transit.reader import Reader, JsonUnmarshaler, MsgPackUnmarshaler 19 | from transit.writer import Writer 20 | from transit.transit_types import Keyword, Symbol, URI, frozendict, TaggedValue, Link, true, false 21 | from io import StringIO, BytesIO 22 | from transit.helpers import mapcat 23 | from helpers import ints_centered_on, hash_of_size, array_of_symbools 24 | from uuid import UUID 25 | from datetime import datetime 26 | import dateutil.tz 27 | from math import isnan 28 | 29 | class ExemplarBaseTest(unittest.TestCase): 30 | pass 31 | 32 | def exemplar(name, val): 33 | class ExemplarTest(ExemplarBaseTest): 34 | 35 | def test_json(self): 36 | with open("../transit-format/examples/0.8/simple/" + name + ".json") as stream: 37 | data = Reader(protocol="json").read(stream) 38 | self.assertEqual(val, data) 39 | 40 | def test_msgpack(self): 41 | with open("../transit-format/examples/0.8/simple/" + name + ".mp", 'rb') as stream: 42 | data = Reader(protocol="msgpack").read(stream) 43 | self.assertEqual(val, data) 44 | 45 | def test_json_verbose(self): 46 | with open("../transit-format/examples/0.8/simple/" + name + ".verbose.json") as stream: 47 | data = Reader(protocol="json_verbose").read(stream) 48 | self.assertEqual(val, data) 49 | 50 | def test_reencode_msgpack(self): 51 | io = BytesIO() 52 | writer = Writer(io, protocol="msgpack") 53 | writer.write(val) 54 | s = io.getvalue() 55 | io = BytesIO(s) 56 | 57 | reader = Reader(protocol="msgpack") 58 | newval = reader.read(io) 59 | self.assertEqual(val, newval) 60 | 61 | def test_reencode_json(self): 62 | io = StringIO() 63 | writer = Writer(io, protocol="json") 64 | writer.write(val) 65 | s = io.getvalue() 66 | # Uncomment when debugging to see what payloads fail 67 | # print(s) 68 | io = StringIO(s) 69 | reader = Reader(protocol="json") 70 | newval = reader.read(io) 71 | self.assertEqual(val, newval) 72 | 73 | # test json verbose 74 | def test_reencode_json_verbose(self): 75 | io = StringIO() 76 | writer = Writer(io, protocol="json_verbose") 77 | writer.write(val) 78 | s = io.getvalue() 79 | io = StringIO(s) 80 | reader = Reader(protocol="json_verbose") 81 | newval = reader.read(io) 82 | self.assertEqual(val, newval) 83 | 84 | def assertEqual(self, val, data): 85 | if type(val) is float or type(data) is float: 86 | if type(val) is float and type(data) is float and isnan(val) and isnan(data): 87 | return true 88 | else: 89 | unittest.TestCase.assertAlmostEqual(self, val, data, places=7, msg=name) 90 | elif type(val) in [list, tuple]: 91 | for v, d in zip(val, data): 92 | self.assertEqual(v, d) 93 | else: 94 | unittest.TestCase.assertEqual(self, val, data, name + " " + str(val) + " vs " + str(data)) 95 | 96 | globals()["test_" + name + "_json"] = ExemplarTest 97 | 98 | ARRAY_SIMPLE = (1, 2, 3) 99 | ARRAY_MIXED = (0, 1, 2.0, true, false, 'five', Keyword("six"), Symbol("seven"), '~eight', None) 100 | ARRAY_NESTED = (ARRAY_SIMPLE, ARRAY_MIXED) 101 | SMALL_STRINGS = ("", "a", "ab", "abc", "abcd", "abcde", "abcdef") 102 | POWERS_OF_TWO = tuple(map(lambda x: pow(2, x), range(66))) 103 | INTERESTING_INTS = tuple(mapcat(lambda x: ints_centered_on(x, 2), POWERS_OF_TWO)) 104 | 105 | SYM_STRS = ["a", "ab", "abc", "abcd", "abcde", "a1", "b2", "c3", "a_b"] 106 | SYMBOLS = tuple(map(Symbol, SYM_STRS)) 107 | KEYWORDS = tuple(map(Keyword, SYM_STRS)) 108 | 109 | 110 | UUIDS = (UUID('5a2cbea3-e8c6-428b-b525-21239370dd55'), 111 | UUID('d1dc64fa-da79-444b-9fa4-d4412f427289'), 112 | UUID('501a978e-3a3e-4060-b3be-1cf2bd4b1a38'), 113 | UUID('b3ba141a-a776-48e4-9fae-a28ea8571f58')) 114 | 115 | 116 | URIS = ( 117 | URI(u'http://example.com'), 118 | URI(u'ftp://example.com'), 119 | URI(u'file:///path/to/file.txt'), 120 | URI(u'http://www.詹姆斯.com/')) 121 | 122 | DATES = tuple(map(lambda x: datetime.fromtimestamp(x/1000.0, tz=dateutil.tz.tzutc()), 123 | [-6106017600000, 0, 946728000000, 1396909037000])) 124 | 125 | SET_SIMPLE = frozenset(ARRAY_SIMPLE) 126 | SET_MIXED = frozenset(ARRAY_MIXED) 127 | SET_NESTED = frozenset([SET_SIMPLE, SET_MIXED]) 128 | 129 | MAP_SIMPLE = frozendict({Keyword("a"): 1, 130 | Keyword("b"): 2, 131 | Keyword("c"): 3}) 132 | 133 | MAP_MIXED = frozendict({Keyword("a"): 1, 134 | Keyword("b"): u"a string", 135 | Keyword("c"): true}) 136 | 137 | MAP_NESTED = frozendict({Keyword("simple"): MAP_SIMPLE, 138 | Keyword("mixed"): MAP_MIXED}) 139 | 140 | exemplar("uris", URIS) 141 | 142 | exemplar("nil", None) 143 | exemplar("true", true) 144 | exemplar("false", false) 145 | exemplar("zero", 0) 146 | exemplar("one", 1) 147 | exemplar("one_string", "hello") 148 | exemplar("one_keyword", Keyword("hello")) 149 | exemplar("one_symbol", Symbol("hello")) 150 | exemplar("one_date", datetime.fromtimestamp(946728000000/1000.0, dateutil.tz.tzutc())) 151 | exemplar("vector_simple", ARRAY_SIMPLE) 152 | exemplar("vector_empty", ()) 153 | exemplar("vector_mixed", ARRAY_MIXED) 154 | exemplar("vector_nested", ARRAY_NESTED) 155 | exemplar("small_strings", SMALL_STRINGS) 156 | exemplar("strings_tilde", tuple(map(lambda x: "~" + x, SMALL_STRINGS))) 157 | exemplar("strings_hash", tuple(map(lambda x: "#" + x, SMALL_STRINGS))) 158 | exemplar("strings_hat", tuple(map(lambda x: "^" + x, SMALL_STRINGS))) 159 | exemplar("ints", tuple(range(128))) 160 | exemplar("small_ints", ints_centered_on(0)) 161 | exemplar("ints_interesting", INTERESTING_INTS) 162 | exemplar("ints_interesting_neg", tuple(map(lambda x: -x, INTERESTING_INTS))) 163 | exemplar("doubles_small", tuple(map(float, ints_centered_on(0)))) 164 | exemplar("doubles_interesting", (-3.14159, 3.14159, 4E11, 2.998E8, 6.626E-34)) 165 | exemplar("one_uuid", UUIDS[0]) 166 | exemplar("uuids", UUIDS) 167 | exemplar("one_uri", URIS[0]) 168 | exemplar("uris", URIS) 169 | exemplar("dates_interesting", DATES) 170 | exemplar("symbols", SYMBOLS) 171 | exemplar("keywords", KEYWORDS) 172 | exemplar("list_simple", ARRAY_SIMPLE) 173 | exemplar("list_empty", ()) 174 | exemplar("list_mixed", ARRAY_MIXED) 175 | exemplar("list_nested", ARRAY_NESTED) 176 | exemplar("set_simple", SET_SIMPLE) 177 | exemplar("set_empty", set()) 178 | exemplar("set_mixed", SET_MIXED) 179 | exemplar("set_nested", SET_NESTED) 180 | exemplar("map_simple", MAP_SIMPLE) 181 | exemplar("map_mixed", MAP_MIXED) 182 | exemplar("map_nested", MAP_NESTED) 183 | exemplar("map_string_keys", {"first": 1, "second": 2, "third": 3}) 184 | exemplar("map_numeric_keys", {1: "one", 2: "two"}) 185 | exemplar("map_vector_keys", frozendict([[(1, 1), "one"], 186 | [(2, 2), "two"]])) 187 | 188 | 189 | exemplar("map_unrecognized_vals", {Keyword("key"): "~Unrecognized"}) 190 | #exemplar("map_unrecognized_keys", ) 191 | exemplar("vector_unrecognized_vals", ("~Unrecognized",)) 192 | exemplar("vector_1935_keywords_repeated_twice", tuple(array_of_symbools(1935, 1935*2))) 193 | exemplar("vector_1936_keywords_repeated_twice", tuple(array_of_symbools(1936, 1936*2))) 194 | exemplar("vector_1937_keywords_repeated_twice", tuple(array_of_symbools(1937, 1937*2))) 195 | 196 | exemplar("map_10_items", hash_of_size(10)) 197 | exemplar("maps_two_char_sym_keys", ({Symbol("aa"): 1, Symbol("bb"): 2}, 198 | {Symbol("aa"): 3, Symbol("bb"): 4}, 199 | {Symbol("aa"): 5, Symbol("bb"): 6})) 200 | 201 | exemplar("maps_three_char_sym_keys", ({Symbol("aaa"): 1, Symbol("bbb"): 2}, 202 | {Symbol("aaa"): 3, Symbol("bbb"): 4}, 203 | {Symbol("aaa"): 5, Symbol("bbb"): 6})) 204 | 205 | exemplar("maps_four_char_sym_keys", ({Symbol("aaaa"): 1, Symbol("bbbb"): 2}, 206 | {Symbol("aaaa"): 3, Symbol("bbbb"): 4}, 207 | {Symbol("aaaa"): 5, Symbol("bbbb"): 6})) 208 | 209 | exemplar("maps_two_char_string_keys", ({"aa": 1, "bb": 2}, 210 | {"aa": 3, "bb": 4}, 211 | {"aa": 5, "bb": 6})) 212 | 213 | exemplar("maps_three_char_string_keys", ({"aaa": 1, "bbb": 2}, 214 | {"aaa": 3, "bbb": 4}, 215 | {"aaa": 5, "bbb": 6})) 216 | 217 | exemplar("maps_four_char_string_keys", ({"aaaa": 1, "bbbb": 2}, 218 | {"aaaa": 3, "bbbb": 4}, 219 | {"aaaa": 5, "bbbb": 6})) 220 | 221 | exemplar("maps_unrecognized_keys", (TaggedValue("abcde", Keyword("anything")), 222 | TaggedValue("fghij", Keyword("anything-else")),)) 223 | 224 | exemplar("vector_special_numbers", (float("nan"), float("inf"), float("-inf"))) 225 | 226 | # Doesn't exist in simple examples but gave me tests to verify Link. 227 | #exemplar("link", Link("http://www.blah.com", "test", "test", "link", "test")) 228 | 229 | def make_hash_exemplar(n): 230 | exemplar("map_%s_nested" % (n,), {Keyword("f"): hash_of_size(n), 231 | Keyword("s"): hash_of_size(n)}) 232 | for n in [10, 1935, 1936, 1937]: 233 | make_hash_exemplar(n) 234 | 235 | if __name__=='__main__': 236 | unittest.main() 237 | #import cProfile 238 | #import pstats 239 | #cProfile.run('unittest.main()', 'exemptests') 240 | #p = pstats.Stats('exemptests') 241 | #p.sort_stats('time') 242 | #p.print_stats() 243 | -------------------------------------------------------------------------------- /tests/helpers.py: -------------------------------------------------------------------------------- 1 | ## Copyright 2014 Cognitect. All Rights Reserved. 2 | ## 3 | ## Licensed under the Apache License, Version 2.0 (the "License"); 4 | ## you may not use this file except in compliance with the License. 5 | ## You may obtain a copy of the License at 6 | ## 7 | ## http://www.apache.org/licenses/LICENSE-2.0 8 | ## 9 | ## Unless required by applicable law or agreed to in writing, software 10 | ## distributed under the License is distributed on an "AS-IS" BASIS, 11 | ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | ## See the License for the specific language governing permissions and 13 | ## limitations under the License. 14 | 15 | import transit 16 | from transit.transit_types import Symbol, Keyword, frozendict 17 | from transit.helpers import cycle, take, pairs 18 | from transit.pyversion import izip 19 | 20 | def ints_centered_on(m, n=5): 21 | return tuple(range(m - n, m + n + 1)) 22 | 23 | 24 | def array_of_symbools(m, n=None): 25 | if n is None: 26 | n = m 27 | 28 | seeds = map(lambda x: Keyword("key"+str(x).zfill(4)), range(0, m)) 29 | return take(n, cycle(seeds)) 30 | 31 | 32 | def hash_of_size(n): 33 | return frozendict(izip(array_of_symbools(n), range(0, n+1))) 34 | -------------------------------------------------------------------------------- /tests/regression.py: -------------------------------------------------------------------------------- 1 | ## Copyright 2014 Cognitect. All Rights Reserved. 2 | ## 3 | ## Licensed under the Apache License, Version 2.0 (the "License"); 4 | ## you may not use this file except in compliance with the License. 5 | ## You may obtain a copy of the License at 6 | ## 7 | ## http://www.apache.org/licenses/LICENSE-2.0 8 | ## 9 | ## Unless required by applicable law or agreed to in writing, software 10 | ## distributed under the License is distributed on an "AS-IS" BASIS, 11 | ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | ## See the License for the specific language governing permissions and 13 | ## limitations under the License. 14 | 15 | # This test suite verifies that issues corrected remain corrected. 16 | import unittest 17 | import json 18 | from transit.reader import Reader 19 | from transit.writer import Writer 20 | from transit.class_hash import ClassDict 21 | from transit.transit_types import Symbol, frozendict, true, false, Keyword, Named 22 | from transit.pyversion import unicode_type 23 | from decimal import Decimal 24 | from io import BytesIO, StringIO 25 | 26 | class RegressionBaseTest(unittest.TestCase): 27 | pass 28 | 29 | def regression(name, value): 30 | class RegressionTest(RegressionBaseTest): 31 | 32 | def test_roundtrip(self): 33 | in_data = value 34 | io = StringIO() 35 | w = Writer(io, "json") 36 | w.write(in_data) 37 | r = Reader("json") 38 | out_data = r.read(StringIO(io.getvalue())) 39 | self.assertEqual(in_data, out_data) 40 | 41 | globals()["test_" + name + "_json"] = RegressionTest 42 | 43 | regression("cache_consistency", ({"Problem?":true}, 44 | Symbol("Here"), 45 | Symbol("Here"))) 46 | regression("one_pair_frozendict", frozendict({"a":1})) 47 | regression("json_int_max", (2**53+100, 2**63+100)) 48 | regression("newline_in_string", "a\nb") 49 | regression("big_decimal", Decimal("190234710272.2394720347203642836434")) 50 | regression("dict_in_set", frozenset(frozendict({"test":"case"}))) 51 | 52 | def json_verbose_cache_bug(): 53 | class JsonVerboseCacheBug(RegressionBaseTest): 54 | """Can't rely on roundtrip behavior to test this bug, have to 55 | actually verify that both keys are written for json_verbose 56 | behavior to be correct.""" 57 | 58 | def test_key_not_cached(self): 59 | io = StringIO() 60 | w = Writer(io, "json_verbose") 61 | w.write([{'myKey1':42},{'myKey1':42}]) 62 | self.assertEqual(io.getvalue(), u"[{\"myKey1\":42},{\"myKey1\":42}]") 63 | 64 | globals()["test_json_verbose_cache_bug"] = JsonVerboseCacheBug 65 | 66 | json_verbose_cache_bug() 67 | 68 | def json_int_boundary(value, expected_type): 69 | class JsonIntBoundaryTest(unittest.TestCase): 70 | 71 | def test_max_is_number(self): 72 | for protocol in ("json", "json_verbose"): 73 | io = StringIO() 74 | w = Writer(io, protocol) 75 | w.write([value]) 76 | actual_type = type(json.loads(io.getvalue())[0]) 77 | self.assertEqual(expected_type, actual_type) 78 | 79 | globals()["test_json_int_boundary_" + str(value)] = JsonIntBoundaryTest 80 | 81 | json_int_boundary(2**53-1, int) 82 | json_int_boundary(2**53, unicode_type) 83 | json_int_boundary(-2**53+1, int) 84 | json_int_boundary(-2**53, unicode_type) 85 | 86 | class BooleanTest(unittest.TestCase): 87 | """Even though we're roundtripping transit_types.true and 88 | transit_types.false now, make sure we can still write Python bools. 89 | 90 | Additionally, make sure we can still do basic logical evaluation on transit 91 | Boolean values. 92 | """ 93 | def test_write_bool(self): 94 | for protocol in ("json", "json_verbose", "msgpack"): 95 | io = BytesIO() if protocol is "msgpack" else StringIO() 96 | w = Writer(io, protocol) 97 | w.write((True, False)) 98 | r = Reader(protocol) 99 | io.seek(0) 100 | out_data = r.read(io) 101 | assert out_data[0] == true 102 | assert out_data[1] == false 103 | 104 | def test_basic_eval(self): 105 | assert true 106 | assert not false 107 | 108 | def test_or(self): 109 | assert true or false 110 | assert not (false or false) 111 | assert true or true 112 | 113 | def test_and(self): 114 | assert not (true and false) 115 | assert true and true 116 | assert not (false and false) 117 | 118 | # Helper classes for inheritance unit test. 119 | class parent(object): 120 | pass 121 | class child(parent): 122 | pass 123 | class grandchild(child): 124 | pass 125 | 126 | class ClassDictInheritanceTest(unittest.TestCase): 127 | """ Fix from issue #18: class_hash should iterate over all ancestors 128 | in proper mro, not just over direct ancestor. 129 | """ 130 | def test_inheritance(self): 131 | cd = ClassDict() 132 | cd[parent] = "test" 133 | assert grandchild in cd 134 | 135 | class NamedTests(unittest.TestCase): 136 | """ Verify behavior for newly introduced built-in Named name/namespace 137 | parsing. Accomplished through transit_types.Named, a mixin for 138 | transit_types.Keyword and transit_types.Symbol. 139 | """ 140 | def test_named(self): 141 | k = Keyword("blah") 142 | s = Symbol("blah") 143 | assert k.name == "blah" 144 | assert s.name == "blah" 145 | 146 | def test_namespaced(self): 147 | k = Keyword("ns/name") 148 | s = Symbol("ns/name") 149 | assert k.name == "name" 150 | assert s.name == "name" 151 | assert k.namespace == "ns" 152 | assert s.namespace == "ns" 153 | 154 | def test_slash(self): 155 | k = Keyword("/") 156 | s = Symbol("/") 157 | assert k.name == "/" 158 | assert s.name == "/" 159 | assert k.namespace is None 160 | assert s.namespace is None 161 | 162 | if __name__ == '__main__': 163 | unittest.main() 164 | -------------------------------------------------------------------------------- /tests/seattle_benchmark.py: -------------------------------------------------------------------------------- 1 | ## Copyright 2014 Cognitect. All Rights Reserved. 2 | ## 3 | ## Licensed under the Apache License, Version 2.0 (the "License"); 4 | ## you may not use this file except in compliance with the License. 5 | ## You may obtain a copy of the License at 6 | ## 7 | ## http://www.apache.org/licenses/LICENSE-2.0 8 | ## 9 | ## Unless required by applicable law or agreed to in writing, software 10 | ## distributed under the License is distributed on an "AS-IS" BASIS, 11 | ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | ## See the License for the specific language governing permissions and 13 | ## limitations under the License. 14 | 15 | from transit.reader import JsonUnmarshaler 16 | from transit.pyversion import unicode_type 17 | import json 18 | import time 19 | from io import StringIO 20 | 21 | def run_tests(data): 22 | datas = StringIO(unicode_type(data)) 23 | t = time.time() 24 | JsonUnmarshaler().load(datas) 25 | et = time.time() 26 | datas = StringIO(unicode_type(data)) 27 | tt = time.time() 28 | json.load(datas) 29 | ett = time.time() 30 | read_delta = (et - t) * 1000.0 31 | print("Done: " + str(read_delta) + " -- raw JSON in: " + str((ett - tt) * 1000.0)) 32 | return read_delta 33 | 34 | 35 | seattle_dir = "../transit-format/examples/0.8/" 36 | means = {} 37 | for jsonfile in [seattle_dir + "example.json", 38 | seattle_dir + "example.verbose.json"]: 39 | data = "" 40 | with open(jsonfile, 'r') as fd: 41 | data = fd.read() 42 | 43 | print("-"*50) 44 | print("Running " + jsonfile) 45 | print("-"*50) 46 | 47 | runs = 200 48 | deltas = [run_tests(data) for x in range(runs)] 49 | means[jsonfile] = sum(deltas)/runs 50 | 51 | for jsonfile, mean in means.items(): 52 | print("\nMean for" + jsonfile + ": "+str(mean)) 53 | 54 | -------------------------------------------------------------------------------- /transit/__init__.py: -------------------------------------------------------------------------------- 1 | ## Copyright 2014 Cognitect. All Rights Reserved. 2 | ## 3 | ## Licensed under the Apache License, Version 2.0 (the "License"); 4 | ## you may not use this file except in compliance with the License. 5 | ## You may obtain a copy of the License at 6 | ## 7 | ## http://www.apache.org/licenses/LICENSE-2.0 8 | ## 9 | ## Unless required by applicable law or agreed to in writing, software 10 | ## distributed under the License is distributed on an "AS-IS" BASIS, 11 | ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | ## See the License for the specific language governing permissions and 13 | ## limitations under the License. 14 | -------------------------------------------------------------------------------- /transit/class_hash.py: -------------------------------------------------------------------------------- 1 | ## Copyright 2014 Cognitect. All Rights Reserved. 2 | ## 3 | ## Licensed under the Apache License, Version 2.0 (the "License"); 4 | ## you may not use this file except in compliance with the License. 5 | ## You may obtain a copy of the License at 6 | ## 7 | ## http://www.apache.org/licenses/LICENSE-2.0 8 | ## 9 | ## Unless required by applicable law or agreed to in writing, software 10 | ## distributed under the License is distributed on an "AS-IS" BASIS, 11 | ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | ## See the License for the specific language governing permissions and 13 | ## limitations under the License. 14 | from transit import pyversion 15 | MutableMapping = pyversion.abc.MutableMapping 16 | 17 | 18 | class ClassDict(MutableMapping): 19 | """A dictionary that looks up class/type keys with inheritance.""" 20 | 21 | def __init__(self, *args, **kwargs): 22 | self.store = dict() 23 | self.update(dict(*args, **kwargs)) 24 | 25 | def __getitem__(self, key): 26 | key = key if isinstance(key, type) else type(key) 27 | if key in self.store: 28 | return self.store[key] 29 | else: 30 | for t in key.__bases__: 31 | value = t in self.store and self.store[t] 32 | if value: 33 | return value 34 | # only use mro if __bases__ doesn't work to 35 | # avoid its perf overhead. 36 | for t in key.mro(): 37 | value = t in self.store and self.store[t] 38 | if value: 39 | return value 40 | raise KeyError("No handler found for: " + str(key)) 41 | 42 | def __setitem__(self, key, value): 43 | self.store[key] = value 44 | 45 | def __delitem__(self, key): 46 | del self.store[key] 47 | 48 | def __iter__(self): 49 | return iter(self.store) 50 | 51 | def __len__(self): 52 | return len(self.store) 53 | -------------------------------------------------------------------------------- /transit/constants.py: -------------------------------------------------------------------------------- 1 | ## Copyright 2014 Cognitect. All Rights Reserved. 2 | ## 3 | ## Licensed under the Apache License, Version 2.0 (the "License"); 4 | ## you may not use this file except in compliance with the License. 5 | ## You may obtain a copy of the License at 6 | ## 7 | ## http://www.apache.org/licenses/LICENSE-2.0 8 | ## 9 | ## Unless required by applicable law or agreed to in writing, software 10 | ## distributed under the License is distributed on an "AS-IS" BASIS, 11 | ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | ## See the License for the specific language governing permissions and 13 | ## limitations under the License. 14 | 15 | # These are constants used throughout the source code. 16 | # They refer to Tag codes used when reading and wrting transit. 17 | 18 | ESC = "~" 19 | SUB = "^" 20 | RES = "`" 21 | TAG = "~#" 22 | QUOTE = "'" 23 | MAP_AS_ARR = "^ " 24 | -------------------------------------------------------------------------------- /transit/decoder.py: -------------------------------------------------------------------------------- 1 | ## Copyright 2014 Cognitect. All Rights Reserved. 2 | ## 3 | ## Licensed under the Apache License, Version 2.0 (the "License"); 4 | ## you may not use this file except in compliance with the License. 5 | ## You may obtain a copy of the License at 6 | ## 7 | ## http://www.apache.org/licenses/LICENSE-2.0 8 | ## 9 | ## Unless required by applicable law or agreed to in writing, software 10 | ## distributed under the License is distributed on an "AS-IS" BASIS, 11 | ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | ## See the License for the specific language governing permissions and 13 | ## limitations under the License. 14 | 15 | from collections import OrderedDict 16 | from transit import pyversion, transit_types 17 | from transit import read_handlers as rh 18 | from transit.constants import MAP_AS_ARR, ESC, SUB, RES 19 | from transit.helpers import pairs 20 | from transit.rolling_cache import RollingCache, is_cacheable, is_cache_key 21 | from transit.transit_types import true, false 22 | 23 | 24 | class Tag(object): 25 | def __init__(self, tag): 26 | self.tag = tag 27 | 28 | default_options = {"decoders": {"_": rh.NoneHandler, 29 | ":": rh.KeywordHandler, 30 | "$": rh.SymbolHandler, 31 | "?": rh.BooleanHandler, 32 | "i": rh.IntHandler, 33 | "d": rh.FloatHandler, 34 | "f": rh.BigDecimalHandler, 35 | "u": rh.UuidHandler, 36 | "r": rh.UriHandler, 37 | "t": rh.DateHandler, 38 | "m": rh.DateHandler, 39 | "n": rh.BigIntegerHandler, 40 | "z": rh.SpecialNumbersHandler, 41 | "link": rh.LinkHandler, 42 | "list": rh.ListHandler, 43 | "set": rh.SetHandler, 44 | "cmap": rh.CmapHandler, 45 | "'": rh.IdentityHandler}, 46 | "default_decoder": rh.DefaultHandler} 47 | 48 | ground_decoders = {"_": rh.NoneHandler, 49 | "?": rh.BooleanHandler, 50 | "i": rh.IntHandler, 51 | "'": rh.IdentityHandler} 52 | 53 | 54 | class Decoder(object): 55 | """The Decoder is the lowest level entry point for parsing, decoding, and 56 | fully converting Transit data into Python objects. 57 | 58 | During the creation of a Decoder object, you can specify custom options 59 | in a dictionary. One such option is 'decoders'. Note that while you 60 | can specify your own decoders and override many of the built in decoders, 61 | some decoders are silently enforced and cannot be overriden. These are 62 | known as Ground Decoders, and are needed to maintain bottom-tier 63 | compatibility. 64 | """ 65 | def __init__(self, options={}): 66 | self.options = default_options.copy() 67 | self.options.update(options) 68 | 69 | self.decoders = self.options["decoders"] 70 | # Always ensure we control the ground decoders 71 | self.decoders.update(ground_decoders) 72 | 73 | def decode(self, node, cache=None, as_map_key=False): 74 | """Given a node of data (any supported decodeable obj - string, dict, 75 | list), return the decoded object. Optionally set the current decode 76 | cache [None]. If None, a new RollingCache is instantiated and used. 77 | You may also hit to the decoder that this node is to be treated as a 78 | map key [False]. This is used internally. 79 | """ 80 | if not cache: 81 | cache = RollingCache() 82 | return self._decode(node, cache, as_map_key) 83 | 84 | def _decode(self, node, cache, as_map_key): 85 | tp = type(node) 86 | if tp is pyversion.unicode_type: 87 | return self.decode_string(node, cache, as_map_key) 88 | elif tp is bytes: 89 | return self.decode_string(node.decode("utf-8"), cache, as_map_key) 90 | elif tp is dict or tp is OrderedDict: 91 | return self.decode_hash(node, cache, as_map_key) 92 | elif tp is list: 93 | return self.decode_list(node, cache, as_map_key) 94 | elif tp is str: 95 | return self.decode_string(unicode(node, "utf-8"), cache, as_map_key) 96 | elif tp is bool: 97 | return true if node else false 98 | return node 99 | 100 | def decode_list(self, node, cache, as_map_key): 101 | """Special case decodes map-as-array. 102 | Otherwise lists are treated as Python lists. 103 | 104 | Arguments follow the same convention as the top-level 'decode' 105 | function. 106 | """ 107 | if node: 108 | if node[0] == MAP_AS_ARR: 109 | # key must be decoded before value for caching to work. 110 | returned_dict = {} 111 | for k, v in pairs(node[1:]): 112 | key = self._decode(k, cache, True) 113 | val = self._decode(v, cache, as_map_key) 114 | returned_dict[key] = val 115 | return transit_types.frozendict(returned_dict) 116 | 117 | decoded = self._decode(node[0], cache, as_map_key) 118 | if isinstance(decoded, Tag): 119 | return self.decode_tag(decoded.tag, 120 | self._decode(node[1], cache, as_map_key)) 121 | return tuple(self._decode(x, cache, as_map_key) for x in node) 122 | 123 | def decode_string(self, string, cache, as_map_key): 124 | """Decode a string - arguments follow the same convention as the 125 | top-level 'decode' function. 126 | """ 127 | if is_cache_key(string): 128 | return self.parse_string(cache.decode(string, as_map_key), 129 | cache, as_map_key) 130 | if is_cacheable(string, as_map_key): 131 | cache.encode(string, as_map_key) 132 | return self.parse_string(string, cache, as_map_key) 133 | 134 | def decode_tag(self, tag, rep): 135 | decoder = self.decoders.get(tag, None) 136 | if decoder: 137 | return decoder.from_rep(rep) 138 | else: 139 | return self.options["default_decoder"].from_rep(tag, rep) 140 | 141 | def decode_hash(self, hash, cache, as_map_key): 142 | if len(hash) != 1: 143 | h = {} 144 | for k, v in hash.items(): 145 | # crude/verbose implementation, but this is only version that 146 | # plays nice w/cache for both msgpack and json thus far. 147 | # -- e.g., we have to specify encode/decode order for key/val 148 | # -- explicitly, all implicit ordering has broken in corner 149 | # -- cases, thus these extraneous seeming assignments 150 | key = self._decode(k, cache, True) 151 | val = self._decode(v, cache, False) 152 | h[key] = val 153 | return transit_types.frozendict(h) 154 | else: 155 | key = list(hash)[0] 156 | value = hash[key] 157 | key = self._decode(key, cache, True) 158 | if isinstance(key, Tag): 159 | return self.decode_tag(key.tag, 160 | self._decode(value, cache, as_map_key)) 161 | return transit_types.frozendict({key: self._decode(value, cache, False)}) 162 | 163 | def parse_string(self, string, cache, as_map_key): 164 | if string.startswith(ESC): 165 | m = string[1] 166 | if m in self.decoders: 167 | return self.decoders[m].from_rep(string[2:]) 168 | elif m == ESC or m == SUB or m == RES: 169 | return string[1:] 170 | elif m == "#": 171 | return Tag(string[2:]) 172 | else: 173 | return self.options["default_decoder"].from_rep(string[1], 174 | string[2:]) 175 | return string 176 | 177 | def register(self, key_or_tag, obj): 178 | """Register a custom Transit tag and new parsing function with the 179 | decoder. Also, you can optionally set the 'default_decoder' with 180 | this function. Your new tag and parse/decode function will be added 181 | to the interal dictionary of decoders for this Decoder object. 182 | """ 183 | if key_or_tag == "default_decoder": 184 | self.options["default_decoder"] = obj 185 | else: 186 | self.decoders[key_or_tag] = obj 187 | -------------------------------------------------------------------------------- /transit/helpers.py: -------------------------------------------------------------------------------- 1 | ## Copyright 2014 Cognitect. All Rights Reserved. 2 | ## 3 | ## Licensed under the Apache License, Version 2.0 (the "License"); 4 | ## you may not use this file except in compliance with the License. 5 | ## You may obtain a copy of the License at 6 | ## 7 | ## http://www.apache.org/licenses/LICENSE-2.0 8 | ## 9 | ## Unless required by applicable law or agreed to in writing, software 10 | ## distributed under the License is distributed on an "AS-IS" BASIS, 11 | ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | ## See the License for the specific language governing permissions and 13 | ## limitations under the License. 14 | 15 | import itertools 16 | from transit.pyversion import imap, izip 17 | 18 | 19 | def mapcat(f, i): 20 | return itertools.chain.from_iterable(imap(f, i)) 21 | 22 | 23 | def pairs(i): 24 | return izip(*[iter(i)] * 2) 25 | 26 | 27 | cycle = itertools.cycle 28 | 29 | 30 | def take(n, i): 31 | return itertools.islice(i, 0, n) 32 | -------------------------------------------------------------------------------- /transit/pyversion.py: -------------------------------------------------------------------------------- 1 | ## Copyright 2016 Cognitect. All Rights Reserved. 2 | ## 3 | ## Licensed under the Apache License, Version 2.0 (the "License"); 4 | ## you may not use this file except in compliance with the License. 5 | ## You may obtain a copy of the License at 6 | ## 7 | ## http://www.apache.org/licenses/LICENSE-2.0 8 | ## 9 | ## Unless required by applicable law or agreed to in writing, software 10 | ## distributed under the License is distributed on an "AS-IS" BASIS, 11 | ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | ## See the License for the specific language governing permissions and 13 | ## limitations under the License. 14 | 15 | import sys 16 | 17 | PY3 = sys.version_info[0] == 3 18 | PY2 = sys.version_info[0] == 2 19 | 20 | if PY3: 21 | import collections.abc 22 | abc = collections.abc 23 | else: 24 | import collections 25 | abc = collections 26 | 27 | if PY3: 28 | string_types = (str,) 29 | unicode_type = str 30 | unicode_f = chr 31 | else: 32 | string_types = (str, unicode) 33 | unicode_type = unicode 34 | unicode_f = unichr 35 | 36 | if PY3: 37 | long_type = int 38 | int_type = int 39 | int_types = (int,) 40 | number_types = (int, float) 41 | else: 42 | long_type = long 43 | int_type = int 44 | int_types = (long, int) 45 | number_types = (long, int, float) 46 | 47 | def isnumber_type(t): 48 | return t in number_types 49 | 50 | if PY3: 51 | imap = map 52 | izip = zip 53 | def iteritems(d): 54 | return d.items() 55 | else: 56 | import itertools 57 | imap = itertools.imap 58 | izip = itertools.izip 59 | def iteritems(d): 60 | return d.iteritems() 61 | -------------------------------------------------------------------------------- /transit/read_handlers.py: -------------------------------------------------------------------------------- 1 | ## Copyright 2014 Cognitect. All Rights Reserved. 2 | ## 3 | ## Licensed under the Apache License, Version 2.0 (the "License"); 4 | ## you may not use this file except in compliance with the License. 5 | ## You may obtain a copy of the License at 6 | ## 7 | ## http://www.apache.org/licenses/LICENSE-2.0 8 | ## 9 | ## Unless required by applicable law or agreed to in writing, software 10 | ## distributed under the License is distributed on an "AS-IS" BASIS, 11 | ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | ## See the License for the specific language governing permissions and 13 | ## limitations under the License. 14 | 15 | from transit import pyversion, transit_types 16 | import uuid 17 | import ctypes 18 | import dateutil.parser 19 | import datetime 20 | import dateutil.tz 21 | from transit.helpers import pairs 22 | from decimal import Decimal 23 | 24 | ## Read handlers are used by the decoder when parsing/reading in Transit 25 | ## data and returning Python objects 26 | 27 | 28 | class DefaultHandler(object): 29 | @staticmethod 30 | def from_rep(t, v): 31 | return transit_types.TaggedValue(t, v) 32 | 33 | 34 | class NoneHandler(object): 35 | @staticmethod 36 | def from_rep(_): 37 | return None 38 | 39 | 40 | class KeywordHandler(object): 41 | @staticmethod 42 | def from_rep(v): 43 | return transit_types.Keyword(v) 44 | 45 | 46 | class SymbolHandler(object): 47 | @staticmethod 48 | def from_rep(v): 49 | return transit_types.Symbol(v) 50 | 51 | 52 | class BigDecimalHandler(object): 53 | @staticmethod 54 | def from_rep(v): 55 | return Decimal(v) 56 | 57 | 58 | class BooleanHandler(object): 59 | @staticmethod 60 | def from_rep(x): 61 | return transit_types.true if x == "t" else transit_types.false 62 | 63 | 64 | class IntHandler(object): 65 | @staticmethod 66 | def from_rep(v): 67 | return int(v) 68 | 69 | 70 | class FloatHandler(object): 71 | @staticmethod 72 | def from_rep(v): 73 | return float(v) 74 | 75 | 76 | class UuidHandler(object): 77 | @staticmethod 78 | def from_rep(u): 79 | """Given a string, return a UUID object.""" 80 | if isinstance(u, pyversion.string_types): 81 | return uuid.UUID(u) 82 | 83 | # hack to remove signs 84 | a = ctypes.c_ulong(u[0]) 85 | b = ctypes.c_ulong(u[1]) 86 | combined = a.value << 64 | b.value 87 | return uuid.UUID(int=combined) 88 | 89 | 90 | class UriHandler(object): 91 | @staticmethod 92 | def from_rep(u): 93 | return transit_types.URI(u) 94 | 95 | 96 | class DateHandler(object): 97 | @staticmethod 98 | def from_rep(d): 99 | if isinstance(d, pyversion.int_types): 100 | return DateHandler._convert_timestamp(d) 101 | if "T" in d: 102 | return dateutil.parser.parse(d) 103 | return DateHandler._convert_timestamp(pyversion.long_type(d)) 104 | 105 | @staticmethod 106 | def _convert_timestamp(ms): 107 | """Given a timestamp in ms, return a DateTime object.""" 108 | return datetime.datetime.fromtimestamp(ms/1000.0, dateutil.tz.tzutc()) 109 | 110 | 111 | if pyversion.PY3: 112 | class BigIntegerHandler(object): 113 | @staticmethod 114 | def from_rep(d): 115 | return int(d) 116 | else: 117 | class BigIntegerHandler(object): 118 | @staticmethod 119 | def from_rep(d): 120 | return long(d) 121 | 122 | 123 | class LinkHandler(object): 124 | @staticmethod 125 | def from_rep(l): 126 | return transit_types.Link(**l) 127 | 128 | 129 | class ListHandler(object): 130 | @staticmethod 131 | def from_rep(l): 132 | return l 133 | 134 | 135 | class SetHandler(object): 136 | @staticmethod 137 | def from_rep(s): 138 | return frozenset(s) 139 | 140 | 141 | class CmapHandler(object): 142 | @staticmethod 143 | def from_rep(cmap): 144 | return transit_types.frozendict(pairs(cmap)) 145 | 146 | 147 | class IdentityHandler(object): 148 | @staticmethod 149 | def from_rep(i): 150 | return i 151 | 152 | 153 | class SpecialNumbersHandler(object): 154 | @staticmethod 155 | def from_rep(z): 156 | if z == 'NaN': 157 | return float('Nan') 158 | if z == 'INF': 159 | return float('Inf') 160 | if z == '-INF': 161 | return float('-Inf') 162 | raise ValueError("Don't know how to handle: " + str(z) + " as \"z\"") 163 | -------------------------------------------------------------------------------- /transit/reader.py: -------------------------------------------------------------------------------- 1 | ## Copyright 2014 Cognitect. All Rights Reserved. 2 | ## 3 | ## Licensed under the Apache License, Version 2.0 (the "License"); 4 | ## you may not use this file except in compliance with the License. 5 | ## You may obtain a copy of the License at 6 | ## 7 | ## http://www.apache.org/licenses/LICENSE-2.0 8 | ## 9 | ## Unless required by applicable law or agreed to in writing, software 10 | ## distributed under the License is distributed on an "AS-IS" BASIS, 11 | ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | ## See the License for the specific language governing permissions and 13 | ## limitations under the License. 14 | 15 | import json 16 | import msgpack 17 | from collections import OrderedDict 18 | from transit import sosjson 19 | from transit.decoder import Decoder 20 | 21 | 22 | class Reader(object): 23 | """The top-level object for reading in Transit data and converting it to 24 | Python objects. During initialization, you must specify the protocol used 25 | for unmarshalling the data- json or msgpack. 26 | """ 27 | def __init__(self, protocol="json"): 28 | if protocol in ("json", "json_verbose"): 29 | self.reader = JsonUnmarshaler() 30 | elif protocol == "msgpack": 31 | self.reader = MsgPackUnmarshaler() 32 | self.unpacker = self.reader.unpacker 33 | else: 34 | raise ValueError("'" + protocol + "' is not a supported. " + 35 | "Protocol must be:" + 36 | "'json', 'json_verbose', or 'msgpack'.") 37 | 38 | def read(self, stream): 39 | """Given a readable file descriptor object (something `load`able by 40 | msgpack or json), read the data, and return the Python representation 41 | of the contents. One-shot reader. 42 | """ 43 | return self.reader.load(stream) 44 | 45 | def register(self, key_or_tag, f_val): 46 | """Register a custom transit tag and decoder/parser function for use 47 | during reads. 48 | """ 49 | self.reader.decoder.register(key_or_tag, f_val) 50 | 51 | def readeach(self, stream, **kwargs): 52 | """Temporary hook for API while streaming reads are in experimental 53 | phase. Read each object from stream as available with generator. 54 | JSON blocks indefinitely waiting on JSON entities to arrive. MsgPack 55 | requires unpacker property to be fed stream using unpacker.feed() 56 | method. 57 | """ 58 | for o in self.reader.loadeach(stream): 59 | yield o 60 | 61 | 62 | class JsonUnmarshaler(object): 63 | """The top-level Unmarshaler used by the Reader for JSON payloads. While 64 | you may use this directly, it is strongly discouraged. 65 | """ 66 | def __init__(self): 67 | self.decoder = Decoder() 68 | 69 | def load(self, stream): 70 | return self.decoder.decode(json.load(stream, 71 | object_pairs_hook=OrderedDict)) 72 | 73 | def loadeach(self, stream): 74 | for o in sosjson.items(stream, object_pairs_hook=OrderedDict): 75 | yield self.decoder.decode(o) 76 | 77 | 78 | class MsgPackUnmarshaler(object): 79 | """The top-level Unmarshaler used by the Reader for MsgPack payloads. 80 | While you may use this directly, it is strongly discouraged. 81 | """ 82 | def __init__(self): 83 | self.decoder = Decoder() 84 | self.unpacker = msgpack.Unpacker(object_pairs_hook=OrderedDict) 85 | 86 | def load(self, stream): 87 | return self.decoder.decode(msgpack.load(stream, 88 | object_pairs_hook=OrderedDict)) 89 | 90 | def loadeach(self, stream): 91 | for o in self.unpacker: 92 | yield self.decoder.decode(o) 93 | -------------------------------------------------------------------------------- /transit/rolling_cache.py: -------------------------------------------------------------------------------- 1 | ## Copyright 2014 Cognitect. All Rights Reserved. 2 | ## 3 | ## Licensed under the Apache License, Version 2.0 (the "License"); 4 | ## you may not use this file except in compliance with the License. 5 | ## You may obtain a copy of the License at 6 | ## 7 | ## http://www.apache.org/licenses/LICENSE-2.0 8 | ## 9 | ## Unless required by applicable law or agreed to in writing, software 10 | ## distributed under the License is distributed on an "AS-IS" BASIS, 11 | ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | ## See the License for the specific language governing permissions and 13 | ## limitations under the License. 14 | 15 | from transit.constants import SUB, MAP_AS_ARR 16 | 17 | FIRST_ORD = 48 18 | CACHE_CODE_DIGITS = 44 19 | CACHE_SIZE = CACHE_CODE_DIGITS * CACHE_CODE_DIGITS 20 | MIN_SIZE_CACHEABLE = 4 21 | 22 | 23 | def is_cache_key(name): 24 | return len(name) and (name[0] == SUB and name != MAP_AS_ARR) 25 | 26 | 27 | def encode_key(i): 28 | lo = i % CACHE_CODE_DIGITS 29 | hi = i // CACHE_CODE_DIGITS 30 | if hi == 0: 31 | return "^" + chr(lo + FIRST_ORD) 32 | return "^" + chr(hi + FIRST_ORD) + chr(lo + FIRST_ORD) 33 | 34 | 35 | def decode_key(s): 36 | sz = len(s) 37 | if sz == 2: 38 | return ord(s[1]) - FIRST_ORD 39 | return (ord(s[2]) - FIRST_ORD) + \ 40 | (CACHE_CODE_DIGITS * (ord(s[1]) - FIRST_ORD)) 41 | 42 | 43 | def is_cacheable(string, as_map_key=False): 44 | return string and len(string) >= MIN_SIZE_CACHEABLE \ 45 | and (as_map_key \ 46 | or (string[:2] in ["~#", "~$", "~:"])) 47 | 48 | 49 | class RollingCache(object): 50 | """This is the internal cache used by python-transit for cacheing and 51 | expanding map keys during writing and reading. The cache enables transit 52 | to minimize the amount of duplicate data sent over the wire, effectively 53 | compressing down the overall payload size. The cache is not intended to 54 | be used directly. 55 | """ 56 | def __init__(self): 57 | self.key_to_value = {} 58 | self.value_to_key = {} 59 | 60 | # if index rolls over... (bug) 61 | def decode(self, name, as_map_key=False): 62 | """Always returns the name""" 63 | if is_cache_key(name) and (name in self.key_to_value): 64 | return self.key_to_value[name] 65 | return self.encache(name) if is_cacheable(name, as_map_key) else name 66 | 67 | def encode(self, name, as_map_key=False): 68 | """Returns the name the first time and the key after that""" 69 | if name in self.key_to_value: 70 | return self.key_to_value[name] 71 | return self.encache(name) if is_cacheable(name, as_map_key) else name 72 | 73 | def size(self): 74 | return len(self.key_to_value) 75 | 76 | def is_cache_full(self): 77 | return len(self.key_to_value) > CACHE_SIZE 78 | 79 | def encache(self, name): 80 | if self.is_cache_full(): 81 | self.clear() 82 | elif name in self.value_to_key: 83 | return self.value_to_key[name] 84 | 85 | key = encode_key(len(self.key_to_value)) 86 | self.key_to_value[key] = name 87 | self.value_to_key[name] = key 88 | 89 | return name 90 | 91 | def clear(self): 92 | self.value_to_key = {} 93 | -------------------------------------------------------------------------------- /transit/sosjson.py: -------------------------------------------------------------------------------- 1 | ## copyright 2014 cognitect. all rights reserved. 2 | ## 3 | ## licensed under the apache license, version 2.0 (the "license"); 4 | ## you may not use this file except in compliance with the license. 5 | ## you may obtain a copy of the license at 6 | ## 7 | ## http://www.apache.org/licenses/license-2.0 8 | ## 9 | ## unless required by applicable law or agreed to in writing, software 10 | ## distributed under the license is distributed on an "as-is" basis, 11 | ## without warranties or conditions of any kind, either express or implied. 12 | ## see the license for the specific language governing permissions and 13 | ## limitations under the license. 14 | # Simple object streaming in Python - just reads one complete JSON object 15 | # at a time and returns json.loads of that string. 16 | 17 | # Ugly implementation at moment 18 | from copy import copy 19 | import json 20 | 21 | SKIP = [" ", "\n", "\t"] 22 | ESCAPE = "\\" 23 | 24 | 25 | def read_chunk(stream): 26 | """Ignore whitespace outside of strings. If we hit a string, read it in 27 | its entirety. 28 | """ 29 | chunk = stream.read(1) 30 | while chunk in SKIP: 31 | chunk = stream.read(1) 32 | if chunk == "\"": 33 | chunk += stream.read(1) 34 | while not chunk.endswith("\""): 35 | if chunk[-1] == ESCAPE: 36 | chunk += stream.read(2) 37 | else: 38 | chunk += stream.read(1) 39 | return chunk 40 | 41 | 42 | def items(stream, **kwargs): 43 | """External facing items. Will return item from stream as available. 44 | Currently waits in loop waiting for next item. Can pass keywords that 45 | json.loads accepts (such as object_pairs_hook) 46 | """ 47 | for s in yield_json(stream): 48 | yield json.loads(s, **kwargs) 49 | 50 | 51 | def yield_json(stream): 52 | """Uses array and object delimiter counts for balancing. 53 | """ 54 | buff = u"" 55 | arr_count = 0 56 | obj_count = 0 57 | while True: 58 | buff += read_chunk(stream) 59 | 60 | # If we finish parsing all objs or arrays, yield a finished JSON 61 | # entity. 62 | if buff.endswith('{'): 63 | obj_count += 1 64 | if buff.endswith('['): 65 | arr_count += 1 66 | if buff.endswith(']'): 67 | arr_count -= 1 68 | if obj_count == arr_count == 0: 69 | json_item = copy(buff) 70 | buff = u"" 71 | yield json_item 72 | if buff.endswith('}'): 73 | obj_count -= 1 74 | if obj_count == arr_count == 0: 75 | json_item = copy(buff) 76 | buff = u"" 77 | yield json_item 78 | -------------------------------------------------------------------------------- /transit/transit_types.py: -------------------------------------------------------------------------------- 1 | ## Copyright 2014 Cognitect. All Rights Reserved. 2 | ## 3 | ## Licensed under the Apache License, Version 2.0 (the "License"); 4 | ## you may not use this file except in compliance with the License. 5 | ## You may obtain a copy of the License at 6 | ## 7 | ## http://www.apache.org/licenses/LICENSE-2.0 8 | ## 9 | ## Unless required by applicable law or agreed to in writing, software 10 | ## distributed under the License is distributed on an "AS-IS" BASIS, 11 | ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | ## See the License for the specific language governing permissions and 13 | ## limitations under the License. 14 | from transit import pyversion 15 | Mapping = pyversion.abc.Mapping 16 | Hashable = pyversion.abc.Hashable 17 | 18 | from transit.pyversion import string_types, unicode_f, unicode_type 19 | 20 | 21 | class Named(object): 22 | def _parse(self): 23 | p = self.str.split('/', 1) 24 | if len(p) == 1: 25 | self._name = self.str 26 | self._namespace = None 27 | else: 28 | self._namespace = p[0] or None 29 | self._name = p[1] or "/" 30 | return self._name, self._namespace 31 | 32 | @property 33 | def name(self): 34 | return self._name if hasattr(self, "_name") else self._parse()[0] 35 | 36 | @property 37 | def namespace(self): 38 | return self._namespace if hasattr(self, "_namespace") \ 39 | else self._parse()[1] 40 | 41 | 42 | class Keyword(Named): 43 | def __init__(self, value): 44 | assert isinstance(value, string_types) 45 | self.str = value 46 | self.hv = value.__hash__() 47 | 48 | def __hash__(self): 49 | return self.hv 50 | 51 | def __eq__(self, other): 52 | return isinstance(other, Keyword) and self.str == other.str 53 | 54 | def __ne__(self, other): 55 | return not self == other 56 | 57 | def __call__(self, mp): 58 | return mp[self] 59 | 60 | def __repr__(self): 61 | return "" 62 | 63 | def __str__(self): 64 | return self.str 65 | 66 | 67 | class Symbol(Named): 68 | def __init__(self, value): 69 | assert isinstance(value, string_types) 70 | self.str = value 71 | self.hv = value.__hash__() 72 | 73 | def __hash__(self): 74 | return self.hv 75 | 76 | def __eq__(self, other): 77 | return isinstance(other, Symbol) and self.str == other.str 78 | 79 | def __ne__(self, other): 80 | return not self == other 81 | 82 | def __call__(self, mp): 83 | return mp[self] 84 | 85 | def __repr__(self): 86 | return self.str 87 | 88 | def __str__(self): 89 | return self.str 90 | 91 | kw_cache = {} 92 | 93 | 94 | class _KWS(object): 95 | def __getattr__(self, item): 96 | value = self(item) 97 | setattr(self, item, value) 98 | return value 99 | 100 | def __call__(self, str): 101 | if str in kw_cache: 102 | return kw_cache[str] 103 | else: 104 | kw_cache[str] = Keyword(str) 105 | return kw_cache[str] 106 | 107 | kws = _KWS() 108 | 109 | 110 | class TaggedValue(object): 111 | def __init__(self, tag, rep): 112 | self.tag = tag 113 | self.rep = rep 114 | 115 | def __eq__(self, other): 116 | if isinstance(other, TaggedValue): 117 | return self.tag == other.tag and \ 118 | self.rep == other.rep 119 | return False 120 | 121 | def __ne__(self, other): 122 | return not (self == other) 123 | 124 | def __hash__(self): 125 | if isinstance(self.rep, list): 126 | return reduce(lambda a, b: hash(a) ^ hash(b), self.rep, 0) 127 | return hash(self.rep) 128 | 129 | def __str__(self): 130 | return repr(self) 131 | 132 | def __repr__(self): 133 | return self.tag + " " + repr(self.rep) 134 | 135 | 136 | class Set(TaggedValue): 137 | def __init__(self, rep): 138 | TaggedValue.__init__(self, "set", rep) 139 | 140 | 141 | class CMap(TaggedValue): 142 | def __init__(self, rep): 143 | TaggedValue.__init__(self, "cmap", rep) 144 | 145 | 146 | class Vector(TaggedValue): 147 | def __init__(self, rep): 148 | TaggedValue.__init__(self, "vector", rep) 149 | 150 | 151 | class Array(TaggedValue): 152 | def __init__(self, rep): 153 | TaggedValue.__init__(self, "array", rep) 154 | 155 | 156 | class List(TaggedValue): 157 | def __init__(self, rep): 158 | TaggedValue.__init__(self, "list", rep) 159 | 160 | 161 | class URI(TaggedValue): 162 | def __init__(self, rep): 163 | # works p3 TaggedValue.__init__(self, "uri", (unicode(rep))) 164 | TaggedValue.__init__(self, "uri", (rep)) 165 | 166 | 167 | class frozendict(Mapping, Hashable): 168 | def __init__(self, *args, **kwargs): 169 | self._dict = dict(*args, **kwargs) 170 | 171 | def __len__(self): 172 | return len(self._dict) 173 | 174 | def __iter__(self): 175 | return iter(self._dict) 176 | 177 | def __getitem__(self, key): 178 | return self._dict[key] 179 | 180 | def __hash__(self): 181 | return hash(frozenset(self._dict.items())) 182 | 183 | def __repr__(self): 184 | return 'frozendict(%r)' % (self._dict,) 185 | 186 | 187 | class Link(object): 188 | # Class property constants for rendering types 189 | LINK = u"link" 190 | IMAGE = u"image" 191 | 192 | # Class property constants for keywords/obj properties. 193 | HREF = u"href" 194 | REL = u"rel" 195 | PROMPT = u"prompt" 196 | NAME = u"name" 197 | RENDER = u"render" 198 | 199 | def __init__(self, href=None, rel=None, name=None, render=None, 200 | prompt=None): 201 | self._dict = frozendict() 202 | assert href and rel 203 | if render: 204 | assert render.lower() in [Link.LINK, Link.IMAGE] 205 | self._dict = {Link.HREF: href, 206 | Link.REL: rel, 207 | Link.NAME: name, 208 | Link.RENDER: render, 209 | Link.PROMPT: prompt} 210 | 211 | def __eq__(self, other): 212 | return self._dict == other._dict 213 | 214 | def __ne__(self, other): 215 | return self._dict != other._dict 216 | 217 | @property 218 | def href(self): 219 | return self._dict[Link.HREF] 220 | 221 | @property 222 | def rel(self): 223 | return self._dict[Link.REL] 224 | 225 | @property 226 | def prompt(self): 227 | return self._dict[Link.PROMPT] 228 | 229 | @property 230 | def name(self): 231 | return self._dict[Link.NAME] 232 | 233 | @property 234 | def render(self): 235 | return self._dict[Link.RENDER] 236 | 237 | @property 238 | def as_map(self): 239 | return self._dict 240 | 241 | @property 242 | def as_array(self): 243 | return [self.href, self.rel, self.name, self.render, self.prompt] 244 | 245 | 246 | class Boolean(object): 247 | """To allow a separate t/f that won't hash as 1/0. Don't call directly, 248 | instead use true and false as singleton objects. Can use with type check. 249 | 250 | Note that the Booleans are for preserving hash/set bools that duplicate 1/0 251 | and not designed for use in Python outside logical evaluation (don't treat 252 | as an int, they're not). You can get a Python bool using bool(x) 253 | where x is a true or false Boolean. 254 | """ 255 | def __init__(self, name): 256 | self.v = True if name == "true" else False 257 | self.name = name 258 | 259 | def __bool__(self): 260 | return self.v 261 | 262 | def __nonzero__(self): 263 | return self.v 264 | 265 | def __repr__(self): 266 | return self.name 267 | 268 | def __str__(self): 269 | return self.name 270 | 271 | # lowercase rep matches java/clojure 272 | 273 | false = Boolean("false") 274 | true = Boolean("true") 275 | -------------------------------------------------------------------------------- /transit/write_handlers.py: -------------------------------------------------------------------------------- 1 | ## Copyright 2014 Cognitect. All Rights Reserved. 2 | ## 3 | ## Licensed under the Apache License, Version 2.0 (the "License"); 4 | ## you may not use this file except in compliance with the License. 5 | ## You may obtain a copy of the License at 6 | ## 7 | ## http://www.apache.org/licenses/LICENSE-2.0 8 | ## 9 | ## Unless required by applicable law or agreed to in writing, software 10 | ## distributed under the License is distributed on an "AS-IS" BASIS, 11 | ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | ## See the License for the specific language governing permissions and 13 | ## limitations under the License. 14 | 15 | import uuid 16 | import datetime 17 | import struct 18 | from transit import pyversion 19 | from transit.class_hash import ClassDict 20 | from transit.transit_types import Keyword, Symbol, URI, frozendict, TaggedValue, Link, Boolean 21 | from decimal import Decimal 22 | from dateutil import tz 23 | from math import isnan 24 | 25 | MAX_INT = 2**63 - 1 26 | MIN_INT = -2**63 27 | 28 | ## This file contains Write Handlers - all the top-level objects used when 29 | ## writing Transit data. These object must all be immutable and pickleable. 30 | 31 | 32 | class TaggedMap(object): 33 | def __init__(self, tag, rep, str): 34 | self._tag = tag 35 | self._rep = rep 36 | self._str = str 37 | 38 | def tag(self): 39 | return self._tag 40 | 41 | def rep(self): 42 | return self._rep 43 | 44 | def string_rep(self): 45 | return self._str 46 | 47 | 48 | class NoneHandler(object): 49 | @staticmethod 50 | def tag(_): 51 | return '_' 52 | 53 | @staticmethod 54 | def rep(_): 55 | return None 56 | 57 | @staticmethod 58 | def string_rep(n): 59 | return None 60 | 61 | 62 | class IntHandler(object): 63 | @staticmethod 64 | def tag(i): 65 | return 'i' 66 | 67 | @staticmethod 68 | def rep(i): 69 | return i 70 | 71 | @staticmethod 72 | def string_rep(i): 73 | return str(i) 74 | 75 | 76 | class BigIntHandler(object): 77 | @staticmethod 78 | def tag(_): 79 | return "n" 80 | 81 | @staticmethod 82 | def rep(n): 83 | return str(n) 84 | 85 | @staticmethod 86 | def string_rep(n): 87 | return str(n) 88 | 89 | class Python3IntHandler(object): 90 | @staticmethod 91 | def tag(n): 92 | if n < MAX_INT and n > MIN_INT: 93 | return "i" 94 | return "n" 95 | 96 | @staticmethod 97 | def rep(n): 98 | return n 99 | 100 | @staticmethod 101 | def string_rep(n): 102 | return str(n) 103 | 104 | 105 | class BigDecimalHandler(object): 106 | @staticmethod 107 | def tag(_): 108 | return "f" 109 | 110 | @staticmethod 111 | def rep(n): 112 | return str(n) 113 | 114 | @staticmethod 115 | def string_rep(n): 116 | return str(n) 117 | 118 | 119 | class FloatHandler(object): 120 | @staticmethod 121 | def tag(f): 122 | return "z" if isnan(f) or f in (float('Inf'), float('-Inf')) else "d" 123 | 124 | @staticmethod 125 | def rep(f): 126 | if isnan(f): 127 | return "NaN" 128 | if f == float('Inf'): 129 | return "INF" 130 | if f == float("-Inf"): 131 | return "-INF" 132 | return f 133 | 134 | @staticmethod 135 | def string_rep(f): 136 | return str(f) 137 | 138 | 139 | class StringHandler(object): 140 | @staticmethod 141 | def tag(s): 142 | return 's' 143 | 144 | @staticmethod 145 | def rep(s): 146 | return s 147 | 148 | @staticmethod 149 | def string_rep(s): 150 | return s 151 | 152 | 153 | class BooleanHandler(object): 154 | @staticmethod 155 | def tag(_): 156 | return '?' 157 | 158 | @staticmethod 159 | def rep(b): 160 | return bool(b) 161 | 162 | @staticmethod 163 | def string_rep(b): 164 | return 't' if b else 'f' 165 | 166 | 167 | class ArrayHandler(object): 168 | @staticmethod 169 | def tag(a): 170 | return 'array' 171 | 172 | @staticmethod 173 | def rep(a): 174 | return a 175 | 176 | @staticmethod 177 | def string_rep(a): 178 | return None 179 | 180 | 181 | class MapHandler(object): 182 | @staticmethod 183 | def tag(m): 184 | return 'map' 185 | 186 | @staticmethod 187 | def rep(m): 188 | return m 189 | 190 | @staticmethod 191 | def string_rep(m): 192 | return None 193 | 194 | 195 | class KeywordHandler(object): 196 | @staticmethod 197 | def tag(k): 198 | return ':' 199 | 200 | @staticmethod 201 | def rep(k): 202 | return str(k) 203 | 204 | @staticmethod 205 | def string_rep(k): 206 | return str(k) 207 | 208 | 209 | class SymbolHandler(object): 210 | @staticmethod 211 | def tag(s): 212 | return '$' 213 | 214 | @staticmethod 215 | def rep(s): 216 | return str(s) 217 | 218 | @staticmethod 219 | def string_rep(s): 220 | return str(s) 221 | 222 | 223 | class UuidHandler(object): 224 | @staticmethod 225 | def tag(_): 226 | return "u" 227 | 228 | @staticmethod 229 | def rep(u): 230 | return struct.unpack('>qq', u.bytes) 231 | 232 | @staticmethod 233 | def string_rep(u): 234 | return str(u) 235 | 236 | 237 | class UriHandler(object): 238 | @staticmethod 239 | def tag(_): 240 | return "r" 241 | 242 | @staticmethod 243 | def rep(u): 244 | return u.rep 245 | 246 | @staticmethod 247 | def string_rep(u): 248 | return u.rep 249 | 250 | 251 | class DateTimeHandler(object): 252 | epoch = datetime.datetime(1970, 1, 1).replace(tzinfo=tz.tzutc()) 253 | 254 | @staticmethod 255 | def tag(_): 256 | return "m" 257 | 258 | @staticmethod 259 | def rep(d): 260 | td = d - DateTimeHandler.epoch 261 | return int((td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / 1e3) 262 | 263 | @staticmethod 264 | def verbose_handler(): 265 | return VerboseDateTimeHandler 266 | 267 | @staticmethod 268 | def string_rep(d): 269 | return str(DateTimeHandler.rep(d)) 270 | 271 | 272 | class VerboseDateTimeHandler(object): 273 | @staticmethod 274 | def tag(_): 275 | return "t" 276 | 277 | @staticmethod 278 | def rep(d): 279 | return d.isoformat() 280 | 281 | @staticmethod 282 | def string_rep(d): 283 | return d.isoformat() 284 | 285 | 286 | class SetHandler(object): 287 | @staticmethod 288 | def tag(_): 289 | return "set" 290 | 291 | @staticmethod 292 | def rep(s): 293 | return TaggedMap("array", tuple(s), None) 294 | 295 | @staticmethod 296 | def string_rep(_): 297 | return None 298 | 299 | 300 | class TaggedValueHandler(object): 301 | @staticmethod 302 | def tag(tv): 303 | return tv.tag 304 | 305 | @staticmethod 306 | def rep(tv): 307 | return tv.rep 308 | 309 | @staticmethod 310 | def string_rep(_): 311 | return None 312 | 313 | 314 | class LinkHandler(object): 315 | @staticmethod 316 | def tag(_): 317 | return "link" 318 | 319 | @staticmethod 320 | def rep(l): 321 | return l.as_map 322 | 323 | @staticmethod 324 | def string_rep(_): 325 | return None 326 | 327 | 328 | class WriteHandler(ClassDict): 329 | """This is the master handler for encoding/writing Python data into 330 | Transit data, based on its type. 331 | The Handler itself is a dispatch map, that resolves on full type/object 332 | inheritance. 333 | 334 | These handlers can be overriden during the creation of a Transit Writer. 335 | """ 336 | 337 | def __init__(self): 338 | super(WriteHandler, self).__init__() 339 | self[type(None)] = NoneHandler 340 | self[bool] = BooleanHandler 341 | self[Boolean] = BooleanHandler 342 | self[str] = StringHandler 343 | self[pyversion.unicode_type] = StringHandler 344 | self[list] = ArrayHandler 345 | self[tuple] = ArrayHandler 346 | self[dict] = MapHandler 347 | 348 | if pyversion.PY3: 349 | self[int] = Python3IntHandler 350 | else: 351 | self[int] = IntHandler 352 | self[long] = BigIntHandler 353 | 354 | self[float] = FloatHandler 355 | self[Keyword] = KeywordHandler 356 | self[Symbol] = SymbolHandler 357 | self[uuid.UUID] = UuidHandler 358 | self[URI] = UriHandler 359 | self[datetime.datetime] = DateTimeHandler 360 | self[set] = SetHandler 361 | self[frozenset] = SetHandler 362 | self[TaggedMap] = TaggedMap 363 | self[dict] = MapHandler 364 | self[frozendict] = MapHandler 365 | self[TaggedValue] = TaggedValueHandler 366 | self[Link] = LinkHandler 367 | self[Decimal] = BigDecimalHandler 368 | -------------------------------------------------------------------------------- /transit/writer.py: -------------------------------------------------------------------------------- 1 | ## Copyright 2014 Cognitect. All Rights Reserved. 2 | ## 3 | ## Licensed under the Apache License, Version 2.0 (the "License"); 4 | ## you may not use this file except in compliance with the License. 5 | ## You may obtain a copy of the License at 6 | ## 7 | ## http://www.apache.org/licenses/LICENSE-2.0 8 | ## 9 | ## Unless required by applicable law or agreed to in writing, software 10 | ## distributed under the License is distributed on an "AS-IS" BASIS, 11 | ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | ## See the License for the specific language governing permissions and 13 | ## limitations under the License. 14 | 15 | import sys 16 | import msgpack 17 | import re 18 | from transit import pyversion 19 | from transit.constants import SUB, ESC, RES, MAP_AS_ARR, QUOTE 20 | from transit.rolling_cache import RollingCache 21 | from transit.write_handlers import WriteHandler 22 | from transit.transit_types import TaggedValue 23 | 24 | ESCAPE_DCT = { 25 | '\\': u'\\\\', 26 | '"': u'\\"', 27 | '\b': u'\\b', 28 | '\f': u'\\f', 29 | '\n': u'\\n', 30 | '\r': u'\\r', 31 | '\t': u'\\t', 32 | } 33 | for i in range(0x20): 34 | ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i)) 35 | 36 | 37 | 38 | class Writer(object): 39 | """The top-level object for writing out Python objects and converting them 40 | to Transit data. During initialization, you must specify the protocol used 41 | for marshalling the data- json or msgpack. You must also specify the io 42 | source used for writing (a file descriptor). You may optionally pass in 43 | an options dictionary that will be forwarded onto the Marshaler. 44 | The cache is enabled by default. 45 | """ 46 | def __init__(self, io, protocol="json", opts={"cache_enabled": True}): 47 | if protocol == "json": 48 | self.marshaler = JsonMarshaler(io, opts=opts) 49 | elif protocol == "json_verbose": 50 | self.marshaler = VerboseJsonMarshaler(io, opts=opts) 51 | elif protocol == "msgpack": 52 | self.marshaler = MsgPackMarshaler(io, opts=opts) 53 | else: 54 | raise ValueError("'" + protocol + "' is not a supported protocol. " + 55 | "Protocol must be 'json', 'json_verbose', or 'msgpack'.") 56 | 57 | def write(self, obj): 58 | """Given a Python object, marshal it into Transit data and write it to 59 | the 'io' source. 60 | """ 61 | self.marshaler.marshal_top(obj) 62 | 63 | def register(self, obj_type, handler_class): 64 | """Register custom converters for object types present in your 65 | application. This allows you to extend Transit to encode new types. 66 | You must specify the obj type to be encoded, and the handler class 67 | that should be used by the Marshaler during write-time. 68 | """ 69 | self.marshaler.register(obj_type, handler_class) 70 | 71 | 72 | def flatten_map(m): 73 | """Expand a dictionary's items into a flat list 74 | """ 75 | # This is the fastest way to do this in Python 76 | return [item for t in m.items() for item in t] 77 | 78 | 79 | def re_fn(pat): 80 | compiled = re.compile(pat) 81 | def re_inner_fn(value): 82 | return compiled.match(value) 83 | 84 | return re_inner_fn 85 | 86 | is_escapable = re_fn("^" + re.escape(SUB) + "|" + ESC + "|" + RES) 87 | 88 | 89 | def escape(s): 90 | if s is MAP_AS_ARR: 91 | return MAP_AS_ARR 92 | if is_escapable(s): 93 | return ESC+s 94 | else: 95 | return s 96 | 97 | 98 | class Marshaler(object): 99 | """The base Marshaler from which all Marshalers inherit. 100 | 101 | The Marshaler specifies how to emit Transit data given encodeable Python 102 | objects. The end of this process is specialized by other Marshalers to 103 | covert the final result into an on-the-wire payload (JSON or MsgPack). 104 | """ 105 | def __init__(self, opts={}): 106 | self.opts = opts 107 | self._init_handlers() 108 | 109 | def _init_handlers(self): 110 | self.handlers = WriteHandler() 111 | 112 | def are_stringable_keys(self, m): 113 | """Test whether the keys within a map are stringable - a simple map, 114 | that can be optimized and whose keys can be cached 115 | """ 116 | for x in m.keys(): 117 | if len(self.handlers[x].tag(x)) != 1: 118 | return False 119 | return True 120 | 121 | def emit_nil(self, _, as_map_key, cache): 122 | return self.emit_string(ESC, "_", "", True, cache) if as_map_key else self.emit_object(None) 123 | 124 | def emit_string(self, prefix, tag, string, as_map_key, cache): 125 | encoded = cache.encode(str(prefix)+tag+string, as_map_key) 126 | # TODO: Remove this optimization for the time being - it breaks cache 127 | #if "cache_enabled" in self.opts and is_cacheable(encoded, as_map_key): 128 | # return self.emit_object(cache.value_to_key[encoded], as_map_key) 129 | return self.emit_object(encoded, as_map_key) 130 | 131 | def emit_boolean(self, b, as_map_key, cache): 132 | return self.emit_string(ESC, "?", b, True, cache) if as_map_key else self.emit_object(b) 133 | 134 | def emit_int(self, tag, i, rep, as_map_key, cache): 135 | if isinstance(rep, int) and i <= self.opts["max_int"] and i >= self.opts["min_int"]: 136 | return self.emit_object(i, as_map_key) 137 | else: 138 | return self.emit_string(ESC, tag, str(rep), as_map_key, cache) 139 | 140 | def emit_double(self, d, as_map_key, cache): 141 | return self.emit_string(ESC, "d", d, True, cache) if as_map_key else self.emit_object(d) 142 | 143 | def emit_array(self, a, _, cache): 144 | self.emit_array_start(len(a)) 145 | for x in a: 146 | self.marshal(x, False, cache) 147 | self.emit_array_end() 148 | 149 | def emit_map(self, m, _, cache):# use map as object from above, have to overwrite default parser. 150 | self.emit_map_start(len(m)) 151 | for k, v in m.items(): 152 | self.marshal(k, True, cache) 153 | self.marshal(v, False, cache) 154 | self.emit_map_end() 155 | 156 | def emit_cmap(self, m, _, cache): 157 | self.emit_map_start(1) 158 | self.emit_string(ESC, "#", "cmap", True, cache) 159 | self.marshal(flatten_map(m), False, cache) 160 | self.emit_map_end() 161 | 162 | def emit_tagged(self, tag, rep, cache): 163 | self.emit_array_start(2) 164 | self.emit_string(ESC, "#", tag, False, cache) 165 | self.marshal(rep, False, cache) 166 | self.emit_array_end() 167 | 168 | def emit_encoded(self, tag, handler, obj, as_map_key, cache): 169 | rep = handler.rep(obj) 170 | if len(tag) == 1: 171 | if isinstance(rep, pyversion.string_types): 172 | self.emit_string(ESC, tag, rep, as_map_key, cache) 173 | elif as_map_key or self.opts["prefer_strings"]: 174 | rep = handler.string_rep(obj) 175 | if isinstance(rep, pyversion.string_types): 176 | self.emit_string(ESC, tag, rep, as_map_key, cache) 177 | else: 178 | raise AssertionError("Cannot be encoded as string: " + str({"tag": tag, 179 | "rep": rep, 180 | "obj": obj})) 181 | else: 182 | self.emit_tagged(tag, rep, cache) 183 | elif as_map_key: 184 | raise AssertionError("Cannot be used as a map key: " + str({"tag": tag, 185 | "rep": rep, 186 | "obj": obj})) 187 | else: 188 | self.emit_tagged(tag, rep, cache) 189 | 190 | def marshal(self, obj, as_map_key, cache): 191 | """Marshal an individual obj, potentially as part of another container 192 | object (like a list/dictionary/etc). Specify if this object is a key 193 | to a map/dict, and pass in the current cache being used. 194 | This method should only be called by a top-level marshalling call 195 | and should not be considered an entry-point for integration. 196 | """ 197 | handler = self.handlers[obj] 198 | tag = handler.tag(obj) 199 | f = marshal_dispatch.get(tag) 200 | 201 | if f: 202 | f(self, obj, handler.string_rep(obj) if as_map_key else handler.rep(obj), as_map_key, cache) 203 | else: 204 | self.emit_encoded(tag, handler, obj, as_map_key, cache) 205 | 206 | def marshal_top(self, obj, cache=None): 207 | """Given a complete object that needs to be marshaled into Transit 208 | data, and optionally a cache, dispatch accordingly, and flush the data 209 | directly into the IO stream. 210 | """ 211 | if not cache: 212 | cache = RollingCache() 213 | 214 | handler = self.handlers[obj] 215 | 216 | tag = handler.tag(obj) 217 | if tag: 218 | if len(tag) == 1: 219 | self.marshal(TaggedValue(QUOTE, obj), False, cache) 220 | else: 221 | self.marshal(obj, False, cache) 222 | self.flush() 223 | else: 224 | raise AssertionError("Handler must provide a non-nil tag: " + str(handler)) 225 | 226 | def dispatch_map(self, rep, as_map_key, cache): 227 | """Used to determine and dipatch the writing of a map - a simple 228 | map with strings as keys, or a complex map, whose keys are also 229 | compound types. 230 | """ 231 | if self.are_stringable_keys(rep): 232 | return self.emit_map(rep, as_map_key, cache) 233 | return self.emit_cmap(rep, as_map_key, cache) 234 | 235 | def register(self, obj_type, handler_class): 236 | """Register custom converters for object types present in your 237 | application. This allows you to extend Transit to encode new types. 238 | You must specify the obj type to be encoded, and the handler class 239 | that should be used by this marshaller. 240 | """ 241 | self.handlers[obj_type] = handler_class 242 | 243 | marshal_dispatch = {"_": lambda self, obj, rep, as_map_key, cache: self.emit_nil(rep, as_map_key, cache), 244 | "?": lambda self, obj, rep, as_map_key, cache: self.emit_boolean(rep, as_map_key, cache), 245 | "s": lambda self, obj, rep, as_map_key, cache: self.emit_string("", "", escape(rep), as_map_key, cache), 246 | "i": lambda self, i, rep, as_map_key, cache: self.emit_int("i", i, rep, as_map_key, cache), 247 | "n": lambda self, i, rep, as_map_key, cache: self.emit_int("n", i, rep, as_map_key, cache), 248 | "d": lambda self, obj, rep, as_map_key, cache: self.emit_double(rep, as_map_key, cache), 249 | "'": lambda self, obj, rep, _, cache: self.emit_tagged("'", rep, cache), 250 | "array": lambda self, obj, rep, as_map_key, cache: self.emit_array(rep, as_map_key, cache), 251 | "map": lambda self, obj, rep, as_map_key, cache: self.dispatch_map(rep, as_map_key, cache)} 252 | 253 | 254 | class MsgPackMarshaler(Marshaler): 255 | """The Marshaler tailor to MsgPack. To use this Marshaler, specify the 256 | 'msgpack' protocol when creating a Writer. 257 | """ 258 | MSGPACK_MAX_INT = pow(2, 63) - 1 259 | MSGPACK_MIN_INT = -pow(2, 63) 260 | 261 | default_opts = {"prefer_strings": False, 262 | "max_int": MSGPACK_MAX_INT, 263 | "min_int": MSGPACK_MIN_INT} 264 | 265 | def __init__(self, io, opts={}): 266 | self.io = io 267 | self.packer = msgpack.Packer(autoreset=False) 268 | nopts = MsgPackMarshaler.default_opts.copy() 269 | nopts.update(opts) 270 | Marshaler.__init__(self, nopts) 271 | 272 | def emit_array_start(self, size): 273 | self.packer.pack_array_header(size) 274 | 275 | def emit_array_end(self): 276 | pass 277 | 278 | def emit_map_start(self, size): 279 | self.packer.pack_map_header(size) 280 | 281 | def emit_map_end(self): 282 | pass 283 | 284 | def emit_object(self, obj, as_map_key=False): 285 | self.packer.pack(obj) 286 | 287 | def flush(self): 288 | self.io.write(self.packer.bytes()) 289 | self.io.flush() 290 | self.packer.reset() 291 | 292 | REPLACE_RE = re.compile("\"") 293 | 294 | 295 | class JsonMarshaler(Marshaler): 296 | """The Marshaler tailor to JSON. To use this Marshaler, specify the 297 | 'json' protocol when creating a Writer. 298 | """ 299 | JSON_MAX_INT = pow(2, 53) - 1 300 | JSON_MIN_INT = -pow(2, 53) + 1 301 | 302 | default_opts = {"prefer_strings": True, 303 | "max_int": JSON_MAX_INT, 304 | "min_int": JSON_MIN_INT} 305 | 306 | def __init__(self, io, opts={}): 307 | self.io = io 308 | nopts = JsonMarshaler.default_opts.copy() 309 | nopts.update(opts) 310 | self.started = [True] 311 | self.is_key = [None] 312 | Marshaler.__init__(self, nopts) 313 | self.flush = self.io.flush 314 | 315 | def push_level(self): 316 | self.started.append(True) 317 | self.is_key.append(None) 318 | 319 | def pop_level(self): 320 | self.started.pop() 321 | self.is_key.pop() 322 | 323 | def push_map(self): 324 | self.started.append(True) 325 | self.is_key.append(True) 326 | 327 | def write_sep(self): 328 | if self.started[-1]: 329 | self.started[-1] = False 330 | else: 331 | last = self.is_key[-1] 332 | if last: 333 | self.io.write(u":") 334 | self.is_key[-1] = False 335 | elif last is False: 336 | self.io.write(u",") 337 | self.is_key[-1] = True 338 | else: 339 | self.io.write(u",") 340 | 341 | def emit_array_start(self, size): 342 | self.write_sep() 343 | self.io.write(u"[") 344 | self.push_level() 345 | 346 | def emit_array_end(self): 347 | self.pop_level() 348 | self.io.write(u"]") 349 | 350 | def emit_map(self, m, _, cache): 351 | """Emits array as per default JSON spec.""" 352 | self.emit_array_start(None) 353 | self.marshal(MAP_AS_ARR, False, cache) 354 | for k, v in m.items(): 355 | self.marshal(k, True, cache) 356 | self.marshal(v, False, cache) 357 | self.emit_array_end() 358 | 359 | def emit_map_start(self, size): 360 | self.write_sep() 361 | self.io.write(u"{") 362 | self.push_map() 363 | 364 | def emit_map_end(self): 365 | self.pop_level() 366 | self.io.write(u"}") 367 | 368 | def emit_object(self, obj, as_map_key=False): 369 | tp = type(obj) 370 | self.write_sep() 371 | if tp in pyversion.string_types: 372 | self.io.write(u"\"") 373 | self.io.write(u"".join([(ESCAPE_DCT[c]) if c in ESCAPE_DCT else c for c in obj])) 374 | self.io.write(u"\"") 375 | elif pyversion.isnumber_type(tp): 376 | self.io.write(pyversion.unicode_type(obj)) 377 | elif tp is bool: 378 | self.io.write(u"true" if obj else u"false") 379 | elif obj is None: 380 | self.io.write(u"null") 381 | else: 382 | raise AssertionError("Don't know how to encode: " + str(obj) + " of type: " + str(type(obj))) 383 | 384 | 385 | class VerboseSettings(object): 386 | """Mixin for JsonMarshaler that adds support for Verbose output/input. 387 | Verbosity is only suggest for debuging/inspecting purposes. 388 | """ 389 | @staticmethod 390 | def _verbose_handlers(handlers): 391 | for k, v in pyversion.iteritems(handlers): 392 | if hasattr(v, "verbose_handler"): 393 | handlers[k] = v.verbose_handler() 394 | return handlers 395 | 396 | def _init_handlers(self): 397 | self.handlers = self._verbose_handlers(WriteHandler()) 398 | 399 | def emit_string(self, prefix, tag, string, as_map_key, cache): 400 | return self.emit_object(pyversion.unicode_type(prefix) + tag + string, as_map_key) 401 | 402 | def emit_map(self, m, _, cache): 403 | self.emit_map_start(len(m)) 404 | for k, v in m.items(): 405 | self.marshal(k, True, cache) 406 | self.marshal(v, False, cache) 407 | self.emit_map_end() 408 | 409 | def emit_tagged(self, tag, rep, cache): 410 | self.emit_map_start(1) 411 | self.emit_object(ESC + "#" + tag, True) 412 | self.marshal(rep, False, cache) 413 | self.emit_map_end() 414 | 415 | 416 | class VerboseJsonMarshaler(VerboseSettings, JsonMarshaler): 417 | """JsonMarshaler class with VerboseSettings mixin.""" 418 | pass # all from inheritance and mixin 419 | --------------------------------------------------------------------------------