├── .gitignore ├── AUTHORS ├── LICENSE ├── README.md ├── benchmark ├── README.md ├── benchmark.py └── person.proto ├── setup.py ├── src └── fastpb │ ├── __init__.py │ ├── generator.py │ ├── plugin_pb2.py │ ├── template │ ├── MANIFEST.jinjain │ ├── module.jinjacc │ ├── setup.jinjapy │ └── test.jinjapy │ └── util.py └── test └── namespace.proto /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[co] 2 | *.egg 3 | *.egg-info 4 | dist 5 | build 6 | *.iml 7 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Greplin 2 | acg 3 | olt 4 | bkad 5 | jparise 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | fast-python-pb: Fast Python Protocol Buffers 2 | ===================== 3 | 4 | Thin wrapper on top of the C++ protocol buffer implementation resulting in significantly faster protocol buffers in 5 | Python. 6 | 7 | 8 | ### Why: 9 | 10 | We wanted a fast implementation of protocol buffers that still felt like Python, hence this implementation. 11 | 12 | For our use case, this module is up to 15 times faster than the standard one and 10 times as fast as 13 | Python's json serializer. 14 | 15 | 16 | ### Status: 17 | 18 | This is a very early stage project. It works for our needs. We haven't verified it works beyond that. Issue reports 19 | and patches are very much appreciated! 20 | 21 | For example, it only supports strint, int32, int64, double, and sub message members at this time. 22 | 23 | 24 | ### Pre-requisites: 25 | 26 | Install [protocol buffers](http://code.google.com/p/protobuf/) 27 | 28 | 29 | ### Installation: 30 | 31 | git clone https://github.com/Cue/fast-python-pb.git 32 | 33 | cd fast-python-pb 34 | 35 | python setup.py install 36 | 37 | 38 | ### Usage: 39 | 40 | protoc --fastpython_out /output/path --cpp_out /output/path --proto_path your/path your/path/file.proto 41 | 42 | 43 | ### Example: 44 | 45 | You can see the example in action in the benchmark directory. 46 | 47 | // person.proto 48 | package person_proto; 49 | 50 | message Fact { 51 | required string name = 1; 52 | 53 | required string content = 2; 54 | } 55 | 56 | message Person { 57 | required string name = 1; 58 | 59 | required int32 birth_year = 2; 60 | 61 | repeated string nicknames = 3; 62 | 63 | repeated Fact facts = 4; 64 | } 65 | 66 | 67 | ```python 68 | # example.py 69 | import person_proto 70 | 71 | lincoln = person_proto.Person(name = 'Abraham Lincoln', birth_year = 1809) 72 | lincoln.nicknames = ['Honest Abe', 'Abe'] 73 | lincoln.facts = [ 74 | person_proto.Fact(name = 'Born In', content = 'Kentucky'), 75 | person_proto.Fact(name = 'Died In', content = 'Washington D.C.'), 76 | person_proto.Fact(name = 'Greatest Speech', content = GETTYSBURG) 77 | ] 78 | 79 | serializedLincoln = lincoln.SerializeToString() 80 | 81 | newLincoln = person_proto.Person() 82 | newLincoln.ParseFromString(serializedLincoln) 83 | ``` 84 | 85 | The `package` definition is mandatory; it determines the Python module name that the code 86 | will generate. If it has dots for namespacing, like `com.cueup.foo`, the last part of the 87 | name (`foo`) will be used for the Python module name. 88 | 89 | ### One more thing 90 | 91 | It's simple, but not that simple. The biggest caveat is that protobuf objects embedded in 92 | other protobuf objects are mutable, but all changes to them are discarded. If you want to 93 | build a protobuf with other protobufs in it, build them separately. To illustrate: 94 | 95 | ```python 96 | import addressbook_proto 97 | 98 | entry = addressbook_proto.Entry(name='Gillian Baskin') 99 | entry.birthplace = addressbook_proto.Location(state='Minnesota', town='Duluth') 100 | 101 | # Now, to modify it. Don't do this: 102 | entry.birthplace.town = 'New Town' 103 | # Instead, do this: 104 | birthplace = entry.birthplace 105 | birthplace.town = 'New Town' 106 | entry.birthplace = birthplace 107 | ``` 108 | 109 | There are also several methods for serializing and deserializing. Here's a list: 110 | 111 | `ParseFromString(str)` parses from a serialized protobuf stream. 112 | 113 | `ParseFromLongString(str)` has the same effect as `ParseFromString(str)`, but is faster 114 | for long strings and slower for short ones. This isn't a huge difference, but could be 115 | important if you're dealing with very large protobufs. 116 | 117 | `SerializeToString()` returns the serialized form of the protobuf, as a string. 118 | 119 | `SerializeMany(protobufs)` takes a sequence of protobuf objects and serializes them to a 120 | single string. The length of each protobuf is marked, so this can be serialized back to a 121 | list of protobufs. 122 | 123 | `ParseMany(str, callback)` takes a string in the format produced by `SerializeMany`, and 124 | calls `callback` with each protobuf object, in order. You can use this to build a list of 125 | protobufs like this: 126 | 127 | ```python 128 | people = [] 129 | addressbook_proto.Person.ParseMany(serializedPeople, people.append) 130 | print people # Will be a list of Person protobuf objects 131 | ``` 132 | 133 | ### Authors: 134 | 135 | [Greplin, Inc.](http://www.greplin.com) 136 | 137 | [Alan Grow](https://github.com/acg) 138 | 139 | [Oliver Tonnhofer](https://github.com/olt) 140 | 141 | [Joe Shaw](https://github.com/joeshaw) 142 | -------------------------------------------------------------------------------- /benchmark/README.md: -------------------------------------------------------------------------------- 1 | fast-python-pb 2 | ===================== 3 | 4 | Running the benchmark 5 | ---------------------------- 6 | 7 | The latest results on my MacBook pro are: 8 | 9 | JSON 10 | 2.87792301178 11 | 12 | SimpleJSON 13 | 0.56374001503 14 | 15 | Protocol Buffer (fast) 16 | 0.24841094017 17 | 18 | Protocol Buffer (standard) 19 | 3.93004989624 20 | 21 | cPickle 22 | 0.637856960297 23 | 24 | 25 | ### How to run the benchmark 26 | 27 | Start all commands in this directory, unless otherwise specified. 28 | 29 | 30 | Create the fast python pb version: 31 | 32 | mkdir /tmp/personproto 33 | 34 | protoc --fastpython_out /tmp/personproto --cpp_out /tmp/personproto person.proto 35 | 36 | cd /tmp/personproto 37 | 38 | python setup.py install 39 | 40 | cd - 41 | 42 | 43 | Create the native python version: 44 | 45 | protoc --python_out . person.proto 46 | 47 | Compile the .proto for lwpb: 48 | 49 | protoc person.proto -o person.pb2 50 | 51 | Run the benchmark: 52 | 53 | python benchmark.py 54 | -------------------------------------------------------------------------------- /benchmark/benchmark.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2011 The fast-python-pb Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | """Compare JSON and protocol buffer serialization times.""" 18 | 19 | from timeit import Timer 20 | 21 | import person_proto 22 | import person_pb2 23 | import json 24 | import simplejson 25 | import cPickle 26 | 27 | try: 28 | import lwpb.codec 29 | except ImportError: 30 | lwpb = None 31 | 32 | 33 | GETTYSBURG = """ 34 | Four score and seven years ago our fathers brought forth on this continent, a new nation, conceived in 35 | Liberty, and dedicated to the proposition that all men are created equal. 36 | 37 | Now we are engaged in a great civil war, testing whether that nation, or any nation so conceived 38 | and so dedicated, can long endure. We are met on a great battle-field of that war. We have come to dedicate 39 | a portion of that field, as a final resting place for those who here gave their lives that that nation might 40 | live. It is altogether fitting and proper that we should do this. 41 | 42 | But, in a larger sense, we can not dedicate -- we can not consecrate -- we can not hallow -- this ground. 43 | The brave men, living and dead, who struggled here, have consecrated it, far above our poor power to add or 44 | detract. The world will little note, nor long remember what we say here, but it can never forget what they did 45 | here. It is for us the living, rather, to be dedicated here to the unfinished work which they who fought here 46 | have thus far so nobly advanced. It is rather for us to be here dedicated to the great task remaining before 47 | us -- that from these honored dead we take increased devotion to that cause for which they gave the last full 48 | measure of devotion -- that we here highly resolve that these dead shall not have died in vain -- that this 49 | nation, under God, shall have a new birth of freedom -- and that government of the people, by the people, for 50 | the people, shall not perish from the earth. 51 | """ 52 | 53 | 54 | def useJson(): 55 | """Test serialization using JSON.""" 56 | lincoln = { 57 | 'name': 'Abraham Lincoln', 58 | 'birth_year': 1809, 59 | 'nicknames': ['Honest Abe', 'Abe'], 60 | 'facts': { 61 | 'Born In': 'Kentucky', 62 | 'Died In': 'Washington D.C.', 63 | 'Greatest Speech': GETTYSBURG 64 | } 65 | } 66 | 67 | serialized = json.dumps(lincoln) 68 | 69 | json.loads(serialized) 70 | 71 | 72 | def useSimpleJson(): 73 | """Test serialization using SimpleJSON.""" 74 | lincoln = { 75 | 'name': 'Abraham Lincoln', 76 | 'birth_year': 1809, 77 | 'nicknames': ['Honest Abe', 'Abe'], 78 | 'facts': { 79 | 'Born In': 'Kentucky', 80 | 'Died In': 'Washington D.C.', 81 | 'Greatest Speech': GETTYSBURG 82 | } 83 | } 84 | 85 | serialized = simplejson.dumps(lincoln) 86 | simplejson.loads(serialized) 87 | 88 | 89 | def usePb(): 90 | """Test protocol buffer serialization.""" 91 | lincoln = person_proto.Person(name = 'Abraham Lincoln', birth_year = 1809) 92 | lincoln.nicknames = ['Honest Abe', 'Abe'] 93 | lincoln.facts = [ 94 | person_proto.Fact(name = 'Born In', content = 'Kentucky'), 95 | person_proto.Fact(name = 'Died In', content = 'Washington D.C.'), 96 | person_proto.Fact(name = 'Greatest Speech', content = GETTYSBURG) 97 | ] 98 | 99 | serializedLincoln = lincoln.SerializeToString() 100 | 101 | newLincoln = person_proto.Person() 102 | newLincoln.ParseFromString(serializedLincoln) 103 | 104 | 105 | def useStandardPb(): 106 | """Test protocol buffer serialization with native protocol buffers.""" 107 | lincoln = person_pb2.Person(name = 'Abraham Lincoln', birth_year = 1809) 108 | lincoln.nicknames.extend(['Honest Abe', 'Abe']) 109 | 110 | fact = lincoln.facts.add() 111 | fact.name = 'Born In' 112 | fact.content = 'Kentucky' 113 | 114 | fact = lincoln.facts.add() 115 | fact.name = 'Died In' 116 | fact.content = 'Washington D.C.' 117 | 118 | fact = lincoln.facts.add() 119 | fact.name = 'Greatest Speech' 120 | fact.content = GETTYSBURG 121 | 122 | serializedLincoln = lincoln.SerializeToString() 123 | 124 | newLincoln = person_pb2.Person() 125 | newLincoln.ParseFromString(serializedLincoln) 126 | 127 | 128 | def useLWPB(codec): 129 | """Test protocol buffer serialization with lwpb.""" 130 | lincoln = { 131 | 'name' : 'Abraham Lincoln', 132 | 'birth_year' : 1809, 133 | 'nicknames' : ['Honest Abe', 'Abe'], 134 | 'facts' : [ 135 | { 'name' : 'Born In', 'content' : 'Kentucky' }, 136 | { 'name' : 'Died In', 'content' : 'Washington D.C.' }, 137 | { 'name' : 'Greatest Speech', 'content' : GETTYSBURG }, 138 | ] 139 | } 140 | 141 | serialized = codec.encode( lincoln ) 142 | newlincoln = codec.decode( serialized ) 143 | 144 | 145 | def useCPickle(): 146 | """Test protocol buffer serialization with cPickle.""" 147 | lincoln = { 148 | 'name' : 'Abraham Lincoln', 149 | 'birth_year' : 1809, 150 | 'nicknames' : ['Honest Abe', 'Abe'], 151 | 'facts' : [ 152 | { 'name' : 'Born In', 'content' : 'Kentucky' }, 153 | { 'name' : 'Died In', 'content' : 'Washington D.C.' }, 154 | { 'name' : 'Greatest Speech', 'content' : GETTYSBURG }, 155 | ] 156 | } 157 | 158 | serialized = cPickle.dumps( lincoln ) 159 | newlincoln = cPickle.loads( serialized ) 160 | 161 | 162 | def main(): 163 | """Runs the PB vs JSON benchmark.""" 164 | print "JSON" 165 | timer = Timer("useJson()", "from __main__ import useJson") 166 | print timer.timeit(10000) 167 | 168 | """Runs the PB vs SimpleJSON benchmark.""" 169 | print "SimpleJSON" 170 | timer = Timer("useSimpleJson()", "from __main__ import useSimpleJson") 171 | print timer.timeit(10000) 172 | 173 | print "Protocol Buffer (fast)" 174 | timer = Timer("usePb()", "from __main__ import usePb") 175 | print timer.timeit(10000) 176 | 177 | print "Protocol Buffer (standard)" 178 | timer = Timer("useStandardPb()", "from __main__ import useStandardPb") 179 | print timer.timeit(10000) 180 | 181 | if lwpb: 182 | print "Protocol Buffer (lwpb)" 183 | timer = Timer("useLWPB(lwpb_codec)", "from __main__ import useLWPB, lwpb_codec") 184 | print timer.timeit(10000) 185 | 186 | print "cPickle" 187 | timer = Timer("useCPickle()", "from __main__ import useCPickle") 188 | print timer.timeit(10000) 189 | 190 | 191 | if __name__ == '__main__': 192 | main() 193 | -------------------------------------------------------------------------------- /benchmark/person.proto: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 The fast-python-pb Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package person_proto; 18 | 19 | message Fact { 20 | required string name = 1; 21 | 22 | required string content = 2; 23 | } 24 | 25 | message Person { 26 | required string name = 1; 27 | 28 | required int32 birth_year = 2; 29 | 30 | repeated string nicknames = 3; 31 | 32 | repeated Fact facts = 4; 33 | } 34 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2011 The fast-python-pb Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | """Setup script for fast python protocol buffers.""" 19 | 20 | try: 21 | from setuptools import setup 22 | except ImportError: 23 | from distutils.core import setup 24 | 25 | setup(name='fastpb', 26 | version='0.1', 27 | description='Fast Python Protocol Buffers', 28 | license='Apache', 29 | author='Greplin, Inc.', 30 | author_email='opensource@greplin.com', 31 | url='https://www.github.com/Cue/fast-python-pb', 32 | package_dir={'': 'src'}, 33 | packages=['fastpb'], 34 | package_data={ 35 | 'fastpb': ['template/*'], 36 | }, 37 | entry_points={ 38 | 'console_scripts': [ 39 | 'protoc-gen-fastpython = fastpb.generator:main' 40 | ] 41 | }, 42 | install_requires=['ez-setup==0.9', 'protobuf >= 2.3.0', 'jinja2 >= 2.0'], 43 | ) 44 | -------------------------------------------------------------------------------- /src/fastpb/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cue/fast-python-pb/1cfa7956d09f4f620807ea38e2f520bed97b3de1/src/fastpb/__init__.py -------------------------------------------------------------------------------- /src/fastpb/generator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2011 The fast-python-pb Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | """Generates a Python wrapper for a C++ protocol buffer.""" 18 | 19 | import plugin_pb2 20 | 21 | from google.protobuf import descriptor_pb2 22 | from fastpb.util import order_dependencies 23 | from jinja2 import Template 24 | 25 | # pylint: disable=E0611 26 | from pkg_resources import resource_string 27 | 28 | import os.path 29 | import sys 30 | 31 | 32 | TYPE = { 33 | 'STRING': descriptor_pb2.FieldDescriptorProto.TYPE_STRING, 34 | 'DOUBLE': descriptor_pb2.FieldDescriptorProto.TYPE_DOUBLE, 35 | 'FLOAT': descriptor_pb2.FieldDescriptorProto.TYPE_FLOAT, 36 | 'INT32': descriptor_pb2.FieldDescriptorProto.TYPE_INT32, 37 | 'SINT32': descriptor_pb2.FieldDescriptorProto.TYPE_SINT32, 38 | 'UINT32': descriptor_pb2.FieldDescriptorProto.TYPE_UINT32, 39 | 'INT64': descriptor_pb2.FieldDescriptorProto.TYPE_INT64, 40 | 'SINT64': descriptor_pb2.FieldDescriptorProto.TYPE_SINT64, 41 | 'UINT64': descriptor_pb2.FieldDescriptorProto.TYPE_UINT64, 42 | 'MESSAGE': descriptor_pb2.FieldDescriptorProto.TYPE_MESSAGE, 43 | 'BYTES': descriptor_pb2.FieldDescriptorProto.TYPE_BYTES, 44 | 'BOOL': descriptor_pb2.FieldDescriptorProto.TYPE_BOOL, 45 | 'ENUM': descriptor_pb2.FieldDescriptorProto.TYPE_ENUM, 46 | 'FIXED32': descriptor_pb2.FieldDescriptorProto.TYPE_FIXED32, 47 | # TODO(robbyw): More types. 48 | } 49 | 50 | LABEL = { 51 | 'REPEATED': descriptor_pb2.FieldDescriptorProto.LABEL_REPEATED 52 | } 53 | 54 | 55 | def template(name): 56 | """Gets a template of the given name.""" 57 | return Template(resource_string(__name__, 'template/' + name)) 58 | 59 | 60 | def sort_messages(fileObject): 61 | """Return a sorted list of messages (sub-messages first). 62 | 63 | This avoids compilation problems involving declaration order. 64 | """ 65 | dependencies = [] 66 | msgDict = {} 67 | 68 | def visit(baseName, messages, parent=None): 69 | """Visitor for the message tree.""" 70 | for msg in messages: 71 | # Build our type name (using the protocol buffer convention) and 72 | # use it to register this message type object in our dictionary. 73 | typeName = baseName + '.' + msg.name 74 | msgDict[typeName] = msg 75 | 76 | # If this is a nested message type, prepend our parent's name to 77 | # our name for all future name lookups (via template expansion). 78 | # This disambiguates nested message names so that two n-level 79 | # messages can both have nested message types with the same name. 80 | # This also matches the generated C++ code's naming convention. 81 | if parent is not None: 82 | msg.name = parent.name + '_' + msg.name 83 | 84 | # If this message has nested message types, recurse. 85 | if msg.nested_type: 86 | visit(typeName, msg.nested_type, parent=msg) 87 | 88 | # Generate the set of messages that this type is dependent upon. 89 | deps = set([field.type_name for field in msg.field 90 | if field.type == TYPE['MESSAGE']]) 91 | dependencies.append((typeName, deps)) 92 | 93 | # Start by visiting the file's top-level message types. 94 | visit('.' + fileObject.package, fileObject.message_type) 95 | 96 | sortedMsgNames = order_dependencies(dependencies) 97 | return [msgDict[n] for n in sortedMsgNames] 98 | 99 | 100 | def writeCFile(response, name, fileObject): 101 | """Writes a C file.""" 102 | messages = sort_messages(fileObject) 103 | context = { 104 | 'fileName': name, 105 | 'moduleName': fileObject.package.lstrip('.'), 106 | 'package': fileObject.package.replace('.', '::'), 107 | 'packageName': fileObject.package.split('.')[-1], 108 | 'messages': messages, 109 | 'enums': fileObject.enum_type, 110 | 'TYPE': TYPE, 111 | 'LABEL': LABEL 112 | } 113 | 114 | cFile = response.file.add() 115 | cFile.name = name + '.cc' 116 | cFile.content = template('module.jinjacc').render(context) 117 | 118 | 119 | def writeSetupPy(response, files, parents): 120 | """Writes the setup.py file.""" 121 | setupFile = response.file.add() 122 | setupFile.name = 'setup.py' 123 | setupFile.content = template('setup.jinjapy').render({ 124 | 'files': files, 125 | 'parents': parents 126 | }) 127 | 128 | 129 | def writeTests(response, files): 130 | """Writes the tests.""" 131 | setupFile = response.file.add() 132 | setupFile.name = 'test.py' 133 | setupFile.content = template('test.jinjapy').render({ 134 | 'files': files, 135 | 'TYPE': TYPE, 136 | 'LABEL': LABEL 137 | }) 138 | 139 | 140 | def writeManifest(response, files): 141 | """Writes the manifest.""" 142 | setupFile = response.file.add() 143 | setupFile.name = 'MANIFEST.in' 144 | setupFile.content = template('MANIFEST.jinjain').render({ 145 | 'files': files 146 | }) 147 | 148 | 149 | def main(): 150 | """Main generation method.""" 151 | request = plugin_pb2.CodeGeneratorRequest() 152 | request.ParseFromString(sys.stdin.read()) 153 | 154 | response = plugin_pb2.CodeGeneratorResponse() 155 | 156 | parents = set() 157 | 158 | generateFiles = set(request.file_to_generate) 159 | files = [] 160 | for fileObject in request.proto_file: 161 | if fileObject.name not in generateFiles: 162 | continue 163 | if not fileObject.package: 164 | sys.stderr.write('%s: package definition required, but not found\n' % fileObject.name) 165 | sys.exit(1) 166 | 167 | name = fileObject.name.split('.')[0] 168 | files.append({ 169 | 'name': name, 170 | 'package': fileObject.package.lstrip('.'), 171 | 'messages': fileObject.message_type 172 | }) 173 | 174 | path = fileObject.package.lstrip('.').split('.')[:-1] 175 | for i in range(len(path)): 176 | filePathParts = path[:i+1] 177 | package = '.'.join(filePathParts) 178 | filePath = os.path.join(*filePathParts) 179 | if package not in parents: 180 | initPy = response.file.add() 181 | initPy.name = os.path.join('src', filePath, '__init__.py') 182 | initPy.content = """ 183 | import pkg_resources 184 | pkg_resources.declare_namespace('%s') 185 | """ % package 186 | parents.add(package) 187 | 188 | # Write the C file. 189 | writeCFile(response, name, fileObject) 190 | 191 | writeSetupPy(response, files, parents) 192 | writeTests(response, files) 193 | writeManifest(response, files) 194 | 195 | sys.stdout.write(response.SerializeToString()) 196 | 197 | 198 | if __name__ == '__main__': 199 | main() 200 | -------------------------------------------------------------------------------- /src/fastpb/plugin_pb2.py: -------------------------------------------------------------------------------- 1 | # Generated by the protocol buffer compiler. DO NOT EDIT! 2 | 3 | from google.protobuf import descriptor 4 | from google.protobuf import message 5 | from google.protobuf import reflection 6 | from google.protobuf import descriptor_pb2 7 | # @@protoc_insertion_point(imports) 8 | 9 | 10 | DESCRIPTOR = descriptor.FileDescriptor( 11 | name='google/protobuf/compiler/plugin.proto', 12 | package='google.protobuf.compiler', 13 | serialized_pb='\n%google/protobuf/compiler/plugin.proto\x12\x18google.protobuf.compiler\x1a google/protobuf/descriptor.proto\"}\n\x14\x43odeGeneratorRequest\x12\x18\n\x10\x66ile_to_generate\x18\x01 \x03(\t\x12\x11\n\tparameter\x18\x02 \x01(\t\x12\x38\n\nproto_file\x18\x0f \x03(\x0b\x32$.google.protobuf.FileDescriptorProto\"\xaa\x01\n\x15\x43odeGeneratorResponse\x12\r\n\x05\x65rror\x18\x01 \x01(\t\x12\x42\n\x04\x66ile\x18\x0f \x03(\x0b\x32\x34.google.protobuf.compiler.CodeGeneratorResponse.File\x1a>\n\x04\x46ile\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x17\n\x0finsertion_point\x18\x02 \x01(\t\x12\x0f\n\x07\x63ontent\x18\x0f \x01(\t') 14 | 15 | 16 | 17 | 18 | _CODEGENERATORREQUEST = descriptor.Descriptor( 19 | name='CodeGeneratorRequest', 20 | full_name='google.protobuf.compiler.CodeGeneratorRequest', 21 | filename=None, 22 | file=DESCRIPTOR, 23 | containing_type=None, 24 | fields=[ 25 | descriptor.FieldDescriptor( 26 | name='file_to_generate', full_name='google.protobuf.compiler.CodeGeneratorRequest.file_to_generate', index=0, 27 | number=1, type=9, cpp_type=9, label=3, 28 | has_default_value=False, default_value=[], 29 | message_type=None, enum_type=None, containing_type=None, 30 | is_extension=False, extension_scope=None, 31 | options=None), 32 | descriptor.FieldDescriptor( 33 | name='parameter', full_name='google.protobuf.compiler.CodeGeneratorRequest.parameter', index=1, 34 | number=2, type=9, cpp_type=9, label=1, 35 | has_default_value=False, default_value=unicode("", "utf-8"), 36 | message_type=None, enum_type=None, containing_type=None, 37 | is_extension=False, extension_scope=None, 38 | options=None), 39 | descriptor.FieldDescriptor( 40 | name='proto_file', full_name='google.protobuf.compiler.CodeGeneratorRequest.proto_file', index=2, 41 | number=15, type=11, cpp_type=10, label=3, 42 | has_default_value=False, default_value=[], 43 | message_type=None, enum_type=None, containing_type=None, 44 | is_extension=False, extension_scope=None, 45 | options=None), 46 | ], 47 | extensions=[ 48 | ], 49 | nested_types=[], 50 | enum_types=[ 51 | ], 52 | options=None, 53 | is_extendable=False, 54 | extension_ranges=[], 55 | serialized_start=101, 56 | serialized_end=226, 57 | ) 58 | 59 | 60 | _CODEGENERATORRESPONSE_FILE = descriptor.Descriptor( 61 | name='File', 62 | full_name='google.protobuf.compiler.CodeGeneratorResponse.File', 63 | filename=None, 64 | file=DESCRIPTOR, 65 | containing_type=None, 66 | fields=[ 67 | descriptor.FieldDescriptor( 68 | name='name', full_name='google.protobuf.compiler.CodeGeneratorResponse.File.name', index=0, 69 | number=1, type=9, cpp_type=9, label=1, 70 | has_default_value=False, default_value=unicode("", "utf-8"), 71 | message_type=None, enum_type=None, containing_type=None, 72 | is_extension=False, extension_scope=None, 73 | options=None), 74 | descriptor.FieldDescriptor( 75 | name='insertion_point', full_name='google.protobuf.compiler.CodeGeneratorResponse.File.insertion_point', index=1, 76 | number=2, type=9, cpp_type=9, label=1, 77 | has_default_value=False, default_value=unicode("", "utf-8"), 78 | message_type=None, enum_type=None, containing_type=None, 79 | is_extension=False, extension_scope=None, 80 | options=None), 81 | descriptor.FieldDescriptor( 82 | name='content', full_name='google.protobuf.compiler.CodeGeneratorResponse.File.content', index=2, 83 | number=15, type=9, cpp_type=9, label=1, 84 | has_default_value=False, default_value=unicode("", "utf-8"), 85 | message_type=None, enum_type=None, containing_type=None, 86 | is_extension=False, extension_scope=None, 87 | options=None), 88 | ], 89 | extensions=[ 90 | ], 91 | nested_types=[], 92 | enum_types=[ 93 | ], 94 | options=None, 95 | is_extendable=False, 96 | extension_ranges=[], 97 | serialized_start=337, 98 | serialized_end=399, 99 | ) 100 | 101 | _CODEGENERATORRESPONSE = descriptor.Descriptor( 102 | name='CodeGeneratorResponse', 103 | full_name='google.protobuf.compiler.CodeGeneratorResponse', 104 | filename=None, 105 | file=DESCRIPTOR, 106 | containing_type=None, 107 | fields=[ 108 | descriptor.FieldDescriptor( 109 | name='error', full_name='google.protobuf.compiler.CodeGeneratorResponse.error', index=0, 110 | number=1, type=9, cpp_type=9, label=1, 111 | has_default_value=False, default_value=unicode("", "utf-8"), 112 | message_type=None, enum_type=None, containing_type=None, 113 | is_extension=False, extension_scope=None, 114 | options=None), 115 | descriptor.FieldDescriptor( 116 | name='file', full_name='google.protobuf.compiler.CodeGeneratorResponse.file', index=1, 117 | number=15, type=11, cpp_type=10, label=3, 118 | has_default_value=False, default_value=[], 119 | message_type=None, enum_type=None, containing_type=None, 120 | is_extension=False, extension_scope=None, 121 | options=None), 122 | ], 123 | extensions=[ 124 | ], 125 | nested_types=[_CODEGENERATORRESPONSE_FILE, ], 126 | enum_types=[ 127 | ], 128 | options=None, 129 | is_extendable=False, 130 | extension_ranges=[], 131 | serialized_start=229, 132 | serialized_end=399, 133 | ) 134 | 135 | import google.protobuf.descriptor_pb2 136 | 137 | _CODEGENERATORREQUEST.fields_by_name['proto_file'].message_type = google.protobuf.descriptor_pb2._FILEDESCRIPTORPROTO 138 | _CODEGENERATORRESPONSE_FILE.containing_type = _CODEGENERATORRESPONSE; 139 | _CODEGENERATORRESPONSE.fields_by_name['file'].message_type = _CODEGENERATORRESPONSE_FILE 140 | 141 | class CodeGeneratorRequest(message.Message): 142 | __metaclass__ = reflection.GeneratedProtocolMessageType 143 | DESCRIPTOR = _CODEGENERATORREQUEST 144 | 145 | # @@protoc_insertion_point(class_scope:google.protobuf.compiler.CodeGeneratorRequest) 146 | 147 | class CodeGeneratorResponse(message.Message): 148 | __metaclass__ = reflection.GeneratedProtocolMessageType 149 | 150 | class File(message.Message): 151 | __metaclass__ = reflection.GeneratedProtocolMessageType 152 | DESCRIPTOR = _CODEGENERATORRESPONSE_FILE 153 | 154 | # @@protoc_insertion_point(class_scope:google.protobuf.compiler.CodeGeneratorResponse.File) 155 | DESCRIPTOR = _CODEGENERATORRESPONSE 156 | 157 | # @@protoc_insertion_point(class_scope:google.protobuf.compiler.CodeGeneratorResponse) 158 | 159 | # @@protoc_insertion_point(module_scope) 160 | -------------------------------------------------------------------------------- /src/fastpb/template/MANIFEST.jinjain: -------------------------------------------------------------------------------- 1 | include MANIFEST.in 2 | include setup.py 3 | include test.py 4 | {% for file in files %} 5 | include {{ file.name }}.cc 6 | include {{ file.name }}.pb.cc 7 | include {{ file.name }}.pb.h 8 | {% endfor %} 9 | -------------------------------------------------------------------------------- /src/fastpb/template/module.jinjacc: -------------------------------------------------------------------------------- 1 | // -*- C++ -*- 2 | #include 3 | #include 4 | #include 5 | #include "structmember.h" 6 | #include "{{ fileName }}.pb.h" 7 | 8 | #include 9 | #include 10 | 11 | 12 | {% macro cpp_type_name(field) -%} 13 | {{ '_'.join(field.type_name.replace('.' + moduleName + '.', '').split('.')) }} 14 | {%- endmacro %} 15 | 16 | static PyObject * 17 | fastpb_convert{{ TYPE.INT32 }}(::google::protobuf::int32 value) 18 | { 19 | return PyLong_FromLong(value); 20 | } 21 | 22 | static PyObject * 23 | fastpb_convert{{ TYPE.INT64 }}(::google::protobuf::int64 value) 24 | { 25 | return PyLong_FromLongLong(value); 26 | } 27 | 28 | static PyObject * 29 | fastpb_convert{{ TYPE.SINT64 }}(::google::protobuf::int64 value) 30 | { 31 | return PyLong_FromLongLong(value); 32 | } 33 | 34 | static PyObject * 35 | fastpb_convert{{ TYPE.SINT32 }}(::google::protobuf::int32 value) 36 | { 37 | return PyLong_FromLong(value); 38 | } 39 | 40 | static PyObject * 41 | fastpb_convert{{ TYPE.UINT32 }}(::google::protobuf::uint32 value) 42 | { 43 | return PyLong_FromUnsignedLong(value); 44 | } 45 | 46 | static PyObject * 47 | fastpb_convert{{ TYPE.FIXED32 }}(::google::protobuf::int32 value) 48 | { 49 | return PyLong_FromLong(value); 50 | } 51 | 52 | static PyObject * 53 | fastpb_convert{{ TYPE.UINT64 }}(::google::protobuf::uint64 value) 54 | { 55 | return PyLong_FromUnsignedLong(value); 56 | } 57 | 58 | static PyObject * 59 | fastpb_convert{{ TYPE.DOUBLE }}(double value) 60 | { 61 | return PyFloat_FromDouble(value); 62 | } 63 | 64 | static PyObject * 65 | fastpb_convert{{ TYPE.FLOAT }}(float value) 66 | { 67 | return PyFloat_FromDouble(value); 68 | } 69 | 70 | static PyObject * 71 | fastpb_convert{{ TYPE.STRING }}(const ::std::string &value) 72 | { 73 | return PyUnicode_Decode(value.data(), value.length(), "utf-8", NULL); 74 | } 75 | 76 | static PyObject * 77 | fastpb_convert{{ TYPE.BYTES }}(const ::std::string &value) 78 | { 79 | return PyString_FromStringAndSize(value.data(), value.length()); 80 | } 81 | 82 | static PyObject * 83 | fastpb_convert{{ TYPE.BOOL }}(bool value) 84 | { 85 | return PyBool_FromLong(value ? 1 : 0); 86 | } 87 | 88 | static PyObject * 89 | fastpb_convert{{ TYPE.ENUM }}(int value) 90 | { 91 | // TODO(robbyw): Check EnumName_IsValid(value) 92 | return PyInt_FromLong(value); 93 | } 94 | 95 | {% for enum in enums %} 96 | typedef struct { 97 | PyObject_HEAD 98 | } enum_{{ enum.name }}; 99 | 100 | static PyTypeObject enum_{{ enum.name }}Type = { 101 | PyObject_HEAD_INIT(NULL) 102 | 0, /*ob_size*/ 103 | "{{ moduleName }}.{{ enum.name }}", /*tp_name*/ 104 | sizeof(enum_{{ enum.name }}), /*tp_basicsize*/ 105 | 0, /*tp_itemsize*/ 106 | 0, /*tp_dealloc*/ 107 | 0, /*tp_print*/ 108 | 0, /*tp_getattr*/ 109 | 0, /*tp_setattr*/ 110 | 0, /*tp_compare*/ 111 | 0, /*tp_repr*/ 112 | 0, /*tp_as_number*/ 113 | 0, /*tp_as_sequence*/ 114 | 0, /*tp_as_mapping*/ 115 | 0, /*tp_hash */ 116 | 0, /*tp_call*/ 117 | 0, /*tp_str*/ 118 | 0, /*tp_getattro*/ 119 | 0, /*tp_setattro*/ 120 | 0, /*tp_as_buffer*/ 121 | Py_TPFLAGS_DEFAULT, /*tp_flags*/ 122 | "{{ enum.name }} enumeration", /* tp_doc */ 123 | }; 124 | {% endfor %} 125 | 126 | {% for message in messages %} 127 | 128 | // Lets try not to pollute the global namespace 129 | namespace { 130 | 131 | // Forward-declaration for recursive structures 132 | extern PyTypeObject {{ message.name }}Type; 133 | 134 | typedef struct { 135 | PyObject_HEAD 136 | 137 | {{ package }}::{{ message.name }} *protobuf; 138 | } {{ message.name }}; 139 | 140 | void 141 | {{ message.name }}_dealloc({{ message.name }}* self) 142 | { 143 | delete self->protobuf; 144 | self->ob_type->tp_free((PyObject*)self); 145 | } 146 | 147 | PyObject * 148 | {{ message.name }}_new(PyTypeObject *type, PyObject *args, PyObject *kwds) 149 | { 150 | {{ message.name }} *self; 151 | 152 | self = ({{ message.name }} *)type->tp_alloc(type, 0); 153 | 154 | self->protobuf = new {{ package }}::{{ message.name }}(); 155 | 156 | return (PyObject *)self; 157 | } 158 | 159 | PyObject * 160 | {{ message.name }}_DebugString({{ message.name }}* self) 161 | { 162 | std::string result; 163 | Py_BEGIN_ALLOW_THREADS 164 | result = self->protobuf->Utf8DebugString(); 165 | Py_END_ALLOW_THREADS 166 | return PyUnicode_FromStringAndSize(result.data(), result.length()); 167 | } 168 | 169 | 170 | PyObject * 171 | {{ message.name }}_SerializeToString({{ message.name }}* self) 172 | { 173 | std::string result; 174 | Py_BEGIN_ALLOW_THREADS 175 | self->protobuf->SerializeToString(&result); 176 | Py_END_ALLOW_THREADS 177 | return PyString_FromStringAndSize(result.data(), result.length()); 178 | } 179 | 180 | 181 | PyObject * 182 | {{ message.name }}_SerializeMany(void *nothing, PyObject *values) 183 | { 184 | std::string result; 185 | google::protobuf::io::ZeroCopyOutputStream* output = 186 | new google::protobuf::io::StringOutputStream(&result); 187 | google::protobuf::io::CodedOutputStream* outputStream = 188 | new google::protobuf::io::CodedOutputStream(output); 189 | 190 | PyObject *sequence = PySequence_Fast(values, "The values to serialize must be a sequence."); 191 | for (Py_ssize_t i = 0, len = PySequence_Length(sequence); i < len; ++i) { 192 | {{ message.name }} *value = ({{ message.name }} *)PySequence_Fast_GET_ITEM(sequence, i); 193 | 194 | Py_BEGIN_ALLOW_THREADS 195 | outputStream->WriteVarint32(value->protobuf->ByteSize()); 196 | value->protobuf->SerializeToCodedStream(outputStream); 197 | Py_END_ALLOW_THREADS 198 | } 199 | 200 | Py_XDECREF(sequence); 201 | delete outputStream; 202 | delete output; 203 | return PyString_FromStringAndSize(result.data(), result.length()); 204 | } 205 | 206 | 207 | PyObject * 208 | {{ message.name }}_ParseFromString({{ message.name }}* self, PyObject *value) 209 | { 210 | std::string serialized(PyString_AsString(value), PyString_Size(value)); 211 | Py_BEGIN_ALLOW_THREADS 212 | self->protobuf->ParseFromString(serialized); 213 | Py_END_ALLOW_THREADS 214 | Py_RETURN_NONE; 215 | } 216 | 217 | 218 | PyObject * 219 | {{ message.name }}_ParseFromLongString({{ message.name }}* self, PyObject *value) 220 | { 221 | google::protobuf::io::ZeroCopyInputStream* input = 222 | new google::protobuf::io::ArrayInputStream(PyString_AsString(value), PyString_Size(value)); 223 | google::protobuf::io::CodedInputStream* inputStream = 224 | new google::protobuf::io::CodedInputStream(input); 225 | inputStream->SetTotalBytesLimit(512 * 1024 * 1024, 512 * 1024 * 1024); 226 | 227 | Py_BEGIN_ALLOW_THREADS 228 | self->protobuf->ParseFromCodedStream(inputStream); 229 | Py_END_ALLOW_THREADS 230 | 231 | delete inputStream; 232 | delete input; 233 | 234 | Py_RETURN_NONE; 235 | } 236 | 237 | 238 | PyObject * 239 | {{ message.name }}_ParseMany(void* nothing, PyObject *args) 240 | { 241 | PyObject *value; 242 | PyObject *callback; 243 | int fail = 0; 244 | 245 | if (!PyArg_ParseTuple(args, "OO", &value, &callback)) { 246 | return NULL; 247 | } 248 | 249 | google::protobuf::io::ZeroCopyInputStream* input = 250 | new google::protobuf::io::ArrayInputStream(PyString_AsString(value), PyString_Size(value)); 251 | google::protobuf::io::CodedInputStream* inputStream = 252 | new google::protobuf::io::CodedInputStream(input); 253 | inputStream->SetTotalBytesLimit(512 * 1024 * 1024, 512 * 1024 * 1024); 254 | 255 | google::protobuf::uint32 bytes; 256 | PyObject *single = NULL; 257 | while (inputStream->ReadVarint32(&bytes)) { 258 | google::protobuf::io::CodedInputStream::Limit messageLimit = inputStream->PushLimit(bytes); 259 | 260 | if (single == NULL) { 261 | single = {{ message.name }}_new(&{{ message.name}}Type, NULL, NULL); 262 | } 263 | 264 | Py_BEGIN_ALLOW_THREADS 265 | (({{message.name}} *)single)->protobuf->ParseFromCodedStream(inputStream); 266 | Py_END_ALLOW_THREADS 267 | 268 | inputStream->PopLimit(messageLimit); 269 | PyObject *result = PyObject_CallFunctionObjArgs(callback, single, NULL); 270 | if (result == NULL) { 271 | fail = 1; 272 | break; 273 | }; 274 | 275 | if (single->ob_refcnt != 1) { 276 | // If the callback saved a reference to the item, don't re-use it. 277 | Py_XDECREF(single); 278 | single = NULL; 279 | } 280 | } 281 | if (single != NULL) { 282 | Py_XDECREF(single); 283 | } 284 | 285 | delete inputStream; 286 | delete input; 287 | 288 | if (fail) { 289 | return NULL; 290 | } else { 291 | Py_RETURN_NONE; 292 | } 293 | } 294 | 295 | 296 | {% for member in message.field %} 297 | {% if member.type == TYPE.MESSAGE %} 298 | PyObject * 299 | fastpb_convert{{ message.name + member.name }}(const ::google::protobuf::Message &value) 300 | { 301 | {{ cpp_type_name(member) }} *obj = ({{ cpp_type_name(member) }} *) 302 | {{ cpp_type_name(member) }}_new(&{{ cpp_type_name(member) }}Type, NULL, NULL); 303 | obj->protobuf->MergeFrom(value); 304 | return (PyObject *)obj; 305 | } 306 | {% endif %} 307 | 308 | PyObject * 309 | {{ message.name }}_get{{ member.name }}({{ message.name }} *self, void *closure) 310 | { 311 | {% if member.label == LABEL.REPEATED %} 312 | int len = self->protobuf->{{ member.name.lower() }}_size(); 313 | PyObject *tuple = PyTuple_New(len); 314 | for (int i = 0; i < len; ++i) { 315 | PyObject *value = 316 | fastpb_convert{{ member.type if member.type != TYPE.MESSAGE else message.name + member.name }}( 317 | self->protobuf->{{ member.name.lower() }}(i)); 318 | if (!value) { 319 | return NULL; 320 | } 321 | PyTuple_SetItem(tuple, i, value); 322 | } 323 | return tuple; 324 | 325 | {% else %} 326 | if (! self->protobuf->has_{{ member.name.lower() }}()) { 327 | Py_RETURN_NONE; 328 | } 329 | 330 | return 331 | fastpb_convert{{ member.type if member.type != TYPE.MESSAGE else message.name + member.name }}( 332 | self->protobuf->{{ member.name.lower() }}()); 333 | 334 | {% endif %} 335 | } 336 | 337 | int 338 | {{ message.name }}_set{{ member.name }}({{ message.name }} *self, PyObject *input, void *closure) 339 | { 340 | if (input == NULL || input == Py_None) { 341 | self->protobuf->clear_{{ member.name.lower() }}(); 342 | return 0; 343 | } 344 | 345 | {% if member.label == LABEL.REPEATED %} 346 | if (PyString_Check(input)) { 347 | PyErr_SetString(PyExc_TypeError, "The {{ member.name }} attribute value must be a sequence"); 348 | return -1; 349 | } 350 | PyObject *sequence = PySequence_Fast(input, "The {{ member.name }} attribute value must be a sequence"); 351 | self->protobuf->clear_{{ member.name.lower() }}(); 352 | for (Py_ssize_t i = 0, len = PySequence_Length(sequence); i < len; ++i) { 353 | PyObject *value = PySequence_Fast_GET_ITEM(sequence, i); 354 | 355 | {% else %} 356 | PyObject *value = input; 357 | {% endif %} 358 | 359 | {% if member.type == TYPE.STRING %} 360 | // string 361 | bool reallocated = false; 362 | if (PyUnicode_Check(value)) { 363 | value = PyUnicode_AsEncodedString(value, "utf-8", NULL); 364 | reallocated = true; 365 | } 366 | 367 | if (! PyString_Check(value)) { 368 | PyErr_SetString(PyExc_TypeError, "The {{ member.name }} attribute value must be a string"); 369 | return -1; 370 | } 371 | 372 | std::string protoValue(PyString_AsString(value), PyString_Size(value)); 373 | if (reallocated) { 374 | Py_XDECREF(value); 375 | } 376 | 377 | {% elif member.type == TYPE.BYTES %} 378 | // string 379 | if (! PyString_Check(value)) { 380 | PyErr_SetString(PyExc_TypeError, "The {{ member.name }} attribute value must be a string"); 381 | return -1; 382 | } 383 | 384 | std::string protoValue(PyString_AsString(value), PyString_Size(value)); 385 | 386 | {% elif member.type == TYPE.DOUBLE or member.type == TYPE.FLOAT %} 387 | {% if member.type == TYPE.DOUBLE %} 388 | double protoValue; 389 | {% elif member.type == TYPE.FLOAT %} 390 | float protoValue; 391 | {% endif %} 392 | if (PyFloat_Check(value)) { 393 | protoValue = PyFloat_AsDouble(value); 394 | } else if (PyInt_Check(value)) { 395 | protoValue = PyInt_AsLong(value); 396 | } else if (PyLong_Check(value)) { 397 | protoValue = PyLong_AsLongLong(value); 398 | } else { 399 | PyErr_SetString(PyExc_TypeError, 400 | {% if member.type == TYPE.DOUBLE %} 401 | "The {{ member.name }} attribute value must be a double"); 402 | {% elif member.type == TYPE.FLOAT %} 403 | "The {{ member.name }} attribute value must be a float"); 404 | {% endif %} 405 | return -1; 406 | } 407 | 408 | {% elif member.type == TYPE.INT64 or member.type == TYPE.SINT64 %} 409 | ::google::protobuf::int64 protoValue; 410 | 411 | // int64 412 | if (PyInt_Check(value)) { 413 | protoValue = PyInt_AsLong(value); 414 | } else if (PyLong_Check(value)) { 415 | protoValue = PyLong_AsLongLong(value); 416 | } else { 417 | PyErr_SetString(PyExc_TypeError, 418 | "The {{ member.name }} attribute value must be an integer"); 419 | return -1; 420 | } 421 | 422 | {% elif member.type == TYPE.INT32 or member.type == TYPE.SINT32 or member.type == TYPE.FIXED32%} 423 | ::google::protobuf::int32 protoValue; 424 | 425 | // int32 426 | if (PyInt_Check(value)) { 427 | protoValue = PyInt_AsLong(value); 428 | } else { 429 | PyErr_SetString(PyExc_TypeError, 430 | "The {{ member.name }} attribute value must be an integer"); 431 | return -1; 432 | } 433 | 434 | {% elif member.type == TYPE.ENUM %} 435 | // {{ member.type_name }} 436 | {{ member.type_name.replace('.', '::') }} protoValue; 437 | 438 | // int32 439 | if (PyInt_Check(value)) { 440 | protoValue = ({{ member.type_name.replace('.', '::') }}) PyInt_AsLong(value); 441 | } else { 442 | PyErr_SetString(PyExc_TypeError, 443 | "The {{ member.name }} attribute value must be an integer"); 444 | return -1; 445 | } 446 | 447 | {% elif member.type == TYPE.BOOL %} 448 | bool protoValue; 449 | 450 | if (PyBool_Check(value)) { 451 | protoValue = (value == Py_True); 452 | } else { 453 | PyErr_SetString(PyExc_TypeError, 454 | "The {{ member.name }} attribute value must be a boolean"); 455 | return -1; 456 | } 457 | 458 | {% elif member.type == TYPE.UINT32 or member.type == TYPE.UINT64 %} 459 | {% if member.type == TYPE.UINT32 %} 460 | ::google::protobuf::uint32 protoValue; 461 | {% elif member.type == TYPE.UINT64 %} 462 | ::google::protobuf::uint64 protoValue; 463 | {% endif %} 464 | 465 | // uint32 466 | if (PyInt_Check(value)) { 467 | protoValue = PyInt_AsUnsignedLongMask(value); 468 | } else if (PyLong_Check(value)) { 469 | protoValue = PyLong_AsUnsignedLong(value); 470 | } else { 471 | PyErr_SetString(PyExc_TypeError, 472 | "The {{ member.name }} attribute value must be an integer"); 473 | return -1; 474 | } 475 | 476 | {% elif member.type == TYPE.MESSAGE %} 477 | 478 | if (!PyType_IsSubtype(value->ob_type, &{{ cpp_type_name(member) }}Type)) { 479 | PyErr_SetString(PyExc_TypeError, 480 | "The {{ member.name }} attribute value must be an instance of {{ cpp_type_name(member) }}"); 481 | return -1; 482 | } 483 | 484 | // {{ member.type_name }} 485 | {{ member.type_name.replace('.', '::') }} *protoValue = 486 | (({{ cpp_type_name(member) }} *) value)->protobuf; 487 | 488 | {% endif %} 489 | 490 | {% if member.label == LABEL.REPEATED %} 491 | {% if member.type == TYPE.MESSAGE %} 492 | self->protobuf->add_{{ member.name.lower() }}()->MergeFrom(*protoValue); 493 | {% else %} 494 | self->protobuf->add_{{ member.name.lower() }}(protoValue); 495 | {% endif %} 496 | } 497 | 498 | Py_XDECREF(sequence); 499 | {% else %} 500 | {% if member.type == TYPE.MESSAGE %} 501 | self->protobuf->clear_{{ member.name.lower() }}(); 502 | self->protobuf->mutable_{{ member.name.lower() }}()->MergeFrom(*protoValue); 503 | {% else %} 504 | self->protobuf->set_{{ member.name.lower() }}(protoValue); 505 | {% endif %} 506 | {% endif %} 507 | 508 | return 0; 509 | } 510 | {% endfor %} 511 | 512 | int 513 | {{ message.name }}_init({{ message.name }} *self, PyObject *args, PyObject *kwds) 514 | { 515 | {% if message.field %} 516 | {% for member in message.field %} 517 | PyObject *{{ member.name }} = NULL; 518 | {% endfor %} 519 | 520 | static char *kwlist[] = { 521 | {% for member in message.field %} 522 | (char *) "{{ member.name }}", 523 | {% endfor %} 524 | NULL 525 | }; 526 | 527 | if (! PyArg_ParseTupleAndKeywords( 528 | args, kwds, "|{{ 'O' * message.field|length }}", kwlist, 529 | {% for member in message.field -%} 530 | &{{ member.name }} 531 | {%- if not loop.last -%} 532 | , 533 | {%- endif -%} 534 | {%- endfor %})) 535 | return -1; 536 | 537 | {% for member in message.field %} 538 | if ({{ member.name }}) { 539 | if ({{ message.name }}_set{{ member.name }}(self, {{ member.name }}, NULL) < 0) { 540 | return -1; 541 | } 542 | } 543 | {% endfor %} 544 | {% endif %} 545 | 546 | return 0; 547 | } 548 | 549 | 550 | PyObject * 551 | {{ message.name }}_richcompare(PyObject *self, PyObject *other, int op) 552 | { 553 | PyObject *result = NULL; 554 | if (!PyType_IsSubtype(other->ob_type, &{{ message.name }}Type)) { 555 | result = Py_NotImplemented; 556 | } else { 557 | // This is not a particularly efficient implementation since it never short circuits, but it's better 558 | // than nothing. It should probably only be used for tests. 559 | {{ message.name }} *selfValue = ({{ message.name }} *)self; 560 | {{ message.name }} *otherValue = ({{ message.name }} *)other; 561 | std::string selfSerialized; 562 | std::string otherSerialized; 563 | Py_BEGIN_ALLOW_THREADS 564 | selfValue->protobuf->SerializeToString(&selfSerialized); 565 | otherValue->protobuf->SerializeToString(&otherSerialized); 566 | Py_END_ALLOW_THREADS 567 | 568 | int cmp = selfSerialized.compare(otherSerialized); 569 | bool value = false; 570 | switch (op) { 571 | case Py_LT: 572 | value = cmp < 0; 573 | break; 574 | case Py_LE: 575 | value = cmp <= 0; 576 | break; 577 | case Py_EQ: 578 | value = cmp == 0; 579 | break; 580 | case Py_NE: 581 | value = cmp != 0; 582 | break; 583 | case Py_GT: 584 | value = cmp > 0; 585 | break; 586 | case Py_GE: 587 | value = cmp >= 0; 588 | break; 589 | } 590 | result = value ? Py_True : Py_False; 591 | } 592 | 593 | Py_XINCREF(result); 594 | return result; 595 | } 596 | 597 | 598 | static PyObject * 599 | {{ message.name }}_repr(PyObject *selfObject) 600 | { 601 | {{ message.name }} *self = ({{ message.name }} *)selfObject; 602 | PyObject *member; 603 | PyObject *memberRepr; 604 | std::stringstream result; 605 | result << "{{ message.name}}("; 606 | 607 | {% for member in message.field %} 608 | {% if not loop.first %} 609 | result << ", "; 610 | {% endif %} 611 | result << "{{ member.name }}="; 612 | member = {{ message.name }}_get{{ member.name }}(self, NULL); 613 | memberRepr = PyObject_Repr(member); 614 | result << PyString_AsString(memberRepr); 615 | Py_XDECREF(memberRepr); 616 | Py_XDECREF(member); 617 | {% endfor %} 618 | 619 | result << ")"; 620 | 621 | std::string resultString = result.str(); 622 | return PyUnicode_Decode(resultString.data(), resultString.length(), "utf-8", NULL); 623 | } 624 | 625 | 626 | PyMemberDef {{ message.name }}_members[] = { 627 | {NULL} // Sentinel 628 | }; 629 | 630 | 631 | PyGetSetDef {{ message.name }}_getsetters[] = { 632 | {% for member in message.field %} 633 | {(char *)"{{ member.name }}", 634 | (getter){{ message.name }}_get{{ member.name }}, (setter){{ message.name }}_set{{ member.name }}, 635 | (char *)"", 636 | NULL}, 637 | {% endfor %} 638 | {NULL} // Sentinel 639 | }; 640 | 641 | 642 | PyMethodDef {{ message.name }}_methods[] = { 643 | {"DebugString", (PyCFunction){{ message.name }}_DebugString, METH_NOARGS, 644 | "Generates a human readable form of this message, useful for debugging and other purposes." 645 | }, 646 | {"SerializeToString", (PyCFunction){{ message.name }}_SerializeToString, METH_NOARGS, 647 | "Serializes the protocol buffer to a string." 648 | }, 649 | {"SerializeMany", (PyCFunction){{ message.name }}_SerializeMany, METH_O | METH_CLASS, 650 | "Serializes a sequence of protocol buffers to a string." 651 | }, 652 | {"ParseFromString", (PyCFunction){{ message.name }}_ParseFromString, METH_O, 653 | "Parses the protocol buffer from a string." 654 | }, 655 | {"ParseFromLongString", (PyCFunction){{ message.name }}_ParseFromLongString, METH_O, 656 | "Parses the protocol buffer from a string as large as 512MB." 657 | }, 658 | {"ParseMany", (PyCFunction){{ message.name }}_ParseMany, METH_VARARGS | METH_CLASS, 659 | "Parses many protocol buffers of this type from a string." 660 | }, 661 | {NULL} // Sentinel 662 | }; 663 | 664 | 665 | PyTypeObject {{ message.name }}Type = { 666 | PyObject_HEAD_INIT(NULL) 667 | 0, /*ob_size*/ 668 | "{{ moduleName }}.{{ message.name }}", /*tp_name*/ 669 | sizeof({{ message.name }}), /*tp_basicsize*/ 670 | 0, /*tp_itemsize*/ 671 | (destructor){{ message.name }}_dealloc, /*tp_dealloc*/ 672 | 0, /*tp_print*/ 673 | 0, /*tp_getattr*/ 674 | 0, /*tp_setattr*/ 675 | 0, /*tp_compare*/ 676 | {{ message.name }}_repr, /*tp_repr*/ 677 | 0, /*tp_as_number*/ 678 | 0, /*tp_as_sequence*/ 679 | 0, /*tp_as_mapping*/ 680 | 0, /*tp_hash */ 681 | 0, /*tp_call*/ 682 | 0, /*tp_str*/ 683 | 0, /*tp_getattro*/ 684 | 0, /*tp_setattro*/ 685 | 0, /*tp_as_buffer*/ 686 | Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_RICHCOMPARE, /*tp_flags*/ 687 | "{{ message.name }} objects", /* tp_doc */ 688 | 0, /* tp_traverse */ 689 | 0, /* tp_clear */ 690 | {{ message.name }}_richcompare, /* tp_richcompare */ 691 | 0, /* tp_weaklistoffset */ 692 | 0, /* tp_iter */ 693 | 0, /* tp_iternext */ 694 | {{ message.name }}_methods, /* tp_methods */ 695 | {{ message.name }}_members, /* tp_members */ 696 | {{ message.name }}_getsetters, /* tp_getset */ 697 | 0, /* tp_base */ 698 | 0, /* tp_dict */ 699 | 0, /* tp_descr_get */ 700 | 0, /* tp_descr_set */ 701 | 0, /* tp_dictoffset */ 702 | (initproc){{ message.name }}_init, /* tp_init */ 703 | 0, /* tp_alloc */ 704 | {{ message.name }}_new, /* tp_new */ 705 | }; 706 | } 707 | 708 | {% endfor %} 709 | 710 | static PyMethodDef module_methods[] = { 711 | {NULL} // Sentinel 712 | }; 713 | 714 | #ifndef PyMODINIT_FUNC // Declarations for DLL import/export. 715 | #define PyMODINIT_FUNC void 716 | #endif 717 | PyMODINIT_FUNC 718 | init{{ packageName }}(void) 719 | { 720 | GOOGLE_PROTOBUF_VERIFY_VERSION; 721 | 722 | PyObject* m; 723 | 724 | {% if enums %} 725 | PyObject *classDict; 726 | {% endif %} 727 | 728 | {% for message in messages %} 729 | if (PyType_Ready(&{{ message.name }}Type) < 0) 730 | return; 731 | {% endfor %} 732 | 733 | m = Py_InitModule3("{{ packageName }}", module_methods, 734 | "{{ moduleDescription }}"); 735 | 736 | if (m == NULL) 737 | return; 738 | 739 | {% for enum in enums %} 740 | classDict = PyDict_New(); 741 | {% for value in enum.value %} 742 | PyDict_SetItemString(classDict, "{{ value.name }}", PyInt_FromLong({{ value.number }})); 743 | {% endfor %} 744 | 745 | enum_{{ enum.name }}Type.tp_dict = classDict; 746 | enum_{{ enum.name }}Type.tp_new = PyType_GenericNew; 747 | if (PyType_Ready(&enum_{{ enum.name }}Type) < 0) 748 | return; 749 | 750 | Py_INCREF(&enum_{{ enum.name }}Type); 751 | PyModule_AddObject(m, "{{ enum.name }}", (PyObject *)&enum_{{ enum.name }}Type); 752 | {% endfor %} 753 | 754 | {% for message in messages %} 755 | Py_INCREF(&{{ message.name }}Type); 756 | PyModule_AddObject(m, "{{ message.name }}", (PyObject *)&{{ message.name }}Type); 757 | {% endfor %} 758 | } 759 | -------------------------------------------------------------------------------- /src/fastpb/template/setup.jinjapy: -------------------------------------------------------------------------------- 1 | try: 2 | from setuptools import setup, Extension 3 | except ImportError: 4 | from distutils.core import setup, Extension 5 | 6 | setup(name="proto_wrapper", 7 | version="1.0", 8 | packages=[ 9 | {% for parent in parents %} 10 | "{{ parent }}", 11 | {% endfor %} 12 | ], 13 | package_dir={ 14 | {% for parent in parents %} 15 | "{{ parent }}": "src/{{ parent.replace('.', '/') }}", 16 | {% endfor %} 17 | }, 18 | ext_modules=[ 19 | {% for file in files %} 20 | Extension("{{ file.package }}", ["{{ file.name }}.cc", "{{ file.name }}.pb.cc"], libraries=['protobuf']), 21 | {% endfor %} 22 | ], 23 | test_suite="test.suite") 24 | -------------------------------------------------------------------------------- /src/fastpb/template/test.jinjapy: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """Auto-generated unit tests.""" 3 | 4 | import unittest 5 | 6 | {% for file in files %} 7 | import {{ file.package }} 8 | {% endfor %} 9 | 10 | {% macro value(field, index) -%} 11 | {%- if field.label == LABEL.REPEATED -%} 12 | ( 13 | {%- endif -%} 14 | {%- if field.type == TYPE.STRING -%} 15 | u'{{ index }}' 16 | {%- elif field.type == TYPE.BYTES -%} 17 | b'\x00\x01\x02' 18 | {%- elif field.type == TYPE.BOOL -%} 19 | True 20 | {%- elif field.type == TYPE.FLOAT -%} 21 | {{ index }}.0 22 | {%- elif field.type == TYPE.INT32 23 | or field.type == TYPE.SINT32 24 | or field.type == TYPE.UINT32 25 | or field.type == TYPE.INT64 26 | or field.type == TYPE.SINT64 27 | or field.type == TYPE.UINT64 -%} 28 | {{ index }} 29 | {%- elif field.type == TYPE.DOUBLE -%} 30 | {{ index }}.{{ index }} 31 | {%- elif field.type == TYPE.ENUM -%} 32 | 0 33 | {%- endif -%} 34 | {%- if field.label == LABEL.REPEATED -%} 35 | ,) 36 | {%- endif -%} 37 | {%- endmacro %} 38 | 39 | {% for file in files %} 40 | class Test_{{ file.package.replace('.', '_') }}(unittest.TestCase): 41 | {% for message in file.messages %} 42 | def test{{ message.name }}_Basics(self): 43 | pb = {{ file.package }}.{{ message.name }}() 44 | {% for field in message.field %} 45 | {% if field.type != TYPE.MESSAGE %} 46 | pb.{{ field.name }} = {{ value(field, loop.index) }} 47 | self.assertEquals({{ value(field, loop.index) }}, pb.{{ field.name }}) 48 | {% endif %} 49 | {% endfor %} 50 | 51 | pb2 = {{ file.package }}.{{ message.name }}() 52 | pb2.ParseFromString(pb.SerializeToString()) 53 | 54 | {% for field in message.field %} 55 | self.assertEquals(pb.{{ field.name }}, pb2.{{ field.name }}) 56 | {% endfor %} 57 | {% endfor %} 58 | {% endfor %} 59 | 60 | 61 | def suite(): 62 | suite = unittest.TestSuite() 63 | {% for file in files %} 64 | suite.addTests(unittest.makeSuite(Test_{{ file.package.replace('.', '_') }})) 65 | {% endfor %} 66 | return suite 67 | -------------------------------------------------------------------------------- /src/fastpb/util.py: -------------------------------------------------------------------------------- 1 | # Copyright 2011 The fast-python-pb Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from functools import reduce 16 | 17 | try: 18 | from collections import OrderedDict 19 | except ImportError: 20 | # just ignore the order with Python <2.7 for now 21 | OrderedDict = dict 22 | 23 | class CyclicError(Exception): 24 | pass 25 | 26 | def order_dependencies(dependencies): 27 | """Produce a topologically-sorted list of the given dependencies. 28 | 29 | >>> list(order_dependencies([ 30 | ... ('a', set(['b', 'c'])), 31 | ... ('b', set(['c'])), 32 | ... ('c', set()), 33 | ... ])) 34 | ['c', 'b', 'a'] 35 | 36 | Flat dependencies simply yield the original order. 37 | 38 | >>> list(order_dependencies([ 39 | ... ('a', set()), 40 | ... ('b', set()), 41 | ... ('c', set()), 42 | ... ])) 43 | ['a', 'b', 'c'] 44 | 45 | Nested and diamond dependencies are also supported. 46 | 47 | >>> list(order_dependencies([ 48 | ... ('a', set()), 49 | ... ('b', set(['c', 'd'])), 50 | ... ('c', set()), 51 | ... ])) 52 | ['a', 'c', 'd', 'b'] 53 | >>> list(order_dependencies([ 54 | ... ('a', set(['b', 'c'])), 55 | ... ('b', set(['d'])), 56 | ... ('c', set(['d'])), 57 | ... ('d', set()), 58 | ... ])) 59 | ['d', 'b', 'c', 'a'] 60 | 61 | An empty dependency list results in an empty generator sequence. 62 | 63 | >>> list(order_dependencies([])) 64 | [] 65 | 66 | Cyclic dependencies result in a CyclicError. 67 | 68 | >>> list(order_dependencies([ 69 | ... ('a', set(['b'])), 70 | ... ('b', set(['c'])), 71 | ... ('c', set(['a'])), 72 | ... ])) 73 | Traceback (most recent call last): 74 | ... 75 | CyclicError: A cyclic dependency exists amongst {'a': set(['b']), 'c': set(['a']), 'b': set(['c'])} 76 | 77 | Based on toposort2() by Paddy McCarthy. 78 | (see http://code.activestate.com/recipes/577413-topological-sort/) 79 | """ 80 | data = OrderedDict(dependencies) 81 | 82 | # Ignore self dependencies. 83 | for k, v in data.items(): 84 | v.discard(k) 85 | 86 | # If we're out of data, return (and produce an empty generator sequence). 87 | if not data: 88 | return 89 | 90 | # Add top-level keys for any unrepresented values. 91 | for item in reduce(set.union, data.values()) - set(data.keys()): 92 | data[item] = set() 93 | 94 | while True: 95 | ordered = set(item for item, dep in data.items() if not dep) 96 | if not ordered: 97 | break 98 | for dep in sorted(ordered): 99 | yield dep 100 | 101 | remaining = {} 102 | for item, dep in data.iteritems(): 103 | if item not in ordered: 104 | remaining[item] = (dep - ordered) 105 | data = remaining 106 | 107 | if data: 108 | raise CyclicError('A cyclic dependency exists amongst %r' % dict(data)) 109 | 110 | if __name__ == "__main__": 111 | import doctest 112 | doctest.testmod() 113 | -------------------------------------------------------------------------------- /test/namespace.proto: -------------------------------------------------------------------------------- 1 | package com.greplin.test; 2 | 3 | message InnerMessage { 4 | required string name = 1; 5 | } 6 | 7 | message OuterMessage { 8 | required InnerMessage message = 1; 9 | 10 | optional OuterMessage recursive = 2; 11 | } 12 | --------------------------------------------------------------------------------