├── setup.py ├── uprotobuf_plugin.bat ├── tests ├── proto │ ├── generate.sh │ └── tests.proto ├── client │ └── client.py └── server │ ├── server.py │ └── tests_pb2.py ├── LICENSE ├── .gitignore ├── README.md ├── uprotobuf_plugin.py └── uprotobuf.py /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | -------------------------------------------------------------------------------- /uprotobuf_plugin.bat: -------------------------------------------------------------------------------- 1 | @python -u uprotobuf_plugin.py 2 | -------------------------------------------------------------------------------- /tests/proto/generate.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh -e 2 | protoc tests.proto --plugin=protoc-gen-custom=../../uprotobuf_plugin.py --custom_out=../client 3 | protoc tests.proto --plugin=python --python_out=../server 4 | -------------------------------------------------------------------------------- /tests/proto/tests.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package Tests; 4 | 5 | message Test1 { 6 | enum Test1Enum { 7 | ValueA=1; 8 | ValueB=2; 9 | ValueC=3; 10 | } 11 | 12 | // Varint types 13 | optional int32 Int32 = 1; 14 | optional sint32 Sint32 = 2; 15 | optional uint32 Uint32 = 3; 16 | 17 | optional int64 Int64 = 4; 18 | optional sint64 Sint64 = 5; 19 | optional uint64 Uint64 = 6; 20 | 21 | optional bool Bool = 7; 22 | 23 | optional Test1Enum Enum = 8 [default=ValueA]; 24 | 25 | // Fixed types 26 | optional fixed32 Fixed32 = 9; 27 | optional fixed64 Fixed64 = 10; 28 | optional sfixed32 Sfixed32 = 11; 29 | optional sfixed64 Sfixed64 = 12; 30 | optional float Float = 13; 31 | optional double Double = 14; 32 | 33 | // Lengths types 34 | optional string String = 15; 35 | } -------------------------------------------------------------------------------- /tests/client/client.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env micropython 2 | import sys 3 | sys.path.append("../..") 4 | 5 | from socket import socket, AF_INET, SOCK_STREAM 6 | 7 | from tests_upb2 import Test1Message, Test1enum 8 | 9 | if __name__=="__main__": 10 | HOST,PORT="localhost",6789 11 | s=socket(AF_INET,SOCK_STREAM) 12 | s.connect((HOST,PORT)) 13 | 14 | t1m=Test1Message() 15 | 16 | # Varint types 17 | t1m.Int32=10 18 | t1m.Sint32=-10 19 | t1m.Uint32=15 20 | 21 | t1m.Int64=20 22 | t1m.Sint64=-20 23 | t1m.Uint64=25 24 | 25 | t1m.Bool=True 26 | 27 | t1m.Enum=Test1enum.ValueB 28 | 29 | # Fixed types 30 | t1m.Fixed32=30 31 | t1m.Sfixed32=-30 32 | 33 | t1m.Fixed64 = 40 34 | t1m.Sfixed64 = -40 35 | 36 | t1m.Float=3.14 37 | t1m.Double=-6.28 38 | 39 | # Length types 40 | t1m.String="My hovercraft is full of eels!" 41 | 42 | data=t1m.serialize() 43 | print("Sending: ",data) 44 | s.sendall(data) 45 | print(b"Received:"+s.recv(1024)) -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Rob Kent 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /tests/server/server.py: -------------------------------------------------------------------------------- 1 | from socketserver import TCPServer, BaseRequestHandler 2 | 3 | from tests_pb2 import Test1 as Test1Message 4 | 5 | class TestHandler(BaseRequestHandler): 6 | def handle(self): 7 | data=self.request.recv(1024).strip() 8 | print(b"Received:"+data) 9 | 10 | t1m=Test1Message() 11 | print("OK:",t1m.ParseFromString(data)) 12 | 13 | print("Int32: ", t1m.Int32) 14 | print("Sint32: ", t1m.Sint32) 15 | print("Uint32: ", t1m.Uint32) 16 | 17 | print("Int64: ", t1m.Int64) 18 | print("Sint64: ", t1m.Sint64) 19 | print("Uint64: ", t1m.Uint64) 20 | 21 | print("Bool: ", t1m.Bool) 22 | 23 | print("Enum: ", t1m.Enum) 24 | 25 | print("Fixed32: ", t1m.Fixed32) 26 | print("Sfixed32: ", t1m.Sfixed32) 27 | 28 | print("Fixed64: ", t1m.Fixed64) 29 | print("Sfixed64: ", t1m.Sfixed64) 30 | 31 | print("Float: ", t1m.Float) 32 | print("Double: ", t1m.Double) 33 | 34 | print("String: ", t1m.String) 35 | 36 | self.request.sendall(data.upper()) 37 | 38 | if __name__=="__main__": 39 | HOST,PORT="localhost",6789 40 | with TCPServer((HOST,PORT), TestHandler) as server: 41 | server.serve_forever() -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # Jupyter Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # SageMath parsed files 80 | *.sage.py 81 | 82 | # dotenv 83 | .env 84 | 85 | # virtualenv 86 | .venv 87 | venv/ 88 | ENV/ 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | .spyproject 93 | 94 | # Rope project settings 95 | .ropeproject 96 | 97 | # mkdocs documentation 98 | /site 99 | 100 | # mypy 101 | .mypy_cache/ 102 | 103 | compile.sh 104 | *_upb2.py 105 | .idea/ 106 | client.sh 107 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # micropython-uprotobuf 2 | 3 | ## Project Status 4 | This project is very much a work in progress and, as such, is incomplete. Currently protocol specifications can be compiled 5 | via the plugin and messages can be parsed. 6 | 7 | ### Things To Do 8 | * Handle repeats and groups 9 | 10 | ## Linux Usage 11 | It is assumed that Google Protobuf has been installed and the compiler (`protoc`) is on the `$PATH`. It is also assumed that 12 | a version of Python (preferably 3) is also available and the `protobuf` module has been installed (`pip install protobuf`). 13 | 14 | Assuming you have a protocol specification file available (in this case called `test1.proto`) containing something like the following: 15 | 16 | ```proto 17 | syntax = "proto2"; 18 | 19 | package test1; 20 | 21 | message test1 { 22 | enum test1enum { 23 | ValueA=1; 24 | ValueB=2; 25 | ValueC=3; 26 | } 27 | 28 | required int32 a = 1; 29 | required string b = 2; 30 | required int64 c = 4; 31 | required float d = 5; 32 | required double e = 6; 33 | required test1enum f = 7 [default=ValueA]; 34 | optional bool g = 8; 35 | repeated fixed32 h = 9; 36 | required sint32 i = 10; 37 | required sfixed64 j = 11; 38 | } 39 | ``` 40 | 41 | then a micropython compatible module can be created using the uprotobuf plugin as follows: 42 | 43 | ```sh 44 | $ git clone https://github.com/jazzycamel/micropython-uprotobuf.git 45 | $ cd micropython-uprotobuf 46 | $ chmod +x uprotobuf_plugin.py 47 | $ protoc test1.proto --plugin=protoc-gen-custom=uprotobuf_plugin.py --custom_out=. test1.proto 48 | ``` 49 | Note: on Windows use 50 | ``` 51 | > protoc test1.proto --plugin=protoc-gen-custom=uprotobuf_plugin.bat --custom_out=. --proto_path=your\path\to\test1.proto test1.proto 52 | ``` 53 | 54 | This will generate a python module named `test1_upb2.py`, which can be imported and used by micropython, containing a 55 | class named `Test1Message`. This class can currently be used to parse binary messages as follows: 56 | 57 | ```python 58 | from test1_upb2 import Test1Message 59 | 60 | message=Test1Message() 61 | ok=message.parse("\x08\x96\x01") 62 | ``` 63 | 64 | ## Windows Usage 65 | It is again assumed, as in "Linux Usage", that you will have installed protoc and placed it on the PATH, installed the protobuf Python module, and created the 'test1.proto' file. 66 | 67 | A micropython compatible module can be created using the uprotobuf plugin as follows: 68 | 69 | 1. Download the code and change into the correct directory: 70 | 71 | ```cmd 72 | git clone https://github.com/jazzycamel/micropython-uprotobuf.git 73 | cd micropython-uprotobuf 74 | ``` 75 | 76 | 2. Create a file in this directory called 'uprotobuf_plugin.bat' with the following contents: 77 | 78 | ```bat 79 | @py 80 | ``` 81 | 82 | For example: 83 | 84 | ```bat 85 | @py C:\Users\me\Desktop\buf\uprotobuf_plugin.py 86 | ``` 87 | Replace @py with the @ sign and then however you run the python command in CMD. This might be python or a filepath to the python executable. 88 | 89 | 3. Run the command 90 | ```bat 91 | protoc test1.proto --custom_out=. --plugin=protoc-gen-custom= test1.proto 92 | ``` 93 | 94 | For example: 95 | 96 | ```bat 97 | protoc test1.proto --custom_out=. --plugin=protoc-gen-custom=C:\Users\me\Desktop\buf\micropython-uprotobuf\uprotobuf_plugin.bat test1.proto 98 | ``` 99 | 100 | Why do we need to do this? Modern versions of Protoc on Windows won't accept .py files for the plugin files but they will accept a batch script which triggers the Python file. 101 | 102 | ## Command Explanation 103 | ```--plugin=protoc-gen-custom=``` means the protobuf generated code will be generated by a plugin with name "custom", which can be run by running ``````. 104 | The output of the plugin to generate the code is saved at the file indicated with ```--_out``` where `````` is the name of the plugin, in this case custom. 105 | 106 | ## Note 107 | This plugin was written for protobuf version 2. Version 3 has not been tested and may or may not work. 108 | 109 | ## Client/Server Example 110 | In the `test` directory there are client and server scripts that demonstrate micropython encoding a message via 111 | micropython-uprotobuf, transmitting that message via TCP and then decoding the message via the standard Google 112 | protobuf python implementation. 113 | 114 | First, run the server as follows: 115 | ```sh 116 | $ cd tests/server 117 | $ python3 server.py 118 | ``` 119 | 120 | Then run the client as follows: 121 | ```sh 122 | $ cd tests/client 123 | $ micropython client.py 124 | ``` 125 | 126 | Both server and client use a protocol specification that can be found in `tests/proto/tests.proto`. There is also a 127 | script named `generate.sh` that is used to generate the python and micropython modules used by the example scripts. 128 | -------------------------------------------------------------------------------- /uprotobuf_plugin.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from itertools import chain 3 | import os.path as osp 4 | from google.protobuf.compiler import plugin_pb2 as plugin 5 | from google.protobuf.descriptor_pb2 import DescriptorProto, EnumDescriptorProto, FieldDescriptorProto as FProto 6 | 7 | from uprotobuf import enum, FieldType 8 | 9 | def traverse(proto_file): 10 | def _traverse(package, items): 11 | for item in items: 12 | yield item,package 13 | 14 | if isinstance(item, DescriptorProto): 15 | for enum in item.enum_type: 16 | yield enum,package 17 | 18 | for nested in item.nested_type: 19 | nested_package=package+item.name 20 | for nested_item in _traverse(nested, nested_package): 21 | yield nested_item, nested_package 22 | return chain( 23 | _traverse(proto_file.package, proto_file.enum_type), 24 | _traverse(proto_file.package, proto_file.message_type) 25 | ) 26 | 27 | def getType(field_type): 28 | if field_type==FProto.TYPE_BOOL: type="WireType.Varint"; subType="VarintSubType.Bool" 29 | elif field_type==FProto.TYPE_BYTES: type="WireType.Length"; subType="LengthSubType.Bytes" 30 | elif field_type==FProto.TYPE_DOUBLE: type="WireType.Bit64"; subType="FixedSubType.Double" 31 | elif field_type==FProto.TYPE_ENUM: type="WireType.Varint"; subType="VarintSubType.Enum" 32 | elif field_type==FProto.TYPE_FIXED32: type="WireType.Bit32"; subType="FixedSubType.Fixed32" 33 | elif field_type==FProto.TYPE_FIXED64: type="WireType.Bit64"; subType="FixedSubType.Fixed64" 34 | elif field_type==FProto.TYPE_FLOAT: type="WireType.Bit32"; subType="FixedSubType.Float" 35 | elif field_type==FProto.TYPE_GROUP: type="WireType.Length"; subType="LengthSubType.Group" 36 | elif field_type==FProto.TYPE_INT32: type="WireType.Varint"; subType="VarintSubType.Int32" 37 | elif field_type==FProto.TYPE_INT64: type="WireType.Varint"; subType="VarintSubType.Int64" 38 | elif field_type==FProto.TYPE_MESSAGE: type="WireType.Length"; subType="LengthSubType.Message" 39 | elif field_type==FProto.TYPE_SFIXED32: type="WireType.Bit32"; subType="FixedSubType.SignedFixed32" 40 | elif field_type==FProto.TYPE_SFIXED64: type="WireType.Bit64"; subType="FixedSubType.SignedFixed64" 41 | elif field_type==FProto.TYPE_SINT32: type="WireType.Varint"; subType="VarintSubType.SInt32" 42 | elif field_type==FProto.TYPE_SINT64: type="WireType.Varint"; subType="VarintSubType.SInt64" 43 | elif field_type==FProto.TYPE_STRING: type="WireType.Length"; subType="LengthSubType.String" 44 | elif field_type==FProto.TYPE_UINT32: type="WireType.Varint"; subType="VarintSubType.UInt32" 45 | elif field_type==FProto.TYPE_UINT64: type="WireType.Varint"; subType="VarintSubType.UInt64" 46 | else: raise Exception() 47 | return type,subType 48 | 49 | def getFieldType(field_type): 50 | assert FieldType.isValid(field_type) 51 | name=FieldType.reverse_mapping[field_type] 52 | return "FieldType.{}".format(name) 53 | 54 | def generateCode(request, response): 55 | for proto_file in request.proto_file: 56 | output="from uprotobuf import *\n\n" 57 | 58 | enums="" 59 | fields="" 60 | 61 | for item,package in traverse(proto_file): 62 | if isinstance(item, DescriptorProto): 63 | fields+="\n@registerMessage\nclass {}Message(Message):\n".format(item.name.capitalize()) 64 | fields+=" _proto_fields=[\n" 65 | 66 | for field in item.field: 67 | type,subType=getType(field.type) 68 | 69 | fields+=" dict(name='{}', type={}, subType={}, fieldType={}, id={}".format( 70 | field.name, 71 | type, 72 | subType, 73 | getFieldType(field.label), 74 | field.number 75 | ) 76 | if field.type==FProto.TYPE_ENUM: 77 | enum_name=field.type_name.split('.')[-1].capitalize() 78 | fields+=", enum={}".format(enum_name) 79 | if field.type==FProto.TYPE_MESSAGE: 80 | fields+=", mType='{}'".format(field.type_name) 81 | fields+="),\n" 82 | 83 | # fields+=" # Label: {}, Default Value: {}, Options: {}\n".format( 84 | # field.label, # FProto.LABEL_OPTIONAL, FProto.LABEL_REPEATED, FProto.LABEL_REQUIRED 85 | # field.default_value, 86 | # field.options, 87 | # ) 88 | 89 | fields+=" ]\n" 90 | 91 | elif isinstance(item, EnumDescriptorProto): 92 | enums+="\n{}=enum(\n".format(item.name.capitalize()) 93 | for value in item.value: 94 | enums+=' {}={},\n'.format(value.name, value.number) 95 | enums+=")\n" 96 | 97 | output+=enums+fields 98 | f=response.file.add() 99 | f.name="{}_upb2.py".format(osp.splitext(proto_file.name)[0]) 100 | f.content=output 101 | 102 | if __name__=="__main__": 103 | from sys import stdin, stdout 104 | 105 | data=stdin.buffer.read() 106 | request=plugin.CodeGeneratorRequest() 107 | request.ParseFromString(data) 108 | response=plugin.CodeGeneratorResponse() 109 | generateCode(request, response) 110 | output=response.SerializeToString() 111 | stdout.buffer.write(output) 112 | -------------------------------------------------------------------------------- /tests/server/tests_pb2.py: -------------------------------------------------------------------------------- 1 | # Generated by the protocol buffer compiler. DO NOT EDIT! 2 | # source: tests.proto 3 | 4 | import sys 5 | _b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) 6 | from google.protobuf import descriptor as _descriptor 7 | from google.protobuf import message as _message 8 | from google.protobuf import reflection as _reflection 9 | from google.protobuf import symbol_database as _symbol_database 10 | from google.protobuf import descriptor_pb2 11 | # @@protoc_insertion_point(imports) 12 | 13 | _sym_db = _symbol_database.Default() 14 | 15 | 16 | 17 | 18 | DESCRIPTOR = _descriptor.FileDescriptor( 19 | name='tests.proto', 20 | package='Tests', 21 | syntax='proto2', 22 | serialized_pb=_b('\n\x0btests.proto\x12\x05Tests\"\xc7\x02\n\x05Test1\x12\r\n\x05Int32\x18\x01 \x01(\x05\x12\x0e\n\x06Sint32\x18\x02 \x01(\x11\x12\x0e\n\x06Uint32\x18\x03 \x01(\r\x12\r\n\x05Int64\x18\x04 \x01(\x03\x12\x0e\n\x06Sint64\x18\x05 \x01(\x12\x12\x0e\n\x06Uint64\x18\x06 \x01(\x04\x12\x0c\n\x04\x42ool\x18\x07 \x01(\x08\x12,\n\x04\x45num\x18\x08 \x01(\x0e\x32\x16.Tests.Test1.Test1Enum:\x06ValueA\x12\x0f\n\x07\x46ixed32\x18\t \x01(\x07\x12\x0f\n\x07\x46ixed64\x18\n \x01(\x06\x12\x10\n\x08Sfixed32\x18\x0b \x01(\x0f\x12\x10\n\x08Sfixed64\x18\x0c \x01(\x10\x12\r\n\x05\x46loat\x18\r \x01(\x02\x12\x0e\n\x06\x44ouble\x18\x0e \x01(\x01\x12\x0e\n\x06String\x18\x0f \x01(\t\"/\n\tTest1Enum\x12\n\n\x06ValueA\x10\x01\x12\n\n\x06ValueB\x10\x02\x12\n\n\x06ValueC\x10\x03') 23 | ) 24 | 25 | 26 | 27 | _TEST1_TEST1ENUM = _descriptor.EnumDescriptor( 28 | name='Test1Enum', 29 | full_name='Tests.Test1.Test1Enum', 30 | filename=None, 31 | file=DESCRIPTOR, 32 | values=[ 33 | _descriptor.EnumValueDescriptor( 34 | name='ValueA', index=0, number=1, 35 | options=None, 36 | type=None), 37 | _descriptor.EnumValueDescriptor( 38 | name='ValueB', index=1, number=2, 39 | options=None, 40 | type=None), 41 | _descriptor.EnumValueDescriptor( 42 | name='ValueC', index=2, number=3, 43 | options=None, 44 | type=None), 45 | ], 46 | containing_type=None, 47 | options=None, 48 | serialized_start=303, 49 | serialized_end=350, 50 | ) 51 | _sym_db.RegisterEnumDescriptor(_TEST1_TEST1ENUM) 52 | 53 | 54 | _TEST1 = _descriptor.Descriptor( 55 | name='Test1', 56 | full_name='Tests.Test1', 57 | filename=None, 58 | file=DESCRIPTOR, 59 | containing_type=None, 60 | fields=[ 61 | _descriptor.FieldDescriptor( 62 | name='Int32', full_name='Tests.Test1.Int32', index=0, 63 | number=1, type=5, cpp_type=1, label=1, 64 | has_default_value=False, default_value=0, 65 | message_type=None, enum_type=None, containing_type=None, 66 | is_extension=False, extension_scope=None, 67 | options=None), 68 | _descriptor.FieldDescriptor( 69 | name='Sint32', full_name='Tests.Test1.Sint32', index=1, 70 | number=2, type=17, cpp_type=1, label=1, 71 | has_default_value=False, default_value=0, 72 | message_type=None, enum_type=None, containing_type=None, 73 | is_extension=False, extension_scope=None, 74 | options=None), 75 | _descriptor.FieldDescriptor( 76 | name='Uint32', full_name='Tests.Test1.Uint32', index=2, 77 | number=3, type=13, cpp_type=3, label=1, 78 | has_default_value=False, default_value=0, 79 | message_type=None, enum_type=None, containing_type=None, 80 | is_extension=False, extension_scope=None, 81 | options=None), 82 | _descriptor.FieldDescriptor( 83 | name='Int64', full_name='Tests.Test1.Int64', index=3, 84 | number=4, type=3, cpp_type=2, label=1, 85 | has_default_value=False, default_value=0, 86 | message_type=None, enum_type=None, containing_type=None, 87 | is_extension=False, extension_scope=None, 88 | options=None), 89 | _descriptor.FieldDescriptor( 90 | name='Sint64', full_name='Tests.Test1.Sint64', index=4, 91 | number=5, type=18, cpp_type=2, label=1, 92 | has_default_value=False, default_value=0, 93 | message_type=None, enum_type=None, containing_type=None, 94 | is_extension=False, extension_scope=None, 95 | options=None), 96 | _descriptor.FieldDescriptor( 97 | name='Uint64', full_name='Tests.Test1.Uint64', index=5, 98 | number=6, type=4, cpp_type=4, label=1, 99 | has_default_value=False, default_value=0, 100 | message_type=None, enum_type=None, containing_type=None, 101 | is_extension=False, extension_scope=None, 102 | options=None), 103 | _descriptor.FieldDescriptor( 104 | name='Bool', full_name='Tests.Test1.Bool', index=6, 105 | number=7, type=8, cpp_type=7, label=1, 106 | has_default_value=False, default_value=False, 107 | message_type=None, enum_type=None, containing_type=None, 108 | is_extension=False, extension_scope=None, 109 | options=None), 110 | _descriptor.FieldDescriptor( 111 | name='Enum', full_name='Tests.Test1.Enum', index=7, 112 | number=8, type=14, cpp_type=8, label=1, 113 | has_default_value=True, default_value=1, 114 | message_type=None, enum_type=None, containing_type=None, 115 | is_extension=False, extension_scope=None, 116 | options=None), 117 | _descriptor.FieldDescriptor( 118 | name='Fixed32', full_name='Tests.Test1.Fixed32', index=8, 119 | number=9, type=7, cpp_type=3, label=1, 120 | has_default_value=False, default_value=0, 121 | message_type=None, enum_type=None, containing_type=None, 122 | is_extension=False, extension_scope=None, 123 | options=None), 124 | _descriptor.FieldDescriptor( 125 | name='Fixed64', full_name='Tests.Test1.Fixed64', index=9, 126 | number=10, type=6, cpp_type=4, label=1, 127 | has_default_value=False, default_value=0, 128 | message_type=None, enum_type=None, containing_type=None, 129 | is_extension=False, extension_scope=None, 130 | options=None), 131 | _descriptor.FieldDescriptor( 132 | name='Sfixed32', full_name='Tests.Test1.Sfixed32', index=10, 133 | number=11, type=15, cpp_type=1, label=1, 134 | has_default_value=False, default_value=0, 135 | message_type=None, enum_type=None, containing_type=None, 136 | is_extension=False, extension_scope=None, 137 | options=None), 138 | _descriptor.FieldDescriptor( 139 | name='Sfixed64', full_name='Tests.Test1.Sfixed64', index=11, 140 | number=12, type=16, cpp_type=2, label=1, 141 | has_default_value=False, default_value=0, 142 | message_type=None, enum_type=None, containing_type=None, 143 | is_extension=False, extension_scope=None, 144 | options=None), 145 | _descriptor.FieldDescriptor( 146 | name='Float', full_name='Tests.Test1.Float', index=12, 147 | number=13, type=2, cpp_type=6, label=1, 148 | has_default_value=False, default_value=float(0), 149 | message_type=None, enum_type=None, containing_type=None, 150 | is_extension=False, extension_scope=None, 151 | options=None), 152 | _descriptor.FieldDescriptor( 153 | name='Double', full_name='Tests.Test1.Double', index=13, 154 | number=14, type=1, cpp_type=5, label=1, 155 | has_default_value=False, default_value=float(0), 156 | message_type=None, enum_type=None, containing_type=None, 157 | is_extension=False, extension_scope=None, 158 | options=None), 159 | _descriptor.FieldDescriptor( 160 | name='String', full_name='Tests.Test1.String', index=14, 161 | number=15, type=9, cpp_type=9, label=1, 162 | has_default_value=False, default_value=_b("").decode('utf-8'), 163 | message_type=None, enum_type=None, containing_type=None, 164 | is_extension=False, extension_scope=None, 165 | options=None), 166 | ], 167 | extensions=[ 168 | ], 169 | nested_types=[], 170 | enum_types=[ 171 | _TEST1_TEST1ENUM, 172 | ], 173 | options=None, 174 | is_extendable=False, 175 | syntax='proto2', 176 | extension_ranges=[], 177 | oneofs=[ 178 | ], 179 | serialized_start=23, 180 | serialized_end=350, 181 | ) 182 | 183 | _TEST1.fields_by_name['Enum'].enum_type = _TEST1_TEST1ENUM 184 | _TEST1_TEST1ENUM.containing_type = _TEST1 185 | DESCRIPTOR.message_types_by_name['Test1'] = _TEST1 186 | _sym_db.RegisterFileDescriptor(DESCRIPTOR) 187 | 188 | Test1 = _reflection.GeneratedProtocolMessageType('Test1', (_message.Message,), dict( 189 | DESCRIPTOR = _TEST1, 190 | __module__ = 'tests_pb2' 191 | # @@protoc_insertion_point(class_scope:Tests.Test1) 192 | )) 193 | _sym_db.RegisterMessage(Test1) 194 | 195 | 196 | # @@protoc_insertion_point(module_scope) 197 | -------------------------------------------------------------------------------- /uprotobuf.py: -------------------------------------------------------------------------------- 1 | try: import ustruct as struct 2 | except ImportError: import struct 3 | 4 | class UnknownTypeException(Exception): pass 5 | class ValueNotSetException(Exception): pass 6 | 7 | def partial(func, *args, **kwargs): 8 | def _partial(*more_args, **more_kwargs): 9 | kw = kwargs.copy() 10 | kw.update(more_kwargs) 11 | return func(*(args + more_args), **kw) 12 | return _partial 13 | 14 | def enum(*sequential, **named): 15 | def isValid(cls, type): 16 | return type in cls.reverse_mapping 17 | 18 | enums=dict(((x,i) for i,x in enumerate(sequential)), **named) 19 | enums['reverse_mapping']=dict((value,key) for key,value in enums.items()) 20 | enums['isValid']=classmethod(isValid) 21 | return type('Enum', (object,), enums) 22 | 23 | _MessageRegister = {} 24 | def registerMessage(cls): 25 | global _MessageRegister 26 | _MessageRegister[".protobuf.{}".format(cls.__name__).lower()] = cls 27 | return cls 28 | def getMessageType(name): 29 | global _MessageRegister 30 | key = name+"Message" 31 | key = key.lower() 32 | return _MessageRegister[key] 33 | 34 | def getBytesForId(_id, typ): 35 | if _id < 0xF:return [(_id<<3)|typ]#we shouldnt get bigger than 0xFF (we shift left 3 bit => 0xF-> 0x78+typ => max 0x7F 36 | else: 37 | data =[] 38 | value = (_id << 3) | typ 39 | while value >= 0x7f: 40 | data.append((value&0x7F)|0x80) 41 | value=value>>7 42 | data.append(value&0x7F) 43 | return data 44 | 45 | WireType=enum(Invalid=-1, Varint=0, Bit64=1, Length=2, Bit32=5) 46 | FieldType=enum(Invalid=-1, Optional=1, Required=2, Repeated=3) 47 | 48 | class VarType(object): 49 | def __init__(self, id=None, data=None, subType=-1, fieldType=-1, **kwargs): 50 | self._id=id 51 | self._data=data 52 | self._value=[] if fieldType is FieldType.Repeated else None 53 | self._subType=subType 54 | self._fieldType=fieldType 55 | self._mType = kwargs.get("mType","") 56 | 57 | def reset(self): 58 | self._data=None 59 | self._value=[] if self._fieldType is FieldType.Repeated else None 60 | 61 | def isValid(self): 62 | if not self._fieldType==FieldType.Required: return True 63 | return self._data!=None and self._value not in (None, []) 64 | 65 | @property 66 | def id(self): return self._id 67 | 68 | @staticmethod 69 | def type(): return WireType.Invalid 70 | 71 | def data(self): return self._data 72 | 73 | def setData(self, data): 74 | if self._data==data: return 75 | self._data=data 76 | 77 | def value(self): return self._value 78 | 79 | def setValue(self, value): 80 | if self._value==value: return 81 | self._value=value 82 | 83 | def __repr__(self): 84 | return "{}({}: {})".format(self.__class__.__name__, self._id, self._value) 85 | 86 | @staticmethod 87 | def encodeZigZag(n, bits=32): return (n<<1)^(n>>(bits-1)) 88 | 89 | @staticmethod 90 | def decodeZigZag(n): return (n>>1)^-(n&1) 91 | 92 | VarintSubType=enum( 93 | Int32=1, 94 | Int64=2, 95 | UInt32=3, 96 | UInt64=4, 97 | SInt32=5, 98 | SInt64=6, 99 | Bool=7, 100 | Enum=8, 101 | ) 102 | 103 | class Varint(VarType): 104 | def __init__(self, id=None, data=None, subType=-1, fieldType=-1, **kwargs): 105 | super().__init__(id, data, subType, fieldType, **kwargs) 106 | 107 | if subType==VarintSubType.Enum: 108 | self._enum=kwargs['enum'] 109 | 110 | @staticmethod 111 | def type(): return WireType.Varint 112 | 113 | def setData(self, data): 114 | if self._data==data: return 115 | self._data=data 116 | 117 | value=0 118 | for i,d in enumerate(self._data): 119 | value|=(d&0x7f)<<(i*7) 120 | 121 | if self._subType in (VarintSubType.SInt32, VarintSubType.SInt64): 122 | value=self.decodeZigZag(value) 123 | elif self._subType==VarintSubType.Bool: 124 | value=bool(value) 125 | 126 | if self._fieldType==FieldType.Repeated: 127 | self._value.append(value) 128 | else: self._value=value 129 | 130 | def setValue(self, value): 131 | if self._value==value: return 132 | self._value=value 133 | 134 | if self._subType in (VarintSubType.SInt32, VarintSubType.SInt64): 135 | value=self.encodeZigZag(value, 32 if self._subType==VarintSubType.SInt32 else 64) 136 | 137 | data=getBytesForId(self._id,WireType.Varint) 138 | if self._subType in (VarintSubType.Int32, VarintSubType.UInt32, VarintSubType.SInt32): 139 | data.append((value&0x7F)|0x80) 140 | value=value>>7 141 | data.append(value&0x7F) 142 | 143 | elif self._subType in (VarintSubType.Int64, VarintSubType.UInt64, VarintSubType.SInt64, VarintSubType.Enum): 144 | for i in range(4): 145 | data.append((value&0x7F)) 146 | value=value>>7 147 | if value==0: break 148 | 149 | for i in range(1,len(data)-1): 150 | data[i]|=0x80 151 | elif self._subType==VarintSubType.Bool: 152 | data.append(int(value)) 153 | 154 | self._data=bytes(data) 155 | 156 | def __repr__(self): 157 | if self._subType!=VarintSubType.Enum: 158 | return super().__repr__() 159 | 160 | valueName=self._enum.reverse_mapping.get(self._value, None) 161 | return "{}({}: {})".format(self.__class__.__name__, self._id, valueName) 162 | 163 | LengthSubType=enum( 164 | String=1, 165 | Message=2, 166 | Group=3, 167 | Bytes=4, 168 | ) 169 | 170 | class Length(VarType): 171 | @staticmethod 172 | def type(): return WireType.Length 173 | 174 | def setData(self, data): 175 | if self._data==data: return 176 | self._data=data 177 | 178 | if self._subType==LengthSubType.String: 179 | value=self._data.decode('utf8') 180 | elif self._subType== LengthSubType.Message: 181 | if not hasattr(self, "_mType"): raise Exception("_mType not set") 182 | value = getMessageType(self._mType)() 183 | value.parse(self._data) 184 | else: raise Error("Not implemented") 185 | 186 | 187 | if self._fieldType==FieldType.Repeated: 188 | self._value.append(value) 189 | else: self._value=value 190 | 191 | def setValue(self, value): 192 | if self._value==value: return 193 | self._value=value 194 | 195 | data=getBytesForId(self._id, self.type()) 196 | data.append(len(value)) 197 | self._data=bytes(data) 198 | if isinstance(value,bytes): self._data += value 199 | else:self._data+= bytes(value, "utf8") 200 | 201 | FixedSubType=enum( 202 | Fixed64=1, 203 | SignedFixed64=2, 204 | Double=3, 205 | Fixed32=4, 206 | SignedFixed32=5, 207 | Float=6, 208 | ) 209 | 210 | class Fixed(VarType): 211 | def __init__(self, id=None, data=None, subType=-1, fieldType=-1, **kwargs): 212 | super().__init__(id,data,subType,fieldType,**kwargs) 213 | if subType==FixedSubType.Float: self._fmt='>3#fast path, single byte 311 | else:#varint 312 | bitpos = 0 313 | result =0 314 | while byte & 0x80: 315 | result |= (byte&0x7F) << bitpos#result 316 | bitpos += 7 317 | i+=1 318 | byte = msg[i] 319 | if bitpos > 32:raise Exception("cant handle >32bit varint") 320 | result |= ((byte&0x7F) << bitpos)#result 321 | type=result&0x7 322 | id=result>>3 323 | 324 | name=self._fieldsLUT[id] 325 | data=None 326 | if type==WireType.Varint: 327 | data=[] 328 | while True: 329 | i+=1 330 | data.append(msg[i]) 331 | if not msg[i]&0x80: break 332 | i+=1 333 | 334 | elif type in (WireType.Bit32, WireType.Bit64, WireType.Length): 335 | if type==WireType.Length: 336 | i+=1 337 | length=int(msg[i]) 338 | else: length=4 if type==WireType.Bit32 else 8 339 | 340 | i+=1 341 | data=msg[i:i+length] 342 | i+=length 343 | 344 | else: raise UnknownTypeException 345 | 346 | if name not in self._fields: continue 347 | self._fields[name].setData(data) 348 | return self.isValid() --------------------------------------------------------------------------------