├── .gitignore ├── LICENSE ├── README.md ├── setup.py └── src ├── protobuf_to_dict.py └── tests ├── __init__.py ├── sample.proto ├── sample_pb2.py └── test_proto_to_dict.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.pyo 3 | .DS_Store 4 | MANIFEST 5 | dist 6 | build 7 | *.egg 8 | .project 9 | .pydevproject 10 | *~ 11 | src/protobuf_to_dict.egg-info 12 | 13 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | This is free and unencumbered software released into the public domain 2 | by its author, Ben Hodgson . 3 | 4 | Anyone is free to copy, modify, publish, use, compile, sell, or 5 | distribute this software, either in source code form or as a compiled 6 | binary, for any purpose, commercial or non-commercial, and by any 7 | means. 8 | 9 | In jurisdictions that recognise copyright laws, the author or authors 10 | of this software dedicate any and all copyright interest in the 11 | software to the public domain. We make this dedication for the benefit 12 | of the public at large and to the detriment of our heirs and 13 | successors. We intend this dedication to be an overt act of 14 | relinquishment in perpetuity of all present and future rights to this 15 | software under copyright law. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 20 | IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 21 | OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 22 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 23 | OTHER DEALINGS IN THE SOFTWARE. 24 | 25 | For more information, please refer to -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # protobuf-to-dict 2 | 3 | protobuf-to-dict is a small Python library for creating dicts from protocol 4 | buffers. It is intended to be used as an intermediate step before 5 | serialization (e.g. to JSON). 6 | 7 | ## Installation 8 | 9 | Use `pip install protobuf-to-dict` or `python setup.py install`. 10 | 11 | ## Example 12 | 13 | Given the `google.protobuf.message.Message` subclass `MyMessage`: 14 | 15 | ```python 16 | >>> from protobuf_to_dict import protobuf_to_dict 17 | >>> my_message = MyMessage() 18 | >>> # pb_my_message is a protobuf string 19 | >>> my_message.ParseFromString(pb_my_message) 20 | >>> protobuf_to_dict(my_message) 21 | {'message': 'Hello'} 22 | ``` 23 | 24 | ## Caveats 25 | 26 | This library grew out of the desire to serialize a protobuf-encoded message to 27 | [JSON](http://json.org/). As JSON has no built-in binary type (all strings in 28 | JSON are Unicode strings), any field whose type is 29 | `FieldDescriptor.TYPE_BYTES` is, by default, converted to a base64-encoded 30 | string. 31 | 32 | If you want to override this behaviour, you may do so by passing 33 | `protobuf_to_dict` a dictionary of protobuf types to callables via the 34 | `type_callable_map` kwarg: 35 | 36 | ```python 37 | >>> from copy import copy 38 | >>> from google.protobuf.descriptor import FieldDescriptor 39 | >>> from protobuf_to_dict import protobuf_to_dict, TYPE_CALLABLE_MAP 40 | >>> 41 | >>> type_callable_map = copy(TYPE_CALLABLE_MAP) 42 | >>> # convert TYPE_BYTES to a Python bytestring 43 | >>> type_callable_map[FieldDescriptor.TYPE_BYTES] = str 44 | >>> 45 | >>> # my_message is a google.protobuf.message.Message instance 46 | >>> protobuf_to_dict(my_message, type_callable_map=type_callable_map) 47 | ``` 48 | 49 | By default, the integer representation is used for enum values. To use their 50 | string labels instead, pass `use_enum_labels=True` into `protobuf_to_dict`: 51 | 52 | ```python 53 | >>> protobuf_to_dict(my_message, use_enum_labels=True) 54 | ``` 55 | 56 | ## Unit testing 57 | 58 | Tests are under `src/tests/`. 59 | 60 | ```sh 61 | $ python setup.py nosetests 62 | ``` 63 | 64 | To regenerate `src/tests/sample_pb2.py`: 65 | 66 | ```sh 67 | $ protoc --python_out=src -Isrc src/tests/sample.proto 68 | ``` 69 | 70 | ## Authors 71 | 72 | protobuf-to-dict is written and maintained by 73 | [Ben Hodgson](http://benhodgson.com/), with significant contributions from 74 | [Nino Walker](https://github.com/ninowalker), 75 | [Jonathan Klaassen](https://github.com/jaklaassen), and 76 | [Tristram Gräbener](http://blog.tristramg.eu/). 77 | 78 | 79 | ## (Un)license 80 | 81 | This is free and unencumbered software released into the public domain. 82 | 83 | Anyone is free to copy, modify, publish, use, compile, sell, or distribute 84 | this software, either in source code form or as a compiled binary, for any 85 | purpose, commercial or non-commercial, and by any means. 86 | 87 | In jurisdictions that recognize copyright laws, the author or authors of this 88 | software dedicate any and all copyright interest in the software to the public 89 | domain. We make this dedication for the benefit of the public at large and to 90 | the detriment of our heirs and successors. We intend this dedication to be an 91 | overt act of relinquishment in perpetuity of all present and future rights to 92 | this software under copyright law. 93 | 94 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 95 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 96 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 97 | AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 98 | ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 99 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 100 | 101 | For more information, please refer to 102 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name='protobuf-to-dict', 5 | description='A teeny Python library for creating Python dicts from ' 6 | 'protocol buffers and the reverse. Useful as an intermediate step ' 7 | 'before serialisation (e.g. to JSON).', 8 | version='0.1.0', 9 | author='Ben Hodgson', 10 | author_email='ben@benhodgson.com', 11 | url='https://github.com/benhodgson/protobuf-to-dict', 12 | license='Public Domain', 13 | keywords=['protobuf', 'json', 'dict'], 14 | install_requires=['protobuf>=2.3.0'], 15 | package_dir={'':'src'}, 16 | py_modules=['protobuf_to_dict'], 17 | setup_requires=['protobuf>=2.3.0', 'nose>=1.0', 'coverage', 'nosexcover'], 18 | test_suite = 'nose.collector', 19 | classifiers=[ 20 | 'Programming Language :: Python', 21 | 'Development Status :: 4 - Beta', 22 | 'Intended Audience :: Developers', 23 | 'License :: Public Domain', 24 | 'Operating System :: OS Independent', 25 | 'Topic :: Software Development :: Libraries :: Python Modules', 26 | ], 27 | ) 28 | -------------------------------------------------------------------------------- /src/protobuf_to_dict.py: -------------------------------------------------------------------------------- 1 | from google.protobuf.message import Message 2 | from google.protobuf.descriptor import FieldDescriptor 3 | 4 | 5 | __all__ = ["protobuf_to_dict", "TYPE_CALLABLE_MAP", "dict_to_protobuf", "REVERSE_TYPE_CALLABLE_MAP"] 6 | 7 | 8 | EXTENSION_CONTAINER = '___X' 9 | 10 | 11 | TYPE_CALLABLE_MAP = { 12 | FieldDescriptor.TYPE_DOUBLE: float, 13 | FieldDescriptor.TYPE_FLOAT: float, 14 | FieldDescriptor.TYPE_INT32: int, 15 | FieldDescriptor.TYPE_INT64: long, 16 | FieldDescriptor.TYPE_UINT32: int, 17 | FieldDescriptor.TYPE_UINT64: long, 18 | FieldDescriptor.TYPE_SINT32: int, 19 | FieldDescriptor.TYPE_SINT64: long, 20 | FieldDescriptor.TYPE_FIXED32: int, 21 | FieldDescriptor.TYPE_FIXED64: long, 22 | FieldDescriptor.TYPE_SFIXED32: int, 23 | FieldDescriptor.TYPE_SFIXED64: long, 24 | FieldDescriptor.TYPE_BOOL: bool, 25 | FieldDescriptor.TYPE_STRING: unicode, 26 | FieldDescriptor.TYPE_BYTES: lambda b: b.encode("base64"), 27 | FieldDescriptor.TYPE_ENUM: int, 28 | } 29 | 30 | 31 | def repeated(type_callable): 32 | return lambda value_list: [type_callable(value) for value in value_list] 33 | 34 | 35 | def enum_label_name(field, value): 36 | return field.enum_type.values_by_number[int(value)].name 37 | 38 | 39 | def protobuf_to_dict(pb, type_callable_map=TYPE_CALLABLE_MAP, use_enum_labels=False): 40 | result_dict = {} 41 | extensions = {} 42 | for field, value in pb.ListFields(): 43 | type_callable = _get_field_value_adaptor(pb, field, type_callable_map, use_enum_labels) 44 | if field.label == FieldDescriptor.LABEL_REPEATED: 45 | type_callable = repeated(type_callable) 46 | 47 | if field.is_extension: 48 | extensions[str(field.number)] = type_callable(value) 49 | continue 50 | 51 | result_dict[field.name] = type_callable(value) 52 | 53 | if extensions: 54 | result_dict[EXTENSION_CONTAINER] = extensions 55 | return result_dict 56 | 57 | 58 | def _get_field_value_adaptor(pb, field, type_callable_map=TYPE_CALLABLE_MAP, use_enum_labels=False): 59 | if field.type == FieldDescriptor.TYPE_MESSAGE: 60 | # recursively encode protobuf sub-message 61 | return lambda pb: protobuf_to_dict(pb, 62 | type_callable_map=type_callable_map, 63 | use_enum_labels=use_enum_labels) 64 | 65 | if use_enum_labels and field.type == FieldDescriptor.TYPE_ENUM: 66 | return lambda value: enum_label_name(field, value) 67 | 68 | if field.type in type_callable_map: 69 | return type_callable_map[field.type] 70 | 71 | raise TypeError("Field %s.%s has unrecognised type id %d" % ( 72 | pb.__class__.__name__, field.name, field.type)) 73 | 74 | 75 | def get_bytes(value): 76 | return value.decode('base64') 77 | 78 | 79 | REVERSE_TYPE_CALLABLE_MAP = { 80 | FieldDescriptor.TYPE_BYTES: get_bytes, 81 | } 82 | 83 | 84 | def dict_to_protobuf(pb_klass_or_instance, values, type_callable_map=REVERSE_TYPE_CALLABLE_MAP, strict=True): 85 | """Populates a protobuf model from a dictionary. 86 | 87 | :param pb_klass_or_instance: a protobuf message class, or an protobuf instance 88 | :type pb_klass_or_instance: a type or instance of a subclass of google.protobuf.message.Message 89 | :param dict values: a dictionary of values. Repeated and nested values are 90 | fully supported. 91 | :param dict type_callable_map: a mapping of protobuf types to callables for setting 92 | values on the target instance. 93 | :param bool strict: complain if keys in the map are not fields on the message. 94 | """ 95 | if isinstance(pb_klass_or_instance, Message): 96 | instance = pb_klass_or_instance 97 | else: 98 | instance = pb_klass_or_instance() 99 | return _dict_to_protobuf(instance, values, type_callable_map, strict) 100 | 101 | 102 | def _get_field_mapping(pb, dict_value, strict): 103 | field_mapping = [] 104 | for key, value in dict_value.items(): 105 | if key == EXTENSION_CONTAINER: 106 | continue 107 | if key not in pb.DESCRIPTOR.fields_by_name: 108 | if strict: 109 | raise KeyError("%s does not have a field called %s" % (pb, key)) 110 | continue 111 | field_mapping.append((pb.DESCRIPTOR.fields_by_name[key], value, getattr(pb, key, None))) 112 | 113 | for ext_num, ext_val in dict_value.get(EXTENSION_CONTAINER, {}).items(): 114 | try: 115 | ext_num = int(ext_num) 116 | except ValueError: 117 | raise ValueError("Extension keys must be integers.") 118 | if ext_num not in pb._extensions_by_number: 119 | if strict: 120 | raise KeyError("%s does not have a extension with number %s. Perhaps you forgot to import it?" % (pb, key)) 121 | continue 122 | ext_field = pb._extensions_by_number[ext_num] 123 | pb_val = None 124 | pb_val = pb.Extensions[ext_field] 125 | field_mapping.append((ext_field, ext_val, pb_val)) 126 | 127 | return field_mapping 128 | 129 | 130 | def _dict_to_protobuf(pb, value, type_callable_map, strict): 131 | fields = _get_field_mapping(pb, value, strict) 132 | 133 | for field, input_value, pb_value in fields: 134 | if field.label == FieldDescriptor.LABEL_REPEATED: 135 | for item in input_value: 136 | if field.type == FieldDescriptor.TYPE_MESSAGE: 137 | m = pb_value.add() 138 | _dict_to_protobuf(m, item, type_callable_map, strict) 139 | elif field.type == FieldDescriptor.TYPE_ENUM and isinstance(item, basestring): 140 | pb_value.append(_string_to_enum(field, item)) 141 | else: 142 | pb_value.append(item) 143 | continue 144 | if field.type == FieldDescriptor.TYPE_MESSAGE: 145 | _dict_to_protobuf(pb_value, input_value, type_callable_map, strict) 146 | continue 147 | 148 | if field.type in type_callable_map: 149 | input_value = type_callable_map[field.type](input_value) 150 | 151 | if field.is_extension: 152 | pb.Extensions[field] = input_value 153 | continue 154 | 155 | if field.type == FieldDescriptor.TYPE_ENUM and isinstance(input_value, basestring): 156 | input_value = _string_to_enum(field, input_value) 157 | 158 | setattr(pb, field.name, input_value) 159 | 160 | return pb 161 | 162 | def _string_to_enum(field, input_value): 163 | enum_dict = field.enum_type.values_by_name 164 | try: 165 | input_value = enum_dict[input_value].number 166 | except KeyError: 167 | raise KeyError("`%s` is not a valid value for field `%s`" % (input_value, field.name)) 168 | return input_value 169 | -------------------------------------------------------------------------------- /src/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/benhodgson/protobuf-to-dict/cd982802cd1a978c9076ca9ce9d0d75b1b426258/src/tests/__init__.py -------------------------------------------------------------------------------- /src/tests/sample.proto: -------------------------------------------------------------------------------- 1 | package tests; 2 | 3 | // protoc --python_out=src -Isrc src/tests/sample.proto 4 | 5 | message MessageOfTypes { 6 | extensions 100 to max; 7 | 8 | required double dubl = 1; 9 | required float flot = 2; 10 | required int32 i32 = 3; 11 | required int64 i64 = 4; 12 | required uint32 ui32 = 5; 13 | required uint64 ui64 = 6; 14 | required sint32 si32 = 7; 15 | required sint64 si64 = 8; 16 | required fixed32 f32 = 9; 17 | required fixed64 f64 = 17; // oops, OOO 18 | required sfixed32 sf32 = 10; 19 | required sfixed64 sf64 = 11; 20 | required bool bol = 12; 21 | required string strng = 13; 22 | required bytes byts = 14; 23 | required NestedType nested = 15; 24 | required Enum enm = 16; 25 | repeated int32 range = 18; 26 | repeated NestedType nestedRepeated = 19; 27 | repeated Enum enmRepeated = 20; 28 | 29 | message NestedType { 30 | required string req = 1; 31 | } 32 | 33 | enum Enum { 34 | A = 0; 35 | B = 1; 36 | C = 2; 37 | } 38 | } 39 | 40 | extend MessageOfTypes { 41 | optional double extDouble = 100; 42 | optional string extString = 101; 43 | } 44 | 45 | message NestedExtension { 46 | extend MessageOfTypes { 47 | optional int32 extInt = 102; 48 | optional MessageOfTypes.NestedType extNested = 103; 49 | } 50 | } -------------------------------------------------------------------------------- /src/tests/sample_pb2.py: -------------------------------------------------------------------------------- 1 | # Generated by the protocol buffer compiler. DO NOT EDIT! 2 | # source: tests/sample.proto 3 | 4 | from google.protobuf import descriptor as _descriptor 5 | from google.protobuf import message as _message 6 | from google.protobuf import reflection as _reflection 7 | from google.protobuf import descriptor_pb2 8 | # @@protoc_insertion_point(imports) 9 | 10 | 11 | 12 | 13 | DESCRIPTOR = _descriptor.FileDescriptor( 14 | name='tests/sample.proto', 15 | package='tests', 16 | serialized_pb='\n\x12tests/sample.proto\x12\x05tests\"\xf5\x03\n\x0eMessageOfTypes\x12\x0c\n\x04\x64ubl\x18\x01 \x02(\x01\x12\x0c\n\x04\x66lot\x18\x02 \x02(\x02\x12\x0b\n\x03i32\x18\x03 \x02(\x05\x12\x0b\n\x03i64\x18\x04 \x02(\x03\x12\x0c\n\x04ui32\x18\x05 \x02(\r\x12\x0c\n\x04ui64\x18\x06 \x02(\x04\x12\x0c\n\x04si32\x18\x07 \x02(\x11\x12\x0c\n\x04si64\x18\x08 \x02(\x12\x12\x0b\n\x03\x66\x33\x32\x18\t \x02(\x07\x12\x0b\n\x03\x66\x36\x34\x18\x11 \x02(\x06\x12\x0c\n\x04sf32\x18\n \x02(\x0f\x12\x0c\n\x04sf64\x18\x0b \x02(\x10\x12\x0b\n\x03\x62ol\x18\x0c \x02(\x08\x12\r\n\x05strng\x18\r \x02(\t\x12\x0c\n\x04\x62yts\x18\x0e \x02(\x0c\x12\x30\n\x06nested\x18\x0f \x02(\x0b\x32 .tests.MessageOfTypes.NestedType\x12\'\n\x03\x65nm\x18\x10 \x02(\x0e\x32\x1a.tests.MessageOfTypes.Enum\x12\r\n\x05range\x18\x12 \x03(\x05\x12\x38\n\x0enestedRepeated\x18\x13 \x03(\x0b\x32 .tests.MessageOfTypes.NestedType\x12/\n\x0b\x65nmRepeated\x18\x14 \x03(\x0e\x32\x1a.tests.MessageOfTypes.Enum\x1a\x19\n\nNestedType\x12\x0b\n\x03req\x18\x01 \x02(\t\"\x1b\n\x04\x45num\x12\x05\n\x01\x41\x10\x00\x12\x05\n\x01\x42\x10\x01\x12\x05\n\x01\x43\x10\x02*\x08\x08\x64\x10\x80\x80\x80\x80\x02\"\x84\x01\n\x0fNestedExtension2%\n\x06\x65xtInt\x12\x15.tests.MessageOfTypes\x18\x66 \x01(\x05\x32J\n\textNested\x12\x15.tests.MessageOfTypes\x18g \x01(\x0b\x32 .tests.MessageOfTypes.NestedType:(\n\textDouble\x12\x15.tests.MessageOfTypes\x18\x64 \x01(\x01:(\n\textString\x12\x15.tests.MessageOfTypes\x18\x65 \x01(\t') 17 | 18 | 19 | EXTDOUBLE_FIELD_NUMBER = 100 20 | extDouble = _descriptor.FieldDescriptor( 21 | name='extDouble', full_name='tests.extDouble', index=0, 22 | number=100, type=1, cpp_type=5, label=1, 23 | has_default_value=False, default_value=0, 24 | message_type=None, enum_type=None, containing_type=None, 25 | is_extension=True, extension_scope=None, 26 | options=None) 27 | EXTSTRING_FIELD_NUMBER = 101 28 | extString = _descriptor.FieldDescriptor( 29 | name='extString', full_name='tests.extString', index=1, 30 | number=101, type=9, cpp_type=9, label=1, 31 | has_default_value=False, default_value=unicode("", "utf-8"), 32 | message_type=None, enum_type=None, containing_type=None, 33 | is_extension=True, extension_scope=None, 34 | options=None) 35 | 36 | _MESSAGEOFTYPES_ENUM = _descriptor.EnumDescriptor( 37 | name='Enum', 38 | full_name='tests.MessageOfTypes.Enum', 39 | filename=None, 40 | file=DESCRIPTOR, 41 | values=[ 42 | _descriptor.EnumValueDescriptor( 43 | name='A', index=0, number=0, 44 | options=None, 45 | type=None), 46 | _descriptor.EnumValueDescriptor( 47 | name='B', index=1, number=1, 48 | options=None, 49 | type=None), 50 | _descriptor.EnumValueDescriptor( 51 | name='C', index=2, number=2, 52 | options=None, 53 | type=None), 54 | ], 55 | containing_type=None, 56 | options=None, 57 | serialized_start=494, 58 | serialized_end=521, 59 | ) 60 | 61 | 62 | _MESSAGEOFTYPES_NESTEDTYPE = _descriptor.Descriptor( 63 | name='NestedType', 64 | full_name='tests.MessageOfTypes.NestedType', 65 | filename=None, 66 | file=DESCRIPTOR, 67 | containing_type=None, 68 | fields=[ 69 | _descriptor.FieldDescriptor( 70 | name='req', full_name='tests.MessageOfTypes.NestedType.req', index=0, 71 | number=1, type=9, cpp_type=9, label=2, 72 | has_default_value=False, default_value=unicode("", "utf-8"), 73 | message_type=None, enum_type=None, containing_type=None, 74 | is_extension=False, extension_scope=None, 75 | options=None), 76 | ], 77 | extensions=[ 78 | ], 79 | nested_types=[], 80 | enum_types=[ 81 | ], 82 | options=None, 83 | is_extendable=False, 84 | extension_ranges=[], 85 | serialized_start=467, 86 | serialized_end=492, 87 | ) 88 | 89 | _MESSAGEOFTYPES = _descriptor.Descriptor( 90 | name='MessageOfTypes', 91 | full_name='tests.MessageOfTypes', 92 | filename=None, 93 | file=DESCRIPTOR, 94 | containing_type=None, 95 | fields=[ 96 | _descriptor.FieldDescriptor( 97 | name='dubl', full_name='tests.MessageOfTypes.dubl', index=0, 98 | number=1, type=1, cpp_type=5, label=2, 99 | has_default_value=False, default_value=0, 100 | message_type=None, enum_type=None, containing_type=None, 101 | is_extension=False, extension_scope=None, 102 | options=None), 103 | _descriptor.FieldDescriptor( 104 | name='flot', full_name='tests.MessageOfTypes.flot', index=1, 105 | number=2, type=2, cpp_type=6, label=2, 106 | has_default_value=False, default_value=0, 107 | message_type=None, enum_type=None, containing_type=None, 108 | is_extension=False, extension_scope=None, 109 | options=None), 110 | _descriptor.FieldDescriptor( 111 | name='i32', full_name='tests.MessageOfTypes.i32', index=2, 112 | number=3, type=5, cpp_type=1, label=2, 113 | has_default_value=False, default_value=0, 114 | message_type=None, enum_type=None, containing_type=None, 115 | is_extension=False, extension_scope=None, 116 | options=None), 117 | _descriptor.FieldDescriptor( 118 | name='i64', full_name='tests.MessageOfTypes.i64', index=3, 119 | number=4, type=3, cpp_type=2, label=2, 120 | has_default_value=False, default_value=0, 121 | message_type=None, enum_type=None, containing_type=None, 122 | is_extension=False, extension_scope=None, 123 | options=None), 124 | _descriptor.FieldDescriptor( 125 | name='ui32', full_name='tests.MessageOfTypes.ui32', index=4, 126 | number=5, type=13, cpp_type=3, label=2, 127 | has_default_value=False, default_value=0, 128 | message_type=None, enum_type=None, containing_type=None, 129 | is_extension=False, extension_scope=None, 130 | options=None), 131 | _descriptor.FieldDescriptor( 132 | name='ui64', full_name='tests.MessageOfTypes.ui64', index=5, 133 | number=6, type=4, cpp_type=4, label=2, 134 | has_default_value=False, default_value=0, 135 | message_type=None, enum_type=None, containing_type=None, 136 | is_extension=False, extension_scope=None, 137 | options=None), 138 | _descriptor.FieldDescriptor( 139 | name='si32', full_name='tests.MessageOfTypes.si32', index=6, 140 | number=7, type=17, cpp_type=1, label=2, 141 | has_default_value=False, default_value=0, 142 | message_type=None, enum_type=None, containing_type=None, 143 | is_extension=False, extension_scope=None, 144 | options=None), 145 | _descriptor.FieldDescriptor( 146 | name='si64', full_name='tests.MessageOfTypes.si64', index=7, 147 | number=8, type=18, cpp_type=2, label=2, 148 | has_default_value=False, default_value=0, 149 | message_type=None, enum_type=None, containing_type=None, 150 | is_extension=False, extension_scope=None, 151 | options=None), 152 | _descriptor.FieldDescriptor( 153 | name='f32', full_name='tests.MessageOfTypes.f32', index=8, 154 | number=9, type=7, cpp_type=3, label=2, 155 | has_default_value=False, default_value=0, 156 | message_type=None, enum_type=None, containing_type=None, 157 | is_extension=False, extension_scope=None, 158 | options=None), 159 | _descriptor.FieldDescriptor( 160 | name='f64', full_name='tests.MessageOfTypes.f64', index=9, 161 | number=17, type=6, cpp_type=4, label=2, 162 | has_default_value=False, default_value=0, 163 | message_type=None, enum_type=None, containing_type=None, 164 | is_extension=False, extension_scope=None, 165 | options=None), 166 | _descriptor.FieldDescriptor( 167 | name='sf32', full_name='tests.MessageOfTypes.sf32', index=10, 168 | number=10, type=15, cpp_type=1, label=2, 169 | has_default_value=False, default_value=0, 170 | message_type=None, enum_type=None, containing_type=None, 171 | is_extension=False, extension_scope=None, 172 | options=None), 173 | _descriptor.FieldDescriptor( 174 | name='sf64', full_name='tests.MessageOfTypes.sf64', index=11, 175 | number=11, type=16, cpp_type=2, label=2, 176 | has_default_value=False, default_value=0, 177 | message_type=None, enum_type=None, containing_type=None, 178 | is_extension=False, extension_scope=None, 179 | options=None), 180 | _descriptor.FieldDescriptor( 181 | name='bol', full_name='tests.MessageOfTypes.bol', index=12, 182 | number=12, type=8, cpp_type=7, label=2, 183 | has_default_value=False, default_value=False, 184 | message_type=None, enum_type=None, containing_type=None, 185 | is_extension=False, extension_scope=None, 186 | options=None), 187 | _descriptor.FieldDescriptor( 188 | name='strng', full_name='tests.MessageOfTypes.strng', index=13, 189 | number=13, type=9, cpp_type=9, label=2, 190 | has_default_value=False, default_value=unicode("", "utf-8"), 191 | message_type=None, enum_type=None, containing_type=None, 192 | is_extension=False, extension_scope=None, 193 | options=None), 194 | _descriptor.FieldDescriptor( 195 | name='byts', full_name='tests.MessageOfTypes.byts', index=14, 196 | number=14, type=12, cpp_type=9, label=2, 197 | has_default_value=False, default_value="", 198 | message_type=None, enum_type=None, containing_type=None, 199 | is_extension=False, extension_scope=None, 200 | options=None), 201 | _descriptor.FieldDescriptor( 202 | name='nested', full_name='tests.MessageOfTypes.nested', index=15, 203 | number=15, type=11, cpp_type=10, label=2, 204 | has_default_value=False, default_value=None, 205 | message_type=None, enum_type=None, containing_type=None, 206 | is_extension=False, extension_scope=None, 207 | options=None), 208 | _descriptor.FieldDescriptor( 209 | name='enm', full_name='tests.MessageOfTypes.enm', index=16, 210 | number=16, type=14, cpp_type=8, label=2, 211 | has_default_value=False, default_value=0, 212 | message_type=None, enum_type=None, containing_type=None, 213 | is_extension=False, extension_scope=None, 214 | options=None), 215 | _descriptor.FieldDescriptor( 216 | name='range', full_name='tests.MessageOfTypes.range', index=17, 217 | number=18, type=5, cpp_type=1, label=3, 218 | has_default_value=False, default_value=[], 219 | message_type=None, enum_type=None, containing_type=None, 220 | is_extension=False, extension_scope=None, 221 | options=None), 222 | _descriptor.FieldDescriptor( 223 | name='nestedRepeated', full_name='tests.MessageOfTypes.nestedRepeated', index=18, 224 | number=19, type=11, cpp_type=10, label=3, 225 | has_default_value=False, default_value=[], 226 | message_type=None, enum_type=None, containing_type=None, 227 | is_extension=False, extension_scope=None, 228 | options=None), 229 | _descriptor.FieldDescriptor( 230 | name='enmRepeated', full_name='tests.MessageOfTypes.enmRepeated', index=19, 231 | number=20, type=14, cpp_type=8, label=3, 232 | has_default_value=False, default_value=[], 233 | message_type=None, enum_type=None, containing_type=None, 234 | is_extension=False, extension_scope=None, 235 | options=None), 236 | ], 237 | extensions=[ 238 | ], 239 | nested_types=[_MESSAGEOFTYPES_NESTEDTYPE, ], 240 | enum_types=[ 241 | _MESSAGEOFTYPES_ENUM, 242 | ], 243 | options=None, 244 | is_extendable=True, 245 | extension_ranges=[(100, 536870912), ], 246 | serialized_start=30, 247 | serialized_end=531, 248 | ) 249 | 250 | 251 | _NESTEDEXTENSION = _descriptor.Descriptor( 252 | name='NestedExtension', 253 | full_name='tests.NestedExtension', 254 | filename=None, 255 | file=DESCRIPTOR, 256 | containing_type=None, 257 | fields=[ 258 | ], 259 | extensions=[ 260 | _descriptor.FieldDescriptor( 261 | name='extInt', full_name='tests.NestedExtension.extInt', index=0, 262 | number=102, type=5, cpp_type=1, label=1, 263 | has_default_value=False, default_value=0, 264 | message_type=None, enum_type=None, containing_type=None, 265 | is_extension=True, extension_scope=None, 266 | options=None), 267 | _descriptor.FieldDescriptor( 268 | name='extNested', full_name='tests.NestedExtension.extNested', index=1, 269 | number=103, type=11, cpp_type=10, label=1, 270 | has_default_value=False, default_value=None, 271 | message_type=None, enum_type=None, containing_type=None, 272 | is_extension=True, extension_scope=None, 273 | options=None), 274 | ], 275 | nested_types=[], 276 | enum_types=[ 277 | ], 278 | options=None, 279 | is_extendable=False, 280 | extension_ranges=[], 281 | serialized_start=534, 282 | serialized_end=666, 283 | ) 284 | 285 | _MESSAGEOFTYPES_NESTEDTYPE.containing_type = _MESSAGEOFTYPES; 286 | _MESSAGEOFTYPES.fields_by_name['nested'].message_type = _MESSAGEOFTYPES_NESTEDTYPE 287 | _MESSAGEOFTYPES.fields_by_name['enm'].enum_type = _MESSAGEOFTYPES_ENUM 288 | _MESSAGEOFTYPES.fields_by_name['nestedRepeated'].message_type = _MESSAGEOFTYPES_NESTEDTYPE 289 | _MESSAGEOFTYPES.fields_by_name['enmRepeated'].enum_type = _MESSAGEOFTYPES_ENUM 290 | _MESSAGEOFTYPES_ENUM.containing_type = _MESSAGEOFTYPES; 291 | DESCRIPTOR.message_types_by_name['MessageOfTypes'] = _MESSAGEOFTYPES 292 | DESCRIPTOR.message_types_by_name['NestedExtension'] = _NESTEDEXTENSION 293 | 294 | class MessageOfTypes(_message.Message): 295 | __metaclass__ = _reflection.GeneratedProtocolMessageType 296 | 297 | class NestedType(_message.Message): 298 | __metaclass__ = _reflection.GeneratedProtocolMessageType 299 | DESCRIPTOR = _MESSAGEOFTYPES_NESTEDTYPE 300 | 301 | # @@protoc_insertion_point(class_scope:tests.MessageOfTypes.NestedType) 302 | DESCRIPTOR = _MESSAGEOFTYPES 303 | 304 | # @@protoc_insertion_point(class_scope:tests.MessageOfTypes) 305 | 306 | class NestedExtension(_message.Message): 307 | __metaclass__ = _reflection.GeneratedProtocolMessageType 308 | DESCRIPTOR = _NESTEDEXTENSION 309 | 310 | # @@protoc_insertion_point(class_scope:tests.NestedExtension) 311 | 312 | MessageOfTypes.RegisterExtension(extDouble) 313 | MessageOfTypes.RegisterExtension(extString) 314 | MessageOfTypes.RegisterExtension(_NESTEDEXTENSION.extensions_by_name['extInt']) 315 | _NESTEDEXTENSION.extensions_by_name['extNested'].message_type = _MESSAGEOFTYPES_NESTEDTYPE 316 | MessageOfTypes.RegisterExtension(_NESTEDEXTENSION.extensions_by_name['extNested']) 317 | 318 | # @@protoc_insertion_point(module_scope) 319 | -------------------------------------------------------------------------------- /src/tests/test_proto_to_dict.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from tests.sample_pb2 import MessageOfTypes, extDouble, extString, NestedExtension 3 | from protobuf_to_dict import protobuf_to_dict, dict_to_protobuf 4 | import base64 5 | import nose.tools 6 | import json 7 | 8 | 9 | class Test(unittest.TestCase): 10 | def test_basics(self): 11 | m = self.populate_MessageOfTypes() 12 | d = protobuf_to_dict(m) 13 | self.compare(m, d, ['nestedRepeated']) 14 | 15 | m2 = dict_to_protobuf(MessageOfTypes, d) 16 | assert m == m2 17 | 18 | def test_use_enum_labels(self): 19 | m = self.populate_MessageOfTypes() 20 | d = protobuf_to_dict(m, use_enum_labels=True) 21 | self.compare(m, d, ['enm', 'enmRepeated', 'nestedRepeated']) 22 | assert d['enm'] == 'C' 23 | assert d['enmRepeated'] == ['A', 'C'] 24 | 25 | m2 = dict_to_protobuf(MessageOfTypes, d) 26 | assert m == m2 27 | 28 | d['enm'] = 'MEOW' 29 | with nose.tools.assert_raises(KeyError): 30 | dict_to_protobuf(MessageOfTypes, d) 31 | 32 | d['enm'] = 'A' 33 | d['enmRepeated'] = ['B'] 34 | dict_to_protobuf(MessageOfTypes, d) 35 | 36 | d['enmRepeated'] = ['CAT'] 37 | with nose.tools.assert_raises(KeyError): 38 | dict_to_protobuf(MessageOfTypes, d) 39 | 40 | def test_repeated_enum(self): 41 | m = self.populate_MessageOfTypes() 42 | d = protobuf_to_dict(m, use_enum_labels=True) 43 | self.compare(m, d, ['enm', 'enmRepeated', 'nestedRepeated']) 44 | assert d['enmRepeated'] == ['A', 'C'] 45 | 46 | m2 = dict_to_protobuf(MessageOfTypes, d) 47 | assert m == m2 48 | 49 | d['enmRepeated'] = ['MEOW'] 50 | with nose.tools.assert_raises(KeyError): 51 | dict_to_protobuf(MessageOfTypes, d) 52 | 53 | def test_nested_repeated(self): 54 | m = self.populate_MessageOfTypes() 55 | m.nestedRepeated.extend([MessageOfTypes.NestedType(req=str(i)) for i in range(10)]) 56 | 57 | d = protobuf_to_dict(m) 58 | self.compare(m, d, exclude=['nestedRepeated']) 59 | assert d['nestedRepeated'] == [{'req': str(i)} for i in range(10)] 60 | 61 | m2 = dict_to_protobuf(MessageOfTypes, d) 62 | assert m == m2 63 | 64 | def test_reverse(self): 65 | m = self.populate_MessageOfTypes() 66 | m2 = dict_to_protobuf(MessageOfTypes, protobuf_to_dict(m)) 67 | assert m == m2 68 | m2.dubl = 0 69 | assert m2 != m 70 | 71 | def test_incomplete(self): 72 | m = self.populate_MessageOfTypes() 73 | d = protobuf_to_dict(m) 74 | d.pop('dubl') 75 | m2 = dict_to_protobuf(MessageOfTypes, d) 76 | assert m2.dubl == 0 77 | assert m != m2 78 | 79 | def test_pass_instance(self): 80 | m = self.populate_MessageOfTypes() 81 | d = protobuf_to_dict(m) 82 | d['dubl'] = 1 83 | m2 = dict_to_protobuf(m, d) 84 | assert m is m2 85 | assert m.dubl == 1 86 | 87 | def test_strict(self): 88 | m = self.populate_MessageOfTypes() 89 | d = protobuf_to_dict(m) 90 | d['meow'] = 1 91 | with nose.tools.assert_raises(KeyError): 92 | m2 = dict_to_protobuf(MessageOfTypes, d) 93 | m2 = dict_to_protobuf(MessageOfTypes, d, strict=False) 94 | assert m == m2 95 | 96 | def populate_MessageOfTypes(self): 97 | m = MessageOfTypes() 98 | m.dubl = 1.7e+308 99 | m.flot = 3.4e+038 100 | m.i32 = 2 ** 31 - 1 # 2147483647 # 101 | m.i64 = 2 ** 63 - 1 #0x7FFFFFFFFFFFFFFF 102 | m.ui32 = 2 ** 32 - 1 103 | m.ui64 = 2 ** 64 - 1 104 | m.si32 = -1 * m.i32 105 | m.si64 = -1 * m.i64 106 | m.f32 = m.i32 107 | m.f64 = m.i64 108 | m.sf32 = m.si32 109 | m.sf64 = m.si64 110 | m.bol = True 111 | m.strng = "string" 112 | m.byts = b'\n\x14\x1e' 113 | assert len(m.byts) == 3, len(m.byts) 114 | m.nested.req = "req" 115 | m.enm = MessageOfTypes.C #@UndefinedVariable 116 | m.enmRepeated.extend([MessageOfTypes.A, MessageOfTypes.C]) 117 | m.range.extend(range(10)) 118 | return m 119 | 120 | def compare(self, m, d, exclude=None): 121 | i = 0 122 | exclude = ['byts', 'nested'] + (exclude or []) 123 | for i, field in enumerate(MessageOfTypes.DESCRIPTOR.fields): #@UndefinedVariable 124 | if field.name not in exclude: 125 | assert field.name in d, field.name 126 | assert d[field.name] == getattr(m, field.name), (field.name, d[field.name]) 127 | assert i > 0 128 | assert m.byts == base64.b64decode(d['byts']) 129 | assert d['nested'] == {'req': m.nested.req} 130 | 131 | def test_extensions(self): 132 | m = MessageOfTypes() 133 | 134 | primitives = {extDouble: 123.4, extString: "string", NestedExtension.extInt: 4} 135 | 136 | for key, value in primitives.items(): 137 | m.Extensions[key] = value 138 | m.Extensions[NestedExtension.extNested].req = "nested" 139 | 140 | # Confirm compatibility with JSON serialization 141 | res = json.loads(json.dumps(protobuf_to_dict(m))) 142 | assert '___X' in res 143 | exts = res['___X'] 144 | assert set(exts.keys()) == set([str(f.number) for f, _ in m.ListFields() if f.is_extension]) 145 | for key, value in primitives.items(): 146 | assert exts[str(key.number)] == value 147 | assert exts[str(NestedExtension.extNested.number)]['req'] == 'nested' 148 | 149 | deser = dict_to_protobuf(MessageOfTypes, res) 150 | assert deser 151 | for key, value in primitives.items(): 152 | assert deser.Extensions[key] == m.Extensions[key] 153 | assert deser.Extensions[NestedExtension.extNested].req == m.Extensions[NestedExtension.extNested].req 154 | --------------------------------------------------------------------------------