├── protod ├── __init__.py ├── definition.py ├── util.py ├── field.py ├── main.py ├── decode.py └── renderer.py ├── setup.py ├── .gitignore ├── LICENSE ├── README.md └── example ├── html_renderer.py ├── json_renderer.py └── mitmproxy_proto_view.py /protod/__init__.py: -------------------------------------------------------------------------------- 1 | __all__ = ["decode"] 2 | from .decode import dump 3 | from .renderer import ConsoleRenderer, Renderer 4 | -------------------------------------------------------------------------------- /protod/definition.py: -------------------------------------------------------------------------------- 1 | class WireType: 2 | Varint = 0 3 | Fixed64 = 1 4 | Struct = 2 5 | Deprecated_3 = 3 6 | Deprecated_4 = 4 7 | Fixed32 = 5 8 | 9 | 10 | def wire_type_str(t): 11 | if t == WireType.Varint: 12 | return "varint" 13 | elif t == WireType.Fixed64: 14 | return "fixed64/double" 15 | elif t == WireType.Struct: 16 | return "string" 17 | elif t == WireType.Fixed32: 18 | return "fixed32/float" 19 | else: 20 | return "Unknown wire type" 21 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | github = "https://github.com/aj3423/protod/" 4 | 5 | setup( 6 | name="protod", 7 | version="24.4.10", 8 | description="Decode protobuf without message definition.", 9 | url=github, 10 | author="aj3423", 11 | packages=find_packages(), 12 | install_requires=["chardet", "charset_normalizer", "protobuf", "termcolor"], 13 | entry_points={"console_scripts": ["protod=protod.main:dummy"]}, 14 | long_description="See: " + github, 15 | ) 16 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | bin/ 10 | build/ 11 | develop-eggs/ 12 | dist/ 13 | eggs/ 14 | lib/ 15 | lib64/ 16 | parts/ 17 | sdist/ 18 | var/ 19 | *.egg-info/ 20 | .installed.cfg 21 | *.egg 22 | 23 | # Installer logs 24 | pip-log.txt 25 | pip-delete-this-directory.txt 26 | 27 | # Unit test / coverage reports 28 | .tox/ 29 | .coverage 30 | .cache 31 | nosetests.xml 32 | coverage.xml 33 | 34 | # Translations 35 | *.mo 36 | 37 | # Mr Developer 38 | .mr.developer.cfg 39 | .project 40 | .pydevproject 41 | 42 | # Rope 43 | .ropeproject 44 | 45 | # Django stuff: 46 | *.log 47 | *.pot 48 | 49 | # Sphinx documentation 50 | docs/_build/ 51 | .vscode/ 52 | sample.proto 53 | -------------------------------------------------------------------------------- /protod/util.py: -------------------------------------------------------------------------------- 1 | import chardet 2 | import charset_normalizer 3 | 4 | 5 | # try to detect the encoding of an string 6 | # return ( 7 | # decoded bytes: bytes 8 | # encoding name: str 9 | # decoding succeeded: bool 10 | # ) 11 | def detect_multi_charset(view) -> tuple[bytes, str, bool]: 12 | view_bytes = view.tobytes() 13 | try: 14 | # `chardet` is way more accurate, but very slow with large bytes(4 seconds on 50k bytes) 15 | # `charset_normalizer` shows wrong result with small bytes, but very performant with long bytes 16 | if len(view_bytes) <= 200: 17 | detected = chardet.detect(view_bytes) 18 | else: 19 | detected = charset_normalizer.detect(view_bytes) 20 | 21 | if detected["confidence"] < 0.9: 22 | raise Exception() 23 | 24 | encoding = detected["encoding"] 25 | 26 | decoded = view_bytes.decode(encoding) 27 | 28 | return decoded, encoding, True 29 | except: 30 | pass 31 | 32 | return view_bytes, "", False 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 aj3423 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Decode protobuf without proto. 2 | ## Try it online 3 | http://168.138.55.177/ 4 | # Screenshot 5 | ![protod](https://github.com/aj3423/protod/assets/4710875/bb8986db-ed7e-4cbf-967b-9d28cc6d4237) 6 | ## Install 7 | `pip install protod` 8 | ## The command line tool 9 | 10 | - `protod 080102...` 11 | - `protod '08 01 02...'` (with space/tab/newline) 12 | - `protod --b64 CAEIAQ==` 13 | - `protod --file ~/pb.bin` 14 | - `protod` for help 15 | 16 | ## library protod 17 | It uses different `Renderer` to generate different output: 18 | - For console: 19 | ```python 20 | print(protod.dump(proto_bytes)) # ConsoleRenderer is used by default 21 | ``` 22 | 23 | There are [examples](https://github.com/aj3423/protod/blob/master/example) demonstrate how to write custom `Renderer`s: 24 | - json 25 | 26 | ![image](https://github.com/aj3423/protod/assets/4710875/2c3bddb2-06e7-44b4-844f-eaaff6a26d6f) 27 | 28 | - html 29 | 30 | ![image](https://github.com/aj3423/protod/assets/4710875/39583ae3-1d77-4c22-b4a0-ed9d12bd8305) 31 | 32 | - Mitmproxy addon: 33 | 34 | ![image](https://github.com/aj3423/protod/assets/4710875/aca8a5b1-4c05-4cc4-8346-f3b91a6ca8d7) 35 | 36 | -------------------------------------------------------------------------------- /example/html_renderer.py: -------------------------------------------------------------------------------- 1 | from protod import ConsoleRenderer 2 | 3 | 4 | # The HtmlRenderer builds a full html div string, 5 | # which can be simply set to a
6 | # 7 | # usage: 8 | # 9 | # html_tag = protod.dump(proto, protod.HtmlRenderer()) 10 | # send the html_tag to client browser 11 | # $('#div').text(html_tag) 12 | # 13 | class HtmlRenderer(ConsoleRenderer): 14 | 15 | def _add_indent(self, level): 16 | self._add(" " * 4 * level) 17 | 18 | def _add_newline(self): 19 | self._add("
") 20 | 21 | def _add_normal(self, s): 22 | self._add(s) 23 | 24 | def _add_idtype(self, s): 25 | self._add(f"{s}") 26 | 27 | def _add_id(self, s): 28 | self._add(f"{s}") 29 | 30 | def _add_type(self, s): 31 | self._add(f"{s}") 32 | 33 | def _add_num(self, s): 34 | self._add(f"{s}") 35 | 36 | def _add_str(self, s): 37 | self._add(f"{escape(s)}") 38 | 39 | def _add_bin(self, s): 40 | self._add( 41 | f"{escape(' '.join(format(x, '02x') for x in s))}" 42 | ) 43 | -------------------------------------------------------------------------------- /protod/field.py: -------------------------------------------------------------------------------- 1 | import ctypes 2 | from abc import ABC, abstractmethod 3 | 4 | from .renderer import Renderer 5 | 6 | 7 | class IdType: 8 | def __init__(self, id, wire_type, idtype_bytes): 9 | self.id = id 10 | self.wire_type = wire_type 11 | self.raw_bytes = idtype_bytes 12 | 13 | 14 | class Field(ABC): 15 | def __init__(self): 16 | self.idtype = None 17 | self.parent = None 18 | 19 | def indent_level(self): 20 | lvl = 0 21 | p = self.parent 22 | while p is not None: 23 | lvl += 1 24 | p = p.parent 25 | 26 | return lvl 27 | 28 | @abstractmethod 29 | def render(self, r: Renderer): 30 | pass 31 | 32 | 33 | class RepeatedField(Field): 34 | def __init__(self, items): 35 | self.items = items 36 | 37 | def render(self, r: Renderer): 38 | r.render_repeated_fields(self) 39 | 40 | 41 | class Varint(Field): 42 | def __init__(self, u64): 43 | super().__init__() 44 | 45 | # convert u64 -> i64 46 | # i64 should be enough, no need for u64 47 | self.u64 = u64 48 | self.i64 = ctypes.c_int64(u64).value 49 | 50 | def render(self, r: Renderer): 51 | r.render_varint(self) 52 | 53 | 54 | class Fixed(Field): 55 | def __init__(self, u, i, f): 56 | super().__init__() 57 | 58 | # u: unsigned form 59 | # i: signed form 60 | # f: float form 61 | self.u, self.i, self.f = u, i, f 62 | 63 | # displayed as: 64 | # unsigned (signed if negative) (hex) (float) 65 | def render(self, r: Renderer): 66 | r.render_fixed(self) 67 | 68 | 69 | class Struct(Field): 70 | 71 | def __init__(self, view: memoryview, as_str: bytes, encoding: str, is_str: bool): 72 | super().__init__() 73 | 74 | self.view = view # raw memoryview 75 | self.as_fields = [] # this struc can be parsed to fields 76 | self.as_str, self.encoding, self.is_str = as_str, encoding, is_str 77 | 78 | def render(self, r: Renderer): 79 | r.render_struct(self) 80 | -------------------------------------------------------------------------------- /protod/main.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import re 3 | import argparse 4 | import protod 5 | from termcolor import cprint 6 | 7 | parser = argparse.ArgumentParser() 8 | parser.add_argument('--file', type=str, help='file path that contains proto data') 9 | parser.add_argument('--hex', action='store_true', help='content is hex string, eg: "080102..."') 10 | parser.add_argument('--b64', action='store_true', help='content is base64') 11 | parser.add_argument('--max_bin', metavar='n', type=int, default=32, help='binary exceeds `n` bytes is truncated and followed by a "..."') 12 | parser.add_argument('rest', help='hex string to parse, eg: "08 01..."', nargs=argparse.REMAINDER) 13 | 14 | args = parser.parse_args() 15 | 16 | proto = bytes() 17 | 18 | # clear all ' \t\n\r' of an str 19 | def cleanup(s) -> str: 20 | if type(s) == bytes: 21 | return re.sub(r'[\n\r\t ]+', '', s.decode()) 22 | return re.sub(r'[\n\r\t ]+', '', s) 23 | 24 | if args.file is not None: # get proto from file 25 | try: 26 | f = open(args.file, "rb") 27 | proto = f.read() 28 | f.close() 29 | except: 30 | cprint(f'failed to read file: {args.file}', 'red') 31 | sys.exit(1) 32 | 33 | if args.b64 or args.hex: 34 | proto = cleanup(proto) 35 | else: # get proto from arguments 36 | if len(args.rest) == 0: 37 | cprint(f'no input', 'red') 38 | parser.print_help() 39 | sys.exit(1) 40 | 41 | if len(args.rest) > 1: # multiple rest arguments, eg: pro 0a 08 01 ... 42 | # concat them together 43 | proto = cleanup(''.join(args.rest)) 44 | else: 45 | # single rest argument, eg: pro "0a 08 01 ..." 46 | proto = cleanup(args.rest[0]) 47 | 48 | # the proto should've already been cleaned up 49 | if args.b64: 50 | try: 51 | import base64 52 | proto = base64.b64decode(proto) 53 | except: 54 | cprint(f'invalid b64 data', 'red') 55 | sys.exit(1) 56 | 57 | if type(proto) == str and re.match(r'^([0-9A-Fa-f]+)$', proto): 58 | args.hex = True 59 | 60 | if args.hex: 61 | proto = bytes.fromhex(proto) 62 | 63 | s = protod.dump( 64 | proto, 65 | protod.ConsoleRenderer( 66 | truncate_after=args.max_bin 67 | ) 68 | ) 69 | print(s) 70 | 71 | # Just a workaround for `pip install` 72 | # the `protod = 'protod.main:dummy'` in pyproject.toml needs to call a function 73 | def dummy(): 74 | pass 75 | 76 | -------------------------------------------------------------------------------- /example/json_renderer.py: -------------------------------------------------------------------------------- 1 | import binascii 2 | import json 3 | 4 | import protod 5 | from protod import Renderer 6 | 7 | 8 | class JsonRenderer(Renderer): 9 | 10 | def __init__(self): 11 | self.result = dict() 12 | self.current = self.result 13 | 14 | def _add(self, id, item): 15 | self.current[id] = item 16 | 17 | def _build_tmp_item(self, chunk): 18 | # use a temporary renderer to build 19 | jr = JsonRenderer() 20 | chunk.render(jr) 21 | tmp_dict = jr.build_result() 22 | 23 | # the tmp_dict only contains 1 item 24 | for _, item in tmp_dict.items(): 25 | return item 26 | 27 | def build_result(self): 28 | return self.result 29 | 30 | def render_repeated_fields(self, repeated): 31 | arr = [] 32 | for ch in repeated.items: 33 | arr.append(self._build_tmp_item(ch)) 34 | self._add(repeated.idtype.id, arr) 35 | 36 | def render_varint(self, varint): 37 | self._add(varint.idtype.id, varint.i64) 38 | 39 | def render_fixed(self, fixed): 40 | self._add(fixed.idtype.id, fixed.i) 41 | 42 | def render_struct(self, struct): 43 | 44 | curr = None 45 | 46 | if struct.as_fields: 47 | curr = {} 48 | for ch in struct.as_fields: 49 | curr[ch.idtype.id] = self._build_tmp_item(ch) 50 | elif struct.is_str: 51 | curr = struct.as_str 52 | 53 | else: 54 | curr = " ".join(format(x, "02x") for x in struct.view) 55 | 56 | self._add(struct.idtype.id, curr) 57 | 58 | 59 | # return ( 60 | # decoded bytes: bytes 61 | # encoding name: str 62 | # decoding succeeded: bool 63 | # ) 64 | def decode_utf8(view) -> tuple[bytes, str, bool]: 65 | view_bytes = view.tobytes() 66 | try: 67 | utf8 = "UTF-8" 68 | decoded = view_bytes.decode(utf8) 69 | return decoded, utf8, True 70 | except: 71 | return view_bytes, "", False 72 | 73 | 74 | sample = "0885ffffffffffffffff011084461a408d17480ac969d3d8fd619e9bc10870688d17480ac969d3d8fd619e9bc10870688d17480ac969d3d8fd619e9bc10870688d17480ac969d3d8fd619e9bc10870682a281d8fc2e842213333333333f34340a2061767656f67726170686963616c20636f6f7264696e617465321b57696e6e69c3a9c3a9c3a9c3a9207468c3a9204469637461746f723208bab8b4d9c0dbc0bd3210c4e3b5c4c3e6b3a3c4e3b5c4c3e6b3a3323ca16dae61bb79a16ea4eaa147a175a767a46ca4a3b3d5a141acb0a8e4addda6e6b463b944ac47a45da143a176a16dbdd7bb79a16ea4aaa147a175a4a3322d8366834283588376838c8343838226233132393b5b836882aa26233134343bdd92e882c582ab82dc82b982f12e" 75 | 76 | proto_bytes = binascii.unhexlify(sample) 77 | 78 | ret = protod.dump( 79 | proto_bytes, 80 | renderer=JsonRenderer(), 81 | str_decoder=decode_utf8, 82 | ) 83 | print(json.dumps(ret, indent=4, ensure_ascii=False)) 84 | -------------------------------------------------------------------------------- /example/mitmproxy_proto_view.py: -------------------------------------------------------------------------------- 1 | from mitmproxy import contentviews, ctx 2 | from mitmproxy.addonmanager import Loader 3 | 4 | import protod 5 | from protod import ConsoleRenderer 6 | 7 | 8 | # Color palettes are defined in: 9 | # https://github.com/mitmproxy/mitmproxy/blob/746537e0511e0316a144e05e7ba8cc6f6e44768b/mitmproxy/tools/console/palettes.py#L154 10 | class MitmproxyRenderer(ConsoleRenderer): 11 | # Long binary data that exceeds `n` bytes are truncated and followed by a '...' 12 | # use a large value like 1000000 to 'not' truncate 13 | # default: 32 14 | def __init__(self, truncate_after=32): 15 | self.truncate_after = truncate_after 16 | 17 | """ 18 | It builds a 2D list like: 19 | [ 20 | [('text', 'white text'), ('offset', 'blue text')], <-- first line 21 | [('header', '...'), ('offset', '...')], <-- second line 22 | ... 23 | ] 24 | """ 25 | self.cells = [[]] 26 | 27 | def _add_newline(self): 28 | self.cells.append([]) 29 | 30 | def build_result(self): 31 | # a workaround to remove the trailing `[]` that causes error 32 | return self.cells[: len(self.cells) - 1] 33 | 34 | def _add(self, cell): 35 | self.cells[len(self.cells) - 1].append(cell) 36 | pass 37 | 38 | def _add_indent(self, level): 39 | for _ in range(level): 40 | self._add(("text", " " * 4)) 41 | 42 | def _add_normal(self, s): # white 43 | self._add(("text", s)) 44 | 45 | def _add_idtype(self, s): # light red 46 | self._add(("error", s)) 47 | 48 | def _add_id(self, s): # light green 49 | self._add(("replay", s)) 50 | 51 | def _add_type(self, s): # yellow 52 | self._add(("alert", s)) 53 | 54 | def _add_num(self, s): # light cyan 55 | self._add(("Token_Literal_String", s)) 56 | 57 | def _add_str(self, s): # light blue 58 | self._add(("content_media", s)) 59 | 60 | def _add_bin(self, s): # light yellow 61 | truncated = s[: self.truncate_after] 62 | self._add(("warn", " ".join(format(x, "02x") for x in truncated))) 63 | if len(s) > self.truncate_after: 64 | self._add_normal(" ...") 65 | 66 | 67 | class ViewProto(contentviews.Contentview): 68 | name = "ViewProto" 69 | 70 | def prettify(self, data: bytes, metadata: contentviews.Metadata) -> str: 71 | # ctx.log.error("111111") 72 | # try: 73 | # ctx.log.warn("start") 74 | # x = protod.dump( 75 | # data, renderer=ConsoleRenderer(no_color=True) 76 | # ) 77 | # ctx.log.warn(x) 78 | # ctx.log.warn("end") 79 | # except Exception as e: 80 | # ctx.log.error(e) 81 | # return "aaaaaaaaaaaaaaaaa" 82 | return protod.dump(data, renderer=ConsoleRenderer(no_color=True)) 83 | 84 | def render_priority(self, data: bytes, metadata: contentviews.Metadata) -> float: 85 | # ctx.log.warn(metadata.flow.server_conn) 86 | return 1000 if "nexus" in str(metadata.flow.server_conn.address) else 0 87 | 88 | 89 | # view = ViewProto() 90 | 91 | 92 | # def load(loader: Loader): 93 | # contentviews.add(view) 94 | 95 | 96 | # def done(): 97 | # contentviews.remove(view) 98 | 99 | contentviews.add(ViewProto) 100 | -------------------------------------------------------------------------------- /protod/decode.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | import struct 3 | from typing import List 4 | 5 | from google.protobuf.internal.decoder import _DecodeVarint 6 | 7 | from .definition import WireType 8 | from .field import Field, Fixed, IdType, RepeatedField, Struct, Varint 9 | from .renderer import ConsoleRenderer 10 | from .util import detect_multi_charset 11 | 12 | 13 | # 0 Varint int32, int64, uint32, uint64, sint32, sint64, bool, enum 14 | # 1 64-bit fixed64, sfixed64, double 15 | # 2 Length-delimited string, bytes, embedded messages, packed repeated fields 16 | # 3 Start group groups (deprecated) 17 | # 4 End group groups (deprecated) 18 | # 5 32-bit fixed32, sfixed32, float 19 | def decode_1_field(str_decoder, parent: Field, view: memoryview) -> tuple[Field, int]: 20 | pos = 0 21 | 22 | id_type, pos = _DecodeVarint(view, 0) 23 | 24 | if pos >= len(view): 25 | raise Exception("not enough data for any further wire type") 26 | 27 | id = id_type >> 3 28 | 29 | if id > 536870911: # max field: 2^29 - 1 == 536870911 30 | raise Exception("field number > max field value 2^29-1") 31 | 32 | wire_type = id_type & 7 33 | 34 | idtype_bytes = view[0:pos] 35 | 36 | ret = None 37 | 38 | if wire_type == WireType.Varint: # 0 39 | if pos >= len(view): 40 | raise Exception("not enough data for wire type 0(varint)") 41 | 42 | u64, pos = _DecodeVarint(view, pos) 43 | 44 | ret = Varint(u64) 45 | 46 | elif wire_type == WireType.Fixed32: # 5 47 | if pos + 4 > len(view): 48 | raise Exception("not enough data for wire type 5(fixed32)") 49 | 50 | _4bytes = view[pos : pos + 4] 51 | pos += 4 52 | 53 | u = struct.unpack(" len(view): 60 | raise Exception("not enough data for wire type 1(fixed64)") 61 | 62 | _8bytes = view[pos : pos + 8] 63 | pos += 8 64 | 65 | u = struct.unpack(" len(view): 75 | raise Exception("not enough data for wire type 2(string)") 76 | 77 | view_field = view[pos : pos + s_len] 78 | pos += s_len 79 | 80 | as_str, encoding, is_str = str_decoder(view_field) 81 | ret = Struct(view_field, as_str, encoding, is_str) 82 | 83 | try: 84 | # if decode successfully, it's child struct, not just binary bytes 85 | ret.as_fields = decode_all_fields(str_decoder, ret, view_field) 86 | except: 87 | pass 88 | 89 | elif wire_type == WireType.Deprecated_3: # 3 90 | raise Exception("[proto 3] found, looks like invalid proto bytes") 91 | 92 | elif wire_type == WireType.Deprecated_4: # 4 93 | raise Exception("[proto 4] found, looks like invalid proto bytes") 94 | else: 95 | raise Exception(f"Unknown wire type {wire_type} of id_type {id_type}") 96 | 97 | ret.idtype = IdType(id, wire_type, idtype_bytes) 98 | ret.parent = parent 99 | 100 | return ret, pos 101 | 102 | 103 | def decode_all_fields(str_decoder, parent: Field, view: memoryview) -> List[Field]: 104 | pos = 0 105 | fields = [] 106 | 107 | while pos < len(view): 108 | try: 109 | field, field_len = decode_1_field(str_decoder, parent, view[pos:]) 110 | except: 111 | raise Exception(f"field: {view[pos:].tobytes()}") 112 | 113 | fields.append(field) 114 | pos += field_len 115 | 116 | # group fields with same id to a RepeatedField 117 | ret = [] 118 | for _, group in itertools.groupby(fields, lambda f: f.idtype.id): 119 | 120 | items = list(group) 121 | 122 | if len(items) == 1: # single field 123 | ret.append(items[0]) 124 | else: # repeated fields 125 | repeated = RepeatedField(items) 126 | repeated.idtype = items[0].idtype 127 | repeated.parent = items[0].parent 128 | ret.append(repeated) 129 | 130 | return ret 131 | 132 | 133 | def dump( 134 | data: bytes, 135 | renderer=None, 136 | str_decoder=detect_multi_charset, 137 | ): 138 | if renderer == None: 139 | renderer = ConsoleRenderer() 140 | 141 | view = memoryview(data) 142 | 143 | fields = decode_all_fields(str_decoder=str_decoder, parent=None, view=view) 144 | 145 | for ch in fields: 146 | ch.render(renderer) 147 | 148 | return renderer.build_result() 149 | -------------------------------------------------------------------------------- /protod/renderer.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | from html import escape 3 | 4 | from termcolor import colored 5 | 6 | from .definition import wire_type_str 7 | 8 | 9 | # Field formatter and colorizer 10 | class Renderer(ABC): 11 | # Return the final result, which can be in different formats, eg: 12 | # for console, it's a string with ansi color 13 | # for mitmproxy, it's an array of tuple 14 | @abstractmethod 15 | def build_result(self): 16 | pass 17 | 18 | # render repeated fields 19 | @abstractmethod 20 | def render_repeated_fields(self, repeated): 21 | pass 22 | 23 | # render varint 24 | @abstractmethod 25 | def render_varint(self, varint): 26 | pass 27 | 28 | # render fixed32/fixed64 29 | @abstractmethod 30 | def render_fixed(self, fixed): 31 | pass 32 | 33 | # render struct 34 | @abstractmethod 35 | def render_struct(self, struct): 36 | pass 37 | 38 | 39 | class ConsoleRenderer(Renderer): 40 | # Long binary data that exceeds `n` bytes is truncated and followed by a '...' 41 | # use a large value like 1000000 to 'not' truncate 42 | # default: 32 43 | def __init__(self, truncate_after=32, no_color=False): 44 | self.cells = [] 45 | self.truncate_after = truncate_after 46 | self.no_color = no_color 47 | 48 | def build_result(self): 49 | return "".join(self.cells) 50 | 51 | def render_repeated_fields(self, repeated): 52 | for ch in repeated.items: 53 | ch.render(self) 54 | 55 | def render_varint(self, varint): 56 | self._render_idtype(varint.indent_level(), varint.idtype) 57 | 58 | self._add_num(str(varint.i64)) 59 | self._add_normal(" (") 60 | self._add_num(str(hex(varint.u64))) 61 | self._add_normal(")") 62 | 63 | self._add_newline() 64 | 65 | def render_fixed(self, fixed): 66 | self._render_idtype(fixed.indent_level(), fixed.idtype) 67 | 68 | u, i, f = fixed.u, fixed.i, fixed.f 69 | 70 | self._add_normal(str(u)) # show unsigned form 71 | if i < 0: # also show signed value if it's negative 72 | self._add_normal(f" ({str(i)})") 73 | 74 | self._add_normal(f" ({hex(u)}) ({str(f)})") # show hex and float form 75 | self._add_newline() 76 | 77 | def render_struct(self, struct): 78 | self._render_idtype(struct.indent_level(), struct.idtype) 79 | 80 | self._add_normal(f"({str(len(struct.view))}) ") 81 | 82 | if struct.is_str: 83 | if struct.as_fields: 84 | if struct.as_str.isprintable(): 85 | self._render_str(struct.as_str, struct.encoding) 86 | self._add_newline() 87 | else: 88 | self._render_str(struct.as_str, struct.encoding) 89 | 90 | # Also show hex if: 91 | # 1. it contains non-printable characters 92 | # 2. it is short, less than 8 bytes 93 | if not struct.as_str.isprintable() or 0 < len(struct.view) <= 8: 94 | self._add_normal(" (") 95 | self._add_bin(struct.view) 96 | self._add_normal(")") 97 | self._add_newline() 98 | 99 | else: 100 | if not struct.as_fields: 101 | # show as binary 102 | self._add_bin(struct.view) 103 | self._add_newline() 104 | 105 | # 2. show as child struct 106 | if struct.as_fields: 107 | self._add_newline() 108 | for ch in struct.as_fields: 109 | ch.render(self) 110 | 111 | ########################### 112 | 113 | def _render_idtype(self, indent_level, idtype): 114 | self._add_indent(indent_level) 115 | self._add_normal("[") 116 | self._add_idtype(" ".join(format(x, "02x") for x in idtype.raw_bytes)) 117 | self._add_normal("] ") 118 | self._add_id(str(idtype.id) + " ") 119 | self._add_type(wire_type_str(idtype.wire_type)) 120 | self._add_normal(": ") 121 | 122 | def _add_newline(self): 123 | self._add("\n") 124 | 125 | def _render_str(self, string: str, encoding: str): 126 | if encoding not in ["ascii"]: 127 | self._add_normal(f"[{encoding}] ") 128 | self._add_str(string) 129 | 130 | def _add(self, cell): 131 | self.cells.append(cell) 132 | pass 133 | 134 | def _add_indent(self, level): 135 | self._add(" " * 4 * level) 136 | 137 | def _add_normal(self, s): 138 | self._add(s) 139 | 140 | def _add_idtype(self, s): 141 | self._add(colored(s, "light_red", no_color=self.no_color)) 142 | 143 | def _add_id(self, s): 144 | self._add(colored(s, "light_green", no_color=self.no_color)) 145 | 146 | def _add_type(self, s): 147 | self._add(colored(s, "yellow", no_color=self.no_color)) 148 | 149 | def _add_num(self, s): 150 | self._add(colored(s, "light_cyan", no_color=self.no_color)) 151 | 152 | def _add_str(self, s): 153 | self._add(colored(s, "light_blue", no_color=self.no_color)) 154 | 155 | def _add_bin(self, s): 156 | truncated = s[: self.truncate_after] 157 | self._add( 158 | colored( 159 | " ".join(format(x, "02x") for x in truncated), 160 | "light_yellow", 161 | no_color=self.no_color, 162 | ) 163 | ) 164 | if len(s) > self.truncate_after: 165 | self._add_normal(" ...") 166 | --------------------------------------------------------------------------------