├── .gitignore ├── LICENSE ├── README.md ├── big_string_chunker.py ├── burp_utils ├── __init__.py ├── burp_grpc_web_editor_tab.py └── grpc_coder_withdout_dependency.py ├── burp_utils_with_dep ├── __init__.py ├── burp_grpc_decodetab.py ├── burp_grpc_insertionpoint.py └── test_old_grpc_web_burp_extension.py ├── grpc_coder.py ├── grpc_scan.py ├── grpc_utils.py ├── grpc_web_burp_extension.py ├── libs ├── blackboxprotobuf │ ├── CLI.md │ ├── Makefile │ ├── README.md │ ├── blackboxprotobuf │ │ ├── __init__.py │ │ ├── __main__.py │ │ ├── lib │ │ │ ├── __init__.py │ │ │ ├── api.py │ │ │ ├── config.py │ │ │ ├── exceptions.py │ │ │ ├── payloads │ │ │ │ ├── __init__.py │ │ │ │ ├── grpc.py │ │ │ │ └── gzip.py │ │ │ ├── protofile.py │ │ │ ├── pytypes.py │ │ │ ├── typedef.py │ │ │ └── types │ │ │ │ ├── __init__.py │ │ │ │ ├── fixed.py │ │ │ │ ├── length_delim.py │ │ │ │ ├── type_maps.py │ │ │ │ ├── varint.py │ │ │ │ └── wiretypes.py │ │ └── py.typed │ ├── poetry.lock │ ├── pyproject.toml │ └── tests │ │ ├── generate_payload.sh │ │ ├── payloads │ │ ├── Test.proto │ │ ├── test_message.in │ │ └── test_message.out │ │ ├── proxy_tests │ │ ├── Makefile │ │ ├── Test.proto │ │ ├── grpc_client.py │ │ ├── grpc_server.py │ │ ├── http_client.py │ │ ├── http_server.py │ │ ├── requirements.txt │ │ ├── websocket_client.py │ │ └── websocket_server.py │ │ ├── py_test │ │ ├── strategies.py │ │ ├── test_exceptions.py │ │ ├── test_fixed.py │ │ ├── test_json.py │ │ ├── test_length_delim.py │ │ ├── test_payloads.py │ │ ├── test_perf.py │ │ ├── test_protobuf.py │ │ ├── test_protofile.py │ │ ├── test_typedef.py │ │ └── test_varint.py │ │ ├── requirements-python2-dev.txt │ │ └── run_decoder.py └── six │ ├── .github │ └── workflows │ │ ├── ci.yml │ │ └── publish.yml │ ├── .gitignore │ ├── CHANGES │ ├── CONTRIBUTORS │ ├── LICENSE │ ├── MANIFEST.in │ ├── README.rst │ ├── documentation │ ├── Makefile │ ├── conf.py │ └── index.rst │ ├── setup.cfg │ ├── setup.py │ ├── six.py │ ├── test_six.py │ └── tox.ini ├── old_grpc_web_burp_extension_with_dependency.py └── requirements.txt /.gitignore: -------------------------------------------------------------------------------- 1 | venv 2 | # Byte-compiled / optimized / DLL files 3 | __pycache__/ 4 | *.py[cod] 5 | *$py.class 6 | # Environments 7 | .env 8 | .venv 9 | env/ 10 | venv*/ 11 | ENV/ 12 | env.bak/ 13 | venv.bak/ 14 | .idea 15 | .idea/ 16 | .vscode 17 | .vscode/ 18 | # some trash files 19 | test 20 | test.py 21 | notes.txt 22 | archives/ 23 | # ignore log files and directory in git 24 | **/*.log 25 | logs/ 26 | 27 | # ingore download directory which downloaded JS files are in that directory 28 | download/ 29 | # js file for testing purposes 30 | main.js 31 | 32 | grpc_coder_output.txt -------------------------------------------------------------------------------- /big_string_chunker.py: -------------------------------------------------------------------------------- 1 | """ 2 | This tool is used for chunk a big string into pieces of 80 characters 3 | """ 4 | from argparse import ArgumentParser 5 | import sys 6 | 7 | def unchunk_string(chunked_string): 8 | new_string = chunked_string.replace('"', '') 9 | new_string = new_string.replace("\n", '') 10 | new_string = new_string.replace(" ", '') 11 | return new_string 12 | 13 | 14 | def chunk_string(big_string, chunk_size=80): 15 | formatted_string = "" 16 | chunks = [big_string[i:i+chunk_size] for i in range(0, len(big_string), chunk_size)] 17 | for i, chunk in enumerate(chunks): 18 | if i == len(chunks) - 1: 19 | formatted_string += f'"{chunk}"\n' 20 | else: 21 | formatted_string += f'"{chunk}"\n ' 22 | # formatted_string += f'"{chunk}"\n ' 23 | return f"1: {{\n {formatted_string}}}" 24 | 25 | 26 | def get_content_from_stdin(): 27 | return sys.stdin.buffer.read().decode() 28 | 29 | 30 | def get_content_from_file(file_path): 31 | try: 32 | with open(file_path, 'r') as file: 33 | return file.read() 34 | 35 | except Exception as e: 36 | print('Error Occurred in Reading Input File: ' + str(e)) 37 | exit(1) 38 | 39 | 40 | def print_parser_help(prog): 41 | help_msg = f"""echo payload | python3 {prog} --stdin [--chunk OR --un-chunk] 42 | Arguments: 43 | --chunk chunk a big string into pieces of 80 chars 44 | --un-chunk un-chunk the chunked data (remove ['"','\\n',' ']) 45 | Input Arguments: 46 | --stdin get input from standard input 47 | --file get input from a file 48 | Help: 49 | --help print help message 50 | 51 | Examples: 52 | echo payload | python3 {prog} --stdin 53 | python3 {prog} --file big_string.txt 54 | """ 55 | 56 | print(help_msg) 57 | 58 | 59 | if __name__ == '__main__': 60 | parser = ArgumentParser(usage='echo payload | python3 %(prog)s --stdin', 61 | allow_abbrev=False, add_help=False) 62 | 63 | parser.add_argument('--help', action='store_true', default=False) 64 | parser.add_argument('--chunk', action='store_true', default=False) 65 | parser.add_argument('--un-chunk', action='store_true', default=False) 66 | parser.add_argument('--stdin', action='store_true', default=False) 67 | parser.add_argument('--file', default=None) 68 | 69 | args, unknown = parser.parse_known_args() 70 | 71 | if (args.stdin is not True) and (args.file is None): 72 | print_parser_help(parser.prog) 73 | print('--stdin or --file is not set!') 74 | exit(1) 75 | 76 | if (args.chunk is not True) and (args.un_chunk is not True): 77 | print_parser_help(parser.prog) 78 | print('--chunk or --un-chunk is not set!') 79 | exit(1) 80 | 81 | if args.file is None: 82 | content = get_content_from_stdin() 83 | else: 84 | content = get_content_from_file(file_path=args.file) 85 | 86 | if args.chunk: 87 | result = chunk_string(content.strip()) 88 | else: 89 | result = unchunk_string(content) 90 | print(result) 91 | -------------------------------------------------------------------------------- /burp_utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxenon/grpc-pentest-suite/9510feef3f4657c9f5a0529c9d176abacf12af71/burp_utils/__init__.py -------------------------------------------------------------------------------- /burp_utils/grpc_coder_withdout_dependency.py: -------------------------------------------------------------------------------- 1 | """ 2 | This is created to remove the protoscope binary dependency for burp suite 3 | """ 4 | 5 | import grpc_coder 6 | import base64 7 | import json 8 | from collections import OrderedDict 9 | import sys 10 | reload(sys) 11 | sys.setdefaultencoding('utf-8') 12 | sys.path.insert(0, "libs/blackboxprotobuf") 13 | sys.path.insert(0, "libs/six") 14 | import blackboxprotobuf 15 | 16 | 17 | def decode_b64_grpc_web_text(payload): 18 | try: 19 | base64_decoded = grpc_coder.decode_b64_payload(payload) 20 | b64_to_hex = grpc_coder.convert_to_hex(base64_decoded) 21 | payload_length_prefix, payload = grpc_coder.split_grpc_length_prefix(b64_to_hex) 22 | length = grpc_coder.calculate_length_from_length_prefix(payload_length_prefix) 23 | main_payload = grpc_coder.read_payload_based_on_length(payload, length) 24 | ascii_payload = grpc_coder.new_method_convert_hex_to_ascii(main_payload) 25 | message, typedef = blackboxprotobuf.protobuf_to_json(ascii_payload) 26 | return message, typedef 27 | except Exception as e: 28 | raise e 29 | 30 | 31 | def decode_grpc_web_proto_payload(payload): 32 | b64_payload = base64.b64encode(payload) 33 | b64_payload = b64_payload.decode('utf-8') 34 | msg, typedef = decode_b64_grpc_web_text(b64_payload) 35 | return msg, typedef 36 | 37 | 38 | def encode_grpc_web_json_to_b64_format(json_payload, typedef): 39 | raw_paylaod = blackboxprotobuf.protobuf_from_json(json_payload, typedef) 40 | hex_converted = grpc_coder.convert_to_hex(raw_paylaod) 41 | hex_length_prefix = grpc_coder.get_padded_length_of_new_payload(hex_converted) 42 | new_payload_with_length_prefix = hex_length_prefix + str(hex_converted.decode()) 43 | ascii_result = grpc_coder.new_method_convert_hex_to_ascii(new_payload_with_length_prefix) 44 | b64_result = grpc_coder.convert_ascii_to_b64(ascii_result) 45 | return b64_result 46 | 47 | 48 | def encode_grpc_web_proto_json_to_proto_format(json_payload, typedef): 49 | raw_paylaod = blackboxprotobuf.protobuf_from_json(json_payload, typedef) 50 | hex_converted = grpc_coder.convert_to_hex(raw_paylaod) 51 | hex_length_prefix = grpc_coder.get_padded_length_of_new_payload(hex_converted) 52 | new_payload_with_length_prefix = hex_length_prefix + str(hex_converted.decode()) 53 | ascii_result = grpc_coder.new_method_convert_hex_to_ascii(new_payload_with_length_prefix) 54 | return ascii_result 55 | 56 | 57 | def get_main_json_from_type_def_ordered_dict(type_def): 58 | temp_dict = {} 59 | for k in type_def.keys(): 60 | temp_dict[k] = type_def[k]['type'] 61 | 62 | pretty_json = json.dumps(temp_dict, indent=1) 63 | return pretty_json 64 | 65 | 66 | def create_bbpb_type_def_from_json(json_type_def): 67 | parsed_json = json.loads(json_type_def) 68 | temp_type_def = {} 69 | for k in parsed_json.keys(): 70 | temp_type_def[str(k)] = OrderedDict([ 71 | ('name', u''), 72 | ('type', parsed_json[k].decode('utf-8')), 73 | ('example_value_ignored', u'') 74 | ]) 75 | 76 | return temp_type_def 77 | -------------------------------------------------------------------------------- /burp_utils_with_dep/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxenon/grpc-pentest-suite/9510feef3f4657c9f5a0529c9d176abacf12af71/burp_utils_with_dep/__init__.py -------------------------------------------------------------------------------- /burp_utils_with_dep/burp_grpc_decodetab.py: -------------------------------------------------------------------------------- 1 | from burp import IBurpExtender 2 | from burp import IContextMenuFactory, IContextMenuInvocation 3 | from java.io import PrintWriter 4 | from javax.swing import JMenuItem 5 | from java.lang import System 6 | from burp import IMessageEditorTabFactory 7 | from burp import IMessageEditorTab 8 | from burp import IScannerInsertionPointProvider 9 | from burp import IScannerInsertionPoint 10 | from burp import IIntruderPayloadProcessor 11 | 12 | import grpc_utils 13 | 14 | 15 | class ProtoDecodeTab(IMessageEditorTab): 16 | 17 | def __init__(self, extender, controller, editable, helpers): 18 | self._extender = extender 19 | self._editable = editable 20 | 21 | # create an instance of Burp's text editor, to display our deserialized data 22 | self._txtInput = extender._callbacks.createTextEditor() 23 | self._txtInput.setEditable(editable) 24 | self._current_payload = "" 25 | 26 | self.stdout = PrintWriter(extender._callbacks.getStdout(), True) 27 | self.stderr = PrintWriter(extender._callbacks.getStderr(), True) 28 | 29 | # 30 | # implement IMessageEditorTab 31 | # 32 | def pprint(self, text): 33 | self.stdout.println(text) 34 | 35 | def getTabCaption(self): 36 | return "Decoded Protobuf" 37 | 38 | def getUiComponent(self): 39 | return self._txtInput.getComponent() 40 | 41 | def isEnabled(self, content, isRequest): 42 | # enable this tab for requests containing a data parameter 43 | return True 44 | 45 | def setMessage(self, content, isRequest): 46 | if (content is None): 47 | # clear our display 48 | self._txtInput.setText(None) 49 | self._txtInput.setEditable(False) 50 | 51 | else: 52 | # retrieve the serialized data 53 | requestInfo = self._extender.helpers.analyzeRequest(content) 54 | headers = requestInfo.getHeaders() 55 | msgBody = content[requestInfo.getBodyOffset():] 56 | 57 | newHeaders = list(headers) 58 | 59 | if not len(newHeaders) > 0: 60 | print("No headers") 61 | print(newHeaders) 62 | return 63 | 64 | query_line = newHeaders[0] 65 | 66 | if " " not in query_line: 67 | print("No space in query line? ") 68 | print(query_line) 69 | return 70 | 71 | # build a new http message 72 | method = query_line.split(" ")[0] 73 | 74 | msgBody = self._extender.helpers.bytesToString(msgBody) 75 | decodedData = grpc_utils.get_decoded_payload_grpc_web_text(msgBody) 76 | self._current_payload = decodedData 77 | # deserialize the parameter value 78 | self._txtInput.setText(decodedData) 79 | self._txtInput.setEditable(self._editable) 80 | 81 | # remember the displayed content 82 | self._currentMessage = content 83 | return 84 | 85 | def getMessage(self): 86 | # determine whether the user modified the deserialized data 87 | if (self._txtInput.isTextModified()): 88 | payload = self._txtInput.getText() 89 | payload = self._extender.helpers.bytesToString(payload) 90 | encoded_data = grpc_utils.get_encoded_payload_grpc_web_text(str(payload)) 91 | requestInfo = self._extender.helpers.analyzeRequest(self._currentMessage) 92 | content = self._currentMessage[:requestInfo.getBodyOffset()] 93 | 94 | new_request_bytes = self._extender.helpers.stringToBytes(encoded_data) 95 | content = content + new_request_bytes 96 | return content 97 | else: 98 | return self._currentMessage 99 | 100 | def isModified(self): 101 | 102 | return self._txtInput.isTextModified() 103 | 104 | def getSelectedData(self): 105 | 106 | return self._txtInput.getSelectedText() -------------------------------------------------------------------------------- /burp_utils_with_dep/burp_grpc_insertionpoint.py: -------------------------------------------------------------------------------- 1 | from burp import IBurpExtender 2 | from burp import IContextMenuFactory, IContextMenuInvocation 3 | from java.io import PrintWriter 4 | from javax.swing import JMenuItem 5 | from java.lang import System 6 | from burp import IMessageEditorTabFactory 7 | from burp import IMessageEditorTab 8 | from burp import IScannerInsertionPointProvider 9 | from burp import IScannerInsertionPoint 10 | from burp import IIntruderPayloadProcessor 11 | from array import array 12 | import traceback 13 | 14 | import grpc_utils 15 | 16 | 17 | class GrpcInsertionPoint(IScannerInsertionPoint): 18 | INS_EXTENSION_PROVIDED = 65 19 | 20 | def __init__(self, extender, baseRequest, offset, decodedData, insertionPointName, field_path=None): 21 | self._extender = extender 22 | self._baseRequest = baseRequest 23 | self._offset = offset 24 | self._originalData = decodedData 25 | self._insertionPointName = insertionPointName 26 | self._field_path = field_path or [] # Track the path to this field in nested structure 27 | self._fullMessage = decodedData 28 | 29 | try: 30 | print("\nParsing:", decodedData) 31 | self._baseValue = grpc_utils.extract_value_from_path(self._field_path, decodedData) 32 | 33 | except: 34 | print("Error parsing gRPC structure in insertion point") 35 | print(traceback.format_exc()) 36 | 37 | def buildRequest(self, payload): 38 | try: 39 | if isinstance(payload, array): 40 | payload = "".join(map(chr, payload)) 41 | elif isinstance(payload, bytes): 42 | payload = self._extender.helpers.bytesToString(payload) 43 | 44 | print("Building request with payload:", payload) 45 | print("Current field path:", self._field_path) 46 | 47 | new_message = grpc_utils.replace_value_at_path(self._fullMessage, self._field_path, payload) 48 | print("Modified message:", new_message) 49 | 50 | encodedPayload = grpc_utils.get_encoded_payload_grpc_web_text(new_message) 51 | print("Decoded: ", grpc_utils.get_decoded_payload_grpc_web_text(encodedPayload)) 52 | # Validate encoded payload 53 | if not encodedPayload: 54 | print("Warning: Empty encoded payload") 55 | return self._baseRequest 56 | 57 | # Construct the new request 58 | prefix = self._baseRequest[:self._offset] 59 | encoded_bytes = self._extender.helpers.stringToBytes(encodedPayload) 60 | 61 | # Build the complete request 62 | result = prefix + encoded_bytes 63 | 64 | # Add any remaining data after the body if needed 65 | if self._offset + len(encoded_bytes) < len(self._baseRequest): 66 | suffix = self._baseRequest[self._offset + len(encoded_bytes):] 67 | result += suffix 68 | 69 | print("Final request length:", len(result)) 70 | return result 71 | 72 | except Exception as e: 73 | print("Error building request:", str(e)) 74 | print(traceback.format_exc()) 75 | return self._baseRequest 76 | 77 | def getPayloadOffsets(self, payload): 78 | print("getPayloadOffsets called with payload:", payload) 79 | try: 80 | encoded_base = grpc_utils.get_encoded_payload_grpc_web_text(self._fullMessage) 81 | base_bytes = self._extender.helpers.stringToBytes(encoded_base) 82 | start = self._offset 83 | end = start + len(base_bytes) 84 | print("Calculated offsets:", [start, end]) 85 | return [start, end] 86 | except: 87 | print("Error calculating offsets") 88 | print(traceback.format_exc()) 89 | return [self._offset, self._offset + len(payload)] 90 | 91 | def getInsertionPointType(self): 92 | print("getInsertionPointType called, returning:", self.INS_EXTENSION_PROVIDED) 93 | return INS_EXTENSION_PROVIDED 94 | 95 | def getInsertionPointName(self): 96 | print("getInsertionPointName called, returning:", self._insertionPointName) 97 | return self._insertionPointName 98 | 99 | def getBaseValue(self): 100 | print("getBaseValue called with base value:", self._baseValue) 101 | return self._baseValue if self._baseValue is not None else "" -------------------------------------------------------------------------------- /burp_utils_with_dep/test_old_grpc_web_burp_extension.py: -------------------------------------------------------------------------------- 1 | # test_grpc_fuzzing.py 2 | import pytest 3 | from grpc_utils import find_insertion_points, replace_value_at_path, extract_value_from_path 4 | from grpc_utils import get_encoded_payload_grpc_web_text, get_decoded_payload_grpc_web_text 5 | 6 | 7 | class TestGRPCFuzzing: 8 | @pytest.fixture 9 | def test_messages(self): 10 | return { 11 | 'simple': '''1: {"test"}''', 12 | 'nested': '''1: {"' AND pg_sleep(20)--"} 13 | 5: {"test"} 14 | 10: {2: 15}''', 15 | 'multi_field': '''1: { 16 | 1: {"1234"} 17 | 3: {"test"} 18 | 5: {"test"} 19 | 10: { 20 | 1: {"test"} 21 | 2: {"test"} 22 | 5: {"18d157ca-72d3-4c26-999f-cf84d8135d8e"} 23 | 6: {"test"} 24 | } 25 | }''' 26 | } 27 | 28 | @pytest.fixture 29 | def test_payloads(self): 30 | return [ 31 | '\'">', 32 | "' OR '1'='1", 33 | '', 34 | ] 35 | 36 | def validate_grpc_text(self, text): 37 | """Check if the text follows gRPC-text format rules""" 38 | if not text: 39 | return False 40 | text = text.decode('utf-8') 41 | # Basic structure validation 42 | lines = text.split('\n') 43 | for line in lines: 44 | line = line.strip() 45 | if not line: 46 | continue 47 | 48 | # Each line should either be a field definition or a nested structure 49 | if ': {' in line: 50 | field_num = line.split(':', 1)[0].strip() 51 | if not field_num.isdigit(): 52 | return False 53 | 54 | value = line.split('{', 1)[1].rstrip('}').strip() 55 | if value.startswith('"'): 56 | if not value.endswith('"'): 57 | return False 58 | elif not value.replace('.', '').isdigit(): 59 | return False 60 | 61 | return True 62 | 63 | def test_find_all_fuzzable_fields(self, test_messages): 64 | """Test that we can locate all fuzzable fields in each message type""" 65 | expected_fields = { 66 | 'simple': 1, # Just field 1 67 | 'nested': 3, # Fields 1, 5, and 10.2 68 | 'multi_field': 7 # All nested fields 69 | } 70 | 71 | for msg_type, message in test_messages.items(): 72 | points = find_insertion_points(message) 73 | assert len(points) == expected_fields[msg_type], f"Wrong number of insertion points for {msg_type}" 74 | 75 | # Verify each point has required properties 76 | for point in points: 77 | assert 'path' in point 78 | assert 'name' in point 79 | assert 'value' in point 80 | 81 | def test_payload_injection_and_encoding(self, test_messages, test_payloads): 82 | """Test injecting payloads into each field and validating the result""" 83 | for msg_type, message in test_messages.items(): 84 | points = find_insertion_points(message) 85 | 86 | for point in points: 87 | for payload in test_payloads: 88 | # Inject payload 89 | modified = replace_value_at_path(message, point['path'], payload) 90 | 91 | if isinstance(modified, bytes): 92 | modified = modified.decode("utf-8") 93 | # Encode 94 | 95 | assert(type(modified) == str) 96 | encoded = get_encoded_payload_grpc_web_text(modified) 97 | assert encoded, f"Encoding failed for {msg_type} at {point['path']}" 98 | 99 | if isinstance(encoded, bytes): 100 | encoded = encoded.decode("utf-8") 101 | # Decode and validate 102 | decoded = get_decoded_payload_grpc_web_text(encoded) 103 | assert decoded, f"Decoding failed for {msg_type} at {point['path']}" 104 | 105 | print('-'*10) 106 | print(decoded.decode('utf-8')) 107 | 108 | 109 | 110 | 111 | def test_encoding_roundtrip(self, test_messages, test_payloads): 112 | """Test that messages remain valid after encode/decode roundtrip""" 113 | for msg_type, message in test_messages.items(): 114 | # First roundtrip without modification 115 | encoded = get_encoded_payload_grpc_web_text(message) 116 | decoded = get_decoded_payload_grpc_web_text(encoded) 117 | 118 | print('-' * 10) 119 | print(decoded.decode('utf-8')) 120 | 121 | 122 | # Then with modifications 123 | points = find_insertion_points(message) 124 | for point in points: 125 | for payload in test_payloads: 126 | modified = replace_value_at_path(message, point['path'], payload) 127 | 128 | encoded = get_encoded_payload_grpc_web_text(modified) 129 | decoded = get_decoded_payload_grpc_web_text(encoded) 130 | 131 | print('-' * 10) 132 | print(decoded.decode('utf-8')) 133 | 134 | 135 | def test_field_value_preservation(self, test_messages): 136 | """Test that non-modified fields retain their values""" 137 | for msg_type, message in test_messages.items(): 138 | points = find_insertion_points(message) 139 | original_values = { 140 | tuple(point['path']): point['value'] 141 | for point in points 142 | } 143 | 144 | # Modify one field at a time 145 | for point in points: 146 | modified = replace_value_at_path(message, point['path'], "TEST_VALUE") 147 | encoded = get_encoded_payload_grpc_web_text(modified) 148 | decoded = get_decoded_payload_grpc_web_text(encoded) 149 | 150 | # Check other fields remained unchanged 151 | for other_point in points: 152 | if other_point['path'] != point['path']: 153 | value = extract_value_from_path(other_point['path'], decoded) 154 | assert value == original_values[tuple(other_point['path'])], \ 155 | f"Unrelated field changed in {msg_type}" -------------------------------------------------------------------------------- /grpc_coder.py: -------------------------------------------------------------------------------- 1 | """ 2 | Encode and Decode GRPC-Web Base64 Encoded Payload for Pentesting GRPC-Web 3 | """ 4 | 5 | import base64 6 | import binascii 7 | import sys 8 | from argparse import ArgumentParser 9 | 10 | 11 | def decode_b64_payload(b64_content): 12 | try: 13 | decoded = base64.b64decode(b64_content) 14 | except Exception as e: 15 | print('Error occurred while decoding b64 payload: ' + str(e)) 16 | raise e 17 | 18 | return decoded 19 | 20 | 21 | def encode_b64_payload(content_input): 22 | base64_encoded = base64.b64encode(content_input) 23 | return base64_encoded.decode('utf-8') 24 | 25 | 26 | def convert_to_hex(content): 27 | try: 28 | hex_rep = binascii.hexlify(content) 29 | except Exception as e: 30 | print('Error occurred while converting payload to hex: ' + str(e)) 31 | raise e 32 | 33 | return hex_rep 34 | 35 | 36 | def new_method_convert_hex_to_ascii(hex_input): 37 | ascii_bytes = bytearray.fromhex(hex_input) 38 | return ascii_bytes 39 | 40 | 41 | def split_grpc_length_prefix(hex_input): 42 | """ 43 | split length prefix and payload from hex input 44 | :param hex_input: 45 | :return: length_prefix, payload 46 | """ 47 | hex_input = hex_input.decode() 48 | length_prefix = hex_input[0:10] 49 | payload = hex_input[10:] 50 | 51 | return length_prefix, payload 52 | 53 | 54 | def calculate_length_from_length_prefix(length_prefix): 55 | try: 56 | tmp = int(length_prefix, 16) * 2 # * 2 is bcs each byte has 2 characters 57 | except Exception as e: 58 | print('Error occurred while calculating length of payload: ' + str(e)) 59 | raise e 60 | 61 | return tmp 62 | 63 | 64 | def read_payload_based_on_length(payload, length): 65 | temp_str = payload[0:length] 66 | return temp_str 67 | 68 | 69 | def convert_payload_hex_to_formatted_output(hex_payload): 70 | # convert for example 0a0d02 to \\x0a\\x0d\\x02 71 | 72 | temp_str = "" 73 | for i in range(0, len(hex_payload)): 74 | 75 | if i % 2 == 0: 76 | temp_str += r"\\x" + hex_payload[i] 77 | else: 78 | temp_str += hex_payload[i] 79 | 80 | return temp_str 81 | 82 | 83 | def convert_hex_to_ascii(hex_input): 84 | return bytes.fromhex(hex_input) 85 | 86 | 87 | def convert_ascii_to_b64(ascii_input): 88 | encoded_b64 = base64.b64encode(ascii_input) 89 | return encoded_b64.decode('utf-8') 90 | 91 | 92 | def get_padded_length_of_new_payload(payload): 93 | length = len(payload) / 2 94 | length = int(length) 95 | tmp = format(length, 'x') 96 | 97 | if len(tmp) < 10: 98 | tmp = "0" * (10 - len(tmp)) + tmp 99 | 100 | return tmp 101 | 102 | 103 | def decoder(content_input): 104 | """ 105 | application/grpc-web-text decoder 106 | :param content_input: 107 | :return: 108 | """ 109 | 110 | base64_decoded = decode_b64_payload(content_input) 111 | b64_to_hex = convert_to_hex(base64_decoded) 112 | payload_length_prefix, payload = split_grpc_length_prefix(b64_to_hex) 113 | length = calculate_length_from_length_prefix(payload_length_prefix) 114 | main_payload = read_payload_based_on_length(payload, length) 115 | # formatted_output = convert_payload_hex_to_formatted_output(main_payload) 116 | ascii_payload = convert_hex_to_ascii(main_payload) 117 | 118 | sys.stdout.buffer.write(ascii_payload) 119 | 120 | 121 | def encoder(content_input): 122 | """ 123 | application/grpc-web-text encoder 124 | :param content_input: 125 | :return: 126 | """ 127 | 128 | hex_converted = convert_to_hex(content_input) 129 | hex_length_prefix = get_padded_length_of_new_payload(hex_converted) 130 | new_payload_with_length_prefix = hex_length_prefix + str(hex_converted.decode()) 131 | ascii_result = convert_hex_to_ascii(new_payload_with_length_prefix) 132 | b64_result = convert_ascii_to_b64(ascii_result) 133 | print(b64_result) 134 | 135 | 136 | def grpc_web_encoder(content_input): 137 | """ 138 | application/grpc-web+proto encoder 139 | :param content_input: 140 | :return: 141 | """ 142 | 143 | hex_converted = convert_to_hex(content_input) 144 | hex_length_prefix = get_padded_length_of_new_payload(hex_converted) 145 | new_payload_with_length_prefix = hex_length_prefix + str(hex_converted.decode()) 146 | ascii_payload = convert_hex_to_ascii(new_payload_with_length_prefix) 147 | 148 | sys.stdout.buffer.write(ascii_payload) 149 | 150 | 151 | def grpc_web_decoder(content_input): 152 | """ 153 | application/grpc-web-text decoder 154 | :param content_input: 155 | :return: 156 | """ 157 | 158 | base64_encoded_content = encode_b64_payload(content_input) 159 | decoder(base64_encoded_content) 160 | 161 | 162 | def print_parser_help(prog): 163 | help_msg = """echo payload | python3 {} [--encode OR --decode] 164 | 165 | General Arguments: 166 | --encode encode protoscope binary output to application/grpc-web-text 167 | --decode decode application/grpc-web-text base64 encoded payload to protoscope format 168 | --type content-type of payload [default: grpc-web-text] available types: [grpc-web-text, grpc-web+proto] 169 | 170 | Input Arguments: 171 | Default Input is Standard Input 172 | --file to get input from a file 173 | 174 | Help: 175 | --help print help message 176 | """.format(prog) 177 | 178 | print(help_msg) 179 | 180 | 181 | def get_content_from_stdin(): 182 | return sys.stdin.buffer.read() 183 | 184 | 185 | def get_content_from_file(file_path): 186 | try: 187 | with open(file_path, 'rb') as file: 188 | return file.read() 189 | 190 | except Exception as e: 191 | print('Error Occurred in Reading Input File: ' + str(e)) 192 | raise e 193 | 194 | 195 | if __name__ == '__main__': 196 | parser = ArgumentParser(usage='echo payload | python3 %(prog)s [--encode or --decode]', 197 | allow_abbrev=False, add_help=False) 198 | 199 | parser.add_argument('--help', action='store_true', default=False) 200 | parser.add_argument('--encode', action='store_true') 201 | parser.add_argument('--decode', action='store_true') 202 | parser.add_argument('--file', default=None) 203 | parser.add_argument('--type', default='grpc-web-text') 204 | 205 | args, unknown = parser.parse_known_args() 206 | 207 | if (args.encode is not True) and (args.decode is not True): 208 | print_parser_help(parser.prog) 209 | exit(1) 210 | 211 | if args.file is None: 212 | content = get_content_from_stdin() 213 | else: 214 | content = get_content_from_file(file_path=args.file) 215 | if args.decode is True: 216 | if args.type == 'grpc-web-text': 217 | decoder(content) 218 | elif args.type == 'grpc-web+proto': 219 | grpc_web_decoder(content) 220 | else: 221 | print('Invalid Content-Type: ' + args.type) 222 | print('Available types: grpc-web-text OR grpc-web+proto') 223 | else: 224 | if args.type == 'grpc-web-text': 225 | encoder(content) 226 | elif args.type == 'grpc-web+proto': 227 | grpc_web_encoder(content) 228 | else: 229 | print('Invalid Content-Type: ' + args.type) 230 | print('Available types: grpc-web-text OR grpc-web+proto') 231 | -------------------------------------------------------------------------------- /grpc_scan.py: -------------------------------------------------------------------------------- 1 | """ 2 | GRPC-Scan 3 | Extracting Methods, Services and Messages (Routes) in JS files (grpc-web) 4 | """ 5 | 6 | import re 7 | from argparse import ArgumentParser 8 | import sys 9 | import jsbeautifier 10 | from texttable import Texttable 11 | 12 | 13 | def create_table(columns_list, rows_list): 14 | table = Texttable() 15 | 16 | table_list = [columns_list] 17 | for i in rows_list: 18 | table_list.append(i) 19 | table.add_rows(table_list) 20 | table_string = table.draw() 21 | # indented_table_string = ' ' + table_string.replace('\n', '\n ') # Add space before each line 22 | 23 | return table_string 24 | 25 | 26 | def beautify_js_content(content): 27 | try: 28 | beautified = jsbeautifier.beautify(content) 29 | 30 | except Exception as e: 31 | print('An error occurred in beautifying Javascript code: ' + str(e)) 32 | print('Enter valid javascript code. Do not copy js code ' 33 | 'from browser dev tools directly! try to download it directly!') 34 | print('If you are still getting this error, try to beautify Javascript code online and then use this tool!') 35 | exit(1) 36 | 37 | return beautified 38 | 39 | 40 | def extract_endpoints(content): 41 | pattern = r'MethodDescriptor\("(\/.*?)"' 42 | compiled_pattern = re.compile(pattern) 43 | matched_items = compiled_pattern.findall(content) 44 | matched_items = list(matched_items) 45 | print('Found Endpoints:') 46 | if matched_items: 47 | for m in matched_items: 48 | print(" " + m) 49 | 50 | print() 51 | 52 | 53 | def extract_messages(content): 54 | pattern = r'proto\.(.*)\.prototype\.set(.*).*=.*function\(.*\).*{\s*.*set(.*)\(.*?,(.*?),' 55 | compiled_pattern = re.compile(pattern) 56 | matched_items = compiled_pattern.findall(content) 57 | matched_items = list(matched_items) 58 | 59 | message_list = {} 60 | 61 | print('Found Messages:') 62 | if matched_items: 63 | for m in matched_items: 64 | 65 | if m[0].strip() not in message_list: 66 | message_list[m[0]] = [] 67 | if m[1].strip() not in message_list[m[0].strip()]: 68 | # add proto field *name* 1, add proto field *type* 2, add proto field *number* 3 69 | temp_list = [m[1].strip(), m[2].strip(), m[3].strip()] 70 | message_list[m[0]].append(temp_list) 71 | 72 | for m2 in message_list.keys(): 73 | print() 74 | print(f'{m2}:') 75 | print(create_table(columns_list=['Field Name', 'Field Type', 'Field Number'], rows_list=message_list[m2])) 76 | 77 | print() 78 | 79 | 80 | def read_file(file): 81 | try: 82 | with open(file, 'r', encoding='utf-8') as file: 83 | return file.read() 84 | 85 | except Exception as e: 86 | print('Error occurred on opening file: ' + str(e)) 87 | exit(1) 88 | 89 | 90 | def read_standard_input(): 91 | return sys.stdin.read() 92 | 93 | 94 | def print_parser_help(prog): 95 | help_msg = f"""python3 {prog} [INPUT] 96 | 97 | Input Arguments: 98 | --file file name of js file 99 | --stdin get input from standard input 100 | Help: 101 | --help print help message 102 | """ 103 | 104 | print(help_msg) 105 | 106 | 107 | if __name__ == "__main__": 108 | parser = ArgumentParser(usage='python3 %(prog)s [INPUT]', 109 | allow_abbrev=False, add_help=False) 110 | 111 | parser.add_argument('--help', action='store_true', default=False) 112 | parser.add_argument('--file') 113 | parser.add_argument('--stdin', action='store_true', default=False) 114 | 115 | args, unknown = parser.parse_known_args() 116 | 117 | if (args.help is True) or (args.file is None): 118 | if args.stdin is not True: 119 | print_parser_help(prog=parser.prog) 120 | exit(0) 121 | 122 | js_content = "" 123 | 124 | if args.file is not None: 125 | js_content = read_file(args.file) 126 | else: 127 | js_content = read_standard_input() 128 | 129 | js_content = beautify_js_content(js_content) 130 | 131 | extract_endpoints(js_content) 132 | extract_messages(js_content) 133 | -------------------------------------------------------------------------------- /grpc_utils.py: -------------------------------------------------------------------------------- 1 | # grpc_utils.py 2 | import subprocess 3 | import os 4 | 5 | import traceback 6 | # Add at the top of the file: 7 | import sys 8 | reload(sys) 9 | sys.setdefaultencoding('utf-8') 10 | 11 | PROTOSCOPE_PATH = "protoscope" 12 | 13 | 14 | def extract_value_from_path(field_path, message): 15 | """Extract the value at the specified field path""" 16 | if not field_path: 17 | return message 18 | 19 | lines = message.split('\n') 20 | path_stack = [] 21 | 22 | for line in lines: 23 | stripped = line.lstrip() 24 | indent = len(line) - len(stripped) 25 | 26 | while path_stack and indent <= path_stack[-1][0]: 27 | path_stack.pop() 28 | 29 | if ': {' in stripped: 30 | field_num = stripped.split(':', 1)[0].strip() 31 | value = stripped.split('{', 1)[1].rstrip('}').strip() 32 | 33 | path_stack.append((indent, field_num)) 34 | current_path = [p[1] for p in path_stack] 35 | 36 | if current_path == field_path: 37 | if value.startswith('"') and value.endswith('"'): 38 | return value[1:-1] 39 | return value 40 | 41 | return None 42 | 43 | 44 | def replace_value_at_path(message, field_path, new_value): 45 | """Replace value at the specified field path (escaping included)""" 46 | 47 | def escape_value(val): 48 | val = str(val) 49 | val = val.replace('\\', '\\\\').replace('"', '\\"') 50 | val = val.replace('\n', '\\n').replace('\r', '\\r') 51 | return '"{}"'.format(val) 52 | 53 | if not field_path: 54 | return message 55 | 56 | lines = message.split('\n') 57 | result = [] 58 | path_stack = [] 59 | 60 | for line in lines: 61 | stripped = line.lstrip() 62 | indent = len(line) - len(stripped) 63 | 64 | while path_stack and indent <= path_stack[-1][0]: 65 | path_stack.pop() 66 | 67 | if ': {' in stripped: 68 | field_num = stripped.split(':', 1)[0].strip() 69 | path_stack.append((indent, field_num)) 70 | 71 | current_path = [p[1] for p in path_stack] 72 | if current_path == field_path: 73 | escaped = escape_value(new_value) 74 | new_line = ' ' * indent + field_num + ': {' + escaped + '}' 75 | result.append(new_line) 76 | continue 77 | 78 | result.append(line) 79 | 80 | return '\n'.join(result) 81 | 82 | 83 | def find_insertion_points(decoded_message): 84 | """Find all possible insertion points in a decoded protobuf message""" 85 | insertion_points = [] 86 | 87 | def parse_message(message, current_path=[]): 88 | """Recursively parse protobuf message to find all fields""" 89 | lines = message.split('\n') 90 | path_stack = [] 91 | current_indent = 0 92 | 93 | for line in lines: 94 | stripped = line.strip() 95 | if not stripped: 96 | continue 97 | 98 | indent = len(line) - len(stripped) 99 | 100 | # Adjust path stack based on indentation 101 | while path_stack and indent <= path_stack[-1][0]: 102 | path_stack.pop() 103 | 104 | if ': {' in stripped: 105 | field_num = stripped.split(':', 1)[0].strip() 106 | value = stripped.split('{', 1)[1].rstrip('}').strip() 107 | 108 | path_stack.append((indent, field_num)) 109 | new_path = [p[1] for p in path_stack] 110 | 111 | if value == '': 112 | continue 113 | # Add the current field as an insertion point 114 | insertion_points.append({ 115 | 'path': new_path, 116 | 'name': "gRPC field {}".format('.'.join(new_path)), 117 | 'value': value 118 | }) 119 | 120 | # If the value contains nested structure, parse it 121 | if value.count('{') > value.count('"'): 122 | parse_message(value, new_path) 123 | 124 | parse_message(decoded_message) 125 | return insertion_points 126 | 127 | 128 | def get_decoded_payload_grpc_web_text(payload): 129 | temp_file_path = 'grpc_coder_output_decode.txt' 130 | temp_proto_path = 'proto_output.txt' 131 | 132 | try: 133 | # If payload is bytes, keep it as-is for base64 134 | if isinstance(payload, bytes): 135 | raw_payload = payload 136 | else: 137 | # If string/unicode, encode to ascii, ignoring problematic chars 138 | raw_payload = payload.encode('ascii', 'ignore') 139 | 140 | # Write raw bytes 141 | with open(temp_file_path, 'wb') as file: 142 | file.write(raw_payload) 143 | 144 | python_name = "python" 145 | if not os.name.startswith("Windows"): 146 | try: 147 | status = subprocess.check_output('which python3', shell=True) 148 | python_name = "python3" 149 | except subprocess.CalledProcessError: 150 | python_name = "python" 151 | 152 | command = [python_name, "grpc_coder.py", "--decode", "--file", temp_file_path] 153 | decoded = subprocess.check_output(command, shell=False) 154 | 155 | with open(temp_proto_path, 'wb') as f: 156 | f.write(decoded) 157 | 158 | try: 159 | protoscope_command = [PROTOSCOPE_PATH, temp_proto_path] 160 | protoscope_output = subprocess.check_output(protoscope_command, shell=False) 161 | return protoscope_output 162 | except subprocess.CalledProcessError as e: 163 | return decoded 164 | 165 | except: 166 | return payload 167 | finally: 168 | for temp_file in [temp_file_path, temp_proto_path]: 169 | if os.path.exists(temp_file): 170 | try: 171 | os.remove(temp_file) 172 | except: 173 | pass 174 | 175 | 176 | def get_encoded_payload_grpc_web_text(payload): 177 | temp_file_path_encoding = 'grpc_coder_output_encode.txt' 178 | temp_proto_path = 'proto_output.txt' 179 | 180 | try: 181 | # If payload is bytes, keep it as-is for base64 182 | if isinstance(payload, bytes): 183 | raw_payload = payload 184 | else: 185 | # If string/unicode, encode to ascii, ignoring problematic chars 186 | raw_payload = payload.encode('ascii', 'ignore') 187 | 188 | # Write raw bytes 189 | with open(temp_file_path_encoding, 'wb') as file: 190 | file.write(raw_payload) 191 | 192 | python_name = "python" 193 | if not os.name.startswith("Windows"): 194 | try: 195 | status = subprocess.check_output('which python3', shell=True) 196 | python_name = "python3" 197 | except subprocess.CalledProcessError: 198 | python_name = "python" 199 | 200 | try: 201 | protoscope_command = [PROTOSCOPE_PATH, "-s", temp_file_path_encoding] 202 | with open(temp_proto_path, 'wb') as f: 203 | subprocess.check_call(protoscope_command, stdout=f, shell=False) 204 | 205 | encode_command = [python_name, "grpc_coder.py", "--encode", "--file", temp_proto_path] 206 | output = subprocess.check_output(encode_command, shell=False) 207 | return output.strip() 208 | 209 | except subprocess.CalledProcessError: 210 | encode_command = [python_name, "grpc_coder.py", "--encode", "--file", temp_file_path_encoding] 211 | output = subprocess.check_output(encode_command, shell=False) 212 | return output.strip() 213 | 214 | except: 215 | return payload 216 | finally: 217 | for temp_file in [temp_file_path_encoding, temp_proto_path]: 218 | if os.path.exists(temp_file): 219 | try: 220 | os.remove(temp_file) 221 | except: 222 | pass -------------------------------------------------------------------------------- /grpc_web_burp_extension.py: -------------------------------------------------------------------------------- 1 | from burp import IBurpExtender, IMessageEditorTabFactory, ITab 2 | from java.io import PrintWriter 3 | from javax.swing import JPanel, JCheckBox, JLabel, BoxLayout 4 | from java.awt.event import ActionListener 5 | from burp_utils.burp_grpc_web_editor_tab import GrpcWebExtensionEditorTab 6 | 7 | 8 | class BurpExtender(IBurpExtender, IMessageEditorTabFactory, ITab, ActionListener): 9 | def registerExtenderCallbacks(self, callbacks): 10 | self._callbacks = callbacks 11 | self._helpers = callbacks.getHelpers() 12 | 13 | # Name of extension 14 | callbacks.setExtensionName("gRPC-Web Pentest Suite") 15 | 16 | # Set stdout and stderr 17 | self.stdout = PrintWriter(callbacks.getStdout(), True) 18 | self.stderr = PrintWriter(callbacks.getStderr(), True) 19 | 20 | # Register TabFactory 21 | callbacks.registerMessageEditorTabFactory(self) 22 | 23 | # Create and register a new UI tab 24 | self._panel = JPanel() 25 | self._panel.setLayout(BoxLayout(self._panel, BoxLayout.Y_AXIS)) 26 | 27 | features_label = JLabel("------ Features ------") 28 | 29 | self._enable_extension_msg_editor_tab_check_box = JCheckBox("Enable Extension Message Editor gRPC-Web Tab by " 30 | "Default on " 31 | "all Requests") 32 | self._enable_extension_msg_editor_tab_check_box.addActionListener(self) 33 | self._enable_extension_msg_editor_tab_check_box.setSelected(True) 34 | self._detect_encode_decode_format_from_ct_header_check_box = JCheckBox("Detect Encode/Decode format from " 35 | "Content-Type header (with value of " 36 | "application/grpc-web-text or " 37 | "application/grpc-web+proto)") 38 | self._detect_encode_decode_format_from_ct_header_check_box.addActionListener(self) 39 | self._detect_encode_decode_format_from_ct_header_check_box.setSelected(True) 40 | self._detect_encode_decode_format_from_x_grpc_header_check_box = JCheckBox("Detect Encode/Decode format from " 41 | "x-grpc-content-type header (with " 42 | "value of " 43 | "application/grpc-web-text or " 44 | "application/grpc-web+proto)") 45 | 46 | self._detect_encode_decode_format_from_x_grpc_header_check_box.addActionListener(self) 47 | self._detect_encode_decode_format_from_x_grpc_header_check_box.setEnabled(False) 48 | content_type_note_label = JLabel("------ If you enable following checkbox, extension tries to decode/encode " 49 | "the body on every request ------") 50 | self._enable_application_grpc_web_text_decode_encode_by_default_check_box = JCheckBox("Enable " 51 | "application/grpc-web" 52 | "-text Decode/Encode by " 53 | "default on all requests") 54 | self._enable_application_grpc_web_text_decode_encode_by_default_check_box.addActionListener(self) 55 | 56 | self._enable_application_grpc_web_proto_decode_encode_by_default_check_box = JCheckBox("Enable " 57 | "application/grpc-web" 58 | "+proto Decode/Encode by " 59 | "default on all requests") 60 | self._enable_application_grpc_web_proto_decode_encode_by_default_check_box.addActionListener(self) 61 | 62 | # Add components to panel 63 | self._panel.add(features_label) 64 | self._panel.add(self._enable_extension_msg_editor_tab_check_box) 65 | self._panel.add(self._detect_encode_decode_format_from_ct_header_check_box) 66 | self._panel.add(self._detect_encode_decode_format_from_x_grpc_header_check_box) 67 | self._panel.add(content_type_note_label) 68 | self._panel.add(self._enable_application_grpc_web_text_decode_encode_by_default_check_box) 69 | self._panel.add(self._enable_application_grpc_web_proto_decode_encode_by_default_check_box) 70 | 71 | # Register the new tab 72 | callbacks.addSuiteTab(self) 73 | 74 | # Handle checkbox clicks 75 | def actionPerformed(self, event): 76 | source = event.getSource() 77 | if source == self._enable_extension_msg_editor_tab_check_box: 78 | pass 79 | 80 | elif source == self._detect_encode_decode_format_from_ct_header_check_box: 81 | if self._detect_encode_decode_format_from_ct_header_check_box.isSelected(): 82 | self._detect_encode_decode_format_from_x_grpc_header_check_box.setEnabled(False) 83 | else: 84 | self._detect_encode_decode_format_from_x_grpc_header_check_box.setEnabled(True) 85 | 86 | elif source == self._detect_encode_decode_format_from_x_grpc_header_check_box: 87 | if self._detect_encode_decode_format_from_x_grpc_header_check_box.isSelected(): 88 | self._detect_encode_decode_format_from_ct_header_check_box.setEnabled(False) 89 | else: 90 | self._detect_encode_decode_format_from_ct_header_check_box.setEnabled(True) 91 | 92 | elif source == self._enable_application_grpc_web_text_decode_encode_by_default_check_box: 93 | if self._enable_application_grpc_web_text_decode_encode_by_default_check_box.isSelected(): 94 | self._detect_encode_decode_format_from_x_grpc_header_check_box.setSelected(False) 95 | self._detect_encode_decode_format_from_x_grpc_header_check_box.setEnabled(False) 96 | 97 | self._detect_encode_decode_format_from_ct_header_check_box.setSelected(False) 98 | self._detect_encode_decode_format_from_ct_header_check_box.setEnabled(False) 99 | 100 | self._enable_extension_msg_editor_tab_check_box.setSelected(True) 101 | self._enable_extension_msg_editor_tab_check_box.setEnabled(False) 102 | 103 | self._enable_application_grpc_web_proto_decode_encode_by_default_check_box.setSelected(False) 104 | self._enable_application_grpc_web_proto_decode_encode_by_default_check_box.setEnabled(False) 105 | 106 | else: 107 | self._detect_encode_decode_format_from_x_grpc_header_check_box.setEnabled(True) 108 | self._detect_encode_decode_format_from_ct_header_check_box.setEnabled(True) 109 | 110 | self._enable_extension_msg_editor_tab_check_box.setSelected(True) 111 | self._enable_extension_msg_editor_tab_check_box.setEnabled(True) 112 | 113 | self._enable_application_grpc_web_proto_decode_encode_by_default_check_box.setEnabled(True) 114 | 115 | elif source == self._enable_application_grpc_web_proto_decode_encode_by_default_check_box: 116 | if self._enable_application_grpc_web_proto_decode_encode_by_default_check_box.isSelected(): 117 | self._detect_encode_decode_format_from_x_grpc_header_check_box.setSelected(False) 118 | self._detect_encode_decode_format_from_x_grpc_header_check_box.setEnabled(False) 119 | 120 | self._detect_encode_decode_format_from_ct_header_check_box.setSelected(False) 121 | self._detect_encode_decode_format_from_ct_header_check_box.setEnabled(False) 122 | 123 | self._enable_extension_msg_editor_tab_check_box.setSelected(True) 124 | self._enable_extension_msg_editor_tab_check_box.setEnabled(False) 125 | 126 | self._enable_application_grpc_web_text_decode_encode_by_default_check_box.setSelected(False) 127 | self._enable_application_grpc_web_text_decode_encode_by_default_check_box.setEnabled(False) 128 | else: 129 | self._detect_encode_decode_format_from_x_grpc_header_check_box.setEnabled(True) 130 | self._detect_encode_decode_format_from_ct_header_check_box.setEnabled(True) 131 | 132 | self._enable_extension_msg_editor_tab_check_box.setSelected(True) 133 | self._enable_extension_msg_editor_tab_check_box.setEnabled(True) 134 | 135 | self._enable_application_grpc_web_text_decode_encode_by_default_check_box.setEnabled(True) 136 | 137 | def getTabCaption(self): 138 | return "gRPC-Web Pentest Suite" 139 | 140 | def getUiComponent(self): 141 | return self._panel 142 | 143 | def createNewInstance(self, controller, editable): 144 | return GrpcWebExtensionEditorTab(self, controller, editable) 145 | 146 | def print_output(self, text): 147 | self.stdout.println(text) 148 | 149 | def print_error(self, text): 150 | self.stderr.println(text) 151 | -------------------------------------------------------------------------------- /libs/blackboxprotobuf/CLI.md: -------------------------------------------------------------------------------- 1 | # Blackbox Protobuf Command Line Interface (CLI) 2 | 3 | ## Description 4 | 5 | The Blackbox Protobuf library has an embedded CLI interface which can be invoked 6 | with `python -m blackboxprotobuf` for use in shell scripts, to plug in to other 7 | tools, or easily decode arbitrary protobuf messages. 8 | 9 | ## Installation 10 | 11 | The Blackbox Protobuf library can be installed with: 12 | 13 | ~~~ 14 | pip install bbpb 15 | ~~~ 16 | 17 | The command line interface can then be run with: 18 | 19 | ~~~ 20 | bbpb 21 | ~~~ 22 | 23 | or 24 | 25 | ~~~ 26 | python3 -m blackboxprotobuf 27 | ~~~ 28 | 29 | ## Usage 30 | 31 | ### Examples 32 | 33 | Simple Decoder: 34 | ~~~ 35 | cat test_data | bbpb -r 36 | ~~~ 37 | 38 | Save type for editing: 39 | ~~~ 40 | cat test_data | bbpb -ot ./saved_type.json 41 | ~~~ 42 | 43 | Decode with type: 44 | ~~~ 45 | cat test_data | bbpb -it ./saved_type.json 46 | ~~~ 47 | 48 | Decode edit and re-encode: 49 | ~~~ 50 | cat test_data | bbpb > message.json 51 | vim message.json 52 | cat message.json | bbpb -e > test_data_out 53 | ~~~ 54 | 55 | 56 | ### Decoding 57 | The CLI decoding mode (default) will take a protobuf payload and an option type 58 | defintion, and output a JSON object which contains the decoded message and a 59 | type definition. 60 | 61 | By default, the binary protobuf message is expected to be provided on stdin. 62 | The input type cannot be provided through stdin and must be saved to file and 63 | provided via the `-it`/`--input-type` argument. 64 | 65 | Alternatively, the `-j`/`--json-protobuf` argument allows the protobuf message 66 | and typedef to be pass in as a single JSON object. The input JSON object should 67 | have a `protobuf_data` field which contains the base64 encoded protobuf data, and can 68 | optionally have a `typedef` field with the input type definition. This option 69 | is useful for tools calling the CLI which may not want to save files to disk 70 | for input types. 71 | 72 | 73 | The default output from the decoder will be a JSON object which contains the 74 | decoded message in the `message` field and the typedef necessary to decode the 75 | message in the `typedef` field. 76 | 77 | The output format matches the expected input for the CLI encoder, allowing the 78 | message to be easily edited and re-encoded. 79 | 80 | Alternatively, the `-r`/`--raw-decode` argument will provide a simpler output 81 | with just the JSON message and no type definition. This is useful if you don't 82 | want to edit the message, just view it, or are saving the type definition to a 83 | file with `-ot`/`--output-type` argument. 84 | 85 | 86 | The `-it`/`--input-type` and `-ot`/`--output-type` arguments will have the CLI 87 | read and/or write type definitions to the provided file. 88 | 89 | ### Encoding 90 | 91 | The `-e`/`--encode` argument put the CLI in encoding mode, which takes a JSON 92 | message type definition, and prints an encoded protobuf message to stdin. 93 | 94 | 95 | By default, the CLI expects a JSON object through stdin which contains a 96 | `message` field with the JSON representation of the message and a `typedef` 97 | field with the type definition. This format should match the output of the CLI 98 | decoder. 99 | 100 | The type definition can also be provided through a file specified with the 101 | `-it`/`--input-type` argument. If the type definition is provided through this 102 | argument and there is no `message` field on the input JSON, the encoder will 103 | use the entire input JSON as the message (eg. the output of the decoder with 104 | `-r`/`--raw-decode`). 105 | 106 | By default, the CLI will output the encoded protobuf bytes to stdout. 107 | 108 | Alternatively, the `-j`/`--json-protobuf` command line flag will output a JSON 109 | payload with `protobuf_data` and `typedef` attributes. The protobuf data field 110 | will contain base64 encoded protobuf data. This format matches the expected 111 | input of the decoder with the `-j`/`--json-protobuf` attribute. 112 | 113 | ### Editing 114 | 115 | The messages and typedefs can be easily edited following the same rules as 116 | other Blackbox Protobuf interfaces. 117 | 118 | The JSON message from the decoder can be edited to easily change field values, 119 | before passing the payload back to the encoder. It is possible to add fields if 120 | the field type is defined in the type definition and the added value matches 121 | the type definition. 122 | 123 | If you wish to edit the type definition to change field names or types, save 124 | the type definition from the output payload or the `-ot`/`--output-typedef` 125 | argument. Edit the type definition and then perform the decoding step again 126 | with `-it`/`--input-typedef`. 127 | 128 | It is not recommended that you edit the typedef from the decoder directly 129 | before passing the message/typedef to the encoder, as this may cause the 130 | payload to be encoded incorrectly. 131 | 132 | ### Payload Encoding 133 | 134 | The Blackbox Protobuf library tries to automatically handle several "wrapper" 135 | encodings. The library currently supports gzip compression and gRPC headers. 136 | During decoding, the library will attempt to detect these wrappers and unpack 137 | the protobuf payload. If a payload encoding is identified, it is stored in 138 | `payload_encoding` field of the output JSON. The encoder will then re-apply the 139 | wrapper when the payload is encoded. 140 | 141 | If the payload encoding is not provided, the encoder will default to "none" 142 | which indicates plain protobuf. The payload encoding is set to "gzip" or "grpc" 143 | for other encoding options. 144 | 145 | The payload encoding process can be overridden during decoding or encoding with 146 | the `-pe`/`--payload-encoding` argument. 147 | -------------------------------------------------------------------------------- /libs/blackboxprotobuf/Makefile: -------------------------------------------------------------------------------- 1 | all: check test_py2 test_py3 2 | setup_py2: 3 | rm -rf /tmp/bbpb-python2-tmp 4 | python2 -m virtualenv /tmp/bbpb-python2-tmp 5 | . /tmp/bbpb-python2-tmp/bin/activate 6 | python2 -m pip install -r tests/requirements-python2-dev.txt 7 | test_py2: setup_py2 8 | python2 -m pytest tests/py_test/ 9 | test_py3: 10 | poetry env use python3 11 | poetry run python -m pytest tests/py_test/ 12 | format: 13 | poetry run black . 14 | check: 15 | poetry run mypy --strict blackboxprotobuf/ 16 | prepublish: 17 | poetry build 18 | poetry config repositories.test-pypi https://test.pypi.org/legacy/ 19 | # poetry config pypi-token.test-pypi TOKEN 20 | # poetry publish -r test-pypi 21 | # poetry config pypi-token.pypi TOKEN 22 | # poetry publish 23 | -------------------------------------------------------------------------------- /libs/blackboxprotobuf/blackboxprotobuf/__init__.py: -------------------------------------------------------------------------------- 1 | """`blackboxprotobuf` provides APIs for decoding and encoding binary protobuf 2 | messages. 3 | 4 | This module re-exports the functions defined in `blackboxprotobuf.lib.api`, 5 | which provides a high level interface and convenience functions. 6 | 7 | This module is split into two submodules: 8 | 9 | - The `lib` submodule contains the decoding/encoding logic and provides a python interface. 10 | - The `burp` submodule defines the Burp Suite plugin and UI. 11 | """ 12 | 13 | # Copyright (c) 2018-2024 NCC Group Plc 14 | # 15 | # Permission is hereby granted, free of charge, to any person obtaining a copy 16 | # of this software and associated documentation files (the "Software"), to deal 17 | # in the Software without restriction, including without limitation the rights 18 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 19 | # copies of the Software, and to permit persons to whom the Software is 20 | # furnished to do so, subject to the following conditions: 21 | # 22 | # The above copyright notice and this permission notice shall be included in all 23 | # copies or substantial portions of the Software. 24 | # 25 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 26 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 27 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 28 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 29 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 30 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 | # SOFTWARE. 32 | 33 | from blackboxprotobuf.lib.api import * 34 | -------------------------------------------------------------------------------- /libs/blackboxprotobuf/blackboxprotobuf/lib/__init__.py: -------------------------------------------------------------------------------- 1 | """The `blackboxprotobuf.lib` module provides APIs for decoding and encoding 2 | binary protobuf messages. 3 | 4 | This module re-exports the functions defined in `blackboxprotobuf.lib.api`, 5 | which provides a high level interface for the module and convenience functions. 6 | """ 7 | 8 | # Copyright (c) 2018-2024 NCC Group Plc 9 | # 10 | # Permission is hereby granted, free of charge, to any person obtaining a copy 11 | # of this software and associated documentation files (the "Software"), to deal 12 | # in the Software without restriction, including without limitation the rights 13 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 14 | # copies of the Software, and to permit persons to whom the Software is 15 | # furnished to do so, subject to the following conditions: 16 | # 17 | # The above copyright notice and this permission notice shall be included in all 18 | # copies or substantial portions of the Software. 19 | # 20 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 21 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 22 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 23 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 24 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 25 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 26 | # SOFTWARE. 27 | 28 | from .api import * 29 | -------------------------------------------------------------------------------- /libs/blackboxprotobuf/blackboxprotobuf/lib/config.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018-2024 NCC Group Plc 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in all 11 | # copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | # SOFTWARE. 20 | 21 | from .types import type_maps 22 | 23 | import six 24 | from blackboxprotobuf.lib.exceptions import ( 25 | DecoderException, 26 | ) 27 | 28 | if six.PY3: 29 | import typing 30 | 31 | if typing.TYPE_CHECKING: 32 | from typing import Dict 33 | from .pytypes import TypeDefDict 34 | 35 | 36 | class Config: 37 | def __init__(self): 38 | # type: (Config) -> None 39 | # Map of message type names to typedefs, previously stored at 40 | # `blackboxprotobuf.known_messages` 41 | self.known_types = {} # type: Dict[str, TypeDefDict] 42 | 43 | # Default type for "bytes" like objects that aren't messages or strings 44 | # Other option is currently just 'bytes_hex' 45 | self.default_binary_type = "bytes" 46 | 47 | # Change the default type for a wiretype (eg. change ints to be signed 48 | # by default or fixed fields to always be float) 49 | self.default_types = {} # type: Dict[int, str] 50 | 51 | # Configure whether bbpb should try to re-encode fields in the same 52 | # order they decoded 53 | # Field order shouldn't matter for real protobufs, but is there to ensure 54 | # that bytes/string are accidentally valid protobufs don't get scrambled 55 | # by decoding/re-encoding 56 | self.preserve_field_order = True 57 | 58 | def get_default_type(self, wiretype): 59 | # type: (Config, int) -> str 60 | default_type = self.default_types.get(wiretype, None) 61 | if default_type is None: 62 | default_type = type_maps.WIRE_TYPE_DEFAULTS.get(wiretype, None) 63 | 64 | if default_type is None: 65 | raise DecoderException( 66 | "Could not find default type for wire type %d" % wiretype 67 | ) 68 | return default_type 69 | 70 | 71 | default = Config() 72 | -------------------------------------------------------------------------------- /libs/blackboxprotobuf/blackboxprotobuf/lib/exceptions.py: -------------------------------------------------------------------------------- 1 | """Exception classes for BlackboxProtobuf""" 2 | 3 | # Copyright (c) 2018-2024 NCC Group Plc 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in all 13 | # copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | 23 | import six 24 | 25 | if six.PY3: 26 | from typing import Any, Optional, List 27 | 28 | 29 | class BlackboxProtobufException(Exception): 30 | """Base class for excepions raised by Blackbox Protobuf""" 31 | 32 | def __init__(self, message, path=None, *args): 33 | # type: (str, Optional[List[str]], Any) -> None 34 | self.path = path 35 | super(BlackboxProtobufException, self).__init__(message, *args) 36 | 37 | def set_path(self, path): 38 | # type: (BlackboxProtobufException, List[str]) -> None 39 | if self.path is None: 40 | self.path = path 41 | 42 | 43 | class TypedefException(BlackboxProtobufException): 44 | """Thrown when an error is identified in the type definition, such as 45 | conflicting or inconsistent values.""" 46 | 47 | def __str__(self): 48 | # type: (TypedefException) -> str 49 | message = super(TypedefException, self).__str__() 50 | if self.path is not None: 51 | message = ( 52 | "Encountered error within typedef for field %s: " 53 | % "->".join(map(str, self.path)) 54 | ) + message 55 | else: 56 | message = ("Encountered error within typedef: ") + message 57 | return message 58 | 59 | 60 | class EncoderException(BlackboxProtobufException, ValueError): 61 | """Thrown when there is an error encoding a dictionary to a type definition""" 62 | 63 | def __str__(self): 64 | # type: (EncoderException) -> str 65 | message = super(EncoderException, self).__str__() 66 | if self.path is not None: 67 | message = ( 68 | "Encountered error encoding field %s: " % "->".join(map(str, self.path)) 69 | ) + message 70 | else: 71 | message = ("Encountered error encoding message: ") + message 72 | return message 73 | 74 | 75 | class DecoderException(BlackboxProtobufException, ValueError): 76 | """Thrown when there is an error decoding a bytestring to a dictionary""" 77 | 78 | def __str__(self): 79 | # type: (DecoderException) -> str 80 | message = super(DecoderException, self).__str__() 81 | if self.path is not None: 82 | message = ( 83 | "Encountered error decoding field %s: " % "->".join(map(str, self.path)) 84 | ) + message 85 | else: 86 | message = ("Encountered error decoding message: ") + message 87 | return message 88 | 89 | 90 | class ProtofileException(BlackboxProtobufException): 91 | def __init__(self, message, path=None, filename=None, *args): 92 | # type: (ProtofileException, str, Optional[List[str]], Optional[str], Any) -> None 93 | self.filename = filename 94 | super(BlackboxProtobufException, self).__init__(message, path, *args) 95 | 96 | def __str__(self): 97 | # type: (ProtofileException) -> str 98 | message = super(ProtofileException, self).__str__() 99 | if self.path is not None: 100 | message = ( 101 | "Encountered error within protofile %s for field %s: " 102 | % (self.filename, "->".join(map(str, self.path))) 103 | ) + message 104 | else: 105 | message = ( 106 | "Encountered error within protofile %s: " % self.filename 107 | ) + message 108 | 109 | return message 110 | -------------------------------------------------------------------------------- /libs/blackboxprotobuf/blackboxprotobuf/lib/payloads/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018-2024 NCC Group Plc 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in all 11 | # copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | # SOFTWARE. 20 | 21 | """ The payloads module is intended to handle different encodings of the 22 | protobuf data, such as compression and grpc header. """ 23 | 24 | from blackboxprotobuf.lib.exceptions import BlackboxProtobufException 25 | from . import gzip, grpc 26 | 27 | import six 28 | 29 | if six.PY3: 30 | from typing import List, Callable, Tuple, Optional 31 | 32 | 33 | # Returns an ordered list of potential decoders, from most specific to least specific 34 | # The consumer should loop through each decoder, try to decode it and then try 35 | # to decode as a protobuf. This should minimize the chance of a false positive 36 | # on any decoders 37 | def find_decoders(buf): 38 | # type: (bytes) -> List[Callable[[bytes], Tuple[bytes | list[bytes], str]]] 39 | # In the future, we can take into account content-type too, such as for 40 | # grpc, but we risk false negatives 41 | decoders = [] # type: List[Callable[[bytes], Tuple[bytes | list[bytes], str]]] 42 | 43 | if gzip.is_gzip(buf): 44 | decoders.append(gzip.decode_gzip) 45 | 46 | if grpc.is_grpc(buf): 47 | decoders.append(grpc.decode_grpc) 48 | 49 | decoders.append(_none_decoder) 50 | return decoders 51 | 52 | 53 | def _none_decoder(buf): 54 | # type: (bytes) -> Tuple[bytes, str] 55 | return buf, "none" 56 | 57 | 58 | # Decoder by name 59 | def decode_payload(buf, decoder): 60 | # type: (bytes, Optional[str]) -> Tuple[bytes | list[bytes], str] 61 | if decoder is None: 62 | return buf, "none" 63 | decoder = decoder.lower() 64 | if decoder == "none": 65 | return buf, "none" 66 | elif decoder.startswith("grpc"): 67 | return grpc.decode_grpc(buf) 68 | elif decoder == "gzip": 69 | return gzip.decode_gzip(buf) 70 | else: 71 | raise BlackboxProtobufException("Unknown decoder: " + decoder) 72 | 73 | 74 | # Encode by name, should pass in the results from the decode function 75 | def encode_payload(buf, encoder): 76 | # type: (bytes | list[bytes], Optional[str]) -> bytes 77 | if encoder is None: 78 | encoder = "none" 79 | 80 | encoder = encoder.lower() 81 | if encoder == "none": 82 | if isinstance(buf, list): 83 | raise BlackboxProtobufException( 84 | "Cannot encode multiple buffers with none/missing encoding" 85 | ) 86 | return buf 87 | elif encoder.startswith("grpc"): 88 | return grpc.encode_grpc(buf, encoder) 89 | elif encoder == "gzip": 90 | return gzip.encode_gzip(buf) 91 | else: 92 | raise BlackboxProtobufException("Unknown encoder: " + encoder) 93 | -------------------------------------------------------------------------------- /libs/blackboxprotobuf/blackboxprotobuf/lib/payloads/grpc.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018-2024 NCC Group Plc 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in all 11 | # copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | # SOFTWARE. 20 | 21 | import six 22 | import struct 23 | from blackboxprotobuf.lib.exceptions import BlackboxProtobufException 24 | 25 | if six.PY3: 26 | from typing import Tuple 27 | 28 | # gRPC over HTTP2 spec: https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md 29 | 30 | HEADER_LEN = 1 + 4 31 | 32 | 33 | def is_grpc(payload): 34 | # type: (bytes) -> bool 35 | if len(payload) < HEADER_LEN: 36 | return False 37 | if six.PY2 and isinstance(payload, bytearray): 38 | payload = bytes(payload) 39 | pos = 0 40 | while pos < len(payload): 41 | compression_byte = six.indexbytes(payload, pos) 42 | # Change this to support 0x1 once we support compression 43 | if compression_byte != 0: 44 | return False 45 | message_length = struct.unpack_from(">I", payload[pos + 1 : pos + 5])[0] 46 | pos += message_length + 5 47 | 48 | if pos != len(payload): 49 | return False 50 | return True 51 | 52 | 53 | def decode_grpc(payload): 54 | # type: (bytes) -> Tuple[bytes | list[bytes], str] 55 | """Decode GRPC. Return the protobuf data""" 56 | if six.PY2 and isinstance(payload, bytearray): 57 | payload = bytes(payload) 58 | 59 | if len(payload) == 0: 60 | raise BlackboxProtobufException("Error decoding GRPC. Payload is empty") 61 | 62 | pos = 0 63 | payloads = [] 64 | while pos + HEADER_LEN <= len(payload): 65 | compression_byte = six.indexbytes(payload, pos) 66 | pos += 1 67 | if compression_byte != 0x00: 68 | if compression_byte == 0x01: 69 | # Payload is compressed 70 | # If a payload is compressed, the compression method is specified in the `grpc-encoding` header 71 | # Options are "identity" / "gzip" / "deflate" / "snappy" / {custom} 72 | raise BlackboxProtobufException( 73 | "Error decoding GRPC. Compressed payloads are not supported" 74 | ) 75 | else: 76 | raise BlackboxProtobufException( 77 | "Error decoding GRPC. First byte must be 0 or 1 to indicate compression" 78 | ) 79 | 80 | message_length = struct.unpack_from(">I", payload[pos : pos + 4])[0] 81 | pos += 4 82 | 83 | if len(payload) < pos + message_length: 84 | raise BlackboxProtobufException( 85 | "Error decoding GRPC. Payload length does not match encoded gRPC length" 86 | ) 87 | 88 | payloads.append(payload[pos : pos + message_length]) 89 | pos += message_length 90 | 91 | if pos != len(payload): 92 | raise BlackboxProtobufException( 93 | "Error decoding GRPC. Payload length does not match encoded gRPC lengths" 94 | ) 95 | 96 | if len(payloads) > 1: 97 | return payloads, "grpc" 98 | else: 99 | return payloads[0], "grpc" 100 | 101 | 102 | def encode_grpc(data, encoding="grpc"): 103 | # type: (bytes | list[bytes], str) -> bytes 104 | if encoding != "grpc": 105 | raise BlackboxProtobufException( 106 | "Error encoding GRPC with encoding %s. GRPC is only supported with no compression" 107 | % encoding 108 | ) 109 | 110 | datas = data if isinstance(data, list) else [data] 111 | 112 | payload = bytearray() 113 | for data in datas: 114 | payload.append(0x00) # No compression 115 | payload.extend(struct.pack(">I", len(data))) # Length 116 | payload.extend(data) 117 | 118 | return payload 119 | -------------------------------------------------------------------------------- /libs/blackboxprotobuf/blackboxprotobuf/lib/payloads/gzip.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018-2024 NCC Group Plc 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in all 11 | # copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | # SOFTWARE. 20 | 21 | import zlib 22 | 23 | from blackboxprotobuf.lib.exceptions import BlackboxProtobufException 24 | 25 | import six 26 | 27 | if six.PY3: 28 | from typing import Tuple 29 | 30 | 31 | def is_gzip(buf): 32 | # type: (bytes) -> bool 33 | return buf.startswith(bytearray([0x1F, 0x8B, 0x08])) 34 | 35 | 36 | def decode_gzip(buf): 37 | # type: (bytes) -> Tuple[bytes, str] 38 | if buf.startswith(bytearray([0x1F, 0x8B, 0x08])): 39 | decompressor = zlib.decompressobj(31) 40 | return decompressor.decompress(buf), "gzip" 41 | else: 42 | raise BlackboxProtobufException( 43 | "Cannot decode as gzip: magic bytes don't match" 44 | ) 45 | 46 | 47 | def encode_gzip(buf): 48 | # type: (bytes | list[bytes]) -> bytes 49 | if isinstance(buf, list): 50 | raise BlackboxProtobufException( 51 | "Cannot encode as gzip: multiple buffers are not supported" 52 | ) 53 | compressor = zlib.compressobj(-1, zlib.DEFLATED, 31) 54 | return compressor.compress(buf) + compressor.flush() 55 | -------------------------------------------------------------------------------- /libs/blackboxprotobuf/blackboxprotobuf/lib/pytypes.py: -------------------------------------------------------------------------------- 1 | """This module provides top level types for adding type definitions to the 2 | blackboxprotobuf library. 3 | """ 4 | 5 | # Copyright (c) 2018-2024 NCC Group Plc 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy 8 | # of this software and associated documentation files (the "Software"), to deal 9 | # in the Software without restriction, including without limitation the rights 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | # copies of the Software, and to permit persons to whom the Software is 12 | # furnished to do so, subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be included in all 15 | # copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | # SOFTWARE. 24 | 25 | import six 26 | 27 | 28 | if six.PY3: 29 | from typing import Any, Dict, List, TypedDict 30 | 31 | # We say messages can have any value 32 | # Functions we define may have fixed types, but someone could add a type 33 | # function that outputs any arbitrary object 34 | Message = Dict[str | int, Any] 35 | 36 | TypeDefDict = Dict[str, "FieldDefDict"] 37 | 38 | FieldDefDict = TypedDict( 39 | "FieldDefDict", 40 | { 41 | "name": str, 42 | "type": str, 43 | "message_type_name": str, 44 | "message_typedef": TypeDefDict, 45 | "alt_typedefs": Dict[str, str | TypeDefDict], 46 | "example_value_ignored": Any, 47 | "seen_repeated": bool, 48 | "field_order": List[str], 49 | }, 50 | total=False, 51 | ) 52 | -------------------------------------------------------------------------------- /libs/blackboxprotobuf/blackboxprotobuf/lib/types/__init__.py: -------------------------------------------------------------------------------- 1 | """This module contains classes/methods for decoding individual field types. 2 | 3 | Grouped into submodules based on wire type. 4 | Re-exports type_maps items for easier access. 5 | """ 6 | 7 | # Copyright (c) 2018-2024 NCC Group Plc 8 | # 9 | # Permission is hereby granted, free of charge, to any person obtaining a copy 10 | # of this software and associated documentation files (the "Software"), to deal 11 | # in the Software without restriction, including without limitation the rights 12 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 13 | # copies of the Software, and to permit persons to whom the Software is 14 | # furnished to do so, subject to the following conditions: 15 | # 16 | # The above copyright notice and this permission notice shall be included in all 17 | # copies or substantial portions of the Software. 18 | # 19 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 20 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 21 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 22 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 23 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 24 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 25 | # SOFTWARE. 26 | 27 | from .type_maps import * 28 | -------------------------------------------------------------------------------- /libs/blackboxprotobuf/blackboxprotobuf/lib/types/fixed.py: -------------------------------------------------------------------------------- 1 | """Functions for encoding and decoding fixed size integers and floats""" 2 | 3 | # Copyright (c) 2018-2024 NCC Group Plc 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in all 13 | # copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | 23 | import struct 24 | import binascii 25 | import six 26 | from blackboxprotobuf.lib.exceptions import DecoderException, EncoderException 27 | 28 | import six 29 | 30 | if six.PY3: 31 | from typing import Any, Tuple 32 | 33 | 34 | # Generic functions for encoding/decoding structs based on the "struct" format 35 | def encode_struct(fmt, value): 36 | # type: (str, Any) -> bytes 37 | """Generic method for encoding arbitrary python "struct" values""" 38 | try: 39 | return struct.pack(fmt, value) 40 | except struct.error as exc: 41 | six.raise_from( 42 | EncoderException( 43 | "Error encoding value %r with format string %s" % (value, fmt) 44 | ), 45 | exc, 46 | ) 47 | 48 | 49 | def decode_struct(fmt, buf, pos): 50 | # type: (str, bytes, int) -> Tuple[Any, int] 51 | """Generic method for decoding arbitrary python "struct" values""" 52 | new_pos = pos + struct.calcsize(fmt) 53 | try: 54 | return struct.unpack(fmt, buf[pos:new_pos])[0], new_pos 55 | except struct.error as exc: 56 | six.raise_from( 57 | DecoderException( 58 | "Error deocding format string %s from bytes: %r" 59 | % (fmt, binascii.hexlify(buf[pos:new_pos])) 60 | ), 61 | exc, 62 | ) 63 | 64 | 65 | _fixed32_fmt = " bytes 74 | """Encode a single 32 bit fixed-size value""" 75 | return encode_struct(_fixed32_fmt, value) 76 | 77 | 78 | def decode_fixed32(buf, pos): 79 | # type: (bytes, int) -> Tuple[Any, int] 80 | """Decode a single 32 bit fixed-size value""" 81 | return decode_struct(_fixed32_fmt, buf, pos) 82 | 83 | 84 | _sfixed32_fmt = " bytes 89 | """Encode a single signed 32 bit fixed-size value""" 90 | return encode_struct(_sfixed32_fmt, value) 91 | 92 | 93 | def decode_sfixed32(buf, pos): 94 | # type: (bytes, int) -> Tuple[Any, int] 95 | """Decode a single signed 32 bit fixed-size value""" 96 | return decode_struct(_sfixed32_fmt, buf, pos) 97 | 98 | 99 | _float_fmt = " bytes 104 | """Encode a single 32 bit floating point value""" 105 | return encode_struct(_float_fmt, value) 106 | 107 | 108 | def decode_float(buf, pos): 109 | # type: (bytes, int) -> Tuple[Any, int] 110 | """Decode a single 32 bit floating point value""" 111 | return decode_struct(_float_fmt, buf, pos) 112 | 113 | 114 | _fixed64_fmt = " bytes 119 | """Encode a single 64 bit fixed-size value""" 120 | return encode_struct(_fixed64_fmt, value) 121 | 122 | 123 | def decode_fixed64(buf, pos): 124 | # type: (bytes, int) -> Tuple[Any, int] 125 | """Decode a single 64 bit fixed-size value""" 126 | return decode_struct(_fixed64_fmt, buf, pos) 127 | 128 | 129 | _sfixed64_fmt = " bytes 134 | """Encode a single signed 64 bit fixed-size value""" 135 | return encode_struct(_sfixed64_fmt, value) 136 | 137 | 138 | def decode_sfixed64(buf, pos): 139 | # type: (bytes, int) -> Tuple[Any, int] 140 | """Decode a single signed 64 bit fixed-size value""" 141 | return decode_struct(_sfixed64_fmt, buf, pos) 142 | 143 | 144 | _double_fmt = " bytes 149 | """Encode a single 64 bit floating point value""" 150 | return encode_struct(_double_fmt, value) 151 | 152 | 153 | def decode_double(buf, pos): 154 | # type: (bytes, int) -> Tuple[Any, int] 155 | """Decode a single 64 bit floating point value""" 156 | return decode_struct(_double_fmt, buf, pos) 157 | -------------------------------------------------------------------------------- /libs/blackboxprotobuf/blackboxprotobuf/lib/types/type_maps.py: -------------------------------------------------------------------------------- 1 | """Contains various maps for protobuf types, including encoding/decoding 2 | functions, wiretypes and default types 3 | """ 4 | 5 | # Copyright (c) 2018-2024 NCC Group Plc 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy 8 | # of this software and associated documentation files (the "Software"), to deal 9 | # in the Software without restriction, including without limitation the rights 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | # copies of the Software, and to permit persons to whom the Software is 12 | # furnished to do so, subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be included in all 15 | # copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | # SOFTWARE. 24 | 25 | from blackboxprotobuf.lib.types import varint, fixed, length_delim, wiretypes 26 | 27 | import six 28 | 29 | if six.PY3: 30 | from typing import Any, Callable, Dict, Tuple 31 | 32 | # Map a blackboxprotobuf type to specific encoder 33 | ENCODERS = { 34 | "uint": varint.encode_uvarint, 35 | "int": varint.encode_varint, 36 | "sint": varint.encode_svarint, 37 | "fixed32": fixed.encode_fixed32, 38 | "sfixed32": fixed.encode_sfixed32, 39 | "float": fixed.encode_float, 40 | "fixed64": fixed.encode_fixed64, 41 | "sfixed64": fixed.encode_sfixed64, 42 | "double": fixed.encode_double, 43 | "bytes": length_delim.encode_bytes, 44 | "bytes_hex": length_delim.encode_bytes_hex, 45 | "string": length_delim.encode_string, 46 | "packed_uint": length_delim.generate_packed_encoder(varint.encode_uvarint), 47 | "packed_int": length_delim.generate_packed_encoder(varint.encode_varint), 48 | "packed_sint": length_delim.generate_packed_encoder(varint.encode_svarint), 49 | "packed_fixed32": length_delim.generate_packed_encoder(fixed.encode_fixed32), 50 | "packed_sfixed32": length_delim.generate_packed_encoder(fixed.encode_sfixed32), 51 | "packed_float": length_delim.generate_packed_encoder(fixed.encode_float), 52 | "packed_fixed64": length_delim.generate_packed_encoder(fixed.encode_fixed64), 53 | "packed_sfixed64": length_delim.generate_packed_encoder(fixed.encode_sfixed64), 54 | "packed_double": length_delim.generate_packed_encoder(fixed.encode_double), 55 | } # type: Dict[str, Callable[[Any], bytes]] 56 | 57 | # Map a blackboxprotobuf type to specific decoder 58 | DECODERS = { 59 | "uint": varint.decode_uvarint, 60 | "int": varint.decode_varint, 61 | "sint": varint.decode_svarint, 62 | "fixed32": fixed.decode_fixed32, 63 | "sfixed32": fixed.decode_sfixed32, 64 | "float": fixed.decode_float, 65 | "fixed64": fixed.decode_fixed64, 66 | "sfixed64": fixed.decode_sfixed64, 67 | "double": fixed.decode_double, 68 | "bytes": length_delim.decode_bytes, 69 | "bytes_hex": length_delim.decode_bytes_hex, 70 | "string": length_delim.decode_string, 71 | "packed_uint": length_delim.generate_packed_decoder(varint.decode_uvarint), 72 | "packed_int": length_delim.generate_packed_decoder(varint.decode_varint), 73 | "packed_sint": length_delim.generate_packed_decoder(varint.decode_svarint), 74 | "packed_fixed32": length_delim.generate_packed_decoder(fixed.decode_fixed32), 75 | "packed_sfixed32": length_delim.generate_packed_decoder(fixed.decode_sfixed32), 76 | "packed_float": length_delim.generate_packed_decoder(fixed.decode_float), 77 | "packed_fixed64": length_delim.generate_packed_decoder(fixed.decode_fixed64), 78 | "packed_sfixed64": length_delim.generate_packed_decoder(fixed.decode_sfixed64), 79 | "packed_double": length_delim.generate_packed_decoder(fixed.decode_double), 80 | } # type: Dict[str, Callable[[bytes, int], Tuple[Any, int] ]] 81 | 82 | WIRETYPES = { 83 | "uint": wiretypes.VARINT, 84 | "int": wiretypes.VARINT, 85 | "sint": wiretypes.VARINT, 86 | "fixed32": wiretypes.FIXED32, 87 | "sfixed32": wiretypes.FIXED32, 88 | "float": wiretypes.FIXED32, 89 | "fixed64": wiretypes.FIXED64, 90 | "sfixed64": wiretypes.FIXED64, 91 | "double": wiretypes.FIXED64, 92 | "bytes": wiretypes.LENGTH_DELIMITED, 93 | "bytes_hex": wiretypes.LENGTH_DELIMITED, 94 | "string": wiretypes.LENGTH_DELIMITED, 95 | "message": wiretypes.LENGTH_DELIMITED, 96 | "group": wiretypes.START_GROUP, 97 | "packed_uint": wiretypes.LENGTH_DELIMITED, 98 | "packed_int": wiretypes.LENGTH_DELIMITED, 99 | "packed_sint": wiretypes.LENGTH_DELIMITED, 100 | "packed_fixed32": wiretypes.LENGTH_DELIMITED, 101 | "packed_sfixed32": wiretypes.LENGTH_DELIMITED, 102 | "packed_float": wiretypes.LENGTH_DELIMITED, 103 | "packed_fixed64": wiretypes.LENGTH_DELIMITED, 104 | "packed_sfixed64": wiretypes.LENGTH_DELIMITED, 105 | "packed_double": wiretypes.LENGTH_DELIMITED, 106 | } # type: Dict[str, int] 107 | 108 | # Default values to use when decoding each wire type 109 | # length delimited is special and handled in the length_delim module 110 | WIRE_TYPE_DEFAULTS = { 111 | wiretypes.VARINT: "int", 112 | wiretypes.FIXED32: "fixed32", 113 | wiretypes.FIXED64: "fixed64", 114 | wiretypes.LENGTH_DELIMITED: None, 115 | wiretypes.START_GROUP: None, 116 | wiretypes.END_GROUP: None, 117 | } # type: Dict[int, str | None] 118 | -------------------------------------------------------------------------------- /libs/blackboxprotobuf/blackboxprotobuf/lib/types/varint.py: -------------------------------------------------------------------------------- 1 | """Classes for encoding and decoding varint types""" 2 | 3 | # Copyright (c) 2018-2024 NCC Group Plc 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in all 13 | # copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | 23 | import binascii 24 | import struct 25 | import six 26 | 27 | from blackboxprotobuf.lib.exceptions import EncoderException, DecoderException 28 | 29 | if six.PY3: 30 | from typing import Any, Tuple 31 | 32 | # These are set in decoder.py 33 | # In theory, uvarints and zigzag varints shouldn't have a max 34 | # But this is enforced by protobuf 35 | MAX_UVARINT = (1 << 64) - 1 36 | MIN_UVARINT = 0 37 | MAX_SVARINT = (1 << 63) - 1 38 | MIN_SVARINT = -(1 << 63) 39 | 40 | 41 | def encode_uvarint(value): 42 | # type: (Any) -> bytes 43 | """Encode a long or int into a bytearray.""" 44 | if not isinstance(value, six.integer_types): 45 | raise EncoderException("Got non-int type for uvarint encoding: %s" % value) 46 | output = bytearray() 47 | if value < MIN_UVARINT: 48 | raise EncoderException( 49 | "Error encoding %d as uvarint. Value must be positive" % value 50 | ) 51 | if value > MAX_UVARINT: 52 | raise EncoderException( 53 | "Error encoding %d as uvarint. Value must be %s or less" 54 | % (value, MAX_UVARINT) 55 | ) 56 | 57 | if not value: 58 | output.append(value & 0x7F) 59 | else: 60 | while value: 61 | next_byte = value & 0x7F 62 | value >>= 7 63 | if value: 64 | next_byte |= 0x80 65 | output.append(next_byte) 66 | 67 | return output 68 | 69 | 70 | def decode_uvarint(buf, pos): 71 | # type: (bytes, int) -> Tuple[int, int] 72 | """Decode bytearray into a long.""" 73 | pos_start = pos 74 | # Convert buffer to string 75 | if six.PY2: 76 | buf = bytes(buf) 77 | 78 | try: 79 | value = 0 80 | shift = 0 81 | while six.indexbytes(buf, pos) & 0x80: 82 | value += (six.indexbytes(buf, pos) & 0x7F) << (shift * 7) 83 | pos += 1 84 | shift += 1 85 | value += (six.indexbytes(buf, pos) & 0x7F) << (shift * 7) 86 | pos += 1 87 | except IndexError: 88 | raise DecoderException( 89 | "Error decoding uvarint: read past the end of the buffer" 90 | ) 91 | 92 | # Validate that this is a cononical encoding by re-encoding the value 93 | try: 94 | test_encode = encode_uvarint(value) 95 | except EncoderException as ex: 96 | raise DecoderException( 97 | "Error decoding uvarint: value (%s) was not able to be re-encoded: %s" 98 | % (value, ex) 99 | ) 100 | if buf[pos_start:pos] != test_encode: 101 | raise DecoderException( 102 | "Error decoding uvarint: Encoding is not standard:\noriginal: %r\nstandard: %r" 103 | % (buf[pos_start:pos], test_encode) 104 | ) 105 | 106 | return (value, pos) 107 | 108 | 109 | def encode_varint(value): 110 | # type: (Any) -> bytes 111 | """Encode a long or int into a bytearray.""" 112 | if not isinstance(value, six.integer_types): 113 | raise EncoderException("Got non-int type for varint encoding: %s" % value) 114 | if value > MAX_SVARINT: 115 | raise EncoderException( 116 | "Error encoding %d as varint. Value must be <= %s" % (value, MAX_SVARINT) 117 | ) 118 | if value < MIN_SVARINT: 119 | raise EncoderException( 120 | "Error encoding %d as varint. Value must be >= %s" % (value, MIN_SVARINT) 121 | ) 122 | if value < 0: 123 | value += 1 << 64 124 | output = encode_uvarint(value) 125 | return output 126 | 127 | 128 | def decode_varint(buf, pos): 129 | # type: (bytes, int) -> Tuple[int, int] 130 | """Decode bytearray into a long.""" 131 | # Convert buffer to string 132 | pos_start = pos 133 | if six.PY2: 134 | buf = bytes(buf) 135 | 136 | value, pos = decode_uvarint(buf, pos) 137 | if value & (1 << 63): 138 | value -= 1 << 64 139 | 140 | # Validate that this is a cononical encoding by re-encoding the value 141 | try: 142 | test_encode = encode_varint(value) 143 | except EncoderException as ex: 144 | raise DecoderException( 145 | "Error decoding varint: value (%s) was not able to be re-encoded: %s" 146 | % (value, ex) 147 | ) 148 | 149 | if buf[pos_start:pos] != test_encode: 150 | raise DecoderException( 151 | "Error decoding varint: Encoding is not standard:\noriginal: %r\nstandard: %r" 152 | % (buf[pos_start:pos], test_encode) 153 | ) 154 | return (value, pos) 155 | 156 | 157 | def encode_zig_zag(value): 158 | # type: (int) -> int 159 | if value < 0: 160 | return (abs(value) << 1) - 1 161 | return value << 1 162 | 163 | 164 | def decode_zig_zag(value): 165 | # type: (int) -> int 166 | if value & 0x1: 167 | # negative 168 | return -((value + 1) >> 1) 169 | return value >> 1 170 | 171 | 172 | def encode_svarint(value): 173 | # type: (Any) -> bytes 174 | """Zigzag encode the potentially signed value prior to encoding""" 175 | if not isinstance(value, six.integer_types): 176 | raise EncoderException("Got non-int type for svarint encoding: %s" % value) 177 | # zigzag encode value 178 | if value > MAX_SVARINT: 179 | raise EncoderException( 180 | "Error encoding %d as svarint. Value must be <= %s" % (value, MAX_SVARINT) 181 | ) 182 | if value < MIN_SVARINT: 183 | raise EncoderException( 184 | "Error encoding %d as svarint. Value must be >= %s" % (value, MIN_SVARINT) 185 | ) 186 | return encode_uvarint(encode_zig_zag(value)) 187 | 188 | 189 | def decode_svarint(buf, pos): 190 | # type: (bytes, int) -> Tuple[int, int] 191 | """Decode bytearray into a long.""" 192 | pos_start = pos 193 | 194 | output, pos = decode_uvarint(buf, pos) 195 | value = decode_zig_zag(output) 196 | 197 | # Validate that this is a cononical encoding by re-encoding the value 198 | test_encode = encode_svarint(value) 199 | if buf[pos_start:pos] != test_encode: 200 | raise DecoderException( 201 | "Error decoding svarint: Encoding is not standard:\noriginal: %r\nstandard: %r" 202 | % (buf[pos_start:pos], test_encode) 203 | ) 204 | 205 | return value, pos 206 | -------------------------------------------------------------------------------- /libs/blackboxprotobuf/blackboxprotobuf/lib/types/wiretypes.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018-2024 NCC Group Plc 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in all 11 | # copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | # SOFTWARE. 20 | 21 | # Define the wiretype int values 22 | VARINT = 0 23 | FIXED64 = 1 24 | LENGTH_DELIMITED = 2 25 | START_GROUP = 3 26 | END_GROUP = 4 27 | FIXED32 = 5 28 | -------------------------------------------------------------------------------- /libs/blackboxprotobuf/blackboxprotobuf/py.typed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxenon/grpc-pentest-suite/9510feef3f4657c9f5a0529c9d176abacf12af71/libs/blackboxprotobuf/blackboxprotobuf/py.typed -------------------------------------------------------------------------------- /libs/blackboxprotobuf/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "bbpb" 3 | version = "1.4.1" 4 | description = "Library for working with protobuf messages without a protobuf type definition." 5 | authors = ["Ryan Winkelmaier "] 6 | license = "MIT" 7 | repository = "https://github.com/nccgroup/blackboxprotobuf" 8 | readme = "README.md" 9 | keywords = ["protobuf"] 10 | exclude = ["./tests"] 11 | packages = [ 12 | { include = "blackboxprotobuf" } 13 | ] 14 | 15 | [tool.poetry.dependencies] 16 | python = "^3.8" 17 | six = "^1.16" 18 | 19 | 20 | [tool.poetry.dev-dependencies] 21 | pytest = "^7.4.2" 22 | hypothesis = "^6.31.6" 23 | black = "^23.9.1" 24 | protobuf = "^3.20" 25 | 26 | [tool.poetry.group.dev.dependencies] 27 | mypy = "^1.10.0" 28 | 29 | [tool.poetry.scripts] 30 | bbpb = "blackboxprotobuf.__main__:main" 31 | 32 | [build-system] 33 | requires = ["poetry-core>=1.0.0"] 34 | build-backend = "poetry.core.masonry.api" 35 | -------------------------------------------------------------------------------- /libs/blackboxprotobuf/tests/generate_payload.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | protoc --encode=TestMessage payloads/Test.proto < payloads/test_message.in 4 | -------------------------------------------------------------------------------- /libs/blackboxprotobuf/tests/payloads/Test.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | // Set up a protobuf message that contains every possible type 3 | 4 | message TestMessage { 5 | double testDouble = 1; 6 | float testFloat = 2; 7 | int32 testInt32 = 4; 8 | int64 testInt64 = 8; 9 | uint32 testUInt32 = 16; 10 | uint64 testUInt64 = 32; 11 | sint32 testSInt32 = 64; 12 | sint64 testSInt64 = 128; 13 | fixed32 testFixed32 = 256; 14 | fixed64 testFixed64 = 512; 15 | sfixed32 testSFixed32 = 1024; 16 | sfixed64 testSFixed64 = 2048; 17 | bool testBool = 4096; 18 | string testString = 8192; 19 | bytes testBytes = 16384; 20 | EmbeddedMessage testEmbed = 32768; 21 | repeated int32 testRepeatedInt32 = 65536; 22 | } 23 | 24 | message EmbeddedMessage { 25 | double embedDouble = 3; 26 | string embedString = 2; 27 | } 28 | -------------------------------------------------------------------------------- /libs/blackboxprotobuf/tests/payloads/test_message.in: -------------------------------------------------------------------------------- 1 | testDouble: 4.55 2 | testFloat: 3.14 3 | testInt32: 5 4 | testInt64: 2 5 | testUInt32: 9 6 | testUInt64: 1024 7 | testSInt32: -2 8 | testSInt64: -1 9 | testFixed32: 20 10 | testFixed64: 10 11 | testSFixed32: -20 12 | testSFixed64: -34 13 | testBool: 1 14 | testString: "Test\n String" 15 | testBytes: "bytes\0aaa" 16 | testEmbed: { 17 | embedDouble: 2.1 18 | embedString: "Test1234" 19 | } 20 | testRepeatedInt32: 2 21 | testRepeatedInt32: 5 22 | -------------------------------------------------------------------------------- /libs/blackboxprotobuf/tests/payloads/test_message.out: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxenon/grpc-pentest-suite/9510feef3f4657c9f5a0529c9d176abacf12af71/libs/blackboxprotobuf/tests/payloads/test_message.out -------------------------------------------------------------------------------- /libs/blackboxprotobuf/tests/proxy_tests/Makefile: -------------------------------------------------------------------------------- 1 | protoc: 2 | python3 -m grpc_tools.protoc -I./ --python_out=./ --grpc_python_out=./ Test.proto 3 | certs: 4 | openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -days 365 -nodes 5 | -------------------------------------------------------------------------------- /libs/blackboxprotobuf/tests/proxy_tests/Test.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | // Set up a protobuf message that contains every possible type 3 | 4 | message TestMessage { 5 | double testDouble = 1; 6 | float testFloat = 2; 7 | int32 testInt32 = 4; 8 | int64 testInt64 = 8; 9 | uint32 testUInt32 = 16; 10 | uint64 testUInt64 = 32; 11 | sint32 testSInt32 = 64; 12 | sint64 testSInt64 = 128; 13 | fixed32 testFixed32 = 256; 14 | fixed64 testFixed64 = 512; 15 | sfixed32 testSFixed32 = 1024; 16 | sfixed64 testSFixed64 = 2048; 17 | bool testBool = 4096; 18 | string testString = 8192; 19 | bytes testBytes = 16384; 20 | EmbeddedMessage testEmbed = 32768; 21 | repeated int32 testRepeatedInt32 = 65536; 22 | } 23 | 24 | message EmbeddedMessage { 25 | double embedDouble = 3; 26 | string embedString = 2; 27 | } 28 | 29 | service TestService { 30 | rpc TestRPC(TestMessage) returns (TestMessage) {} 31 | } 32 | -------------------------------------------------------------------------------- /libs/blackboxprotobuf/tests/proxy_tests/grpc_client.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018-2024 NCC Group Plc 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in all 11 | # copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | # SOFTWARE. 20 | 21 | import grpc 22 | import Test_pb2 23 | import Test_pb2_grpc 24 | 25 | grpc_options = [ 26 | ("grpc.enable_http_proxy", True), 27 | ("grpc.http_proxy", "http://127.0.0.1:8080"), 28 | ] 29 | 30 | with open("proxy_cert.pem", "rb") as f: 31 | ssl_cert = f.read() 32 | 33 | credentials = grpc.ssl_channel_credentials(ssl_cert) 34 | with grpc.secure_channel( 35 | "127.0.0.1:8000", credentials, options=grpc_options 36 | ) as channel: 37 | stub = Test_pb2_grpc.TestServiceStub(channel) 38 | response = stub.TestRPC(Test_pb2.TestMessage(testString="test123")) 39 | print("Got response: %s" % response) 40 | -------------------------------------------------------------------------------- /libs/blackboxprotobuf/tests/proxy_tests/grpc_server.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018-2024 NCC Group Plc 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in all 11 | # copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | # SOFTWARE. 20 | 21 | import grpc 22 | from concurrent import futures 23 | 24 | 25 | import Test_pb2_grpc 26 | 27 | 28 | class TestService(Test_pb2_grpc.TestService): 29 | def TestRPC(self, msg, ctx): 30 | print("Got RPC message: %s" % msg) 31 | return msg 32 | 33 | 34 | def serve(): 35 | with open("key.pem", "rb") as f: 36 | ssl_key = f.read() 37 | with open("cert.pem", "rb") as f: 38 | ssl_cert = f.read() 39 | credentials = grpc.ssl_server_credentials([(ssl_key, ssl_cert)]) 40 | 41 | server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) 42 | Test_pb2_grpc.add_TestServiceServicer_to_server(TestService(), server) 43 | server.add_secure_port("127.0.0.1:8000", credentials) 44 | server.start() 45 | server.wait_for_termination() 46 | 47 | 48 | if __name__ == "__main__": 49 | serve() 50 | -------------------------------------------------------------------------------- /libs/blackboxprotobuf/tests/proxy_tests/http_client.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018-2024 NCC Group Plc 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in all 11 | # copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | # SOFTWARE. 20 | 21 | import requests 22 | import zlib 23 | import Test_pb2 24 | import struct 25 | 26 | 27 | for payload_type in ["none", "gzip", "grpc"]: 28 | message = Test_pb2.TestMessage(testString="test123").SerializeToString() 29 | print(f"Sending payload encoded with {payload_type}") 30 | if payload_type == "gzip": 31 | message = zlib.compress(message, level=9, wbits=31) 32 | elif payload_type == "grpc": 33 | # Fake grpc wrapper 34 | length = len(message) 35 | old_message = message 36 | message = bytearray() 37 | message.append(0x00) 38 | message.extend(struct.pack(">I", length)) 39 | message.extend(old_message) 40 | 41 | response = requests.post( 42 | "http://localhost:8000", 43 | data=message, 44 | headers={ 45 | "content-type": "application/protobuf", 46 | "payload_encoding": payload_type, 47 | }, 48 | proxies={"http": "http://localhost:8080"}, 49 | ) 50 | print(f"Got response: {response.status_code} {response.text}") 51 | response_content = response.content 52 | 53 | if payload_type == "gzip": 54 | response_content = zlib.decompress(response_content, wbits=31) 55 | elif payload_type == "grpc": 56 | old_response_content = response_content 57 | compression_byte = response_content[0] 58 | assert compression_byte == 0 59 | length = struct.unpack_from(">I", response_content[1:])[0] 60 | response_content = old_response_content[5:] 61 | assert length == len(response_content) 62 | 63 | response_message = Test_pb2.TestMessage() 64 | response_message.ParseFromString(response_content) 65 | 66 | print(f"Got response message: {response_message}") 67 | -------------------------------------------------------------------------------- /libs/blackboxprotobuf/tests/proxy_tests/http_server.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018-2024 NCC Group Plc 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in all 11 | # copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | # SOFTWARE. 20 | 21 | import Test_pb2 22 | from http.server import BaseHTTPRequestHandler, HTTPServer 23 | import zlib 24 | import struct 25 | 26 | 27 | class TestHandler(BaseHTTPRequestHandler): 28 | def do_POST(self): 29 | payload_type = self.headers.get("payload_encoding", "none") 30 | print(f"Got connection with payload encoding: {payload_type}") 31 | data = self.rfile.read1() 32 | if payload_type == "gzip": 33 | data = zlib.decompress(data, wbits=31) 34 | elif payload_type == "grpc": 35 | old_data = data 36 | compression_byte = data[0] 37 | assert compression_byte == 0 38 | length = struct.unpack_from(">I", data[1:])[0] 39 | data = old_data[5:] 40 | assert length == len(data) 41 | print("Got data: %s" % data) 42 | message = Test_pb2.TestMessage() 43 | message.ParseFromString(data) 44 | print("Got message: %s" % data) 45 | 46 | output = message.SerializeToString() 47 | 48 | if payload_type == "gzip": 49 | output = zlib.compress(output, level=9, wbits=31) 50 | elif payload_type == "grpc": 51 | # Fake grpc wrapper 52 | length = len(output) 53 | old_output = output 54 | output = bytearray() 55 | output.append(0x00) 56 | output.extend(struct.pack(">I", length)) 57 | output.extend(old_output) 58 | 59 | self.send_response(200) 60 | self.send_header("content-type", "application/protobuf") 61 | self.send_header("content-length", len(output)) 62 | self.end_headers() 63 | self.wfile.write(output) 64 | 65 | 66 | server = HTTPServer(("", 8000), TestHandler) 67 | server.serve_forever() 68 | -------------------------------------------------------------------------------- /libs/blackboxprotobuf/tests/proxy_tests/requirements.txt: -------------------------------------------------------------------------------- 1 | grpcio 2 | protobuf 3 | requests 4 | websockets 5 | websocket-client 6 | -------------------------------------------------------------------------------- /libs/blackboxprotobuf/tests/proxy_tests/websocket_client.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018-2024 NCC Group Plc 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in all 11 | # copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | # SOFTWARE. 20 | 21 | import zlib 22 | import Test_pb2 23 | import struct 24 | 25 | import websocket 26 | 27 | payload_type = "grpc" 28 | 29 | message = Test_pb2.TestMessage(testString="test123").SerializeToString() 30 | 31 | if payload_type == "gzip": 32 | message = zlib.compress(message, level=9, wbits=31) 33 | elif payload_type == "grpc": 34 | # Fake grpc wrapper 35 | length = len(message) 36 | old_message = message 37 | message = bytearray() 38 | message.append(0x00) 39 | message.extend(struct.pack(">I", length)) 40 | message.extend(old_message) 41 | 42 | ws = websocket.WebSocket() 43 | ws.connect( 44 | "ws://localhost:8000", 45 | proxy_type="http", 46 | http_proxy_host="localhost", 47 | http_proxy_port="8080", 48 | http_no_proxy=["test"], 49 | ) 50 | print("Connected") 51 | ws.send(bytes(message), websocket.ABNF.OPCODE_BINARY) 52 | print("Sent message") 53 | response_content = ws.recv() 54 | 55 | if len(response_content) == 0: 56 | print("Got an Empty response") 57 | else: 58 | if payload_type == "gzip": 59 | response_content = zlib.decompress(response_content, wbits=31) 60 | elif payload_type == "grpc": 61 | old_response_content = response_content 62 | compression_byte = response_content[0] 63 | assert compression_byte == 0 64 | length = struct.unpack_from(">I", response_content[1:])[0] 65 | response_content = old_response_content[5:] 66 | assert length == len(response_content) 67 | 68 | response_message = Test_pb2.TestMessage() 69 | response_message.ParseFromString(response_content) 70 | 71 | print(f"Got response message: {response_message}") 72 | 73 | ws.close() 74 | -------------------------------------------------------------------------------- /libs/blackboxprotobuf/tests/proxy_tests/websocket_server.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018-2024 NCC Group Plc 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in all 11 | # copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | # SOFTWARE. 20 | 21 | import Test_pb2 22 | import zlib 23 | import struct 24 | import asyncio 25 | from websockets.server import serve 26 | 27 | payload_type = "grpc" 28 | 29 | 30 | async def handle_messages(websocket): 31 | async for message in websocket: 32 | print(f"Got message: {type(message)} {message}") 33 | data = message 34 | if payload_type == "gzip": 35 | data = zlib.decompress(data, wbits=31) 36 | elif payload_type == "grpc": 37 | old_data = data 38 | compression_byte = data[0] 39 | assert compression_byte == 0 40 | length = struct.unpack_from(">I", data[1:])[0] 41 | data = old_data[5:] 42 | assert length == len(data) 43 | print("Got data: %s" % data) 44 | message = Test_pb2.TestMessage() 45 | message.ParseFromString(data) 46 | print("Got message: %s" % data) 47 | 48 | message.testString += "_server" 49 | output = message.SerializeToString() 50 | 51 | if payload_type == "gzip": 52 | output = zlib.compress(output, level=9, wbits=31) 53 | elif payload_type == "grpc": 54 | # Fake grpc wrapper 55 | length = len(output) 56 | old_output = output 57 | output = bytearray() 58 | output.append(0x00) 59 | output.extend(struct.pack(">I", length)) 60 | output.extend(old_output) 61 | 62 | await websocket.send(output) 63 | 64 | 65 | async def main(): 66 | async with serve(handle_messages, "localhost", 8000): 67 | await asyncio.Future() # run forever 68 | 69 | 70 | asyncio.run(main()) 71 | -------------------------------------------------------------------------------- /libs/blackboxprotobuf/tests/py_test/strategies.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018-2024 NCC Group Plc 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in all 11 | # copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | # SOFTWARE. 20 | 21 | import six 22 | import binascii 23 | import hypothesis.strategies as st 24 | import blackboxprotobuf 25 | from blackboxprotobuf.lib.types import varint 26 | from blackboxprotobuf.lib.types import type_maps 27 | 28 | from hypothesis import settings 29 | from hypothesis import database 30 | from datetime import timedelta 31 | 32 | settings.register_profile( 33 | "quick", 34 | max_examples=100, 35 | database=database.ExampleDatabase(".hypothesis-db"), 36 | print_blob=True, 37 | ) 38 | settings.register_profile( 39 | "extended", 40 | max_examples=1000, 41 | database=database.ExampleDatabase(".hypothesis-db"), 42 | print_blob=True, 43 | ) 44 | settings.register_profile( 45 | "superextended", 46 | max_examples=10000, 47 | database=database.ExampleDatabase(".hypothesis-db"), 48 | print_blob=True, 49 | ) 50 | settings.load_profile("quick") 51 | 52 | 53 | @st.composite 54 | def message_typedef_gen(draw, max_depth=3, anon=False, types=None, named_fields=True): 55 | output = {} 56 | field_numbers = draw( 57 | st.lists(st.integers(min_value=1, max_value=2000).map(str), min_size=1) 58 | ) 59 | # pre-generate names so we can be sure they're unique 60 | field_names = draw( 61 | st.lists( 62 | st.from_regex(blackboxprotobuf.NAME_REGEX), 63 | min_size=len(field_numbers), 64 | max_size=len(field_numbers), 65 | unique_by=lambda x: x.lower(), 66 | ) 67 | ) 68 | if types is None: 69 | message_types = [ 70 | field_type 71 | for field_type in type_maps.WIRETYPES.keys() 72 | if field_type in input_map and input_map[field_type] is not None 73 | ] 74 | else: 75 | message_types = types 76 | 77 | for field_number, field_name in zip(field_numbers, field_names): 78 | field_number = six.ensure_text(str(field_number)) 79 | if max_depth == 0 and "message" in message_types: 80 | message_types.remove("message") 81 | field_type = draw(st.sampled_from(message_types)) 82 | output[field_number] = {} 83 | output[field_number]["type"] = field_type 84 | 85 | if field_type.startswith("packed"): 86 | output[field_number]["seen_repeated"] = True 87 | elif anon and field_type in [ 88 | "message", 89 | "string", 90 | "bytes", 91 | ]: # repeated fields can't be tested easily with anonymous types due to the type guessing 92 | output[field_number]["seen_repeated"] = False 93 | else: 94 | output[field_number]["seen_repeated"] = draw(st.booleans()) 95 | 96 | if field_type == "message": 97 | output[field_number]["message_typedef"] = draw( 98 | message_typedef_gen( 99 | max_depth=max_depth - 1, 100 | anon=anon, 101 | types=types, 102 | named_fields=named_fields, 103 | ) 104 | ) 105 | 106 | # decide whether to give it a name 107 | if named_fields and not anon and draw(st.booleans()): 108 | output[field_number]["name"] = six.ensure_text(field_name) 109 | 110 | return output 111 | 112 | 113 | @st.composite 114 | def gen_message_data(draw, type_def): 115 | output = {} 116 | for number, field in type_def.items(): 117 | if "name" in field and field["name"] != "": 118 | field_label = field["name"] 119 | else: 120 | field_label = six.ensure_text(str(number)) 121 | 122 | field_type = field["type"] 123 | if field_type == "message": 124 | strat = gen_message_data(field["message_typedef"]) 125 | else: 126 | strat = input_map[field["type"]] 127 | 128 | if field.get("seen_repeated", False) and not field_type.startswith("packed"): 129 | output[field_label] = draw(st.lists(strat, min_size=2)) 130 | else: 131 | output[field_label] = draw(strat) 132 | 133 | return output 134 | 135 | 136 | @st.composite 137 | # if anon is True, typedef will only have "default" types that it will decode 138 | # to without a typedef 139 | def gen_message(draw, anon=False, named_fields=True): 140 | allowed_types = None 141 | if anon: 142 | allowed_types = list( 143 | filter(lambda x: x is not None, type_maps.WIRE_TYPE_DEFAULTS.values()) 144 | ) 145 | # add length delim wiretypes 146 | allowed_types += ["message", "string", "bytes"] 147 | type_def = draw( 148 | message_typedef_gen(anon=anon, types=allowed_types, named_fields=named_fields) 149 | ) 150 | message = draw(gen_message_data(type_def)) 151 | return type_def, message 152 | 153 | 154 | # Map types to generators 155 | input_map = { 156 | "fixed32": st.integers(min_value=0, max_value=(2**32) - 1), 157 | "sfixed32": st.integers(min_value=-(2**31), max_value=(2**31) - 1), 158 | "fixed64": st.integers(min_value=0, max_value=(2**64) - 1), 159 | "sfixed64": st.integers(min_value=-(2**63), max_value=(2**63) - 1), 160 | "float": st.floats(width=32, allow_nan=False), 161 | "double": st.floats(width=64, allow_nan=False), 162 | "uint": st.integers(min_value=varint.MIN_UVARINT, max_value=varint.MAX_UVARINT), 163 | "int": st.integers(min_value=varint.MIN_SVARINT, max_value=varint.MAX_SVARINT), 164 | "sint": st.integers(min_value=varint.MIN_SVARINT, max_value=varint.MAX_SVARINT), 165 | "bytes": st.binary(), 166 | "string": st.text(), 167 | #'bytes_hex': st.binary().map(binascii.hexlify), 168 | "message": gen_message(), 169 | } 170 | input_map.update( 171 | { 172 | "packed_uint": st.lists(input_map["uint"], min_size=1), 173 | "packed_int": st.lists(input_map["int"], min_size=1), 174 | "packed_sint": st.lists(input_map["sint"], min_size=1), 175 | "packed_fixed32": st.lists(input_map["fixed32"], min_size=1), 176 | "packed_sfixed32": st.lists(input_map["sfixed32"], min_size=1), 177 | "packed_float": st.lists(input_map["float"], min_size=1), 178 | "packed_fixed64": st.lists(input_map["fixed64"], min_size=1), 179 | "packed_sfixed64": st.lists(input_map["sfixed64"], min_size=1), 180 | "packed_double": st.lists(input_map["double"], min_size=1), 181 | "packed_bytes": st.lists(input_map["bytes"], min_size=1), 182 | "packed_string": st.lists(input_map["string"], min_size=1), 183 | "packed_bytes": st.lists(input_map["bytes"], min_size=1), 184 | #'packed_bytes_hex': st.lists(input_map['bytes_hex'], min_size=1), 185 | } 186 | ) 187 | -------------------------------------------------------------------------------- /libs/blackboxprotobuf/tests/py_test/test_exceptions.py: -------------------------------------------------------------------------------- 1 | """Try to test the exception generation by the library. Everything should 2 | throw some form of BlackboxProtobufException.""" 3 | 4 | # Copyright (c) 2018-2024 NCC Group Plc 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | from hypothesis import given 25 | import hypothesis.strategies as st 26 | 27 | from blackboxprotobuf.lib import config 28 | from blackboxprotobuf.lib.types import fixed, varint, length_delim 29 | from blackboxprotobuf.lib.exceptions import ( 30 | BlackboxProtobufException, 31 | DecoderException, 32 | EncoderException, 33 | ) 34 | 35 | # Fixed exception tests 36 | 37 | 38 | ## Encoding 39 | @given(value=st.integers()) 40 | def test_encode_fixed32(value): 41 | try: 42 | fixed.encode_fixed32(value) 43 | except BlackboxProtobufException as exc: 44 | assert not isinstance(exc, DecoderException) 45 | pass 46 | 47 | 48 | @given(value=st.integers()) 49 | def test_encode_sfixed32(value): 50 | try: 51 | fixed.encode_sfixed32(value) 52 | except BlackboxProtobufException as exc: 53 | assert not isinstance(exc, DecoderException) 54 | pass 55 | 56 | 57 | @given(value=st.decimals()) 58 | def test_encode_float(value): 59 | try: 60 | fixed.encode_float(value) 61 | except BlackboxProtobufException as exc: 62 | assert not isinstance(exc, DecoderException) 63 | pass 64 | 65 | 66 | @given(value=st.integers()) 67 | def test_encode_fixed64(value): 68 | try: 69 | fixed.encode_fixed64(value) 70 | except BlackboxProtobufException as exc: 71 | assert not isinstance(exc, DecoderException) 72 | pass 73 | 74 | 75 | @given(value=st.integers()) 76 | def test_encode_sfixed64(value): 77 | try: 78 | fixed.encode_sfixed64(value) 79 | except BlackboxProtobufException as exc: 80 | assert not isinstance(exc, DecoderException) 81 | pass 82 | 83 | 84 | @given(value=st.decimals()) 85 | def test_encode_double(value): 86 | try: 87 | fixed.encode_double(value) 88 | except BlackboxProtobufException as exc: 89 | assert not isinstance(exc, DecoderException) 90 | pass 91 | 92 | 93 | ## Decoding 94 | 95 | 96 | @given(buf=st.binary(max_size=100), pos=st.integers(max_value=200)) 97 | def test_decode_fixed32(buf, pos): 98 | try: 99 | fixed.decode_fixed32(buf, pos) 100 | except BlackboxProtobufException as exc: 101 | assert not isinstance(exc, EncoderException) 102 | pass 103 | 104 | 105 | @given(buf=st.binary(max_size=100), pos=st.integers(max_value=200)) 106 | def test_decode_sfixed32(buf, pos): 107 | try: 108 | fixed.decode_sfixed32(buf, pos) 109 | except BlackboxProtobufException as exc: 110 | assert not isinstance(exc, EncoderException) 111 | pass 112 | 113 | 114 | @given(buf=st.binary(max_size=100), pos=st.integers(max_value=200)) 115 | def test_decode_float(buf, pos): 116 | try: 117 | fixed.decode_float(buf, pos) 118 | except BlackboxProtobufException as exc: 119 | assert not isinstance(exc, EncoderException) 120 | pass 121 | 122 | 123 | @given(buf=st.binary(max_size=100), pos=st.integers(max_value=200)) 124 | def test_decode_fixed64(buf, pos): 125 | try: 126 | fixed.decode_fixed64(buf, pos) 127 | except BlackboxProtobufException as exc: 128 | assert not isinstance(exc, EncoderException) 129 | pass 130 | 131 | 132 | @given(buf=st.binary(max_size=100), pos=st.integers(max_value=200)) 133 | def test_decode_sfixed64(buf, pos): 134 | try: 135 | fixed.decode_sfixed64(buf, pos) 136 | except BlackboxProtobufException as exc: 137 | assert not isinstance(exc, EncoderException) 138 | pass 139 | 140 | 141 | @given(buf=st.binary(max_size=100), pos=st.integers(max_value=200)) 142 | def test_decode_double(buf, pos): 143 | try: 144 | fixed.decode_double(buf, pos) 145 | except BlackboxProtobufException as exc: 146 | assert not isinstance(exc, EncoderException) 147 | pass 148 | 149 | 150 | # Varint exception tests 151 | @given(value=st.integers()) 152 | def test_encode_uvarint(value): 153 | try: 154 | varint.encode_uvarint(value) 155 | except BlackboxProtobufException as exc: 156 | assert not isinstance(exc, DecoderException) 157 | pass 158 | 159 | 160 | @given(value=st.integers()) 161 | def test_encode_varint(value): 162 | try: 163 | varint.encode_varint(value) 164 | except BlackboxProtobufException as exc: 165 | assert not isinstance(exc, DecoderException) 166 | pass 167 | 168 | 169 | @given(value=st.integers()) 170 | def test_encode_svarint(value): 171 | try: 172 | varint.encode_svarint(value) 173 | except BlackboxProtobufException as exc: 174 | assert not isinstance(exc, DecoderException) 175 | pass 176 | 177 | 178 | @given(buf=st.binary(max_size=32), pos=st.integers(max_value=64)) 179 | def test_decode_uvarint(buf, pos): 180 | try: 181 | varint.decode_uvarint(buf, pos) 182 | except BlackboxProtobufException as exc: 183 | assert not isinstance(exc, EncoderException) 184 | pass 185 | 186 | 187 | @given(buf=st.binary(max_size=32), pos=st.integers(max_value=64)) 188 | def test_decode_varint(buf, pos): 189 | try: 190 | varint.decode_varint(buf, pos) 191 | except BlackboxProtobufException as exc: 192 | assert not isinstance(exc, EncoderException) 193 | pass 194 | 195 | 196 | @given(buf=st.binary(max_size=32), pos=st.integers(max_value=64)) 197 | def test_decode_svarint(buf, pos): 198 | try: 199 | varint.decode_svarint(buf, pos) 200 | except BlackboxProtobufException as exc: 201 | assert not isinstance(exc, EncoderException) 202 | pass 203 | 204 | 205 | # length_delim exception tests 206 | 207 | 208 | @given(value=st.binary()) 209 | def encode_bytes(value): 210 | try: 211 | length_delim.encode_bytes(value) 212 | except BlackboxProtobufException as exc: 213 | assert not isinstance(exc, DecoderException) 214 | pass 215 | 216 | 217 | @given(value=st.binary(), pos=st.integers(max_value=2000)) 218 | def test_decode_bytes(value, pos): 219 | try: 220 | length_delim.decode_bytes(value, pos) 221 | except BlackboxProtobufException as exc: 222 | assert not isinstance(exc, EncoderException) 223 | pass 224 | 225 | 226 | @given(value=st.binary()) 227 | def test_encode_bytes_hex(value): 228 | try: 229 | length_delim.encode_bytes_hex(value) 230 | except BlackboxProtobufException as exc: 231 | assert not isinstance(exc, DecoderException) 232 | pass 233 | 234 | 235 | @given(buf=st.binary(), pos=st.integers(max_value=2000)) 236 | def test_decode_bytes_hex(buf, pos): 237 | try: 238 | length_delim.decode_bytes_hex(buf, pos) 239 | except BlackboxProtobufException as exc: 240 | assert not isinstance(exc, EncoderException) 241 | pass 242 | 243 | 244 | @given(value=st.binary(), pos=st.integers(max_value=2000)) 245 | def test_decode_string(value, pos): 246 | try: 247 | length_delim.decode_string(value, pos) 248 | except BlackboxProtobufException as exc: 249 | assert not isinstance(exc, EncoderException) 250 | pass 251 | 252 | 253 | @given(buf=st.binary()) 254 | def test_decode_message(buf): 255 | try: 256 | length_delim.decode_message(buf, config.Config()) 257 | except BlackboxProtobufException as exc: 258 | assert not isinstance(exc, EncoderException) 259 | pass 260 | 261 | 262 | @given(buf=st.binary()) 263 | def test_decode_lendelim_message(buf): 264 | try: 265 | length_delim.decode_lendelim_message(buf, config.Config()) 266 | except BlackboxProtobufException as exc: 267 | assert not isinstance(exc, EncoderException) 268 | pass 269 | -------------------------------------------------------------------------------- /libs/blackboxprotobuf/tests/py_test/test_fixed.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018-2024 NCC Group Plc 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in all 11 | # copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | # SOFTWARE. 20 | 21 | import math 22 | from hypothesis import given 23 | import hypothesis.strategies as st 24 | import strategies 25 | 26 | from blackboxprotobuf.lib.types import fixed 27 | 28 | 29 | # Inverse checks. Ensure a value encoded by bbp decodes to the same value 30 | @given(x=strategies.input_map["fixed32"]) 31 | def test_fixed32_inverse(x): 32 | encoded = fixed.encode_fixed32(x) 33 | decoded, pos = fixed.decode_fixed32(encoded, 0) 34 | assert pos == len(encoded) 35 | assert decoded == x 36 | 37 | 38 | @given(x=st.binary(min_size=4)) 39 | def test_fixed32_idem(x): 40 | try: 41 | value, pos = fixed.decode_fixed32(x, 0) 42 | except DecoderException: 43 | assume(True) 44 | return 45 | 46 | encoded = fixed.encode_fixed32(value) 47 | assert encoded == x[:pos] 48 | 49 | 50 | @given(x=strategies.input_map["sfixed32"]) 51 | def test_sfixed32_inverse(x): 52 | encoded = fixed.encode_sfixed32(x) 53 | decoded, pos = fixed.decode_sfixed32(encoded, 0) 54 | assert pos == len(encoded) 55 | assert decoded == x 56 | 57 | 58 | @given(x=st.binary(min_size=4)) 59 | def test_sfixed32_idem(x): 60 | try: 61 | value, pos = fixed.decode_sfixed32(x, 0) 62 | except DecoderException: 63 | assume(True) 64 | return 65 | 66 | encoded = fixed.encode_sfixed32(value) 67 | assert encoded == x[:pos] 68 | 69 | 70 | @given(x=strategies.input_map["fixed64"]) 71 | def test_fixed64_inverse(x): 72 | encoded = fixed.encode_fixed64(x) 73 | decoded, pos = fixed.decode_fixed64(encoded, 0) 74 | assert pos == len(encoded) 75 | assert decoded == x 76 | 77 | 78 | @given(x=st.binary(min_size=8)) 79 | def test_fixed64_idem(x): 80 | try: 81 | value, pos = fixed.decode_fixed64(x, 0) 82 | except DecoderException: 83 | assume(True) 84 | return 85 | 86 | encoded = fixed.encode_fixed64(value) 87 | assert encoded == x[:pos] 88 | 89 | 90 | @given(x=strategies.input_map["sfixed64"]) 91 | def test_sfixed64_inverse(x): 92 | encoded = fixed.encode_sfixed64(x) 93 | decoded, pos = fixed.decode_sfixed64(encoded, 0) 94 | assert pos == len(encoded) 95 | assert decoded == x 96 | 97 | 98 | @given(x=st.binary(min_size=8)) 99 | def test_sfixed64_idem(x): 100 | try: 101 | value, pos = fixed.decode_sfixed64(x, 0) 102 | except DecoderException: 103 | assume(True) 104 | return 105 | 106 | encoded = fixed.encode_sfixed64(value) 107 | assert encoded == x[:pos] 108 | 109 | 110 | @given(x=strategies.input_map["float"]) 111 | def test_float_inverse(x): 112 | encoded = fixed.encode_float(x) 113 | decoded, pos = fixed.decode_float(encoded, 0) 114 | assert pos == len(encoded) 115 | if math.isnan(x): 116 | assert math.isnan(decoded) 117 | else: 118 | assert decoded == x 119 | 120 | 121 | # Would be nice, but not a default type, so probably ok 122 | # Probably asking for trouble to have a float decode then encode the same 123 | # @given(x=st.binary(min_size=4)) 124 | # def test_float_idem(x): 125 | # try: 126 | # value, pos = fixed.decode_float(x, 0) 127 | # except DecoderException: 128 | # assume(True) 129 | # return 130 | # 131 | # encoded = fixed.encode_float(value) 132 | # assert encoded == x[:pos] 133 | 134 | 135 | @given(x=strategies.input_map["double"]) 136 | def test_double_inverse(x): 137 | encoded = fixed.encode_double(x) 138 | decoded, pos = fixed.decode_double(encoded, 0) 139 | assert pos == len(encoded) 140 | if math.isnan(x): 141 | assert math.isnan(decoded) 142 | else: 143 | assert decoded == x 144 | 145 | 146 | # @given(x=st.binary(min_size=8)) 147 | # def test_double_idem(x): 148 | # try: 149 | # value, pos = fixed.decode_double(x, 0) 150 | # except DecoderException: 151 | # assume(True) 152 | # return 153 | # 154 | # encoded = fixed.encode_double(value) 155 | # assert encoded == x[:pos] 156 | -------------------------------------------------------------------------------- /libs/blackboxprotobuf/tests/py_test/test_json.py: -------------------------------------------------------------------------------- 1 | """Tests similar to the length_delim or protobuf tests, but make sure we can round trip through the JSON encode/decode """ 2 | 3 | # Copyright (c) 2018-2024 NCC Group Plc 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in all 13 | # copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | 23 | from hypothesis import given, assume, note, example, reproduce_failure 24 | import hypothesis.strategies as st 25 | import strategies 26 | import json 27 | import six 28 | import binascii 29 | 30 | from blackboxprotobuf.lib.config import Config 31 | from blackboxprotobuf.lib.types import length_delim 32 | from blackboxprotobuf.lib.types import type_maps 33 | from blackboxprotobuf.lib.typedef import TypeDef 34 | from blackboxprotobuf.lib.payloads import grpc, gzip 35 | import blackboxprotobuf 36 | 37 | 38 | @given(x=strategies.gen_message()) 39 | def test_message_json_inverse(x): 40 | config = Config() 41 | typedef, message = x 42 | encoded = length_delim.encode_message(message, config, TypeDef.from_dict(typedef)) 43 | decoded_json, typedef_out = blackboxprotobuf.protobuf_to_json( 44 | encoded, config=config, message_type=typedef 45 | ) 46 | blackboxprotobuf.validate_typedef(typedef_out) 47 | encoded_json = blackboxprotobuf.protobuf_from_json( 48 | decoded_json, config=config, message_type=typedef_out 49 | ) 50 | assert not isinstance(encoded_json, list) 51 | decoded, typedef_out = blackboxprotobuf.decode_message( 52 | encoded_json, config=config, message_type=typedef 53 | ) 54 | blackboxprotobuf.validate_typedef(typedef_out) 55 | assert isinstance(encoded, bytearray) 56 | assert isinstance(decoded, dict) 57 | assert message == decoded 58 | 59 | 60 | @given(x=strategies.gen_message(), n=st.integers(min_value=2, max_value=10)) 61 | def test_multiple_encoding(x, n): 62 | config = Config() 63 | typedef, message = x 64 | encoded = length_delim.encode_message(message, config, TypeDef.from_dict(typedef)) 65 | 66 | bufs = [encoded] * n 67 | message_json, typedef_out = blackboxprotobuf.protobuf_to_json(bufs, typedef, config) 68 | messages = json.loads(message_json) 69 | assert isinstance(messages, list) 70 | assert len(messages) == n 71 | 72 | encoded2 = blackboxprotobuf.protobuf_from_json(message_json, typedef, config) 73 | assert isinstance(encoded2, list) 74 | assert len(encoded2) == n 75 | 76 | 77 | @given(x=strategies.gen_message(anon=True)) 78 | def test_anon_json_decode(x): 79 | """Create a new encoded message, the try to decode without a typedef into a 80 | json, from json back to binary and then finally decode the message back to 81 | the original format. Makes sure json decoding can handle any types and does 82 | not change the essage. 83 | """ 84 | config = Config() 85 | typedef, message = x 86 | encoded = blackboxprotobuf.encode_message( 87 | message, config=config, message_type=typedef 88 | ) 89 | decoded_json, typedef_out = blackboxprotobuf.protobuf_to_json( 90 | encoded, config=config 91 | ) 92 | blackboxprotobuf.validate_typedef(typedef_out) 93 | note("To Json Typedef: %r" % dict(typedef_out)) 94 | encoded_json = blackboxprotobuf.protobuf_from_json( 95 | decoded_json, config=config, message_type=typedef_out 96 | ) 97 | assert not isinstance(encoded_json, list) 98 | decoded, typedef_out = blackboxprotobuf.decode_message( 99 | encoded_json, config=config, message_type=typedef 100 | ) 101 | blackboxprotobuf.validate_typedef(typedef_out) 102 | note("Original message: %r" % message) 103 | note("Decoded JSON: %r" % decoded_json) 104 | note("Decoded message: %r" % decoded) 105 | note("Original typedef: %r" % typedef) 106 | note("Decoded typedef: %r" % typedef_out) 107 | 108 | def check_message(orig, orig_typedef, new, new_typedef): 109 | for field_number in set(orig.keys()) | set(new.keys()): 110 | # verify all fields are there 111 | assert field_number in orig 112 | assert field_number in orig_typedef 113 | assert field_number in new 114 | assert field_number in new_typedef 115 | 116 | orig_values = orig[field_number] 117 | new_values = new[field_number] 118 | orig_type = orig_typedef[field_number]["type"] 119 | new_type = new_typedef[field_number]["type"] 120 | 121 | note("Parsing field# %s" % field_number) 122 | note("orig_values: %r" % orig_values) 123 | note("new_values: %r" % new_values) 124 | note("orig_type: %s" % orig_type) 125 | note("new_type: %s" % new_type) 126 | # Fields might be lists. Just convert everything to a list 127 | if not isinstance(orig_values, list): 128 | orig_values = [orig_values] 129 | assert not isinstance(new_values, list) 130 | new_values = [new_values] 131 | assert isinstance(orig_values, list) 132 | assert isinstance(new_values, list) 133 | 134 | # if the types don't match, then try to convert them 135 | if new_type == "message" and orig_type in ["bytes", "string"]: 136 | # if the type is a message, we want to convert the orig type to a message 137 | # this isn't ideal, we'll be using the unintended type, but 138 | # best way to compare. Re-encoding a message to binary might 139 | # not keep the field order 140 | new_field_typedef = new_typedef[field_number]["message_typedef"] 141 | for i, orig_value in enumerate(orig_values): 142 | if orig_type == "bytes": 143 | ( 144 | orig_values[i], 145 | orig_field_typedef, 146 | _, 147 | _, 148 | ) = length_delim.decode_lendelim_message( 149 | length_delim.encode_bytes(orig_value), 150 | config, 151 | TypeDef.from_dict(new_field_typedef), 152 | ) 153 | orig_field_typedef = orig_field_typedef.to_dict() 154 | else: 155 | # string value 156 | ( 157 | orig_values[i], 158 | orig_field_typedef, 159 | _, 160 | _, 161 | ) = length_delim.decode_lendelim_message( 162 | length_delim.encode_string(orig_value), 163 | config, 164 | TypeDef.from_dict(new_field_typedef), 165 | ) 166 | orig_field_typedef = orig_field_typedef.to_dict() 167 | orig_typedef[field_number]["message_typedef"] = orig_field_typedef 168 | orig_type = "message" 169 | 170 | if new_type == "string" and orig_type == "bytes": 171 | # our bytes were accidently valid string 172 | new_type = "bytes" 173 | for i, new_value in enumerate(new_values): 174 | new_values[i], _ = length_delim.decode_bytes( 175 | length_delim.encode_string(new_value), 0 176 | ) 177 | note("New values: %r" % new_values) 178 | # sort the lists with special handling for dicts 179 | orig_values.sort(key=lambda x: x if not isinstance(x, dict) else x.items()) 180 | new_values.sort(key=lambda x: x if not isinstance(x, dict) else x.items()) 181 | for orig_value, new_value in zip(orig_values, new_values): 182 | if orig_type == "message": 183 | check_message( 184 | orig_value, 185 | orig_typedef[field_number]["message_typedef"], 186 | new_value, 187 | new_typedef[field_number]["message_typedef"], 188 | ) 189 | else: 190 | assert orig_value == new_value 191 | 192 | check_message(message, typedef, decoded, typedef_out) 193 | # assert message == decoded 194 | -------------------------------------------------------------------------------- /libs/blackboxprotobuf/tests/py_test/test_payloads.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018-2024 NCC Group Plc 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in all 11 | # copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | # SOFTWARE. 20 | 21 | from hypothesis import assume, given, reproduce_failure, example 22 | import hypothesis.strategies as st 23 | import strategies 24 | import pytest 25 | 26 | from blackboxprotobuf.lib import payloads 27 | from blackboxprotobuf.lib.payloads import grpc, gzip 28 | from blackboxprotobuf.lib.exceptions import BlackboxProtobufException 29 | 30 | 31 | def test_grpc(): 32 | message = bytearray([0x00, 0x00, 0x00, 0x00, 0x01, 0xAA]) 33 | data, encoding = grpc.decode_grpc(message) 34 | assert data == bytearray([0xAA]) 35 | assert encoding == "grpc" 36 | 37 | # Compression flag 38 | with pytest.raises(BlackboxProtobufException): 39 | message = bytearray([0x01, 0x00, 0x00, 0x00, 0x01, 0xAA]) 40 | data = grpc.decode_grpc(message) 41 | 42 | # Unknown flag 43 | with pytest.raises(BlackboxProtobufException): 44 | message = bytearray([0x11, 0x00, 0x00, 0x00, 0x01, 0xAA]) 45 | data = grpc.decode_grpc(message) 46 | 47 | # Incorrect length 48 | with pytest.raises(BlackboxProtobufException): 49 | message = bytearray([0x00, 0x00, 0x01, 0x00, 0x01, 0xAA]) 50 | data = grpc.decode_grpc(message) 51 | 52 | # Incorrect length 53 | with pytest.raises(BlackboxProtobufException): 54 | message = bytearray([0x00, 0x00, 0x00, 0x00, 0x01, 0xAA, 0xBB]) 55 | data = grpc.decode_grpc(message) 56 | 57 | # Empty 58 | message = bytearray([0x00, 0x00, 0x00, 0x00, 0x00]) 59 | data, encoding = grpc.decode_grpc(message) 60 | assert len(data) == 0 61 | assert encoding == "grpc" 62 | 63 | 64 | @given(payloads=st.lists(st.binary(), min_size=2)) 65 | def test_grpc_multiple(payloads): 66 | # Test grpc encoding with multiple payloads 67 | 68 | # Manually encode multiple grpc payloads and string them together 69 | encoded = b"" 70 | for payload in payloads: 71 | encoded += grpc.encode_grpc(payload) 72 | 73 | assert grpc.is_grpc(encoded) 74 | 75 | # Make sure we can decode bytes with multiple grpc 76 | decoded, encoding = grpc.decode_grpc(encoded) 77 | assert isinstance(decoded, list) 78 | assert len(decoded) == len(payloads) 79 | 80 | for x, y in zip(decoded, payloads): 81 | assert x == y 82 | 83 | # Make sure we can encode to the same bytes 84 | encoded2 = grpc.encode_grpc(decoded) 85 | assert encoded == encoded2 86 | 87 | 88 | @given(data=st.binary()) 89 | def test_grpc_inverse(data): 90 | encoding = "grpc" 91 | encoded = grpc.encode_grpc(data) 92 | decoded, encoding_out = grpc.decode_grpc(encoded) 93 | 94 | assert data == decoded 95 | assert encoding == encoding_out 96 | 97 | 98 | @given(data=st.binary()) 99 | def test_gzip_inverse(data): 100 | encoded = gzip.encode_gzip(data) 101 | decoded, encoding_out = gzip.decode_gzip(encoded) 102 | 103 | assert data == decoded 104 | assert "gzip" == encoding_out 105 | 106 | 107 | @given(data=st.binary(), alg=st.sampled_from(["grpc", "gzip", "none"])) 108 | def test_find_payload_inverse(data, alg): 109 | encoded = payloads.encode_payload(data, alg) 110 | decoders = payloads.find_decoders(encoded) 111 | 112 | valid_decoders = {} 113 | for decoder in decoders: 114 | try: 115 | decoded, decoder_alg = decoder(encoded) 116 | valid_decoders[decoder_alg] = decoded 117 | except: 118 | pass 119 | assert "none" in valid_decoders 120 | assert alg in valid_decoders 121 | assert valid_decoders[alg] == data 122 | -------------------------------------------------------------------------------- /libs/blackboxprotobuf/tests/py_test/test_perf.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import blackboxprotobuf 3 | 4 | 5 | @pytest.mark.skip() 6 | def test_wide(): 7 | typedef = {"1": {"type": "int"}} 8 | 9 | message = {"1": [1] * 10000000} 10 | 11 | encoded = blackboxprotobuf.lib.encode_message(message, typedef) 12 | decoded, _ = blackboxprotobuf.lib.decode_message(encoded, typedef) 13 | 14 | 15 | @pytest.mark.skip() 16 | def test_deep(): 17 | config = blackboxprotobuf.lib.config.Config() 18 | 19 | typedef = { 20 | "1": {"type": "message", "message_type_name": "test"}, 21 | "2": {"type": "int"}, 22 | } 23 | config.known_types["test"] = typedef 24 | target_depth = 100 25 | message = {} 26 | last_layer = message 27 | 28 | while target_depth: 29 | new_layer = {"2": 1} 30 | last_layer["1"] = new_layer 31 | last_layer = new_layer 32 | 33 | target_depth -= 1 34 | 35 | encoded = blackboxprotobuf.lib.encode_message(message, typedef, config) 36 | decoded, _ = blackboxprotobuf.lib.decode_message(encoded, typedef, config) 37 | 38 | 39 | @pytest.mark.skip() 40 | def test_large_multilayer(): 41 | config = blackboxprotobuf.lib.config.Config() 42 | 43 | typedef = { 44 | "1": {"type": "message", "message_type_name": "test"}, 45 | "2": {"type": "int"}, 46 | } 47 | config.known_types["test"] = typedef 48 | target_depth = 10 49 | message = {} 50 | last_layer = message 51 | 52 | while target_depth: 53 | new_layer = {"2": [1] * 10000} 54 | last_layer["1"] = [new_layer] * 2 55 | last_layer = new_layer 56 | 57 | target_depth -= 1 58 | 59 | encoded = blackboxprotobuf.lib.encode_message(message, typedef, config) 60 | decoded, _ = blackboxprotobuf.lib.decode_message(encoded, typedef, config) 61 | -------------------------------------------------------------------------------- /libs/blackboxprotobuf/tests/py_test/test_protobuf.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018-2024 NCC Group Plc 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in all 11 | # copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | # SOFTWARE. 20 | 21 | from hypothesis import given, example, note 22 | import hypothesis.strategies as st 23 | import hypothesis 24 | import strategies 25 | import warnings 26 | import base64 27 | import json 28 | import six 29 | 30 | import blackboxprotobuf 31 | 32 | 33 | warnings.filterwarnings( 34 | "ignore", 35 | "Call to deprecated create function.*", 36 | ) 37 | 38 | try: 39 | import Test_pb2 40 | except: 41 | import os 42 | 43 | os.system( 44 | "cd tests/payloads; protoc --python_out ../py_test/ Test.proto; cd ../../" 45 | ) 46 | import Test_pb2 47 | 48 | testMessage_typedef = { 49 | "1": {"type": "double", "name": six.u("testDouble")}, 50 | "2": {"type": "float", "name": six.u("testFloat")}, 51 | # "4": {"type": "int", "name": "testInt32"}, 52 | "8": {"type": "int", "name": six.u("testInt64")}, 53 | # "16": {"type": "uint", "name": "testUInt32"}, 54 | "32": {"type": "uint", "name": six.u("testUInt64")}, 55 | # "64": {"type": "sint", "name": "testSInt32"}, 56 | "128": {"type": "sint", "name": six.u("testSInt64")}, 57 | "256": {"type": "fixed32", "name": six.u("testFixed32")}, 58 | "512": {"type": "fixed64", "name": six.u("testFixed64")}, 59 | "1024": {"type": "sfixed32", "name": six.u("testSFixed32")}, 60 | "2048": {"type": "sfixed64", "name": six.u("testSFixed64")}, 61 | # "4096": {"type": "int", "name": "testBool"}, 62 | "8192": {"type": "string", "name": six.u("testString")}, 63 | "16384": {"type": "bytes", "name": six.u("testBytes")}, 64 | # "32768": {"type": "message", "name": "testEmbed", 65 | # "message_typedef": { 66 | # "3": {"type": "double", "name": "embedDouble"}, 67 | # "2": {"type": "bytes", "name": "embedString"}} 68 | # }, 69 | # "65536": {"type": "packed_int", "name": "testRepeatedInt32"} 70 | } 71 | 72 | 73 | # Test decoding from blackboxprotobuf 74 | @given(x=strategies.gen_message_data(testMessage_typedef)) 75 | def test_decode(x): 76 | message = Test_pb2.TestMessage() 77 | for key, value in x.items(): 78 | setattr(message, key, value) 79 | encoded = message.SerializeToString() 80 | decoded, typedef = blackboxprotobuf.decode_message(encoded, testMessage_typedef) 81 | blackboxprotobuf.validate_typedef(typedef) 82 | hypothesis.note("Decoded: %r" % decoded) 83 | for key in decoded.keys(): 84 | assert x[key] == decoded[key] 85 | 86 | 87 | # Test encoding with blackboxprotobuf 88 | @given(x=strategies.gen_message_data(testMessage_typedef)) 89 | def test_encode(x): 90 | encoded = blackboxprotobuf.encode_message(x, testMessage_typedef) 91 | message = Test_pb2.TestMessage() 92 | message.ParseFromString(encoded) 93 | 94 | for key in x.keys(): 95 | assert getattr(message, key) == x[key] 96 | 97 | 98 | # Try to modify a random key with blackbox and re-encode 99 | # TODO: In the future do more random modifications, like swap the whole value 100 | @given( 101 | x=strategies.gen_message_data(testMessage_typedef), 102 | modify_num=st.sampled_from(sorted(testMessage_typedef.keys())), 103 | ) 104 | def test_modify(x, modify_num): 105 | modify_key = testMessage_typedef[modify_num]["name"] 106 | message = Test_pb2.TestMessage() 107 | for key, value in x.items(): 108 | setattr(message, key, value) 109 | encoded = message.SerializeToString() 110 | decoded, typedef = blackboxprotobuf.decode_message(encoded, testMessage_typedef) 111 | blackboxprotobuf.validate_typedef(typedef) 112 | 113 | # eliminate any cases where protobuf defaults out a field 114 | hypothesis.assume(modify_key in decoded) 115 | 116 | if isinstance(decoded[modify_key], six.text_type): 117 | mod_func = lambda x: six.u("test") 118 | elif isinstance(decoded[modify_key], bytes): 119 | mod_func = lambda x: b"test" 120 | elif isinstance(decoded[modify_key], six.integer_types): 121 | mod_func = lambda x: 10 122 | elif isinstance(decoded[modify_key], float): 123 | mod_func = lambda x: 10 124 | else: 125 | hypothesis.note( 126 | "Failed to modify key: %s (%r)" % (modify_key, type(decoded[modify_key])) 127 | ) 128 | assert False 129 | 130 | decoded[modify_key] = mod_func(decoded[modify_key]) 131 | x[modify_key] = mod_func(x[modify_key]) 132 | 133 | encoded = blackboxprotobuf.encode_message(decoded, testMessage_typedef) 134 | message = Test_pb2.TestMessage() 135 | message.ParseFromString(encoded) 136 | 137 | for key in decoded.keys(): 138 | assert getattr(message, key) == x[key] 139 | 140 | 141 | ## Second copies of the above methods that use the protobuf to/from json functions 142 | 143 | 144 | @given(x=strategies.gen_message_data(testMessage_typedef)) 145 | @example(x={"testBytes": b"test123"}) 146 | @example(x={"testBytes": b"\x80"}) 147 | def test_decode_json(x): 148 | # Test with JSON payload 149 | message = Test_pb2.TestMessage() 150 | for key, value in x.items(): 151 | setattr(message, key, value) 152 | encoded = message.SerializeToString() 153 | 154 | decoded_json, typedef_json = blackboxprotobuf.protobuf_to_json( 155 | encoded, testMessage_typedef 156 | ) 157 | blackboxprotobuf.validate_typedef(typedef_json) 158 | hypothesis.note("Encoded JSON:") 159 | hypothesis.note(decoded_json) 160 | decoded = json.loads(decoded_json) 161 | hypothesis.note("Original value:") 162 | hypothesis.note(x) 163 | hypothesis.note("Decoded valuec:") 164 | hypothesis.note(decoded) 165 | for key in decoded.keys(): 166 | if key == "testBytes": 167 | decoded[key] = six.ensure_binary(decoded[key], encoding="latin1") 168 | assert x[key] == decoded[key] 169 | 170 | 171 | @given(x=strategies.gen_message_data(testMessage_typedef)) 172 | @example(x={"testBytes": b"\x80"}) 173 | def test_encode_json(x): 174 | # Test with JSON payload 175 | if "testBytes" in x: 176 | x["testBytes"] = x["testBytes"].decode("latin1") 177 | json_str = json.dumps(x) 178 | 179 | hypothesis.note("JSON Str Input:") 180 | hypothesis.note(json_str) 181 | hypothesis.note(json.loads(json_str)) 182 | 183 | encoded = blackboxprotobuf.protobuf_from_json(json_str, testMessage_typedef) 184 | assert not isinstance(encoded, list) 185 | hypothesis.note("BBP decoding:") 186 | 187 | test_decode, _ = blackboxprotobuf.decode_message(encoded, testMessage_typedef) 188 | hypothesis.note(test_decode) 189 | 190 | message = Test_pb2.TestMessage() 191 | message.ParseFromString(encoded) 192 | hypothesis.note("Message:") 193 | hypothesis.note(message) 194 | 195 | for key in x.keys(): 196 | hypothesis.note("Message value") 197 | hypothesis.note(type(getattr(message, key))) 198 | hypothesis.note("Original value") 199 | hypothesis.note(type(x[key])) 200 | if key == "testBytes": 201 | x[key] = six.ensure_binary(x[key], encoding="latin1") 202 | assert getattr(message, key) == x[key] 203 | 204 | 205 | @given( 206 | x=strategies.gen_message_data(testMessage_typedef), 207 | modify_num=st.sampled_from(sorted(testMessage_typedef.keys())), 208 | ) 209 | def test_modify_json(x, modify_num): 210 | modify_key = testMessage_typedef[modify_num]["name"] 211 | message = Test_pb2.TestMessage() 212 | for key, value in x.items(): 213 | setattr(message, key, value) 214 | encoded = message.SerializeToString() 215 | decoded_json, typedef = blackboxprotobuf.protobuf_to_json( 216 | encoded, testMessage_typedef 217 | ) 218 | blackboxprotobuf.validate_typedef(typedef) 219 | decoded = json.loads(decoded_json) 220 | 221 | # eliminate any cases where protobuf defaults out a field 222 | hypothesis.assume(modify_key in decoded) 223 | 224 | if isinstance(decoded[modify_key], six.text_type): 225 | mod_func = lambda x: six.u("test") 226 | elif isinstance(decoded[modify_key], bytes): 227 | mod_func = lambda x: b"test" 228 | elif isinstance(decoded[modify_key], six.integer_types): 229 | mod_func = lambda x: 10 230 | elif isinstance(decoded[modify_key], float): 231 | mod_func = lambda x: 10 232 | else: 233 | hypothesis.note( 234 | "Failed to modify key: %s (%r)" % (modify_key, type(decoded[modify_key])) 235 | ) 236 | assert False 237 | 238 | decoded[modify_key] = mod_func(decoded[modify_key]) 239 | x[modify_key] = mod_func(x[modify_key]) 240 | 241 | encoded = blackboxprotobuf.protobuf_from_json( 242 | json.dumps(decoded), testMessage_typedef 243 | ) 244 | assert not isinstance(encoded, list) 245 | message = Test_pb2.TestMessage() 246 | message.ParseFromString(encoded) 247 | 248 | for key in decoded.keys(): 249 | hypothesis.note("Message value:") 250 | hypothesis.note(type(getattr(message, key))) 251 | hypothesis.note("Orig value:") 252 | hypothesis.note((x[key])) 253 | if key == "testBytes": 254 | x[key] = six.ensure_binary(x[key], encoding="latin1") 255 | assert getattr(message, key) == x[key] 256 | -------------------------------------------------------------------------------- /libs/blackboxprotobuf/tests/py_test/test_typedef.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018-2024 NCC Group Plc 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in all 11 | # copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | # SOFTWARE. 20 | 21 | from hypothesis import given, assume, note, example, reproduce_failure 22 | import hypothesis.strategies as st 23 | import collections 24 | import strategies 25 | import six 26 | import binascii 27 | 28 | from blackboxprotobuf.lib.config import Config 29 | from blackboxprotobuf.lib.types import length_delim 30 | from blackboxprotobuf.lib.types import type_maps 31 | from blackboxprotobuf.lib.typedef import TypeDef 32 | 33 | 34 | # Test for bug when alt typedef string is unicode/string 35 | def test_alt_typedef_unicode(): 36 | config = Config() 37 | 38 | typedef = { 39 | "1": {"type": "message", "message_typedef": {}, "alt_typedefs": {"1": "string"}} 40 | } 41 | 42 | message = {"1-1": "test"} 43 | 44 | data = length_delim.encode_message(message, config, TypeDef.from_dict(typedef)) 45 | length_delim.decode_message(data, config, TypeDef.from_dict(typedef)) 46 | 47 | # try unicode too 48 | typedef = { 49 | "1": {"type": "message", "message_typedef": {}, "alt_typedefs": {"1": "string"}} 50 | } 51 | data = length_delim.encode_message(message, config, TypeDef.from_dict(typedef)) 52 | length_delim.decode_message(data, config, TypeDef.from_dict(typedef)) 53 | 54 | 55 | def test_alt_field_id_unicode(): 56 | # Check for bug when field id is a str and not unicode in python2 57 | config = Config() 58 | 59 | typedef = { 60 | "1": {"type": "message", "message_typedef": {}, "alt_typedefs": {"1": "string"}} 61 | } 62 | 63 | message = {"1-1": "test"} 64 | 65 | data = length_delim.encode_message(message, config, TypeDef.from_dict(typedef)) 66 | length_delim.decode_message(data, config, TypeDef.from_dict(typedef)) 67 | 68 | # try unicode 69 | message = {"1-1": "test"} 70 | 71 | data = length_delim.encode_message(message, config, TypeDef.from_dict(typedef)) 72 | length_delim.decode_message(data, config, TypeDef.from_dict(typedef)) 73 | -------------------------------------------------------------------------------- /libs/blackboxprotobuf/tests/py_test/test_varint.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018-2024 NCC Group Plc 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in all 11 | # copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | # SOFTWARE. 20 | 21 | from hypothesis import given, example, note, assume 22 | import hypothesis.strategies as st 23 | import strategies 24 | import pytest 25 | import six 26 | 27 | from google.protobuf.internal import wire_format, encoder, decoder 28 | 29 | from blackboxprotobuf.lib.types import varint 30 | from blackboxprotobuf.lib.exceptions import EncoderException, DecoderException 31 | 32 | 33 | # Test that for any given bytes, we don't alter them when decoding as a varint 34 | @given(x=st.binary(max_size=10)) 35 | @example(x=b"\x80\x01") 36 | @example(x=b"\x80\x81") 37 | @example(x=b"\x81\x80\x80\x80\x80\x80\x80\x80\x01\x00") 38 | @example(x=b"\x80\x80\x80\x80\x80\x80\x80\x80\x81\x00") 39 | @example(x=b"\x80\x80\x80\x80\x80\x80\x80\x80\x81\x80") 40 | def test_varint_idem_uvarint(x): 41 | try: 42 | decoded, pos = varint.decode_uvarint(x, 0) 43 | except DecoderException: 44 | assume(True) 45 | return 46 | 47 | encoded = varint.encode_uvarint(decoded) 48 | assert encoded == x[:pos] 49 | 50 | 51 | # Test that for any given bytes, we don't alter them when decoding as a varint 52 | @given(x=st.binary(min_size=10, max_size=10)) 53 | @example(x=b"\x80\x01") 54 | @example(x=b"\x80\x81") 55 | @example(x=b"\x81\x80\x80\x80\x80\x80\x80\x80\x01\x00") 56 | @example(x=b"\x80\x80\x80\x80\x80\x80\x80\x80\x81\x00") 57 | @example(x=b"\x80\x80\x80\x80\x80\x80\x80\x80\x81\x80") 58 | @example(x=b"\x8d\x9b\xb0\xcc\xcf\xdc\xea\xf4\xf9\x02") 59 | def test_varint_idem_varint(x): 60 | try: 61 | decoded, pos = varint.decode_varint(x, 0) 62 | except DecoderException: 63 | assume(True) 64 | return 65 | encoded = varint.encode_varint(decoded) 66 | assert encoded == x[:pos] 67 | 68 | 69 | # Test that for any given bytes, we don't alter them when decoding as a varint 70 | @given(x=st.binary(max_size=10)) 71 | @example(x=b"\x80\x01") 72 | @example(x=b"\x80\x81") 73 | @example(x=b"\x81\x80\x80\x80\x80\x80\x80\x80\x01\x00") 74 | @example(x=b"\x80\x80\x80\x80\x80\x80\x80\x80\x81\x00") 75 | @example( 76 | x=b"\x80\x80\x80\x80\x80\x80\x80\x80\x81\x80" 77 | ) # I think this is overflowing and getting truncated on decode 78 | def test_varint_idem_svarint(x): 79 | try: 80 | decoded, pos = varint.decode_svarint(x, 0) 81 | except DecoderException: 82 | assume(True) 83 | return 84 | encoded = varint.encode_svarint(decoded) 85 | assert encoded == x[:pos] 86 | 87 | 88 | # Inverse checks. Ensure a value encoded by bbp decodes to the same value 89 | @given(x=strategies.input_map["uint"]) 90 | @example(x=18446744073709551615) 91 | def test_uvarint_inverse(x): 92 | encoded = varint.encode_uvarint(x) 93 | decoded, pos = varint.decode_uvarint(encoded, 0) 94 | assert pos == len(encoded) 95 | assert decoded == x 96 | 97 | 98 | @given(x=strategies.input_map["int"]) 99 | @example(x=-1143843382522404608) 100 | @example(x=-1) 101 | @example(x=8784740448578833805) 102 | def test_varint_inverse(x): 103 | encoded = varint.encode_varint(x) 104 | decoded, pos = varint.decode_varint(encoded, 0) 105 | assert pos == len(encoded) 106 | assert decoded == x 107 | 108 | 109 | @given(x=strategies.input_map["sint"]) 110 | def test_svarint_inverse(x): 111 | encoded = varint.encode_svarint(x) 112 | decoded, pos = varint.decode_svarint(encoded, 0) 113 | assert pos == len(encoded) 114 | assert decoded == x 115 | 116 | 117 | @given(x=st.integers(min_value=varint.MAX_UVARINT + 1)) 118 | def test_bounds_varints(x): 119 | with pytest.raises(EncoderException): 120 | varint.encode_uvarint(x) 121 | 122 | with pytest.raises(EncoderException): 123 | varint.encode_uvarint(-x) 124 | 125 | with pytest.raises(EncoderException): 126 | varint.encode_varint(x) 127 | 128 | with pytest.raises(EncoderException): 129 | varint.encode_varint(-x) 130 | 131 | with pytest.raises(EncoderException): 132 | varint.encode_svarint(x) 133 | 134 | with pytest.raises(EncoderException): 135 | varint.encode_svarint(-x) 136 | 137 | 138 | def _gen_append_bytearray(arr): 139 | def _append_bytearray(x): 140 | if isinstance(x, (str, int)): 141 | arr.append(x) 142 | elif isinstance(x, bytes): 143 | arr.extend(x) 144 | else: 145 | raise EncoderException("Unknown type returned by protobuf library") 146 | 147 | return _append_bytearray 148 | 149 | 150 | # Test our varint functions against google 151 | @given(x=strategies.input_map["uint"]) 152 | def test_uvarint_encode(x): 153 | encoded_google = bytearray() 154 | encoder._EncodeVarint(_gen_append_bytearray(encoded_google), x) 155 | encoded_bbpb = varint.encode_uvarint(x) 156 | assert encoded_google == encoded_bbpb 157 | 158 | 159 | @given(x=strategies.input_map["uint"]) 160 | def test_uvarint_decode(x): 161 | buf = bytearray() 162 | encoder._EncodeVarint(_gen_append_bytearray(buf), x) 163 | 164 | if six.PY2: 165 | buf = str(buf) 166 | decoded_google, _ = decoder._DecodeVarint(buf, 0) 167 | decoded_bbpb, _ = varint.decode_uvarint(buf, 0) 168 | assert decoded_google == decoded_bbpb 169 | 170 | 171 | @given(x=strategies.input_map["int"]) 172 | def test_varint_encode(x): 173 | encoded_google = bytearray() 174 | encoder._EncodeSignedVarint(_gen_append_bytearray(encoded_google), x) 175 | encoded_bbpb = varint.encode_varint(x) 176 | assert encoded_google == encoded_bbpb 177 | 178 | 179 | @given(x=strategies.input_map["int"]) 180 | def test_varint_decode(x): 181 | buf = bytearray() 182 | encoder._EncodeSignedVarint(_gen_append_bytearray(buf), x) 183 | 184 | if six.PY2: 185 | buf = bytes(buf) 186 | decoded_google, _ = decoder._DecodeSignedVarint(buf, 0) 187 | decoded_bbpb, _ = varint.decode_varint(buf, 0) 188 | assert decoded_google == decoded_bbpb 189 | 190 | 191 | @given(x=strategies.input_map["sint"]) 192 | def test_svarint_encode(x): 193 | encoded_google = bytearray() 194 | encoder._EncodeSignedVarint( 195 | _gen_append_bytearray(encoded_google), wire_format.ZigZagEncode(x) 196 | ) 197 | encoded_bbpb = varint.encode_svarint(x) 198 | assert encoded_google == encoded_bbpb 199 | 200 | 201 | @given(x=strategies.input_map["sint"]) 202 | @example(x=-1) 203 | def test_svarint_decode(x): 204 | buf = bytearray() 205 | encoder._EncodeSignedVarint(_gen_append_bytearray(buf), wire_format.ZigZagEncode(x)) 206 | 207 | if six.PY2: 208 | buf = bytes(buf) 209 | decoded_google_uint, _ = decoder._DecodeVarint(buf, 0) 210 | decoded_google = wire_format.ZigZagDecode(decoded_google_uint) 211 | decoded_bbpb, _ = varint.decode_svarint(buf, 0) 212 | 213 | assert decoded_google == decoded_bbpb 214 | -------------------------------------------------------------------------------- /libs/blackboxprotobuf/tests/requirements-python2-dev.txt: -------------------------------------------------------------------------------- 1 | protobuf==3.17.3 2 | six==1.16 3 | pytest==4.6.11 4 | more_itertools>=5.0 5 | hypothesis==4.57.1 6 | -------------------------------------------------------------------------------- /libs/blackboxprotobuf/tests/run_decoder.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright (c) 2018-2024 NCC Group Plc 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in all 13 | # copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | 23 | import sys 24 | 25 | sys.path.insert(0, "../") 26 | 27 | import blackboxprotobuf as bbp 28 | 29 | typedef = {} 30 | 31 | # Take a protobuf binary from stdin and decode it to JSON 32 | protobuf = sys.stdin.read() 33 | json, typedef = bbp.protobuf_to_json(protobuf, typedef) 34 | print(json) 35 | print(typedef) 36 | -------------------------------------------------------------------------------- /libs/six/.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | branches: 9 | - main 10 | 11 | jobs: 12 | test: 13 | strategy: 14 | matrix: 15 | python-version: [ 16 | "2.7", 17 | "3.5", 18 | "3.6", 19 | "3.7", 20 | "3.8", 21 | "3.9", 22 | "3.10", 23 | "3.11", 24 | "3.12", 25 | "3.13", 26 | "pypy-2.7", 27 | "pypy-3.8", 28 | ] 29 | os: [ubuntu-latest, windows-latest, macos-latest] 30 | exclude: 31 | - python-version: "2.7" 32 | os: "ubuntu-latest" 33 | - python-version: "2.7" 34 | os: "windows-latest" 35 | - python-version: "2.7" 36 | os: "macos-latest" 37 | - python-version: "3.5" 38 | os: "macos-latest" 39 | - python-version: "3.6" 40 | os: "macos-latest" 41 | - python-version: "3.7" 42 | os: "macos-latest" 43 | - python-version: "3.5" 44 | os: "ubuntu-latest" 45 | - python-version: "3.6" 46 | os: "ubuntu-latest" 47 | include: 48 | - python-version: "3.5" 49 | os: "macos-13" 50 | - python-version: "3.6" 51 | os: "macos-13" 52 | - python-version: "3.7" 53 | os: "macos-13" 54 | - python-version: "2.7" 55 | os: "ubuntu-20.04" 56 | - python-version: "3.5" 57 | os: "ubuntu-20.04" 58 | - python-version: "3.6" 59 | os: "ubuntu-20.04" 60 | runs-on: ${{ matrix.os }} 61 | env: 62 | TOXENV: py 63 | steps: 64 | - uses: actions/checkout@v4 65 | - if: ${{ matrix.python-version == '2.7' }} 66 | run: | 67 | sudo apt-get install python-is-python2 68 | curl -sSL https://bootstrap.pypa.io/pip/2.7/get-pip.py -o get-pip.py 69 | python get-pip.py 70 | name: Set up Python ${{ matrix.python-version }} on ${{ matrix.os }} 71 | - if: ${{ matrix.python-version != '2.7' }} 72 | name: Set up Python ${{ matrix.python-version }} on ${{ matrix.os }} 73 | uses: actions/setup-python@v5 74 | with: 75 | python-version: ${{ matrix.python-version }} 76 | allow-prereleases: true 77 | env: 78 | PIP_TRUSTED_HOST: ${{ contains(fromJson('["3.5"]'), matrix.python-version) && 'pypi.python.org pypi.org files.pythonhosted.org' || '' }} 79 | - name: Install dependencies 80 | run: python -m pip install -U tox 81 | - name: Run tox 82 | run: python -m tox 83 | -------------------------------------------------------------------------------- /libs/six/.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Upload package 2 | 3 | on: 4 | push: 5 | tags: 6 | - '*' 7 | workflow_dispatch: 8 | 9 | jobs: 10 | deploy: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v4 14 | - name: Set up Python 15 | uses: actions/setup-python@v5 16 | with: 17 | python-version: '3.13' 18 | - name: Install dependencies 19 | run: | 20 | python -m pip install --upgrade pip 21 | pip install -U build twine 22 | - name: Build package 23 | run: | 24 | python -m build 25 | - name: Publish package 26 | env: 27 | TWINE_USERNAME: "__token__" 28 | run: | 29 | if [[ "$GITHUB_EVENT_NAME" == "workflow_dispatch" ]]; then 30 | export TWINE_REPOSITORY="testpypi" 31 | export TWINE_PASSWORD="${{ secrets.TEST_PYPI_UPLOAD_TOKEN }}" 32 | elif [[ "$GITHUB_EVENT_NAME" == "push" ]]; then 33 | export TWINE_REPOSITORY="pypi" 34 | export TWINE_PASSWORD="${{ secrets.PYPI_UPLOAD_TOKEN }}" 35 | else 36 | echo "Unknown event name: ${GITHUB_EVENT_NAME}" 37 | exit 1 38 | fi 39 | python -m twine upload dist/* 40 | -------------------------------------------------------------------------------- /libs/six/.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | build 3 | dist 4 | MANIFEST 5 | documentation/_build 6 | .tox 7 | six.egg-info 8 | -------------------------------------------------------------------------------- /libs/six/CHANGES: -------------------------------------------------------------------------------- 1 | Changelog for six 2 | ================= 3 | 4 | This file lists the changes in each six version. 5 | 6 | 1.17.0 7 | ------ 8 | 9 | - Pull request #388: Remove `URLopener` and `FancyURLopener` classes from 10 | `urllib.request` when running on Python 3.14 or greater. 11 | 12 | - Pull request #365, issue #283: `six.moves.UserDict` now points to 13 | `UserDict.IterableUserDict` instead of `UserDict.UserDict` on Python 2. 14 | 15 | 1.16.0 16 | ------ 17 | 18 | - Pull request #343, issue #341, pull request #349: Port _SixMetaPathImporter to 19 | Python 3.10. 20 | 21 | 1.15.0 22 | ------ 23 | 24 | - Pull request #331: Optimize `six.ensure_str` and `six.ensure_binary`. 25 | 26 | 1.14.0 27 | ------ 28 | 29 | - Issue #288, pull request #289: Add `six.assertNotRegex`. 30 | 31 | - Issue #317: `six.moves._dummy_thread` now points to the `_thread` module on 32 | Python 3.9+. Python 3.7 and later requires threading and deprecated the 33 | `_dummy_thread` module. 34 | 35 | - Issue #308, pull request #314: Remove support for Python 2.6 and Python 3.2. 36 | 37 | - Issue #250, issue #165, pull request #251: `six.wraps` now ignores missing 38 | attributes. This follows the Python 3.2+ standard library behavior. 39 | 40 | 1.13.0 41 | ------ 42 | 43 | - Issue #298, pull request #299: Add `six.moves.dbm_ndbm`. 44 | 45 | - Issue #155: Add `six.moves.collections_abc`, which aliases the `collections` 46 | module on Python 2-3.2 and the `collections.abc` on Python 3.3 and greater. 47 | 48 | - Pull request #304: Re-add distutils fallback in `setup.py`. 49 | 50 | - Pull request #305: On Python 3.7, `with_metaclass` supports classes using PEP 51 | 560 features. 52 | 53 | 1.12.0 54 | ------ 55 | 56 | - Issue #259, pull request #260: `six.add_metaclass` now preserves 57 | `__qualname__` from the original class. 58 | 59 | - Pull request #204: Add `six.ensure_binary`, `six.ensure_text`, and 60 | `six.ensure_str`. 61 | 62 | 1.11.0 63 | ------ 64 | 65 | - Pull request #178: `with_metaclass` now properly proxies `__prepare__` to the 66 | underlying metaclass. 67 | 68 | - Pull request #191: Allow `with_metaclass` to work with metaclasses implemented 69 | in C. 70 | 71 | - Pull request #203: Add parse_http_list and parse_keqv_list to moved 72 | urllib.request. 73 | 74 | - Pull request #172 and issue #171: Add unquote_to_bytes to moved urllib.parse. 75 | 76 | - Pull request #167: Add `six.moves.getoutput`. 77 | 78 | - Pull request #80: Add `six.moves.urllib_parse.splitvalue`. 79 | 80 | - Pull request #75: Add `six.moves.email_mime_image`. 81 | 82 | - Pull request #72: Avoid creating reference cycles through tracebacks in 83 | `reraise`. 84 | 85 | 1.10.0 86 | ------ 87 | 88 | - Issue #122: Improve the performance of `six.int2byte` on Python 3. 89 | 90 | - Pull request #55 and issue #99: Don't add the `winreg` module to `six.moves` 91 | on non-Windows platforms. 92 | 93 | - Pull request #60 and issue #108: Add `six.moves.getcwd` and 94 | `six.moves.getcwdu`. 95 | 96 | - Pull request #64: Add `create_unbound_method` to create unbound methods. 97 | 98 | 1.9.0 99 | ----- 100 | 101 | - Issue #106: Support the `flush` parameter to `six.print_`. 102 | 103 | - Pull request #48 and issue #15: Add the `python_2_unicode_compatible` 104 | decorator. 105 | 106 | - Pull request #57 and issue #50: Add several compatibility methods for unittest 107 | assertions that were renamed between Python 2 and 3. 108 | 109 | - Issue #105 and pull request #58: Ensure `six.wraps` respects the *updated* and 110 | *assigned* arguments. 111 | 112 | - Issue #102: Add `raise_from` to abstract out Python 3's raise from syntax. 113 | 114 | - Issue #97: Optimize `six.iterbytes` on Python 2. 115 | 116 | - Issue #98: Fix `six.moves` race condition in multi-threaded code. 117 | 118 | - Pull request #51: Add `six.view(keys|values|items)`, which provide dictionary 119 | views on Python 2.7+. 120 | 121 | - Issue #112: `six.moves.reload_module` now uses the importlib module on 122 | Python 3.4+. 123 | 124 | 1.8.0 125 | ----- 126 | 127 | - Issue #90: Add `six.moves.shlex_quote`. 128 | 129 | - Issue #59: Add `six.moves.intern`. 130 | 131 | - Add `six.urllib.parse.uses_(fragment|netloc|params|query|relative)`. 132 | 133 | - Issue #88: Fix add_metaclass when the class has `__slots__` containing 134 | `__weakref__` or `__dict__`. 135 | 136 | - Issue #89: Make six use absolute imports. 137 | 138 | - Issue #85: Always accept *updated* and *assigned* arguments for `wraps()`. 139 | 140 | - Issue #86: In `reraise()`, instantiate the exception if the second argument is 141 | `None`. 142 | 143 | - Pull request #45: Add `six.moves.email_mime_nonmultipart`. 144 | 145 | - Issue #81: Add `six.urllib.request.splittag` mapping. 146 | 147 | - Issue #80: Add `six.urllib.request.splituser` mapping. 148 | 149 | 1.7.3 150 | ----- 151 | 152 | - Issue #77: Fix import six on Python 3.4 with a custom loader. 153 | 154 | - Issue #74: `six.moves.xmlrpc_server` should map to `SimpleXMLRPCServer` on Python 155 | 2 as documented not `xmlrpclib`. 156 | 157 | 1.7.2 158 | ----- 159 | 160 | - Issue #72: Fix installing on Python 2. 161 | 162 | 1.7.1 163 | ----- 164 | 165 | - Issue #71: Make the six.moves meta path importer handle reloading of the six 166 | module gracefully. 167 | 168 | 1.7.0 169 | ----- 170 | 171 | - Pull request #30: Implement six.moves with a PEP 302 meta path hook. 172 | 173 | - Pull request #32: Add six.wraps, which is like functools.wraps but always sets 174 | the __wrapped__ attribute. 175 | 176 | - Pull request #35: Improve add_metaclass, so that it doesn't end up inserting 177 | another class into the hierarchy. 178 | 179 | - Pull request #34: Add import mappings for dummy_thread. 180 | 181 | - Pull request #33: Add import mappings for UserDict and UserList. 182 | 183 | - Pull request #31: Select the implementations of dictionary iterator routines 184 | at import time for a 20% speed boost. 185 | 186 | 1.6.1 187 | ----- 188 | 189 | - Raise an AttributeError for six.moves.X when X is a module not available in 190 | the current interpreter. 191 | 192 | 1.6.0 193 | ----- 194 | 195 | - Raise an AttributeError for every attribute of unimportable modules. 196 | 197 | - Issue #56: Make the fake modules six.moves puts into sys.modules appear not to 198 | have a __path__ unless they are loaded. 199 | 200 | - Pull request #28: Add support for SplitResult. 201 | 202 | - Issue #55: Add move mapping for xmlrpc.server. 203 | 204 | - Pull request #29: Add move for urllib.parse.splitquery. 205 | 206 | 1.5.2 207 | ----- 208 | 209 | - Issue #53: Make the fake modules six.moves puts into sys.modules appear not to 210 | have a __name__ unless they are loaded. 211 | 212 | 1.5.1 213 | ----- 214 | 215 | - Issue #51: Hack around the Django autoreloader after recent six.moves changes. 216 | 217 | 1.5.0 218 | ----- 219 | 220 | - Removed support for Python 2.4. This is because py.test no longer supports 221 | 2.4. 222 | 223 | - Fix various import problems including issues #19 and #41. six.moves modules 224 | are now lazy wrappers over the underlying modules instead of the actual 225 | modules themselves. 226 | 227 | - Issue #49: Add six.moves mapping for tkinter.ttk. 228 | 229 | - Pull request #24: Add __dir__ special method to six.moves modules. 230 | 231 | - Issue #47: Fix add_metaclass on classes with a string for the __slots__ 232 | variable. 233 | 234 | - Issue #44: Fix interpretation of backslashes on Python 2 in the u() function. 235 | 236 | - Pull request #21: Add import mapping for urllib's proxy_bypass function. 237 | 238 | - Issue #43: Add import mapping for the Python 2 xmlrpclib module. 239 | 240 | - Issue #39: Add import mapping for the Python 2 thread module. 241 | 242 | - Issue #40: Add import mapping for the Python 2 gdbm module. 243 | 244 | - Issue #35: On Python versions less than 2.7, print_ now encodes unicode 245 | strings when outputting to standard streams. (Python 2.7 handles this 246 | automatically.) 247 | 248 | 1.4.1 249 | ----- 250 | 251 | - Issue #32: urllib module wrappings don't work when six is not a toplevel file. 252 | 253 | 1.4.0 254 | ----- 255 | 256 | - Issue #31: Add six.moves mapping for UserString. 257 | 258 | - Pull request #12: Add six.add_metaclass, a decorator for adding a metaclass to 259 | a class. 260 | 261 | - Add six.moves.zip_longest and six.moves.filterfalse, which correspond 262 | respectively to itertools.izip_longest and itertools.ifilterfalse on Python 2 263 | and itertools.zip_longest and itertools.filterfalse on Python 3. 264 | 265 | - Issue #25: Add the unichr function, which returns a string for a Unicode 266 | codepoint. 267 | 268 | - Issue #26: Add byte2int function, which complements int2byte. 269 | 270 | - Add a PY2 constant with obvious semantics. 271 | 272 | - Add helpers for indexing and iterating over bytes: iterbytes and indexbytes. 273 | 274 | - Add create_bound_method() wrapper. 275 | 276 | - Issue #23: Allow multiple base classes to be passed to with_metaclass. 277 | 278 | - Issue #24: Add six.moves.range alias. This exactly the same as the current 279 | xrange alias. 280 | 281 | - Pull request #5: Create six.moves.urllib, which contains abstractions for a 282 | bunch of things which are in urllib in Python 3 and spread out across urllib, 283 | urllib2, and urlparse in Python 2. 284 | 285 | 1.3.0 286 | ----- 287 | 288 | - Issue #21: Add methods to access the closure and globals of a function. 289 | 290 | - In six.iter(items/keys/values/lists), passed keyword arguments through to the 291 | underlying method. 292 | 293 | - Add six.iterlists(). 294 | 295 | - Issue #20: Fix tests if tkinter is not available. 296 | 297 | - Issue #17: Define callable to be builtin callable when it is available again 298 | in Python 3.2+. 299 | 300 | - Issue #16: Rename Python 2 exec_'s arguments, so casually calling exec_ with 301 | keyword arguments will raise. 302 | 303 | - Issue #14: Put the six.moves package in sys.modules based on the name six is 304 | imported under. 305 | 306 | - Fix Jython detection. 307 | 308 | - Pull request #4: Add email_mime_multipart, email_mime_text, and 309 | email_mime_base to six.moves. 310 | 311 | 1.2.0 312 | ----- 313 | 314 | - Issue #13: Make iterkeys/itervalues/iteritems return iterators on Python 3 315 | instead of iterables. 316 | 317 | - Issue #11: Fix maxsize support on Jython. 318 | 319 | - Add six.next() as an alias for six.advance_iterator(). 320 | 321 | - Use the builtin next() function for advance_iterator() where is available 322 | (2.6+), not just Python 3. 323 | 324 | - Add the Iterator class for writing portable iterators. 325 | 326 | 1.1.0 327 | ----- 328 | 329 | - Add the int2byte function. 330 | 331 | - Add compatibility mappings for iterators over the keys, values, and items of a 332 | dictionary. 333 | 334 | - Fix six.MAXSIZE on platforms where sizeof(long) != sizeof(Py_ssize_t). 335 | 336 | - Issue #3: Add six.moves mappings for filter, map, and zip. 337 | 338 | 1.0.0 339 | ----- 340 | 341 | - Issue #2: u() on Python 2.x now resolves unicode escapes. 342 | 343 | - Expose an API for adding mappings to six.moves. 344 | 345 | 1.0 beta 1 346 | ---------- 347 | 348 | - Reworked six into one .py file. This breaks imports. Please tell me if you 349 | are interested in an import compatibility layer. 350 | -------------------------------------------------------------------------------- /libs/six/CONTRIBUTORS: -------------------------------------------------------------------------------- 1 | The primary author and maintainer of six is Benjamin Peterson. He would like to 2 | acknowledge the following people who submitted bug reports, pull requests, and 3 | otherwise worked to improve six: 4 | 5 | Marc Abramowitz 6 | immerrr again 7 | Alexander Artemenko 8 | Aymeric Augustin 9 | Lee Ball 10 | Ben Bariteau 11 | Ned Batchelder 12 | Wouter Bolsterlee 13 | Brett Cannon 14 | Jason R. Coombs 15 | Julien Danjou 16 | Ben Darnell 17 | Ben Davis 18 | Jon Dufresne 19 | Tim Graham 20 | Thomas Grainger 21 | Max Grender-Jones 22 | Pierre Grimaud 23 | Joshua Harlow 24 | Toshiki Kataoka 25 | Hugo van Kemenade 26 | Anselm Kruis 27 | Ivan Levkivskyi 28 | Alexander Lukanin 29 | James Mills 30 | Jordan Moldow 31 | Berker Peksag 32 | Sridhar Ratnakumar 33 | Erik Rose 34 | Mirko Rossini 35 | Peter Ruibal 36 | Miroslav Shubernetskiy 37 | Eli Schwartz 38 | Bart Skowron 39 | Anthony Sottile 40 | Victor Stinner 41 | Jonathan Vanasco 42 | Lucas Wiman 43 | Jingxin Zhu 44 | 45 | If you think you belong on this list, please let me know! --Benjamin 46 | -------------------------------------------------------------------------------- /libs/six/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2010-2024 Benjamin Peterson 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of 4 | this software and associated documentation files (the "Software"), to deal in 5 | the Software without restriction, including without limitation the rights to 6 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 7 | the Software, and to permit persons to whom the Software is furnished to do so, 8 | subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 15 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 16 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 17 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 18 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 19 | -------------------------------------------------------------------------------- /libs/six/MANIFEST.in: -------------------------------------------------------------------------------- 1 | include CHANGES 2 | include LICENSE 3 | include test_six.py 4 | 5 | recursive-include documentation * 6 | prune documentation/_build 7 | -------------------------------------------------------------------------------- /libs/six/README.rst: -------------------------------------------------------------------------------- 1 | .. image:: https://img.shields.io/pypi/v/six.svg 2 | :target: https://pypi.org/project/six/ 3 | :alt: six on PyPI 4 | 5 | .. image:: https://readthedocs.org/projects/six/badge/?version=latest 6 | :target: https://six.readthedocs.io/ 7 | :alt: six's documentation on Read the Docs 8 | 9 | .. image:: https://img.shields.io/badge/license-MIT-green.svg 10 | :target: https://github.com/benjaminp/six/blob/master/LICENSE 11 | :alt: MIT License badge 12 | 13 | Six is a Python 2 and 3 compatibility library. It provides utility functions 14 | for smoothing over the differences between the Python versions with the goal of 15 | writing Python code that is compatible on both Python versions. See the 16 | documentation for more information on what is provided. 17 | 18 | Six supports Python 2.7 and 3.3+. It is contained in only one Python 19 | file, so it can be easily copied into your project. (The copyright and license 20 | notice must be retained.) 21 | 22 | Online documentation is at https://six.readthedocs.io/. 23 | 24 | Bugs can be reported to https://github.com/benjaminp/six. The code can also 25 | be found there. 26 | -------------------------------------------------------------------------------- /libs/six/documentation/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 14 | 15 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest 16 | 17 | help: 18 | @echo "Please use \`make ' where is one of" 19 | @echo " html to make standalone HTML files" 20 | @echo " dirhtml to make HTML files named index.html in directories" 21 | @echo " singlehtml to make a single large HTML file" 22 | @echo " pickle to make pickle files" 23 | @echo " json to make JSON files" 24 | @echo " htmlhelp to make HTML files and a HTML help project" 25 | @echo " qthelp to make HTML files and a qthelp project" 26 | @echo " devhelp to make HTML files and a Devhelp project" 27 | @echo " epub to make an epub" 28 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 29 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 30 | @echo " text to make text files" 31 | @echo " man to make manual pages" 32 | @echo " changes to make an overview of all changed/added/deprecated items" 33 | @echo " linkcheck to check all external links for integrity" 34 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 35 | 36 | clean: 37 | -rm -rf $(BUILDDIR)/* 38 | 39 | html: 40 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 41 | @echo 42 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 43 | 44 | dirhtml: 45 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 46 | @echo 47 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 48 | 49 | singlehtml: 50 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 51 | @echo 52 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 53 | 54 | pickle: 55 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 56 | @echo 57 | @echo "Build finished; now you can process the pickle files." 58 | 59 | json: 60 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 61 | @echo 62 | @echo "Build finished; now you can process the JSON files." 63 | 64 | htmlhelp: 65 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 66 | @echo 67 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 68 | ".hhp project file in $(BUILDDIR)/htmlhelp." 69 | 70 | qthelp: 71 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 72 | @echo 73 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 74 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 75 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/six.qhcp" 76 | @echo "To view the help file:" 77 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/six.qhc" 78 | 79 | devhelp: 80 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 81 | @echo 82 | @echo "Build finished." 83 | @echo "To view the help file:" 84 | @echo "# mkdir -p $$HOME/.local/share/devhelp/six" 85 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/six" 86 | @echo "# devhelp" 87 | 88 | epub: 89 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 90 | @echo 91 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 92 | 93 | latex: 94 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 95 | @echo 96 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 97 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 98 | "(use \`make latexpdf' here to do that automatically)." 99 | 100 | latexpdf: 101 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 102 | @echo "Running LaTeX files through pdflatex..." 103 | make -C $(BUILDDIR)/latex all-pdf 104 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 105 | 106 | text: 107 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 108 | @echo 109 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 110 | 111 | man: 112 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 113 | @echo 114 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 115 | 116 | changes: 117 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 118 | @echo 119 | @echo "The overview file is in $(BUILDDIR)/changes." 120 | 121 | linkcheck: 122 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 123 | @echo 124 | @echo "Link check complete; look for any errors in the above output " \ 125 | "or in $(BUILDDIR)/linkcheck/output.txt." 126 | 127 | doctest: 128 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 129 | @echo "Testing of doctests in the sources finished, look at the " \ 130 | "results in $(BUILDDIR)/doctest/output.txt." 131 | -------------------------------------------------------------------------------- /libs/six/documentation/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # six documentation build configuration file 4 | 5 | import os 6 | import sys 7 | 8 | # If extensions (or modules to document with autodoc) are in another directory, 9 | # add these directories to sys.path here. If the directory is relative to the 10 | # documentation root, use os.path.abspath to make it absolute, like shown here. 11 | #sys.path.append(os.path.abspath('.')) 12 | 13 | # -- General configuration ----------------------------------------------------- 14 | 15 | # If your documentation needs a minimal Sphinx version, state it here. 16 | needs_sphinx = "1.0" 17 | 18 | # Add any Sphinx extension module names here, as strings. They can be extensions 19 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 20 | extensions = ["sphinx.ext.intersphinx"] 21 | 22 | # Add any paths that contain templates here, relative to this directory. 23 | templates_path = ["_templates"] 24 | 25 | # The suffix of source filenames. 26 | source_suffix = ".rst" 27 | 28 | # The encoding of source files. 29 | #source_encoding = "utf-8-sig" 30 | 31 | # The master toctree document. 32 | master_doc = "index" 33 | 34 | # General information about the project. 35 | project = u"six" 36 | copyright = u"2010-2024, Benjamin Peterson" 37 | 38 | sys.path.append(os.path.abspath(os.path.join(".", ".."))) 39 | from six import __version__ as six_version 40 | sys.path.pop() 41 | 42 | # The version info for the project you're documenting, acts as replacement for 43 | # |version| and |release|, also used in various other places throughout the 44 | # built documents. 45 | # 46 | # The short X.Y version. 47 | version = six_version[:-2] 48 | # The full version, including alpha/beta/rc tags. 49 | release = six_version 50 | 51 | # The language for content autogenerated by Sphinx. Refer to documentation 52 | # for a list of supported languages. 53 | #language = None 54 | 55 | # There are two options for replacing |today|: either, you set today to some 56 | # non-false value, then it is used: 57 | #today = '' 58 | # Else, today_fmt is used as the format for a strftime call. 59 | #today_fmt = '%B %d, %Y' 60 | 61 | # List of patterns, relative to source directory, that match files and 62 | # directories to ignore when looking for source files. 63 | exclude_patterns = ["_build"] 64 | 65 | # The reST default role (used for this markup: `text`) to use for all documents. 66 | #default_role = None 67 | 68 | # If true, '()' will be appended to :func: etc. cross-reference text. 69 | #add_function_parentheses = True 70 | 71 | # If true, the current module name will be prepended to all description 72 | # unit titles (such as .. function::). 73 | #add_module_names = True 74 | 75 | # If true, sectionauthor and moduleauthor directives will be shown in the 76 | # output. They are ignored by default. 77 | #show_authors = False 78 | 79 | # The name of the Pygments (syntax highlighting) style to use. 80 | pygments_style = "sphinx" 81 | 82 | # A list of ignored prefixes for module index sorting. 83 | #modindex_common_prefix = [] 84 | 85 | 86 | # -- Options for HTML output --------------------------------------------------- 87 | 88 | # The theme to use for HTML and HTML Help pages. See the documentation for 89 | # a list of builtin themes. 90 | html_theme = "default" 91 | 92 | # Theme options are theme-specific and customize the look and feel of a theme 93 | # further. For a list of options available for each theme, see the 94 | # documentation. 95 | #html_theme_options = {} 96 | 97 | # Add any paths that contain custom themes here, relative to this directory. 98 | #html_theme_path = [] 99 | 100 | # The name for this set of Sphinx documents. If None, it defaults to 101 | # " v documentation". 102 | #html_title = None 103 | 104 | # A shorter title for the navigation bar. Default is the same as html_title. 105 | #html_short_title = None 106 | 107 | # The name of an image file (relative to this directory) to place at the top 108 | # of the sidebar. 109 | #html_logo = None 110 | 111 | # The name of an image file (within the static path) to use as favicon of the 112 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 113 | # pixels large. 114 | #html_favicon = None 115 | 116 | # Add any paths that contain custom static files (such as style sheets) here, 117 | # relative to this directory. They are copied after the builtin static files, 118 | # so a file named "default.css" will overwrite the builtin "default.css". 119 | html_static_path = ["_static"] 120 | 121 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 122 | # using the given strftime format. 123 | #html_last_updated_fmt = '%b %d, %Y' 124 | 125 | # If true, SmartyPants will be used to convert quotes and dashes to 126 | # typographically correct entities. 127 | #html_use_smartypants = True 128 | 129 | # Custom sidebar templates, maps document names to template names. 130 | #html_sidebars = {} 131 | 132 | # Additional templates that should be rendered to pages, maps page names to 133 | # template names. 134 | #html_additional_pages = {} 135 | 136 | # If false, no module index is generated. 137 | #html_domain_indices = True 138 | 139 | # If false, no index is generated. 140 | #html_use_index = True 141 | 142 | # If true, the index is split into individual pages for each letter. 143 | #html_split_index = False 144 | 145 | # If true, links to the reST sources are added to the pages. 146 | #html_show_sourcelink = True 147 | 148 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 149 | #html_show_sphinx = True 150 | 151 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 152 | #html_show_copyright = True 153 | 154 | # If true, an OpenSearch description file will be output, and all pages will 155 | # contain a tag referring to it. The value of this option must be the 156 | # base URL from which the finished HTML is served. 157 | #html_use_opensearch = '' 158 | 159 | # If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). 160 | #html_file_suffix = '' 161 | 162 | # Output file base name for HTML help builder. 163 | htmlhelp_basename = 'sixdoc' 164 | 165 | 166 | # -- Options for LaTeX output -------------------------------------------------- 167 | 168 | # The paper size ('letter' or 'a4'). 169 | #latex_paper_size = 'letter' 170 | 171 | # The font size ('10pt', '11pt' or '12pt'). 172 | #latex_font_size = '10pt' 173 | 174 | # Grouping the document tree into LaTeX files. List of tuples 175 | # (source start file, target name, title, author, documentclass [howto/manual]). 176 | latex_documents = [ 177 | ("index", "six.tex", u"six Documentation", 178 | u"Benjamin Peterson", "manual"), 179 | ] 180 | 181 | # The name of an image file (relative to this directory) to place at the top of 182 | # the title page. 183 | #latex_logo = None 184 | 185 | # For "manual" documents, if this is true, then toplevel headings are parts, 186 | # not chapters. 187 | #latex_use_parts = False 188 | 189 | # If true, show page references after internal links. 190 | #latex_show_pagerefs = False 191 | 192 | # If true, show URL addresses after external links. 193 | #latex_show_urls = False 194 | 195 | # Additional stuff for the LaTeX preamble. 196 | #latex_preamble = '' 197 | 198 | # Documents to append as an appendix to all manuals. 199 | #latex_appendices = [] 200 | 201 | # If false, no module index is generated. 202 | #latex_domain_indices = True 203 | 204 | 205 | # -- Options for manual page output -------------------------------------------- 206 | 207 | # One entry per manual page. List of tuples 208 | # (source start file, name, description, authors, manual section). 209 | man_pages = [ 210 | ("index", "six", u"six Documentation", 211 | [u"Benjamin Peterson"], 1) 212 | ] 213 | 214 | # -- Intersphinx --------------------------------------------------------------- 215 | 216 | intersphinx_mapping = {"py2" : ("https://docs.python.org/2/", None), 217 | "py3" : ("https://docs.python.org/3/", None)} 218 | -------------------------------------------------------------------------------- /libs/six/setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | universal = 1 3 | 4 | [flake8] 5 | max-line-length = 100 6 | ignore = F821 7 | 8 | [metadata] 9 | license_files = LICENSE 10 | 11 | [tool:pytest] 12 | minversion=2.2.0 13 | -------------------------------------------------------------------------------- /libs/six/setup.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2010-2024 Benjamin Peterson 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in all 11 | # copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | # SOFTWARE. 20 | 21 | from __future__ import with_statement 22 | 23 | # Six is a dependency of setuptools, so using setuptools creates a 24 | # circular dependency when building a Python stack from source. We 25 | # therefore allow falling back to distutils to install six. 26 | try: 27 | from setuptools import setup 28 | except ImportError: 29 | from distutils.core import setup 30 | 31 | import six 32 | 33 | six_classifiers = [ 34 | "Development Status :: 5 - Production/Stable", 35 | "Programming Language :: Python :: 2", 36 | "Programming Language :: Python :: 3", 37 | "Intended Audience :: Developers", 38 | "License :: OSI Approved :: MIT License", 39 | "Topic :: Software Development :: Libraries", 40 | "Topic :: Utilities", 41 | ] 42 | 43 | with open("README.rst", "r") as fp: 44 | six_long_description = fp.read() 45 | 46 | setup(name="six", 47 | version=six.__version__, 48 | author="Benjamin Peterson", 49 | author_email="benjamin@python.org", 50 | url="https://github.com/benjaminp/six", 51 | tests_require=["pytest"], 52 | py_modules=["six"], 53 | description="Python 2 and 3 compatibility utilities", 54 | long_description=six_long_description, 55 | license="MIT", 56 | classifiers=six_classifiers, 57 | python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*", 58 | ) 59 | -------------------------------------------------------------------------------- /libs/six/tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist=py27,py33,py34,py35,py36,py37,py38,pypy,flake8 3 | 4 | [testenv] 5 | deps= pytest 6 | commands= python -m pytest -rfsxX {posargs} 7 | 8 | [testenv:flake8] 9 | basepython=python 10 | deps=flake8 11 | commands= flake8 six.py 12 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | EditorConfig==0.12.3 2 | jsbeautifier==1.14.8 3 | six==1.16.0 4 | texttable==1.6.7 5 | wcwidth==0.2.6 6 | --------------------------------------------------------------------------------