├── tests ├── __init__.py ├── marine │ ├── __init__.py │ ├── benchmark │ │ ├── __init__.py │ │ ├── utils.py │ │ ├── benchmark_generator.py │ │ ├── conversation_generators.py │ │ └── main.py │ ├── run_benchmark.sh │ ├── test_marine_pool.py │ └── test_marine.py ├── conftest.py └── fixtures │ ├── __init__.py │ └── marine │ ├── __init__.py │ └── marine_fixtures.py ├── scripts ├── wheel_abi.patch ├── expose_auditwheel.sh ├── patch_auditwheel_recursive_dependency_bug.sh └── modify_auditwheel_policy.py ├── .dockerignore ├── setup.cfg ├── marine ├── encap_consts.py ├── exceptions.py ├── __init__.py ├── marine_prefs.py ├── marine_pool.py ├── simple_marine.py └── marine.py ├── tox.ini ├── setup.py ├── .github └── workflows │ └── test.yml ├── Dockerfile ├── .gitignore ├── README.md └── LICENSE /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/marine/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/marine/benchmark/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | from .fixtures import * 2 | -------------------------------------------------------------------------------- /tests/fixtures/__init__.py: -------------------------------------------------------------------------------- 1 | from .marine import * 2 | -------------------------------------------------------------------------------- /tests/fixtures/marine/__init__.py: -------------------------------------------------------------------------------- 1 | from .marine_fixtures import * 2 | -------------------------------------------------------------------------------- /tests/marine/run_benchmark.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | python3.8 -m benchmark.main "$@" -------------------------------------------------------------------------------- /scripts/wheel_abi.patch: -------------------------------------------------------------------------------- 1 | 76c76 2 | < if is_py_ext: 3 | --- 4 | > if True: 5 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | Dockerfile 2 | .git 3 | .tox 4 | venv 5 | .venv 6 | .pytest_cache 7 | build/ 8 | dist/ 9 | *egg-info 10 | marine/.wslibs/ 11 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | ignore = E203, E266, E501, F403, F405, W503 3 | max-line-length = 120 4 | max-complexity = 10 5 | exclude = tests/conftest.py __init__.py 6 | -------------------------------------------------------------------------------- /marine/encap_consts.py: -------------------------------------------------------------------------------- 1 | """ 2 | Encapsulation consts 3 | For more values look at wireshark's wiretap/wtap.h 4 | """ 5 | 6 | ENCAP_ETHERNET = 1 7 | 8 | """Your friendly neighbourhood wifi""" 9 | ENCAP_IEEE_802_11_RADIOTAP = 23 10 | -------------------------------------------------------------------------------- /scripts/expose_auditwheel.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | to="$1" 3 | python_exe=$(head -n1 $(which auditwheel) | cut -d'!' -f2-) 4 | site_packages=$($python_exe -c 'import site; print(site.getsitepackages()[0])') 5 | ln -s "$site_packages/auditwheel" "$to/auditwheel" 6 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py38 3 | 4 | [testenv] 5 | changedir = tests 6 | commands = pytest --basetemp="{envtmpdir}" {posargs} 7 | flake8 {toxinidir}/marine/ {toxinidir}/tests/ 8 | deps = 9 | pytest 10 | pypacker 11 | psutil 12 | flake8-black 13 | -------------------------------------------------------------------------------- /marine/exceptions.py: -------------------------------------------------------------------------------- 1 | class BadBPFException(ValueError): 2 | pass 3 | 4 | 5 | class BadDisplayFilterException(ValueError): 6 | pass 7 | 8 | 9 | class InvalidFieldException(ValueError): 10 | pass 11 | 12 | 13 | class UnknownInternalException(Exception): 14 | pass 15 | -------------------------------------------------------------------------------- /scripts/patch_auditwheel_recursive_dependency_bug.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | # This is a patch to handle auditwheel's bug with recursive depenencides: 4 | # https://github.com/pypa/auditwheel/issues/48 5 | 6 | patch -i "$(dirname ${BASH_SOURCE[0]})/wheel_abi.patch" /auditwheel/wheel_abi.py 7 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name="marine", 5 | version="3.1.2", 6 | description="Python client for Marine", 7 | packages=["marine"], 8 | include_package_data=True, 9 | package_data={ 10 | "marine": [".ws/libs/*.so*", ".ws/data/*"], 11 | }, 12 | ) 13 | -------------------------------------------------------------------------------- /marine/__init__.py: -------------------------------------------------------------------------------- 1 | from .simple_marine import ( 2 | filter_packet, 3 | parse_packet, 4 | filter_and_parse_packet, 5 | parse_all_packet_fields, 6 | report_fields, 7 | validate_bpf, 8 | validate_fields, 9 | validate_display_filter, 10 | get_marine, 11 | ) 12 | from .marine_pool import MarinePool 13 | from .exceptions import * 14 | from .marine import Marine 15 | from . import encap_consts 16 | -------------------------------------------------------------------------------- /scripts/modify_auditwheel_policy.py: -------------------------------------------------------------------------------- 1 | import json 2 | from typing import List, TypedDict 3 | 4 | 5 | class Policy(TypedDict): 6 | name: str 7 | lib_whitelist: List[str] 8 | # There are more fields but they don't concern us 9 | 10 | policy_path = "/auditwheel/policy/manylinux-policy.json" 11 | 12 | with open(policy_path) as f: 13 | policies: List[Policy] = json.load(f) 14 | 15 | for policy in policies: 16 | policy["lib_whitelist"].append("libpcap.so.1") 17 | 18 | with open(policy_path, "w") as f: 19 | json.dump(policies, f, indent=2) 20 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: push 4 | 5 | jobs: 6 | test: 7 | 8 | runs-on: ubuntu-latest 9 | 10 | steps: 11 | - uses: actions/checkout@v2 12 | 13 | - name: Extract branch name 14 | shell: bash 15 | run: echo "##[set-output name=tag;]$(echo ${GITHUB_REF#refs/heads/} | tr / -)" 16 | id: extract_branch 17 | 18 | - name: "Pull core image" 19 | shell: bash 20 | run: echo -n "##[set-output name=tag;]" && docker pull tomlegkov/marine-core:${{ steps.extract_branch.outputs.tag }} > /dev/null && echo "${{ steps.extract_branch.outputs.tag }}" || echo marine 21 | id: detect_tag 22 | 23 | - name: "Build marine-python docker image" 24 | run: docker build --build-arg MARINE_CORE_TAG=${{ steps.detect_tag.outputs.tag }} --pull -t marine-python . 25 | 26 | - name: "Extract wheel from docker image" 27 | run: docker run -i --rm -v $(pwd)/dist:/io marine-python sh -c "cp /dist/marine*.whl /io/" 28 | 29 | - uses: actions/upload-artifact@v2 30 | with: 31 | path: dist/*.whl 32 | 33 | - name: "Run marine-python tests" 34 | run: docker run -i --rm marine-python 35 | -------------------------------------------------------------------------------- /tests/marine/benchmark/utils.py: -------------------------------------------------------------------------------- 1 | import struct 2 | from dataclasses import dataclass 3 | from typing import List, Dict, Callable 4 | 5 | 6 | @dataclass 7 | class BenchmarkPacket: 8 | packet: bytes 9 | good_bpf: str 10 | good_display_filter: str 11 | # TODO: add bad_bpf and bad_display_filter to drop some packets 12 | # bad_bpf: str 13 | # bad_display_filter: str 14 | fields_to_extract: List[str] 15 | expected_parse_result: Dict[str, str] 16 | 17 | 18 | @dataclass(frozen=True) 19 | class Layer3Conversation: 20 | src_mac: str 21 | dst_mac: str 22 | src_ip: str 23 | dst_ip: str 24 | 25 | 26 | @dataclass 27 | class ConversationGenerator: 28 | percentage_of_packets: float 29 | generator: Callable[[Layer3Conversation, int], List[BenchmarkPacket]] 30 | 31 | 32 | def write_cap(file_path: str, packets: List[bytes]): 33 | """ 34 | This is a good util for debugging, which is why I'm keeping it here. 35 | """ 36 | PCAP_HEADER = bytes.fromhex("D4C3B2A10200040000000000000000000000040001000000") 37 | data = b"" 38 | for packet in packets: 39 | data += struct.pack(" Packet: 11 | src_mac = "00:00:00:12:34:ff" 12 | broadcast_mac = "ff:ff:ff:ff:ff:ff" 13 | 14 | return ethernet.Ethernet(src_s=src_mac, dst_s=broadcast_mac) + arp.ARP( 15 | sha_s=src_mac, spa_s=src_ip, tha_s=broadcast_mac, tpa_s=target_ip 16 | ) 17 | 18 | 19 | @pytest.fixture 20 | def passing_src_ip() -> str: 21 | return "2.2.2.2" 22 | 23 | 24 | @pytest.fixture 25 | def not_passing_src_ip() -> str: 26 | return "3.3.3.3" 27 | 28 | 29 | @pytest.fixture 30 | def arp_packets(passing_src_ip: str, not_passing_src_ip: str) -> List[Packet]: 31 | packets = [] 32 | for i in range(10): 33 | target_ip = f"1.1.1.{i}" 34 | src_ip = passing_src_ip if i % 2 == 0 else not_passing_src_ip 35 | packets.append(generate_arp_packet(src_ip, target_ip)) 36 | return packets 37 | 38 | 39 | def test_marine_pool_preserves_order( 40 | marine_pool_instance: MarinePool, arp_packets: List[Packet], passing_src_ip: str 41 | ) -> None: 42 | TARGET_IP_FIELD_NAME = "arp.dst.proto_ipv4" 43 | 44 | raw_packets = [packet.bin() for packet in arp_packets] 45 | results = marine_pool_instance.filter_and_parse( 46 | raw_packets, bpf=f"arp net {passing_src_ip}", fields=[TARGET_IP_FIELD_NAME] 47 | ) 48 | 49 | for arp_packet, filter_and_parse_result in zip(arp_packets, results): 50 | expected_target_ip = arp_packet.highest_layer.tpa_s 51 | passed, actual_fields = filter_and_parse_result 52 | assert passed == (arp_packet.highest_layer.spa_s == passing_src_ip) 53 | if passed: 54 | assert expected_target_ip == actual_fields[TARGET_IP_FIELD_NAME] 55 | -------------------------------------------------------------------------------- /tests/fixtures/marine/marine_fixtures.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from typing import List, Union 3 | 4 | import pytest 5 | from pypacker.layer12 import ethernet 6 | from pypacker.layer3 import ip 7 | from pypacker.layer4 import tcp 8 | 9 | from marine.marine import Marine 10 | from marine.marine_pool import MarinePool 11 | 12 | 13 | @pytest.fixture 14 | def extracted_fields_from_tcp_packet() -> List[str]: 15 | return ["eth.src", "eth.dst", "ip.src", "ip.dst", "tcp.srcport", "tcp.dstport"] 16 | 17 | 18 | @pytest.fixture 19 | def tcp_payload() -> bytes: 20 | return b"payload" 21 | 22 | 23 | @pytest.fixture 24 | def tcp_packet(tcp_payload) -> bytes: 25 | packet = ( 26 | ethernet.Ethernet(src_s="00:00:00:12:34:ff", dst_s="00:00:00:ff:00:1e") 27 | + ip.IP(src_s="10.0.0.255", dst_s="21.53.78.255") 28 | + tcp.TCP(sport=16424, dport=41799, flags=tcp.TH_ACK, body_bytes=tcp_payload) 29 | ) 30 | return packet.bin() 31 | 32 | 33 | @pytest.fixture 34 | def tcp_packet_fields(marine_instance: Marine, tcp_packet: bytes): 35 | return marine_instance.parse_all_fields(tcp_packet) 36 | 37 | 38 | @pytest.fixture(scope="session") 39 | def marine_so_path() -> str: 40 | path = Path(__file__).parent / "libmarine.so" 41 | return str(path.resolve()) 42 | 43 | 44 | @pytest.fixture(scope="session") 45 | def epan_auto_reset_count() -> int: 46 | return 5000 47 | 48 | 49 | @pytest.fixture(scope="session") 50 | def marine_instance(marine_so_path: str, epan_auto_reset_count: int) -> Marine: 51 | return Marine(epan_auto_reset_count=epan_auto_reset_count) 52 | 53 | 54 | @pytest.fixture(scope="session") 55 | def marine_pool_instance(marine_so_path: str, epan_auto_reset_count: int) -> MarinePool: 56 | with MarinePool(epan_auto_reset_count) as mp: 57 | yield mp 58 | 59 | 60 | @pytest.fixture(scope="session", params=["marine_instance", "marine_pool_instance"]) 61 | def marine_or_marine_pool(request) -> Union[Marine, MarinePool]: 62 | return request.getfixturevalue(request.param) 63 | -------------------------------------------------------------------------------- /tests/marine/benchmark/benchmark_generator.py: -------------------------------------------------------------------------------- 1 | from ipaddress import IPv4Address 2 | from random import randint, getrandbits, shuffle 3 | from typing import List 4 | 5 | from .conversation_generators import ( 6 | generate_raw_tcp_conversation, 7 | generate_raw_udp_conversation, 8 | ) 9 | from .utils import ConversationGenerator, Layer3Conversation, BenchmarkPacket 10 | 11 | CONVERSATION_GENERATORS = [ 12 | ConversationGenerator(0.5, generate_raw_tcp_conversation), 13 | ConversationGenerator(0.5, generate_raw_udp_conversation), 14 | ] 15 | 16 | 17 | def create_random_mac() -> str: 18 | return "00:00:00:%02x:%02x:%02x" % ( 19 | randint(0, 255), 20 | randint(0, 255), 21 | randint(0, 255), 22 | ) 23 | 24 | 25 | def generate_macs(count: int) -> List[str]: 26 | if count % 2 != 0: 27 | count += 1 28 | macs = set() 29 | while len(macs) < count: 30 | macs.add(str(create_random_mac())) 31 | mac_list = list(macs) 32 | shuffle(mac_list) 33 | return mac_list 34 | 35 | 36 | def generate_ips(count: int) -> List[str]: 37 | if count % 2 != 0: 38 | count += 1 39 | ips = set() 40 | while len(ips) < count: 41 | ips.add(str(IPv4Address(getrandbits(32)))) 42 | ip_list = list(ips) 43 | shuffle(ip_list) 44 | return ip_list 45 | 46 | 47 | def generate_layer_3_conversations(count: int) -> List[Layer3Conversation]: 48 | ips = generate_ips(count * 2) 49 | macs = generate_macs(count * 2) 50 | # The list is already randomly generated, so taking consecutive values is random enough 51 | return [ 52 | Layer3Conversation(macs[i], macs[i + 1], ips[i], ips[i + 1]) 53 | for i in range(0, count * 2, 2) 54 | ] 55 | 56 | 57 | def generate_packets(count: int) -> List[BenchmarkPacket]: 58 | benchmark_packets = [] 59 | layer_3_conversations = generate_layer_3_conversations(count // 1000) 60 | packets_per_conversation = count // len(layer_3_conversations) 61 | for conversation in layer_3_conversations: 62 | for conversation_generator in CONVERSATION_GENERATORS: 63 | benchmark_packets.extend( 64 | conversation_generator.generator( 65 | conversation, 66 | int( 67 | packets_per_conversation 68 | * conversation_generator.percentage_of_packets 69 | ), 70 | ) 71 | ) 72 | return benchmark_packets 73 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | .idea/ 132 | tests/fixtures/marine/libmarine.so 133 | .wslibs 134 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Marine Python Client 2 | Python client for [Marine](https://github.com/tomlegkov/marine-core) 3 | 4 | ## Installation 5 | Clone the repo and run: 6 | ```shell 7 | python setup.py install 8 | ``` 9 | 10 | ## Usage 11 | ### Basic Usage (filter and parse packet) 12 | ```python 13 | import marine 14 | 15 | passed, result = marine.filter_and_parse_packet(b"your packet", "ip host 1.1.1.1", "tcp.port == 80", ["ip.src", "ip.dst"]) 16 | if passed: 17 | print(f"{result['ip.src']} -> {result['ip.dst']}") 18 | ``` 19 | 20 | ### Other Available API 21 | #### Filter packet 22 | ```python 23 | passed = marine.filter_packet(b"your packet", "ip host 1.1.1.1", "tcp.port == 80") 24 | if not passed: 25 | print("Packet didn't pass filter") 26 | ``` 27 | 28 | #### Parse packet 29 | ```python 30 | result = marine.parse_packet(b"your packet", ["macro.ip.src", "tcp.port"], {"macro.ip.src": ["ip.src", "ipv6.src"]}) 31 | print(f"Parsed IP: {result['macro.ip.src']} and port: {result['tcp.port']}") 32 | ``` 33 | 34 | #### BPF validation 35 | ```python 36 | validation_result = marine.validate_bpf("ip host 1.1.1.1") 37 | if not validation_result: 38 | print(f"BPF validation error: {validation_result.error}") 39 | ``` 40 | 41 | #### Display filter validation 42 | ```python 43 | validation_result = marine.validate_display_filter("tcp.port == 80") 44 | if not validation_result: 45 | print(f"Display filter validation error: {validation_result.error}") 46 | ``` 47 | 48 | #### Fields validation 49 | ```python 50 | validation_result = marine.validate_fields(["macro.ip.src", "tcp.port"], {"macro.ip.src": ["ip.src", "ipv6.src"]}) 51 | if not validation_result: 52 | print(f"The following fields don't exist: {validation_result.errors}") 53 | ``` 54 | 55 | #### Pool 56 | `MarinePool` allows to run multiple instances of Marine using multiple cores. 57 | The exported API is identical to Marine's: `filter`, `parse`, `filter_and_parse`. 58 | 59 | ```python 60 | pool = MarinePool(process_count=4) 61 | parsed_packets = pool.filter_and_parse(packets, bpf="udp", fields=["macro.ip.src", "udp.port"], field_templates={"macro.ip.src": ["ip.src", "ipv6.src"]}) 62 | for passed, result in parsed_packets: 63 | if passed: 64 | print(f"Parsed IP: {result['macro.ip.src']} and UDP port: {result['udp.port']}") 65 | ``` 66 | 67 | 68 | #### Advanced 69 | For advanced usages (not recommended) see the `get_marine()` function and the `Marine` class . 70 | 71 | ## Contributing 72 | ### Guidelines 73 | Syntax formatting is done using [Black](https://github.com/psf/black) 74 | 75 | ### Running Tests 76 | The tests are written using pytest. To run the tests, you need to provide the library file (`libmarine.so`) and its dependencies. 77 | `marine` expects `libmarine.so` to reside in `marine/.wslibs`. Inside a development environment, you can simply create a link from `marine/.wslibs` to where you compile `marine-core`. 78 | 79 | Then, simply run `tox`. 80 | 81 | Additionally, syntax is checked with flake8 by running `flake8 marine tests` from the root directory of the project. 82 | 83 | ### Packaging 84 | An `x86_64` `manylinux2014` wheel is built by our CI, based on a `manylinux` image supplied by `marine-core` and patched to meet our needs. 85 | There is currently no support for installing on other platforms. -------------------------------------------------------------------------------- /marine/marine_prefs.py: -------------------------------------------------------------------------------- 1 | from ctypes import CDLL, POINTER, byref, c_char_p, c_int, c_ubyte, c_uint 2 | 3 | 4 | class MarinePreferences: 5 | def __init__(self, marine_cdll: CDLL): 6 | self._marine = marine_cdll 7 | 8 | self.MARINE_PREFS_BAD_MODULE_NAME = c_int.in_dll( 9 | self._marine, "MARINE_PREFS_BAD_MODULE_NAME" 10 | ).value 11 | self.MARINE_PREFS_BAD_PREF_NAME = c_int.in_dll( 12 | self._marine, "MARINE_PREFS_BAD_PREF_NAME" 13 | ).value 14 | self.MARINE_PREFS_BAD_PREF_TYPE = c_int.in_dll( 15 | self._marine, "MARINE_PREFS_BAD_PREF_TYPE" 16 | ).value 17 | 18 | self._marine.marine_prefs_set_bool.argtypes = [c_char_p, c_char_p, c_ubyte] 19 | self._marine.marine_prefs_set_bool.restype = c_int 20 | 21 | self._marine.marine_prefs_get_bool.argtypes = [ 22 | c_char_p, 23 | c_char_p, 24 | POINTER(c_ubyte), 25 | ] 26 | self._marine.marine_prefs_get_bool.restype = c_int 27 | 28 | self._marine.marine_prefs_set_uint.argtypes = [c_char_p, c_char_p, c_uint] 29 | self._marine.marine_prefs_set_uint.restype = c_int 30 | 31 | self._marine.marine_prefs_get_uint.argtypes = [ 32 | c_char_p, 33 | c_char_p, 34 | POINTER(c_uint), 35 | ] 36 | self._marine.marine_prefs_get_uint.restype = c_int 37 | 38 | self._marine.marine_prefs_set_str.argtypes = [c_char_p, c_char_p, c_char_p] 39 | self._marine.marine_prefs_set_str.restype = c_int 40 | 41 | self._marine.marine_prefs_get_str.argtypes = [ 42 | c_char_p, 43 | c_char_p, 44 | POINTER(c_char_p), 45 | ] 46 | self._marine.marine_prefs_get_str.restype = c_int 47 | 48 | def set_bool(self, module_name: str, preference_name: str, value: bool): 49 | status = self._marine.marine_prefs_set_bool( 50 | module_name.encode(), preference_name.encode(), value 51 | ) 52 | self._raise_for_pref_status(status, module_name, preference_name, "bool") 53 | 54 | def set_uint(self, module_name: str, preference_name: str, value: int): 55 | status = self._marine.marine_prefs_set_uint( 56 | module_name.encode(), preference_name.encode(), value 57 | ) 58 | self._raise_for_pref_status(status, module_name, preference_name, "uint") 59 | 60 | def set_str(self, module_name: str, preference_name: str, value: str): 61 | status = self._marine.marine_prefs_set_str( 62 | module_name.encode(), preference_name.encode(), value.encode() 63 | ) 64 | self._raise_for_pref_status(status, module_name, preference_name, "string") 65 | 66 | def get_bool(self, module_name: str, preference_name: str) -> bool: 67 | value = c_ubyte() 68 | status = self._marine.marine_prefs_get_bool( 69 | module_name.encode(), preference_name.encode(), byref(value) 70 | ) 71 | self._raise_for_pref_status(status, module_name, preference_name, "bool") 72 | return bool(value) 73 | 74 | def get_uint(self, module_name: str, preference_name: str) -> int: 75 | value = c_uint() 76 | status = self._marine.marine_prefs_get_uint( 77 | module_name.encode(), preference_name.encode(), byref(value) 78 | ) 79 | self._raise_for_pref_status(status, module_name, preference_name, "uint") 80 | return int.from_bytes(value, "little") 81 | 82 | def get_str(self, module_name: str, preference_name: str) -> str: 83 | value = c_char_p(b"") 84 | status = self._marine.marine_prefs_get_str( 85 | module_name.encode(), preference_name.encode(), byref(value) 86 | ) 87 | self._raise_for_pref_status(status, module_name, preference_name, "string") 88 | return value.value.decode("utf-8") 89 | 90 | def _raise_for_pref_status( 91 | self, status: int, module_name: str, preference_name: str, desired_type: str 92 | ): 93 | if status == self.MARINE_PREFS_BAD_MODULE_NAME: 94 | raise ValueError(f"Bad module name {module_name}") 95 | elif status == self.MARINE_PREFS_BAD_PREF_NAME: 96 | raise ValueError(f"Bad preference name {preference_name}") 97 | elif status == self.MARINE_PREFS_BAD_PREF_TYPE: 98 | raise TypeError( 99 | f"Preference {module_name}.{preference_name} is not a {desired_type}" 100 | ) 101 | -------------------------------------------------------------------------------- /marine/marine_pool.py: -------------------------------------------------------------------------------- 1 | import math 2 | import multiprocessing 3 | from itertools import repeat 4 | from typing import List, Dict, Optional, Tuple, ClassVar 5 | 6 | from .marine import Marine, ParsedPacket 7 | 8 | 9 | class MarinePool: 10 | _marine_instance: ClassVar[Optional[Marine]] = None 11 | 12 | def __init__( 13 | self, epan_auto_reset_count: Optional[int] = None, process_count: int = 4 14 | ): 15 | self._epan_auto_reset_count = epan_auto_reset_count 16 | self._process_count = process_count 17 | 18 | ctx = multiprocessing.get_context("spawn") 19 | # Using spawn so child processes won't get the already initialized marine from the parent process. 20 | # We do that because initializing marine more than one time in a process causes SIGTRAP 21 | self.pool = ctx.Pool( 22 | self._process_count, 23 | initializer=self._init_marine, 24 | initargs=[self._epan_auto_reset_count], 25 | ) 26 | 27 | def __enter__(self): 28 | return self 29 | 30 | def filter( 31 | self, 32 | packets: List[bytes], 33 | bpf: Optional[str] = None, 34 | display_filter: Optional[str] = None, 35 | encapsulation_type: Optional[int] = None, 36 | ) -> List[bool]: 37 | result = self.filter_and_parse( 38 | packets=packets, 39 | bpf=bpf, 40 | display_filter=display_filter, 41 | encapsulation_type=encapsulation_type, 42 | ) 43 | 44 | return [passed for passed, _ in result] 45 | 46 | def parse( 47 | self, 48 | packets: List[bytes], 49 | fields: Optional[List[str]] = None, 50 | encapsulation_type: Optional[int] = None, 51 | field_templates: Optional[Dict[str, List[str]]] = None, 52 | ) -> List[Dict[str, Optional[str]]]: 53 | result = self.filter_and_parse( 54 | packets=packets, 55 | fields=fields, 56 | encapsulation_type=encapsulation_type, 57 | field_templates=field_templates, 58 | ) 59 | 60 | return [values for _, values in result] 61 | 62 | def filter_and_parse( 63 | self, 64 | packets: List[bytes], 65 | bpf: Optional[str] = None, 66 | display_filter: Optional[str] = None, 67 | fields: Optional[List[str]] = None, 68 | encapsulation_type: Optional[int] = None, 69 | field_templates: Optional[Dict[str, List[str]]] = None, 70 | ) -> List[Tuple[bool, Dict[str, Optional[str]]]]: 71 | if len(packets) == 0: 72 | return [] 73 | 74 | chunk_size = int(math.ceil(len(packets) / float(self._process_count))) 75 | return self.pool.starmap( 76 | self._filter_and_parse, 77 | zip( 78 | packets, 79 | repeat(bpf), 80 | repeat(display_filter), 81 | repeat(fields), 82 | repeat(encapsulation_type), 83 | repeat(field_templates), 84 | ), 85 | chunksize=chunk_size, 86 | ) 87 | 88 | def parse_all_fields( 89 | self, packets: List[bytes], encapsulation_type: Optional[int] = None 90 | ) -> List[ParsedPacket]: 91 | chunk_size = int(math.ceil(len(packets) / float(self._process_count))) 92 | return self.pool.starmap( 93 | self._parse_all_fields, 94 | zip(packets, repeat(encapsulation_type)), 95 | chunksize=chunk_size, 96 | ) 97 | 98 | @classmethod 99 | def _init_marine(cls, epan_auto_reset_count: int) -> None: 100 | cls._marine_instance = Marine(epan_auto_reset_count) 101 | 102 | @classmethod 103 | def _filter_and_parse( 104 | cls, 105 | packet: bytes, 106 | bpf: Optional[str] = None, 107 | display_filter: Optional[str] = None, 108 | fields: Optional[list] = None, 109 | encapsulation_type: Optional[int] = None, 110 | field_templates: Optional[Dict[str, List[str]]] = None, 111 | ) -> (bool, Dict[str, str]): 112 | return cls._marine_instance.filter_and_parse( 113 | packet, bpf, display_filter, fields, encapsulation_type, field_templates 114 | ) 115 | 116 | @classmethod 117 | def _parse_all_fields( 118 | cls, packet: bytes, encapsulation_type: Optional[int] = None 119 | ) -> ParsedPacket: 120 | return cls._marine_instance.parse_all_fields(packet, encapsulation_type) 121 | 122 | def __exit__(self, exc_type, exc_val, exc_tb): 123 | self.pool.close() 124 | 125 | def close(self): 126 | self.pool.close() 127 | -------------------------------------------------------------------------------- /marine/simple_marine.py: -------------------------------------------------------------------------------- 1 | from . import encap_consts 2 | from .marine import ( 3 | Marine, 4 | MarineFilterValidationResult, 5 | MarineFieldsValidationResult, 6 | ParsedPacket, 7 | ) 8 | from typing import Optional, List, Dict, Tuple 9 | 10 | marine_instance = None 11 | 12 | 13 | def init_instance(epan_auto_reset_count: Optional[int] = None) -> Marine: 14 | global marine_instance 15 | 16 | if marine_instance is None: 17 | marine_instance = Marine(epan_auto_reset_count) 18 | return marine_instance 19 | 20 | 21 | def filter_packet( 22 | packet: bytes, 23 | bpf: Optional[str] = None, 24 | display_filter: Optional[str] = None, 25 | encapsulation_type: Optional[int] = None, 26 | ) -> bool: 27 | """ 28 | Filters a packet with BPF and a Wireshark-style display filter. 29 | At least one form of filtering is required. 30 | By default the packet is parsed as an ethernet packet, 31 | to view other possible encapsulation values view encap_consts. 32 | """ 33 | return init_instance().filter( 34 | packet=packet, 35 | bpf=bpf, 36 | display_filter=display_filter, 37 | encapsulation_type=encapsulation_type, 38 | ) 39 | 40 | 41 | def parse_packet( 42 | packet: bytes, 43 | fields: Optional[List[str]] = None, 44 | field_templates: Optional[Dict[str, List[str]]] = None, 45 | encapsulation_type: Optional[int] = None, 46 | ) -> Dict[str, Optional[str]]: 47 | """ 48 | Parses the given fields from the packet. Fields have the same name as specified for Wireshark. 49 | If you want to add a custom field, you need to have the required dissector in your Wireshark plugins folder. 50 | Fields that are not available in the packet will be returned as "". 51 | Field templates can be used to expand a field - Example field template format: {"macro.ip.src" : ["ip.src", "ipv6.src"]}. 52 | By default the packet is parsed as an ethernet packet, 53 | to view other possible encapsulation values view encap_consts. 54 | """ 55 | return init_instance().parse( 56 | packet=packet, 57 | fields=fields, 58 | encapsulation_type=encapsulation_type, 59 | field_templates=field_templates, 60 | ) 61 | 62 | 63 | def filter_and_parse_packet( 64 | packet: bytes, 65 | bpf: Optional[List[str]] = None, 66 | display_filter: Optional[str] = None, 67 | fields: Optional[List[str]] = None, 68 | field_templates: Optional[Dict[str, List[str]]] = None, 69 | encapsulation_type: Optional[int] = None, 70 | ) -> Tuple[bool, Dict[str, Optional[str]]]: 71 | """ 72 | Filters a packet with BPF and a Wireshark-style display filter. 73 | If the filter passes, parses the packet according to the fields. 74 | Fields have the same name as specified for Wireshark. 75 | Either the bpf, display filter or fields must be not None. 76 | If the packet does not pass the filter, or fields is None, result fields will be None as well. 77 | Fields that are not available in the packet will be returned as "". 78 | If you want to add a custom field, you need to have the required dissector in your Wireshark plugins folder. 79 | By default the packet is parsed as an ethernet packet, 80 | to view other possible encapsulation values view encap_consts. 81 | """ 82 | return init_instance().filter_and_parse( 83 | packet=packet, 84 | bpf=bpf, 85 | display_filter=display_filter, 86 | fields=fields, 87 | field_templates=field_templates, 88 | encapsulation_type=encapsulation_type, 89 | ) 90 | 91 | 92 | def parse_all_packet_fields( 93 | packet: bytes, encapsulation_type: Optional[int] = None 94 | ) -> ParsedPacket: 95 | """ 96 | Parses the given packet with wireshark's packet parser, returning a dict with all it's fields. 97 | """ 98 | return init_instance().parse_all_fields(packet, encapsulation_type) 99 | 100 | 101 | def validate_bpf( 102 | bpf: str, encapsulation_type: int = encap_consts.ENCAP_ETHERNET 103 | ) -> MarineFilterValidationResult: 104 | """ 105 | Validates the given BPF. 106 | By default the BPF is parsed with ethernet encapsulation, 107 | to view other possible encapsulation values view encap_consts. 108 | """ 109 | return init_instance().validate_bpf(bpf=bpf, encapsulation_type=encapsulation_type) 110 | 111 | 112 | def validate_display_filter(display_filter: str) -> MarineFilterValidationResult: 113 | """ 114 | Validates the given display filter. 115 | """ 116 | return init_instance().validate_display_filter(display_filter=display_filter) 117 | 118 | 119 | def validate_fields( 120 | fields: List[str], field_templates: Optional[Dict[str, List[str]]] = None 121 | ) -> MarineFieldsValidationResult: 122 | """ 123 | Validates the given fields. Fields have the same name as specified for Wireshark. 124 | If you want to add a custom field, you need to have the required dissector in your Wireshark plugins folder. 125 | Field templates can be used to expand a field - Example field template format: {"macro.ip.src" : ["ip.src", "ipv6.src"]}. 126 | """ 127 | return init_instance().validate_fields( 128 | fields=fields, field_templates=field_templates 129 | ) 130 | 131 | 132 | def get_marine() -> Marine: 133 | """ 134 | Gets the used marine object. 135 | """ 136 | return init_instance() 137 | 138 | 139 | def report_fields() -> None: 140 | """ 141 | Dumps to stdout all of marine, similiarly to `tshark -G` 142 | """ 143 | return get_marine().report_fields() 144 | -------------------------------------------------------------------------------- /tests/marine/benchmark/conversation_generators.py: -------------------------------------------------------------------------------- 1 | import os 2 | from random import randint 3 | from typing import List, Callable, Dict, Tuple 4 | 5 | from pypacker.layer12 import ethernet 6 | from pypacker.layer3 import ip 7 | from pypacker.layer4 import tcp, udp 8 | from pypacker.pypacker import Packet 9 | 10 | from .utils import Layer3Conversation, BenchmarkPacket 11 | 12 | 13 | def _generate_port() -> int: 14 | return randint(10000, 60000) 15 | 16 | 17 | def _create_tcp_base_packets(conversation: Layer3Conversation) -> Tuple[Packet, Packet]: 18 | src_port, dst_port = _generate_port(), _generate_port() 19 | base_src_to_dst = ( 20 | ethernet.Ethernet( 21 | src_s=conversation.src_mac, 22 | dst_s=conversation.dst_mac, 23 | type=ethernet.ETH_TYPE_IP, 24 | ) 25 | + ip.IP(p=ip.IP_PROTO_TCP, src_s=conversation.src_ip, dst_s=conversation.dst_ip) 26 | + tcp.TCP(sport=src_port, dport=dst_port) 27 | ) 28 | base_dst_to_src = ( 29 | ethernet.Ethernet( 30 | src_s=conversation.dst_mac, 31 | dst_s=conversation.src_mac, 32 | type=ethernet.ETH_TYPE_IP, 33 | ) 34 | + ip.IP(p=ip.IP_PROTO_TCP, src_s=conversation.dst_ip, dst_s=conversation.src_ip) 35 | + tcp.TCP(sport=dst_port, dport=src_port) 36 | ) 37 | 38 | return base_src_to_dst, base_dst_to_src 39 | 40 | 41 | def _create_udp_base_packets(conversation: Layer3Conversation) -> Tuple[Packet, Packet]: 42 | src_port, dst_port = _generate_port(), _generate_port() 43 | base_src_to_dst = ( 44 | ethernet.Ethernet( 45 | src_s=conversation.src_mac, 46 | dst_s=conversation.dst_mac, 47 | type=ethernet.ETH_TYPE_IP, 48 | ) 49 | + ip.IP(p=ip.IP_PROTO_UDP, src_s=conversation.src_ip, dst_s=conversation.dst_ip) 50 | + udp.UDP(sport=src_port, dport=dst_port) 51 | ) 52 | base_dst_to_src = ( 53 | ethernet.Ethernet( 54 | src_s=conversation.dst_mac, 55 | dst_s=conversation.src_mac, 56 | type=ethernet.ETH_TYPE_IP, 57 | ) 58 | + ip.IP(p=ip.IP_PROTO_UDP, src_s=conversation.dst_ip, dst_s=conversation.src_ip) 59 | + udp.UDP(sport=dst_port, dport=src_port) 60 | ) 61 | 62 | return base_src_to_dst, base_dst_to_src 63 | 64 | 65 | def _generate_conversation( 66 | base_src_to_dst: Packet, 67 | base_dst_to_src: Packet, 68 | packet_generator: Callable[[Packet], BenchmarkPacket], 69 | conversation_length: int, 70 | ) -> List[BenchmarkPacket]: 71 | packets: List[BenchmarkPacket] = [] 72 | total_sent = 0 73 | clients_turn = True 74 | 75 | while total_sent < conversation_length: 76 | packet_count = randint(1, 3) 77 | if packet_count + total_sent > conversation_length: 78 | packet_count = conversation_length - total_sent 79 | 80 | for i in range(packet_count): 81 | packet = base_src_to_dst if clients_turn else base_dst_to_src 82 | packets.append(packet_generator(packet)) 83 | 84 | clients_turn = not clients_turn 85 | total_sent += packet_count 86 | 87 | return packets 88 | 89 | 90 | def _get_up_to_layer_3_expected_fields(packet: Packet) -> Dict[str, str]: 91 | return { 92 | "eth.src": packet[ethernet.Ethernet].src_s.lower(), 93 | "eth.dst": packet[ethernet.Ethernet].dst_s.lower(), 94 | "ip.src": packet[ip.IP].src_s, 95 | "ip.dst": packet[ip.IP].dst_s, 96 | "ip.proto": str(packet[ip.IP].p), 97 | } 98 | 99 | 100 | def generate_raw_tcp_conversation( 101 | conversation: Layer3Conversation, conversation_length: int 102 | ) -> List[BenchmarkPacket]: 103 | base_src_to_dst, base_dst_to_src = _create_tcp_base_packets(conversation) 104 | 105 | def _create_packet(base_layer: Packet) -> BenchmarkPacket: 106 | src_port = base_layer[tcp.TCP].sport 107 | dst_port = base_layer[tcp.TCP].dport 108 | data_len = randint(100, 1000) 109 | base_layer[tcp.TCP].body_bytes = os.urandom(data_len) 110 | layer_3_expected_fields = _get_up_to_layer_3_expected_fields(base_layer) 111 | expected_parse_result = { 112 | "tcp.len": str(data_len), 113 | "tcp.srcport": str(src_port), 114 | "tcp.dstport": str(dst_port), # TODO change this when Marine supports types 115 | } 116 | expected_parse_result.update(layer_3_expected_fields) 117 | return BenchmarkPacket( 118 | base_layer.bin(), 119 | good_bpf=f"tcp src port {src_port} and tcp dst port {dst_port}", 120 | good_display_filter=f"tcp.srcport == {src_port} and tcp.dstport == {dst_port}", 121 | fields_to_extract=list(layer_3_expected_fields.keys()) 122 | + ["tcp.len", "tcp.srcport", "tcp.dstport"], 123 | expected_parse_result=expected_parse_result, 124 | ) 125 | 126 | return _generate_conversation( 127 | base_src_to_dst, base_dst_to_src, _create_packet, conversation_length 128 | ) 129 | 130 | 131 | def generate_raw_udp_conversation( 132 | conversation: Layer3Conversation, conversation_length: int 133 | ) -> List[BenchmarkPacket]: 134 | base_src_to_dst, base_dst_to_src = _create_udp_base_packets(conversation) 135 | 136 | def _create_packet(base_layer: Packet) -> BenchmarkPacket: 137 | src_port = base_layer[udp.UDP].sport 138 | dst_port = base_layer[udp.UDP].dport 139 | data_len = randint(100, 1000) 140 | base_layer[udp.UDP].body_bytes = os.urandom(data_len) 141 | layer_3_expected_fields = _get_up_to_layer_3_expected_fields(base_layer) 142 | expected_parse_result = { 143 | "udp.length": str(data_len + base_layer[udp.UDP].header_len), 144 | "udp.srcport": str(src_port), 145 | "udp.dstport": str(dst_port), # TODO change this when Marine supports types 146 | } 147 | expected_parse_result.update(layer_3_expected_fields) 148 | return BenchmarkPacket( 149 | base_layer.bin(), 150 | good_bpf=f"udp src port {src_port} and udp dst port {dst_port}", 151 | good_display_filter=f"udp.srcport == {src_port} and udp.dstport == {dst_port}", 152 | fields_to_extract=list(layer_3_expected_fields) 153 | + ["udp.length", "udp.srcport", "udp.dstport"], 154 | expected_parse_result=expected_parse_result, 155 | ) 156 | 157 | return _generate_conversation( 158 | base_src_to_dst, base_dst_to_src, _create_packet, conversation_length 159 | ) 160 | -------------------------------------------------------------------------------- /tests/marine/benchmark/main.py: -------------------------------------------------------------------------------- 1 | """" 2 | Marine Python Benchmark 3 | 4 | The benchmark is written to achieve 2 goals: 5 | 1) Understand how fast Marine performs (specifically in the Python client) and find bottlenecks 6 | 2) See that memory usage doesn't increase 7 | 8 | The following things are benchmarked: 9 | 1) Filter only with BPF 10 | 2) Filter only with display filter 11 | 3) Filter with BPF + display filter 12 | 4) Extract 3 fields 13 | 5) Extract 8 fields 14 | 6) Filter with BPF + display filter + extract 3 fields 15 | 7) Filter with BPF + display filter + extract 8 fields 16 | 17 | By default, all of the benchmarks are executed. 18 | 19 | For accurate memory testing, the packet count should be passed as a multiple of AUTO_RESET_COUNT. 20 | TODO: allow passing packet count as a parameter 21 | TODO: allow setting AUTO_RESET_COUNT as a parameter (blocked on issue #4 in marine-core) 22 | 23 | The generated packets will contain multiple protocols, and will purposely contain many different conversations, 24 | in order for the benchmarks to fill up the conversation table "naturally" and see how it affects performance. 25 | TODO: add HTTP, DNS, ARP. 26 | TODO: simulate PL in conversations 27 | TODO: add support for real TCP conversations with ack and seq management 28 | """ 29 | import argparse 30 | import os 31 | import time 32 | from typing import List, Callable, Dict 33 | 34 | import psutil 35 | 36 | from marine import Marine, encap_consts 37 | from .benchmark_generator import generate_packets 38 | from .utils import BenchmarkPacket 39 | 40 | AUTO_RESET_COUNT = ( 41 | 20000 # TODO use marine_instance.get_epan_reset_count() when it's implemented 42 | ) 43 | 44 | marine_instance = Marine( 45 | os.path.join( 46 | os.path.dirname(os.path.abspath(__file__)), 47 | "..", 48 | "..", 49 | "fixtures", 50 | "marine", 51 | "libmarine.so", 52 | ) 53 | ) 54 | process = psutil.Process() 55 | benchmark_functions: Dict[str, Callable[[List[BenchmarkPacket]], None]] = dict() 56 | 57 | 58 | def get_used_memory_in_mb() -> float: 59 | return process.memory_info().rss / 1024.0 / 1024.0 60 | 61 | 62 | def create_3_expected_fields(packet: BenchmarkPacket) -> Dict[str, str]: 63 | return { 64 | field: packet.expected_parse_result[field] 65 | for field in packet.fields_to_extract[:3] 66 | } 67 | 68 | 69 | def benchmark_wrapper(f: Callable[[List[BenchmarkPacket]], None]): 70 | def benchmark_timer(packets: List[BenchmarkPacket]): 71 | start_used_memory = get_used_memory_in_mb() 72 | start_time = time.time() 73 | f(packets) 74 | end_time = time.time() 75 | end_used_memory = get_used_memory_in_mb() 76 | delta_time = end_time - start_time 77 | delta_used_memory = end_used_memory - start_used_memory 78 | print( 79 | f""" 80 | Executed {f.__name__} on {len(packets)} packets in {delta_time:.2f} seconds, 81 | which is {(len(packets) / delta_time):.2f} packets per second. 82 | Started with {start_used_memory:.2f} MB, ended with {end_used_memory:.2f}. Delta is {delta_used_memory:.2f} MB. 83 | """ 84 | ) 85 | 86 | benchmark_functions[f.__name__] = benchmark_timer 87 | return benchmark_timer 88 | 89 | 90 | @benchmark_wrapper 91 | def benchmark_bpf(packets: List[BenchmarkPacket]): 92 | for packet in packets: 93 | passed, result = marine_instance.filter_and_parse( 94 | packet.packet, 95 | bpf=packet.good_bpf, 96 | encapsulation_type=encap_consts.ENCAP_ETHERNET, 97 | ) 98 | assert passed 99 | assert result is None 100 | 101 | 102 | @benchmark_wrapper 103 | def benchmark_display_filter(packets: List[BenchmarkPacket]): 104 | for packet in packets: 105 | passed, result = marine_instance.filter_and_parse( 106 | packet.packet, 107 | display_filter=packet.good_display_filter, 108 | encapsulation_type=encap_consts.ENCAP_ETHERNET, 109 | ) 110 | assert passed 111 | assert result is None 112 | 113 | 114 | @benchmark_wrapper 115 | def benchmark_bpf_and_display_filter(packets: List[BenchmarkPacket]): 116 | for packet in packets: 117 | passed, result = marine_instance.filter_and_parse( 118 | packet.packet, 119 | bpf=packet.good_bpf, 120 | display_filter=packet.good_display_filter, 121 | encapsulation_type=encap_consts.ENCAP_ETHERNET, 122 | ) 123 | assert passed 124 | assert result is None 125 | 126 | 127 | @benchmark_wrapper 128 | def benchmark_3_fields(packets: List[BenchmarkPacket]): 129 | for packet in packets: 130 | passed, result = marine_instance.filter_and_parse( 131 | packet.packet, 132 | fields=packet.fields_to_extract[:3], 133 | encapsulation_type=encap_consts.ENCAP_ETHERNET, 134 | ) 135 | assert passed 136 | assert create_3_expected_fields(packet) == result 137 | 138 | 139 | @benchmark_wrapper 140 | def benchmark_8_fields(packets: List[BenchmarkPacket]): 141 | for packet in packets: 142 | passed, result = marine_instance.filter_and_parse( 143 | packet.packet, 144 | fields=packet.fields_to_extract, 145 | encapsulation_type=encap_consts.ENCAP_ETHERNET, 146 | ) 147 | assert passed 148 | assert packet.expected_parse_result == result 149 | 150 | 151 | @benchmark_wrapper 152 | def benchmark_bpf_and_display_filter_and_3_fields(packets: List[BenchmarkPacket]): 153 | for packet in packets: 154 | passed, result = marine_instance.filter_and_parse( 155 | packet.packet, 156 | bpf=packet.good_bpf, 157 | display_filter=packet.good_display_filter, 158 | fields=packet.fields_to_extract[:3], 159 | encapsulation_type=encap_consts.ENCAP_ETHERNET, 160 | ) 161 | assert passed 162 | assert create_3_expected_fields(packet) == result 163 | 164 | 165 | @benchmark_wrapper 166 | def benchmark_bpf_and_display_filter_and_8_fields(packets: List[BenchmarkPacket]): 167 | for packet in packets: 168 | passed, result = marine_instance.filter_and_parse( 169 | packet.packet, 170 | bpf=packet.good_bpf, 171 | display_filter=packet.good_display_filter, 172 | fields=packet.fields_to_extract, 173 | encapsulation_type=encap_consts.ENCAP_ETHERNET, 174 | ) 175 | assert passed 176 | assert packet.expected_parse_result == result 177 | 178 | 179 | @benchmark_wrapper 180 | def benchmark_bpf_and_display_filter_and_8_fields_auto_detection( 181 | packets: List[BenchmarkPacket], 182 | ): 183 | for packet in packets: 184 | passed, result = marine_instance.filter_and_parse( 185 | packet.packet, 186 | bpf=packet.good_bpf, 187 | display_filter=packet.good_display_filter, 188 | fields=packet.fields_to_extract, 189 | ) 190 | assert passed 191 | assert packet.expected_parse_result == result 192 | 193 | 194 | @benchmark_wrapper 195 | def benchmark_bpf_and_display_filter_and_8_fields_with_macros( 196 | packets: List[BenchmarkPacket], 197 | ): 198 | for packet in packets: 199 | passed, result = marine_instance.filter_and_parse( 200 | packet.packet, 201 | bpf=packet.good_bpf, 202 | display_filter=packet.good_display_filter, 203 | fields=packet.fields_to_extract, 204 | field_templates=Marine.SUGGESTED_FIELD_TEMPLATES, 205 | encapsulation_type=encap_consts.ENCAP_ETHERNET, 206 | ) 207 | assert passed 208 | assert packet.expected_parse_result == result 209 | 210 | 211 | if __name__ == "__main__": 212 | parser = argparse.ArgumentParser() 213 | parser.add_argument( 214 | "--benchmark", choices=["all"] + list(benchmark_functions.keys()) 215 | ) 216 | args = parser.parse_args() 217 | run_all_functions = not args.benchmark or args.benchmark == "all" 218 | packet_multiplier = len(benchmark_functions) if run_all_functions else 1 219 | print("Generating packets...") 220 | generated_packets = generate_packets(AUTO_RESET_COUNT * packet_multiplier) 221 | print("Done generating, executing benchmarks...") 222 | benchmark_start_used_memory = get_used_memory_in_mb() 223 | 224 | if run_all_functions: 225 | for i, benchmark_function in enumerate(benchmark_functions.values()): 226 | benchmark_function( 227 | generated_packets[i * AUTO_RESET_COUNT : (i + 1) * AUTO_RESET_COUNT] 228 | ) 229 | elif args.benchmark in benchmark_functions: 230 | benchmark_functions[args.benchmark](generated_packets) 231 | else: 232 | raise ValueError("Pick a benchmark") 233 | 234 | benchmark_end_used_memory = get_used_memory_in_mb() 235 | memory_delta = benchmark_end_used_memory - benchmark_start_used_memory 236 | print( 237 | f"Total memory usage (over all of the benchmarks) increased by {memory_delta:.2f} MB" 238 | ) 239 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /marine/marine.py: -------------------------------------------------------------------------------- 1 | from ctypes import ( 2 | CDLL, 3 | POINTER, 4 | Structure, 5 | c_char, 6 | c_char_p, 7 | c_long, 8 | c_int, 9 | c_ulong, 10 | c_uint, 11 | c_uint8, 12 | c_ubyte, 13 | cdll, 14 | string_at, 15 | pointer, 16 | ) 17 | from pathlib import Path 18 | from typing import Optional, List, Dict, Tuple, NamedTuple, Union 19 | import ctypes 20 | import os 21 | 22 | from .exceptions import ( 23 | BadBPFException, 24 | BadDisplayFilterException, 25 | InvalidFieldException, 26 | UnknownInternalException, 27 | ) 28 | from .marine_prefs import MarinePreferences 29 | from . import encap_consts 30 | 31 | 32 | class MarineResult(Structure): 33 | _fields_ = [("output", POINTER(c_char_p)), ("len", c_uint), ("result", c_int)] 34 | 35 | 36 | class MarinePacketFieldValues(ctypes.Union): 37 | pass 38 | 39 | 40 | class MarinePacketFieldValue(Structure): 41 | pass 42 | 43 | 44 | MarinePacketFieldValues._fields_ = [ 45 | ("int_value", c_long), 46 | ("uint_value", c_ulong), 47 | ("bool_value", c_ubyte), 48 | ("str_value", c_char_p), 49 | ("bytes_value", c_char_p), 50 | ("list_value", POINTER(MarinePacketFieldValue)), 51 | ] 52 | 53 | MarinePacketFieldValue._anonymous_ = ["values"] 54 | MarinePacketFieldValue._fields_ = [ 55 | ("values", MarinePacketFieldValues), 56 | ("len", c_uint), 57 | ("type", c_uint), 58 | ] 59 | 60 | 61 | class MarinePacketFieldValueType: 62 | NONE = 0 63 | INT = 1 64 | UINT = 2 65 | BOOL = 3 66 | STR = 4 67 | BYTES = 5 68 | LIST = 6 69 | 70 | 71 | class MarinePacketFieldChildren(Structure): 72 | pass 73 | 74 | 75 | class MarinePacketField(Structure): 76 | _fields_ = [ 77 | ("name", c_char_p), 78 | ("children", POINTER(MarinePacketFieldChildren)), 79 | ("value", MarinePacketFieldValue), 80 | ] 81 | 82 | 83 | MarinePacketFieldChildren._fields_ = [ 84 | ("data", POINTER(MarinePacketField)), 85 | ("len", c_uint), 86 | ] 87 | 88 | 89 | class MarinePacket(Structure): 90 | _fields_ = [ 91 | ("source_packet", POINTER(c_uint8)), 92 | ("source_packet_length", c_uint), 93 | ("layer_tree", POINTER(MarinePacketField)), 94 | ] 95 | 96 | 97 | ParsedPacket = Dict[str, Union[bytes, int, str, "ParsedPacket"]] 98 | 99 | MARINE_RESULT_POINTER = POINTER(MarineResult) 100 | MARINE_PACKET_POINTER = POINTER(MarinePacket) 101 | 102 | MARINE_BASE_DIR = Path(__file__).parent / ".ws" 103 | MARINE_NAME = MARINE_BASE_DIR / "libs" / "libmarine.so" 104 | MARINE_DATA_DIR = MARINE_BASE_DIR / "data" 105 | 106 | 107 | class MarineFieldsValidationResult(NamedTuple): 108 | valid: bool 109 | errors: List[str] 110 | 111 | def __bool__(self) -> bool: 112 | return self.valid 113 | 114 | 115 | class MarineFilterValidationResult(NamedTuple): 116 | valid: bool 117 | error: Optional[str] 118 | 119 | def __bool__(self) -> bool: 120 | return self.valid 121 | 122 | 123 | class Marine: 124 | SUGGESTED_FIELD_TEMPLATES = { 125 | "macro.ip.src": ["ip.src", "arp.src.proto_ipv4"], 126 | "macro.ip.dst": ["ip.dst", "arp.dst.proto_ipv4"], 127 | "macro.src_port": ["tcp.srcport", "udp.srcport"], 128 | "macro.dst_port": ["tcp.dstport", "udp.dstport"], 129 | } 130 | WIFI_RADIO_PROTOCOLS = frozenset(["radiotap", "wlan", "wlan_radio"]) 131 | 132 | def __init__(self, epan_auto_reset_count: Optional[int] = None): 133 | if not os.getenv("WIRESHARK_DATA_DIR"): 134 | os.putenv("WIRESHARK_DATA_DIR", str(MARINE_DATA_DIR)) 135 | try: 136 | cdll.LoadLibrary(MARINE_NAME) 137 | except Exception: 138 | raise OSError( 139 | "Could not load Marine. Please make sure you have put marine in LD_LIBRARY_PATH." 140 | ) 141 | 142 | self._filters_cache = dict() 143 | self._field_templates_cache = dict() 144 | self._encap_cache = dict() 145 | self._marine = CDLL(MARINE_NAME) 146 | self._marine.marine_dissect_packet.restype = MARINE_RESULT_POINTER 147 | self._marine.marine_free.argtypes = [MARINE_RESULT_POINTER] 148 | self._marine.marine_dissect_all_packet_fields.restype = MARINE_PACKET_POINTER 149 | self._marine.marine_packet_free.argtypes = [MARINE_PACKET_POINTER] 150 | 151 | return_code = self._marine.init_marine() 152 | if return_code < 0: 153 | if ( 154 | return_code 155 | == c_int.in_dll( 156 | self._marine, "MARINE_ALREADY_INITIALIZED_ERROR_CODE" 157 | ).value 158 | ): 159 | raise RuntimeError("Marine is already initialized") 160 | raise RuntimeError("Could not initialize Marine") 161 | 162 | if epan_auto_reset_count: 163 | self._marine.set_epan_auto_reset_count(epan_auto_reset_count) 164 | 165 | self.prefs = MarinePreferences(self._marine) 166 | 167 | @property 168 | def epan_auto_reset_count(self) -> int: 169 | return self._marine.get_epan_auto_reset_count() 170 | 171 | @epan_auto_reset_count.setter 172 | def epan_auto_reset_count(self, value: int) -> None: 173 | self._marine.set_epan_auto_reset_count(value) 174 | 175 | def filter( 176 | self, 177 | packet: bytes, 178 | bpf: Optional[str] = None, 179 | display_filter: Optional[str] = None, 180 | encapsulation_type: Optional[int] = None, 181 | ) -> bool: 182 | passed, _ = self.filter_and_parse( 183 | packet=packet, 184 | bpf=bpf, 185 | display_filter=display_filter, 186 | encapsulation_type=encapsulation_type, 187 | ) 188 | 189 | return passed 190 | 191 | def parse( 192 | self, 193 | packet: bytes, 194 | fields: Optional[List[str]] = None, 195 | encapsulation_type: Optional[int] = None, 196 | field_templates: Optional[Dict[str, List[str]]] = None, 197 | ) -> Dict[str, Optional[str]]: 198 | _, result = self.filter_and_parse( 199 | packet=packet, 200 | fields=fields, 201 | encapsulation_type=encapsulation_type, 202 | field_templates=field_templates, 203 | ) 204 | 205 | return result 206 | 207 | def filter_and_parse( 208 | self, 209 | packet: bytes, 210 | bpf: Optional[str] = None, 211 | display_filter: Optional[str] = None, 212 | fields: Optional[List[str]] = None, 213 | encapsulation_type: Optional[int] = None, 214 | field_templates: Optional[Dict[str, List[str]]] = None, 215 | ) -> (bool, Dict[str, Optional[str]]): 216 | if bpf is None and display_filter is None and fields is None: 217 | raise ValueError( 218 | "At least one form of dissection must be passed to the function" 219 | ) 220 | 221 | if isinstance(bpf, str): 222 | bpf = bpf.encode("utf-8") 223 | if isinstance(display_filter, str): 224 | display_filter = display_filter.encode("utf-8") 225 | 226 | if fields is not None: 227 | expanded_fields, field_template_indices = self._expand_field_templates( 228 | fields, field_templates 229 | ) 230 | encoded_fields = [ 231 | f.encode("utf-8") if isinstance(f, str) else f for f in expanded_fields 232 | ] 233 | else: 234 | expanded_fields, field_template_indices = None, None 235 | encoded_fields = None 236 | 237 | if encapsulation_type is None: 238 | encapsulation_type = self._detect_encap(expanded_fields) 239 | 240 | filter_key = ( 241 | bpf, 242 | display_filter, 243 | tuple(encoded_fields) if fields is not None else None, 244 | tuple(field_template_indices) 245 | if field_template_indices is not None 246 | else None, 247 | encapsulation_type, 248 | ) 249 | if filter_key in self._filters_cache: 250 | filter_id = self._filters_cache[filter_key] 251 | else: 252 | filter_id = self._add_or_get_filter( 253 | bpf, 254 | display_filter, 255 | encoded_fields, 256 | field_template_indices, 257 | encapsulation_type, 258 | ) 259 | self._filters_cache[filter_key] = filter_id 260 | 261 | marine_result = self._marine.marine_dissect_packet( 262 | filter_id, packet, len(packet) 263 | ) 264 | success, result = False, None 265 | if marine_result.contents.result == 1: 266 | success = True 267 | if fields is not None: 268 | parsed_output = self._parse_output( 269 | marine_result.contents.output, marine_result.contents.len 270 | ) 271 | result = dict(zip(fields, parsed_output)) 272 | 273 | self._marine.marine_free(marine_result) 274 | return success, result 275 | 276 | def parse_all_fields( 277 | self, packet: bytes, encapsulation_type: Optional[int] = None 278 | ) -> ParsedPacket: 279 | encapsulation_type = ( 280 | encapsulation_type if encapsulation_type else encap_consts.ENCAP_ETHERNET 281 | ) 282 | marine_packet = self._marine.marine_dissect_all_packet_fields( 283 | packet, len(packet), encapsulation_type 284 | ) 285 | packet_dict = self._load_marine_packet( 286 | marine_packet.contents.layer_tree.contents 287 | ) 288 | self._marine.marine_packet_free(marine_packet) 289 | return packet_dict 290 | 291 | def _resolve_err_msg(self, err_msg: POINTER(POINTER(c_char))) -> Optional[str]: 292 | if not err_msg.contents: 293 | return None 294 | error = string_at(err_msg.contents) 295 | self._marine.marine_free_err_msg(err_msg.contents) 296 | return error.decode("utf-8") 297 | 298 | def validate_bpf( 299 | self, bpf: str, encapsulation_type: int = encap_consts.ENCAP_ETHERNET 300 | ) -> MarineFilterValidationResult: 301 | bpf = bpf.encode("utf-8") 302 | err_msg = pointer(POINTER(c_char)()) 303 | valid = bool(self._marine.validate_bpf(bpf, encapsulation_type, err_msg)) 304 | error = self._resolve_err_msg(err_msg) 305 | return MarineFilterValidationResult(valid, error) 306 | 307 | def validate_display_filter( 308 | self, display_filter: str 309 | ) -> MarineFilterValidationResult: 310 | display_filter = display_filter.encode("utf-8") 311 | err_msg = pointer(POINTER(c_char)()) 312 | valid = bool(self._marine.validate_display_filter(display_filter, err_msg)) 313 | error = self._resolve_err_msg(err_msg) 314 | return MarineFilterValidationResult(valid, error) 315 | 316 | def validate_fields( 317 | self, fields: List[str], field_templates: Optional[Dict[str, List[str]]] = None 318 | ) -> MarineFieldsValidationResult: 319 | fields, _ = self._expand_field_templates(fields, field_templates) 320 | fields_len = len(fields) 321 | fields = [field.encode("utf-8") for field in fields] 322 | fields_c_arr = (c_char_p * fields_len)(*fields) 323 | err_msg = pointer(POINTER(c_char)()) 324 | valid = bool(self._marine.validate_fields(fields_c_arr, fields_len, err_msg)) 325 | error = self._resolve_err_msg(err_msg) 326 | return MarineFieldsValidationResult( 327 | valid, [] if error is None else error.split("\t") 328 | ) 329 | 330 | @staticmethod 331 | def _parse_output(output: POINTER(c_char_p), length: int) -> List[Optional[str]]: 332 | return list( 333 | output[i][:].decode("utf-8") if output[i] is not None else None 334 | for i in range(length) 335 | ) 336 | 337 | @classmethod 338 | def _load_marine_packet(cls, field): 339 | if field.children: 340 | return cls._load_child_fields(field.children.contents) 341 | else: 342 | return cls._load_field_value(field.value) 343 | 344 | @classmethod 345 | def _load_child_fields(cls, children): 346 | child_fields = {} 347 | for child in children.data[: children.len]: 348 | name = cls._safe_decode(child.name) 349 | value = cls._load_marine_packet(child) 350 | child_fields[name] = value 351 | return child_fields 352 | 353 | @classmethod 354 | def _load_field_value(cls, value): 355 | value_len = value.len 356 | value_type = value.type 357 | if value_type == 0 or value_len <= 0: 358 | return None 359 | elif value_type == MarinePacketFieldValueType.INT: 360 | return value.int_value 361 | elif value_type == MarinePacketFieldValueType.UINT: 362 | return value.uint_value 363 | elif value_type == MarinePacketFieldValueType.BOOL: 364 | return bool(value.bool_value) 365 | elif value_type == MarinePacketFieldValueType.STR: 366 | return cls._safe_decode(value.str_value) 367 | elif value_type == MarinePacketFieldValueType.BYTES: 368 | return value.str_value[:value_len] 369 | elif value_type == MarinePacketFieldValueType.LIST: 370 | return [cls._load_field_value(v) for v in value.list_value[:value_len]] 371 | else: 372 | raise ValueError(f"Unknown value type {value_type}") 373 | 374 | @staticmethod 375 | def _safe_decode(raw_value: bytes) -> str: 376 | try: 377 | return raw_value.decode("utf-8") 378 | except UnicodeDecodeError: 379 | return raw_value.hex() 380 | 381 | def _add_or_get_filter( 382 | self, 383 | bpf: Optional[bytes] = None, 384 | display_filter: Optional[bytes] = None, 385 | fields: Optional[List[bytes]] = None, 386 | field_template_indices: Optional[List[int]] = None, 387 | encapsulation_type: int = encap_consts.ENCAP_ETHERNET, 388 | ) -> int: 389 | if fields is not None: 390 | fields_len = len(fields) 391 | fields_c_arr = (c_char_p * fields_len)(*fields) 392 | else: 393 | fields_len = 0 394 | fields_c_arr = None 395 | 396 | field_template_indices_c_arr = ( 397 | (c_int * fields_len)(*field_template_indices) 398 | if field_template_indices is not None 399 | else None 400 | ) 401 | err_msg = pointer(POINTER(c_char)()) 402 | filter_id = self._marine.marine_add_filter( 403 | bpf, 404 | display_filter, 405 | fields_c_arr, 406 | field_template_indices_c_arr, 407 | fields_len, 408 | encapsulation_type, 409 | err_msg, 410 | ) 411 | error = self._resolve_err_msg(err_msg) 412 | if filter_id < 0: 413 | if filter_id == c_int.in_dll(self._marine, "BAD_BPF_ERROR_CODE").value: 414 | raise BadBPFException(error) 415 | elif ( 416 | filter_id 417 | == c_int.in_dll(self._marine, "BAD_DISPLAY_FILTER_ERROR_CODE").value 418 | ): 419 | raise BadDisplayFilterException(error) 420 | elif ( 421 | filter_id 422 | == c_int.in_dll(self._marine, "INVALID_FIELD_ERROR_CODE").value 423 | ): 424 | raise InvalidFieldException(error) 425 | raise UnknownInternalException(error) 426 | return filter_id 427 | 428 | def __del__(self): 429 | if getattr(self, "_marine", None): 430 | self._marine.destroy_marine() 431 | 432 | def _expand_field_templates( 433 | self, fields: List[str], field_templates: Optional[Dict[str, List[str]]] 434 | ) -> Tuple[Tuple[str, ...], Optional[Tuple[int, ...]]]: 435 | if not field_templates: 436 | return tuple(fields), None 437 | 438 | field_template_key = ( 439 | tuple(fields), 440 | frozenset((key, tuple(value)) for key, value in field_templates.items()), 441 | ) 442 | if field_template_key in self._field_templates_cache: 443 | return self._field_templates_cache[field_template_key] 444 | else: 445 | expanded_with_indices = [ 446 | (possible_field, field_template_id) 447 | for field_template_id, field in enumerate(fields) 448 | for possible_field in field_templates.get(field, [field]) 449 | ] 450 | ret_value = tuple(zip(*expanded_with_indices)) 451 | self._field_templates_cache[field_template_key] = ret_value 452 | return ret_value 453 | 454 | def _detect_encap(self, fields: Optional[List[str]]) -> int: 455 | if not fields: 456 | return encap_consts.ENCAP_ETHERNET 457 | 458 | encap_key = frozenset(fields) 459 | if encap_key in self._encap_cache: 460 | return self._encap_cache[encap_key] 461 | 462 | fields_protocols = frozenset(field.split(".")[0].lower() for field in fields) 463 | if fields_protocols.intersection(self.WIFI_RADIO_PROTOCOLS): 464 | self._encap_cache[encap_key] = encap_consts.ENCAP_IEEE_802_11_RADIOTAP 465 | return encap_consts.ENCAP_IEEE_802_11_RADIOTAP 466 | 467 | self._encap_cache[encap_key] = encap_consts.ENCAP_ETHERNET 468 | return encap_consts.ENCAP_ETHERNET 469 | 470 | def report_fields(self) -> None: 471 | self._marine.marine_report_fields() 472 | -------------------------------------------------------------------------------- /tests/marine/test_marine.py: -------------------------------------------------------------------------------- 1 | """ 2 | Note: in order to run the tests, you must put libmarine.so next to the marine_fixtures.py file 3 | """ 4 | import pytest 5 | from typing import List, Union, Optional, Dict 6 | from marine.marine import Marine, MarineFieldsValidationResult 7 | from marine.marine_pool import MarinePool 8 | 9 | from pypacker.layer12 import ethernet, arp, radiotap, ieee80211, llc 10 | from pypacker.layer3 import ip, icmp 11 | from pypacker.layer4 import tcp, udp 12 | from pypacker.layer567 import dns, http, dhcp 13 | 14 | from marine import encap_consts 15 | from marine import BadBPFException, BadDisplayFilterException, InvalidFieldException 16 | 17 | 18 | # TODO: Add a test for FTP. 19 | 20 | 21 | def filter_and_parse( 22 | marine_or_marine_pool: Union[Marine, MarinePool], 23 | packet: bytes, 24 | packet_encapsulation: Optional[int], 25 | bpf_filter: Optional[str] = None, 26 | display_filter: Optional[str] = None, 27 | fields: Optional[List[str]] = None, 28 | field_templates: Optional[Dict[str, List[str]]] = None, 29 | ): 30 | return ( 31 | marine_or_marine_pool.filter_and_parse( 32 | packet, 33 | bpf_filter, 34 | display_filter, 35 | fields, 36 | packet_encapsulation, 37 | field_templates, 38 | ) 39 | if isinstance(marine_or_marine_pool, Marine) 40 | else marine_or_marine_pool.filter_and_parse( 41 | [packet], 42 | bpf_filter, 43 | display_filter, 44 | fields, 45 | packet_encapsulation, 46 | field_templates, 47 | )[0] 48 | ) 49 | 50 | 51 | def general_filter_and_parse_test( 52 | marine_or_marine_pool: Union[Marine, MarinePool], 53 | packet: bytes, 54 | packet_encapsulation: Optional[int], 55 | bpf_filter: Optional[str], 56 | display_filter: Optional[str], 57 | field_templates: Optional[Dict[str, List[str]]], 58 | expected_passed: bool, 59 | expected_output: Optional[Dict[str, Optional[Union[int, str]]]], 60 | ): 61 | expected_fields = list(expected_output.keys()) if expected_output else None 62 | passed, output = filter_and_parse( 63 | marine_or_marine_pool, 64 | packet, 65 | packet_encapsulation, 66 | bpf_filter, 67 | display_filter, 68 | expected_fields, 69 | field_templates, 70 | ) 71 | 72 | expected_output = ( 73 | {k: str(v) if v is not None else v for k, v in expected_output.items()} 74 | if expected_output 75 | else None 76 | ) 77 | 78 | assert expected_passed == passed 79 | assert expected_output == output 80 | 81 | 82 | def test_arp_packet_filter_and_parse(marine_or_marine_pool: Union[Marine, MarinePool]): 83 | src_mac = "00:00:00:12:34:ff" 84 | broadcast_mac = "ff:ff:ff:ff:ff:ff" 85 | src_ip = "21.53.78.255" 86 | target_ip = "10.0.0.255" 87 | bpf_filter = "arp" 88 | display_filter = "arp" 89 | expected_output = { 90 | "eth.src": src_mac, 91 | "eth.dst": broadcast_mac, 92 | "arp.src.hw_mac": src_mac, 93 | "arp.src.proto_ipv4": src_ip, 94 | "arp.dst.hw_mac": broadcast_mac, 95 | "arp.dst.proto_ipv4": target_ip, 96 | } 97 | packet = ethernet.Ethernet(src_s=src_mac, dst_s=broadcast_mac) + arp.ARP( 98 | sha_s=src_mac, spa_s=src_ip, tha_s=broadcast_mac, tpa_s=target_ip 99 | ) 100 | 101 | general_filter_and_parse_test( 102 | marine_or_marine_pool=marine_or_marine_pool, 103 | packet=packet.bin(), 104 | packet_encapsulation=encap_consts.ENCAP_ETHERNET, 105 | bpf_filter=bpf_filter, 106 | display_filter=display_filter, 107 | field_templates=None, 108 | expected_passed=True, 109 | expected_output=expected_output, 110 | ) 111 | 112 | 113 | def test_icmp_packet_filter_and_parse(marine_or_marine_pool: Union[Marine, MarinePool]): 114 | src_mac = "00:00:00:12:34:ff" 115 | dst_mac = "00:00:00:ff:00:1e" 116 | src_ip = "21.53.78.255" 117 | dst_ip = "10.0.0.255" 118 | icmp_echo_type = 8 119 | bpf_filter = "ip" 120 | display_filter = "icmp" 121 | expected_output = { 122 | "eth.src": src_mac, 123 | "eth.dst": dst_mac, 124 | "ip.src": src_ip, 125 | "ip.dst": dst_ip, 126 | "icmp.type": icmp_echo_type, 127 | } 128 | 129 | packet = ( 130 | ethernet.Ethernet(src_s=src_mac, dst_s=dst_mac) 131 | + ip.IP(src_s=src_ip, dst_s=dst_ip, p=ip.IP_PROTO_ICMP) 132 | + icmp.ICMP(type=icmp_echo_type) 133 | + icmp.ICMP.Echo() 134 | ) 135 | 136 | general_filter_and_parse_test( 137 | marine_or_marine_pool=marine_or_marine_pool, 138 | packet=packet.bin(), 139 | packet_encapsulation=encap_consts.ENCAP_ETHERNET, 140 | bpf_filter=bpf_filter, 141 | display_filter=display_filter, 142 | field_templates=None, 143 | expected_passed=True, 144 | expected_output=expected_output, 145 | ) 146 | 147 | 148 | def test_tcp_packet_filter_and_parse(marine_or_marine_pool: Union[Marine, MarinePool]): 149 | src_mac = "00:00:00:12:34:ff" 150 | dst_mac = "00:00:00:ff:00:1e" 151 | src_ip = "21.53.78.255" 152 | dst_ip = "10.0.0.255" 153 | src_port = 16424 154 | dst_port = 41799 155 | bpf_filter = "ip" 156 | display_filter = "tcp" 157 | expected_output = { 158 | "eth.src": src_mac, 159 | "eth.dst": dst_mac, 160 | "ip.src": src_ip, 161 | "ip.dst": dst_ip, 162 | "tcp.srcport": src_port, 163 | "tcp.dstport": dst_port, 164 | } 165 | 166 | packet = ( 167 | ethernet.Ethernet(src_s=src_mac, dst_s=dst_mac) 168 | + ip.IP(src_s=src_ip, dst_s=dst_ip) 169 | + tcp.TCP(sport=src_port, dport=dst_port) 170 | ) 171 | 172 | general_filter_and_parse_test( 173 | marine_or_marine_pool=marine_or_marine_pool, 174 | packet=packet.bin(), 175 | packet_encapsulation=encap_consts.ENCAP_ETHERNET, 176 | bpf_filter=bpf_filter, 177 | display_filter=display_filter, 178 | field_templates=None, 179 | expected_passed=True, 180 | expected_output=expected_output, 181 | ) 182 | 183 | 184 | def test_dns_packet_filter_and_parse(marine_or_marine_pool: Union[Marine, MarinePool]): 185 | src_mac = "00:00:00:12:34:ff" 186 | dst_mac = "00:00:00:ff:00:1e" 187 | src_ip = "21.53.78.255" 188 | dst_ip = "10.0.0.255" 189 | src_port = 16424 190 | dst_port = 53 191 | bpf_filter = "ip" 192 | display_filter = "dns" 193 | domain_name = "www.testwebsite.com" 194 | expected_output = { 195 | "eth.src": src_mac, 196 | "eth.dst": dst_mac, 197 | "ip.src": src_ip, 198 | "ip.dst": dst_ip, 199 | "udp.srcport": src_port, 200 | "udp.dstport": dst_port, 201 | "dns.qry.name": domain_name, 202 | } 203 | 204 | packet = ( 205 | ethernet.Ethernet(src_s=src_mac, dst_s=dst_mac) 206 | + ip.IP(src_s=src_ip, dst_s=dst_ip, p=ip.IP_PROTO_UDP) 207 | + udp.UDP(sport=src_port, dport=dst_port) 208 | + dns.DNS(queries=[dns.DNS.Query(name_s=domain_name, type=1, cls=1)]) 209 | ) 210 | 211 | general_filter_and_parse_test( 212 | marine_or_marine_pool=marine_or_marine_pool, 213 | packet=packet.bin(), 214 | packet_encapsulation=encap_consts.ENCAP_ETHERNET, 215 | bpf_filter=bpf_filter, 216 | display_filter=display_filter, 217 | field_templates=None, 218 | expected_passed=True, 219 | expected_output=expected_output, 220 | ) 221 | 222 | 223 | def test_dhcp_packet_filter_and_parse(marine_or_marine_pool: Union[Marine, MarinePool]): 224 | src_mac = "00:00:00:12:34:ff" 225 | dst_mac = "00:00:00:ff:00:1e" 226 | src_ip = "21.53.78.255" 227 | given_ip = "10.0.0.255" 228 | broadcast_ip = "255.255.255.255" 229 | src_port = 16424 230 | dst_port = 68 231 | bpf_filter = "ip" 232 | display_filter = "dhcp" 233 | expected_output = { 234 | "eth.src": src_mac, 235 | "eth.dst": dst_mac, 236 | "ip.src": src_ip, 237 | "ip.dst": broadcast_ip, 238 | "udp.srcport": src_port, 239 | "udp.dstport": dst_port, 240 | "dhcp.ip.your": given_ip, 241 | "dhcp.option.dhcp_server_id": src_ip, 242 | } 243 | 244 | packet = ( 245 | ethernet.Ethernet(src_s=src_mac, dst_s=dst_mac) 246 | + ip.IP(src_s=src_ip, dst_s=broadcast_ip, p=ip.IP_PROTO_UDP) 247 | + udp.UDP(sport=src_port, dport=dst_port) 248 | + dhcp.DHCP( 249 | yiaddr_s=given_ip, 250 | magic=dhcp.DHCP_MAGIC, 251 | opts=[ 252 | dhcp.DHCPOpt( 253 | type=dhcp.DHCP_OPT_SERVER_ID, 254 | len=4, 255 | body_bytes=bytes(int(num) for num in src_ip.split(".")), 256 | ) 257 | ], 258 | ) 259 | ) 260 | 261 | general_filter_and_parse_test( 262 | marine_or_marine_pool=marine_or_marine_pool, 263 | packet=packet.bin(), 264 | packet_encapsulation=encap_consts.ENCAP_ETHERNET, 265 | bpf_filter=bpf_filter, 266 | display_filter=display_filter, 267 | field_templates=None, 268 | expected_passed=True, 269 | expected_output=expected_output, 270 | ) 271 | 272 | 273 | def test_http_packet_filter_and_parse(marine_or_marine_pool: Union[Marine, MarinePool]): 274 | src_mac = "00:00:00:12:34:ff" 275 | dst_mac = "00:00:00:ff:00:1e" 276 | src_ip = "21.53.78.255" 277 | dst_ip = "10.0.0.255" 278 | src_port = 16424 279 | dst_port = 80 280 | http_type = "GET" 281 | uri = "/subtest/subsubtest" 282 | version = "HTTP/1.1" 283 | domain_name = "www.testwebsite.com" 284 | body = "random body \x09\xff\x00" 285 | bpf_filter = "ip" 286 | display_filter = "http" 287 | expected_output = { 288 | "eth.src": src_mac, 289 | "eth.dst": dst_mac, 290 | "ip.src": src_ip, 291 | "ip.dst": dst_ip, 292 | "tcp.srcport": src_port, 293 | "tcp.dstport": dst_port, 294 | "http.request.method": http_type, 295 | "http.request.uri": uri, 296 | "http.request.version": version, 297 | "http.host": domain_name, 298 | } 299 | packet = ( 300 | ethernet.Ethernet(src_s=src_mac, dst_s=dst_mac) 301 | + ip.IP(src_s=src_ip, dst_s=dst_ip) 302 | + tcp.TCP(sport=src_port, dport=dst_port) 303 | + http.HTTP( 304 | f"{http_type} {uri} {version}\r\nHost: {domain_name}\r\n\r\n{body}\r\n".encode() 305 | ) 306 | ) 307 | 308 | general_filter_and_parse_test( 309 | marine_or_marine_pool=marine_or_marine_pool, 310 | packet=packet.bin(), 311 | packet_encapsulation=encap_consts.ENCAP_ETHERNET, 312 | bpf_filter=bpf_filter, 313 | display_filter=display_filter, 314 | field_templates=None, 315 | expected_passed=True, 316 | expected_output=expected_output, 317 | ) 318 | 319 | 320 | def test_tcp_packet_filter_and_parse_with_field_template( 321 | marine_or_marine_pool: Union[Marine, MarinePool] 322 | ): 323 | src_mac = "00:00:00:12:34:ff" 324 | dst_mac = "00:00:00:ff:00:1e" 325 | src_ip = "21.53.78.255" 326 | dst_ip = "10.0.0.255" 327 | src_port = 16424 328 | dst_port = 41799 329 | bpf_filter = "ip" 330 | display_filter = "tcp" 331 | field_templates = {"macro.ip.src": ["ip.src", "ipv6.src"]} 332 | expected_output = { 333 | "eth.src": src_mac, 334 | "eth.dst": dst_mac, 335 | "macro.ip.src": src_ip, 336 | "ip.dst": dst_ip, 337 | "tcp.srcport": src_port, 338 | "tcp.dstport": dst_port, 339 | } 340 | 341 | packet = ( 342 | ethernet.Ethernet(src_s=src_mac, dst_s=dst_mac) 343 | + ip.IP(src_s=src_ip, dst_s=dst_ip) 344 | + tcp.TCP(sport=src_port, dport=dst_port) 345 | ) 346 | 347 | general_filter_and_parse_test( 348 | marine_or_marine_pool=marine_or_marine_pool, 349 | packet=packet.bin(), 350 | packet_encapsulation=encap_consts.ENCAP_ETHERNET, 351 | bpf_filter=bpf_filter, 352 | display_filter=display_filter, 353 | field_templates=field_templates, 354 | expected_passed=True, 355 | expected_output=expected_output, 356 | ) 357 | 358 | 359 | def test_tcp_packet_filter_and_parse_with_multiple_field_templates( 360 | marine_or_marine_pool: Union[Marine, MarinePool] 361 | ): 362 | src_mac = "00:00:00:12:34:ff" 363 | dst_mac = "00:00:00:ff:00:1e" 364 | src_ip = "21.53.78.255" 365 | dst_ip = "10.0.0.255" 366 | src_port = 16424 367 | dst_port = 41799 368 | bpf_filter = "ip" 369 | display_filter = "tcp" 370 | field_templates = { 371 | "macro.ip.src": ["ip.src", "ipv6.src"], 372 | "macro.ip.dst": ["ip.dst", "ipv6.dst"], 373 | "macro.srcport": ["tcp.srcport", "udp.srcport"], 374 | "macro.dstport": ["tcp.dstport", "udp.dstport"], 375 | } 376 | expected_output = { 377 | "eth.src": src_mac, 378 | "eth.dst": dst_mac, 379 | "macro.ip.src": src_ip, 380 | "macro.ip.dst": dst_ip, 381 | "macro.srcport": src_port, 382 | "macro.dstport": dst_port, 383 | } 384 | 385 | packet = ( 386 | ethernet.Ethernet(src_s=src_mac, dst_s=dst_mac) 387 | + ip.IP(src_s=src_ip, dst_s=dst_ip) 388 | + tcp.TCP(sport=src_port, dport=dst_port) 389 | ) 390 | 391 | general_filter_and_parse_test( 392 | marine_or_marine_pool=marine_or_marine_pool, 393 | packet=packet.bin(), 394 | packet_encapsulation=encap_consts.ENCAP_ETHERNET, 395 | bpf_filter=bpf_filter, 396 | display_filter=display_filter, 397 | field_templates=field_templates, 398 | expected_passed=True, 399 | expected_output=expected_output, 400 | ) 401 | 402 | 403 | def test_tcp_packet_filter_and_parse_with_field_template_with_non_existing_field_first( 404 | marine_or_marine_pool: Union[Marine, MarinePool] 405 | ): 406 | src_mac = "00:00:00:12:34:ff" 407 | dst_mac = "00:00:00:ff:00:1e" 408 | src_ip = "21.53.78.255" 409 | dst_ip = "10.0.0.255" 410 | src_port = 16424 411 | dst_port = 41799 412 | bpf_filter = "ip" 413 | display_filter = "tcp" 414 | field_templates = {"macro.ip.src": ["ipv6.src", "ip.src"]} 415 | expected_output = { 416 | "macro.ip.src": src_ip, 417 | } 418 | 419 | packet = ( 420 | ethernet.Ethernet(src_s=src_mac, dst_s=dst_mac) 421 | + ip.IP(src_s=src_ip, dst_s=dst_ip) 422 | + tcp.TCP(sport=src_port, dport=dst_port) 423 | ) 424 | 425 | general_filter_and_parse_test( 426 | marine_or_marine_pool=marine_or_marine_pool, 427 | packet=packet.bin(), 428 | packet_encapsulation=encap_consts.ENCAP_ETHERNET, 429 | bpf_filter=bpf_filter, 430 | display_filter=display_filter, 431 | field_templates=field_templates, 432 | expected_passed=True, 433 | expected_output=expected_output, 434 | ) 435 | 436 | 437 | def test_tcp_packet_filter_and_parse_with_multiple_different_field_templates( 438 | marine_or_marine_pool: Union[Marine, MarinePool] 439 | ): 440 | src_mac = "00:00:00:12:34:ff" 441 | dst_mac = "00:00:00:ff:00:1e" 442 | src_ip = "21.53.78.255" 443 | dst_ip = "10.0.0.255" 444 | src_port = 16424 445 | dst_port = 41799 446 | bpf_filter = "ip" 447 | display_filter = "tcp" 448 | first_field_templates = { 449 | "macro.ip.src": ["ip.src", "ipv6.src"], 450 | "macro.ip.dst": ["ip.dst", "ipv6.dst"], 451 | } 452 | second_field_templates = { 453 | "macro.ip.src": ["ip.src", "ipv6.src"], 454 | "macro.ip.dst": ["ip.dst", "ipv6.dst"], 455 | "macro.srcport": ["tcp.srcport", "udp.srcport"], 456 | "macro.dstport": ["tcp.dstport", "udp.dstport"], 457 | } 458 | third_field_templates = { 459 | "macro.ip.src": ["ip.src", "ipv6.src"], 460 | "macro.ip.dst": ["ip.dst", "ipv6.dst"], 461 | "macro.dstport": ["tcp.dstport", "udp.dstport"], 462 | } 463 | first_expected_output = { 464 | "eth.src": src_mac, 465 | "eth.dst": dst_mac, 466 | "macro.ip.src": src_ip, 467 | "macro.ip.dst": dst_ip, 468 | "tcp.srcport": src_port, 469 | "tcp.dstport": dst_port, 470 | } 471 | second_expected_output = { 472 | "eth.src": src_mac, 473 | "eth.dst": dst_mac, 474 | "macro.ip.src": src_ip, 475 | "macro.ip.dst": dst_ip, 476 | "macro.srcport": src_port, 477 | "macro.dstport": dst_port, 478 | } 479 | third_expected_output = { 480 | "eth.src": src_mac, 481 | "eth.dst": dst_mac, 482 | "macro.ip.src": src_ip, 483 | "macro.ip.dst": dst_ip, 484 | "tcp.srcport": src_port, 485 | "macro.dstport": dst_port, 486 | } 487 | 488 | packet = ( 489 | ethernet.Ethernet(src_s=src_mac, dst_s=dst_mac) 490 | + ip.IP(src_s=src_ip, dst_s=dst_ip) 491 | + tcp.TCP(sport=src_port, dport=dst_port) 492 | ) 493 | 494 | general_filter_and_parse_test( 495 | marine_or_marine_pool=marine_or_marine_pool, 496 | packet=packet.bin(), 497 | packet_encapsulation=encap_consts.ENCAP_ETHERNET, 498 | bpf_filter=bpf_filter, 499 | display_filter=display_filter, 500 | field_templates=first_field_templates, 501 | expected_passed=True, 502 | expected_output=first_expected_output, 503 | ) 504 | general_filter_and_parse_test( 505 | marine_or_marine_pool=marine_or_marine_pool, 506 | packet=packet.bin(), 507 | packet_encapsulation=encap_consts.ENCAP_ETHERNET, 508 | bpf_filter=bpf_filter, 509 | display_filter=display_filter, 510 | field_templates=second_field_templates, 511 | expected_passed=True, 512 | expected_output=second_expected_output, 513 | ) 514 | general_filter_and_parse_test( 515 | marine_or_marine_pool=marine_or_marine_pool, 516 | packet=packet.bin(), 517 | packet_encapsulation=encap_consts.ENCAP_ETHERNET, 518 | bpf_filter=bpf_filter, 519 | display_filter=display_filter, 520 | field_templates=third_field_templates, 521 | expected_passed=True, 522 | expected_output=third_expected_output, 523 | ) 524 | 525 | 526 | def test_tcp_packet_filter_and_parse_with_multiple_field_templates_sharing_fields( 527 | marine_or_marine_pool: Union[Marine, MarinePool] 528 | ): 529 | src_mac = "00:00:00:12:34:ff" 530 | dst_mac = "00:00:00:ff:00:1e" 531 | src_ip = "21.53.78.255" 532 | dst_ip = "10.0.0.255" 533 | src_port = 16424 534 | dst_port = 41799 535 | bpf_filter = "ip" 536 | display_filter = "tcp" 537 | field_templates = { 538 | "macro.ip.src": ["ip.src", "ipv6.src"], 539 | "test.macro.ip.src": ["ipv6.src", "arp.src.proto_ipv4", "ip.src"], 540 | "macro.ip.dst": ["ip.dst", "ipv6.dst"], 541 | "macro.srcport": ["tcp.srcport", "udp.srcport"], 542 | "macro.dstport": ["tcp.dstport", "udp.dstport"], 543 | } 544 | expected_output = { 545 | "eth.src": src_mac, 546 | "eth.dst": dst_mac, 547 | "macro.ip.src": src_ip, 548 | "test.macro.ip.src": src_ip, 549 | "macro.ip.dst": dst_ip, 550 | "macro.srcport": src_port, 551 | "macro.dstport": dst_port, 552 | } 553 | 554 | packet = ( 555 | ethernet.Ethernet(src_s=src_mac, dst_s=dst_mac) 556 | + ip.IP(src_s=src_ip, dst_s=dst_ip) 557 | + tcp.TCP(sport=src_port, dport=dst_port) 558 | ) 559 | 560 | general_filter_and_parse_test( 561 | marine_or_marine_pool=marine_or_marine_pool, 562 | packet=packet.bin(), 563 | packet_encapsulation=encap_consts.ENCAP_ETHERNET, 564 | bpf_filter=bpf_filter, 565 | display_filter=display_filter, 566 | field_templates=field_templates, 567 | expected_passed=True, 568 | expected_output=expected_output, 569 | ) 570 | 571 | 572 | def test_radiotap_packet_filter_and_parse( 573 | marine_or_marine_pool: Union[Marine, MarinePool] 574 | ): 575 | src_ip = "78.78.78.255" 576 | dst_ip = "10.0.0.255" 577 | bpf_filter = "ip" 578 | display_filter = "ip" 579 | expected_output = { 580 | "radiotap.present.tsft": 0, 581 | "radiotap.present.channel": 1, 582 | "radiotap.present.rate": 1, 583 | "wlan.fc.type_subtype": 40, 584 | "llc.type": "0x00000800", 585 | "ip.src": src_ip, 586 | "ip.dst": dst_ip, 587 | } 588 | 589 | packet = ( 590 | radiotap.Radiotap(present_flags=radiotap.CHANNEL_MASK + radiotap.RATE_MASK) 591 | + ieee80211.IEEE80211(framectl=0x8801) 592 | + ieee80211.IEEE80211.Dataframe(sec_param=None) 593 | + llc.LLC( 594 | dsap=170, ssap=170, ctrl=3, snap=int.to_bytes(llc.LLC_TYPE_IP, 5, "big") 595 | ) 596 | + ip.IP(src_s=src_ip, dst_s=dst_ip) 597 | ) 598 | 599 | general_filter_and_parse_test( 600 | marine_or_marine_pool=marine_or_marine_pool, 601 | packet=packet.bin(), 602 | packet_encapsulation=encap_consts.ENCAP_IEEE_802_11_RADIOTAP, 603 | bpf_filter=bpf_filter, 604 | display_filter=display_filter, 605 | field_templates=None, 606 | expected_passed=True, 607 | expected_output=expected_output, 608 | ) 609 | 610 | 611 | def test_radiotap_packet_filter_and_parse_failing_wrong_encapsulation( 612 | marine_or_marine_pool: Union[Marine, MarinePool] 613 | ): 614 | src_ip = "78.78.78.255" 615 | dst_ip = "10.0.0.255" 616 | bpf_filter = "ip" 617 | display_filter = "ip" 618 | 619 | packet = ( 620 | radiotap.Radiotap(present_flags=radiotap.CHANNEL_MASK + radiotap.RATE_MASK) 621 | + ieee80211.IEEE80211(framectl=0x8801) 622 | + ieee80211.IEEE80211.Dataframe(sec_param=None) 623 | + llc.LLC( 624 | dsap=170, ssap=170, ctrl=3, snap=int.to_bytes(llc.LLC_TYPE_IP, 5, "big") 625 | ) 626 | + ip.IP(src_s=src_ip, dst_s=dst_ip) 627 | ) 628 | 629 | general_filter_and_parse_test( 630 | marine_or_marine_pool=marine_or_marine_pool, 631 | packet=packet.bin(), 632 | packet_encapsulation=encap_consts.ENCAP_ETHERNET, 633 | bpf_filter=bpf_filter, 634 | display_filter=display_filter, 635 | field_templates=None, 636 | expected_passed=False, 637 | expected_output={}, 638 | ) 639 | 640 | 641 | def test_radiotap_packet_filter_and_parse_parsing_wrong_encapsulation( 642 | marine_or_marine_pool: Union[Marine, MarinePool] 643 | ): 644 | src_ip = "78.78.78.255" 645 | dst_ip = "10.0.0.255" 646 | expected_output = { 647 | "radiotap.present.tsft": None, 648 | "radiotap.present.channel": None, 649 | "radiotap.present.rate": None, 650 | "wlan.fc.type_subtype": None, 651 | "llc.type": None, 652 | "ip.src": None, 653 | "ip.dst": None, 654 | } 655 | 656 | packet = ( 657 | radiotap.Radiotap(present_flags=radiotap.CHANNEL_MASK + radiotap.RATE_MASK) 658 | + ieee80211.IEEE80211(framectl=0x8801) 659 | + ieee80211.IEEE80211.Dataframe(sec_param=None) 660 | + llc.LLC( 661 | dsap=170, ssap=170, ctrl=3, snap=int.to_bytes(llc.LLC_TYPE_IP, 5, "big") 662 | ) 663 | + ip.IP(src_s=src_ip, dst_s=dst_ip) 664 | ) 665 | 666 | general_filter_and_parse_test( 667 | marine_or_marine_pool=marine_or_marine_pool, 668 | packet=packet.bin(), 669 | packet_encapsulation=encap_consts.ENCAP_ETHERNET, 670 | bpf_filter=None, 671 | display_filter=None, 672 | field_templates=None, 673 | expected_passed=True, 674 | expected_output=expected_output, 675 | ) 676 | 677 | 678 | def test_tcp_packet_filter_and_parse_with_auto_encap( 679 | marine_or_marine_pool: Union[Marine, MarinePool] 680 | ): 681 | src_mac = "00:00:00:12:34:ff" 682 | dst_mac = "00:00:00:ff:00:1e" 683 | src_ip = "21.53.78.255" 684 | dst_ip = "10.0.0.255" 685 | src_port = 16424 686 | dst_port = 41799 687 | bpf_filter = "ip" 688 | display_filter = "tcp" 689 | expected_output = { 690 | "eth.src": src_mac, 691 | "eth.dst": dst_mac, 692 | "ip.src": src_ip, 693 | "ip.dst": dst_ip, 694 | "tcp.srcport": src_port, 695 | "tcp.dstport": dst_port, 696 | } 697 | 698 | packet = ( 699 | ethernet.Ethernet(src_s=src_mac, dst_s=dst_mac) 700 | + ip.IP(src_s=src_ip, dst_s=dst_ip) 701 | + tcp.TCP(sport=src_port, dport=dst_port) 702 | ) 703 | 704 | general_filter_and_parse_test( 705 | marine_or_marine_pool=marine_or_marine_pool, 706 | packet=packet.bin(), 707 | packet_encapsulation=None, 708 | bpf_filter=bpf_filter, 709 | display_filter=display_filter, 710 | field_templates=None, 711 | expected_passed=True, 712 | expected_output=expected_output, 713 | ) 714 | 715 | 716 | def test_radiotap_packet_filter_and_parse_with_auto_encap( 717 | marine_or_marine_pool: Union[Marine, MarinePool] 718 | ): 719 | src_ip = "78.78.78.255" 720 | dst_ip = "10.0.0.255" 721 | bpf_filter = "ip" 722 | display_filter = "ip" 723 | expected_output = { 724 | "radiotap.present.tsft": 0, 725 | "radiotap.present.channel": 1, 726 | "radiotap.present.rate": 1, 727 | "wlan.fc.type_subtype": 40, 728 | "llc.type": "0x00000800", 729 | "ip.src": src_ip, 730 | "ip.dst": dst_ip, 731 | } 732 | 733 | packet = ( 734 | radiotap.Radiotap(present_flags=radiotap.CHANNEL_MASK + radiotap.RATE_MASK) 735 | + ieee80211.IEEE80211(framectl=0x8801) 736 | + ieee80211.IEEE80211.Dataframe(sec_param=None) 737 | + llc.LLC( 738 | dsap=170, ssap=170, ctrl=3, snap=int.to_bytes(llc.LLC_TYPE_IP, 5, "big") 739 | ) 740 | + ip.IP(src_s=src_ip, dst_s=dst_ip) 741 | ) 742 | 743 | general_filter_and_parse_test( 744 | marine_or_marine_pool=marine_or_marine_pool, 745 | packet=packet.bin(), 746 | packet_encapsulation=None, 747 | bpf_filter=bpf_filter, 748 | display_filter=display_filter, 749 | field_templates=None, 750 | expected_passed=True, 751 | expected_output=expected_output, 752 | ) 753 | 754 | 755 | def test_radiotap_packet_filter_and_parse_with_auto_encap_in_field_template( 756 | marine_or_marine_pool: Union[Marine, MarinePool] 757 | ): 758 | src_ip = "78.78.78.255" 759 | dst_ip = "10.0.0.255" 760 | bpf_filter = "ip" 761 | display_filter = "ip" 762 | field_templates = {"macro.dummy": ["radiotap.present.rate"]} 763 | expected_output = { 764 | "macro.dummy": 1, 765 | "ip.src": src_ip, 766 | "ip.dst": dst_ip, 767 | } 768 | 769 | packet = ( 770 | radiotap.Radiotap(present_flags=radiotap.CHANNEL_MASK + radiotap.RATE_MASK) 771 | + ieee80211.IEEE80211(framectl=0x8801) 772 | + ieee80211.IEEE80211.Dataframe(sec_param=None) 773 | + llc.LLC( 774 | dsap=170, ssap=170, ctrl=3, snap=int.to_bytes(llc.LLC_TYPE_IP, 5, "big") 775 | ) 776 | + ip.IP(src_s=src_ip, dst_s=dst_ip) 777 | ) 778 | 779 | general_filter_and_parse_test( 780 | marine_or_marine_pool=marine_or_marine_pool, 781 | packet=packet.bin(), 782 | packet_encapsulation=None, 783 | bpf_filter=bpf_filter, 784 | display_filter=display_filter, 785 | field_templates=field_templates, 786 | expected_passed=True, 787 | expected_output=expected_output, 788 | ) 789 | 790 | 791 | def test_filter_and_parse_without_filters( 792 | marine_or_marine_pool: Union[Marine, MarinePool] 793 | ): 794 | src_mac = "00:00:00:12:34:ff" 795 | dst_mac = "00:00:00:ff:00:1e" 796 | src_ip = "21.53.78.255" 797 | dst_ip = "10.0.0.255" 798 | src_port = 16424 799 | dst_port = 41799 800 | expected_output = { 801 | "eth.src": src_mac, 802 | "eth.dst": dst_mac, 803 | "ip.src": src_ip, 804 | "ip.dst": dst_ip, 805 | "tcp.srcport": src_port, 806 | "tcp.dstport": dst_port, 807 | } 808 | 809 | packet = ( 810 | ethernet.Ethernet(src_s=src_mac, dst_s=dst_mac) 811 | + ip.IP(src_s=src_ip, dst_s=dst_ip) 812 | + tcp.TCP(sport=src_port, dport=dst_port) 813 | ) 814 | 815 | general_filter_and_parse_test( 816 | marine_or_marine_pool=marine_or_marine_pool, 817 | packet=packet.bin(), 818 | packet_encapsulation=encap_consts.ENCAP_ETHERNET, 819 | bpf_filter=None, 820 | display_filter=None, 821 | field_templates=None, 822 | expected_passed=True, 823 | expected_output=expected_output, 824 | ) 825 | 826 | 827 | def test_filter_and_parse_without_fields( 828 | marine_or_marine_pool: Union[Marine, MarinePool], tcp_packet: bytes 829 | ): 830 | general_filter_and_parse_test( 831 | marine_or_marine_pool=marine_or_marine_pool, 832 | packet=tcp_packet, 833 | packet_encapsulation=encap_consts.ENCAP_ETHERNET, 834 | bpf_filter="ip", 835 | display_filter="tcp", 836 | field_templates=None, 837 | expected_passed=True, 838 | expected_output=None, 839 | ) 840 | 841 | 842 | def test_packet_doesnt_pass_filter_because_of_bpf( 843 | marine_instance: Marine, 844 | tcp_packet: bytes, 845 | extracted_fields_from_tcp_packet: List[str], 846 | ): 847 | passed, output = filter_and_parse( 848 | marine_instance, 849 | tcp_packet, 850 | encap_consts.ENCAP_ETHERNET, 851 | "arp", 852 | fields=extracted_fields_from_tcp_packet, 853 | ) 854 | 855 | assert not passed 856 | assert output is None 857 | 858 | 859 | def test_packet_doesnt_pass_filter_because_of_display_filter( 860 | marine_instance: Marine, 861 | tcp_packet: bytes, 862 | extracted_fields_from_tcp_packet: List[str], 863 | ): 864 | passed, output = filter_and_parse( 865 | marine_instance, 866 | tcp_packet, 867 | encap_consts.ENCAP_ETHERNET, 868 | display_filter="udp", 869 | fields=extracted_fields_from_tcp_packet, 870 | ) 871 | 872 | assert not passed 873 | assert output is None 874 | 875 | 876 | def test_illegal_bpf_in_filter_and_parse( 877 | marine_or_marine_pool: Union[Marine, MarinePool], tcp_packet: bytes 878 | ): 879 | with pytest.raises(BadBPFException, match="syntax error"): 880 | filter_and_parse( 881 | marine_or_marine_pool, 882 | tcp_packet, 883 | encap_consts.ENCAP_ETHERNET, 884 | bpf_filter="what is this bpf?", 885 | ) 886 | 887 | 888 | def test_illegal_display_filter_in_filter_and_parse( 889 | marine_or_marine_pool: Union[Marine, MarinePool], tcp_packet: bytes 890 | ): 891 | with pytest.raises( 892 | BadDisplayFilterException, match="neither a field nor a protocol name" 893 | ): 894 | filter_and_parse( 895 | marine_or_marine_pool, 896 | tcp_packet, 897 | encap_consts.ENCAP_ETHERNET, 898 | display_filter="illegal_filter", 899 | ) 900 | 901 | 902 | def test_illegal_fields_in_filter_and_parse( 903 | marine_or_marine_pool: Union[Marine, MarinePool], tcp_packet: bytes 904 | ): 905 | with pytest.raises(InvalidFieldException) as excinfo: 906 | filter_and_parse( 907 | marine_or_marine_pool, 908 | tcp_packet, 909 | encap_consts.ENCAP_ETHERNET, 910 | fields=["illegal_field_1", "illegal_field_2", "ip.src"], 911 | ) 912 | err_msg = str(excinfo) 913 | 914 | assert "illegal_field_1" in err_msg 915 | assert "illegal_field_2" in err_msg 916 | assert "ip.src" not in err_msg 917 | 918 | 919 | def test_filter_and_parse_with_no_parameters( 920 | marine_or_marine_pool: Union[Marine, MarinePool], tcp_packet: bytes 921 | ): 922 | with pytest.raises(ValueError, match="must be passed"): 923 | filter_and_parse(marine_or_marine_pool, tcp_packet, encap_consts.ENCAP_ETHERNET) 924 | 925 | 926 | def test_validate_bpf_success(marine_instance: Union[Marine, MarinePool]): 927 | assert marine_instance.validate_bpf("arp") 928 | 929 | 930 | def test_validate_bpf_failure(marine_instance: Union[Marine, MarinePool]): 931 | res = marine_instance.validate_bpf("what is this bpf?") 932 | assert not res 933 | assert "syntax error" in res.error 934 | 935 | 936 | def test_validate_bpf_failure_on_encapsulation(marine_instance: Marine): 937 | bpf = "ether host 00:01:45:aa:aa:aa" 938 | assert marine_instance.validate_bpf(bpf, encap_consts.ENCAP_ETHERNET) 939 | assert not marine_instance.validate_bpf(bpf, 9) # PPP encapsulation type 940 | 941 | 942 | def test_validate_display_filter_success(marine_instance: Marine): 943 | assert marine_instance.validate_display_filter("tcp") 944 | 945 | 946 | def test_validate_display_filter_failure(marine_instance: Marine): 947 | res = marine_instance.validate_display_filter("illegal_filter") 948 | assert not res 949 | assert "neither a field nor a protocol" in res.error 950 | 951 | 952 | def test_get_epan_auto_reset_count(marine_instance: Marine, epan_auto_reset_count: int): 953 | assert marine_instance.epan_auto_reset_count == epan_auto_reset_count 954 | 955 | 956 | def test_set_epan_auto_reset_count(marine_instance: Marine): 957 | SOME_VALUE = 1 958 | assert marine_instance.epan_auto_reset_count != SOME_VALUE 959 | marine_instance.epan_auto_reset_count = SOME_VALUE 960 | assert marine_instance.epan_auto_reset_count == SOME_VALUE 961 | 962 | 963 | def test_validate_fields_success(marine_instance: Marine): 964 | assert marine_instance.validate_fields(["ip.src", "eth.dst"]) 965 | 966 | 967 | def test_validate_fields_failure(marine_instance: Marine): 968 | result = marine_instance.validate_fields( 969 | ["ip.src", "this.field.is.bad", "eth.dst", "another.bad"] 970 | ) 971 | assert not result 972 | assert set(result.errors) == {"this.field.is.bad", "another.bad"} 973 | 974 | 975 | def test_validate_fields_with_field_template(marine_instance: Marine): 976 | assert marine_instance.validate_fields( 977 | ["macro.ip.src"], {"macro.ip.src": ["ip.src", "ipv6.src"]} 978 | ) 979 | 980 | 981 | def test_validate_fields_with_field_of_length_23(marine_instance: Marine): 982 | """ 983 | We did not allocate err_msg in parse_output_fields in marine.c properly, 984 | and fields of length (23 + 16*n) specifically would cause an error. 985 | This error was probably was caused by overriding memory that was used by the allocator. 986 | """ 987 | fields = ["a" * 23] 988 | assert marine_instance.validate_fields(fields) == MarineFieldsValidationResult( 989 | False, fields 990 | ) 991 | 992 | 993 | def test_validate_fields_errors_order(marine_instance: Marine): 994 | fields = ["wrong", "this too", "wtf"] 995 | assert marine_instance.validate_fields(fields).errors == fields 996 | 997 | 998 | def test_auto_encap_on_empty_fields(marine_instance: Marine): 999 | assert marine_instance._detect_encap(None) == encap_consts.ENCAP_ETHERNET 1000 | 1001 | 1002 | def test_auto_encap_ethernet(marine_instance: Marine): 1003 | assert ( 1004 | marine_instance._detect_encap(["ip.src", "ip.dst"]) 1005 | == encap_consts.ENCAP_ETHERNET 1006 | ) 1007 | 1008 | 1009 | def test_auto_encap_wireless(marine_instance: Marine): 1010 | assert ( 1011 | marine_instance._detect_encap(["ip.src", "ip.dst", "radiotap.channel"]) 1012 | == encap_consts.ENCAP_IEEE_802_11_RADIOTAP 1013 | ) 1014 | 1015 | 1016 | def test_report_fields(marine_instance: Marine, capfd: pytest.CaptureFixture): 1017 | marine_instance.report_fields() 1018 | out = capfd.readouterr().out 1019 | assert "eth.src" in out 1020 | assert "ip.src" in out 1021 | 1022 | 1023 | def test_parse_fields_preserves_order(marine_instance: Marine, tcp_packet: bytes): 1024 | assert marine_instance.parse(tcp_packet, fields=["udp.srcport", "tcp.srcport"]) == { 1025 | "udp.srcport": None, 1026 | "tcp.srcport": "16424", 1027 | } 1028 | 1029 | 1030 | def test_parse_all_fields_int_value(tcp_packet_fields): 1031 | tcp_source_port = tcp_packet_fields["tcp"]["tcp.srcport"] 1032 | assert isinstance(tcp_source_port, int) 1033 | assert tcp_source_port == 16424 1034 | 1035 | 1036 | def test_parse_all_fields_str_value(tcp_packet_fields): 1037 | ip_src = tcp_packet_fields["ip"]["ip.src"] 1038 | assert isinstance(ip_src, str) 1039 | assert ip_src == "10.0.0.255" 1040 | 1041 | 1042 | def test_parse_all_fields_list_value(tcp_packet_fields): 1043 | ip_addr = tcp_packet_fields["ip"]["ip.addr"] 1044 | assert isinstance(ip_addr, list) 1045 | assert "10.0.0.255" in ip_addr 1046 | assert "21.53.78.255" in ip_addr 1047 | 1048 | 1049 | def test_parse_all_fields_bool_value(tcp_packet_fields): 1050 | tcp_ack_flag = tcp_packet_fields["tcp"]["tcp.flags_tree"]["tcp.flags.ack"] 1051 | tcp_fin_flag = tcp_packet_fields["tcp"]["tcp.flags_tree"]["tcp.flags.fin"] 1052 | assert isinstance(tcp_ack_flag, bool) 1053 | assert isinstance(tcp_fin_flag, bool) 1054 | assert tcp_ack_flag 1055 | assert not tcp_fin_flag 1056 | 1057 | 1058 | def test_parse_all_fields_bytes_value(tcp_packet_fields, tcp_payload): 1059 | parsed_tcp_payload = tcp_packet_fields["tcp"]["tcp.payload"] 1060 | assert isinstance(parsed_tcp_payload, bytes) 1061 | assert parsed_tcp_payload == tcp_payload 1062 | 1063 | 1064 | def test_boolean_preference(marine_instance): 1065 | marine_instance.prefs.set_bool("tcp", "reassemble_out_of_order", True) 1066 | 1067 | assert marine_instance.prefs.get_bool("tcp", "reassemble_out_of_order") 1068 | 1069 | 1070 | def test_uint_preference(marine_instance): 1071 | marine_instance.prefs.set_uint("amqp", "tls.port", 1234) 1072 | 1073 | assert marine_instance.prefs.get_uint("amqp", "tls.port") == 1234 1074 | 1075 | 1076 | def test_string_preference(marine_instance): 1077 | marine_instance.prefs.set_str("lbmr", "mc_outgoing_address", "1234,1234,1234,1234") 1078 | 1079 | assert ( 1080 | marine_instance.prefs.get_str("lbmr", "mc_outgoing_address") 1081 | == "1234,1234,1234,1234" 1082 | ) 1083 | 1084 | 1085 | def test_value_error_for_unknown_module_name(marine_instance): 1086 | with pytest.raises(ValueError): 1087 | marine_instance.prefs.set_str("marine", "test", "test") 1088 | 1089 | 1090 | def test_value_error_for_unknown_pref_name(marine_instance): 1091 | with pytest.raises(ValueError): 1092 | marine_instance.prefs.set_str("amqp", "marine", "test") 1093 | 1094 | 1095 | def test_type_error_for_invalid_pref_type(marine_instance): 1096 | with pytest.raises(TypeError): 1097 | marine_instance.prefs.set_str("amqp", "tls.port", "1234") 1098 | --------------------------------------------------------------------------------