├── .github └── workflows │ ├── ci.yml │ └── rustdoc.yml ├── .gitignore ├── Cargo.toml ├── INSTALL.md ├── LICENSE.md ├── README.md ├── configs ├── offline.toml ├── online-vdev.toml └── online.toml ├── core ├── Cargo.toml ├── build.rs └── src │ ├── config.rs │ ├── conntrack │ ├── conn │ │ ├── conn_info.rs │ │ ├── mod.rs │ │ ├── tcp_conn │ │ │ ├── mod.rs │ │ │ └── reassembly.rs │ │ └── udp_conn │ │ │ └── mod.rs │ ├── conn_id.rs │ ├── mod.rs │ ├── pdu.rs │ └── timerwheel.rs │ ├── dpdk │ ├── dpdk_headers.h │ ├── inlined.c │ └── mod.rs │ ├── filter │ ├── actions.rs │ ├── ast.rs │ ├── datatypes.rs │ ├── grammar.pest │ ├── hardware │ │ ├── flow_action.rs │ │ ├── flow_attr.rs │ │ ├── flow_item.rs │ │ └── mod.rs │ ├── macros.rs │ ├── mod.rs │ ├── parser.rs │ ├── pattern.rs │ ├── ptree.rs │ └── ptree_flat.rs │ ├── lcore │ ├── mod.rs │ ├── monitor.rs │ ├── ring.rs │ └── rx_core.rs │ ├── lib.rs │ ├── memory │ ├── mbuf.rs │ ├── mempool.rs │ └── mod.rs │ ├── port │ ├── info.rs │ ├── mod.rs │ └── statistics.rs │ ├── protocols │ ├── mod.rs │ ├── packet │ │ ├── ethernet.rs │ │ ├── ipv4.rs │ │ ├── ipv6.rs │ │ ├── mod.rs │ │ ├── tcp.rs │ │ └── udp.rs │ └── stream │ │ ├── conn │ │ ├── layer3.rs │ │ ├── layer4.rs │ │ └── mod.rs │ │ ├── dns │ │ ├── mod.rs │ │ ├── parser.rs │ │ └── transaction.rs │ │ ├── http │ │ ├── mod.rs │ │ ├── parser.rs │ │ └── transaction.rs │ │ ├── mod.rs │ │ ├── quic │ │ ├── crypto.rs │ │ ├── frame.rs │ │ ├── header.rs │ │ ├── mod.rs │ │ └── parser.rs │ │ ├── ssh │ │ ├── handshake.rs │ │ ├── mod.rs │ │ └── parser.rs │ │ └── tls │ │ ├── handshake.rs │ │ ├── mod.rs │ │ └── parser.rs │ ├── runtime │ ├── mod.rs │ ├── offline.rs │ └── online.rs │ ├── stats │ ├── mod.rs │ └── prometheus.rs │ ├── subscription │ └── mod.rs │ ├── timing │ ├── macros.rs │ ├── mod.rs │ └── timer.rs │ └── utils │ ├── base64.rs │ ├── mod.rs │ └── types.rs ├── datatypes ├── Cargo.toml └── src │ ├── conn_fts.rs │ ├── connection.rs │ ├── dns_transaction.rs │ ├── http_transaction.rs │ ├── lib.rs │ ├── packet.rs │ ├── packet_list.rs │ ├── quic_stream.rs │ ├── ssh_handshake.rs │ ├── static_type.rs │ ├── streaming.rs │ ├── tls_handshake.rs │ └── typedefs.rs ├── docs ├── DEVELOPER.md └── figures │ ├── combined_states_example.png │ ├── filtergen_example.png │ ├── tls_single_flow_multi.png │ └── tls_sub_ex_code.png ├── examples ├── basic │ ├── Cargo.toml │ ├── README.md │ └── src │ │ └── main.rs ├── basic_file │ ├── Cargo.toml │ ├── README.md │ ├── spec.toml │ └── src │ │ └── main.rs ├── filter_stats │ ├── Cargo.toml │ ├── README.md │ ├── spec.toml │ └── src │ │ └── main.rs ├── log_ssh │ ├── Cargo.toml │ └── src │ │ └── main.rs ├── port_count │ ├── Cargo.toml │ ├── README.md │ └── src │ │ └── main.rs ├── protocols │ ├── Cargo.toml │ ├── README.md │ └── src │ │ └── main.rs ├── run_repeat.py ├── streaming │ ├── Cargo.toml │ └── src │ │ └── main.rs ├── websites-prometheus │ ├── Cargo.toml │ ├── README.md │ └── src │ │ └── main.rs └── websites │ ├── Cargo.toml │ ├── README.md │ └── src │ └── main.rs ├── filtergen ├── Cargo.toml └── src │ ├── cache.rs │ ├── data.rs │ ├── deliver_filter.rs │ ├── lib.rs │ ├── packet_filter.rs │ ├── parse.rs │ ├── proto_filter.rs │ ├── session_filter.rs │ └── utils.rs └── traces ├── README.md ├── quic.pcap ├── quic_kyber.pcapng ├── quic_retry.pcapng ├── quic_xargs.pcap ├── small_flows.pcap └── tls_ciphers.pcap /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | env: 10 | CARGO_TERM_COLOR: always 11 | 12 | jobs: 13 | build: 14 | strategy: 15 | fail-fast: false 16 | matrix: 17 | dpdk_version: 18 | - "20.11" 19 | - "23.11" 20 | - "24.11" 21 | env: 22 | DPDK_VERSION: ${{ matrix.dpdk_version }} 23 | runs-on: ubuntu-latest 24 | steps: 25 | - name: Check out repo 26 | uses: actions/checkout@v3 27 | - name: Install Rust stable 28 | uses: actions-rs/toolchain@v1 29 | with: 30 | toolchain: 1.85.0 31 | override: true 32 | components: rustfmt, clippy 33 | - name: Set up Python ${{ matrix.python-version }} 34 | uses: actions/setup-python@v2 35 | with: 36 | python-version: 3.11 37 | - name: Install pip 38 | run: | 39 | python -m pip install --upgrade pip 40 | - name: Install pyelftools 41 | run: | 42 | pip3 install pyelftools 43 | - name: Install apt dependencies (Linux) 44 | run: | 45 | sudo apt-get update 46 | sudo apt-get install libpcap-dev libnuma-dev 47 | - name: Cache DPDK 48 | id: cache-dpdk 49 | uses: actions/cache@v4 50 | with: 51 | path: dpdk-${{ matrix.dpdk_version }} 52 | key: ${{ runner.os }}-dpdk-${{ matrix.dpdk_version }} 53 | - name: Export enviroment variables 54 | run: | 55 | echo "PKG_CONFIG_PATH=$(pwd)/dpdk-$DPDK_VERSION/install/lib/x86_64-linux-gnu/pkgconfig/" >> $GITHUB_ENV 56 | echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$(pwd)/dpdk-$DPDK_VERSION/install/lib/x86_64-linux-gnu/" >> $GITHUB_ENV 57 | echo "DPDK_PATH=$(pwd)/dpdk-$DPDK_VERSION/install/" >> $GITHUB_ENV 58 | - name: Retrive or compile DPDK 59 | if: steps.cache-dpdk.outputs.cache-hit != 'true' 60 | run: | 61 | echo "Compiling DPDK..." ; 62 | if [ ! -e "dpdk-$DPDK_VERSION" ] || ! pkg-config --exists libdpdk ; then 63 | wget http://fast.dpdk.org/rel/dpdk-$DPDK_VERSION.tar.gz && 64 | tar -zxf dpdk-$DPDK_VERSION.tar.gz && 65 | cd dpdk-$DPDK_VERSION && 66 | pip3 install meson ninja && 67 | meson -Dprefix=$(pwd)/install/ build && 68 | cd build && 69 | ninja && 70 | ninja install && 71 | cd .. && 72 | cd .. ; 73 | fi 74 | - name: Clippy retina-core (no mlx5) 75 | run: cargo clippy --manifest-path core/Cargo.toml --no-default-features -- --deny warnings 76 | - name: Unit test retina-core (no mlx5) 77 | run: cargo test core --manifest-path core/Cargo.toml --no-default-features 78 | - name: Clippy retina-core (with timing) 79 | run: cargo clippy --manifest-path core/Cargo.toml --no-default-features --features timing 80 | - name: Clippy retina-filtergen (no mlx5) 81 | run: cargo clippy --manifest-path filtergen/Cargo.toml --no-default-features -- --deny warnings 82 | 83 | format: 84 | runs-on: ubuntu-latest 85 | steps: 86 | - name: Check out repo 87 | uses: actions/checkout@v3 88 | - name: cargo fmt 89 | run: cargo fmt --all -- --check 90 | 91 | 92 | 93 | 94 | 95 | -------------------------------------------------------------------------------- /.github/workflows/rustdoc.yml: -------------------------------------------------------------------------------- 1 | name: rustdoc 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | 7 | env: 8 | CARGO_TERM_COLOR: always 9 | 10 | jobs: 11 | rustdoc: 12 | env: 13 | DPDK_VERSION: "20.11" 14 | runs-on: ubuntu-latest 15 | steps: 16 | - name: Checkout repository 17 | uses: actions/checkout@v3 18 | - name: Install Rust stable 19 | uses: actions-rs/toolchain@v1 20 | with: 21 | toolchain: 1.85.0 22 | override: true 23 | components: rustfmt, clippy 24 | - name: Set up Python ${{ matrix.python-version }} 25 | uses: actions/setup-python@v2 26 | with: 27 | python-version: 3.11 28 | - name: Install pip 29 | run: | 30 | python -m pip install --upgrade pip 31 | - name: Install pyelftools 32 | run: | 33 | pip3 install pyelftools 34 | - name: Install libpcap (Linux) 35 | run: | 36 | sudo apt-get update 37 | sudo apt-get install libpcap-dev 38 | - name: Cache DPDK 39 | id: cache-dpdk 40 | uses: actions/cache@v4 41 | with: 42 | path: dpdk-$DPDK_VERSION 43 | key: ${{ runner.os }}-dpdk-$DPDK_VERSION 44 | - name: Export enviroment variables 45 | run: | 46 | echo "PKG_CONFIG_PATH=$(pwd)/dpdk-$DPDK_VERSION/install/lib/x86_64-linux-gnu/pkgconfig/" >> $GITHUB_ENV 47 | echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$(pwd)/dpdk-$DPDK_VERSION/install/lib/x86_64-linux-gnu/" >> $GITHUB_ENV 48 | echo "DPDK_PATH=$(pwd)/dpdk-$DPDK_VERSION/install/" >> $GITHUB_ENV 49 | - name: Retrive or compile DPDK 50 | if: steps.cache-dpdk.outputs.cache-hit != 'true' 51 | run: | 52 | echo "Compiling DPDK..." ; 53 | if [ ! -e "dpdk-$DPDK_VERSION" ] || ! pkg-config --exists libdpdk ; then 54 | wget http://fast.dpdk.org/rel/dpdk-$DPDK_VERSION.tar.gz && 55 | tar -zxf dpdk-$DPDK_VERSION.tar.gz && 56 | cd dpdk-$DPDK_VERSION && 57 | pip3 install meson ninja && 58 | meson -Dprefix=$(pwd)/install/ build && 59 | cd build && 60 | ninja && 61 | ninja install && 62 | cd .. && 63 | cd .. ; 64 | fi 65 | 66 | - name: Build Documentation (v0.1.0) 67 | run: git fetch --all --tags && git checkout v0.1.0 && cargo doc --lib --no-deps 68 | 69 | - name: Move Documentation (v0.1.0) 70 | run: | 71 | mkdir -p docs/v0.1.0 72 | cp -r target/doc/* docs/v0.1.0/ 73 | 74 | - name: Build Documentation 75 | run: git checkout main && cargo doc --lib --no-deps 76 | 77 | - name: Move Documentation 78 | run: | 79 | cp -r target/doc/* docs/ 80 | 81 | - name: Deploy Documentation 82 | uses: peaceiris/actions-gh-pages@v3 83 | with: 84 | github_token: ${{ secrets.GITHUB_TOKEN }} 85 | publish_branch: gh-pages 86 | publish_dir: ./docs 87 | force_orphan: true 88 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /log 3 | /logs 4 | /*.pcap 5 | Cargo.lock 6 | *.csv 7 | *.json 8 | *.jsonl 9 | *.DS_Store 10 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = [ 3 | "core", 4 | "datatypes", 5 | "examples/websites", 6 | "examples/websites-prometheus", 7 | "examples/port_count", 8 | # Exclude from compilation; many subscriptions takes a long time to compile 9 | # "examples/filter_stats", 10 | "examples/protocols", 11 | "examples/basic", 12 | "examples/basic_file", 13 | "examples/log_ssh", 14 | "examples/streaming", 15 | ] 16 | resolver = "2" 17 | 18 | [profile.release] 19 | lto = true 20 | 21 | [patch.crates-io] 22 | pcap = { git = 'https://github.com/thegwan/pcap', branch = 'compile-optimized' } 23 | 24 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Retina 2 | 3 | [![build-status](https://github.com/stanford-esrg/retina/actions/workflows/ci.yml/badge.svg)](https://github.com/stanford-esrg/retina/actions) 4 | [![doc-status](https://github.com/stanford-esrg/retina/actions/workflows/rustdoc.yml/badge.svg)](https://stanford-esrg.github.io/retina/retina_core) 5 | 6 | Retina is a network analysis framework that enables operators and researchers to ask complex questions about high-speed (>100gbE) network links. Retina allows users to easily *subscribe* to subsets of parsed application-layer sessions, reassembled network flows, or raw packets in real-time and to run arbitrary analysis code in a standard Rust-based software environment. Retina optimizes for: 7 | 8 | - **Expressiveness:** Retina supports arbitrarily complex processing of 9 | individual packets, reassembled connections, or parsed application-layer 10 | sessions using a simple filter and callback interface. 11 | 12 | - **Performance:** Retina is capable of real-time traffic analysis in high 13 | volume (100G+) environments, such as ISPs or academic institutions. 14 | 15 | - **Deployability:** Retina is readily deployable on a single multi-core server 16 | with commodity 100gbE NICs (e.g., Mellanox ConnectX-5 or Intel E810). 17 | 18 | - **Security:** Retina leverages compile-time memory safety guarantees offered 19 | by Rust to safely and efficiently process network traffic. 20 | 21 | Retina v0.1.0 supported subscribing to a single, pre-defined datatype. v1.1.0 (2024) introduces significant framework changes that allow a user to specify multiple subscriptions. That is, users can now request multiple callbacks, each associated with a filter and one or more datatypes. 22 | 23 | ## Documentation 24 | 25 | A detailed description of Retina's (v0.1.0) architecture and its performance can be 26 | found in our SIGCOMM'22 paper: *[Retina: Analyzing 100 GbE Traffic on Commodity 27 | Hardware](https://thegwan.github.io/files/retina.pdf)*. 28 | 29 | Documentation for using and developing against Retina can be found 30 | [here](https://stanford-esrg.github.io/retina/retina_core/). 31 | 32 | 33 | ## Getting Started 34 | 35 | Install [Rust](https://www.rust-lang.org/tools/install) and 36 | [DPDK](http://core.dpdk.org/download/). Detailed instructions can be found in 37 | [INSTALL](INSTALL.md). 38 | 39 | Add `$DPDK_PATH/lib/x86_64-linux-gnu` to your `LD_LIBRARY_PATH`, where `DPDK_PATH` points to the DPDK installation directory. 40 | 41 | Fork or clone the main git repository: 42 | 43 | `git clone git@github.com:stanford-esrg/retina.git` 44 | 45 | Write your first Retina application (see [examples](https://github.com/stanford-esrg/retina/tree/main/examples); `basic` and `basic_file` are good starters). 46 | 47 | Writing a Retina application consists of defining one or more subscriptions. A subscription is defined by (1) [writing a filter](https://stanford-esrg.github.io/retina/retina_filtergen/index.html) to describe what subset of network traffic you're interested in, (2) choosing [data types to subscribe to](https://stanford-esrg.github.io/retina/retina_datatypes/index.html), and (3) defining a callback function that takes in a subscribable data type and performs operations on the filtered, delivered data. 48 | 49 | Build all examples in release mode: 50 | 51 | `cargo build --release` 52 | 53 | Run `basic` in release mode: 54 | 55 | `sudo env LD_LIBRARY_PATH=$LD_LIBRARY_PATH RUST_LOG=error ./target/release/basic` 56 | 57 | ## Development 58 | 59 | Build a single application in debug mode: 60 | 61 | `cargo build --bin my_app` 62 | 63 | Run in debug mode: 64 | 65 | `sudo env LD_LIBRARY_PATH=$LD_LIBRARY_PATH RUST_LOG=debug ./target/debug/my_app` 66 | 67 | View generated code (requires Rust nightly): 68 | 69 | `cargo expand --manifest-path=examples/my_app/Cargo.toml` 70 | 71 | ### Interfaces 72 | 73 | Retina v1.1.0 provides two interfaces for Retina programs: 74 | * Specify subscriptions via a config file: this tends to be most useful 75 | for applications that wish to subscribe to a large number of IP addresses, 76 | strings, etc., and can map these filters to a relatively small number of 77 | callbacks. 78 | * Specify subscriptions in code: tag callbacks with a filter, then tag 79 | `main` with the number of expected subscriptions. 80 | 81 | For more detail, see [retina-filtergen](https://stanford-esrg.github.io/retina/retina_filtergen/). 82 | 83 | ## Contributing 84 | 85 | Contributions welcome! Please run `cargo fmt` and `cargo clippy` before making a pull request. 86 | 87 | ## Reproducibility 88 | 89 | A [Docker image](https://github.com/tbarbette/retina-docker) is available to run Retina without the hassle of installing DPDK and other dependencies. It is, however, not suitable for performance testing as it uses the DPDK PCAP driver and is limited to a single core. The GitHub repository also includes a tutorial and a video to start learning about Retina (v0.1.0). 90 | 91 | A [CloudLab image](https://github.com/tbarbette/retina-expe) is available to reproduce a few of the experiments shown in the paper on the CloudLab public testbed. The repository also includes the scripts and information to reproduce these experiments on your own testbed. 92 | 93 | ## Acknowledgements 94 | 95 | Retina was developed with support from the National Science Foundation under 96 | award CNS-2124424, and through gifts from Google, Inc., Cisco Systems, Inc., 97 | and Comcast Corporation. 98 | 99 | -------------------------------------------------------------------------------- /configs/offline.toml: -------------------------------------------------------------------------------- 1 | # This configuration is an example to show Retina in "offline" mode, which 2 | # reads from a PCAP file. 3 | # 4 | # See https://stanford-esrg.github.io/retina/retina_core/config/index.html 5 | # for configuration options. 6 | 7 | main_core = 0 8 | nb_memory_channels = 6 9 | 10 | [mempool] 11 | capacity = 262_144 12 | cache_size = 512 13 | 14 | [offline] 15 | pcap = "./traces/small_flows.pcap" 16 | mtu = 9702 17 | 18 | [conntrack] 19 | max_connections = 10_000_000 20 | max_out_of_order = 500 21 | timeout_resolution = 100 22 | udp_inactivity_timeout = 60_000 23 | tcp_inactivity_timeout = 300_000 24 | tcp_establish_timeout = 5000 -------------------------------------------------------------------------------- /configs/online-vdev.toml: -------------------------------------------------------------------------------- 1 | # This configuration is an example to show how to use a DPDK virtual device 2 | # for instance af_packet sockets, memory ring, or pcap-backed device. 3 | # !!! Those virtual devices do not expose hardware capabilities, this is only 4 | # intended for limited functional testing. !!! 5 | # 6 | # See https://stanford-esrg.github.io/retina/retina_core/config/index.html 7 | # for configuration options. 8 | 9 | main_core = 0 10 | nb_memory_channels = 6 11 | 12 | [mempool] 13 | capacity = 8192 14 | cache_size = 512 15 | 16 | [online] 17 | duration = 60 18 | nb_rxd = 32768 19 | promiscuous = true 20 | mtu = 64000 21 | hardware_assist = false 22 | dpdk_supl_args = ["--no-huge","--no-pci","-m 512M","--vdev=net_pcap0,iface=ens0"] 23 | 24 | [online.prometheus] 25 | port = 9898 26 | 27 | [online.monitor.display] 28 | throughput = true 29 | mempool_usage = true 30 | port_stats = ["rx_good_packets", 31 | "rx_good_bytes", 32 | "rx_mbuf_allocation_errors", 33 | "rx_phy_discard_packets", 34 | "rx_missed_errors", 35 | ] 36 | [online.monitor.log] 37 | directory = "./log" 38 | interval = 1000 39 | 40 | [[online.ports]] 41 | device = "net_pcap0" 42 | cores = [1] 43 | 44 | [conntrack] 45 | max_connections = 10_000_000 46 | max_out_of_order = 500 47 | timeout_resolution = 100 48 | udp_inactivity_timeout = 60_000 49 | tcp_inactivity_timeout = 300_000 50 | tcp_establish_timeout = 5000 51 | -------------------------------------------------------------------------------- /configs/online.toml: -------------------------------------------------------------------------------- 1 | main_core = 0 2 | nb_memory_channels = 6 3 | 4 | [mempool] 5 | capacity = 8_388_608 6 | cache_size = 512 7 | 8 | [online] 9 | duration = 120 10 | nb_rxd = 32768 11 | promiscuous = true 12 | mtu = 1500 13 | hardware_assist = false 14 | [[online.ports]] 15 | device = "0000:3b:00.0" 16 | cores = [ 1, 2, 3, 4, 5, 6, 7, 8 ] 17 | 18 | #[online.ports.sink] 19 | #core = 17 20 | #nb_buckets = 16 21 | 22 | [[online.ports]] 23 | device = "0000:d8:00.0" 24 | cores = [ 9, 10, 11, 12, 13, 14, 15, 16 ] 25 | 26 | #[online.ports.sink] 27 | #core = 18 28 | #nb_buckets = 16 29 | 30 | [conntrack] 31 | max_connections = 10000000 32 | max_out_of_order = 500 33 | timeout_resolution = 100 34 | udp_inactivity_timeout = 60000 35 | tcp_inactivity_timeout = 300000 36 | tcp_establish_timeout = 5000 37 | 38 | [online.prometheus] 39 | port = 9898 40 | 41 | [online.monitor.display] 42 | throughput = true 43 | mempool_usage = true 44 | port_stats = [ "rx_good_packets", "rx_good_bytes", "rx_phy_packets", "rx_phy_bytes", "rx_mbuf_allocation_errors", "rx_phy_discard_packets", "rx_missed_errors",] -------------------------------------------------------------------------------- /core/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "retina-core" 3 | version = "1.0.0" 4 | edition = "2021" 5 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 6 | 7 | [build-dependencies] 8 | bindgen = "0.70.1" 9 | cc = "1.0.79" 10 | 11 | [dependencies] 12 | anyhow = "1.0.70" 13 | base64 = "0.13.1" 14 | bimap = "0.6.3" 15 | byteorder = "1.4.3" 16 | chrono = "0.4" 17 | colored = "2" 18 | cpu-time = "1.0.0" 19 | csv = "1.2.1" 20 | ctrlc = { version = "3.2.5", features = ["termination"] } 21 | dns-parser = { git = "https://github.com/stanford-esrg/dns-parser" } 22 | hashlink = "0.7.0" 23 | hdrhistogram = "7.5.2" 24 | hex = { version = "0.4.3", features = ["serde"] } 25 | httparse = "1.8" 26 | indexmap = "1.9.3" 27 | ipnet = "2.7.2" 28 | itertools = "0.10.5" 29 | lazy_static = "1.4.0" 30 | log = { version = "0.4", features = ["release_max_level_info"] } 31 | maplit = "1.0.2" 32 | md5 = "0.7.0" 33 | nom = "7.1.3" 34 | pcap = "0.8.1" 35 | pest = "2.5.7" 36 | pest_derive = "2.5" 37 | petgraph = "0.5.1" 38 | pnet = "0.33.0" 39 | prettytable-rs = "0.10.0" 40 | serde = { version = "1.0", features = ["derive"] } 41 | serde_json = "1.0.96" 42 | strum = "0.20" 43 | strum_macros = "0.20" 44 | thiserror = "1.0" 45 | tls-parser = { git = "https://github.com/stanford-esrg/tls-parser" } 46 | toml = "0.5.11" 47 | x509-parser = "0.13.2" 48 | bitmask-enum = "2.2.4" 49 | quote = "1.0.26" 50 | proc-macro2 = "1.0.56" 51 | ssh-parser = "0.5.0" 52 | syn = { version = "2.0.15" } 53 | regex = "1.7.3" 54 | ring = "0.17.8" 55 | aes-gcm = "0.10.3" 56 | memchr = "2.7.4" 57 | 58 | # Statistics 59 | tokio = { version = "1.42.0", features = ["full"] } 60 | array-init = "2.0" 61 | hyper = { version = "1", features = ["full"], optional = true } 62 | hyper-util = { version = "0.1", features = ["full"], optional = true } 63 | prometheus-client = { version = "0.22.3", optional = true } 64 | http-body-util = { version = "0.1.2", optional = true } 65 | 66 | [features] 67 | timing = [] 68 | prometheus = [ 69 | "dep:hyper", 70 | "dep:hyper-util", 71 | "dep:prometheus-client", 72 | "dep:http-body-util", 73 | ] 74 | mlx5 = [] 75 | default = [] 76 | -------------------------------------------------------------------------------- /core/build.rs: -------------------------------------------------------------------------------- 1 | use bindgen::Builder; 2 | use std::env; 3 | use std::path::Path; 4 | use std::process::exit; 5 | use std::process::Command; 6 | 7 | fn main() { 8 | // modified from https://github.com/deeptir18/cornflakes/blob/master/cornflakes-libos/build.rs 9 | 10 | println!("cargo:rerun-if-env-changed=DPDK_PATH"); 11 | println!("cargo:rerun-if-env-changed=DPDK_VERSION"); 12 | println!("cargo:rerun-if-changed=build.rs"); 13 | println!("cargo:rerun-if-changed=src/dpdk/inlined.c"); 14 | let cargo_manifest_dir = env::var("CARGO_MANIFEST_DIR").unwrap(); 15 | let cargo_dir = Path::new(&cargo_manifest_dir); 16 | 17 | println!("cargo:rustc-check-cfg=cfg(dpdk_ge_2011)"); 18 | println!("cargo:rustc-check-cfg=cfg(dpdk_ge_2108)"); 19 | println!("cargo:rustc-check-cfg=cfg(dpdk_ge_2111)"); 20 | println!("cargo:rustc-check-cfg=cfg(dpdk_ge_2311)"); 21 | println!("cargo:rustc-check-cfg=cfg(dpdk_ge_2411)"); 22 | let dpdk_version = env::var("DPDK_VERSION").expect("Set DPDK_VERSION env variable"); 23 | 24 | if !["20.11", "21.08", "23.11", "24.11"].contains(&dpdk_version.as_str()) { 25 | println!("Unsupported dpdk version"); 26 | exit(1); 27 | } 28 | 29 | println!("cargo:rustc-cfg=dpdk_ge_2011"); 30 | if dpdk_version != "20.11" { 31 | println!("cargo:rustc-cfg=dpdk_ge_2108"); 32 | if dpdk_version != "21.08" { 33 | println!("cargo:rustc-cfg=dpdk_ge_2111"); 34 | if dpdk_version != "21.11" { 35 | println!("cargo:rustc-cfg=dpdk_ge_2311"); 36 | if dpdk_version != "23.11" { 37 | println!("cargo:rustc-cfg=dpdk_ge_2411"); 38 | } 39 | } 40 | } 41 | } 42 | 43 | let out_dir_s = env::var("OUT_DIR").unwrap(); 44 | let out_dir = Path::new(&out_dir_s); 45 | let load_lib_path_s = env::var("LD_LIBRARY_PATH").unwrap(); 46 | let load_lib_path = Path::new(&load_lib_path_s); 47 | let pkg_config_path = load_lib_path.join("pkgconfig"); 48 | let cflags_bytes = Command::new("pkg-config") 49 | .env("PKG_CONFIG_PATH", &pkg_config_path) 50 | .args(["--cflags", "libdpdk"]) 51 | .output() 52 | .unwrap_or_else(|e| panic!("Failed pkg-config cflags: {:?}", e)) 53 | .stdout; 54 | let cflags = String::from_utf8(cflags_bytes).unwrap(); 55 | 56 | let mut header_locations = vec![]; 57 | 58 | for flag in cflags.split(' ') { 59 | if let Some(stripped) = flag.strip_prefix("-I") { 60 | let header_location = stripped.trim(); 61 | header_locations.push(header_location); 62 | } 63 | } 64 | 65 | let ldflags_bytes = Command::new("pkg-config") 66 | .env("PKG_CONFIG_PATH", &pkg_config_path) 67 | .args(["--libs", "libdpdk"]) 68 | .output() 69 | .unwrap_or_else(|e| panic!("Failed pkg-config ldflags: {:?}", e)) 70 | .stdout; 71 | 72 | if ldflags_bytes.is_empty() { 73 | println!("Could not get DPDK's LDFLAGS. Are DPDK_PATH, LD_LIBRARY_PATH set correctly?"); 74 | exit(1); 75 | }; 76 | 77 | let ldflags = String::from_utf8(ldflags_bytes).unwrap(); 78 | 79 | let mut library_location = None; 80 | let mut lib_names = vec![]; 81 | 82 | for flag in ldflags.split(' ') { 83 | if let Some(stripped) = flag.strip_prefix("-L") { 84 | library_location = Some(stripped); 85 | } else if let Some(stripped) = flag.strip_prefix("-l") { 86 | lib_names.push(stripped); 87 | } 88 | } 89 | 90 | // Link in `librte_net_mlx5` and its dependencies if desired. 91 | #[cfg(feature = "mlx5")] 92 | { 93 | lib_names.extend(&[ 94 | "rte_net_mlx5", 95 | "rte_bus_pci", 96 | "rte_bus_vdev", 97 | "rte_common_mlx5", 98 | ]); 99 | } 100 | 101 | // Step 1: Now that we've compiled and installed DPDK, point cargo to the libraries. 102 | println!( 103 | "cargo:rustc-link-search=native={}", 104 | library_location.unwrap() 105 | ); 106 | for lib_name in &lib_names { 107 | println!("cargo:rustc-link-lib={}", lib_name); 108 | } 109 | 110 | // Step 2: Generate bindings for the DPDK headers. 111 | let mut builder = Builder::default(); 112 | for header_location in &header_locations { 113 | builder = builder.clang_arg(format!("-I{}", header_location)); 114 | } 115 | 116 | let headers_file = Path::new(&cargo_dir) 117 | .join("src") 118 | .join("dpdk") 119 | .join("dpdk_headers.h"); 120 | let bindings = builder 121 | .header(headers_file.to_str().unwrap()) 122 | .clang_macro_fallback() 123 | // mark as opaque per bindgen bug on packed+aligned structs: 124 | // https://github.com/rust-lang/rust-bindgen/issues/1538 125 | .opaque_type(r"rte_arp_ipv4|rte_arp_hdr") 126 | .opaque_type(r"(rte_ecpri|rte_l2tpv2)_.*") 127 | .allowlist_type(r"(rte|eth|pcap)_.*") 128 | .allowlist_function(r"(_rte|rte|eth|numa|pcap)_.*") 129 | .allowlist_var(r"(RTE|DEV|ETH|MEMPOOL|PKT|rte)_.*") 130 | .parse_callbacks(Box::new(bindgen::CargoCallbacks::new())) 131 | .generate_comments(false) 132 | .generate() 133 | .unwrap_or_else(|e| panic!("Failed to generate bindings: {:?}", e)); 134 | let bindings_out = out_dir.join("dpdk.rs"); 135 | bindings 136 | .write_to_file(bindings_out) 137 | .expect("Failed to write bindings"); 138 | 139 | // Step 3: Compile a stub file so Rust can access `inline` functions in the headers 140 | // that aren't compiled into the libraries. 141 | let mut builder = cc::Build::new(); 142 | builder.opt_level(3); 143 | builder.pic(true); 144 | builder.flag("-march=native"); 145 | 146 | let inlined_file = Path::new(&cargo_dir) 147 | .join("src") 148 | .join("dpdk") 149 | .join("inlined.c"); 150 | builder.file(inlined_file.to_str().unwrap()); 151 | for header_location in &header_locations { 152 | builder.include(header_location); 153 | } 154 | builder.compile("inlined"); 155 | } 156 | -------------------------------------------------------------------------------- /core/src/conntrack/conn/tcp_conn/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod reassembly; 2 | 3 | use self::reassembly::TcpFlow; 4 | use crate::conntrack::conn::conn_info::ConnInfo; 5 | use crate::conntrack::pdu::{L4Context, L4Pdu}; 6 | use crate::protocols::packet::tcp::{FIN, RST}; 7 | use crate::protocols::stream::ParserRegistry; 8 | use crate::subscription::{Subscription, Trackable}; 9 | 10 | pub(crate) struct TcpConn { 11 | pub(crate) ctos: TcpFlow, 12 | pub(crate) stoc: TcpFlow, 13 | } 14 | 15 | impl TcpConn { 16 | pub(crate) fn new_on_syn(ctxt: L4Context, max_ooo: usize) -> Self { 17 | let flags = ctxt.flags; 18 | let next_seq = ctxt.seq_no.wrapping_add(1 + ctxt.length as u32); 19 | let ack = ctxt.ack_no; 20 | TcpConn { 21 | ctos: TcpFlow::new(max_ooo, next_seq, flags, ack), 22 | stoc: TcpFlow::default(max_ooo), 23 | } 24 | } 25 | 26 | /// Insert TCP segment ordered into ctos or stoc flow 27 | #[inline] 28 | pub(crate) fn reassemble( 29 | &mut self, 30 | segment: L4Pdu, 31 | info: &mut ConnInfo, 32 | subscription: &Subscription, 33 | registry: &ParserRegistry, 34 | ) { 35 | if segment.dir { 36 | self.ctos 37 | .insert_segment::(segment, info, subscription, registry); 38 | } else { 39 | self.stoc 40 | .insert_segment::(segment, info, subscription, registry); 41 | } 42 | } 43 | 44 | /// Returns `true` if the connection should be terminated 45 | #[inline] 46 | pub(crate) fn is_terminated(&self) -> bool { 47 | // Both sides have sent, reassembled, and acknowledged FIN, or RST has been sent 48 | (self.ctos.consumed_flags & self.stoc.consumed_flags & FIN != 0 49 | && self.ctos.last_ack == self.stoc.next_seq 50 | && self.stoc.last_ack == self.ctos.next_seq) 51 | || (self.ctos.consumed_flags & RST | self.stoc.consumed_flags & RST) != 0 52 | } 53 | 54 | /// Updates connection termination flags 55 | // Useful if desired to track TCP connections without reassembly 56 | #[inline] 57 | pub(super) fn update_flags(&mut self, flags: u8, dir: bool) { 58 | if dir { 59 | self.ctos.consumed_flags |= flags; 60 | } else { 61 | self.stoc.consumed_flags |= flags; 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /core/src/conntrack/conn/udp_conn/mod.rs: -------------------------------------------------------------------------------- 1 | pub(crate) struct UdpConn; 2 | -------------------------------------------------------------------------------- /core/src/conntrack/conn_id.rs: -------------------------------------------------------------------------------- 1 | //! Bidirectional connection identifiers. 2 | //! 3 | //! Provides endpoint-specific (distinguishes originator and responder) and generic identifiers for bi-directional connections. 4 | //! Retina defines a "connection" by five tuple (source/destination addresses, ports, and transport protocol). 5 | 6 | use crate::conntrack::L4Context; 7 | 8 | use crate::protocols::packet::tcp::TCP_PROTOCOL; 9 | use crate::protocols::packet::udp::UDP_PROTOCOL; 10 | use std::cmp; 11 | use std::fmt; 12 | use std::net::{Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddr::V4, SocketAddr::V6}; 13 | 14 | use serde::Serialize; 15 | 16 | /// Connection 5-tuple. 17 | /// 18 | /// The sender of the first observed packet in the connection becomes the originator `orig`, and the 19 | /// recipient becomes the responder `resp`. 20 | #[derive(Debug, Copy, Clone, Hash, Eq, PartialEq, Serialize)] 21 | pub struct FiveTuple { 22 | /// The originator connection endpoint. 23 | pub orig: SocketAddr, 24 | /// The responder connection endpoint. 25 | pub resp: SocketAddr, 26 | /// The layer-4 protocol. 27 | pub proto: usize, 28 | } 29 | 30 | impl FiveTuple { 31 | /// Creates a new 5-tuple from `ctxt`. 32 | pub fn from_ctxt(ctxt: L4Context) -> Self { 33 | FiveTuple { 34 | orig: ctxt.src, 35 | resp: ctxt.dst, 36 | proto: ctxt.proto, 37 | } 38 | } 39 | 40 | /// Converts a 5-tuple to a non-directional connection identifier. 41 | pub fn conn_id(&self) -> ConnId { 42 | ConnId::new(self.orig, self.resp, self.proto) 43 | } 44 | 45 | /// Utility for returning a string representation of the dst. subnet 46 | /// /24 for IPv4, /64 for IPv6; no mask for broadcast 47 | pub fn dst_subnet_str(&self) -> String { 48 | if let V4(_) = self.orig { 49 | if let V4(dst) = self.resp { 50 | if dst.ip().is_broadcast() || dst.ip().is_multicast() { 51 | return dst.ip().to_string(); 52 | } else { 53 | let mask = !0u32 << (32 - 24); // Convert to a /24 54 | return Ipv4Addr::from(dst.ip().to_bits() & mask).to_string(); 55 | } 56 | } 57 | } else if let V6(_) = self.orig { 58 | if let V6(dst) = self.resp { 59 | let mask = !0u128 << (128 - 64); // Convert to a /64 60 | return Ipv6Addr::from(dst.ip().to_bits() & mask).to_string(); 61 | } 62 | } 63 | String::new() 64 | } 65 | 66 | /// Utility for returning a string representation of the dst. IP 67 | pub fn dst_ip_str(&self) -> String { 68 | if let V4(_) = self.orig { 69 | if let V4(dst) = self.resp { 70 | return dst.ip().to_string(); 71 | } 72 | } else if let V6(_) = self.orig { 73 | if let V6(dst) = self.resp { 74 | return dst.ip().to_string(); 75 | } 76 | } 77 | String::new() 78 | } 79 | 80 | /// Utility for returning a string representation of the transport 81 | /// protocol and source/destination ports 82 | pub fn transp_proto_str(&self) -> String { 83 | let src_port = self.orig.port(); 84 | let dst_port = self.resp.port(); 85 | let proto = match self.proto { 86 | UDP_PROTOCOL => "udp", 87 | TCP_PROTOCOL => "tcp", 88 | _ => "none", 89 | }; 90 | format!( 91 | "{{ \"proto\": \"{}\", \"src\": \"{}\", \"dst\": \"{}\" }}", 92 | proto, src_port, dst_port 93 | ) 94 | } 95 | } 96 | 97 | impl fmt::Display for FiveTuple { 98 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 99 | write!(f, "{} -> ", self.orig)?; 100 | write!(f, "{}", self.resp)?; 101 | write!(f, " protocol {}", self.proto)?; 102 | Ok(()) 103 | } 104 | } 105 | 106 | /// A generic connection identifier. 107 | /// 108 | /// Identifies a connection independent of the source and destination socket address order. Does not 109 | /// distinguish between the originator and responder of the connection. 110 | #[derive(Debug, Clone, Hash, Eq, PartialEq)] 111 | pub struct ConnId(SocketAddr, SocketAddr, usize); 112 | 113 | impl ConnId { 114 | /// Returns the connection ID of a packet with `src` and `dst` IP/port pairs. 115 | pub(super) fn new(src: SocketAddr, dst: SocketAddr, protocol: usize) -> Self { 116 | ConnId(cmp::max(src, dst), cmp::min(src, dst), protocol) 117 | } 118 | } 119 | 120 | impl fmt::Display for ConnId { 121 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 122 | write!(f, "{} <> ", self.0)?; 123 | write!(f, "{}", self.1)?; 124 | write!(f, " protocol {}", self.2)?; 125 | Ok(()) 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /core/src/conntrack/timerwheel.rs: -------------------------------------------------------------------------------- 1 | use crate::conntrack::{Conn, ConnId}; 2 | use crate::subscription::{Subscription, Trackable}; 3 | 4 | use hashlink::linked_hash_map::LinkedHashMap; 5 | use hashlink::linked_hash_map::RawEntryMut; 6 | use std::collections::VecDeque; 7 | use std::time::{Duration, Instant}; 8 | 9 | /// Tracks inactive connection expiration. 10 | pub(super) struct TimerWheel { 11 | /// Period to check for inactive connections. 12 | period: Duration, 13 | /// Start time of the `TimerWheel`. 14 | start_ts: Instant, 15 | /// Previous check time of the `TimerWheel`. 16 | prev_ts: Instant, 17 | /// Index of the next bucket to expire. 18 | next_bucket: usize, 19 | /// List of timers. 20 | timers: Vec>, 21 | } 22 | 23 | impl TimerWheel { 24 | /// Creates a new `TimerWheel` with a maximum timeout of `max_timeout` and a timeout check 25 | /// period of `timeout_resolution`. 26 | pub(super) fn new(max_timeout: usize, timeout_resolution: usize) -> Self { 27 | if timeout_resolution > max_timeout { 28 | panic!("Timeout check period must be smaller than maximum inactivity timeout") 29 | } 30 | let start_ts = Instant::now(); 31 | let period = Duration::from_millis(timeout_resolution as u64); 32 | TimerWheel { 33 | period, 34 | start_ts, 35 | prev_ts: start_ts, 36 | next_bucket: 0, 37 | timers: vec![VecDeque::new(); max_timeout / timeout_resolution], 38 | } 39 | } 40 | 41 | /// Insert a new connection ID into the timerwheel. 42 | #[inline] 43 | pub(super) fn insert( 44 | &mut self, 45 | conn_id: &ConnId, 46 | last_seen_ts: Instant, 47 | inactivity_window: usize, 48 | ) { 49 | let current_time = (last_seen_ts - self.start_ts).as_millis() as usize; 50 | let period = self.period.as_millis() as usize; 51 | let timer_index = ((current_time + inactivity_window) / period) % self.timers.len(); 52 | log::debug!("Inserting into index: {}, {:?}", timer_index, current_time); 53 | self.timers[timer_index].push_back(conn_id.to_owned()); 54 | } 55 | 56 | /// Checks for and remove inactive connections. 57 | #[inline] 58 | pub(super) fn check_inactive( 59 | &mut self, 60 | table: &mut LinkedHashMap>, 61 | subscription: &Subscription, 62 | now: Instant, 63 | ) { 64 | let table_len = table.len(); 65 | if now - self.prev_ts >= self.period { 66 | self.prev_ts = now; 67 | let nb_removed = self.remove_inactive(now, table, subscription); 68 | log::debug!( 69 | "expired: {} ({})", 70 | nb_removed, 71 | nb_removed as f64 / table_len as f64 72 | ); 73 | log::debug!("new table size: {}", table.len()); 74 | } 75 | } 76 | 77 | /// Removes connections that have been inactive for at least their inactivity window time 78 | /// period. 79 | /// 80 | /// Returns the number of connections removed. 81 | #[inline] 82 | pub(super) fn remove_inactive( 83 | &mut self, 84 | now: Instant, 85 | table: &mut LinkedHashMap>, 86 | subscription: &Subscription, 87 | ) -> usize { 88 | let period = self.period.as_millis() as usize; 89 | let nb_buckets = self.timers.len(); 90 | let mut not_expired: Vec<(usize, ConnId)> = vec![]; 91 | let check_time = (now - self.start_ts).as_millis() as usize / period * period; 92 | 93 | let mut cnt_exp = 0; 94 | let last_expire_bucket = check_time / period; 95 | log::debug!( 96 | "check time: {}, next: {}, last: {}", 97 | check_time, 98 | self.next_bucket, 99 | last_expire_bucket 100 | ); 101 | 102 | for expire_bucket in self.next_bucket..last_expire_bucket { 103 | log::debug!( 104 | "bucket: {}, index: {}", 105 | expire_bucket, 106 | expire_bucket % nb_buckets 107 | ); 108 | let list = &mut self.timers[expire_bucket % nb_buckets]; 109 | 110 | for conn_id in list.drain(..) { 111 | if let RawEntryMut::Occupied(mut occupied) = 112 | table.raw_entry_mut().from_key(&conn_id) 113 | { 114 | let conn = occupied.get_mut(); 115 | let last_seen_time = (conn.last_seen_ts - self.start_ts).as_millis() as usize; 116 | log::debug!("Last seen time: {}", last_seen_time); 117 | let expire_time = last_seen_time + conn.inactivity_window; 118 | if expire_time < check_time { 119 | cnt_exp += 1; 120 | conn.terminate(subscription); 121 | occupied.remove(); 122 | } else { 123 | let timer_index = (expire_time / period) % nb_buckets; 124 | not_expired.push((timer_index, conn_id)); 125 | } 126 | } 127 | } 128 | for (timer_index, conn_id) in not_expired.drain(..) { 129 | self.timers[timer_index].push_back(conn_id); 130 | } 131 | } 132 | self.next_bucket = last_expire_bucket; 133 | cnt_exp 134 | } 135 | } 136 | -------------------------------------------------------------------------------- /core/src/dpdk/dpdk_headers.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | -------------------------------------------------------------------------------- /core/src/dpdk/inlined.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | void rte_pktmbuf_free_(struct rte_mbuf *packet) { 9 | rte_pktmbuf_free(packet); 10 | } 11 | 12 | struct rte_mbuf* rte_pktmbuf_alloc_(struct rte_mempool *mp) { 13 | return rte_pktmbuf_alloc(mp); 14 | } 15 | 16 | uint16_t rte_eth_tx_burst_(uint16_t port_id, uint16_t queue_id, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { 17 | return rte_eth_tx_burst(port_id, queue_id, tx_pkts, nb_pkts); 18 | } 19 | 20 | uint16_t rte_eth_rx_burst_(uint16_t port_id, uint16_t queue_id, struct rte_mbuf **rx_pkts, const uint16_t nb_pkts) { 21 | return rte_eth_rx_burst(port_id, queue_id, rx_pkts, nb_pkts); 22 | } 23 | 24 | uint16_t rte_mbuf_refcnt_read_(const struct rte_mbuf* m) { 25 | return rte_mbuf_refcnt_read(m); 26 | } 27 | 28 | uint16_t rte_mbuf_refcnt_update_(struct rte_mbuf* m, int16_t value) { 29 | return rte_mbuf_refcnt_update(m, value); 30 | } 31 | 32 | void rte_mbuf_refcnt_set_(struct rte_mbuf* m, int16_t value) { 33 | return rte_mbuf_refcnt_set(m, value); 34 | } 35 | 36 | char* rte_pktmbuf_adj_(struct rte_mbuf* m, uint16_t len) { 37 | return rte_pktmbuf_adj(m, len); 38 | } 39 | 40 | int rte_pktmbuf_trim_(struct rte_mbuf* m, uint16_t len) { 41 | return rte_pktmbuf_trim(m, len); 42 | } 43 | 44 | unsigned rte_lcore_id_(void) { 45 | return rte_lcore_id(); 46 | } 47 | 48 | uint64_t rte_rdtsc_(void) { 49 | return rte_rdtsc(); 50 | } 51 | 52 | /* RTE_RING functions */ 53 | 54 | int rte_ring_enqueue_(struct rte_ring* r, void* obj) { 55 | return rte_ring_enqueue(r, obj); 56 | } 57 | 58 | int rte_ring_sp_enqueue_(struct rte_ring* r, void* obj) { 59 | return rte_ring_sp_enqueue(r, obj); 60 | } 61 | 62 | int rte_ring_mp_enqueue_(struct rte_ring* r, void* obj) { 63 | return rte_ring_mp_enqueue(r, obj); 64 | } 65 | 66 | int rte_ring_dequeue_(struct rte_ring* r, void** obj_p) { 67 | return rte_ring_dequeue(r, obj_p); 68 | } 69 | 70 | int rte_ring_sc_dequeue_(struct rte_ring* r, void** obj_p) { 71 | return rte_ring_sc_dequeue(r, obj_p); 72 | } 73 | 74 | int rte_ring_mc_dequeue_(struct rte_ring* r, void** obj_p) { 75 | return rte_ring_mc_dequeue(r, obj_p); 76 | } 77 | 78 | unsigned rte_ring_count_(const struct rte_ring* r) { 79 | return rte_ring_count(r); 80 | } 81 | 82 | unsigned rte_ring_free_count_(const struct rte_ring* r) { 83 | return rte_ring_free_count(r); 84 | } 85 | 86 | int rte_ring_full_(const struct rte_ring* r) { 87 | return rte_ring_full(r); 88 | } 89 | 90 | int rte_ring_empty_(const struct rte_ring* r) { 91 | return rte_ring_empty(r); 92 | } 93 | 94 | unsigned rte_ring_get_size_(const struct rte_ring* r) { 95 | return rte_ring_get_size(r); 96 | } 97 | 98 | unsigned rte_ring_get_capacity_(const struct rte_ring* r) { 99 | return rte_ring_get_capacity(r); 100 | } 101 | -------------------------------------------------------------------------------- /core/src/filter/grammar.pest: -------------------------------------------------------------------------------- 1 | // Modified from https://github.com/cloudflare/wirefilter/blob/master/wirefilter-parser/src/grammar.pest 2 | 3 | // Expressions 4 | // ---------------------------------------------------------------------- 5 | filter = _{ SOI ~ expr? ~ EOI } 6 | raw_predicates = _{ SOI ~ ( (non_identifier)* ~ predicate ~ (non_identifier)* )* ~ EOI } 7 | 8 | // encodes operator precedence (AND over OR) 9 | expr = { sub_expr ~ (or_op ~ sub_expr)* } 10 | sub_expr = { term ~ (and_op ~ term)* } 11 | term = _{ predicate | "(" ~ expr ~ ")" } 12 | predicate = { protocol ~ ("." ~ (combined_field | field) ~ bin_op ~ value)? } 13 | 14 | // Identifiers 15 | // ---------------------------------------------------------------------- 16 | protocol = @{ ASCII_ALPHA ~ (ASCII_ALPHANUMERIC| "_")* } 17 | field = @{ ASCII_ALPHA ~ (ASCII_ALPHANUMERIC | "_")* } 18 | combined_field = @{ "addr" | "port" } 19 | 20 | // order matters! Parser will try from left to right 21 | value = { ipv4_lit | ipv6_lit | int_range | int_lit | byte_lit | str_lit } 22 | 23 | ipv4_addr = @{ 24 | ASCII_DIGIT{1,3} ~ ("." ~ ASCII_DIGIT{1,3}){3} 25 | } 26 | ipv4_prefix = @{ ASCII_DIGIT{1,2} } 27 | ipv4_lit = ${ ipv4_addr ~ ("/" ~ ipv4_prefix)? } 28 | 29 | ipv6_addr = @{ 30 | (":" | ASCII_ALPHANUMERIC{1,4}) ~ ":" ~ (ipv4_addr | ASCII_ALPHANUMERIC{1,4} | ":")* 31 | } 32 | ipv6_prefix = @{ ASCII_DIGIT{1,3} } 33 | ipv6_lit = ${ ipv6_addr ~ ("/" ~ ipv6_prefix)? } 34 | 35 | // Integers 36 | int_lit = @{ ASCII_DIGIT+ } 37 | int_range = ${ int_lit ~ ".." ~ int_lit } 38 | 39 | // Strings 40 | str_lit = _{ "\'" ~ text ~ "\'" } 41 | text = { (!("\'") ~ ANY)+ } 42 | 43 | // Bytes 44 | byte_lit = { "|" ~ (byte)+ ~ "|" } 45 | byte = { !("|") ~ ASCII_HEX_DIGIT{2} ~ " "? } 46 | 47 | // Logical operators 48 | // ---------------------------------------------------------------------- 49 | or_op = { "||" | "or" | "OR" } 50 | and_op = { "&&" | "and" | "AND" } 51 | not_op = { "!" | "not" | "NOT" } // not yet supported, temp placeholder 52 | 53 | // Binary operators 54 | // ---------------------------------------------------------------------- 55 | bin_op = { 56 | eq_op | ne_op | ge_op | le_op | gt_op | lt_op | in_op | byte_re_op | re_op | en_op | contains_op | not_contains_op 57 | } 58 | 59 | eq_op = { "=" } 60 | ne_op = { "!=" | "ne" } 61 | ge_op = { ">=" | "ge" } 62 | le_op = { "<=" | "le" } 63 | gt_op = { ">" | "gt" } 64 | lt_op = { "<" | "lt" } 65 | in_op = { "in" } 66 | re_op = { "~" | "matches" } 67 | en_op = { "eq" } 68 | byte_re_op = { "~b" } 69 | contains_op = { "contains" } 70 | not_contains_op = { "!contains" | "not contains" } 71 | 72 | // Miscellaneous 73 | // ---------------------------------------------------------------------- 74 | WHITESPACE = _{ " " | NEWLINE } 75 | non_identifier = _{ or_op | and_op | not_op | "(" | ")" } 76 | -------------------------------------------------------------------------------- /core/src/filter/hardware/flow_action.rs: -------------------------------------------------------------------------------- 1 | use crate::dpdk; 2 | use crate::port::{PortId, SYMMETRIC_RSS_KEY}; 3 | 4 | use std::mem; 5 | 6 | pub(super) type ActionRules = Vec; 7 | 8 | // Builds a vector of rte_flow_action 9 | // #[derive(Debug, Clone)] 10 | pub(super) struct FlowAction { 11 | pub(super) rules: ActionRules, 12 | port_id: PortId, 13 | 14 | // The following vectors contain configurations [conf] for special action 15 | pub(super) rss: Vec, 16 | pub(super) jump: Vec, 17 | pub(super) mark: Vec, 18 | // drop has no config 19 | } 20 | 21 | impl FlowAction { 22 | pub(super) fn new(port_id: PortId) -> FlowAction { 23 | FlowAction { 24 | rules: ActionRules::new(), 25 | port_id, 26 | rss: Vec::::new(), 27 | jump: Vec::::new(), 28 | #[allow(dead_code)] 29 | mark: Vec::::new(), 30 | } 31 | } 32 | 33 | pub(super) fn finish(&mut self) { 34 | // Add END terminator 35 | let mut a_end: dpdk::rte_flow_action = unsafe { mem::zeroed() }; 36 | a_end.type_ = dpdk::rte_flow_action_type_RTE_FLOW_ACTION_TYPE_END; 37 | self.rules.push(a_end); 38 | } 39 | 40 | pub(super) fn append_jump(&mut self, group: u32) { 41 | let mut a_jump: dpdk::rte_flow_action = unsafe { mem::zeroed() }; 42 | a_jump.type_ = dpdk::rte_flow_action_type_RTE_FLOW_ACTION_TYPE_JUMP; 43 | self.rules.push(a_jump); 44 | 45 | let mut jump_conf: dpdk::rte_flow_action_jump = unsafe { mem::zeroed() }; 46 | jump_conf.group = group; 47 | 48 | self.jump.push(jump_conf); 49 | } 50 | 51 | #[allow(dead_code)] 52 | pub(super) fn append_mark(&mut self, mark: u32) { 53 | let mut a_mark: dpdk::rte_flow_action = unsafe { mem::zeroed() }; 54 | a_mark.type_ = dpdk::rte_flow_action_type_RTE_FLOW_ACTION_TYPE_MARK; 55 | self.rules.push(a_mark); 56 | 57 | let mut mark_conf: dpdk::rte_flow_action_mark = unsafe { mem::zeroed() }; 58 | mark_conf.id = mark; 59 | 60 | self.mark.push(mark_conf); 61 | } 62 | 63 | pub(super) fn append_rss(&mut self) { 64 | let mut a_rss: dpdk::rte_flow_action = unsafe { mem::zeroed() }; 65 | a_rss.type_ = dpdk::rte_flow_action_type_RTE_FLOW_ACTION_TYPE_RSS; 66 | self.rules.push(a_rss); 67 | 68 | let mut rss_conf: dpdk::rte_eth_rss_conf = unsafe { mem::zeroed() }; 69 | let ret = unsafe { dpdk::rte_eth_dev_rss_hash_conf_get(self.port_id.raw(), &mut rss_conf) }; 70 | assert_eq!(ret, 0); 71 | 72 | let mut a_rss_conf: dpdk::rte_flow_action_rss = unsafe { mem::zeroed() }; 73 | a_rss_conf.func = dpdk::rte_eth_hash_function_RTE_ETH_HASH_FUNCTION_TOEPLITZ; 74 | // Innermost encapsulation level PMD can handle 75 | a_rss_conf.level = 0; 76 | a_rss_conf.types = rss_conf.rss_hf; 77 | a_rss_conf.key_len = rss_conf.rss_key_len as u32; 78 | 79 | // Since the RSS key needs to outlive this method, we use the static 80 | // SYMMETRIC_RSS_KEY instead of the key queried from the existing rss_conf 81 | a_rss_conf.key = SYMMETRIC_RSS_KEY.as_ptr(); 82 | 83 | self.rss.push(a_rss_conf); 84 | } 85 | 86 | #[allow(dead_code)] 87 | pub(super) fn append_drop(&mut self) { 88 | let mut a_drop: dpdk::rte_flow_action = unsafe { mem::zeroed() }; 89 | a_drop.type_ = dpdk::rte_flow_action_type_RTE_FLOW_ACTION_TYPE_DROP; 90 | self.rules.push(a_drop); 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /core/src/filter/hardware/flow_attr.rs: -------------------------------------------------------------------------------- 1 | use crate::dpdk; 2 | use std::mem; 3 | 4 | /// An ingress flow attribute 5 | pub(super) struct FlowAttribute(dpdk::rte_flow_attr); 6 | 7 | impl FlowAttribute { 8 | pub(super) fn new(group: u32, priority: u32) -> Self { 9 | let mut attr: dpdk::rte_flow_attr = unsafe { mem::zeroed() }; 10 | attr.set_ingress(1); 11 | attr.group = group; 12 | attr.priority = priority; 13 | FlowAttribute(attr) 14 | } 15 | 16 | /// Returns a reference to the inner rte_flow_attr for use with DPDK functions. 17 | pub(super) fn raw(&self) -> &dpdk::rte_flow_attr { 18 | &self.0 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /core/src/filter/macros.rs: -------------------------------------------------------------------------------- 1 | macro_rules! unwrap_or_ret_false { 2 | ( $e:expr ) => { 3 | match $e { 4 | Some(x) => x, 5 | None => return false, 6 | } 7 | }; 8 | } 9 | 10 | #[doc(hidden)] 11 | #[macro_export] 12 | macro_rules! protocol { 13 | ( $x:expr ) => { 14 | $crate::filter::ast::ProtocolName($x.to_owned()) 15 | }; 16 | } 17 | 18 | #[doc(hidden)] 19 | #[macro_export] 20 | macro_rules! field { 21 | ( $x:expr ) => { 22 | $crate::filter::ast::FieldName($x.to_owned()) 23 | }; 24 | } 25 | -------------------------------------------------------------------------------- /core/src/lcore/mod.rs: -------------------------------------------------------------------------------- 1 | //! Utilities for managing and monitoring Retina cores. 2 | 3 | pub(crate) mod monitor; 4 | // pub(crate) mod ring; 5 | pub(crate) mod rx_core; 6 | 7 | use crate::dpdk; 8 | 9 | use std::fmt; 10 | 11 | use serde::{Deserialize, Serialize}; 12 | 13 | #[derive(Debug, Copy, Clone, Hash, Ord, Eq, PartialEq, PartialOrd)] 14 | pub(crate) struct SocketId(pub(crate) u32); 15 | 16 | impl SocketId { 17 | // For DPDK functions 18 | pub(crate) fn raw(&self) -> u32 { 19 | self.0 20 | } 21 | } 22 | 23 | impl fmt::Display for SocketId { 24 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 25 | write!(f, "{}", self.0) 26 | } 27 | } 28 | 29 | /* --------------------------------------------------------------------------------- */ 30 | 31 | /// An identifier for a core running Retina (sink, monitoring, or RX). 32 | #[derive(Debug, Copy, Clone, Hash, Ord, Eq, PartialEq, PartialOrd, Deserialize, Serialize)] 33 | pub struct CoreId(pub u32); 34 | 35 | impl CoreId { 36 | pub(crate) fn socket_id(&self) -> SocketId { 37 | unsafe { SocketId(dpdk::rte_lcore_to_socket_id(self.0)) } 38 | } 39 | 40 | /// The core ID as u32, primarily for DPDK functions 41 | pub fn raw(&self) -> u32 { 42 | self.0 43 | } 44 | } 45 | 46 | impl fmt::Display for CoreId { 47 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 48 | write!(f, "{}", self.0) 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /core/src/lcore/ring.rs: -------------------------------------------------------------------------------- 1 | use crate::dpdk; 2 | use crate::lcore::SocketId; 3 | 4 | use anyhow::{bail, Result}; 5 | use std::ffi::{CStr, CString}; 6 | use std::fmt; 7 | use std::os::raw::{c_int, c_uint, c_void}; 8 | use std::ptr::NonNull; 9 | 10 | /// A wrapper around a ring structure 11 | pub(crate) struct Ring { 12 | raw: NonNull, 13 | } 14 | 15 | unsafe impl Send for Ring {} 16 | unsafe impl Sync for Ring {} 17 | 18 | impl Ring { 19 | pub(crate) fn new(size: u32, socket_id: SocketId, flags: u32) -> Result { 20 | if size == 0 || ((size & size - 1) != 0) { 21 | bail!("Ring size must be a power of 2"); 22 | } 23 | 24 | let name = format!("event_ring_{}", socket_id); 25 | let cname = CString::new(name.clone()).unwrap(); 26 | log::debug!("Ring size: {}", size); 27 | let ring = unsafe { 28 | dpdk::rte_ring_create( 29 | cname.as_ptr(), 30 | size, 31 | socket_id.raw() as c_int, 32 | flags as c_uint, 33 | ) 34 | }; 35 | 36 | let ring_nn = NonNull::new(ring); 37 | match ring_nn { 38 | Some(raw) => Ok(Ring { raw }), 39 | None => bail!("Failed to create ring {}", name), 40 | } 41 | } 42 | 43 | /// For DPDK functions 44 | pub(crate) fn raw(&self) -> &dpdk::rte_ring { 45 | unsafe { self.raw.as_ref() } 46 | } 47 | 48 | /// For DPDK functions 49 | pub(crate) fn raw_mut(&mut self) -> &mut dpdk::rte_ring { 50 | unsafe { self.raw.as_mut() } 51 | } 52 | 53 | /// Returns the name of the Ring 54 | pub(crate) fn name(&self) -> &str { 55 | let cstr = unsafe { CStr::from_ptr(self.raw().name.as_ptr()) }; 56 | cstr.to_str().unwrap() 57 | } 58 | 59 | /// Returns the size of the data store used by the ring (NOT the usable space) 60 | pub(crate) fn size(&self) -> u32 { 61 | unsafe { dpdk::rte_ring_get_size(self.raw()) as u32 } 62 | } 63 | 64 | /// Returns the number of objects can be stored in the ring 65 | pub(crate) fn capacity(&self) -> u32 { 66 | unsafe { dpdk::rte_ring_get_capacity(self.raw()) as u32 } 67 | } 68 | 69 | /// Returns `true` if the ring is full 70 | pub(crate) fn is_full(&self) -> bool { 71 | unsafe { dpdk::rte_ring_full(self.raw()) == 1 } 72 | } 73 | 74 | /// Returns `true` if the ring is empty 75 | pub(crate) fn is_empty(&self) -> bool { 76 | unsafe { dpdk::rte_ring_empty(self.raw()) == 1 } 77 | } 78 | 79 | /// Returns the number of entries in the ring 80 | pub(crate) fn count(&self) -> u32 { 81 | unsafe { dpdk::rte_ring_count(self.raw()) as u32 } 82 | } 83 | 84 | /// Returns the number of free entries in the ring 85 | pub(crate) fn free_count(&self) -> u32 { 86 | unsafe { dpdk::rte_ring_free_count(self.raw()) as u32 } 87 | } 88 | 89 | /// Enqueue object of type `T` onto the ring (multi-producers safe) 90 | pub(crate) fn mp_enqueue(&mut self, obj: T) -> Result<()> { 91 | let ret = unsafe { 92 | dpdk::rte_ring_mp_enqueue( 93 | self.raw_mut(), 94 | Box::into_raw(Box::new(obj)) as *mut _ as *mut c_void, 95 | ) 96 | }; 97 | if ret != 0 { 98 | bail!("Failed to enqueue object"); 99 | } 100 | Ok(()) 101 | } 102 | 103 | /// Enqueue object of type `T` onto the ring (NOT multi-producers safe) 104 | pub(crate) fn sp_enqueue(&mut self, obj: T) -> Result<()> { 105 | let ret = unsafe { 106 | dpdk::rte_ring_sp_enqueue( 107 | self.raw_mut(), 108 | Box::into_raw(Box::new(obj)) as *mut _ as *mut c_void, 109 | ) 110 | }; 111 | if ret != 0 { 112 | bail!("Failed to enqueue object"); 113 | } 114 | Ok(()) 115 | } 116 | 117 | /// Dequeue one object from the ring and return it as `T` (multi-consumers safe) 118 | pub(crate) fn mc_dequeue(&mut self) -> Result { 119 | let mut ptr: *mut c_void = std::ptr::null_mut(); 120 | let tmp: *mut *mut c_void = &mut ptr; 121 | let ret = unsafe { dpdk::rte_ring_mc_dequeue(self.raw_mut(), tmp) }; 122 | if ret != 0 { 123 | bail!("Nothing to dequeue"); 124 | } 125 | let obj = unsafe { Box::from_raw(ptr as *mut T) }; 126 | Ok(*obj) 127 | } 128 | 129 | /// Dequeue one object from the ring and return it as `T` (NOT multi-consumers safe) 130 | pub(crate) fn sc_dequeue(&mut self) -> Result { 131 | let mut ptr: *mut c_void = std::ptr::null_mut(); 132 | let tmp: *mut *mut c_void = &mut ptr; 133 | let ret = unsafe { dpdk::rte_ring_sc_dequeue(self.raw_mut(), tmp) }; 134 | if ret != 0 { 135 | bail!("Nothing to dequeue"); 136 | } 137 | let obj = unsafe { Box::from_raw(ptr as *mut T) }; 138 | Ok(*obj) 139 | } 140 | } 141 | 142 | impl Drop for Ring { 143 | fn drop(&mut self) { 144 | log::info!("Dropping {}.", self.name()); 145 | unsafe { dpdk::rte_ring_free(self.raw_mut()) }; 146 | } 147 | } 148 | 149 | impl fmt::Debug for Ring { 150 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 151 | f.debug_struct(self.name()) 152 | .field("size", &self.size()) 153 | .field("capacity", &self.capacity()) 154 | .field("count", &self.count()) 155 | .field("free_count", &self.free_count()) 156 | .field("is_full", &self.is_full()) 157 | .field("is_empty", &self.is_empty()) 158 | .finish() 159 | } 160 | } 161 | -------------------------------------------------------------------------------- /core/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![allow(clippy::needless_doctest_main)] 2 | // #![warn(missing_docs)] 3 | 4 | //! An ergonomic framework for high speed network traffic analysis on commodity hardware. 5 | //! 6 | //! Retina provides a simple filter and callback interface that lets users subscribe to network 7 | //! traffic in real-time and run user-defined analysis code in a standard software environment. It 8 | //! is a passive analysis framework that supports access to network traffic at one of four 9 | //! abstraction levels: 10 | //! 11 | //! - Individual packets 12 | //! - Reassembled connections 13 | //! - Parsed application-layer sessions 14 | //! - Static (inferrable at first packet and constant throughout the connection) 15 | //! 16 | //! Retina is designed with a focus on performance in real-world, high-volume network environments 17 | //! (e.g., full-network or full-uplink analysis). It employs an efficient filtering mechanism to 18 | //! discard out-of-scope traffic. Due to performance, is not specifically geared towards deep 19 | //! inspection of all packets, though it can be customized to do so with sampling. 20 | //! 21 | //! For filter and callback syntax and usage, see [retina_filtergen](../retina_filtergen). 22 | //! All built-in subscribable datatypes are defined in [retina_datatypes](../retina_datatypes). 23 | //! Additional datatypes in this crate are welcome and encouraged! 24 | //! 25 | //! The following example shows a simple Retina application with two subscriptions, which print 26 | //! (1) parsed TLS handshakes and (2) parsed DNS transactions to stdout: 27 | //! 28 | //! ```rust,ignore 29 | //! use retina_core::config::default_config; 30 | //! use retina_core::Runtime; 31 | //! use retina_filtergen::{retina_main, filter}; 32 | //! use retina_datatypes::*; 33 | //! 34 | //! // Specify a subscription: filter, datatype(s), and callback. The filter determines what 35 | //! // subset of traffic is delivered to the callback. The datatype(s) determine what data is 36 | //! // delivered (here, a parsed TLS handshake). Datatypes are defined in the retina_datatypes 37 | //! // crate and must be passed by immutable reference. 38 | //! // The callback is executed when the filter (here, TLS connection with matching sni) 39 | //! // is matched and the specified data is ready to be delivered (here, when the TLS handshake 40 | //! // is fully parsed). 41 | //! #[filter("tls.sni ~ '^.*\\.com$'")] 42 | //! fn log_tls(tls: &TlsHandshake) { 43 | //! println!("{:?}", tls); 44 | //! } 45 | //! 46 | //! // A Retina application consists of one or more subscriptions. 47 | //! // Define other subscriptions in the same file. 48 | //! #[filter("dns")] 49 | //! fn log_dns(dns: &DnsTransaction) { 50 | //! println!("{:?}", dns); 51 | //! } 52 | //! 53 | //! // When using the `filter` macro to identify subscriptions, include the 54 | //! // `retina_main` attribute with the number of expected subscriptions. 55 | //! #[retina_main(2)] 56 | //! fn main() { 57 | //! // Specify the runtime config (default or from a config file) 58 | //! let cfg = default_config(); 59 | //! // SubscribedWrapper is the type generated at compile-time to "wrap" all 60 | //! // data tracking and delivering functionality, while `filter` wraps all filtering. 61 | //! let runtime:: = Runtime::new(cfg, filter).unwrap(); 62 | //! // Starts Retina 63 | //! runtime.run(); 64 | //! } 65 | //! ``` 66 | //! 67 | //! For programs that require many filters (e.g., searching for 100s of attack signatures), using 68 | //! the [subscription](../retina_filtergen/attr.subscription.html) macro to specify an input TOML 69 | //! file may be preferable to specifying each subscription individually as above. 70 | //! 71 | 72 | #[macro_use] 73 | mod timing; 74 | pub mod config; 75 | pub mod conntrack; 76 | #[doc(hidden)] 77 | #[allow(clippy::all)] 78 | mod dpdk; 79 | pub mod filter; 80 | pub mod lcore; 81 | pub mod memory; 82 | mod port; 83 | pub mod protocols; 84 | mod runtime; 85 | pub mod stats; 86 | #[doc(hidden)] 87 | pub mod subscription; 88 | pub mod utils; 89 | 90 | pub use self::conntrack::conn_id::{ConnId, FiveTuple}; 91 | pub use self::conntrack::pdu::L4Pdu; 92 | pub use self::lcore::CoreId; 93 | pub use self::memory::mbuf::Mbuf; 94 | pub use self::runtime::Runtime; 95 | 96 | pub use dpdk::rte_lcore_id; 97 | pub use dpdk::rte_rdtsc; 98 | 99 | #[macro_use] 100 | extern crate pest_derive; 101 | #[macro_use] 102 | extern crate lazy_static; 103 | #[macro_use] 104 | extern crate maplit; 105 | -------------------------------------------------------------------------------- /core/src/memory/mempool.rs: -------------------------------------------------------------------------------- 1 | //! Memory pools to allocate DPDK message buffers. 2 | 3 | use crate::config::MempoolConfig; 4 | use crate::dpdk; 5 | use crate::lcore::SocketId; 6 | use std::cmp; 7 | use std::ffi::{CStr, CString}; 8 | use std::fmt; 9 | use std::os::raw::{c_int, c_uint}; 10 | use std::ptr::NonNull; 11 | 12 | use anyhow::{Context, Result}; 13 | use thiserror::Error; 14 | 15 | const RX_BUF_ALIGN: u32 = 1024; 16 | 17 | /// A wrapper around a DPDK `rte_mempool` for packet mbufs. 18 | /// It is recommended to allocate one Mempool per NUMA node. 19 | pub(crate) struct Mempool { 20 | raw: NonNull, 21 | } 22 | 23 | impl Mempool { 24 | /// Creates a new mbuf pool on socket_id 25 | pub(crate) fn new(config: &MempoolConfig, socket_id: SocketId, mtu: usize) -> Result { 26 | let data_room = crate::port::mtu_to_max_frame_len(mtu as u32); 27 | let data_room_aligned = round_up(data_room, RX_BUF_ALIGN); 28 | let mbuf_size = data_room_aligned + dpdk::RTE_PKTMBUF_HEADROOM; 29 | let mbuf_size = cmp::max(mbuf_size, dpdk::RTE_MBUF_DEFAULT_BUF_SIZE); 30 | 31 | let name = format!("mempool_{}", socket_id); 32 | let cname = CString::new(name.clone()).expect("Invalid CString conversion"); 33 | let mempool = unsafe { 34 | dpdk::rte_pktmbuf_pool_create( 35 | cname.as_ptr(), 36 | config.capacity as c_uint, 37 | config.cache_size as c_uint, 38 | 0, 39 | mbuf_size.try_into().with_context(|| { 40 | format!("mbuf size {mbuf_size} is larger than 65535, please adjust mtu") 41 | })?, 42 | socket_id.raw() as c_int, 43 | ) 44 | }; 45 | Ok(Mempool { 46 | raw: NonNull::new(mempool).ok_or(MempoolError::Create(name))?, 47 | }) 48 | } 49 | 50 | /// For DPDK functions 51 | pub(crate) fn raw(&self) -> &dpdk::rte_mempool { 52 | unsafe { self.raw.as_ref() } 53 | } 54 | 55 | /// For DPDK functions 56 | pub(crate) fn raw_mut(&mut self) -> &mut dpdk::rte_mempool { 57 | unsafe { self.raw.as_mut() } 58 | } 59 | 60 | /// Mempool name. 61 | pub(crate) fn name(&self) -> &str { 62 | let cstr = unsafe { CStr::from_ptr(self.raw().name.as_ptr()) }; 63 | cstr.to_str().unwrap() 64 | } 65 | 66 | /// Default mbuf size in bytes. 67 | pub(crate) fn default_mtu() -> usize { 68 | 1500 69 | } 70 | } 71 | 72 | impl Drop for Mempool { 73 | fn drop(&mut self) { 74 | log::info!("Dropping {}.", self.name()); 75 | unsafe { dpdk::rte_mempool_free(self.raw_mut()) }; 76 | } 77 | } 78 | 79 | impl fmt::Debug for Mempool { 80 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 81 | let raw = self.raw(); 82 | f.debug_struct(self.name()) 83 | .field("capacity", &raw.size) 84 | .field("cache_size", &raw.cache_size) 85 | .field("socket_id", &raw.socket_id) 86 | .finish() 87 | } 88 | } 89 | 90 | /// Rounds `n` up to the nearest multiple of `s` 91 | fn round_up(n: u32, s: u32) -> u32 { 92 | n.div_ceil(s) * s 93 | } 94 | 95 | #[derive(Error, Debug)] 96 | pub(crate) enum MempoolError { 97 | #[error("Mempool {0} creation failed")] 98 | Create(String), 99 | 100 | #[error("Mbuf allocation failed: mempool exhausted.")] 101 | Exhausted, 102 | } 103 | -------------------------------------------------------------------------------- /core/src/memory/mod.rs: -------------------------------------------------------------------------------- 1 | //! Packet memory buffer management. 2 | 3 | pub mod mbuf; 4 | pub(crate) mod mempool; 5 | -------------------------------------------------------------------------------- /core/src/port/info.rs: -------------------------------------------------------------------------------- 1 | use super::PortId; 2 | use crate::dpdk; 3 | 4 | use std::mem; 5 | 6 | use anyhow::{bail, Result}; 7 | 8 | /* --------------------------------------------------------------------------------- */ 9 | 10 | #[derive(Debug)] 11 | pub(crate) struct PortInfo { 12 | raw: dpdk::rte_eth_dev_info, 13 | } 14 | 15 | impl PortInfo { 16 | pub(crate) fn collect(port_id: PortId) -> Result { 17 | let mut dev_info: dpdk::rte_eth_dev_info = unsafe { mem::zeroed() }; 18 | let ret = unsafe { dpdk::rte_eth_dev_info_get(port_id.raw(), &mut dev_info) }; 19 | if ret < 0 { 20 | bail!("Failed retrieving port information."); 21 | } 22 | 23 | Ok(PortInfo { raw: dev_info }) 24 | } 25 | 26 | /// Displays debug output for the raw device information. 27 | pub(crate) fn display(&self) { 28 | log::debug!("{:#?}", self.raw); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /core/src/protocols/mod.rs: -------------------------------------------------------------------------------- 1 | //! Protocol parsing and manipulation. 2 | pub mod packet; 3 | pub mod stream; 4 | 5 | pub use stream::{ConnData, Session}; 6 | -------------------------------------------------------------------------------- /core/src/protocols/packet/ethernet.rs: -------------------------------------------------------------------------------- 1 | //! Ethernet packet. 2 | 3 | use crate::memory::mbuf::Mbuf; 4 | use crate::protocols::packet::{Packet, PacketHeader, PacketParseError}; 5 | use crate::utils::types::*; 6 | 7 | use anyhow::{bail, Result}; 8 | use pnet::datalink::MacAddr; 9 | 10 | const VLAN_802_1Q: u16 = 0x8100; 11 | const VLAN_802_1AD: u16 = 0x88a8; 12 | 13 | const TAG_SIZE: usize = 4; 14 | const HDR_SIZE: usize = 14; 15 | const HDR_SIZE_802_1Q: usize = HDR_SIZE + TAG_SIZE; 16 | const HDR_SIZE_802_1AD: usize = HDR_SIZE_802_1Q + TAG_SIZE; 17 | 18 | /// An Ethernet frame. 19 | /// 20 | /// On networks that support virtual LANs, the frame may include a VLAN tag after the source MAC 21 | /// address. Double-tagged frames (QinQ) are not yet supported. 22 | #[derive(Debug)] 23 | pub struct Ethernet<'a> { 24 | /// Fixed header. 25 | header: EthernetHeader, 26 | /// Offset to `header` from the start of `mbuf`. 27 | offset: usize, 28 | /// Packet buffer. 29 | mbuf: &'a Mbuf, 30 | } 31 | 32 | impl Ethernet<'_> { 33 | /// Returns the destination MAC address. 34 | #[inline] 35 | pub fn dst(&self) -> MacAddr { 36 | self.header.dst 37 | } 38 | 39 | /// Returns the source MAC address. 40 | #[inline] 41 | pub fn src(&self) -> MacAddr { 42 | self.header.src 43 | } 44 | 45 | /// Returns the encapsulated protocol identifier for untagged and single-tagged frames, and `0` 46 | /// for incorrectly fornatted and (not yet supported) double-tagged frames,. 47 | #[inline] 48 | pub fn ether_type(&self) -> u16 { 49 | self.next_header().unwrap_or(0) as u16 50 | } 51 | 52 | /// Returns the Tag Control Information field from a 802.1Q (single-tagged) 53 | /// frame, if available. 54 | pub fn tci(&self) -> Option { 55 | let ether_type: u16 = u16::from(self.header.ether_type); 56 | match ether_type { 57 | VLAN_802_1Q => { 58 | if let Ok(dot1q) = self.mbuf.get_data(HDR_SIZE) { 59 | let dot1q: Dot1q = unsafe { *dot1q }; 60 | Some(dot1q.tci.into()) 61 | } else { 62 | None 63 | } 64 | } 65 | _ => None, 66 | } 67 | } 68 | } 69 | 70 | impl<'a> Packet<'a> for Ethernet<'a> { 71 | fn mbuf(&self) -> &Mbuf { 72 | self.mbuf 73 | } 74 | 75 | fn header_len(&self) -> usize { 76 | self.header.length() 77 | } 78 | 79 | fn next_header_offset(&self) -> usize { 80 | self.offset + self.header_len() 81 | } 82 | 83 | fn next_header(&self) -> Option { 84 | let ether_type: u16 = u16::from(self.header.ether_type); 85 | match ether_type { 86 | VLAN_802_1Q => { 87 | if let Ok(dot1q) = self.mbuf.get_data(HDR_SIZE) { 88 | let dot1q: Dot1q = unsafe { *dot1q }; 89 | Some(u16::from(dot1q.ether_type).into()) 90 | } else { 91 | None 92 | } 93 | } 94 | VLAN_802_1AD => { 95 | // Unimplemented. TODO: support QinQ 96 | None 97 | } 98 | _ => Some(ether_type.into()), 99 | } 100 | } 101 | 102 | fn parse_from(outer: &'a impl Packet<'a>) -> Result 103 | where 104 | Self: Sized, 105 | { 106 | if let Ok(header) = outer.mbuf().get_data(0) { 107 | Ok(Ethernet { 108 | header: unsafe { *header }, 109 | offset: 0, 110 | mbuf: outer.mbuf(), 111 | }) 112 | } else { 113 | bail!(PacketParseError::InvalidRead) 114 | } 115 | } 116 | } 117 | 118 | /// Fixed portion of an Ethernet header. 119 | #[derive(Debug, Clone, Copy)] 120 | #[repr(C, packed)] 121 | struct EthernetHeader { 122 | dst: MacAddr, 123 | src: MacAddr, 124 | ether_type: u16be, 125 | } 126 | 127 | impl PacketHeader for EthernetHeader { 128 | fn length(&self) -> usize { 129 | match self.ether_type.into() { 130 | VLAN_802_1Q => HDR_SIZE_802_1Q, 131 | VLAN_802_1AD => HDR_SIZE_802_1AD, 132 | _ => HDR_SIZE, 133 | } 134 | } 135 | } 136 | 137 | /// 802.1Q tag control information and next EtherType. 138 | /// 139 | /// ## Remarks 140 | /// This is not a 801.1Q header. The first 16 bits of `Dot1q` is the TCI field and the second 16 141 | /// bits is the EtherType of the encapsulated protocol. 142 | #[derive(Debug, Clone, Copy)] 143 | #[repr(C, packed)] 144 | struct Dot1q { 145 | tci: u16be, 146 | ether_type: u16be, 147 | } 148 | 149 | impl PacketHeader for Dot1q { 150 | /// The four bytes that make up the second byte of the 802.1Q header and the EtherType of the 151 | /// encapsulated protocol. 152 | fn length(&self) -> usize { 153 | TAG_SIZE 154 | } 155 | } 156 | 157 | // TODO: Implement QinQ. 158 | -------------------------------------------------------------------------------- /core/src/protocols/packet/ipv6.rs: -------------------------------------------------------------------------------- 1 | //! IPv6 packet. 2 | 3 | use crate::memory::mbuf::Mbuf; 4 | use crate::protocols::packet::{Packet, PacketHeader, PacketParseError}; 5 | use crate::utils::types::*; 6 | 7 | use std::net::Ipv6Addr; 8 | 9 | use anyhow::{bail, Result}; 10 | 11 | const IPV6_PROTOCOL: usize = 0x86DD; 12 | const IPV6_HEADER_LEN: usize = 40; 13 | 14 | /// An IPv6 packet. 15 | /// 16 | /// Optional IPv6 extension headers are not parsed by default. 17 | #[derive(Debug)] 18 | pub struct Ipv6<'a> { 19 | /// Fixed header. 20 | header: Ipv6Header, 21 | /// Offset to `header` from the start of `mbuf`. 22 | offset: usize, 23 | /// Packet buffer. 24 | mbuf: &'a Mbuf, 25 | } 26 | 27 | impl Ipv6<'_> { 28 | /// Returns the IP protocol version. 29 | #[inline] 30 | pub fn version(&self) -> u8 { 31 | let v: u32 = (self.header.version_to_flow_label & u32be::from(0xf000_0000)).into(); 32 | (v >> 28) as u8 33 | } 34 | 35 | /// Returns the differentiated services code point (DSCP). 36 | #[inline] 37 | pub fn dscp(&self) -> u8 { 38 | let v: u32 = (self.header.version_to_flow_label & u32be::from(0x0fc0_0000)).into(); 39 | (v >> 22) as u8 40 | } 41 | 42 | /// Returns the explicit congestion notification (ECN). 43 | #[inline] 44 | pub fn ecn(&self) -> u8 { 45 | let v: u32 = (self.header.version_to_flow_label & u32be::from(0x0030_0000)).into(); 46 | (v >> 20) as u8 47 | } 48 | 49 | /// Returns the traffic class (former name of differentiated services field). 50 | #[inline] 51 | pub fn traffic_class(&self) -> u8 { 52 | let v: u32 = (self.header.version_to_flow_label & u32be::from(0x0ff0_0000)).into(); 53 | (v >> 20) as u8 54 | } 55 | 56 | /// Returns the flow label. 57 | #[inline] 58 | pub fn flow_label(&self) -> u32 { 59 | (self.header.version_to_flow_label & u32be::from(0x000f_ffff)).into() 60 | } 61 | 62 | /// Returns the 32-bit field containing the version, traffic class, and flow label. 63 | #[inline] 64 | pub fn version_to_flow_label(&self) -> u32 { 65 | self.header.version_to_flow_label.into() 66 | } 67 | 68 | /// Returns the length of the payload in bytes. 69 | #[inline] 70 | pub fn payload_length(&self) -> u16 { 71 | self.header.payload_length.into() 72 | } 73 | 74 | /// Returns the encapsulated protocol identifier. 75 | #[inline] 76 | pub fn next_header(&self) -> u8 { 77 | self.header.next_header 78 | } 79 | 80 | /// Returns hop limit/time to live of the packet. 81 | #[inline] 82 | pub fn hop_limit(&self) -> u8 { 83 | self.header.hop_limit 84 | } 85 | 86 | /// Returns the sender's IPv6 address. 87 | #[inline] 88 | pub fn src_addr(&self) -> Ipv6Addr { 89 | self.header.src_addr 90 | } 91 | 92 | /// Returns the receiver's IPv6 address. 93 | #[inline] 94 | pub fn dst_addr(&self) -> Ipv6Addr { 95 | self.header.dst_addr 96 | } 97 | } 98 | 99 | impl<'a> Packet<'a> for Ipv6<'a> { 100 | fn mbuf(&self) -> &Mbuf { 101 | self.mbuf 102 | } 103 | 104 | fn header_len(&self) -> usize { 105 | self.header.length() 106 | } 107 | 108 | fn next_header_offset(&self) -> usize { 109 | self.offset + self.header_len() 110 | } 111 | 112 | fn next_header(&self) -> Option { 113 | Some(self.next_header().into()) 114 | } 115 | 116 | fn parse_from(outer: &'a impl Packet<'a>) -> Result 117 | where 118 | Self: Sized, 119 | { 120 | let offset = outer.next_header_offset(); 121 | if let Ok(header) = outer.mbuf().get_data(offset) { 122 | match outer.next_header() { 123 | Some(IPV6_PROTOCOL) => Ok(Ipv6 { 124 | header: unsafe { *header }, 125 | offset, 126 | mbuf: outer.mbuf(), 127 | }), 128 | _ => bail!(PacketParseError::InvalidProtocol), 129 | } 130 | } else { 131 | bail!(PacketParseError::InvalidRead) 132 | } 133 | } 134 | } 135 | 136 | // Fixed portion of Ipv6 header TODO: handle extension headers 137 | #[derive(Debug, Clone, Copy)] 138 | #[repr(C)] 139 | struct Ipv6Header { 140 | version_to_flow_label: u32be, 141 | payload_length: u16be, 142 | next_header: u8, 143 | hop_limit: u8, 144 | src_addr: Ipv6Addr, 145 | dst_addr: Ipv6Addr, 146 | } 147 | 148 | impl PacketHeader for Ipv6Header { 149 | /// Payload offset 150 | fn length(&self) -> usize { 151 | IPV6_HEADER_LEN 152 | } 153 | } 154 | -------------------------------------------------------------------------------- /core/src/protocols/packet/mod.rs: -------------------------------------------------------------------------------- 1 | //! Types for parsing and manipulating packet-level network protocols. 2 | //! 3 | //! The structure of this module is adapted from 4 | //! [capsule::packets](https://docs.rs/capsule/0.1.5/capsule/packets/index.html) and 5 | //! [pnet::packet](https://docs.rs/pnet/latest/pnet/packet/index.html). Every packet type represents 6 | //! a single frame on the wire. 7 | 8 | pub mod ethernet; 9 | pub mod ipv4; 10 | pub mod ipv6; 11 | pub mod tcp; 12 | pub mod udp; 13 | use crate::memory::mbuf::Mbuf; 14 | 15 | use anyhow::Result; 16 | use thiserror::Error; 17 | 18 | /// Represents a single packet. 19 | pub trait Packet<'a> { 20 | /// Reference to the underlying packet buffer. 21 | fn mbuf(&self) -> &Mbuf; 22 | 23 | /// Offset from the beginning of the header to the start of the payload. 24 | fn header_len(&self) -> usize; 25 | 26 | /// Offset from the beginning of the packet buffer to the start of the payload. 27 | fn next_header_offset(&self) -> usize; 28 | 29 | /// Next level IANA protocol number. 30 | fn next_header(&self) -> Option; 31 | 32 | /// Parses the `Packet`'s payload as a new `Packet` of type `T`. 33 | fn parse_to>(&'a self) -> Result 34 | where 35 | Self: Sized, 36 | { 37 | T::parse_from(self) 38 | } 39 | 40 | /// Parses a `Packet` from the outer encapsulating `Packet`'s payload. 41 | fn parse_from(outer: &'a impl Packet<'a>) -> Result 42 | where 43 | Self: Sized; 44 | } 45 | 46 | /// Represents a packet header. 47 | pub trait PacketHeader { 48 | /// Offset from beginning of the header to start of the payload. It includes the length of any 49 | /// variable-sized options and tags. 50 | fn length(&self) -> usize; 51 | 52 | /// Size of the fixed portion of the header in bytes. 53 | fn size_of() -> usize 54 | where 55 | Self: Sized, 56 | { 57 | std::mem::size_of::() 58 | } 59 | } 60 | 61 | #[derive(Error, Debug)] 62 | pub(crate) enum PacketParseError { 63 | #[error("Invalid protocol")] 64 | InvalidProtocol, 65 | 66 | #[error("Invalid data read")] 67 | InvalidRead, 68 | } 69 | -------------------------------------------------------------------------------- /core/src/protocols/packet/udp.rs: -------------------------------------------------------------------------------- 1 | //! UDP packet. 2 | 3 | use crate::memory::mbuf::Mbuf; 4 | use crate::protocols::packet::{Packet, PacketHeader, PacketParseError}; 5 | use crate::utils::types::*; 6 | 7 | use anyhow::{bail, Result}; 8 | 9 | /// UDP assigned protocol number. 10 | pub const UDP_PROTOCOL: usize = 17; 11 | const UDP_HEADER_LEN: usize = 8; 12 | 13 | /// A UDP packet. 14 | #[derive(Debug)] 15 | pub struct Udp<'a> { 16 | /// Fixed header. 17 | header: UdpHeader, 18 | /// Offset to `header` from the start of `mbuf`. 19 | offset: usize, 20 | /// Packet buffer. 21 | mbuf: &'a Mbuf, 22 | } 23 | 24 | impl Udp<'_> { 25 | /// Returns the sending port. 26 | #[inline] 27 | pub fn src_port(&self) -> u16 { 28 | self.header.src_port.into() 29 | } 30 | 31 | /// Returns the receiving port. 32 | #[inline] 33 | pub fn dst_port(&self) -> u16 { 34 | self.header.dst_port.into() 35 | } 36 | 37 | /// Returns the length of packet (both header and payload) in bytes. 38 | #[inline] 39 | pub fn length(&self) -> u16 { 40 | self.header.length.into() 41 | } 42 | 43 | /// Returns the UDP checksum. 44 | #[inline] 45 | pub fn checksum(&self) -> u16 { 46 | self.header.checksum.into() 47 | } 48 | } 49 | 50 | impl<'a> Packet<'a> for Udp<'a> { 51 | fn mbuf(&self) -> &Mbuf { 52 | self.mbuf 53 | } 54 | 55 | fn header_len(&self) -> usize { 56 | self.header.length() 57 | } 58 | 59 | fn next_header_offset(&self) -> usize { 60 | self.offset + self.header_len() 61 | } 62 | 63 | fn next_header(&self) -> Option { 64 | None 65 | } 66 | 67 | fn parse_from(outer: &'a impl Packet<'a>) -> Result 68 | where 69 | Self: Sized, 70 | { 71 | let offset = outer.next_header_offset(); 72 | if let Ok(header) = outer.mbuf().get_data(offset) { 73 | match outer.next_header() { 74 | Some(UDP_PROTOCOL) => Ok(Udp { 75 | header: unsafe { *header }, 76 | offset, 77 | mbuf: outer.mbuf(), 78 | }), 79 | _ => bail!(PacketParseError::InvalidProtocol), 80 | } 81 | } else { 82 | bail!(PacketParseError::InvalidRead) 83 | } 84 | } 85 | } 86 | 87 | /// UDP header. 88 | #[derive(Debug, Clone, Copy)] 89 | #[repr(C, packed)] 90 | struct UdpHeader { 91 | src_port: u16be, 92 | dst_port: u16be, 93 | length: u16be, 94 | checksum: u16be, 95 | } 96 | 97 | impl PacketHeader for UdpHeader { 98 | /// Header length measured in bytes. Equivalent to the payload offset. 99 | fn length(&self) -> usize { 100 | UDP_HEADER_LEN 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /core/src/protocols/stream/conn/layer3.rs: -------------------------------------------------------------------------------- 1 | use crate::protocols::stream::ConnData; 2 | use std::net::{Ipv4Addr, Ipv6Addr, SocketAddr}; 3 | 4 | use super::{ConnDataError, ConnField}; 5 | use anyhow::{bail, Result}; 6 | 7 | /// IPv4 Connection Metadata. 8 | #[derive(Debug)] 9 | pub struct Ipv4CData { 10 | src_addr: Ipv4Addr, 11 | dst_addr: Ipv4Addr, 12 | } 13 | 14 | impl Ipv4CData { 15 | #[inline] 16 | pub fn src_addr(&self) -> Ipv4Addr { 17 | self.src_addr 18 | } 19 | 20 | #[inline] 21 | pub fn dst_addr(&self) -> Ipv4Addr { 22 | self.dst_addr 23 | } 24 | } 25 | 26 | impl ConnField for Ipv4CData { 27 | fn supported_fields() -> Vec<&'static str> { 28 | vec!["src_addr", "dst_addr"] 29 | } 30 | 31 | fn parse_from(conn_data: &ConnData) -> Result { 32 | if let SocketAddr::V4(src) = conn_data.five_tuple.orig { 33 | if let SocketAddr::V4(dst) = conn_data.five_tuple.resp { 34 | return Ok(Self { 35 | src_addr: *src.ip(), 36 | dst_addr: *dst.ip(), 37 | }); 38 | } 39 | } 40 | bail!(ConnDataError::InvalidProtocol) 41 | } 42 | } 43 | 44 | /// IPv6 Connection Metadata. 45 | #[derive(Debug)] 46 | pub struct Ipv6CData { 47 | src_addr: Ipv6Addr, 48 | dst_addr: Ipv6Addr, 49 | } 50 | 51 | impl Ipv6CData { 52 | #[inline] 53 | pub fn src_addr(&self) -> Ipv6Addr { 54 | self.src_addr 55 | } 56 | 57 | #[inline] 58 | pub fn dst_addr(&self) -> Ipv6Addr { 59 | self.dst_addr 60 | } 61 | } 62 | 63 | impl ConnField for Ipv6CData { 64 | fn supported_fields() -> Vec<&'static str> { 65 | vec!["src_addr", "dst_addr"] 66 | } 67 | 68 | fn parse_from(conn_data: &ConnData) -> Result { 69 | if let SocketAddr::V6(src) = conn_data.five_tuple.orig { 70 | if let SocketAddr::V6(dst) = conn_data.five_tuple.resp { 71 | return Ok(Self { 72 | src_addr: *src.ip(), 73 | dst_addr: *dst.ip(), 74 | }); 75 | } 76 | } 77 | bail!(ConnDataError::InvalidProtocol) 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /core/src/protocols/stream/conn/layer4.rs: -------------------------------------------------------------------------------- 1 | use super::{ConnDataError, ConnField}; 2 | use crate::protocols::packet::tcp::TCP_PROTOCOL; 3 | use crate::protocols::packet::udp::UDP_PROTOCOL; 4 | use crate::protocols::stream::ConnData; 5 | use anyhow::{bail, Result}; 6 | use std::net::SocketAddr; 7 | 8 | /// TCP Connection Metadata, parsed from ConnData. 9 | #[derive(Debug)] 10 | pub struct TcpCData { 11 | src_port: u16, 12 | dst_port: u16, 13 | } 14 | 15 | impl TcpCData { 16 | #[inline] 17 | pub fn src_port(&self) -> u16 { 18 | self.src_port 19 | } 20 | 21 | /// Returns the receiving port. 22 | #[inline] 23 | pub fn dst_port(&self) -> u16 { 24 | self.dst_port 25 | } 26 | } 27 | 28 | impl ConnField for TcpCData { 29 | fn supported_fields() -> Vec<&'static str> { 30 | vec!["src_port", "dst_port"] 31 | } 32 | 33 | fn parse_from(conn_data: &ConnData) -> Result { 34 | if matches!(conn_data.five_tuple.proto, TCP_PROTOCOL) { 35 | if let SocketAddr::V4(src) = conn_data.five_tuple.orig { 36 | if let SocketAddr::V4(dst) = conn_data.five_tuple.resp { 37 | return Ok(Self { 38 | src_port: src.port(), 39 | dst_port: dst.port(), 40 | }); 41 | } 42 | } else if let SocketAddr::V6(src) = conn_data.five_tuple.orig { 43 | if let SocketAddr::V6(dst) = conn_data.five_tuple.resp { 44 | return Ok(Self { 45 | src_port: src.port(), 46 | dst_port: dst.port(), 47 | }); 48 | } 49 | } 50 | } 51 | bail!(ConnDataError::InvalidProtocol) 52 | } 53 | } 54 | /// UDP Connection Metadata. 55 | #[derive(Debug)] 56 | pub struct UdpCData { 57 | src_port: u16, 58 | dst_port: u16, 59 | } 60 | 61 | impl UdpCData { 62 | #[inline] 63 | pub fn src_port(&self) -> u16 { 64 | self.src_port 65 | } 66 | 67 | /// Returns the receiving port. 68 | #[inline] 69 | pub fn dst_port(&self) -> u16 { 70 | self.dst_port 71 | } 72 | } 73 | 74 | impl ConnField for UdpCData { 75 | fn supported_fields() -> Vec<&'static str> { 76 | vec!["src_port", "dst_port"] 77 | } 78 | 79 | fn parse_from(conn_data: &ConnData) -> Result { 80 | if matches!(conn_data.five_tuple.proto, UDP_PROTOCOL) { 81 | if let SocketAddr::V4(src) = conn_data.five_tuple.orig { 82 | if let SocketAddr::V4(dst) = conn_data.five_tuple.resp { 83 | return Ok(Self { 84 | src_port: src.port(), 85 | dst_port: dst.port(), 86 | }); 87 | } 88 | } else if let SocketAddr::V6(src) = conn_data.five_tuple.orig { 89 | if let SocketAddr::V6(dst) = conn_data.five_tuple.resp { 90 | return Ok(Self { 91 | src_port: src.port(), 92 | dst_port: dst.port(), 93 | }); 94 | } 95 | } 96 | } 97 | bail!(ConnDataError::InvalidProtocol) 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /core/src/protocols/stream/conn/mod.rs: -------------------------------------------------------------------------------- 1 | /// Types for parsing FiveTuple data from a Conn struct. 2 | /// This is used by retina-filtergen if a packet-level field must be checked when the raw 3 | /// packet is not available, but connection data is. 4 | pub mod layer3; 5 | pub mod layer4; 6 | 7 | pub use layer3::{Ipv4CData, Ipv6CData}; 8 | pub use layer4::{TcpCData, UdpCData}; 9 | 10 | use crate::protocols::stream::ConnData; 11 | use anyhow::Result; 12 | use thiserror::Error; 13 | 14 | #[derive(Error, Debug)] 15 | pub(crate) enum ConnDataError { 16 | #[error("Invalid protocol")] 17 | InvalidProtocol, 18 | } 19 | 20 | /// A trait that all extractable ConnData fields must implement 21 | /// This is used for filtering on L3/L4 fields following the packet 22 | /// filter stage (i.e., when protocol, port, etc. cannot be extracted 23 | /// fro a packet). 24 | pub trait ConnField { 25 | /// Parse from the ConnData 26 | fn parse_from(conn: &ConnData) -> Result 27 | where 28 | Self: Sized; 29 | 30 | /// Supported methods, as strings (e.g., src_port, dst_addr...) 31 | /// This allows for better error-checking at compile-time 32 | fn supported_fields() -> Vec<&'static str>; 33 | } 34 | -------------------------------------------------------------------------------- /core/src/protocols/stream/dns/mod.rs: -------------------------------------------------------------------------------- 1 | //! DNS transaction parsing. 2 | 3 | pub mod parser; 4 | mod transaction; 5 | 6 | pub use self::transaction::*; 7 | 8 | use serde::Serialize; 9 | 10 | /// Parsed DNS transaction contents. 11 | /// 12 | /// A DNS transaction consists of a query and a response. 13 | #[derive(Clone, Debug, Serialize)] 14 | pub struct Dns { 15 | /// DNS transaction ID. 16 | pub transaction_id: u16, 17 | /// DNS Query. 18 | pub query: Option, 19 | /// DNS Response. 20 | pub response: Option, 21 | } 22 | 23 | impl Dns { 24 | /// Returns the DNS query domain name, or `""` if no query was observed in the transaction. 25 | pub fn query_domain(&self) -> &str { 26 | if let Some(query) = &self.query { 27 | if !query.queries.is_empty() { 28 | return &query.queries[0]; 29 | } 30 | } 31 | "" 32 | } 33 | 34 | /// Returns a string representation of the answers 35 | pub fn answers(&self) -> String { 36 | if let Some(resp) = &self.response { 37 | if !resp.answers.is_empty() { 38 | return serde_json::to_string(&resp.answers).unwrap_or(String::new()); 39 | } 40 | } 41 | String::new() 42 | } 43 | 44 | /// Returns a string representation of the response nameservers 45 | pub fn nameservers(&self) -> String { 46 | if let Some(resp) = &self.response { 47 | if !resp.nameservers.is_empty() { 48 | return serde_json::to_string(&resp.nameservers).unwrap_or(String::new()); 49 | } 50 | } 51 | String::new() 52 | } 53 | 54 | /// Returns a string representation of the response additionals 55 | pub fn additionals(&self) -> String { 56 | if let Some(resp) = &self.response { 57 | if !resp.additionals.is_empty() { 58 | return serde_json::to_string(&resp.additionals).unwrap_or(String::new()); 59 | } 60 | } 61 | String::new() 62 | } 63 | 64 | /// Returns a string representation of the response 65 | pub fn response(&self) -> String { 66 | if let Some(resp) = &self.response { 67 | return serde_json::to_string(&resp).unwrap_or(String::new()); 68 | } 69 | String::new() 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /core/src/protocols/stream/dns/transaction.rs: -------------------------------------------------------------------------------- 1 | //! DNS transaction components. 2 | 3 | use dns_parser::rdata::{Aaaa, RData, A}; 4 | use dns_parser::{Packet, ResponseCode}; 5 | 6 | use serde::Serialize; 7 | 8 | /// A DNS Query. 9 | #[derive(Clone, Debug, Serialize)] 10 | pub struct DnsQuery { 11 | pub num_questions: u16, 12 | pub recursion_desired: bool, // appears in query & answer 13 | pub queries: Vec, // typically only one question per query, could have multiple 14 | } 15 | 16 | impl DnsQuery { 17 | pub(super) fn parse_query(pkt: &Packet) -> Self { 18 | let mut queries = Vec::new(); 19 | for q in &pkt.questions { 20 | log::debug!(" query: {}/{:?}", q.qname, q.qtype); 21 | queries.push(q.qname.to_string()); 22 | } 23 | DnsQuery { 24 | num_questions: pkt.header.questions, 25 | recursion_desired: pkt.header.recursion_desired, 26 | queries, 27 | } 28 | } 29 | } 30 | 31 | /// A DNS Response. 32 | #[derive(Clone, Debug, Serialize)] 33 | pub struct DnsResponse { 34 | pub response_code: ResponseCode, 35 | pub authoritative: bool, // if the DNS server is authoritative for the queried hostname, appear in answer 36 | pub recursion_available: bool, // appear in answer 37 | pub num_answers: u16, 38 | pub num_additional: u16, // the number of records in Additional section in answer 39 | pub num_nameservers: u16, // the number of records in Authority section in answer 40 | pub answers: Vec, 41 | pub nameservers: Vec, 42 | pub additionals: Vec, 43 | } 44 | 45 | impl DnsResponse { 46 | pub(super) fn parse_response(pkt: &Packet) -> Self { 47 | let mut answers = Vec::new(); 48 | for answer in &pkt.answers { 49 | log::debug!(" answer: {}/{:?}", answer.name, answer.data); 50 | let data = Data::new(&answer.data); 51 | answers.push(DnsRecord { 52 | name: answer.name.to_string(), 53 | data, 54 | ttl: answer.ttl, 55 | }); 56 | } 57 | let mut nameservers = Vec::new(); 58 | for nameserver in &pkt.nameservers { 59 | let data = Data::new(&nameserver.data); 60 | nameservers.push(DnsRecord { 61 | name: nameserver.name.to_string(), 62 | data, 63 | ttl: nameserver.ttl, 64 | }); 65 | } 66 | let mut additionals = Vec::new(); 67 | for additional in &pkt.additional { 68 | let data = Data::new(&additional.data); 69 | additionals.push(DnsRecord { 70 | name: additional.name.to_string(), 71 | data, 72 | ttl: additional.ttl, 73 | }); 74 | } 75 | DnsResponse { 76 | response_code: pkt.header.response_code, 77 | authoritative: pkt.header.authoritative, 78 | recursion_available: pkt.header.recursion_available, 79 | num_answers: pkt.header.answers, 80 | num_additional: pkt.header.additional, 81 | num_nameservers: pkt.header.nameservers, 82 | answers, 83 | nameservers, 84 | additionals, 85 | } 86 | } 87 | } 88 | 89 | /// A DNS Record. 90 | #[derive(Clone, Debug, Serialize)] 91 | pub struct DnsRecord { 92 | pub name: String, 93 | pub data: Data, 94 | pub ttl: u32, 95 | } 96 | 97 | /// RData types. 98 | #[derive(Clone, Debug, Serialize)] 99 | pub enum Data { 100 | A(A), 101 | Aaaa(Aaaa), 102 | Cname(String), 103 | Mx(Mx), 104 | Ns(String), 105 | Ptr(String), 106 | Soa(Soa), 107 | Srv(Srv), 108 | Txt(String), 109 | Unknown, 110 | } 111 | 112 | impl Data { 113 | fn new(data: &RData) -> Self { 114 | match data { 115 | RData::A(a) => Data::A(*a), 116 | RData::AAAA(a) => Data::Aaaa(*a), 117 | RData::CNAME(a) => Data::Cname(a.0.to_string()), 118 | RData::MX(a) => Data::Mx(Mx { 119 | preference: a.preference, 120 | exchange: a.exchange.to_string(), 121 | }), 122 | RData::NS(a) => Data::Ns(a.0.to_string()), 123 | RData::PTR(a) => Data::Ptr(a.0.to_string()), 124 | RData::SOA(a) => Data::Soa(Soa { 125 | primary_ns: a.primary_ns.to_string(), 126 | mailbox: a.mailbox.to_string(), 127 | serial: a.serial, 128 | refresh: a.refresh, 129 | retry: a.retry, 130 | expire: a.expire, 131 | minimum_ttl: a.minimum_ttl, 132 | }), 133 | RData::SRV(a) => Data::Srv(Srv { 134 | priority: a.priority, 135 | weight: a.weight, 136 | port: a.port, 137 | target: a.target.to_string(), 138 | }), 139 | RData::TXT(a) => Data::Txt(String::from_utf8_lossy(a.bytes).to_string()), 140 | RData::Unknown(..) => Data::Unknown, 141 | } 142 | } 143 | } 144 | 145 | /// A DNS mail exchange (MX) record. 146 | #[derive(Debug, PartialEq, Eq, Clone, Serialize)] 147 | pub struct Mx { 148 | pub preference: u16, 149 | pub exchange: String, 150 | } 151 | 152 | /// A DNS start of authority (SOA) record. 153 | #[derive(Clone, Debug, PartialEq, Eq, Serialize)] 154 | pub struct Soa { 155 | pub primary_ns: String, 156 | pub mailbox: String, 157 | pub serial: u32, 158 | pub refresh: u32, 159 | pub retry: u32, 160 | pub expire: u32, 161 | pub minimum_ttl: u32, 162 | } 163 | 164 | /// A DNS service (SRV) record. 165 | #[derive(Clone, Debug, PartialEq, Eq, Serialize)] 166 | pub struct Srv { 167 | pub priority: u16, 168 | pub weight: u16, 169 | pub port: u16, 170 | pub target: String, 171 | } 172 | -------------------------------------------------------------------------------- /core/src/protocols/stream/http/mod.rs: -------------------------------------------------------------------------------- 1 | //! HTTP transaction parsing. 2 | //! 3 | //! ## Remarks 4 | //! Retina currently only parses HTTP 1.x request and response headers, and does not attempt to 5 | //! parse or defragment HTTP bodies that may span multiple packets. This is enough for basic HTTP 6 | //! header analysis, but not for deep inspection of message body contents. Support for 7 | //! request/response continuations, chunked transfer encoding, and body content retrieval, are in 8 | //! progress. 9 | //! 10 | //! This module does support parsing pipelined requests and maintains state for linking requests and 11 | //! responses. 12 | //! 13 | /* 14 | TODO: support request/response continuations (body that spans multiple packets) 15 | TODO: support chunked transfer encoding 16 | TODO: provide request/response body 17 | TODO: HTTP/2 support 18 | */ 19 | 20 | pub mod parser; 21 | mod transaction; 22 | 23 | pub use self::transaction::{HttpRequest, HttpResponse}; 24 | 25 | use serde::Serialize; 26 | 27 | /// Parsed HTTP transaction contents. 28 | #[derive(Debug, Serialize, Clone)] 29 | pub struct Http { 30 | /// HTTP Request. 31 | pub request: HttpRequest, 32 | /// HTTP Response. 33 | pub response: HttpResponse, 34 | /// The pipelined depth into the connection of this transaction. 35 | pub trans_depth: usize, 36 | } 37 | 38 | impl Http { 39 | /// Returns the request URI, or `""` if it does not exist. 40 | pub fn uri(&self) -> &str { 41 | self.request.uri.as_deref().unwrap_or("") 42 | } 43 | 44 | /// Returns the HTTP method, or `""` if it does not exist. 45 | pub fn method(&self) -> &str { 46 | self.request.method.as_deref().unwrap_or("") 47 | } 48 | 49 | /// Returns the HTTP request version, or `""` if it does not exist. 50 | pub fn request_version(&self) -> &str { 51 | self.request.version.as_deref().unwrap_or("") 52 | } 53 | 54 | /// Returns the user agent string of the user agent, or `""` if it does not exist. 55 | pub fn user_agent(&self) -> &str { 56 | self.request.user_agent.as_deref().unwrap_or("") 57 | } 58 | 59 | /// Returns HTTP cookies sent by the client, or `""` if it does not exist. 60 | pub fn cookie(&self) -> &str { 61 | self.request.cookie.as_deref().unwrap_or("") 62 | } 63 | 64 | /// Returns the domain name of the server specified by the client, or `""` if it does not exist. 65 | pub fn host(&self) -> &str { 66 | self.request.host.as_deref().unwrap_or("") 67 | } 68 | 69 | /// Returns the size of the request body in bytes, or `0` if it does not exist. 70 | pub fn request_content_length(&self) -> usize { 71 | self.request.content_length.unwrap_or(0) 72 | } 73 | 74 | /// Returns the media type of the request resource, or `""` if it does not exist. 75 | pub fn request_content_type(&self) -> &str { 76 | self.request.content_type.as_deref().unwrap_or("") 77 | } 78 | 79 | /// Returns the form of encoding used to transfer the request body, or `""` if it does not 80 | /// exist. 81 | pub fn request_transfer_encoding(&self) -> &str { 82 | self.request.transfer_encoding.as_deref().unwrap_or("") 83 | } 84 | 85 | /// Returns the HTTP response version, or `""` if it does not exist. 86 | pub fn response_version(&self) -> &str { 87 | self.response.version.as_deref().unwrap_or("") 88 | } 89 | 90 | /// Returns the HTTP status code, or `0` if it does not exist. 91 | pub fn status_code(&self) -> u16 { 92 | self.response.status_code.unwrap_or(0) 93 | } 94 | 95 | /// Returns the HTTP status tet, or `0` if it does not exist. 96 | pub fn status_msg(&self) -> &str { 97 | self.response.status_msg.as_deref().unwrap_or("") 98 | } 99 | 100 | /// Returns the size of the request body in bytes, or `0` if it does not exist. 101 | pub fn response_content_length(&self) -> usize { 102 | self.response.content_length.unwrap_or(0) 103 | } 104 | 105 | /// Returns the media type of the response resource, or `""` if it does not exist. 106 | pub fn response_content_type(&self) -> &str { 107 | self.response.content_type.as_deref().unwrap_or("") 108 | } 109 | 110 | /// Returns the form of encoding used to transfer the response body, or `""` if it does not 111 | /// exist. 112 | pub fn response_transfer_encoding(&self) -> &str { 113 | self.response.transfer_encoding.as_deref().unwrap_or("") 114 | } 115 | 116 | // TODO: more methods... 117 | } 118 | -------------------------------------------------------------------------------- /core/src/protocols/stream/http/parser.rs: -------------------------------------------------------------------------------- 1 | // modified from https://github.com/rusticata/rusticata/blob/master/src/http.rs 2 | //! HTTP transaction parser. 3 | //! 4 | //! The HTTP transaction parser uses the [httparse](https://docs.rs/httparse/latest/httparse/) crate to parse HTTP request/responses. It handles HTTP pipelining, but does not currently support defragmenting message bodies. 5 | //! 6 | 7 | use super::transaction::{HttpRequest, HttpResponse}; 8 | use super::Http; 9 | use crate::conntrack::pdu::L4Pdu; 10 | use crate::protocols::stream::{ 11 | ConnParsable, ParseResult, ParsingState, ProbeResult, Session, SessionData, 12 | }; 13 | 14 | use httparse::{Request, EMPTY_HEADER}; 15 | use std::collections::HashMap; 16 | 17 | #[derive(Default, Debug)] 18 | pub struct HttpParser { 19 | /// Pending requests: maps session ID to HTTP transaction. 20 | pending: HashMap, 21 | /// Current outstanding request ID (transaction depth). 22 | current_trans: usize, 23 | /// The current deepest transaction (total transactions ever seen). 24 | cnt: usize, 25 | } 26 | 27 | impl HttpParser { 28 | /// Process data segments from client to server 29 | pub(crate) fn process_ctos(&mut self, data: &[u8]) -> ParseResult { 30 | if let Ok(request) = HttpRequest::parse_from(data) { 31 | let session_id = self.cnt; 32 | let http = Http { 33 | request, 34 | response: HttpResponse::default(), 35 | trans_depth: session_id, 36 | }; 37 | self.cnt += 1; 38 | self.pending.insert(session_id, http); 39 | ParseResult::Continue(session_id) 40 | } else { 41 | // request continuation data or parse error. 42 | // TODO: parse request continuation data 43 | ParseResult::Skipped 44 | } 45 | } 46 | 47 | /// Process data segments from server to client 48 | pub(crate) fn process_stoc(&mut self, data: &[u8], pdu: &L4Pdu) -> ParseResult { 49 | if let Ok(response) = HttpResponse::parse_from(data) { 50 | if let Some(http) = self.pending.get_mut(&self.current_trans) { 51 | http.response = response; 52 | // TODO: Handle response continuation data instead of returning 53 | // ParseResult::Done immediately on Response start-line 54 | ParseResult::Done(self.current_trans) 55 | } else { 56 | log::warn!("HTTP response without oustanding request: {:?}", pdu.ctxt); 57 | ParseResult::Skipped 58 | } 59 | } else { 60 | // response continuation data or parse error. 61 | // TODO: parse response continuation data 62 | ParseResult::Skipped 63 | } 64 | } 65 | } 66 | 67 | impl ConnParsable for HttpParser { 68 | fn parse(&mut self, pdu: &L4Pdu) -> ParseResult { 69 | let offset = pdu.offset(); 70 | let length = pdu.length(); 71 | if length == 0 { 72 | return ParseResult::Skipped; 73 | } 74 | 75 | if let Ok(data) = (pdu.mbuf_ref()).get_data_slice(offset, length) { 76 | if pdu.dir { 77 | self.process_ctos(data) 78 | } else { 79 | self.process_stoc(data, pdu) 80 | } 81 | } else { 82 | log::warn!("Malformed packet on parse"); 83 | ParseResult::Skipped 84 | } 85 | } 86 | 87 | fn probe(&self, pdu: &L4Pdu) -> ProbeResult { 88 | // adapted from [the Rusticata HTTP parser](https://github.com/rusticata/rusticata/blob/master/src/http.rs) 89 | 90 | // number of headers to parse at once 91 | const NUM_OF_HEADERS: usize = 4; 92 | 93 | if pdu.length() < 6 { 94 | return ProbeResult::Unsure; 95 | } 96 | let offset = pdu.offset(); 97 | let length = pdu.length(); 98 | if let Ok(data) = (pdu.mbuf_ref()).get_data_slice(offset, length) { 99 | // check if first characters match start of "request-line" 100 | match &data[..4] { 101 | b"OPTI" | b"GET " | b"HEAD" | b"POST" | b"PUT " | b"PATC" | b"COPY" | b"MOVE" 102 | | b"DELE" | b"LINK" | b"UNLI" | b"TRAC" | b"WRAP" => (), 103 | _ => return ProbeResult::NotForUs, 104 | } 105 | // try parsing request 106 | let mut headers = [EMPTY_HEADER; NUM_OF_HEADERS]; 107 | let mut req = Request::new(&mut headers[..]); 108 | let status = req.parse(data); 109 | if let Err(e) = status { 110 | if e != httparse::Error::TooManyHeaders { 111 | log::trace!( 112 | "data could be HTTP, but got error {:?} while parsing", 113 | status 114 | ); 115 | return ProbeResult::Unsure; 116 | } 117 | } 118 | ProbeResult::Certain 119 | } else { 120 | log::warn!("Malformed packet"); 121 | ProbeResult::Error 122 | } 123 | } 124 | 125 | fn remove_session(&mut self, session_id: usize) -> Option { 126 | // Increment to next outstanding transaction in request order 127 | self.current_trans = session_id + 1; 128 | self.pending.remove(&session_id).map(|http| Session { 129 | data: SessionData::Http(Box::new(http)), 130 | id: session_id, 131 | }) 132 | } 133 | 134 | fn drain_sessions(&mut self) -> Vec { 135 | self.pending 136 | .drain() 137 | .map(|(session_id, http)| Session { 138 | data: SessionData::Http(Box::new(http)), 139 | id: session_id, 140 | }) 141 | .collect() 142 | } 143 | 144 | fn session_parsed_state(&self) -> ParsingState { 145 | ParsingState::Parsing 146 | } 147 | } 148 | -------------------------------------------------------------------------------- /core/src/protocols/stream/quic/header.rs: -------------------------------------------------------------------------------- 1 | //! Quic header types 2 | 3 | use serde::Serialize; 4 | 5 | use crate::protocols::stream::quic::QuicError; 6 | 7 | /// Quic Long Header 8 | #[derive(Debug, Serialize, Clone)] 9 | pub struct QuicLongHeader { 10 | pub packet_type: LongHeaderPacketType, 11 | pub type_specific: u8, 12 | pub version: u32, 13 | pub dcid_len: u8, // length of dcid in bytes 14 | pub dcid: String, // hex string 15 | pub scid_len: u8, // length of scid in bytes 16 | pub scid: String, // hex string 17 | pub token_len: Option, // length of token in bytes, if packet is of type Init or Retry 18 | pub token: Option, // hex string, if packet is of type Init or Retry 19 | pub retry_tag: Option, // hex string, if packet is of type Retry 20 | } 21 | 22 | /// Quic Short Header 23 | #[derive(Debug, Serialize, Clone)] 24 | pub struct QuicShortHeader { 25 | pub dcid: Option, // optional. If not pre-existing cid then none. 26 | } 27 | 28 | // Long Header Packet Types from RFC 9000 Table 5 29 | #[derive(Debug, Clone, Serialize, Copy)] 30 | pub enum LongHeaderPacketType { 31 | Initial, 32 | ZeroRTT, 33 | Handshake, 34 | Retry, 35 | } 36 | 37 | impl LongHeaderPacketType { 38 | pub fn from_u8(value: u8) -> Result { 39 | match value { 40 | 0x00 => Ok(LongHeaderPacketType::Initial), 41 | 0x01 => Ok(LongHeaderPacketType::ZeroRTT), 42 | 0x02 => Ok(LongHeaderPacketType::Handshake), 43 | 0x03 => Ok(LongHeaderPacketType::Retry), 44 | _ => Err(QuicError::UnknowLongHeaderPacketType), 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /core/src/protocols/stream/quic/mod.rs: -------------------------------------------------------------------------------- 1 | //! QUIC protocol parser. 2 | //! 3 | //! ## Remarks 4 | //! [QUIC-INVARIANTS] https://datatracker.ietf.org/doc/rfc8999/ 5 | //! [QUIC-RFC9000] https://datatracker.ietf.org/doc/rfc9000/ (Quic V1) 6 | //! Retina currently only parses Quic Long and Short Headers and does not attempt to parse TLS or HTTP/3 out of 7 | //! Quic packets. The Quic protocol parser makes several assumptions about the way that quic 8 | //! packets will behave: 9 | //! - Assume that the Quic version is one as listed in the QuicVersion Enum in the quic/parser.rs file 10 | //! - Assume that the dcid of a short header is a maximum of 20 bytes. 11 | //! - Assume that the packet will not try to grease the fixed bit. 12 | //! [QUIC-GREASE](https://www.rfc-editor.org/rfc/rfc9287.html) 13 | //! 14 | //! Additionally, there are a couple decisions made in the design of the quic parser: 15 | //! - The parser will not parse a short header dcid if it is not a part of a pre-identified connection 16 | //! - The payload bytes count is a lazy counter which does not try to exclude tokens for encryption, 17 | //! which is a process that happens in wireshark. 18 | /* 19 | TODO: support parsing the tls out of the initial quic packet setup 20 | TODO support dns over quic 21 | TODO: support HTTP/3 22 | */ 23 | pub(crate) mod parser; 24 | 25 | use std::collections::HashSet; 26 | 27 | pub use self::header::{QuicLongHeader, QuicShortHeader}; 28 | use crypto::Open; 29 | use frame::QuicFrame; 30 | use header::LongHeaderPacketType; 31 | use serde::Serialize; 32 | 33 | use super::tls::Tls; 34 | pub(crate) mod crypto; 35 | pub(crate) mod frame; 36 | pub(crate) mod header; 37 | 38 | /// Errors Thrown throughout QUIC parsing. These are handled by retina and used to skip packets. 39 | #[derive(Debug)] 40 | pub enum QuicError { 41 | FixedBitNotSet, 42 | PacketTooShort, 43 | UnknownVersion, 44 | ShortHeader, 45 | UnknowLongHeaderPacketType, 46 | NoLongHeader, 47 | UnsupportedVarLen, 48 | InvalidDataIndices, 49 | CryptoFail, 50 | FailedHeaderProtection, 51 | UnknownFrameType, 52 | TlsParseFail, 53 | MissingCryptoFrames, 54 | } 55 | 56 | /// Parsed Quic connections 57 | #[derive(Debug, Serialize)] 58 | pub struct QuicConn { 59 | // All packets associated with the connection 60 | pub packets: Vec, 61 | 62 | // All cids, both src and destination, seen in Long Header packets 63 | pub cids: HashSet, 64 | 65 | // Parsed TLS messsages 66 | pub tls: Tls, 67 | 68 | // Crypto needed to decrypt initial packets sent by client 69 | pub client_opener: Option, 70 | 71 | // Crypto needed to decrypt initial packets sent by server 72 | pub server_opener: Option, 73 | 74 | // Client buffer for multi-packet TLS messages 75 | #[serde(skip_serializing)] 76 | pub client_buffer: Vec, 77 | 78 | // Server buffer for multi-packet TLS messages 79 | #[serde(skip_serializing)] 80 | pub server_buffer: Vec, 81 | } 82 | 83 | /// Parsed Quic Packet contents 84 | #[derive(Debug, Serialize)] 85 | pub struct QuicPacket { 86 | /// Quic Short header 87 | pub short_header: Option, 88 | 89 | /// Quic Long header 90 | pub long_header: Option, 91 | 92 | /// The number of bytes contained in the estimated payload 93 | pub payload_bytes_count: Option, 94 | 95 | pub frames: Option>, 96 | } 97 | 98 | impl QuicPacket { 99 | /// Returns the header type of the Quic packet (ie. "long" or "short") 100 | pub fn header_type(&self) -> &str { 101 | match &self.long_header { 102 | Some(_) => "long", 103 | None => match &self.short_header { 104 | Some(_) => "short", 105 | None => "", 106 | }, 107 | } 108 | } 109 | 110 | /// Returns the packet type of the Quic packet 111 | pub fn packet_type(&self) -> Result { 112 | match &self.long_header { 113 | Some(long_header) => Ok(long_header.packet_type), 114 | None => Err(QuicError::NoLongHeader), 115 | } 116 | } 117 | 118 | /// Returns the version of the Quic packet 119 | pub fn version(&self) -> u32 { 120 | match &self.long_header { 121 | Some(long_header) => long_header.version, 122 | None => 0, 123 | } 124 | } 125 | 126 | /// Returns the destination connection ID of the Quic packet or an empty string if it does not exist 127 | pub fn dcid(&self) -> &str { 128 | match &self.long_header { 129 | Some(long_header) => { 130 | if long_header.dcid_len > 0 { 131 | &long_header.dcid 132 | } else { 133 | "" 134 | } 135 | } 136 | None => { 137 | if let Some(short_header) = &self.short_header { 138 | short_header.dcid.as_deref().unwrap_or("") 139 | } else { 140 | "" 141 | } 142 | } 143 | } 144 | } 145 | 146 | /// Returns the source connection ID of the Quic packet or an empty string if it does not exist 147 | pub fn scid(&self) -> &str { 148 | match &self.long_header { 149 | Some(long_header) => { 150 | if long_header.scid_len > 0 { 151 | &long_header.scid 152 | } else { 153 | "" 154 | } 155 | } 156 | None => "", 157 | } 158 | } 159 | 160 | /// Returns the number of bytes in the payload of the Quic packet 161 | pub fn payload_bytes_count(&self) -> u64 { 162 | self.payload_bytes_count.unwrap_or_default() 163 | } 164 | } 165 | -------------------------------------------------------------------------------- /core/src/protocols/stream/ssh/handshake.rs: -------------------------------------------------------------------------------- 1 | //! SSH handshake components. 2 | //! 3 | 4 | use crate::utils::base64; 5 | 6 | use serde::Serialize; 7 | 8 | /// A parsed SSH Protocol Version Exchange message. 9 | #[derive(Clone, Debug, Default, PartialEq, Serialize)] 10 | pub struct SshVersionExchange { 11 | pub protoversion: Option, 12 | pub softwareversion: Option, 13 | pub comments: Option, 14 | } 15 | 16 | /// A parsed SSH Key Exchange message. 17 | #[derive(Debug, PartialEq, Serialize)] 18 | pub struct SshKeyExchange { 19 | #[serde(with = "base64")] 20 | pub cookie: Vec, 21 | pub kex_algs: Vec, 22 | pub server_host_key_algs: Vec, 23 | pub encryption_algs_client_to_server: Vec, 24 | pub encryption_algs_server_to_client: Vec, 25 | pub mac_algs_client_to_server: Vec, 26 | pub mac_algs_server_to_client: Vec, 27 | pub compression_algs_client_to_server: Vec, 28 | pub compression_algs_server_to_client: Vec, 29 | pub languages_client_to_server: Vec, 30 | pub languages_server_to_client: Vec, 31 | pub first_kex_packet_follows: bool, 32 | } 33 | 34 | /// A parsed Diffie-Hellman Key Exchange message sent by the client. 35 | #[derive(Debug, Default, Serialize)] 36 | pub struct SshDhInit { 37 | pub e: Vec, 38 | } 39 | 40 | /// A parsed Diffie-Hellman Key Exchange message sent by the server. 41 | #[derive(Debug, Default, Serialize)] 42 | pub struct SshDhResponse { 43 | pub pubkey_and_certs: Vec, 44 | pub f: Vec, 45 | pub signature: Vec, 46 | } 47 | 48 | #[derive(Debug, Default, Serialize)] 49 | pub struct SshNewKeys; 50 | -------------------------------------------------------------------------------- /core/src/protocols/stream/tls/handshake.rs: -------------------------------------------------------------------------------- 1 | //! TLS handshake components. 2 | //! 3 | //! See [tls-parser](https://docs.rs/tls-parser/latest/tls_parser/) for dependency type definitions. 4 | 5 | use crate::utils::base64; 6 | 7 | use serde::Serialize; 8 | use tls_parser::{ 9 | NamedGroup, SignatureScheme, TlsCipherSuiteID, TlsCompressionID, TlsExtensionType, TlsVersion, 10 | }; 11 | 12 | /// A parsed TLS ClientHello message. 13 | #[derive(Clone, Debug, Default, Serialize)] 14 | pub struct ClientHello { 15 | pub version: TlsVersion, 16 | #[serde(with = "base64")] 17 | pub random: Vec, 18 | #[serde(with = "base64")] 19 | pub session_id: Vec, 20 | pub cipher_suites: Vec, 21 | pub compression_algs: Vec, 22 | pub extension_list: Vec, 23 | pub server_name: Option, 24 | pub supported_groups: Vec, 25 | pub ec_point_formats: Vec, 26 | pub alpn_protocols: Vec, 27 | pub signature_algs: Vec, 28 | pub key_shares: Vec, 29 | pub supported_versions: Vec, 30 | } 31 | 32 | /// A parsed TLS ServerHello message. 33 | #[derive(Clone, Debug, Default, Serialize)] 34 | pub struct ServerHello { 35 | pub version: TlsVersion, 36 | #[serde(with = "base64")] 37 | pub random: Vec, 38 | #[serde(with = "base64")] 39 | pub session_id: Vec, 40 | pub cipher_suite: TlsCipherSuiteID, 41 | pub compression_alg: TlsCompressionID, 42 | pub extension_list: Vec, 43 | pub ec_point_formats: Vec, 44 | pub alpn_protocol: Option, 45 | pub key_share: Option, 46 | pub selected_version: Option, 47 | } 48 | 49 | /// A raw X509 certificate. 50 | #[derive(Clone, Debug, Default, Serialize)] 51 | pub struct Certificate { 52 | #[serde(with = "base64")] 53 | pub raw: Vec, 54 | // TODO: parsed certificate 55 | } 56 | 57 | /// Key data sent by the server in a ServerKeyExchange message. 58 | #[derive(Clone, Debug, Serialize)] 59 | #[serde(rename_all = "snake_case")] 60 | pub enum ServerKeyExchange { 61 | Ecdh(ServerECDHParams), 62 | Dh(ServerDHParams), 63 | Rsa(ServerRSAParams), 64 | #[serde(with = "base64")] 65 | Unknown(Vec), 66 | // TODO: parse signature 67 | } 68 | 69 | impl Default for ServerKeyExchange { 70 | fn default() -> Self { 71 | ServerKeyExchange::Unknown(vec![]) 72 | } 73 | } 74 | 75 | /// Key data sent by the client in a ClientKeyExchange message. 76 | #[derive(Clone, Debug, Serialize)] 77 | #[serde(rename_all = "snake_case")] 78 | pub enum ClientKeyExchange { 79 | Ecdh(ClientECDHParams), 80 | Dh(ClientDHParams), 81 | Rsa(ClientRSAParams), 82 | #[serde(with = "base64")] 83 | Unknown(Vec), 84 | } 85 | 86 | impl Default for ClientKeyExchange { 87 | fn default() -> Self { 88 | ClientKeyExchange::Unknown(vec![]) 89 | } 90 | } 91 | 92 | /// RSA parameters sent by the server in a ServerKeyExchange message. (RSA_EXPORT cipher suites). 93 | #[derive(Clone, Debug, Default, Serialize)] 94 | pub struct ServerRSAParams { 95 | #[serde(with = "base64")] 96 | pub modulus: Vec, 97 | #[serde(with = "base64")] 98 | pub exponent: Vec, 99 | } 100 | 101 | /// Stores the encrypted premaster secret sent by the client in a ClientKeyExchange message in an 102 | /// RSA handshake. 103 | #[derive(Clone, Debug, Default, Serialize)] 104 | pub struct ClientRSAParams { 105 | #[serde(with = "base64")] 106 | pub encrypted_pms: Vec, 107 | } 108 | 109 | /// Finite-field Diffie-Hellman parameters sent by the server in a ServerKeyExchange message. 110 | #[derive(Clone, Debug, Default, Serialize)] 111 | pub struct ServerDHParams { 112 | #[serde(with = "base64")] 113 | pub prime: Vec, 114 | #[serde(with = "base64")] 115 | pub generator: Vec, 116 | #[serde(with = "base64")] 117 | pub kx_data: Vec, 118 | } 119 | 120 | /// Finite-field Diffie-Hellman parameters sent by the client in a ClientKeyExchange message. 121 | #[derive(Clone, Debug, Default, Serialize)] 122 | pub struct ClientDHParams { 123 | #[serde(with = "base64")] 124 | pub kx_data: Vec, 125 | } 126 | 127 | /// Elliptic-curve Diffie-Hellman parameters sent by the server in a ServerKeyExchange message. 128 | #[derive(Clone, Debug, Default, Serialize)] 129 | pub struct ServerECDHParams { 130 | pub curve: NamedGroup, 131 | #[serde(with = "base64")] 132 | pub kx_data: Vec, 133 | } 134 | 135 | /// Elliptic-curve Diffie-Hellman parameters sent by the client in a ClientKeyExchange message. 136 | #[derive(Clone, Debug, Default, Serialize)] 137 | pub struct ClientECDHParams { 138 | #[serde(with = "base64")] 139 | pub kx_data: Vec, 140 | } 141 | 142 | /// A TLS 1.3 key share entry. 143 | /// 144 | /// ## Remarks. 145 | /// TLS 1.3 only. `kx_data` contents are determined by the specified group. For Finite Field DH, 146 | /// `kx_data` contains the DH public value. For ECDH, `kx_data` contains the uncompressed x,y EC 147 | /// point prepended with the value 0x4. See [Key 148 | /// Share](https://datatracker.ietf.org/doc/html/rfc8446#section-4.2.8) for details. 149 | #[derive(Clone, Debug, Default, Serialize)] 150 | pub struct KeyShareEntry { 151 | pub group: NamedGroup, 152 | #[serde(with = "base64")] 153 | pub kx_data: Vec, 154 | } 155 | -------------------------------------------------------------------------------- /core/src/runtime/mod.rs: -------------------------------------------------------------------------------- 1 | //! Retina runtime. 2 | //! 3 | //! The runtime initializes the DPDK environment abstraction layer, creates memory pools, launches 4 | //! the packet processing cores, and manages logging and display output. 5 | 6 | mod offline; 7 | mod online; 8 | use self::offline::*; 9 | use self::online::*; 10 | 11 | use crate::config::*; 12 | use crate::dpdk; 13 | use crate::filter::FilterFactory; 14 | use crate::lcore::SocketId; 15 | use crate::memory::mempool::Mempool; 16 | use crate::subscription::*; 17 | 18 | use std::collections::BTreeMap; 19 | use std::ffi::CString; 20 | use std::sync::Arc; 21 | 22 | use anyhow::{bail, Result}; 23 | 24 | /// The Retina runtime. 25 | /// 26 | /// The runtime initializes the DPDK environment abstraction layer, creates memory pools, launches 27 | /// the packet processing cores, and manages logging and display output. 28 | pub struct Runtime 29 | where 30 | S: Subscribable, 31 | { 32 | #[allow(dead_code)] 33 | mempools: BTreeMap, 34 | online: Option>, 35 | offline: Option>, 36 | #[cfg(feature = "timing")] 37 | subscription: Arc>, 38 | } 39 | 40 | impl Runtime 41 | where 42 | S: Subscribable, 43 | { 44 | /// Creates a new runtime from the `config` settings, filter, and callback. 45 | /// 46 | /// # Remarks 47 | /// 48 | /// The `factory` parameter is a macro-generated function pointer based on the user-defined 49 | /// filter string, and must take the value "`filter`". `cb` is the name of the user-defined 50 | /// callback function. 51 | /// 52 | /// # Example 53 | /// 54 | /// let mut runtime = Runtime::new(config, filter, callback)?; 55 | pub fn new(config: RuntimeConfig, factory: fn() -> FilterFactory) -> Result { 56 | let factory = factory(); 57 | let filter_str = factory.filter_str.clone(); 58 | let subscription = Arc::new(Subscription::new(factory)); 59 | 60 | println!("Initializing Retina runtime..."); 61 | log::info!("Initializing EAL..."); 62 | dpdk::load_drivers(); 63 | { 64 | let eal_params = config.get_eal_params(); 65 | let eal_params_len = eal_params.len() as i32; 66 | 67 | let mut args = vec![]; 68 | let mut ptrs = vec![]; 69 | for arg in eal_params.into_iter() { 70 | let s = CString::new(arg).unwrap(); 71 | ptrs.push(s.as_ptr() as *mut u8); 72 | args.push(s); 73 | } 74 | 75 | let ret = unsafe { dpdk::rte_eal_init(eal_params_len, ptrs.as_ptr() as *mut _) }; 76 | if ret < 0 { 77 | bail!("Failure initializing EAL"); 78 | } 79 | } 80 | 81 | log::info!("Initializing Mempools..."); 82 | let mut mempools = BTreeMap::new(); 83 | let socket_ids = config.get_all_socket_ids(); 84 | let mtu = if let Some(online) = &config.online { 85 | online.mtu 86 | } else if let Some(offline) = &config.offline { 87 | offline.mtu 88 | } else { 89 | Mempool::default_mtu() 90 | }; 91 | for socket_id in socket_ids { 92 | log::debug!("Socket ID: {}", socket_id); 93 | let mempool = Mempool::new(&config.mempool, socket_id, mtu)?; 94 | mempools.insert(socket_id, mempool); 95 | } 96 | 97 | let online = config.online.as_ref().map(|cfg| { 98 | log::info!("Initializing Online Runtime..."); 99 | let online_opts = OnlineOptions { 100 | online: cfg.clone(), 101 | conntrack: config.conntrack.clone(), 102 | }; 103 | OnlineRuntime::new( 104 | &config, 105 | online_opts, 106 | &mut mempools, 107 | filter_str.clone(), 108 | Arc::clone(&subscription), 109 | ) 110 | }); 111 | 112 | let offline = config.offline.as_ref().map(|cfg| { 113 | log::info!("Initializing Offline Analysis..."); 114 | let offline_opts = OfflineOptions { 115 | offline: cfg.clone(), 116 | conntrack: config.conntrack.clone(), 117 | }; 118 | OfflineRuntime::new(offline_opts, &mempools, Arc::clone(&subscription)) 119 | }); 120 | 121 | log::info!("Runtime ready."); 122 | Ok(Runtime { 123 | mempools, 124 | online, 125 | offline, 126 | #[cfg(feature = "timing")] 127 | subscription, 128 | }) 129 | } 130 | 131 | /// Run Retina for the duration specified in the configuration or until `ctrl-c` to terminate. 132 | /// 133 | /// # Example 134 | /// 135 | /// runtime.run(); 136 | pub fn run(&mut self) { 137 | if let Some(online) = &mut self.online { 138 | online.run(); 139 | } else if let Some(offline) = &self.offline { 140 | offline.run(); 141 | } else { 142 | log::error!("No runtime"); 143 | } 144 | #[cfg(feature = "timing")] 145 | { 146 | self.subscription.timers.display_stats(); 147 | self.subscription.timers.dump_stats(); 148 | } 149 | log::info!("Done."); 150 | } 151 | } 152 | -------------------------------------------------------------------------------- /core/src/runtime/offline.rs: -------------------------------------------------------------------------------- 1 | use crate::config::{ConnTrackConfig, OfflineConfig}; 2 | use crate::conntrack::{ConnTracker, TrackerConfig}; 3 | use crate::dpdk; 4 | use crate::lcore::{CoreId, SocketId}; 5 | use crate::memory::mbuf::Mbuf; 6 | use crate::memory::mempool::Mempool; 7 | use crate::subscription::*; 8 | 9 | use std::collections::BTreeMap; 10 | use std::ffi::CString; 11 | use std::sync::Arc; 12 | 13 | use cpu_time::ProcessTime; 14 | use pcap::Capture; 15 | 16 | pub(crate) struct OfflineRuntime 17 | where 18 | S: Subscribable, 19 | { 20 | pub(crate) mempool_name: String, 21 | pub(crate) subscription: Arc>, 22 | pub(crate) options: OfflineOptions, 23 | id: CoreId, 24 | } 25 | 26 | impl OfflineRuntime 27 | where 28 | S: Subscribable, 29 | { 30 | pub(crate) fn new( 31 | options: OfflineOptions, 32 | mempools: &BTreeMap, 33 | subscription: Arc>, 34 | ) -> Self { 35 | let core_id = CoreId(unsafe { dpdk::rte_lcore_id() } as u32); 36 | let mempool_name = mempools 37 | .get(&core_id.socket_id()) 38 | .expect("Get offline mempool") 39 | .name() 40 | .to_string(); 41 | OfflineRuntime { 42 | mempool_name, 43 | subscription, 44 | options, 45 | id: core_id, 46 | } 47 | } 48 | 49 | pub(crate) fn run(&self) { 50 | log::info!( 51 | "Launched offline analysis. Processing pcap: {}", 52 | self.options.offline.pcap, 53 | ); 54 | 55 | let mut nb_pkts = 0; 56 | let mut nb_bytes = 0; 57 | 58 | let config = TrackerConfig::from(&self.options.conntrack); 59 | let registry = S::Tracked::parsers(); 60 | log::debug!("{:#?}", registry); 61 | let mut stream_table = ConnTracker::::new(config, registry, self.id); 62 | 63 | let mempool_raw = self.get_mempool_raw(); 64 | let pcap = self.options.offline.pcap.as_str(); 65 | let mut cap = Capture::from_file(pcap).expect("Error opening pcap. Aborting."); 66 | let start = ProcessTime::try_now().expect("Getting process time failed"); 67 | while let Ok(frame) = cap.next() { 68 | if frame.header.len as usize > self.options.offline.mtu { 69 | continue; 70 | } 71 | let mbuf = Mbuf::from_bytes(frame.data, mempool_raw) 72 | .expect("Unable to allocate mbuf. Try increasing mempool size."); 73 | nb_pkts += 1; 74 | nb_bytes += mbuf.data_len() as u64; 75 | 76 | /* Apply the packet filter to get actions */ 77 | let actions = self.subscription.continue_packet(&mbuf, &self.id); 78 | if !actions.drop() { 79 | self.subscription 80 | .process_packet(mbuf, &mut stream_table, actions); 81 | } 82 | } 83 | 84 | // // Deliver remaining data in table 85 | stream_table.drain(&self.subscription); 86 | let cpu_time = start.elapsed(); 87 | println!("Processed: {} pkts, {} bytes", nb_pkts, nb_bytes); 88 | println!("CPU time: {:?}ms", cpu_time.as_millis()); 89 | } 90 | 91 | fn get_mempool_raw(&self) -> *mut dpdk::rte_mempool { 92 | let cname = CString::new(self.mempool_name.clone()).expect("Invalid CString conversion"); 93 | unsafe { dpdk::rte_mempool_lookup(cname.as_ptr()) } 94 | } 95 | } 96 | 97 | /// Read-only runtime options for the offline core 98 | #[derive(Debug)] 99 | pub(crate) struct OfflineOptions { 100 | pub(crate) offline: OfflineConfig, 101 | pub(crate) conntrack: ConnTrackConfig, 102 | } 103 | -------------------------------------------------------------------------------- /core/src/stats/mod.rs: -------------------------------------------------------------------------------- 1 | use std::cell::Cell; 2 | 3 | #[cfg(feature = "prometheus")] 4 | mod prometheus; 5 | 6 | #[cfg(feature = "prometheus")] 7 | pub use prometheus::*; 8 | 9 | thread_local! { 10 | pub(crate) static IGNORED_BY_PACKET_FILTER_PKT: Cell = const { Cell::new(0) }; 11 | pub(crate) static IGNORED_BY_PACKET_FILTER_BYTE: Cell = const { Cell::new(0) }; 12 | pub(crate) static DROPPED_MIDDLE_OF_CONNECTION_TCP_PKT: Cell = const { Cell::new(0) }; 13 | pub(crate) static DROPPED_MIDDLE_OF_CONNECTION_TCP_BYTE: Cell = const { Cell::new(0) }; 14 | pub(crate) static TOTAL_PKT: Cell = const { Cell::new(0) }; 15 | pub(crate) static TOTAL_BYTE: Cell = const { Cell::new(0) }; 16 | pub(crate) static TCP_PKT: Cell = const { Cell::new(0) }; 17 | pub(crate) static TCP_BYTE: Cell = const { Cell::new(0) }; 18 | pub(crate) static UDP_PKT: Cell = const { Cell::new(0) }; 19 | pub(crate) static UDP_BYTE: Cell = const { Cell::new(0) }; 20 | pub(crate) static TCP_NEW_CONNECTIONS: Cell = const { Cell::new(0) }; 21 | pub(crate) static UDP_NEW_CONNECTIONS: Cell = const { Cell::new(0) }; 22 | pub(crate) static IDLE_CYCLES: Cell = const { Cell::new(0) }; 23 | pub(crate) static TOTAL_CYCLES: Cell = const { Cell::new(0) }; 24 | 25 | #[cfg(feature = "prometheus")] 26 | pub(crate) static PROMETHEUS: std::cell::OnceCell = const { std::cell::OnceCell::new() }; 27 | } 28 | 29 | pub(crate) trait StatExt: Sized { 30 | fn inc(&'static self) { 31 | self.inc_by(1); 32 | } 33 | fn inc_by(&'static self, val: u64); 34 | } 35 | 36 | impl StatExt for std::thread::LocalKey> { 37 | fn inc_by(&'static self, val: u64) { 38 | self.set(self.get() + val); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /core/src/subscription/mod.rs: -------------------------------------------------------------------------------- 1 | use crate::conntrack::pdu::{L4Context, L4Pdu}; 2 | use crate::conntrack::ConnTracker; 3 | use crate::filter::*; 4 | use crate::lcore::CoreId; 5 | use crate::memory::mbuf::Mbuf; 6 | use crate::protocols::packet::tcp::TCP_PROTOCOL; 7 | use crate::protocols::packet::udp::UDP_PROTOCOL; 8 | use crate::protocols::stream::{ConnData, ParserRegistry, Session}; 9 | use crate::stats::{StatExt, TCP_BYTE, TCP_PKT, UDP_BYTE, UDP_PKT}; 10 | 11 | #[cfg(feature = "timing")] 12 | use crate::timing::timer::Timers; 13 | 14 | pub trait Subscribable { 15 | type Tracked: Trackable; 16 | } 17 | 18 | pub trait Trackable { 19 | type Subscribed: Subscribable; 20 | 21 | /// Create a new struct for tracking connection data for user delivery 22 | fn new(first_pkt: &L4Pdu, core_id: CoreId) -> Self; 23 | 24 | /// When tracking, parsing, or buffering frames, 25 | /// update tracked data with new PDU 26 | fn update(&mut self, pdu: &L4Pdu, reassembled: bool); 27 | 28 | /// Get a reference to all sessions that matched filter(s) in connection 29 | fn sessions(&self) -> &Vec; 30 | 31 | /// Store a session that matched 32 | fn track_session(&mut self, session: Session); 33 | 34 | /// Store packets for (possible) future delivery 35 | fn buffer_packet(&mut self, pdu: &L4Pdu, actions: &Actions, reassembled: bool); 36 | 37 | /// Get reference to stored packets (those buffered for delivery) 38 | fn packets(&self) -> &Vec; 39 | 40 | /// Drain data from all types that require storing packets 41 | /// Can help free mbufs for future use 42 | fn drain_tracked_packets(&mut self); 43 | 44 | /// Check and potentially deliver to streaming callbacks 45 | fn stream_deliver(&mut self, actions: &mut Actions, pdu: &L4Pdu); 46 | 47 | /// Drain data from packets cached for future potential delivery 48 | /// Used after these packets have been delivered or when associated 49 | /// subscription fails to match 50 | fn drain_cached_packets(&mut self); 51 | 52 | /// Return the core ID that this tracked conn. is on 53 | fn core_id(&self) -> &CoreId; 54 | 55 | /// Parsers needed by all datatypes 56 | /// Parsers needed by filter are generated on program startup 57 | fn parsers() -> ParserRegistry; 58 | 59 | /// Clear all internal data 60 | fn clear(&mut self); 61 | } 62 | 63 | pub struct Subscription 64 | where 65 | S: Subscribable, 66 | { 67 | packet_continue: PacketContFn, 68 | packet_filter: PacketFilterFn, 69 | proto_filter: ProtoFilterFn, 70 | session_filter: SessionFilterFn, 71 | packet_deliver: PacketDeliverFn, 72 | conn_deliver: ConnDeliverFn, 73 | #[cfg(feature = "timing")] 74 | pub(crate) timers: Timers, 75 | } 76 | 77 | impl Subscription 78 | where 79 | S: Subscribable, 80 | { 81 | pub fn new(factory: FilterFactory) -> Self { 82 | Subscription { 83 | packet_continue: factory.packet_continue, 84 | packet_filter: factory.packet_filter, 85 | proto_filter: factory.proto_filter, 86 | session_filter: factory.session_filter, 87 | packet_deliver: factory.packet_deliver, 88 | conn_deliver: factory.conn_deliver, 89 | #[cfg(feature = "timing")] 90 | timers: Timers::new(), 91 | } 92 | } 93 | 94 | pub fn process_packet( 95 | &self, 96 | mbuf: Mbuf, 97 | conn_tracker: &mut ConnTracker, 98 | actions: Actions, 99 | ) { 100 | if actions.data.intersects(ActionData::PacketContinue) { 101 | if let Ok(ctxt) = L4Context::new(&mbuf) { 102 | match ctxt.proto { 103 | TCP_PROTOCOL => { 104 | TCP_PKT.inc(); 105 | TCP_BYTE.inc_by(mbuf.data_len() as u64); 106 | } 107 | UDP_PROTOCOL => { 108 | UDP_PKT.inc(); 109 | UDP_BYTE.inc_by(mbuf.data_len() as u64); 110 | } 111 | _ => {} 112 | } 113 | conn_tracker.process(mbuf, ctxt, self); 114 | } 115 | } 116 | } 117 | 118 | // TODO: packet continue filter should ideally be built at 119 | // compile-time based on what the NIC supports (what has 120 | // already been filtered out in HW). 121 | // Ideally, NIC would `mark` mbufs as `deliver` and/or `continue`. 122 | /// Invokes the software packet filter. 123 | /// Used for each packet to determine 124 | /// forwarding to conn. tracker. 125 | pub fn continue_packet(&self, mbuf: &Mbuf, core_id: &CoreId) -> Actions { 126 | (self.packet_continue)(mbuf, core_id) 127 | } 128 | 129 | /// Invokes the five-tuple filter. 130 | /// Applied to the first packet in the connection. 131 | pub fn filter_packet(&self, mbuf: &Mbuf, tracked: &mut S::Tracked) -> Actions { 132 | (self.packet_filter)(mbuf, tracked) 133 | } 134 | 135 | /// Invokes the end-to-end protocol filter. 136 | /// Applied once a parser identifies the application-layer protocol. 137 | pub fn filter_protocol(&self, conn: &ConnData, tracked: &mut S::Tracked) -> Actions { 138 | (self.proto_filter)(conn, tracked) 139 | } 140 | 141 | /// Invokes the application-layer session filter. 142 | /// Delivers sessions to callbacks if applicable. 143 | pub fn filter_session( 144 | &self, 145 | session: &Session, 146 | conn: &ConnData, 147 | tracked: &mut S::Tracked, 148 | ) -> Actions { 149 | (self.session_filter)(session, conn, tracked) 150 | } 151 | 152 | /// Delivery functions, including delivery to the correct callback 153 | pub fn deliver_packet(&self, mbuf: &Mbuf, conn_data: &ConnData, tracked: &S::Tracked) { 154 | (self.packet_deliver)(mbuf, conn_data, tracked) 155 | } 156 | 157 | pub fn deliver_conn(&self, conn_data: &ConnData, tracked: &S::Tracked) { 158 | (self.conn_deliver)(conn_data, tracked) 159 | } 160 | } 161 | -------------------------------------------------------------------------------- /core/src/timing/macros.rs: -------------------------------------------------------------------------------- 1 | #![allow(unused_macros)] 2 | macro_rules! tsc_start { 3 | ( $start:ident ) => { 4 | #[cfg(feature = "timing")] 5 | let $start = unsafe { $crate::dpdk::rte_rdtsc() }; 6 | }; 7 | } 8 | 9 | macro_rules! tsc_record { 10 | ( $timers:expr, $timer:expr, $start:ident ) => { 11 | #[cfg(feature = "timing")] 12 | $timers.record($timer, unsafe { $crate::dpdk::rte_rdtsc() } - $start, 1); 13 | }; 14 | ( $timers:expr, $timer:expr, $start:ident, $sample:literal ) => { 15 | #[cfg(feature = "timing")] 16 | $timers.record( 17 | $timer, 18 | unsafe { $crate::dpdk::rte_rdtsc() } - $start, 19 | $sample, 20 | ); 21 | }; 22 | } 23 | -------------------------------------------------------------------------------- /core/src/timing/mod.rs: -------------------------------------------------------------------------------- 1 | #[macro_use] 2 | pub(crate) mod macros; 3 | #[cfg(feature = "timing")] 4 | pub(crate) mod timer; 5 | -------------------------------------------------------------------------------- /core/src/utils/base64.rs: -------------------------------------------------------------------------------- 1 | //! Helper functions for Base64 encoding and decoding. 2 | 3 | use serde::{Deserialize, Serialize}; 4 | use serde::{Deserializer, Serializer}; 5 | 6 | /// Encodes byte slice as a Base64 string. 7 | pub fn serialize(v: &[u8], s: S) -> Result { 8 | let base64 = base64::encode(v); 9 | String::serialize(&base64, s) 10 | } 11 | 12 | /// Decodes Base64 string as a byte vector. 13 | pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result, D::Error> { 14 | let base64 = String::deserialize(d)?; 15 | base64::decode(base64.as_bytes()).map_err(serde::de::Error::custom) 16 | } 17 | -------------------------------------------------------------------------------- /core/src/utils/mod.rs: -------------------------------------------------------------------------------- 1 | //! Utility modules. 2 | 3 | pub mod base64; 4 | pub mod types; 5 | -------------------------------------------------------------------------------- /core/src/utils/types.rs: -------------------------------------------------------------------------------- 1 | //! Wrapper types for converting packet data to/from network and host byte order. 2 | //! 3 | //! Adapted from [Capsule primitive wrapper types](https://docs.rs/capsule/0.1.5/capsule/packets/types/index.html). 4 | 5 | use std::ops::{BitAnd, BitOr}; 6 | 7 | /// 16-bit unsigned integer in big-endian order. 8 | #[allow(non_camel_case_types)] 9 | #[derive(Clone, Copy, Debug, Default, Hash, PartialEq, Eq, PartialOrd, Ord)] 10 | #[repr(C, packed)] 11 | pub struct u16be(pub u16); 12 | 13 | impl From for u16be { 14 | fn from(item: u16) -> Self { 15 | u16be(u16::to_be(item)) 16 | } 17 | } 18 | 19 | impl From for u16 { 20 | fn from(item: u16be) -> Self { 21 | u16::from_be(item.0) 22 | } 23 | } 24 | 25 | impl BitAnd for u16be { 26 | type Output = Self; 27 | 28 | fn bitand(self, rhs: Self) -> Self::Output { 29 | u16be(self.0 & rhs.0) 30 | } 31 | } 32 | 33 | impl BitOr for u16be { 34 | type Output = Self; 35 | 36 | fn bitor(self, rhs: Self) -> Self { 37 | Self(self.0 | rhs.0) 38 | } 39 | } 40 | 41 | // ------------------------------------------------------- 42 | 43 | /// 32-bit unsigned integer in big-endian order. 44 | #[allow(non_camel_case_types)] 45 | #[derive(Clone, Copy, Debug, Default, Hash, PartialEq, Eq, PartialOrd, Ord)] 46 | #[repr(C, packed)] 47 | pub struct u32be(pub u32); 48 | 49 | impl From for u32be { 50 | fn from(item: u32) -> Self { 51 | u32be(u32::to_be(item)) 52 | } 53 | } 54 | 55 | impl From<::std::net::Ipv4Addr> for u32be { 56 | fn from(item: ::std::net::Ipv4Addr) -> Self { 57 | u32be::from(u32::from(item)) 58 | } 59 | } 60 | 61 | impl From for u32 { 62 | fn from(item: u32be) -> Self { 63 | u32::from_be(item.0) 64 | } 65 | } 66 | 67 | impl BitAnd for u32be { 68 | type Output = Self; 69 | 70 | fn bitand(self, rhs: Self) -> Self::Output { 71 | u32be(self.0 & rhs.0) 72 | } 73 | } 74 | 75 | impl BitOr for u32be { 76 | type Output = Self; 77 | 78 | fn bitor(self, rhs: Self) -> Self { 79 | Self(self.0 | rhs.0) 80 | } 81 | } 82 | 83 | // ------------------------------------------------------- 84 | 85 | /// 64-bit unsigned integer in big-endian order. 86 | #[allow(non_camel_case_types)] 87 | #[derive(Clone, Copy, Debug, Default, Hash, PartialEq, Eq, PartialOrd, Ord)] 88 | #[repr(C, packed)] 89 | pub struct u64be(pub u64); 90 | 91 | impl From for u64be { 92 | fn from(item: u64) -> Self { 93 | u64be(u64::to_be(item)) 94 | } 95 | } 96 | 97 | impl From for u64 { 98 | fn from(item: u64be) -> Self { 99 | u64::from_be(item.0) 100 | } 101 | } 102 | 103 | impl BitAnd for u64be { 104 | type Output = Self; 105 | 106 | fn bitand(self, rhs: Self) -> Self::Output { 107 | u64be(self.0 & rhs.0) 108 | } 109 | } 110 | 111 | impl BitOr for u64be { 112 | type Output = Self; 113 | 114 | fn bitor(self, rhs: Self) -> Self { 115 | Self(self.0 | rhs.0) 116 | } 117 | } 118 | 119 | // ------------------------------------------------------- 120 | 121 | /// 128-bit unsigned integer in big-endian order. 122 | #[allow(non_camel_case_types)] 123 | #[derive(Clone, Copy, Debug, Default, Hash, PartialEq, Eq, PartialOrd, Ord)] 124 | #[repr(C, packed)] 125 | pub struct u128be(pub u128); 126 | 127 | impl From for u128be { 128 | fn from(item: u128) -> Self { 129 | u128be(u128::to_be(item)) 130 | } 131 | } 132 | 133 | impl From for u128 { 134 | fn from(item: u128be) -> Self { 135 | u128::from_be(item.0) 136 | } 137 | } 138 | 139 | impl BitAnd for u128be { 140 | type Output = Self; 141 | 142 | fn bitand(self, rhs: Self) -> Self::Output { 143 | u128be(self.0 & rhs.0) 144 | } 145 | } 146 | 147 | impl BitOr for u128be { 148 | type Output = Self; 149 | 150 | fn bitor(self, rhs: Self) -> Self { 151 | Self(self.0 | rhs.0) 152 | } 153 | } 154 | -------------------------------------------------------------------------------- /datatypes/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "retina-datatypes" 3 | version = "1.0.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | lazy_static = "1.4.0" 10 | retina-core = { path = "../core" } 11 | quote = "1.0.26" 12 | proc-macro2 = "1.0.56" 13 | syn = { version = "2.0.15" } 14 | serde = { version = "1.0", features = ["derive"] } 15 | pnet = "0.33.0" -------------------------------------------------------------------------------- /datatypes/src/dns_transaction.rs: -------------------------------------------------------------------------------- 1 | //! A DNS transaction. 2 | //! Subscribable alias for [`retina_core::protocols::stream::dns::Dns`] 3 | 4 | use retina_core::protocols::stream::dns::Dns; 5 | use retina_core::protocols::stream::{Session, SessionData}; 6 | 7 | use super::{FromSession, SessionList}; 8 | 9 | pub type DnsTransaction = Box; 10 | 11 | impl FromSession for DnsTransaction { 12 | fn stream_protocols() -> Vec<&'static str> { 13 | vec!["dns"] 14 | } 15 | 16 | fn from_session(session: &Session) -> Option<&Self> { 17 | if let SessionData::Dns(dns) = &session.data { 18 | return Some(dns); 19 | } 20 | None 21 | } 22 | 23 | fn from_sessionlist(session_list: &SessionList) -> Option<&Self> { 24 | for session in session_list { 25 | if let SessionData::Dns(dns) = &session.data { 26 | return Some(dns); 27 | } 28 | } 29 | None 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /datatypes/src/http_transaction.rs: -------------------------------------------------------------------------------- 1 | //! An Http transaction. 2 | //! Subscribable alias for [`retina_core::protocols::stream::http::Http`] 3 | 4 | use retina_core::protocols::stream::http::Http; 5 | use retina_core::protocols::stream::{Session, SessionData}; 6 | 7 | use super::{FromSession, SessionList}; 8 | 9 | pub type HttpTransaction = Box; 10 | 11 | impl FromSession for HttpTransaction { 12 | fn stream_protocols() -> Vec<&'static str> { 13 | vec!["http"] 14 | } 15 | 16 | fn from_session(session: &Session) -> Option<&Self> { 17 | if let SessionData::Http(http) = &session.data { 18 | return Some(http); 19 | } 20 | None 21 | } 22 | 23 | fn from_sessionlist(session_list: &SessionList) -> Option<&Self> { 24 | for session in session_list { 25 | if let SessionData::Http(http) = &session.data { 26 | return Some(http); 27 | } 28 | } 29 | None 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /datatypes/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! Subscribable data types. 3 | //! 4 | //! A subscription is a request for a callback on a subset of network traffic specified by a filter. 5 | //! Each callback function requires one or more *subscribable data types* as parameter(s), which it 6 | //! immutably borrows. 7 | //! 8 | //! Each subscribable datatype must: 9 | //! 10 | //! - Be defined as a [DataType](retina_core::filter::DataType), with appropriate parameters and [retina_core::filter::Level]. 11 | //! - Implement one of the traits defined in this module (Tracked, FromSession, etc.) 12 | //! - Be added to the [DATATYPES](`crate::typedefs::DATATYPES`) map (note: we are actively working on an approach that eliminates this requirement). 13 | //! 14 | //! 15 | 16 | pub mod conn_fts; 17 | pub mod typedefs; 18 | pub use conn_fts::*; 19 | pub mod connection; 20 | pub use connection::ConnRecord; 21 | pub mod http_transaction; 22 | pub use http_transaction::HttpTransaction; 23 | pub mod dns_transaction; 24 | pub use dns_transaction::DnsTransaction; 25 | pub mod tls_handshake; 26 | pub use tls_handshake::TlsHandshake; 27 | pub mod quic_stream; 28 | pub use quic_stream::QuicStream; 29 | pub mod ssh_handshake; 30 | pub use ssh_handshake::SshHandshake; 31 | pub mod packet; 32 | pub use packet::{Payload, ZcFrame}; 33 | pub mod static_type; 34 | pub use static_type::*; 35 | pub mod packet_list; 36 | pub use packet_list::*; 37 | pub use typedefs::*; 38 | pub mod streaming; 39 | #[doc(hidden)] 40 | pub use streaming::CallbackTimer; 41 | 42 | use retina_core::conntrack::pdu::L4Pdu; 43 | use retina_core::filter::SubscriptionSpec; 44 | use retina_core::protocols::stream::Session; 45 | use retina_core::Mbuf; 46 | 47 | /// Trait implemented by datatypes that require inline tracking. 48 | /// This is typically required for subscribable types that require 49 | /// calculating metrics throughout a connection, e.g. QoS metrics. 50 | pub trait Tracked { 51 | /// Initialize internal data; called once per connection. 52 | /// Note `first_pkt` will also be delivered to `update`. 53 | fn new(first_pkt: &L4Pdu) -> Self; 54 | /// New packet in connection received (or reassembled, if reassembled=true) 55 | /// Note this may be invoked both pre- and post-reassembly; types 56 | /// should check `reassembled` to avoid double-counting. 57 | fn update(&mut self, pdu: &L4Pdu, reassembled: bool); 58 | /// The stream protocols (lower-case) required for this datatype. 59 | /// See `IMPLEMENTED_PROTOCOLS` in retina_core for list of supported protocols. 60 | fn stream_protocols() -> Vec<&'static str>; 61 | /// Clear internal data; called if connection no longer matches filter 62 | /// that requires the Tracked type. 63 | fn clear(&mut self); 64 | } 65 | 66 | /// Trait implemented by datatypes that are built from session data. 67 | /// This is used when subscribing to specific parsed application-layer data. 68 | pub trait FromSession { 69 | /// The stream protocols (lower-case) required for this datatype. 70 | /// See `IMPLEMENTED_PROTOCOLS` in retina_core for list of supported protocols. 71 | fn stream_protocols() -> Vec<&'static str>; 72 | /// Build Self from a parsed session, or return None if impossible. 73 | /// Invoked when the session is fully matched, parsed, and ready to 74 | /// be delivered to a callback. 75 | fn from_session(session: &Session) -> Option<&Self>; 76 | /// Build Self from a *list* of sessions, or return None if impossible. 77 | /// Invoked when the connection has terminated and a FromSession datatype 78 | /// must be delivered to a callback. 79 | fn from_sessionlist(sessionlist: &SessionList) -> Option<&Self>; 80 | } 81 | 82 | /// Trait implemented by datatypes that are built from a packet (Mbuf). 83 | /// This is used when subscribing to packet-level data. 84 | pub trait FromMbuf { 85 | fn from_mbuf(mbuf: &Mbuf) -> Option<&Self>; 86 | } 87 | 88 | /// Trait implemented by datatypes that are constant throughout 89 | /// a connection and inferrable at first packet. 90 | pub trait StaticData { 91 | fn new(first_pkt: &L4Pdu) -> Self; 92 | } 93 | 94 | /// Trait for a datatype that is built from a subscription specification. 95 | /// [retina-filtergen](../filtergen) assumes that FilterStr is the only use-case for this. 96 | #[doc(hidden)] 97 | pub trait FromSubscription { 98 | /// Output the literal tokenstream (e.g., string literal) representing 99 | /// the constant value (e.g., matched filter string). 100 | fn from_subscription(spec: &SubscriptionSpec) -> proc_macro2::TokenStream; 101 | } 102 | -------------------------------------------------------------------------------- /datatypes/src/packet.rs: -------------------------------------------------------------------------------- 1 | //! Raw packet-level datatypes. 2 | 3 | use super::FromMbuf; 4 | use retina_core::{conntrack::pdu::L4Context, Mbuf}; 5 | 6 | /// Subscribable alias for [`retina_core::Mbuf`] 7 | pub type ZcFrame = Mbuf; 8 | 9 | impl FromMbuf for ZcFrame { 10 | fn from_mbuf(mbuf: &Mbuf) -> Option<&Self> { 11 | Some(mbuf) 12 | } 13 | } 14 | 15 | /// Payload after TCP/UDP headers 16 | pub type Payload = [u8]; 17 | 18 | impl FromMbuf for Payload { 19 | fn from_mbuf(mbuf: &Mbuf) -> Option<&Self> { 20 | if let Ok(ctxt) = L4Context::new(mbuf) { 21 | let offset = ctxt.offset; 22 | let payload_len = ctxt.length; 23 | if let Ok(data) = mbuf.get_data_slice(offset, payload_len) { 24 | return Some(data); 25 | } 26 | } 27 | None 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /datatypes/src/quic_stream.rs: -------------------------------------------------------------------------------- 1 | //! A Quic stream. 2 | //! Subscribable alias for [`retina_core::protocols::stream::quic::QuicConn`] 3 | 4 | use retina_core::protocols::stream::quic::QuicConn; 5 | use retina_core::protocols::stream::{Session, SessionData}; 6 | 7 | use super::{FromSession, SessionList}; 8 | 9 | pub type QuicStream = Box; 10 | 11 | impl FromSession for QuicStream { 12 | fn stream_protocols() -> Vec<&'static str> { 13 | vec!["quic"] 14 | } 15 | 16 | fn from_session(session: &Session) -> Option<&Self> { 17 | if let SessionData::Quic(quic) = &session.data { 18 | return Some(quic); 19 | } 20 | None 21 | } 22 | 23 | fn from_sessionlist(session_list: &SessionList) -> Option<&Self> { 24 | for session in session_list { 25 | if let SessionData::Quic(quic) = &session.data { 26 | return Some(quic); 27 | } 28 | } 29 | None 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /datatypes/src/ssh_handshake.rs: -------------------------------------------------------------------------------- 1 | //! A SSH handshake. 2 | //! Subscribable alias for [`retina_core::protocols::stream::ssh::Ssh`] 3 | 4 | use retina_core::protocols::stream::ssh::Ssh; 5 | use retina_core::protocols::stream::{Session, SessionData}; 6 | 7 | use super::{FromSession, SessionList}; 8 | 9 | pub type SshHandshake = Box; 10 | 11 | impl FromSession for SshHandshake { 12 | fn stream_protocols() -> Vec<&'static str> { 13 | vec!["ssh"] 14 | } 15 | 16 | fn from_session(session: &Session) -> Option<&Self> { 17 | if let SessionData::Ssh(ssh) = &session.data { 18 | return Some(ssh); 19 | } 20 | None 21 | } 22 | 23 | fn from_sessionlist(session_list: &SessionList) -> Option<&Self> { 24 | for session in session_list { 25 | if let SessionData::Ssh(ssh) = &session.data { 26 | return Some(ssh); 27 | } 28 | } 29 | None 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /datatypes/src/static_type.rs: -------------------------------------------------------------------------------- 1 | //! Static-level datatypes. 2 | //! A data type is considered "static" if it can be inferred at or before 3 | //! the first packet in a connection and it stays constant throughout a connection. 4 | //! See datatypes, including foreign types, that implement [StaticData](trait.StaticData.html). 5 | 6 | use super::StaticData; 7 | use pnet::datalink::MacAddr; 8 | use retina_core::conntrack::conn_id::FiveTuple; 9 | use retina_core::conntrack::pdu::L4Pdu; 10 | 11 | /// Subscribable alias for [`retina_core::FiveTuple`] 12 | impl StaticData for FiveTuple { 13 | fn new(first_pkt: &L4Pdu) -> Self { 14 | FiveTuple::from_ctxt(first_pkt.ctxt) 15 | } 16 | } 17 | 18 | use retina_core::protocols::packet::{ethernet::Ethernet, Packet}; 19 | 20 | /// Tag Control Information field on the first packet, or none 21 | #[derive(Clone, Copy, Debug, PartialEq)] 22 | pub struct EtherTCI(Option); 23 | 24 | impl StaticData for EtherTCI { 25 | fn new(first_pkt: &L4Pdu) -> Self { 26 | if let Ok(ethernet) = &Packet::parse_to::(first_pkt.mbuf_ref()) { 27 | if let Some(tci) = ethernet.tci() { 28 | return EtherTCI(Some(tci)); 29 | } 30 | } 31 | EtherTCI(None) 32 | } 33 | } 34 | 35 | /// The src/dst MAC of a connection 36 | #[derive(Clone, Debug)] 37 | pub struct EthAddr { 38 | pub src: MacAddr, 39 | pub dst: MacAddr, 40 | } 41 | 42 | impl StaticData for EthAddr { 43 | fn new(first_pkt: &L4Pdu) -> Self { 44 | if let Ok(ethernet) = &Packet::parse_to::(first_pkt.mbuf_ref()) { 45 | Self { 46 | src: ethernet.src(), 47 | dst: ethernet.dst(), 48 | } 49 | } else { 50 | panic!("Non-ethernet packets not supported"); 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /datatypes/src/streaming.rs: -------------------------------------------------------------------------------- 1 | use crate::Tracked; 2 | use retina_core::conntrack::pdu::L4Pdu; 3 | /// Infrastructure for managing the state of streaming subscriptions. 4 | /// This should not be accessed directly by the user. 5 | use retina_core::filter::datatypes::Streaming; 6 | use std::time::{Duration, Instant}; 7 | 8 | /// Callback timer wrapper 9 | pub struct CallbackTimer 10 | where 11 | T: Tracked, 12 | { 13 | /// The type of counter (time-based, packet-based, or byte-based) 14 | counter_type: Streaming, 15 | /// Whether a callback has "unsubscribed" from streaming data 16 | unsubscribed: bool, 17 | /// Whether the subscription's filter has matched 18 | deliverable: bool, 19 | /// For time-based counters, when the callback was last invoked 20 | last_invoked: Option, 21 | /// For packet- and byte-based counters, the number of packets/bytes 22 | /// remaining until the callback will be invoked again. 23 | /// Can be set to 0 for time-based counters. 24 | count_remaining: Option, 25 | /// TMP - TODO move this into the TrackedWrapper to be shared 26 | /// TODO - ideally could have multiple tracked datatypes 27 | data: T, 28 | } 29 | 30 | impl CallbackTimer 31 | where 32 | T: Tracked, 33 | { 34 | /// Create a new CallbackTimer with the given counter type and data. 35 | pub fn new(counter_type: Streaming, first_pkt: &L4Pdu) -> Self { 36 | Self { 37 | counter_type, 38 | unsubscribed: false, 39 | deliverable: false, 40 | last_invoked: None, 41 | count_remaining: None, 42 | data: T::new(first_pkt), 43 | } 44 | } 45 | 46 | /// Clear internal data. 47 | /// Should be invoked after delivery. 48 | #[inline] 49 | pub fn clear(&mut self) { 50 | self.data.clear(); 51 | } 52 | 53 | #[inline] 54 | pub fn update(&mut self, pdu: &L4Pdu, reassembled: bool) { 55 | if !self.unsubscribed { 56 | self.data.update(pdu, reassembled); 57 | } 58 | } 59 | 60 | pub fn stream_protocols() -> Vec<&'static str> { 61 | T::stream_protocols() 62 | } 63 | 64 | #[inline] 65 | pub fn unsubscribe(&mut self) { 66 | self.unsubscribed = true; 67 | } 68 | 69 | #[inline] 70 | pub fn matched(&mut self) { 71 | if !self.unsubscribed { 72 | self.deliverable = true; 73 | } 74 | } 75 | 76 | /// Check if the callback should be invoked. Update counters. 77 | pub fn invoke(&mut self, pdu: &L4Pdu) -> bool { 78 | if self.unsubscribed || !self.deliverable { 79 | return false; 80 | } 81 | match self.counter_type { 82 | Streaming::Seconds(duration) => { 83 | // Deliver when first ready for delivery, then every N seconds 84 | if self.last_invoked.is_none() { 85 | self.last_invoked = Some(Instant::now()); 86 | return true; 87 | } 88 | if self.last_invoked.unwrap().elapsed() 89 | >= Duration::from_millis((duration * 1000.0).round() as u64) 90 | { 91 | self.last_invoked = Some(Instant::now()); 92 | return true; 93 | } 94 | false 95 | } 96 | Streaming::Packets(count) => { 97 | // Deliver when first ready for delivery, then every N packets 98 | if self.count_remaining.is_none() { 99 | self.count_remaining = Some(count); 100 | return true; 101 | } 102 | // New packet received 103 | let count_remaining = self.count_remaining.unwrap(); 104 | if count_remaining == 1 { 105 | self.count_remaining = Some(count); 106 | return true; 107 | } 108 | self.count_remaining = Some(count_remaining - 1); 109 | false 110 | } 111 | Streaming::Bytes(count) => { 112 | // Deliver when first ready for delivery, then every N packets 113 | if self.count_remaining.is_none() { 114 | self.count_remaining = Some(count); 115 | return true; 116 | } 117 | let count_remaining = self.count_remaining.unwrap(); 118 | let len = pdu.mbuf_ref().data_len() as u32; 119 | if count_remaining <= len { 120 | self.count_remaining = Some(count); 121 | return true; 122 | } 123 | self.count_remaining = Some(count_remaining - len); 124 | false 125 | } 126 | } 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /datatypes/src/tls_handshake.rs: -------------------------------------------------------------------------------- 1 | //! A TLS handshake. 2 | //! Subscribable alias for [`retina_core::protocols::stream::tls::Tls`] 3 | 4 | use retina_core::protocols::stream::tls::Tls; 5 | use retina_core::protocols::stream::{Session, SessionData}; 6 | 7 | use super::{FromSession, SessionList}; 8 | 9 | pub type TlsHandshake = Box; 10 | 11 | impl FromSession for TlsHandshake { 12 | fn stream_protocols() -> Vec<&'static str> { 13 | vec!["tls"] 14 | } 15 | 16 | fn from_session(session: &Session) -> Option<&Self> { 17 | if let SessionData::Tls(tls) = &session.data { 18 | return Some(tls); 19 | } 20 | None 21 | } 22 | 23 | fn from_sessionlist(session_list: &SessionList) -> Option<&Self> { 24 | for session in session_list { 25 | if let SessionData::Tls(tls) = &session.data { 26 | return Some(tls); 27 | } 28 | } 29 | None 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /datatypes/src/typedefs.rs: -------------------------------------------------------------------------------- 1 | //! New datatypes are defined in this module. 2 | //! Newly-defined datatypes must be added to the DATATYPES map in this module. 3 | 4 | use lazy_static::lazy_static; 5 | use proc_macro2::Span; 6 | use quote::quote; 7 | use retina_core::filter::{DataType, Level, SubscriptionSpec}; 8 | use std::collections::HashMap; 9 | 10 | use crate::*; 11 | 12 | lazy_static! { 13 | /// To add a datatype, add it to the DATATYPES map 14 | /// This is read by the filtergen crate to generate code 15 | pub static ref DATATYPES: HashMap<&'static str, DataType> = { 16 | HashMap::from([ 17 | ("ConnRecord", DataType::new_default_connection("ConnRecord")), 18 | ( 19 | "ConnDuration", 20 | DataType::new_default_connection("ConnDuration"), 21 | ), 22 | ("PktCount", DataType::new_default_connection("PktCount")), 23 | ("ByteCount", DataType::new_default_connection("ByteCount")), 24 | ( 25 | "InterArrivals", 26 | DataType::new_default_connection("InterArrivals"), 27 | ), 28 | ( 29 | "ConnHistory", 30 | DataType::new_default_connection("ConnHistory"), 31 | ), 32 | ( 33 | "HttpTransaction", 34 | DataType::new_default_session( 35 | "HttpTransaction", 36 | HttpTransaction::stream_protocols(), 37 | ), 38 | ), 39 | ( 40 | "DnsTransaction", 41 | DataType::new_default_session("DnsTransaction", DnsTransaction::stream_protocols()), 42 | ), 43 | ( 44 | "TlsHandshake", 45 | DataType::new_default_session("TlsHandshake", TlsHandshake::stream_protocols()), 46 | ), 47 | ( 48 | "QuicStream", 49 | DataType::new_default_session("QuicStream", QuicStream::stream_protocols()), 50 | ), 51 | ( 52 | "SshHandshake", 53 | DataType::new_default_session("SshHandshake", SshHandshake::stream_protocols()), 54 | ), 55 | ("ZcFrame", DataType::new_default_packet("ZcFrame")), 56 | ("Payload", DataType::new_default_packet("Payload")), 57 | ("SessionList", { 58 | DataType { 59 | level: Level::Connection, 60 | needs_parse: true, 61 | track_sessions: true, 62 | needs_update: false, 63 | needs_reassembly: false, 64 | needs_packet_track: false, 65 | stream_protos: vec!["tls", "dns", "http", "quic", "ssh"], 66 | as_str: "SessionList", 67 | } 68 | }), 69 | ("BidirZcPktStream", { DataType::new_default_pktlist("BidirZcPktStream", false) }), 70 | ("OrigZcPktStream", { DataType::new_default_pktlist("OrigZcPktStream", false) }), 71 | ("RespZcPktStream", { DataType::new_default_pktlist("RespZcPktStream", false) }), 72 | ("OrigZcPktsReassembled", { DataType::new_default_pktlist("OrigZcPktsReassembled", true) }), 73 | ("RespZcPktsReassembled", { DataType::new_default_pktlist("RespZcPktsReassembled", true) }), 74 | ("BidirPktStream", { DataType::new_default_pktlist("BidirPktStream", false) }), 75 | ("OrigPktStream", { DataType::new_default_pktlist("OrigPktStream", false) }), 76 | ("RespPktStream", { DataType::new_default_pktlist("RespPktStream", false) }), 77 | ("OrigPktsReassembled", { DataType::new_default_pktlist("OrigPktsReassembled", true) }), 78 | ("RespPktsReassembled", { DataType::new_default_pktlist("RespPktsReassembled", true) }), 79 | ("CoreId", { DataType::new_default_static("CoreId") }), 80 | ("FiveTuple", { DataType::new_default_static("FiveTuple") }), 81 | ("EtherTCI", { DataType::new_default_static("EtherTCI") }), 82 | ("EthAddr", { DataType::new_default_static("EthAddr") }), 83 | ("FilterStr", { DataType::new_default_static("FilterStr") }), 84 | ]) 85 | }; 86 | } 87 | 88 | // Special cases: have specific conditions in generated code 89 | // \Note ideally these would be implemented more cleanly 90 | lazy_static! { 91 | /// To avoid copying, the `Tracked` structure in the framework -- 92 | /// built at compile time -- will track certain generic, raw datatypes 93 | /// if a subset of subscriptions require them. 94 | /// 95 | /// For example: buffering packets may be required as a pre-match action for a 96 | /// packet-level datatype; it may also be required if one or more subscriptions request 97 | /// a connection-level `packet list`. Rather than maintaining these lists separately -- 98 | /// one for filtering and one for delivery -- the tracked packets are stored once. 99 | /// 100 | /// Core ID is a special case, as it cannot be derived from connection, 101 | /// session, or packet data. It is simpler to define it as a directly tracked datatype. 102 | /// 103 | /// The directly tracked datatypes are SessionList and CoreId 104 | pub static ref DIRECTLY_TRACKED: HashMap<&'static str, &'static str> = HashMap::from([ 105 | ("SessionList", "sessions"), 106 | ("CoreId", "core_id") 107 | ]); 108 | 109 | /// See `FilterStr` 110 | #[doc(hidden)] 111 | pub static ref FILTER_STR: &'static str = "FilterStr"; 112 | } 113 | 114 | /// A list of all sessions (zero-copy) parsed in the connection. 115 | pub type SessionList = Vec; 116 | 117 | /// The string literal representing a matched filter. 118 | pub type FilterStr<'a> = &'a str; 119 | 120 | impl FromSubscription for FilterStr<'_> { 121 | fn from_subscription(spec: &SubscriptionSpec) -> proc_macro2::TokenStream { 122 | let str = syn::LitStr::new(&spec.filter, Span::call_site()); 123 | quote! { &#str } 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /docs/figures/combined_states_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stanford-esrg/retina/8c90da88db32f6bc6a9878c774834fef10f2a520/docs/figures/combined_states_example.png -------------------------------------------------------------------------------- /docs/figures/filtergen_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stanford-esrg/retina/8c90da88db32f6bc6a9878c774834fef10f2a520/docs/figures/filtergen_example.png -------------------------------------------------------------------------------- /docs/figures/tls_single_flow_multi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stanford-esrg/retina/8c90da88db32f6bc6a9878c774834fef10f2a520/docs/figures/tls_single_flow_multi.png -------------------------------------------------------------------------------- /docs/figures/tls_sub_ex_code.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stanford-esrg/retina/8c90da88db32f6bc6a9878c774834fef10f2a520/docs/figures/tls_sub_ex_code.png -------------------------------------------------------------------------------- /examples/basic/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "basic" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | env_logger = "0.8.4" 8 | retina-core = { path = "../../core" } 9 | retina-filtergen = { path = "../../filtergen" } 10 | retina-datatypes = { path = "../../datatypes" } 11 | lazy_static = "1.4.0" 12 | serde = { version = "1.0", features = ["derive"] } 13 | regex = "1.7.3" -------------------------------------------------------------------------------- /examples/basic/README.md: -------------------------------------------------------------------------------- 1 | # Basic 2 | 3 | An introductory example that logs TLS and DNS transactions, each with associated connection metrics. -------------------------------------------------------------------------------- /examples/basic/src/main.rs: -------------------------------------------------------------------------------- 1 | use retina_core::{config::default_config, Runtime}; 2 | use retina_datatypes::{ConnRecord, DnsTransaction, TlsHandshake}; 3 | use retina_filtergen::{filter, retina_main}; 4 | 5 | #[filter("tls")] 6 | fn tls_cb(tls: &TlsHandshake, conn_record: &ConnRecord) { 7 | println!("Tls SNI: {}, conn. metrics: {:?}", tls.sni(), conn_record); 8 | } 9 | 10 | #[filter("dns")] 11 | fn dns_cb(dns: &DnsTransaction, conn_record: &ConnRecord) { 12 | println!( 13 | "DNS query domain: {}, conn. metrics: {:?}", 14 | dns.query_domain(), 15 | conn_record 16 | ); 17 | } 18 | 19 | #[retina_main(2)] 20 | fn main() { 21 | let config = default_config(); 22 | let mut runtime: Runtime = Runtime::new(config, filter).unwrap(); 23 | runtime.run(); 24 | } 25 | -------------------------------------------------------------------------------- /examples/basic_file/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "basic_file" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | env_logger = "0.8.4" 8 | retina-core = { path = "../../core" } 9 | retina-filtergen = { path = "../../filtergen" } 10 | retina-datatypes = { path = "../../datatypes" } 11 | lazy_static = "1.4.0" 12 | serde = { version = "1.0", features = ["derive"] } 13 | regex = "1.7.3" -------------------------------------------------------------------------------- /examples/basic_file/README.md: -------------------------------------------------------------------------------- 1 | # Basic (from specification TOML file) 2 | 3 | Basic application demonstrating the from-TOML file interface. Logs TLS handshakes matching certain SNIs to a file. -------------------------------------------------------------------------------- /examples/basic_file/spec.toml: -------------------------------------------------------------------------------- 1 | [[subscriptions]] 2 | filter = "tls.sni ~ '^.*\\.com$'" 3 | datatypes = [ 4 | "TlsHandshake", 5 | "FilterStr", 6 | ] 7 | callback = "tls_cb" 8 | 9 | [[subscriptions]] 10 | filter = "tls.sni ~ '^.*\\.net$'" 11 | datatypes = [ 12 | "TlsHandshake", 13 | "FilterStr", 14 | ] 15 | callback = "tls_cb" 16 | 17 | [[subscriptions]] 18 | filter = "tls.sni ~ '^.*\\.edu$'" 19 | datatypes = [ 20 | "TlsHandshake", 21 | "FilterStr", 22 | ] 23 | callback = "tls_cb" -------------------------------------------------------------------------------- /examples/basic_file/src/main.rs: -------------------------------------------------------------------------------- 1 | use retina_core::{config::default_config, Runtime}; 2 | use retina_datatypes::{FilterStr, TlsHandshake}; 3 | use retina_filtergen::subscription; 4 | 5 | fn tls_cb(tls: &TlsHandshake, filter_str: &FilterStr) { 6 | println!("Matched filter {}: {:?}", filter_str, tls); 7 | } 8 | 9 | #[subscription("./examples/basic_file/spec.toml")] 10 | fn main() { 11 | let config = default_config(); 12 | let mut runtime: Runtime = Runtime::new(config, filter).unwrap(); 13 | runtime.run(); 14 | } 15 | -------------------------------------------------------------------------------- /examples/filter_stats/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "filter_stats" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | clap = { version = "3.2.23", features = ["derive"] } 10 | env_logger = "0.8.4" 11 | jsonl = "4.0.1" 12 | retina-core = { path = "../../core" } 13 | retina-filtergen = { path = "../../filtergen" } 14 | retina-datatypes = { path = "../../datatypes" } 15 | serde_json = "1.0.96" 16 | lazy_static = "1.4.0" 17 | array-init = "2.0" 18 | serde = { version = "1.0", features = ["derive"] } 19 | ipcrypt = "0.1.0" 20 | regex = "1.7.3" -------------------------------------------------------------------------------- /examples/filter_stats/README.md: -------------------------------------------------------------------------------- 1 | # Filter Statistics 2 | 3 | A sample program that applies a list of filters inspired by Snort rules. When a filter is matched, the filter and corresponding data are written to a file. Since many different filters use the same callback, a config file (spec.toml) is used. -------------------------------------------------------------------------------- /examples/filter_stats/src/main.rs: -------------------------------------------------------------------------------- 1 | use array_init::array_init; 2 | use retina_core::config::load_config; 3 | use retina_core::{CoreId, Runtime}; 4 | use retina_datatypes::*; 5 | use retina_filtergen::subscription; 6 | use std::sync::atomic::{AtomicPtr, Ordering}; 7 | 8 | use clap::Parser; 9 | use lazy_static::lazy_static; 10 | use std::fs::File; 11 | use std::io::{BufWriter, Write}; 12 | use std::path::PathBuf; 13 | 14 | // Number of cores being used by the runtime; should match config file 15 | // Should be defined at compile-time so that we can use a 16 | // statically-sized array for RESULTS 17 | const NUM_CORES: usize = 16; 18 | // Add 1 for ARR_LEN to avoid overflow; one core is used as main_core 19 | const ARR_LEN: usize = NUM_CORES + 1; 20 | // Temporary per-core files 21 | const OUTFILE_PREFIX: &str = "filter_stats_"; 22 | 23 | lazy_static! { 24 | static ref RESULTS: [AtomicPtr>; ARR_LEN] = { 25 | let mut results = vec![]; 26 | for core_id in 0..ARR_LEN { 27 | let file_name = String::from(OUTFILE_PREFIX) + &format!("{}", core_id) + ".jsonl"; 28 | let core_wtr = BufWriter::new(File::create(&file_name).unwrap()); 29 | let core_wtr = Box::into_raw(Box::new(core_wtr)); 30 | results.push(core_wtr); 31 | } 32 | array_init(|i| AtomicPtr::new(results[i].clone())) 33 | }; 34 | } 35 | 36 | fn init() { 37 | println!("Initializing {} results", RESULTS.len()); 38 | } 39 | 40 | #[derive(Parser, Debug)] 41 | struct Args { 42 | #[clap(short, long, parse(from_os_str), value_name = "FILE")] 43 | config: PathBuf, 44 | #[clap( 45 | short, 46 | long, 47 | parse(from_os_str), 48 | value_name = "FILE", 49 | default_value = "filter_stats.jsonl" 50 | )] 51 | outfile: PathBuf, 52 | } 53 | 54 | fn write_result(key: &str, value: String, core_id: &CoreId) { 55 | if value.is_empty() { 56 | return; 57 | } // Would it be helpful to count these? 58 | let with_proto = format!("\n{}: {}", key, value); 59 | let ptr = RESULTS[core_id.raw() as usize].load(Ordering::Relaxed); 60 | let wtr = unsafe { &mut *ptr }; 61 | wtr.write_all(with_proto.as_bytes()).unwrap(); 62 | } 63 | 64 | fn dns_cb(dns: &DnsTransaction, core_id: &CoreId, filter_str: &FilterStr) { 65 | let query_domain = (*dns).query_domain().to_string(); 66 | write_result(*filter_str, query_domain, core_id); 67 | } 68 | 69 | fn http_cb(http: &HttpTransaction, core_id: &CoreId, filter_str: &FilterStr) { 70 | let uri = (*http).uri().to_string(); 71 | write_result(*filter_str, uri, core_id); 72 | } 73 | 74 | fn tls_cb(tls: &TlsHandshake, core_id: &CoreId, filter_str: &FilterStr) { 75 | let sni = (*tls).sni().to_string(); 76 | write_result(*filter_str, sni, core_id); 77 | } 78 | 79 | #[allow(dead_code)] 80 | fn quic_cb(quic: &QuicStream, core_id: &CoreId, filter_str: &FilterStr) { 81 | let sni = (*quic).tls.sni().to_string(); 82 | write_result(*filter_str, sni, core_id); 83 | } 84 | 85 | fn packet_cb(_frame: &ZcFrame, core_id: &CoreId, filter_str: &FilterStr) { 86 | write_result(*filter_str, String::from(""), core_id); 87 | } 88 | 89 | fn conn_cb(core_id: &CoreId, filter_str: &FilterStr) { 90 | write_result(*filter_str, String::from(""), core_id); 91 | } 92 | 93 | fn combine_results(outfile: &PathBuf) { 94 | println!("Combining results from {} cores...", NUM_CORES); 95 | let mut results = Vec::new(); 96 | for core_id in 0..ARR_LEN { 97 | let fp = String::from(OUTFILE_PREFIX) + &format!("{}", core_id) + ".jsonl"; 98 | let content = std::fs::read(fp.clone()).unwrap(); 99 | results.extend_from_slice(&content); 100 | std::fs::remove_file(fp).unwrap(); 101 | } 102 | let mut file = std::fs::File::create(outfile).unwrap(); 103 | file.write_all(&results).unwrap(); 104 | } 105 | 106 | #[subscription("./examples/filter_stats/spec.toml")] 107 | fn main() { 108 | init(); 109 | let args = Args::parse(); 110 | let config = load_config(&args.config); 111 | let cores = config.get_all_rx_core_ids(); 112 | let num_cores = cores.len(); 113 | if num_cores > NUM_CORES { 114 | panic!( 115 | "Compile-time NUM_CORES ({}) must be <= num cores ({}) in config file", 116 | NUM_CORES, num_cores 117 | ); 118 | } 119 | if cores.len() > 1 && !cores.windows(2).all(|w| w[1].raw() - w[0].raw() == 1) { 120 | panic!("Cores in config file should be consecutive for zero-lock indexing"); 121 | } 122 | if cores[0].raw() > 1 { 123 | panic!("RX core IDs should start at 0 or 1"); 124 | } 125 | let mut runtime: Runtime = Runtime::new(config, filter).unwrap(); 126 | runtime.run(); 127 | combine_results(&args.outfile); 128 | } 129 | -------------------------------------------------------------------------------- /examples/log_ssh/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "log_ssh" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | clap = { version = "3.2.23", features = ["derive"] } 8 | retina-core = { path = "../../core" } 9 | retina-filtergen = { path = "../../filtergen" } 10 | retina-datatypes = { path = "../../datatypes" } 11 | serde = { version = "1.0", features = ["derive"] } 12 | serde_json = "1.0.96" 13 | lazy_static = "1.4.0" 14 | regex = "1.7.3" 15 | memchr = "2.7.4" 16 | -------------------------------------------------------------------------------- /examples/log_ssh/src/main.rs: -------------------------------------------------------------------------------- 1 | use retina_core::{config::load_config, Runtime}; 2 | use retina_datatypes::SshHandshake; 3 | use retina_filtergen::{filter, retina_main}; 4 | 5 | use clap::Parser; 6 | use lazy_static::lazy_static; 7 | use std::fs::File; 8 | use std::io::{BufWriter, Write}; 9 | use std::path::PathBuf; 10 | use std::sync::Mutex; 11 | 12 | lazy_static! { 13 | static ref file: Mutex> = 14 | Mutex::new(BufWriter::new(File::create("ssh.jsonl").unwrap())); 15 | } 16 | 17 | #[derive(Parser, Debug)] 18 | struct Args { 19 | #[clap(short, long, parse(from_os_str), value_name = "FILE")] 20 | config: PathBuf, 21 | #[clap( 22 | short, 23 | long, 24 | parse(from_os_str), 25 | value_name = "FILE", 26 | default_value = "ssh.jsonl" 27 | )] 28 | outfile: PathBuf, 29 | } 30 | 31 | #[filter("ssh.protocol_version_ctos = |32 2E 30|")] 32 | fn ssh_byte_match_cb(ssh: &SshHandshake) { 33 | if let Ok(serialized) = serde_json::to_string(&ssh) { 34 | let mut wtr = file.lock().unwrap(); 35 | wtr.write_all(serialized.as_bytes()).unwrap(); 36 | wtr.write_all(b"\n").unwrap(); 37 | } 38 | } 39 | 40 | #[filter("ssh.key_exchange_cookie_stoc contains |15 1A|")] 41 | fn ssh_contains_bytes_cb(ssh: &SshHandshake) { 42 | if let Ok(serialized) = serde_json::to_string(&ssh) { 43 | let mut wtr = file.lock().unwrap(); 44 | wtr.write_all(serialized.as_bytes()).unwrap(); 45 | wtr.write_all(b"\n").unwrap(); 46 | } 47 | } 48 | 49 | #[filter("ssh.software_version_ctos not contains 'OpenSSH'")] 50 | fn ssh_contains_str_cb(ssh: &SshHandshake) { 51 | if let Ok(serialized) = serde_json::to_string(&ssh) { 52 | let mut wtr = file.lock().unwrap(); 53 | wtr.write_all(serialized.as_bytes()).unwrap(); 54 | wtr.write_all(b"\n").unwrap(); 55 | } 56 | } 57 | 58 | #[filter("ssh.key_exchange_cookie_stoc ~b '(?-u)^\x15\x1A.+\x78$'")] 59 | fn ssh_byte_regex_byte_cb(ssh: &SshHandshake) { 60 | if let Ok(serialized) = serde_json::to_string(&ssh) { 61 | let mut wtr = file.lock().unwrap(); 62 | wtr.write_all(serialized.as_bytes()).unwrap(); 63 | wtr.write_all(b"\n").unwrap(); 64 | } 65 | } 66 | 67 | #[retina_main(4)] 68 | fn main() { 69 | let args = Args::parse(); 70 | let config = load_config(&args.config); 71 | 72 | let mut runtime: Runtime = Runtime::new(config, filter).unwrap(); 73 | runtime.run(); 74 | 75 | let mut wtr = file.lock().unwrap(); 76 | wtr.flush().unwrap(); 77 | } 78 | -------------------------------------------------------------------------------- /examples/port_count/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "port_count" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | clap = { version = "3.2.23", features = ["derive"] } 10 | env_logger = "0.8.4" 11 | jsonl = "4.0.1" 12 | retina-core = { path = "../../core" } 13 | retina-filtergen = { path = "../../filtergen" } 14 | retina-datatypes = { path = "../../datatypes" } 15 | serde_json = "1.0.96" 16 | lazy_static = "1.4.0" 17 | array-init = "2.0" 18 | serde = { version = "1.0", features = ["derive"] } 19 | -------------------------------------------------------------------------------- /examples/port_count/README.md: -------------------------------------------------------------------------------- 1 | # Port Counts 2 | 3 | A sample program that counts the instances of UDP/TCP ports. When sessions are identified by Retina's application-layer parsers, it also tracks which ports the application-layer protocols appear on. -------------------------------------------------------------------------------- /examples/protocols/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "protocols" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | clap = { version = "3.2.23", features = ["derive"] } 8 | env_logger = "0.8.4" 9 | jsonl = "4.0.1" 10 | retina-core = { path = "../../core" } 11 | retina-filtergen = { path = "../../filtergen" } 12 | retina-datatypes = { path = "../../datatypes" } 13 | serde_json = "1.0.96" 14 | lazy_static = "1.4.0" 15 | array-init = "2.0" 16 | serde = { version = "1.0", features = ["derive"] } -------------------------------------------------------------------------------- /examples/protocols/README.md: -------------------------------------------------------------------------------- 1 | # Protocol Data 2 | 3 | A sample program that gathers basic data about services hosted on unexpected ports. -------------------------------------------------------------------------------- /examples/run_repeat.py: -------------------------------------------------------------------------------- 1 | import subprocess, re, os, toml, argparse 2 | 3 | TERMINATE = 10 # Stop if drops > 10% 4 | GRACE_PD = 5 # Grace period to allow for small/temporary spikes 5 | 6 | def execute(cmd, executable): 7 | print(f"Starting {executable}") 8 | popen = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, universal_newlines=True) 9 | grace_pd = 0 10 | for stdout_line in iter(popen.stdout.readline, ''): 11 | print(stdout_line, end='') 12 | if 'SW Dropped' in stdout_line: 13 | num = re.findall('\d*\.*\d+\%', stdout_line) 14 | if not num: 15 | continue 16 | value = float(num[0].split('%')[0]) 17 | if value > TERMINATE: 18 | grace_pd += 1 19 | if grace_pd < GRACE_PD: 20 | continue 21 | print(f'{value}% dropped; terminating') 22 | pid = os.popen(f'pidof {executable}').read() 23 | os.system(f'sudo kill -INT {pid}') 24 | return True # Stop iterations 25 | 26 | return False # Continue 27 | 28 | def main(args): 29 | # Config file (duration) 30 | config_file = args.config 31 | config = toml.load(config_file) 32 | config['online']['duration'] = int(args.duration) 33 | f = open(config_file, 'w') 34 | toml.dump(config, f) 35 | f.close() 36 | 37 | # Cmd 38 | executable = f'/home/tcr6/retina/target/release/{args.binary}' 39 | cmd = f'sudo env LD_LIBRARY_PATH=$LD_LIBRARY_PATH RUST_LOG=error {executable} --config {config_file}' 40 | print(cmd) 41 | 42 | outfiles = [f'{outdir}/{i}_{args.duration}s_{args.outfile}' for i in range(int(args.iterations))] 43 | 44 | iters = 0 45 | for outfile in outfiles: 46 | cmd_i = cmd + f' --outfile {outfile}' 47 | stop = execute(cmd_i, executable) 48 | iters += 1 49 | if stop: 50 | print(f"Terminated at {iters} iterations") 51 | break 52 | 53 | 54 | 55 | def parse_args(): 56 | parser = argparse.ArgumentParser() 57 | parser.add_argument('-b', '--binary') 58 | parser.add_argument('-d', '--duration') 59 | parser.add_argument('-i', '--iterations') 60 | parser.add_argument('-c', '--config') 61 | parser.add_argument('-o', '--outfile') 62 | parser.add_argument('-d', '--outdir') 63 | return parser.parse_args() 64 | 65 | if __name__ == '__main__': 66 | print("Start running program...") 67 | main(parse_args()) -------------------------------------------------------------------------------- /examples/streaming/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "streaming" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | clap = { version = "3.2.23", features = ["derive"] } 8 | env_logger = "0.8.4" 9 | jsonl = "4.0.1" 10 | retina-core = { path = "../../core" } 11 | retina-filtergen = { path = "../../filtergen" } 12 | retina-datatypes = { path = "../../datatypes" } 13 | serde_json = "1.0.96" 14 | lazy_static = "1.4.0" 15 | array-init = "2.0" 16 | serde = { version = "1.0", features = ["derive"] } 17 | -------------------------------------------------------------------------------- /examples/streaming/src/main.rs: -------------------------------------------------------------------------------- 1 | use clap::Parser; 2 | /// An example to illustrate the streaming callback interface. 3 | use retina_core::{config::load_config, FiveTuple, Runtime}; 4 | use retina_datatypes::*; 5 | use retina_filtergen::{filter, retina_main, streaming}; 6 | use std::path::PathBuf; 7 | 8 | // Argument parsing 9 | #[derive(Parser, Debug)] 10 | struct Args { 11 | #[clap(short, long, parse(from_os_str), value_name = "FILE")] 12 | config: PathBuf, 13 | } 14 | 15 | /// This callback will be invoked on every packet for connections 16 | /// that are identified as TLS on TCP port 52152. 17 | /// The callback will begin to be invoked for the first packet 18 | /// following filter match (identification of TLS connection). 19 | /// That is, we expect the initial value of PktCount to be >3 20 | /// (after the TCP handshake and first data packet). 21 | #[filter("tls")] 22 | #[streaming("packets=1")] 23 | fn tls_cb_pkts(pkts: &PktCount, ft: &FiveTuple) -> bool { 24 | println!("{} Packet count: {:?}", ft, pkts); 25 | true 26 | } 27 | 28 | /// This callback will be invoked every 5,000 bytes for connections 29 | /// that are identified as TLS and are not on TCP port 52152. 30 | /// Since the callback is invoked at most every packet, it may not 31 | /// be invoked at intervals of exactly 5,000 bytes. 32 | /// The "timer" for callback invocation will begin once the filter 33 | /// has matched; that is, the initial value of `bytes` will be >5,000 34 | /// as it will include the preamble *plus* ≥5,000 bytes of data. 35 | #[filter("tls and tcp.port != 52152")] 36 | #[streaming("bytes=5000")] 37 | fn tls_cb_bytes(bytes: &ByteCount, ft: &FiveTuple) -> bool { 38 | println!("{} Byte count: {}", ft, bytes.raw()); 39 | true 40 | } 41 | 42 | #[retina_main(2)] 43 | fn main() { 44 | let args = Args::parse(); 45 | let config = load_config(&args.config); 46 | let mut runtime: Runtime = Runtime::new(config, filter).unwrap(); 47 | runtime.run(); 48 | } 49 | -------------------------------------------------------------------------------- /examples/websites-prometheus/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "websites-prometheus" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | clap = { version = "3.2.23", features = ["derive"] } 10 | env_logger = "0.8.4" 11 | retina-core = { path = "../../core", features = ["prometheus"] } 12 | retina-filtergen = { path = "../../filtergen" } 13 | retina-datatypes = { path = "../../datatypes" } 14 | prometheus-client = "0.22.3" 15 | -------------------------------------------------------------------------------- /examples/websites-prometheus/README.md: -------------------------------------------------------------------------------- 1 | # Websites Prometheus 2 | 3 | Records URLs visited by protocol and stores them in the Prometheus. 4 | -------------------------------------------------------------------------------- /examples/websites-prometheus/src/main.rs: -------------------------------------------------------------------------------- 1 | use prometheus_client::encoding::EncodeLabelSet; 2 | use prometheus_client::metrics::counter::Counter; 3 | use prometheus_client::metrics::family::Family; 4 | use prometheus_client::registry::Registry; 5 | use retina_core::config::load_config; 6 | use retina_core::{stats::register_base_prometheus_registry, CoreId, Runtime}; 7 | use retina_datatypes::*; 8 | use retina_filtergen::{filter, retina_main}; 9 | 10 | use clap::Parser; 11 | use std::path::PathBuf; 12 | use std::sync::LazyLock; 13 | 14 | #[derive(Parser, Debug)] 15 | struct Args { 16 | #[clap(short, long, parse(from_os_str), value_name = "FILE")] 17 | config: PathBuf, 18 | } 19 | 20 | // Note: Using unbounded and high cardinality label set (like website field here) is bad practice 21 | // and can lead to high memory and disk usage in Prometheus. This is just an example. 22 | #[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)] 23 | struct Labels { 24 | protocol: &'static str, 25 | website: String, 26 | core_id: u32, 27 | } 28 | 29 | static FAMILY: LazyLock> = LazyLock::new(Family::default); 30 | 31 | fn init() { 32 | let mut r = Registry::default(); 33 | r.register( 34 | "myapp_site_hits", 35 | "Number of callback calls per each website and protocol", 36 | FAMILY.clone(), 37 | ); 38 | register_base_prometheus_registry(r); 39 | } 40 | 41 | fn write_result(protocol: &'static str, website: String, core_id: &CoreId) { 42 | if website.is_empty() { 43 | return; 44 | } // Would it be helpful to count these? 45 | FAMILY 46 | .get_or_create(&Labels { 47 | protocol, 48 | website, 49 | core_id: core_id.raw(), 50 | }) 51 | .inc(); 52 | } 53 | 54 | #[filter("dns")] 55 | fn dns_cb(dns: &DnsTransaction, core_id: &CoreId) { 56 | let query_domain = (*dns).query_domain().to_string(); 57 | write_result("dns", query_domain, core_id); 58 | } 59 | 60 | #[filter("http")] 61 | fn http_cb(http: &HttpTransaction, core_id: &CoreId) { 62 | let uri = (*http).uri().to_string(); 63 | write_result("http", uri, core_id); 64 | } 65 | 66 | #[filter("tls")] 67 | fn tls_cb(tls: &TlsHandshake, core_id: &CoreId) { 68 | let sni = (*tls).sni().to_string(); 69 | write_result("tls", sni, core_id); 70 | } 71 | 72 | #[filter("quic")] 73 | fn quic_cb(quic: &QuicStream, core_id: &CoreId) { 74 | let sni = quic.tls.sni().to_string(); 75 | write_result("quic", sni, core_id); 76 | } 77 | 78 | #[retina_main(4)] 79 | fn main() { 80 | init(); 81 | let args = Args::parse(); 82 | let config = load_config(&args.config); 83 | let mut runtime: Runtime = Runtime::new(config, filter).unwrap(); 84 | runtime.run(); 85 | } 86 | -------------------------------------------------------------------------------- /examples/websites/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "websites" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | clap = { version = "3.2.23", features = ["derive"] } 10 | env_logger = "0.8.4" 11 | jsonl = "4.0.1" 12 | retina-core = { path = "../../core" } 13 | retina-filtergen = { path = "../../filtergen" } 14 | retina-datatypes = { path = "../../datatypes" } 15 | serde_json = "1.0.96" 16 | lazy_static = "1.4.0" 17 | array-init = "2.0" 18 | serde = { version = "1.0", features = ["derive"] } 19 | -------------------------------------------------------------------------------- /examples/websites/README.md: -------------------------------------------------------------------------------- 1 | # Websites 2 | 3 | Records URLs visited by protocol. 4 | 5 | The script provided re-runs this multiple times in order to record changes over time. -------------------------------------------------------------------------------- /examples/websites/src/main.rs: -------------------------------------------------------------------------------- 1 | use array_init::array_init; 2 | use retina_core::config::load_config; 3 | use retina_core::{CoreId, Runtime}; 4 | use retina_datatypes::*; 5 | use retina_filtergen::{filter, retina_main}; 6 | use std::sync::atomic::{AtomicPtr, Ordering}; 7 | 8 | use clap::Parser; 9 | use std::fs::File; 10 | use std::io::{BufWriter, Write}; 11 | use std::path::PathBuf; 12 | use std::sync::OnceLock; 13 | 14 | // Number of cores being used by the runtime; should match config file 15 | // Should be defined at compile-time so that we can use a 16 | // statically-sized array for results() 17 | const NUM_CORES: usize = 16; 18 | // Add 1 for ARR_LEN to avoid overflow; one core is used as main_core 19 | const ARR_LEN: usize = NUM_CORES + 1; 20 | // Temporary per-core files 21 | const OUTFILE_PREFIX: &str = "websites_"; 22 | 23 | static RESULTS: OnceLock<[AtomicPtr>; ARR_LEN]> = OnceLock::new(); 24 | 25 | fn results() -> &'static [AtomicPtr>; ARR_LEN] { 26 | RESULTS.get_or_init(|| { 27 | let mut outp = vec![]; 28 | for core_id in 0..ARR_LEN { 29 | let file_name = String::from(OUTFILE_PREFIX) + &format!("{}", core_id) + ".jsonl"; 30 | let core_wtr = BufWriter::new(File::create(&file_name).unwrap()); 31 | let core_wtr = Box::into_raw(Box::new(core_wtr)); 32 | outp.push(core_wtr); 33 | } 34 | array_init(|i| AtomicPtr::new(outp[i])) 35 | }) 36 | } 37 | 38 | fn init() { 39 | let _ = results(); 40 | } 41 | 42 | #[derive(Parser, Debug)] 43 | struct Args { 44 | #[clap(short, long, parse(from_os_str), value_name = "FILE")] 45 | config: PathBuf, 46 | #[clap( 47 | short, 48 | long, 49 | parse(from_os_str), 50 | value_name = "FILE", 51 | default_value = "websites.jsonl" 52 | )] 53 | outfile: PathBuf, 54 | } 55 | 56 | fn write_result(key: &str, value: String, core_id: &CoreId) { 57 | if value.is_empty() { 58 | return; 59 | } // Would it be helpful to count these? 60 | let with_proto = format!("\n{}: {}", key, value); 61 | let ptr = results()[core_id.raw() as usize].load(Ordering::Relaxed); 62 | let wtr = unsafe { &mut *ptr }; 63 | wtr.write_all(with_proto.as_bytes()).unwrap(); 64 | } 65 | 66 | #[filter("dns")] 67 | fn dns_cb(dns: &DnsTransaction, core_id: &CoreId) { 68 | let query_domain = (*dns).query_domain().to_string(); 69 | write_result("dns", query_domain, core_id); 70 | } 71 | 72 | #[filter("http")] 73 | fn http_cb(http: &HttpTransaction, core_id: &CoreId) { 74 | let uri = (*http).uri().to_string(); 75 | write_result("http", uri, core_id); 76 | } 77 | 78 | #[filter("tls")] 79 | fn tls_cb(tls: &TlsHandshake, core_id: &CoreId) { 80 | let sni = (*tls).sni().to_string(); 81 | write_result("tls", sni, core_id); 82 | } 83 | 84 | #[filter("quic")] 85 | fn quic_cb(quic: &QuicStream, core_id: &CoreId) { 86 | let sni = quic.tls.sni().to_string(); 87 | write_result("quic", sni, core_id); 88 | } 89 | 90 | fn combine_results(outfile: &PathBuf) { 91 | println!("Combining results from {} cores...", NUM_CORES); 92 | let mut output = Vec::new(); 93 | for core_id in 0..ARR_LEN { 94 | let ptr = results()[core_id].load(Ordering::Relaxed); 95 | let wtr = unsafe { &mut *ptr }; 96 | wtr.flush().unwrap(); 97 | let fp = String::from(OUTFILE_PREFIX) + &format!("{}", core_id) + ".jsonl"; 98 | let content = std::fs::read(fp.clone()).unwrap(); 99 | output.extend_from_slice(&content); 100 | std::fs::remove_file(fp).unwrap(); 101 | } 102 | let mut file = std::fs::File::create(outfile).unwrap(); 103 | file.write_all(&output).unwrap(); 104 | } 105 | 106 | #[retina_main(4)] 107 | fn main() { 108 | init(); 109 | let args = Args::parse(); 110 | let config = load_config(&args.config); 111 | let cores = config.get_all_rx_core_ids(); 112 | let num_cores = cores.len(); 113 | if num_cores > NUM_CORES { 114 | panic!( 115 | "Compile-time NUM_CORES ({}) must be <= num cores ({}) in config file", 116 | NUM_CORES, num_cores 117 | ); 118 | } 119 | if cores.len() > 1 && !cores.windows(2).all(|w| w[1].raw() - w[0].raw() == 1) { 120 | panic!("Cores in config file should be consecutive for zero-lock indexing"); 121 | } 122 | if cores[0].raw() > 1 { 123 | panic!("RX core IDs should start at 0 or 1"); 124 | } 125 | let mut runtime: Runtime = Runtime::new(config, filter).unwrap(); 126 | runtime.run(); 127 | combine_results(&args.outfile); 128 | } 129 | -------------------------------------------------------------------------------- /filtergen/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "retina-filtergen" 3 | version = "1.0.0" 4 | edition = "2021" 5 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 6 | 7 | [lib] 8 | proc-macro = true 9 | 10 | [dependencies] 11 | heck = "0.3.3" 12 | proc-macro2 = "1.0.56" 13 | quote = "1.0.26" 14 | regex = "1.7.3" 15 | retina-core = { path = "../core", default-features = false } 16 | syn = { version = "2.0.15", features = ["full", "extra-traits"] } 17 | serde_yaml = "0.9.3" 18 | serde_with = "1.6.1" 19 | serde = { version = "1.0", features = ["derive"] } 20 | toml = "0.5.11" 21 | retina-datatypes = { path = "../datatypes" } 22 | lazy_static = "1.4.0" 23 | memchr = "2.7.4" 24 | -------------------------------------------------------------------------------- /filtergen/src/cache.rs: -------------------------------------------------------------------------------- 1 | use super::parse::{ConfigRaw, SubscriptionRaw}; 2 | use quote::ToTokens; 3 | use retina_core::filter::datatypes::Streaming; 4 | use std::sync::{ 5 | atomic::{AtomicUsize, Ordering}, 6 | Mutex, 7 | }; 8 | 9 | lazy_static! { 10 | pub(crate) static ref NUM_SUBSCRIPTIONS: AtomicUsize = AtomicUsize::new(0); 11 | pub(crate) static ref CACHED_SUBSCRIPTIONS: Mutex = Mutex::new(ConfigRaw { 12 | subscriptions: vec![] 13 | }); 14 | pub(crate) static ref STREAMING_SUBSCRIPTIONS: Mutex> = Mutex::new(None); 15 | } 16 | 17 | pub(crate) fn parse_input(input: &syn::ItemFn) -> (Vec, String) { 18 | let datatypes = input 19 | .sig 20 | .inputs 21 | .iter() 22 | .filter_map(|arg| { 23 | if let syn::FnArg::Typed(syn::PatType { pat: _, ty, .. }) = arg { 24 | let mut param_type: String = (*ty).to_token_stream().to_string(); 25 | if !param_type.contains("&") { 26 | panic!( 27 | "Parameters to callbacks must be passed by reference ({})", 28 | param_type 29 | ); 30 | } 31 | param_type = param_type.replace("&", "").trim().to_string(); 32 | return Some(param_type); 33 | } 34 | None 35 | }) 36 | .collect(); 37 | 38 | let callback = input.sig.ident.to_token_stream().to_string(); 39 | 40 | (datatypes, callback) 41 | } 42 | 43 | pub(crate) fn add_subscription(callback: String, datatypes: Vec, filter: String) { 44 | let streaming = { 45 | let mut lock = STREAMING_SUBSCRIPTIONS.lock().unwrap(); 46 | lock.take() // Move, replacing with None 47 | }; 48 | if streaming.is_some() { 49 | println!("Streaming callback: {}={:?}", callback, streaming.unwrap()); 50 | } 51 | CACHED_SUBSCRIPTIONS 52 | .lock() 53 | .unwrap() 54 | .subscriptions 55 | .push(SubscriptionRaw { 56 | filter, 57 | datatypes, 58 | callback, 59 | streaming, 60 | }); 61 | } 62 | 63 | pub(crate) fn add_streaming(callback: String, key: &str, value: f32) { 64 | let mut lock = CACHED_SUBSCRIPTIONS.lock().unwrap(); 65 | if let Some(entry) = lock.subscriptions.last_mut() { 66 | if entry.callback == callback { 67 | entry.streaming = Some(Streaming::from((key, value))); 68 | println!("Streaming callback: {}={:?}", callback, entry.streaming); 69 | return; 70 | } 71 | } 72 | 73 | *STREAMING_SUBSCRIPTIONS.lock().unwrap() = Some(Streaming::from((key, value))); 74 | } 75 | 76 | pub(crate) fn is_done() -> bool { 77 | let present = CACHED_SUBSCRIPTIONS.lock().unwrap().subscriptions.len(); 78 | let expected = NUM_SUBSCRIPTIONS.load(Ordering::SeqCst); 79 | if present > expected && expected > 0 { 80 | panic!("Too many subscriptions present; expected: {}", expected); 81 | } 82 | present == expected 83 | } 84 | 85 | pub(crate) fn set_count(count: usize) { 86 | NUM_SUBSCRIPTIONS.store(count, Ordering::SeqCst); 87 | } 88 | -------------------------------------------------------------------------------- /filtergen/src/deliver_filter.rs: -------------------------------------------------------------------------------- 1 | use heck::CamelCase; 2 | use proc_macro2::{Ident, Span}; 3 | use quote::quote; 4 | 5 | use crate::utils::*; 6 | use retina_core::filter::ast::*; 7 | use retina_core::filter::ptree::{FilterLayer, PNode, PTree}; 8 | 9 | pub(crate) fn gen_deliver_filter( 10 | ptree: &PTree, 11 | statics: &mut Vec, 12 | filter_layer: FilterLayer, 13 | ) -> proc_macro2::TokenStream { 14 | let mut body: Vec = vec![]; 15 | 16 | // Ensure that "always deliver" case is covered. 17 | // \Note if more outer protocols are added (beyond ethernet), this would 18 | // need to be changed. 19 | if !ptree.root.deliver.is_empty() { 20 | update_body(&mut body, &ptree.root, filter_layer, false); 21 | } 22 | 23 | gen_deliver_util(&mut body, statics, &ptree.root, filter_layer); 24 | // Ensure root protocol is extracted 25 | match filter_layer { 26 | FilterLayer::PacketDeliver => PacketDataFilter::add_root_pred(&ptree.root, &body), 27 | _ => quote! { #( #body )* }, 28 | } 29 | } 30 | 31 | fn gen_deliver_util( 32 | code: &mut Vec, 33 | statics: &mut Vec, 34 | node: &PNode, 35 | filter_layer: FilterLayer, 36 | ) { 37 | let mut first_unary = true; 38 | for child in node.children.iter() { 39 | match &child.pred { 40 | Predicate::Unary { protocol } => { 41 | if child.pred.on_packet() { 42 | if matches!(filter_layer, FilterLayer::PacketDeliver) { 43 | PacketDataFilter::add_unary_pred( 44 | code, 45 | statics, 46 | child, 47 | node.pred.get_protocol(), 48 | protocol, 49 | first_unary, 50 | filter_layer, 51 | &gen_deliver_util, 52 | ); 53 | } else { 54 | ConnDataFilter::add_unary_pred( 55 | code, 56 | statics, 57 | child, 58 | protocol, 59 | first_unary, 60 | filter_layer, 61 | &gen_deliver_util, 62 | ); 63 | } 64 | first_unary = false; 65 | } else if child.pred.on_proto() { 66 | ConnDataFilter::add_service_pred( 67 | code, 68 | statics, 69 | child, 70 | protocol, 71 | filter_layer, 72 | &gen_deliver_util, 73 | ); 74 | } else { 75 | panic!("Unary predicate on session filter"); 76 | } 77 | } 78 | Predicate::Binary { 79 | protocol, 80 | field, 81 | op, 82 | value, 83 | } => { 84 | if child.pred.on_packet() { 85 | if matches!(filter_layer, FilterLayer::PacketDeliver) { 86 | PacketDataFilter::add_binary_pred( 87 | code, 88 | statics, 89 | child, 90 | protocol, 91 | field, 92 | op, 93 | value, 94 | filter_layer, 95 | &gen_deliver_util, 96 | ); 97 | } else { 98 | ConnDataFilter::add_binary_pred( 99 | code, 100 | statics, 101 | child, 102 | protocol, 103 | field, 104 | op, 105 | value, 106 | filter_layer, 107 | &gen_deliver_util, 108 | ); 109 | } 110 | } else if child.pred.on_session() { 111 | add_session_pred( 112 | code, 113 | statics, 114 | child, 115 | protocol, 116 | field, 117 | op, 118 | value, 119 | filter_layer, 120 | ); 121 | } else { 122 | panic!("Binary predicate on protocol filter"); 123 | } 124 | } 125 | } 126 | } 127 | } 128 | 129 | #[allow(clippy::too_many_arguments)] 130 | pub(crate) fn add_session_pred( 131 | code: &mut Vec, 132 | statics: &mut Vec, 133 | node: &PNode, 134 | protocol: &ProtocolName, 135 | field: &FieldName, 136 | op: &BinOp, 137 | value: &Value, 138 | layer: FilterLayer, 139 | ) { 140 | let mut body: Vec = vec![]; 141 | gen_deliver_util(&mut body, statics, node, layer); 142 | update_body(&mut body, node, layer, true); 143 | let pred_tokenstream = binary_to_tokens(protocol, field, op, value, statics); 144 | 145 | let service = protocol.name(); 146 | let proto_name = Ident::new(service, Span::call_site()); 147 | let proto_variant = Ident::new(&service.to_camel_case(), Span::call_site()); 148 | let proto_condition = quote! { let retina_core::protocols::stream::SessionData::#proto_variant(#proto_name) = &session.data }; 149 | 150 | code.push(quote! { 151 | for session in tracked.sessions() { 152 | if #proto_condition { 153 | if #pred_tokenstream { 154 | #( #body )* 155 | } 156 | } 157 | } 158 | }); 159 | } 160 | -------------------------------------------------------------------------------- /filtergen/src/packet_filter.rs: -------------------------------------------------------------------------------- 1 | use quote::quote; 2 | 3 | use crate::utils::*; 4 | use retina_core::filter::ast::*; 5 | use retina_core::filter::ptree::{FilterLayer, PNode, PTree}; 6 | 7 | pub(crate) fn gen_packet_filter( 8 | ptree: &PTree, 9 | statics: &mut Vec, 10 | filter_layer: FilterLayer, 11 | ) -> proc_macro2::TokenStream { 12 | let mut body: Vec = vec![]; 13 | 14 | // Ensure root is covered 15 | if !ptree.root.actions.drop() || !ptree.root.deliver.is_empty() { 16 | update_body(&mut body, &ptree.root, filter_layer, false); 17 | } 18 | 19 | gen_packet_filter_util(&mut body, statics, &ptree.root, filter_layer); 20 | 21 | let body = PacketDataFilter::add_root_pred(&ptree.root, &body); 22 | 23 | let packet_filter = quote! { 24 | let mut result = retina_core::filter::Actions::new(); 25 | #body 26 | result 27 | }; 28 | packet_filter 29 | } 30 | 31 | fn gen_packet_filter_util( 32 | code: &mut Vec, 33 | statics: &mut Vec, 34 | node: &PNode, 35 | filter_layer: FilterLayer, 36 | ) { 37 | let mut first_unary = true; 38 | for child in node.children.iter().filter(|n| n.pred.on_packet()) { 39 | match &child.pred { 40 | Predicate::Unary { protocol } => { 41 | PacketDataFilter::add_unary_pred( 42 | code, 43 | statics, 44 | child, 45 | node.pred.get_protocol(), 46 | protocol, 47 | first_unary, 48 | filter_layer, 49 | &gen_packet_filter_util, 50 | ); 51 | first_unary = false; 52 | } 53 | Predicate::Binary { 54 | protocol, 55 | field, 56 | op, 57 | value, 58 | } => { 59 | PacketDataFilter::add_binary_pred( 60 | code, 61 | statics, 62 | child, 63 | protocol, 64 | field, 65 | op, 66 | value, 67 | filter_layer, 68 | &gen_packet_filter_util, 69 | ); 70 | } 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /filtergen/src/parse.rs: -------------------------------------------------------------------------------- 1 | use retina_core::filter::{datatypes::Streaming, Level, SubscriptionSpec}; 2 | use retina_datatypes::DATATYPES; 3 | use serde::{Deserialize, Serialize}; 4 | use serde_with::serde_as; 5 | 6 | // Specify subscription specs from a file 7 | #[derive(Serialize, Deserialize)] 8 | pub(crate) struct ConfigRaw { 9 | pub(crate) subscriptions: Vec, 10 | } 11 | 12 | #[serde_as] 13 | #[derive(Serialize, Deserialize, Debug, Clone)] 14 | pub(crate) struct SubscriptionRaw { 15 | pub(crate) filter: String, 16 | #[serde_as(as = "serde_with::OneOrMany<_>")] 17 | pub(crate) datatypes: Vec, 18 | pub(crate) callback: String, 19 | pub(crate) streaming: Option, 20 | } 21 | 22 | #[derive(Debug, Clone)] 23 | pub(crate) struct SubscriptionConfig { 24 | pub(crate) subscriptions: Vec, 25 | } 26 | 27 | impl SubscriptionConfig { 28 | pub(crate) fn from_raw(config: &ConfigRaw) -> Self { 29 | let mut subscriptions = vec![]; 30 | for s in &config.subscriptions { 31 | assert!(!s.datatypes.is_empty()); 32 | let mut spec = SubscriptionSpec::new(s.filter.clone(), s.callback.clone()); 33 | if let Some(streaming) = s.streaming { 34 | spec.level = Level::Streaming(streaming); 35 | } 36 | for datatype_str in &s.datatypes { 37 | Self::validate_datatype(datatype_str.as_str()); 38 | let datatype = DATATYPES.get(datatype_str.as_str()).unwrap().clone(); 39 | spec.add_datatype(datatype); 40 | } 41 | spec.validate_spec(); 42 | subscriptions.push(spec); 43 | } 44 | Self { subscriptions } 45 | } 46 | 47 | pub(crate) fn from_file(filepath_in: &str) -> Self { 48 | let config_str = std::fs::read_to_string(filepath_in) 49 | .unwrap_or_else(|err| panic!("ERROR: File read failed {}: {:?}", filepath_in, err)); 50 | 51 | let config: ConfigRaw = toml::from_str(&config_str) 52 | .unwrap_or_else(|err| panic!("ERROR: Config file invalid {}: {:?}", filepath_in, err)); 53 | Self::from_raw(&config) 54 | } 55 | 56 | fn validate_datatype(datatype: &str) { 57 | if !DATATYPES.contains_key(datatype) { 58 | let valid_types: Vec<&str> = DATATYPES.keys().copied().collect(); 59 | panic!( 60 | "Invalid datatype: {};\nDid you mean:\n {}", 61 | datatype, 62 | valid_types.join(",\n") 63 | ); 64 | } 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /filtergen/src/proto_filter.rs: -------------------------------------------------------------------------------- 1 | use quote::quote; 2 | 3 | use crate::utils::*; 4 | use retina_core::filter::ast::*; 5 | use retina_core::filter::ptree::{FilterLayer, PNode, PTree}; 6 | 7 | pub(crate) fn gen_proto_filter( 8 | ptree: &PTree, 9 | statics: &mut Vec, 10 | ) -> proc_macro2::TokenStream { 11 | let mut body: Vec = vec![]; 12 | 13 | if !ptree.root.actions.drop() || !ptree.root.deliver.is_empty() { 14 | update_body(&mut body, &ptree.root, FilterLayer::Protocol, false); 15 | } 16 | 17 | gen_proto_filter_util(&mut body, statics, &ptree.root, FilterLayer::Protocol); 18 | 19 | let start = quote! { let mut result = retina_core::filter::Actions::new(); }; 20 | let ret = quote! { result }; 21 | 22 | let connection_filter = quote! { 23 | #start 24 | #( #body )* 25 | #ret 26 | }; 27 | connection_filter 28 | } 29 | 30 | fn gen_proto_filter_util( 31 | code: &mut Vec, 32 | statics: &mut Vec, 33 | node: &PNode, 34 | _filter_layer: FilterLayer, 35 | ) { 36 | let mut first_unary = true; 37 | for child in node 38 | .children 39 | .iter() 40 | .filter(|n| n.pred.on_packet() || n.pred.on_proto()) 41 | { 42 | match &child.pred { 43 | Predicate::Unary { protocol } => { 44 | if child.pred.on_packet() { 45 | ConnDataFilter::add_unary_pred( 46 | code, 47 | statics, 48 | child, 49 | protocol, 50 | first_unary, 51 | FilterLayer::Protocol, 52 | &gen_proto_filter_util, 53 | ); 54 | first_unary = false; 55 | } else if child.pred.on_proto() { 56 | ConnDataFilter::add_service_pred( 57 | code, 58 | statics, 59 | child, 60 | protocol, 61 | FilterLayer::Protocol, 62 | &gen_proto_filter_util, 63 | ); 64 | } 65 | } 66 | Predicate::Binary { 67 | protocol, 68 | field, 69 | op, 70 | value, 71 | } => { 72 | assert!(child.pred.on_packet()); 73 | ConnDataFilter::add_binary_pred( 74 | code, 75 | statics, 76 | child, 77 | protocol, 78 | field, 79 | op, 80 | value, 81 | FilterLayer::Protocol, 82 | &gen_proto_filter_util, 83 | ); 84 | } 85 | } 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /filtergen/src/session_filter.rs: -------------------------------------------------------------------------------- 1 | use quote::quote; 2 | 3 | use retina_core::filter::ast::*; 4 | use retina_core::filter::ptree::{FilterLayer, PNode, PTree}; 5 | 6 | use crate::utils::*; 7 | 8 | pub(crate) fn gen_session_filter( 9 | ptree: &PTree, 10 | statics: &mut Vec, 11 | ) -> proc_macro2::TokenStream { 12 | let mut body: Vec = vec![]; 13 | if !ptree.root.actions.drop() || !ptree.root.deliver.is_empty() { 14 | update_body(&mut body, &ptree.root, FilterLayer::Session, false); 15 | } 16 | 17 | gen_session_filter_util(&mut body, statics, &ptree.root, FilterLayer::Session); 18 | 19 | let start = quote! { let mut result = retina_core::filter::Actions::new(); }; 20 | let ret = quote! { result }; 21 | 22 | let session_filter = quote! { 23 | #start 24 | #( #body )* 25 | #ret 26 | }; 27 | session_filter 28 | } 29 | 30 | fn gen_session_filter_util( 31 | code: &mut Vec, 32 | statics: &mut Vec, 33 | node: &PNode, 34 | _filter_layer: FilterLayer, 35 | ) { 36 | let mut first_unary = true; 37 | for child in node.children.iter() { 38 | match &child.pred { 39 | Predicate::Unary { protocol } => { 40 | if child.pred.on_packet() { 41 | ConnDataFilter::add_unary_pred( 42 | code, 43 | statics, 44 | child, 45 | protocol, 46 | first_unary, 47 | FilterLayer::Session, 48 | &gen_session_filter_util, 49 | ); 50 | first_unary = false; 51 | } else if child.pred.on_proto() { 52 | SessionDataFilter::add_service_pred( 53 | code, 54 | statics, 55 | child, 56 | protocol, 57 | first_unary, 58 | FilterLayer::Session, 59 | &gen_session_filter_util, 60 | ); 61 | first_unary = false; 62 | } else { 63 | panic!("Found unary predicate in session filter pattern"); 64 | } 65 | } 66 | Predicate::Binary { 67 | protocol, 68 | field, 69 | op, 70 | value, 71 | } => { 72 | if child.pred.on_packet() { 73 | ConnDataFilter::add_binary_pred( 74 | code, 75 | statics, 76 | child, 77 | protocol, 78 | field, 79 | op, 80 | value, 81 | FilterLayer::Session, 82 | &gen_session_filter_util, 83 | ); 84 | } else if child.pred.on_session() { 85 | add_binary_pred(code, statics, child, protocol, field, op, value); 86 | } else { 87 | panic!("Found binary predicate in connection filter pattern"); 88 | } 89 | } 90 | } 91 | } 92 | } 93 | 94 | #[allow(clippy::too_many_arguments)] 95 | pub(crate) fn add_binary_pred( 96 | code: &mut Vec, 97 | statics: &mut Vec, 98 | node: &PNode, 99 | protocol: &ProtocolName, 100 | field: &FieldName, 101 | op: &BinOp, 102 | value: &Value, 103 | ) { 104 | let mut body: Vec = vec![]; 105 | gen_session_filter_util(&mut body, statics, node, FilterLayer::Session); 106 | let pred_tokenstream = binary_to_tokens(protocol, field, op, value, statics); 107 | update_body(&mut body, node, FilterLayer::Session, false); 108 | 109 | if node.if_else { 110 | code.push(quote! { 111 | else if #pred_tokenstream { 112 | #( #body )* 113 | } 114 | }); 115 | } else { 116 | code.push(quote! { 117 | if #pred_tokenstream { 118 | #( #body )* 119 | } 120 | }); 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /traces/README.md: -------------------------------------------------------------------------------- 1 | # traces 2 | 3 | A collection of sample packet captures pulled from a variety of sources. 4 | 5 | | Trace | Source | Description | 6 | |--------------------|-------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------| 7 | | `small_flows.pcap` | [Tcpreplay Sample Captures](https://tcpreplay.appneta.com/wiki/captures.html) | A synthetic combination of a few different applications and protocols at a relatively low network traffic rate. | 8 | | `tls_ciphers.pcap` | [Wireshark Sample Captures](https://wiki.wireshark.org/SampleCaptures) | OpenSSL client/server GET requests over TLS 1.2 with 73 different cipher suites. | 9 | | `quic_retry.pcapng`| [Wireshark Issue](https://gitlab.com/wireshark/wireshark/-/issues/18757) | An example of a QUIC Retry Packet. Original Pcap modified to remove CookedLinux and add Ether | 10 | | `quic_xargs.pcap` | [illustrated-quic GitHub](https://github.com/syncsynchalt/illustrated-quic/blob/main/captures/capture.pcap) | The pcap used in the creation of [The Illustrated QUIC Connection](https://quic.xargs.org). | 11 | | `quic_kyber.pcap` | Captured from Chrome 124 | A QUIC packet demonstrating the use of the Kyber keyshare, exceeding MTU, and requiring CRYPTO buffers. | 12 | -------------------------------------------------------------------------------- /traces/quic.pcap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stanford-esrg/retina/8c90da88db32f6bc6a9878c774834fef10f2a520/traces/quic.pcap -------------------------------------------------------------------------------- /traces/quic_kyber.pcapng: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stanford-esrg/retina/8c90da88db32f6bc6a9878c774834fef10f2a520/traces/quic_kyber.pcapng -------------------------------------------------------------------------------- /traces/quic_retry.pcapng: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stanford-esrg/retina/8c90da88db32f6bc6a9878c774834fef10f2a520/traces/quic_retry.pcapng -------------------------------------------------------------------------------- /traces/quic_xargs.pcap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stanford-esrg/retina/8c90da88db32f6bc6a9878c774834fef10f2a520/traces/quic_xargs.pcap -------------------------------------------------------------------------------- /traces/small_flows.pcap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stanford-esrg/retina/8c90da88db32f6bc6a9878c774834fef10f2a520/traces/small_flows.pcap -------------------------------------------------------------------------------- /traces/tls_ciphers.pcap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stanford-esrg/retina/8c90da88db32f6bc6a9878c774834fef10f2a520/traces/tls_ciphers.pcap --------------------------------------------------------------------------------