├── .gitmodules ├── src ├── xdp_prog.h ├── loader.c └── xdp_prog.c ├── Makefile └── README.md /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "libbpf"] 2 | path = libbpf 3 | url = https://github.com/libbpf/libbpf 4 | -------------------------------------------------------------------------------- /src/xdp_prog.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #ifndef htons 4 | #define htons(x) ((__be16)___constant_swab16((x))) 5 | #endif 6 | 7 | #define NULL ((void*)0) -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | CC = clang 2 | 3 | BUILDDIR = build 4 | SRCDIR = src 5 | 6 | LIBBPFSRC = libbpf/src 7 | LIBBPFOBJS += $(LIBBPFSRC)/staticobjs/bpf_prog_linfo.o $(LIBBPFSRC)/staticobjs/bpf.o $(LIBBPFSRC)/staticobjs/btf_dump.o 8 | LIBBPFOBJS += $(LIBBPFSRC)/staticobjs/btf.o $(LIBBPFSRC)/staticobjs/hashmap.o $(LIBBPFSRC)/staticobjs/libbpf_errno.o 9 | LIBBPFOBJS += $(LIBBPFSRC)/staticobjs/libbpf_probes.o $(LIBBPFSRC)/staticobjs/libbpf.o $(LIBBPFSRC)/staticobjs/netlink.o 10 | LIBBPFOBJS += $(LIBBPFSRC)/staticobjs/nlattr.o $(LIBBPFSRC)/staticobjs/str_error.o 11 | LIBBPFOBJS += $(LIBBPFSRC)/staticobjs/xsk.o 12 | 13 | LOADERSRC = loader.c 14 | LOADEROUT = tcpopts 15 | LOADERFLAGS = -lelf -lz 16 | 17 | XDPSRC = xdp_prog.c 18 | XDPBC = xdp.bc 19 | XDPOBJ = xdp.o 20 | 21 | INCS = -I $(LIBBPFSRC) 22 | 23 | all: loader xdp 24 | loader: libbpf 25 | mkdir -p $(BUILDDIR) 26 | $(CC) $(INCS) $(LOADERFLAGS) -O2 -o $(BUILDDIR)/$(LOADEROUT) $(LIBBPFOBJS) $(SRCDIR)/$(LOADERSRC) 27 | xdp: 28 | mkdir -p $(BUILDDIR) 29 | $(CC) $(INCS) -D__BPF__ -Wall -Wextra -O2 -emit-llvm -c $(SRCDIR)/$(XDPSRC) -o $(BUILDDIR)/$(XDPBC) 30 | llc -march=bpf -filetype=obj $(BUILDDIR)/$(XDPBC) -o $(BUILDDIR)/$(XDPOBJ) 31 | libbpf: 32 | $(MAKE) -C $(LIBBPFSRC) 33 | install: 34 | mkdir -p /etc/tcpopts 35 | cp $(BUILDDIR)/$(XDPOBJ) /etc/tcpopts/$(XDPOBJ) 36 | cp $(BUILDDIR)/$(LOADEROUT) /usr/bin/$(LOADEROUT) 37 | clean: 38 | $(MAKE) -C $(LIBBPFSRC) clean 39 | rm -f $(BUILDDIR)/* 40 | .PHONY: libbpf xdp loader 41 | .DEFAULT: all -------------------------------------------------------------------------------- /src/loader.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include 11 | #include 12 | #include 13 | 14 | #include 15 | #include 16 | 17 | const struct option longopts[] = 18 | { 19 | {"interface", required_argument, NULL, 'i'}, 20 | {"obj", required_argument, NULL, 'o'}, 21 | {NULL, 0, NULL, 0} 22 | }; 23 | 24 | __u8 cont = 1; 25 | 26 | char *dev = NULL; 27 | char *objfile = NULL; 28 | 29 | void sighndl(int tmp) 30 | { 31 | cont = 0; 32 | } 33 | 34 | /** 35 | * Raises the RLimit. 36 | * 37 | * @return Returns 0 on success (EXIT_SUCCESS) or 1 on failure (EXIT_FAILURE). 38 | */ 39 | int raise_rlimit() 40 | { 41 | struct rlimit r = {RLIM_INFINITY, RLIM_INFINITY}; 42 | 43 | if (setrlimit(RLIMIT_MEMLOCK, &r)) 44 | { 45 | return EXIT_FAILURE; 46 | } 47 | 48 | return EXIT_SUCCESS; 49 | } 50 | 51 | void parsecmdline(int argc, char *argv[]) 52 | { 53 | int c = -1; 54 | 55 | while ((c = getopt_long(argc, argv, "i:o:", longopts, NULL)) != -1) 56 | { 57 | switch (c) 58 | { 59 | case 'i': 60 | dev = optarg; 61 | 62 | break; 63 | 64 | case 'o': 65 | objfile = optarg; 66 | 67 | break; 68 | 69 | case '?': 70 | fprintf(stderr, "Missing argument value.\n"); 71 | 72 | break; 73 | } 74 | } 75 | } 76 | 77 | int main(int argc, char *argv[]) 78 | { 79 | // Raise RLimit 80 | if (raise_rlimit() != 0) 81 | { 82 | fprintf(stderr, "Error setting rlimit. Please ensure you're running this program as a privileged user.\n"); 83 | 84 | return EXIT_FAILURE; 85 | } 86 | 87 | // Parse command line. 88 | parsecmdline(argc, argv); 89 | 90 | // Check if we have an interface specified. 91 | if (dev == NULL) 92 | { 93 | fprintf(stderr, "Missing interface argument.\n"); 94 | 95 | return EXIT_FAILURE; 96 | } 97 | 98 | // Check to see if the interface exists and get its index. 99 | int ifidx = if_nametoindex(dev); 100 | 101 | if (ifidx < 0) 102 | { 103 | fprintf(stderr, "Interface index less than 0 (Not Found).\n"); 104 | 105 | return EXIT_FAILURE; 106 | } 107 | 108 | // If we don't have an object path, set to default. 109 | if (objfile == NULL) 110 | { 111 | objfile = "/etc/tcpopts/xdp.o"; 112 | } 113 | 114 | fprintf(stdout, "Attempting to load BPF object file => %s.\n", objfile); 115 | 116 | struct bpf_object *obj = NULL; 117 | int bpffd = -1; 118 | 119 | // Load BPF/XDP program. 120 | bpf_prog_load(objfile, BPF_PROG_TYPE_XDP, &obj, &bpffd); 121 | 122 | if (bpffd < 0) 123 | { 124 | fprintf(stderr, "Error loading BPF program.\n"); 125 | 126 | return EXIT_FAILURE; 127 | } 128 | 129 | __u32 flags = XDP_FLAGS_UPDATE_IF_NOEXIST | XDP_FLAGS_DRV_MODE; 130 | 131 | // Attach XDP program with DRV mode. 132 | int err = bpf_set_link_xdp_fd(ifidx, bpffd, flags); 133 | 134 | if (err) 135 | { 136 | fprintf(stdout, "DRV mode not supported. Trying SKB instead.\n"); 137 | 138 | // Attempt to try SKB mode. 139 | flags = XDP_FLAGS_UPDATE_IF_NOEXIST | XDP_FLAGS_SKB_MODE; 140 | 141 | err = bpf_set_link_xdp_fd(ifidx, bpffd, flags); 142 | 143 | if (err) 144 | { 145 | fprintf(stderr, "Error attaching XDP program in DRV or SKB mode :: %s (%d).\n", strerror(-err), -err); 146 | 147 | return EXIT_FAILURE; 148 | } 149 | } 150 | 151 | // Setup signal. 152 | signal(SIGINT, sighndl); 153 | 154 | // Loop (we sleep every second to avoid CPU consumption). 155 | while (cont) 156 | { 157 | sleep(1); 158 | } 159 | 160 | // Detach XDP program. 161 | bpf_set_link_xdp_fd(ifidx, -1, flags); 162 | 163 | return EXIT_SUCCESS; 164 | } -------------------------------------------------------------------------------- /src/xdp_prog.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include "xdp_prog.h" 10 | 11 | //#define PRINT 12 | 13 | #ifdef PRINT 14 | #define bpf_printk(fmt, ...) \ 15 | ({ \ 16 | char ____fmt[] = fmt; \ 17 | bpf_trace_printk(____fmt, sizeof(____fmt), \ 18 | ##__VA_ARGS__); \ 19 | }) 20 | #endif 21 | 22 | SEC("xdp_prog") 23 | int prog(struct xdp_md *ctx) 24 | { 25 | // Initialize data and data_end along with needed headers along with checks. 26 | void *data = (void *)(long)ctx->data; 27 | void *data_end = (void *)(long)ctx->data_end; 28 | 29 | struct ethhdr *eth = data; 30 | 31 | if (eth + 1 > (struct ethhdr *)data_end) 32 | { 33 | return XDP_DROP; 34 | } 35 | 36 | if (eth->h_proto != htons(ETH_P_IP)) 37 | { 38 | return XDP_PASS; 39 | } 40 | 41 | struct iphdr *iph = data + sizeof (struct ethhdr); 42 | 43 | if (iph + 1 > (struct iphdr *)data_end) 44 | { 45 | return XDP_DROP; 46 | } 47 | 48 | if (iph->protocol != IPPROTO_TCP) 49 | { 50 | return XDP_PASS; 51 | } 52 | 53 | struct tcphdr *tcph = data + sizeof(struct ethhdr) + (iph->ihl * 4); 54 | 55 | if (tcph + 1 > (struct tcphdr *)data_end) 56 | { 57 | return XDP_DROP; 58 | } 59 | 60 | // These variables will indicate the timestamp values in memory if found (otherwise NULL). 61 | __u32 *senderts = NULL; 62 | __u32 *recvts = NULL; 63 | 64 | // Check to see if we have additional TCP header options. 65 | if (tcph->doff > 5) 66 | { 67 | #ifdef PRINT 68 | bpf_printk("[TCPOPTS] Have TCP header options. Header length => %d. Beginning to parse options.\n", tcph->doff * 5); 69 | #endif 70 | 71 | __u16 off = 0; 72 | __u8 *opts = data + sizeof(struct ethhdr) + (iph->ihl * 4) + 20; 73 | 74 | if (opts + 1 > (__u8 *)data_end) 75 | { 76 | return XDP_PASS; 77 | } 78 | 79 | __u16 optdata = 0; 80 | 81 | while (optdata <= 40) 82 | { 83 | // Initialize the byte we're parsing and ensure it isn't outside of data_end. 84 | __u8 *val = opts + optdata; 85 | 86 | if (val + 1 > (__u8 *)data_end || val < (__u8 *)data) 87 | { 88 | break; 89 | } 90 | 91 | #ifdef PRINT 92 | bpf_printk("[TCPOPTS] Received %d as type code.\n", *val); 93 | #endif 94 | 95 | // 0x01 indicates a NOP which must be skipped. 96 | if (*val == 0x01) 97 | { 98 | #ifdef PRINT 99 | bpf_printk("[TCPOPTS] Skipping NOP.\n"); 100 | #endif 101 | 102 | optdata++; 103 | 104 | continue; 105 | } 106 | // 0x00 indicates end of TCP header options, so break loop. 107 | else if (*val == 0x00) 108 | { 109 | break; 110 | } 111 | // 0x08 indicates timestamps. 112 | else if (*val == 0x08) 113 | { 114 | // Adjust offset by two since +1 = option length and +2 = start of timestamps data. 115 | off = optdata + 2; 116 | 117 | #ifdef PRINT 118 | bpf_printk("[TCPOPTS] Found start of timestamps! Offset => %d.\n", off); 119 | #endif 120 | 121 | break; 122 | } 123 | // We need to increase by the option's length field for other options. 124 | else 125 | { 126 | #ifdef PRINT 127 | bpf_printk("[TCPOPTS] Found another TCP option! Adjusting by its length.\n"); 128 | #endif 129 | 130 | // Increase by option length (which is val + 1 since the option length is the second field). 131 | __u8 *len = val + 1; 132 | 133 | // Check to make sure the length pointer doesn't go outside of data_end and data (the packet). 134 | if (len + 1 > (__u8 *)data_end || len < (__u8 *)data) 135 | { 136 | break; 137 | } 138 | 139 | #ifdef PRINT 140 | bpf_printk("[TCPOPTS] Found option length => %d! Option type => %d.\n", *len, *val); 141 | #endif 142 | 143 | // This check shouldn't be needed, but just for safe measure, perform another check before incrementing optdata by the option's length. 144 | if (len <= (__u8 *)data_end && len >= (__u8 *)data) 145 | { 146 | optdata += (*len > 0) ? *len : 1; 147 | } 148 | else 149 | { 150 | // Avoid an infinite loop. 151 | optdata++; 152 | } 153 | 154 | continue; 155 | } 156 | 157 | // We shouldn't get here, but increment to prevent an infinite loop either way. 158 | optdata++; 159 | } 160 | 161 | // If the offset is above 0, that means we've found the timestamps. 162 | if (off > 0) 163 | { 164 | // We'll need to make sure 8 bytes after the offset is not outside of the packet since the two timestamps variables are 32 bits (8 bytes). 165 | if (opts + off + 8 <= (__u8 *)data_end && opts + off + 8 >= (__u8 *)data) 166 | { 167 | // Sender timestamp should be the first 32-bit value. 168 | senderts = (__u32 *)opts + off; 169 | 170 | // Receive timestamp should be the second 32-bit value. 171 | recvts = (__u32 *)opts + off + 4; 172 | 173 | #ifdef PRINT 174 | bpf_printk("[TCPOPTS] Sender TS Value => %lu. Receive TS Value => %lu.\n", *senderts, *recvts); 175 | #endif 176 | } 177 | } 178 | } 179 | 180 | return XDP_PASS; 181 | } 182 | 183 | char _license[] SEC("license") = "GPL"; -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # XDP/BPF TCP Header Options Parsing 2 | ## Update 11-9-2021 3 | Around a month or so ago, I asked how to locate the TCP header timestamp options inside of XDP and thanks to Toke Høiland-Jørgensen [here](https://marc.info/?l=xdp-newbies&m=163178833212690&w=2), I was able to parse the TCP header timestamp options. I ended up using a modified version of [this](https://github.com/xdp-project/bpf-examples/blob/master/pping/pping_kern.c#L83) function which can be found below. 4 | 5 | ```C 6 | #define MAX_TCP_OPTIONS 10 7 | 8 | static __always_inline int parse_tcp_ts(struct tcphdr *tcph, void *data_end, __u32 **tsval, __u32 **tsecr) 9 | { 10 | int len = tcph->doff << 2; 11 | void *opt_end = (void *)tcph + len; 12 | __u8 *pos = (__u8 *)(tcph + 1); 13 | __u8 i, opt; 14 | volatile __u8 opt_size; 15 | 16 | if (tcph + 1 > (struct tcphdr *)data_end || len <= sizeof(struct tcphdr)) 17 | { 18 | #ifdef DEBUG 19 | bpf_printk("parse_tcp_ts() :: tcph + 1 > (struct tcphdr *)data_end || len <= sizeof(struct tcphdr)\n"); 20 | #endif 21 | 22 | return -1; 23 | } 24 | 25 | #pragma unroll 26 | for (i = 0; i < MAX_TCP_OPTIONS; i++) 27 | { 28 | if (pos + 1 > (__u8 *)opt_end || pos + 1 > (__u8 *)data_end) 29 | { 30 | #ifdef DEBUG 31 | bpf_printk("parse_tcp_ts() :: pos + 1 > (__u8 *)opt_end || pos + 1 > (__u8 *)data_end\n"); 32 | #endif 33 | 34 | return -1; 35 | } 36 | 37 | opt = *pos; 38 | 39 | if (opt == 0) 40 | { 41 | #ifdef DEBUG 42 | bpf_printk("parse_tcp_ts() :: opt == 0\n"); 43 | #endif 44 | 45 | return -1; 46 | } 47 | 48 | if (opt == 1) 49 | { 50 | pos++; 51 | 52 | continue; 53 | } 54 | 55 | if (pos + 2 > (__u8 *)opt_end || pos + 2 > (__u8 *)data_end) 56 | { 57 | #ifdef DEBUG 58 | bpf_printk("parse_tcp_ts() :: pos + 2 > (__u8 *)opt_end || pos + 2 > (__u8 *)data_end\n"); 59 | #endif 60 | 61 | return -1; 62 | } 63 | 64 | opt_size = *(pos + 1); 65 | 66 | if (opt_size < 2) 67 | { 68 | #ifdef DEBUG 69 | bpf_printk("parse_tcp_ts() :: opt_size < 2\n"); 70 | #endif 71 | 72 | return -1; 73 | } 74 | 75 | if (opt == 8 && opt_size == 10) 76 | { 77 | if (pos + 10 > (__u8 *)opt_end || pos + 10 > (__u8 *)data_end) 78 | { 79 | #ifdef DEBUG 80 | bpf_printk("parse_tcp_ts() :: pos + 10 > (__u8 *)opt_end || pos + 10 > (__u8 *)data_end\n"); 81 | #endif 82 | 83 | return -1; 84 | } 85 | 86 | *tsval = (__u32 *)(pos + 2); 87 | *tsecr = (__u32 *)(pos + 6); 88 | 89 | return 0; 90 | } 91 | 92 | pos += opt_size; 93 | } 94 | 95 | #ifdef DEBUG 96 | bpf_printk("parse_tcp_ts() :: Reached end (return -1).\n"); 97 | #endif 98 | 99 | return -1; 100 | } 101 | 102 | ... 103 | 104 | __u32 *sendval = NULL; 105 | __u32 *echoval = NULL; 106 | 107 | parse_tcp_ts(tcph, data_end, &sendval, &echoval); 108 | 109 | if (sendval != NULL && echoval != NULL) 110 | { 111 | // Timestamps found and pointers are valid. 112 | } 113 | ``` 114 | 115 | ## Description 116 | A repository to show attempts at dynamically parsing the TCP header options (specifically timestamps in this repository) within XDP/BPF. 117 | 118 | ## Command Line Options 119 | There are two command line options for this program which may be found below. 120 | 121 | * `-i --interface` => The interface name to attempt to attach the XDP program to (**required**). 122 | * `-o --obj` => A path to the BPF object file (default is `/etc/tcpopts/xdp.o` which `make install` installs to). 123 | 124 | ## Building 125 | You may use the following to build the program. 126 | 127 | ``` 128 | # Clone the repository and libbpf (with the --recursive flag). 129 | git clone --recursive https://github.com/gamemann/XDP-TCP-Header-Options.git 130 | 131 | # Change directory to the repository. 132 | cd XDP-TCP-Header-Options/ 133 | 134 | # Build the program. 135 | make 136 | 137 | # Install the program. The program is installed to /usr/bin/tcpopts 138 | sudo make install 139 | ``` 140 | 141 | ## Fails 142 | The current code fails with the following: 143 | 144 | ``` 145 | 49: (2d) if r6 > r2 goto pc+24 146 | R0_w=inv2 R1=pkt(id=0,off=0,r=34,imm=0) R2=pkt_end(id=0,off=0,imm=0) R3=pkt(id=1,off=34,r=36,umax_value=60,var_off=(0x0; 0x3c)) R4=inv41 R5_w=inv(id=0,umax_value=65535,var_off=(0x0; 0xffff)) R6_w=pkt(id=258,off=35,r=0,umax_value=65595,var_off=(0x0; 0x1ffff)) R7_w=pkt(id=258,off=34,r=0,umax_value=65595,var_off=(0x0; 0x1ffff)) R10=fp0 147 | 50: (71) r7 = *(u8 *)(r7 +0) 148 | invalid access to packet, off=34 size=1, R7(id=258,off=34,r=0) 149 | R7 offset is outside of the packet 150 | processed 8789 insns (limit 1000000) max_states_per_insn 4 total_states 142 peak_states 142 mark_read 4 151 | 152 | libbpf: -- END LOG -- 153 | libbpf: failed to load program 'xdp_prog' 154 | libbpf: failed to load object '/etc/tcpopts/xdp.o' 155 | Error loading BPF program. 156 | ``` 157 | 158 | The full log may be found in the `logs/` directory. 159 | 160 | The error is caused by this piece of code: 161 | 162 | ```C 163 | // This check shouldn't be needed, but just for safe measure, perform another check before incrementing optdata by the option's length. 164 | if (len <= (__u8 *)data_end && len >= (__u8 *)data) 165 | { 166 | optdata += (*len > 0) ? *len : 1; 167 | } 168 | else 169 | { 170 | // Avoid an infinite loop. 171 | optdata++; 172 | } 173 | ``` 174 | 175 | If you stop incrementing `optdata` by `*len`, the XDP program loads. For example: 176 | 177 | ```C 178 | if (len <= (__u8 *)data_end && len >= (__u8 *)data) 179 | { 180 | optdata++; 181 | } 182 | else 183 | { 184 | // Avoid an infinite loop. 185 | optdata++; 186 | } 187 | ``` 188 | 189 | Loads without any issues. 190 | 191 | ## Other Notes 192 | I found an article [here](https://legacy.netdevconf.info/0x14/pub/slides/50/Issuing%20SYN%20Cookies%20in%20XDP.pdf) where it appears the creator was having similar issues. At the end, under challenges/next steps you can see: 193 | 194 | > Parsing variable number of TCP options is challenging for the verifier 195 | 196 | Being able to parse TCP header options in XDP/BPF would be very useful in my opinion. The code I have right now has many checks that I don't think are needed, but I'm not able to tell the BPF verifier the code is safe/within the packet range for some reason when incrementing by a dynamic value. 197 | 198 | ## Credits 199 | * [Christian Deacon](https://github.com/gamemann) --------------------------------------------------------------------------------