├── .clang-format ├── .gitignore ├── README.md ├── cmd └── badbox.go ├── go.mod ├── go.sum ├── include └── vmlinux.h └── internal ├── pidfd └── pidfd.go └── probe ├── probe.c ├── probe.go └── probe_bpf.go /.clang-format: -------------------------------------------------------------------------------- 1 | --- 2 | Language: Cpp 3 | BasedOnStyle: LLVM 4 | AlignAfterOpenBracket: DontAlign 5 | AlignConsecutiveAssignments: true 6 | AlignEscapedNewlines: DontAlign 7 | AlwaysBreakBeforeMultilineStrings: true 8 | AlwaysBreakTemplateDeclarations: false 9 | AllowAllParametersOfDeclarationOnNextLine: false 10 | BreakBeforeBraces: Linux 11 | IndentWidth: 4 12 | KeepEmptyLinesAtTheStartOfBlocks: false 13 | TabWidth: 4 14 | UseTab: ForContinuationAndIndentation 15 | ColumnLimit: 1000 16 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *_bpf.o 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # find-bad-middleboxes 2 | 3 | This is the proof-of-concept code that accompanies the eBPF Summit lightning talk I gave called Bad middlebox! 4 | 5 | You can try this yourself to see if anyone connecting to you over IPv6 has a bad middlebox that alters their flow labels. 6 | This also functions as an example of using the cilium/ebpf bpf2go project to compile and embed eBPF objects in go code. 7 | -------------------------------------------------------------------------------- /cmd/badbox.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "os" 8 | "os/signal" 9 | 10 | "github.com/markpash/find-bad-middleboxes/internal/pidfd" 11 | "github.com/markpash/find-bad-middleboxes/internal/probe" 12 | "golang.org/x/sys/unix" 13 | ) 14 | 15 | func main() { 16 | pid := flag.Uint64("pid", 0, "provide a pid to get socket file descriptor and set TCP_SAVED_SYN sockopt") 17 | flag.Parse() 18 | 19 | if *pid != 0 { 20 | if err := stealAndSetSockOpt(*pid); err != nil { 21 | panic(err) 22 | } 23 | } 24 | 25 | ctx := context.Background() 26 | ctx, cancel := context.WithCancel(ctx) 27 | 28 | s := make(chan os.Signal, 1) 29 | signal.Notify(s, os.Interrupt) 30 | 31 | go func() { 32 | <-s 33 | signal.Stop(s) 34 | cancel() 35 | }() 36 | 37 | if err := probe.Run(ctx); err != nil { 38 | panic(err) 39 | } 40 | } 41 | 42 | func stealAndSetSockOpt(pid uint64) error { 43 | fds, err := pidfd.Files(int(pid)) 44 | if err != nil { 45 | return err 46 | } 47 | 48 | for _, fd := range fds { 49 | if err := unix.SetsockoptInt(int(fd.Fd()), unix.SOL_TCP, unix.TCP_SAVE_SYN, 1); err != nil { 50 | return err 51 | } 52 | } 53 | return nil 54 | } 55 | 56 | func panic(err error) { 57 | fmt.Fprintf(os.Stderr, "%s\n", err) 58 | os.Exit(1) 59 | } 60 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/markpash/find-bad-middleboxes 2 | 3 | go 1.16 4 | 5 | require ( 6 | github.com/cilium/ebpf v0.6.3-0.20210803095552-6b7634f668de 7 | golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c 8 | inet.af/netaddr v0.0.0-20210729200904-31d5ee66059c 9 | ) 10 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/cilium/ebpf v0.6.3-0.20210803095552-6b7634f668de h1:hfwB8ypnnagT5z4ZDATBzrcyTuYuL+9TpwTtw6XPTSE= 2 | github.com/cilium/ebpf v0.6.3-0.20210803095552-6b7634f668de/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= 3 | github.com/dvyukov/go-fuzz v0.0.0-20210103155950-6a8e9d1f2415/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= 4 | github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY= 5 | github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= 6 | github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= 7 | github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 8 | github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= 9 | github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 10 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 11 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 12 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 13 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 14 | go4.org/intern v0.0.0-20210108033219-3eb7198706b2 h1:VFTf+jjIgsldaz/Mr00VaCSswHJrI2hIjQygE/W4IMg= 15 | go4.org/intern v0.0.0-20210108033219-3eb7198706b2/go.mod h1:vLqJ+12kCw61iCWsPto0EOHhBS+o4rO5VIucbc9g2Cc= 16 | go4.org/unsafe/assume-no-moving-gc v0.0.0-20201222175341-b30ae309168e/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E= 17 | go4.org/unsafe/assume-no-moving-gc v0.0.0-20201222180813-1025295fd063 h1:1tk03FUNpulq2cuWpXZWj649rwJpk0d20rxWiopKRmc= 18 | go4.org/unsafe/assume-no-moving-gc v0.0.0-20201222180813-1025295fd063/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E= 19 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 20 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 21 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 22 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 23 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 24 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 25 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 26 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 27 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 28 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 29 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 30 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 31 | golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 32 | golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 33 | golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= 34 | golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 35 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 36 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 37 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 38 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 39 | golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= 40 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 41 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 42 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 43 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= 44 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 45 | inet.af/netaddr v0.0.0-20210729200904-31d5ee66059c h1:AmHBNAHZCfyMGFk5kd060kpnQ6GJLHmGwdJGGQgr1Y8= 46 | inet.af/netaddr v0.0.0-20210729200904-31d5ee66059c/go.mod h1:z0nx+Dh+7N7CC8V5ayHtHGpZpxLQZZxkIaaz6HN65Ls= 47 | -------------------------------------------------------------------------------- /internal/pidfd/pidfd.go: -------------------------------------------------------------------------------- 1 | package pidfd 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "os" 7 | 8 | "golang.org/x/sys/unix" 9 | ) 10 | 11 | func Files(pid int) (files []*os.File, err error) { 12 | const maxFDGap = 32 13 | 14 | defer func() { 15 | if err != nil { 16 | for _, file := range files { 17 | file.Close() 18 | } 19 | } 20 | }() 21 | 22 | if pid == os.Getpid() { 23 | // Retrieving files from the current process makes the loop below 24 | // never finish. 25 | return nil, fmt.Errorf("can't retrieve files from the same process") 26 | } 27 | 28 | pidfd, err := pidfdOpen(pid, 0) 29 | if err != nil { 30 | return nil, err 31 | } 32 | defer unix.Close(pidfd) 33 | 34 | for i, gap := 0, 0; i < int(^uint(0)>>1) && gap < maxFDGap; i++ { 35 | target, err := pidfdGetFD(pidfd, i, 0) 36 | if errors.Is(err, unix.EBADF) { 37 | gap++ 38 | continue 39 | } 40 | if err != nil { 41 | return nil, fmt.Errorf("target fd %d: %s", i, err) 42 | } 43 | gap = 0 44 | 45 | keep, err := filter(target) 46 | if errors.Is(err, unix.ENOTSOCK) { 47 | unix.Close(target) 48 | } else if err != nil { 49 | unix.Close(target) 50 | return nil, fmt.Errorf("target fd %d: %w", i, err) 51 | } else if keep { 52 | files = append(files, os.NewFile(uintptr(target), "")) 53 | } else { 54 | unix.Close(target) 55 | } 56 | } 57 | 58 | return files, nil 59 | } 60 | 61 | func filter(fd int) (bool, error) { 62 | // get the domain of the socket and reject it if it's not ipv6 63 | domain, err := unix.GetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_DOMAIN) 64 | if err != nil { 65 | return false, err 66 | } 67 | if domain != unix.AF_INET6 { 68 | return false, nil 69 | } 70 | 71 | // get type of the socket, and if it's not tcp, reject it 72 | soType, err := unix.GetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_TYPE) 73 | if err != nil { 74 | return false, fmt.Errorf("getsockopt(SO_TYPE): %s", err) 75 | } 76 | 77 | if soType != unix.SOCK_STREAM { 78 | return false, nil 79 | } 80 | 81 | // see if the socket is a listening one, if it isn't then reject it 82 | acceptConn, err := unix.GetsockoptInt(int(fd), unix.SOL_SOCKET, unix.SO_ACCEPTCONN) 83 | if err != nil { 84 | return false, fmt.Errorf("getsockopt(SO_ACCEPTCONN): %s", err) 85 | } 86 | 87 | if acceptConn == 0 { 88 | // Not a listening socket 89 | return false, nil 90 | } 91 | 92 | return true, nil 93 | } 94 | 95 | func pidfdOpen(pid, flags int) (int, error) { 96 | fd, _, errNo := unix.Syscall(unix.SYS_PIDFD_OPEN, uintptr(pid), uintptr(flags), 0) 97 | if errNo != 0 { 98 | return -1, fmt.Errorf("pidfd_open(%d): %w", pid, errNo) 99 | } 100 | return int(fd), nil 101 | } 102 | 103 | func pidfdGetFD(pidfd, target, flags int) (int, error) { 104 | fd, _, errNo := unix.Syscall(unix.SYS_PIDFD_GETFD, uintptr(pidfd), uintptr(target), uintptr(flags)) 105 | if errNo != 0 { 106 | return -1, fmt.Errorf("pidfd_getfd(%d, %d): %w", pidfd, target, errNo) 107 | } 108 | return int(fd), nil 109 | } 110 | -------------------------------------------------------------------------------- /internal/probe/probe.c: -------------------------------------------------------------------------------- 1 | // +build ignore 2 | 3 | #include "../include/vmlinux.h" 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #define IPV6_FLOWLABEL_MASK __bpf_constant_htonl(0x000FFFFF) 11 | #define PT_REGS_PARM1(x) ((x)->di) 12 | #define PT_REGS_PARM2(x) ((x)->si) 13 | 14 | SEC("license") char _license[] = "Dual MIT/GPL"; 15 | 16 | struct { 17 | __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); 18 | } pipe SEC(".maps"); 19 | 20 | enum { 21 | AF_INET = 2, 22 | AF_INET6 = 10, 23 | }; 24 | 25 | struct flow_t { 26 | __u32 syn_flow_lbl, ack_flow_lbl; 27 | struct in6_addr laddr, raddr; 28 | __le16 lport; 29 | __be16 rport; 30 | }; 31 | 32 | SEC("kprobe/tcp_set_state") 33 | int BPF_KPROBE(kprobe__tcp_set_state, struct sock *sk, int state) 34 | { 35 | if (state != BPF_TCP_ESTABLISHED) 36 | return 0; 37 | 38 | if (BPF_CORE_READ(sk, __sk_common.skc_family) != AF_INET6) 39 | return 0; 40 | 41 | struct saved_syn *saved_syn = BPF_CORE_READ((struct tcp_sock *)sk, saved_syn); 42 | if (!saved_syn) 43 | return 0; 44 | 45 | // First 32 bit word of saved_syn is the length of the buffer. 46 | __u32 syn_flow_lbl; 47 | if (bpf_probe_read(&syn_flow_lbl, sizeof(syn_flow_lbl), saved_syn + 1)) 48 | return 0; 49 | 50 | syn_flow_lbl &= IPV6_FLOWLABEL_MASK; 51 | 52 | __u32 ack_flow_lbl = BPF_CORE_READ((struct tcp6_sock *)sk, inet6.rcv_flowinfo) & IPV6_FLOWLABEL_MASK; 53 | if (syn_flow_lbl == ack_flow_lbl) 54 | return 0; 55 | 56 | struct flow_t flow = { 57 | .syn_flow_lbl = syn_flow_lbl, 58 | .ack_flow_lbl = ack_flow_lbl, 59 | .laddr = BPF_CORE_READ(sk, __sk_common.skc_v6_rcv_saddr), 60 | .raddr = BPF_CORE_READ(sk, __sk_common.skc_v6_daddr), 61 | .lport = BPF_CORE_READ(sk, __sk_common.skc_num), 62 | .rport = BPF_CORE_READ(sk, __sk_common.skc_dport), 63 | }; 64 | 65 | bpf_perf_event_output(ctx, &pipe, BPF_F_CURRENT_CPU, &flow, sizeof(flow)); 66 | 67 | return 0; 68 | } 69 | -------------------------------------------------------------------------------- /internal/probe/probe.go: -------------------------------------------------------------------------------- 1 | package probe 2 | 3 | import ( 4 | "context" 5 | "encoding/binary" 6 | "fmt" 7 | 8 | "github.com/cilium/ebpf/link" 9 | "github.com/cilium/ebpf/perf" 10 | "golang.org/x/sys/unix" 11 | "inet.af/netaddr" 12 | ) 13 | 14 | //go:generate go run github.com/cilium/ebpf/cmd/bpf2go -target bpf -cc clang probe probe.c -- -Wall -I../../include 15 | 16 | type flow struct { 17 | synLabel uint32 18 | ackLabel uint32 19 | lAddr netaddr.IP 20 | rAddr netaddr.IP 21 | lPort uint16 22 | rPort uint16 23 | } 24 | 25 | func Run(ctx context.Context) error { 26 | if err := unix.Setrlimit(unix.RLIMIT_MEMLOCK, &unix.Rlimit{ 27 | Cur: unix.RLIM_INFINITY, 28 | Max: unix.RLIM_INFINITY, 29 | }); err != nil { 30 | return err 31 | } 32 | 33 | var probe probeObjects 34 | if err := loadProbeObjects(&probe, nil); err != nil { 35 | return err 36 | } 37 | 38 | kp, err := link.Kprobe("tcp_set_state", probe.KprobeTcpSetState) 39 | if err != nil { 40 | return err 41 | } 42 | defer kp.Close() 43 | 44 | pipe := probe.Pipe 45 | rd, err := perf.NewReader(pipe, 10) 46 | if err != nil { 47 | return err 48 | } 49 | defer rd.Close() 50 | 51 | c := make(chan []byte) 52 | go func() { 53 | for { 54 | event, err := rd.Read() 55 | if err != nil { 56 | fmt.Println(err) 57 | continue 58 | } 59 | c <- event.RawSample 60 | } 61 | }() 62 | 63 | for { 64 | select { 65 | case <-ctx.Done(): 66 | kp.Close() 67 | return probe.Close() 68 | case raw := <-c: 69 | var lAddr, rAddr [16]byte 70 | copy(lAddr[:], raw[8:24]) 71 | copy(rAddr[:], raw[24:40]) 72 | 73 | flow := flow{ 74 | synLabel: binary.BigEndian.Uint32(raw[0:4]), 75 | ackLabel: binary.BigEndian.Uint32(raw[4:8]), 76 | lAddr: netaddr.IPFrom16(lAddr), 77 | rAddr: netaddr.IPFrom16(rAddr), 78 | lPort: binary.LittleEndian.Uint16(raw[40:42]), // this is little endian for some reason 79 | rPort: binary.BigEndian.Uint16(raw[42:44]), 80 | } 81 | fmt.Printf("src: %v : %d\n", flow.rAddr.String(), flow.rPort) 82 | fmt.Printf("dest: %v : %d\n", flow.lAddr.String(), flow.lPort) 83 | fmt.Printf("flow label modified: %v\n", flow.synLabel != flow.ackLabel) 84 | fmt.Printf("\n") 85 | } 86 | } 87 | } 88 | --------------------------------------------------------------------------------