├── .gitignore ├── Makefile ├── README.md ├── Vagrantfile ├── echo_dispatch.bpf.c ├── sk_lookup_attach.c ├── sockmap_update.c ├── syscall.h └── talk └── eBPF Summit 2020 - Steering connections to sockets with BPF socket lookup hook.pdf /.gitignore: -------------------------------------------------------------------------------- 1 | linux-5.9.1/ 2 | .vagrant/ 3 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 | # Copyright (c) 2020 Cloudflare 3 | 4 | KERNEL := linux-5.9.1 5 | KERNEL_INC := $(KERNEL)/usr/include 6 | LIBBPF_INC := $(KERNEL)/tools/lib 7 | LIBBPF_LIB := $(KERNL)/tools/lib/bpf/libbpf.a 8 | 9 | CC := clang 10 | CFLAGS := -g -O2 -Wall -Wextra 11 | CPPFLAGS := -I$(KERNEL_INC) -I$(LIBBPF_INC) 12 | 13 | PROGS := sockmap-update sk-lookup-attach echo_dispatch.bpf.o bpftool 14 | 15 | .PHONY: all 16 | all: $(PROGS) 17 | 18 | sockmap-update: sockmap_update.c $(KERNEL_INC) 19 | $(CC) $(CPPFLAGS) $(CFLAGS) -o $@ $< 20 | 21 | sk-lookup-attach: sk_lookup_attach.c $(KERNEL_INC) 22 | $(CC) $(CPPFLAGS) $(CFLAGS) -o $@ $< 23 | 24 | echo_dispatch.bpf.o: echo_dispatch.bpf.c $(KERNEL_INC) $(LIBBPF_INC) $(LIBBPF_LIB) 25 | $(CC) $(CPPFLAGS) $(CFLAGS) -target bpf -c -o $@ $< 26 | 27 | # Download kernel sources 28 | $(KERNEL).tar.xz: 29 | curl -O https://cdn.kernel.org/pub/linux/kernel/v5.x/$(KERNEL).tar.xz 30 | 31 | # Unpack kernel sources 32 | $(KERNEL): $(KERNEL).tar.xz 33 | tar axf $< 34 | 35 | # Install kernel headers 36 | $(KERNEL_INC): $(KERNEL) 37 | make -C $< headers_install INSTALL_HDR_PATH=$@ 38 | 39 | # Build libbpf to generate helper definitions header 40 | $(LIBBPF_LIB): $(KERNEL) 41 | make -C $ mtu 1500 qdisc fq_codel state UP group default qlen 1000 48 | altname enp0s5 49 | altname ens5 50 | inet 192.168.122.247/24 brd 192.168.122.255 scope global dynamic noprefixroute eth0 51 | valid_lft 2834sec preferred_lft 2834sec 52 | ``` 53 | 54 | VM external IP is `192.168.122.247`. 55 | 56 | ``` 57 | host $ nmap -sT -p 1-1000 192.168.122.247 58 | Starting Nmap 7.80 ( https://nmap.org ) at 2020-10-28 17:10 CET 59 | Nmap scan report for 192.168.122.247 60 | Host is up (0.00021s latency). 61 | Not shown: 999 closed ports 62 | PORT STATE SERVICE 63 | 22/tcp open ssh 64 | 65 | Nmap done: 1 IP address (1 host up) scanned in 0.08 seconds 66 | ``` 67 | 68 | Only port `22` is open. 69 | 70 | ## Load `echo_dispatch` BPF program 71 | 72 | ``` 73 | vm $ cd /vagrant 74 | vm $ sudo ./bpftool prog load ./echo_dispatch.bpf.o /sys/fs/bpf/echo_dispatch_prog 75 | vm $ sudo ./bpftool prog show pinned /sys/fs/bpf/echo_dispatch_prog 76 | 49: sk_lookup name echo_dispatch tag da043673afd29081 gpl 77 | loaded_at 2020-10-28T16:13:42+0000 uid 0 78 | xlated 272B jited 164B memlock 4096B map_ids 3,4 79 | btf_id 4 80 | ``` 81 | 82 | ## Pin BPF maps used by `echo_dispatch` 83 | 84 | Mount a dedicated bpf file-system for our user `vagrant`: 85 | 86 | ``` 87 | vm $ mkdir ~/bpffs 88 | vm $ sudo mount -t bpf none ~/bpffs 89 | vm $ sudo chown vagrant.vagrant ~/bpffs 90 | ``` 91 | 92 | Pin `echo_ports` map 93 | 94 | ``` 95 | vm $ sudo ./bpftool map show name echo_ports 96 | 3: hash name echo_ports flags 0x0 97 | key 2B value 1B max_entries 1024 memlock 86016B 98 | vm $ sudo ./bpftool map pin name echo_ports ~/bpffs/echo_ports 99 | ``` 100 | 101 | Pin `echo_socket` map 102 | 103 | ``` 104 | vm $ sudo ./bpftool map show name echo_socket 105 | 4: sockmap name echo_socket flags 0x0 106 | key 4B value 8B max_entries 1 memlock 4096B 107 | vm $ sudo ./bpftool map pin name echo_socket ~/bpffs/echo_socket 108 | ``` 109 | 110 | Give user `vagrant` access to pinned maps: 111 | 112 | ``` 113 | vm $ sudo chown vagrant.vagrant ~/bpffs/{echo_ports,echo_socket} 114 | ``` 115 | 116 | ## Insert Ncat socket into `echo_socket` map 117 | 118 | Find socket owner PID and FD number: 119 | 120 | ``` 121 | vm $ ss -tlpne 'sport = 7777' 122 | State Recv-Q Send-Q Local Address:Port Peer Address:Port Process 123 | LISTEN 0 10 127.0.0.1:7777 0.0.0.0:* users:(("nc",pid=24533,fd=3)) uid:1000 ino:38462 sk:1 <-> 124 | ``` 125 | 126 | Put the socket into `echo_socket` map using `socket-update` tool: 127 | 128 | ``` 129 | vm $ ./sockmap-update 24533 3 ~/bpffs/echo_socket 130 | vm $ ./bpftool map dump pinned ~/bpffs/echo_socket 131 | $ ./bpftool map dump pinned ~/bpffs/echo_socket 132 | key: 00 00 00 00 value: 01 00 00 00 00 00 00 00 133 | Found 1 element 134 | ``` 135 | 136 | Notice the value under key `0x00` is the socket cookie (`0x01`) we saw in `ss` 137 | output (`sk:1`). Socket cookie is a unique identifier for a socket description 138 | inside the kernel. 139 | 140 | ## Attach `echo_dispatch` to network namespace 141 | 142 | Create a BPF link between the current network namespace and the loaded 143 | `echo_dispatch` program with the `sk-lookup-attach` tool: 144 | 145 | ``` 146 | vm $ sudo ./sk-lookup-attach /sys/fs/bpf/echo_dispatch_prog /sys/fs/bpf/echo_dispatch_link 147 | ``` 148 | 149 | Examine the created BPF link: 150 | 151 | ``` 152 | vm $ sudo ./bpftool link show pinned /sys/fs/bpf/echo_dispatch_link 153 | 4: netns prog 49 154 | netns_ino 4026531992 attach_type sk_lookup 155 | vm $ ls -l /proc/self/ns/net 156 | lrwxrwxrwx. 1 vagrant vagrant 0 Oct 28 16:29 /proc/self/ns/net -> 'net:[4026531992]' 157 | ``` 158 | 159 | Notice the BPF link can be matched with the network namespace via its inode number. 160 | 161 | ## Enable echo service on ports 7, 77, 777 162 | 163 | Populate `echo_ports` map with entries for open ports `7` (`0x7`), `77` 164 | (`0x4d`), and `7777` (`0x0309`): 165 | 166 | ``` 167 | vm $ ./bpftool map update pinned ~/bpffs/echo_ports key 0x07 0x00 value 0x00 168 | vm $ ./bpftool map update pinned ~/bpffs/echo_ports key 0x4d 0x00 value 0x00 169 | vm $ ./bpftool map update pinned ~/bpffs/echo_ports key 0x09 0x03 value 0x00 170 | vm $ ./bpftool map dump pinned ~/bpffs/echo_ports 171 | key: 4d 00 value: 00 172 | key: 07 00 value: 00 173 | key: 09 03 value: 00 174 | Found 3 elements 175 | ``` 176 | 177 | ## Re-scan open ports on VM 178 | 179 | ``` 180 | host $ nmap -sT -p 1-1000 192.168.122.247 181 | Starting Nmap 7.80 ( https://nmap.org ) at 2020-10-28 17:34 CET 182 | Nmap scan report for 192.168.122.247 183 | Host is up (0.00017s latency). 184 | Not shown: 996 closed ports 185 | PORT STATE SERVICE 186 | 7/tcp open echo 187 | 22/tcp open ssh 188 | 77/tcp open priv-rje 189 | 777/tcp open multiling-http 190 | 191 | Nmap done: 1 IP address (1 host up) scanned in 0.09 seconds 192 | ``` 193 | 194 | Notice echo ports we have just configured are open. 195 | 196 | ## Test the echo service on all open ports 197 | 198 | ``` 199 | host $ { echo 'Hip'; sleep 0.1; } | nc -4 192.168.122.247 7 && \ 200 | { echo 'hip'; sleep 0.1; } | nc -4 192.168.122.247 77 && \ 201 | { echo 'hooray!'; sleep 0.1; } | nc -4 192.168.122.247 777 202 | Hip 203 | hip 204 | hooray! 205 | ``` 206 | -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | 4 | Vagrant.configure("2") do |config| 5 | config.vm.box = "fedora/32-cloud-base" 6 | config.vm.box_version = "32.20200422.0" 7 | 8 | config.vm.hostname = "bpf-sk-lookup" 9 | 10 | config.vm.provider :libvirt do |libvirt| 11 | libvirt.memory = 8192 12 | libvirt.cpus = 4 13 | end 14 | 15 | config.vm.provision "shell", inline: <<-SHELL 16 | # Configure dnf repo for vanilla kernel 17 | curl -s https://repos.fedorapeople.org/repos/thl/kernel-vanilla.repo | \ 18 | sudo tee /etc/yum.repos.d/kernel-vanilla.repo 19 | dnf config-manager --set-enabled kernel-vanilla-stable 20 | # Install latest vanilla kernel 21 | dnf update -y kernel-core 22 | # Install development tools 23 | dnf install -y clang gcc make elfutils-libelf-devel llvm nmap-ncat 24 | # Reboot into new kernel 25 | reboot 26 | SHELL 27 | 28 | config.vm.synced_folder ".", "/vagrant", type: "rsync", 29 | rsync__exclude: [".git/", ".vagrant/", "talk/"] 30 | end 31 | -------------------------------------------------------------------------------- /echo_dispatch.bpf.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 | /* Copyright (c) 2020 Cloudflare */ 3 | /* 4 | * BPF socket lookup program that dispatches connections destined to a 5 | * configured set of open ports, to the echo service socket. 6 | * 7 | * Program expects the echo service socket to be in the `echo_socket` BPF map. 8 | * Port is considered open when an entry for that port number exists in the 9 | * `echo_ports` BPF hashmap. 10 | * 11 | */ 12 | 13 | #include 14 | 15 | #include 16 | #include 17 | 18 | /* Declare BPF maps */ 19 | 20 | /* List of open echo service ports. Key is the port number. */ 21 | struct bpf_map_def SEC("maps") echo_ports = { 22 | .type = BPF_MAP_TYPE_HASH, 23 | .max_entries = 1024, 24 | .key_size = sizeof(__u16), 25 | .value_size = sizeof(__u8), 26 | }; 27 | 28 | /* Echo server socket */ 29 | struct bpf_map_def SEC("maps") echo_socket = { 30 | .type = BPF_MAP_TYPE_SOCKMAP, 31 | .max_entries = 1, 32 | .key_size = sizeof(__u32), 33 | .value_size = sizeof(__u64), 34 | }; 35 | 36 | /* Dispatcher program for the echo service */ 37 | SEC("sk_lookup/echo_dispatch") 38 | int echo_dispatch(struct bpf_sk_lookup *ctx) 39 | { 40 | const __u32 zero = 0; 41 | struct bpf_sock *sk; 42 | __u16 port; 43 | __u8 *open; 44 | long err; 45 | 46 | /* Is echo service enabled on packets destination port? */ 47 | port = ctx->local_port; 48 | open = bpf_map_lookup_elem(&echo_ports, &port); 49 | if (!open) 50 | return SK_PASS; 51 | 52 | /* Get echo server socket */ 53 | sk = bpf_map_lookup_elem(&echo_socket, &zero); 54 | if (!sk) 55 | return SK_DROP; 56 | 57 | /* Dispatch the packet to echo server socket */ 58 | err = bpf_sk_assign(ctx, sk, 0); 59 | bpf_sk_release(sk); 60 | return err ? SK_DROP : SK_PASS; 61 | } 62 | 63 | SEC("license") const char __license[] = "Dual BSD/GPL"; 64 | -------------------------------------------------------------------------------- /sk_lookup_attach.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 | /* Copyright (c) 2020 Cloudflare */ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include 14 | 15 | #include "syscall.h" 16 | 17 | int main(int argc, char **argv) 18 | { 19 | const char *prog_path; 20 | const char *link_path; 21 | union bpf_attr attr; 22 | int prog_fd, netns_fd, link_fd, err; 23 | 24 | if (argc != 3) { 25 | fprintf(stderr, "Usage: %s \n", argv[0]); 26 | exit(EXIT_SUCCESS); 27 | } 28 | 29 | prog_path = argv[1]; 30 | link_path = argv[2]; 31 | 32 | /* 1. Open the pinned BPF program */ 33 | memset(&attr, 0, sizeof(attr)); 34 | attr.pathname = (uint64_t) prog_path; 35 | attr.file_flags = BPF_F_RDONLY; 36 | 37 | prog_fd = bpf(BPF_OBJ_GET, &attr, sizeof(attr)); 38 | if (prog_fd == -1) 39 | error(EXIT_FAILURE, errno, "bpf(OBJ_GET)"); 40 | 41 | /* 2. Get an FD for this process network namespace (netns) */ 42 | netns_fd = open("/proc/self/ns/net", O_RDONLY | O_CLOEXEC); 43 | if (netns_fd == -1) 44 | error(EXIT_FAILURE, errno, "open"); 45 | 46 | /* 3. Attach BPF sk_lookup program to the (netns) with a BPF link */ 47 | memset(&attr, 0, sizeof(attr)); 48 | attr.link_create.prog_fd = prog_fd; 49 | attr.link_create.target_fd = netns_fd; 50 | attr.link_create.attach_type = BPF_SK_LOOKUP; 51 | attr.link_create.flags = 0; 52 | 53 | link_fd = bpf(BPF_LINK_CREATE, &attr, sizeof(attr)); 54 | if (link_fd == -1) 55 | error(EXIT_FAILURE, errno, "bpf(LINK_CREATE)"); 56 | 57 | /* 4. Pin the BPF link (otherwise would be destroyed on FD close) */ 58 | memset(&attr, 0, sizeof(attr)); 59 | attr.pathname = (uint64_t) link_path; 60 | attr.bpf_fd = link_fd; 61 | attr.file_flags = 0; 62 | 63 | err = bpf(BPF_OBJ_PIN, &attr, sizeof(attr)); 64 | if (err) 65 | error(EXIT_FAILURE, errno, "bpf(OBJ_PIN)"); 66 | 67 | close(link_fd); 68 | close(netns_fd); 69 | close(prog_fd); 70 | 71 | exit(EXIT_SUCCESS); 72 | } 73 | -------------------------------------------------------------------------------- /sockmap_update.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 | /* Copyright (c) 2020 Cloudflare */ 3 | /* 4 | * Inserts a socket belonging to another process, as specified by the target PID 5 | * and FD number, into a given BPF map. 6 | */ 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #include 17 | 18 | #include "syscall.h" 19 | 20 | int main(int argc, char **argv) 21 | { 22 | pid_t target_pid; 23 | int pid_fd, target_fd, sock_fd, map_fd, err; 24 | uint32_t key; 25 | uint64_t value; 26 | const char *map_path; 27 | union bpf_attr attr; 28 | 29 | if (argc < 4) { 30 | fprintf(stderr, "Usage: %s [map key]\n", argv[0]); 31 | exit(EXIT_SUCCESS); 32 | } 33 | 34 | target_pid = atoi(argv[1]); 35 | target_fd = atoi(argv[2]); 36 | map_path = argv[3]; 37 | key = 0; 38 | 39 | if (argc == 5) 40 | key = atoi(argv[4]); 41 | 42 | /* Get duplicate FD for the socket */ 43 | pid_fd = pidfd_open(target_pid, 0); 44 | if (pid_fd == -1) 45 | error(EXIT_FAILURE, errno, "pidfd_open"); 46 | 47 | sock_fd = pidfd_getfd(pid_fd, target_fd, 0); 48 | if (sock_fd == -1) 49 | error(EXIT_FAILURE, errno, "pidfd_getfd"); 50 | 51 | /* Open BPF map for storing the socket */ 52 | memset(&attr, 0, sizeof(attr)); 53 | attr.pathname = (uint64_t) map_path; 54 | attr.bpf_fd = 0; 55 | attr.file_flags = 0; 56 | 57 | map_fd = bpf(BPF_OBJ_GET, &attr, sizeof(attr)); 58 | if (map_fd == -1) 59 | error(EXIT_FAILURE, errno, "bpf(OBJ_GET)"); 60 | 61 | /* Insert socket FD into the BPF map */ 62 | value = (uint64_t) sock_fd; 63 | memset(&attr, 0, sizeof(attr)); 64 | attr.map_fd = map_fd; 65 | attr.key = (uint64_t) &key; 66 | attr.value = (uint64_t) &value; 67 | attr.flags = BPF_ANY; 68 | 69 | err = bpf(BPF_MAP_UPDATE_ELEM, &attr, sizeof(attr)); 70 | if (err) 71 | error(EXIT_FAILURE, errno, "bpf(MAP_UPDATE_ELEM)"); 72 | 73 | close(map_fd); 74 | close(sock_fd); 75 | close(pid_fd); 76 | } 77 | -------------------------------------------------------------------------------- /syscall.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /* 4 | * Syscall wrappers missing in glibc 5 | */ 6 | 7 | #define _GNU_SOURCE 8 | #include 9 | #include 10 | 11 | #include 12 | 13 | static inline int bpf(enum bpf_cmd cmd, union bpf_attr *attr, unsigned int size) 14 | { 15 | return syscall(__NR_bpf, cmd, attr, size); 16 | } 17 | 18 | static inline int pidfd_open(pid_t target_pid, unsigned int flags) 19 | { 20 | return syscall(__NR_pidfd_open, target_pid, flags); 21 | } 22 | 23 | static inline int pidfd_getfd(int pidfd, int targetfd, unsigned int flags) 24 | { 25 | return syscall(__NR_pidfd_getfd, pidfd, targetfd, flags); 26 | } 27 | -------------------------------------------------------------------------------- /talk/eBPF Summit 2020 - Steering connections to sockets with BPF socket lookup hook.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsitnicki/ebpf-summit-2020/92203ca2d7edb5f7bd7032473950f5256b6b7701/talk/eBPF Summit 2020 - Steering connections to sockets with BPF socket lookup hook.pdf --------------------------------------------------------------------------------