├── .gitignore ├── serverConn.go ├── client.go ├── server.go ├── README.md └── main.go /.gitignore: -------------------------------------------------------------------------------- 1 | .*.swp 2 | tcp-fast-open 3 | -------------------------------------------------------------------------------- /serverConn.go: -------------------------------------------------------------------------------- 1 | // Interfaces for the server's establish tcp connection with a client 2 | 3 | package main 4 | 5 | import ( 6 | "log" 7 | "syscall" 8 | ) 9 | 10 | // A client/server connection accepted by TFOServer 11 | type TFOServerConn struct { 12 | sockaddr *syscall.SockaddrInet4 13 | fd int 14 | } 15 | 16 | // Read the data from the client and immediately close the connection 17 | func (cxn *TFOServerConn) Handle() { 18 | 19 | defer cxn.Close() 20 | 21 | log.Printf("Server Conn: Connection received from remote addr: %v, remote port: %d\n", 22 | cxn.sockaddr.Addr, cxn.sockaddr.Port) 23 | 24 | // Create a small buffer to store data from client 25 | buf := make([]byte, 24) 26 | 27 | // Read from the socket, assign to buffer 28 | n, err := syscall.Read(cxn.fd, buf) 29 | if err != nil { 30 | log.Println("Failed to read() client:", err) 31 | return 32 | } 33 | 34 | // Do nothing in particular with the response, just print it 35 | log.Printf("Server Conn: Read %d bytes: %#v", n, string(buf[:n])) 36 | 37 | // The defer will close the connection now 38 | 39 | } 40 | 41 | // Gracefully close the connection to a client 42 | func (cxn *TFOServerConn) Close() { 43 | 44 | // Gracefull close the connection 45 | err := syscall.Shutdown(cxn.fd, syscall.SHUT_RDWR) 46 | if err != nil { 47 | log.Println("Failed to shutdown() connection:", err) 48 | } 49 | 50 | // Close the file descriptor 51 | err = syscall.Close(cxn.fd) 52 | if err != nil { 53 | log.Println("Failed to close() connection:", err) 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /client.go: -------------------------------------------------------------------------------- 1 | // Interfaces for a client to establish a TFO connection to a server 2 | 3 | package main 4 | 5 | import ( 6 | "errors" 7 | "fmt" 8 | "log" 9 | "syscall" 10 | ) 11 | 12 | type TFOClient struct { 13 | ServerAddr [4]byte 14 | ServerPort int 15 | fd int 16 | } 17 | 18 | // Create a tcp socket and send data on it. This uses the sendto() system call 19 | // instead of connect() - because connect() calls does not support sending 20 | // data in the syn packet, but the sendto() system call does (as often used in 21 | // connectionless protocols such as udp. 22 | func (c *TFOClient) Send() (err error) { 23 | 24 | c.fd, err = syscall.Socket(syscall.AF_INET, syscall.SOCK_STREAM, 0) 25 | if err != nil { 26 | return 27 | } 28 | defer syscall.Close(c.fd) 29 | 30 | sa := &syscall.SockaddrInet4{Addr: c.ServerAddr, Port: c.ServerPort} 31 | 32 | // Data to appear, if an existing tcp fast open cookie is available, this 33 | // data will appear in the SYN packet, if not, it will appear in the ACK. 34 | data := []byte("Hello TCP Fast Open") 35 | 36 | log.Printf("Client: Sending to server: %#v\n", string(data)) 37 | 38 | // Use the sendto() syscall, instead of connect() 39 | err = syscall.Sendto(c.fd, data, syscall.MSG_FASTOPEN, sa) 40 | if err != nil { 41 | if err == syscall.EOPNOTSUPP { 42 | err = errors.New("TCP Fast Open client support is unavailable (unsupported kernel or disabled, see /proc/sys/net/ipv4/tcp_fastopen).") 43 | } 44 | err = errors.New(fmt.Sprintf("Received error in sendTo():", err)) 45 | return 46 | } 47 | 48 | // Note, this exists before waiting for response and is meant to illustrate 49 | // the use of the sendto() system call, not of a complete and proper socket 50 | // setup and teardown processes. 51 | 52 | return 53 | } 54 | -------------------------------------------------------------------------------- /server.go: -------------------------------------------------------------------------------- 1 | // Interface to listen on a TFO enabled TCP socket 2 | package main 3 | 4 | import ( 5 | "errors" 6 | "fmt" 7 | "log" 8 | "syscall" 9 | ) 10 | 11 | type TFOServer struct { 12 | ServerAddr [4]byte 13 | ServerPort int 14 | fd int 15 | } 16 | 17 | const TCP_FASTOPEN int = 23 18 | const LISTEN_BACKLOG int = 23 19 | 20 | // Create a tcp socket, setting the TCP_FASTOPEN socket option. 21 | func (s *TFOServer) Bind() (err error) { 22 | 23 | s.fd, err = syscall.Socket(syscall.AF_INET, syscall.SOCK_STREAM, 0) 24 | if err != nil { 25 | if err == syscall.ENOPROTOOPT { 26 | err = errors.New("TCP Fast Open server support is unavailable (unsupported kernel).") 27 | } 28 | return 29 | } 30 | 31 | err = syscall.SetsockoptInt(s.fd, syscall.SOL_TCP, TCP_FASTOPEN, 1) 32 | if err != nil { 33 | err = errors.New(fmt.Sprintf("Failed to set necessary TCP_FASTOPEN socket option: %s", err)) 34 | return 35 | } 36 | 37 | sa := &syscall.SockaddrInet4{Addr: s.ServerAddr, Port: s.ServerPort} 38 | 39 | err = syscall.Bind(s.fd, sa) 40 | if err != nil { 41 | err = errors.New(fmt.Sprintf("Failed to bind to Addr: %v, Port: %d, Reason: %s", s.ServerAddr, s.ServerPort, err)) 42 | return 43 | } 44 | 45 | log.Printf("Server: Bound to addr: %v, port: %d\n", s.ServerAddr, s.ServerPort) 46 | 47 | err = syscall.Listen(s.fd, LISTEN_BACKLOG) 48 | if err != nil { 49 | err = errors.New(fmt.Sprintf("Failed to listen: %s", err)) 50 | return 51 | } 52 | 53 | return 54 | 55 | } 56 | 57 | // Block, waiting for connections, handling each connection in its own go 58 | // routine 59 | func (s *TFOServer) Accept() { 60 | 61 | log.Println("Server: Waiting for connections") 62 | 63 | defer syscall.Close(s.fd) 64 | 65 | for { 66 | 67 | fd, sockaddr, err := syscall.Accept(s.fd) 68 | if err != nil { 69 | log.Fatalln("Failed to accept(): ", err) 70 | } 71 | 72 | cxn := TFOServerConn{fd: fd, sockaddr: sockaddr.(*syscall.SockaddrInet4)} 73 | 74 | go cxn.Handle() 75 | 76 | } 77 | 78 | } 79 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | TCP Fast Open 2 | ============= 3 | 4 | Go example of TCP Fast Open (TFO) as described by RFC7413 and available in Linux Kernel 3.7 (server and client support). 5 | 6 | Check support for TCP Fast Open by checking: `/proc/sys/net/ipv4/tcp_fastopen`, ensure this value is 3 for client and 7 | server support. If necessary, echo 3 to this file, eg: 8 | 9 | ```bash 10 | # echo 0 > /proc/sys/net/ipv4/tcp_fastopen 11 | # cat /proc/sys/net/ipv4/tcp_fastopen 12 | 0 13 | # echo 3 > /proc/sys/net/ipv4/tcp_fastopen 14 | # cat /proc/sys/net/ipv4/tcp_fastopen 15 | 3 16 | ``` 17 | 18 | Using standard Linux Kernel system calls, this program shows steps required to configure a server and client to 19 | establish a TCP connection using TFO. 20 | 21 | The program simply establishes a connection to itself, using go routines, but doesn't demonstrate a complete client 22 | server architecture with correct tear down. 23 | 24 | Note: Go itself provides better handlers for establishing and listening to sockets via the 25 | [net](https://golang.org/pkg/net/) package, but these packages don't currently provide TFO support. 26 | 27 | Usage 28 | ===== 29 | 30 | ```bash 31 | Usage of ./tcp-fast-open: 32 | Options: 33 | -s 127.0.0.1 --server=127.0.0.1 Server to connect to (and listen if listening) 34 | -p 2222 --port=2222 Port to connect to (and listen to if listening) 35 | -l --listen Create a listening TFO socket 36 | --help show usage message 37 | ``` 38 | 39 | Create a listening socket, then connect to it: 40 | ```bash 41 | tcp-fast-open -l 42 | ``` 43 | 44 | Connect to remote server: 45 | 46 | ```bash 47 | tcp-fast-open -s 192.0.2.1 -p 80 48 | ``` 49 | 50 | Once connected, you'll need to use `ip tcp_metrics` (check for `fo_cookie`) or `tcpdump` to determine whether a TFO connection was successful. 51 | 52 | 53 | Behaviour 54 | ========= 55 | 56 | TFO's goal is to establish a connection regardless of client, server or middleware support. The system calls provided by 57 | the Kernel therefore abstract these details from the caller. For this reason this program itself can't determine whether: 58 | - the connection was not established using cookies, but have been successfully transferred for consecutive connections 59 | - the connection was establish using cookies 60 | - the connection used an invalid cookie 61 | 62 | Therefore, for this program, you'll need to use tcpdump to analyse the packets yourself: `tcpdump -s 0 -XX -nn -i lo port 63 | 2222` 64 | 65 | When analysing the traces, pay particular attention to the following: 66 | 67 | 1. SYN packet should not contain data (like most standard SYN packets), but should contain to TFO option. 68 | 2. SYN-ACK response should contain a cookie to be cached by the client 69 | 3. ACK packet, as usual, contains data (and PUSH bit) 70 | 71 | On the second, and consecutive connections, the following behaviour should occur: 72 | 73 | 1. SYN packet contains data, as well as TFO option and cookie 74 | 2. Usual SYN-ACK response 75 | 3. ACK packet contains no data 76 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "log" 6 | "net" 7 | "os/exec" 8 | "regexp" 9 | "strings" 10 | "time" 11 | 12 | "github.com/droundy/goopt" 13 | ) 14 | 15 | var connect = goopt.String([]string{"-s", "--server"}, "127.0.0.1", "Server to connect to (and listen if listening)") 16 | var port = goopt.Int([]string{"-p", "--port"}, 2222, "Port to connect to (and listen to if listening)") 17 | 18 | var listen = goopt.Flag([]string{"-l", "--listen"}, []string{}, "Create a listening TFO socket", "") 19 | 20 | func main() { 21 | 22 | goopt.Parse(nil) 23 | 24 | // IPv4 only for no real reason, could be v6 by adjusting the sizes 25 | // here and where it's used 26 | var serverAddr [4]byte 27 | 28 | IP := net.ParseIP(*connect) 29 | if IP == nil { 30 | log.Fatal("Unable to process IP: ", *connect) 31 | } 32 | 33 | copy(serverAddr[:], IP[12:16]) 34 | 35 | if *listen { 36 | 37 | server := TFOServer{ServerAddr: serverAddr, ServerPort: *port} 38 | err := server.Bind() 39 | if err != nil { 40 | log.Fatalln("Failed to bind socket:", err) 41 | } 42 | 43 | // Create a new routine ("thread") and wait for connection from client 44 | go server.Accept() 45 | 46 | } 47 | 48 | client := TFOClient{ServerAddr: serverAddr, ServerPort: *port} 49 | 50 | err := client.Send() 51 | if err != nil { 52 | log.Fatalln("Failed to send to server:", err) 53 | } 54 | 55 | // Give the server a chance to receive, process the packet and print results 56 | time.Sleep(100 * time.Millisecond) 57 | 58 | success, cached, err := checkTcpMetrics(*connect) 59 | if err != nil { 60 | log.Println("ip tcp_metrics failure:", err) 61 | } else { 62 | var response string 63 | if success { 64 | response = "TFO success to IP " + *connect 65 | } else { 66 | response = "TFO failure to IP " + *connect 67 | } 68 | if len(cached) > 0 { 69 | response += " " + strings.Join(cached, ", ") 70 | } 71 | log.Println(response) 72 | } 73 | 74 | } 75 | 76 | // Use `ip tcp_metrics` to check whether we received a cookie or not. Only 77 | // available in later versions of iproute 78 | func checkTcpMetrics(ip string) (success bool, cached []string, err error) { 79 | 80 | cmd := exec.Command("ip", "tcp_metrics", "show", ip) 81 | 82 | var out bytes.Buffer 83 | cmd.Stdout = &out 84 | err = cmd.Run() 85 | if err != nil { 86 | return 87 | } 88 | 89 | reFOc := regexp.MustCompile(" fo_cookie ([a-z0-9]+)") 90 | reFOmss := regexp.MustCompile(" fo_mss ([0-9]+)") 91 | reFOdrop := regexp.MustCompile(" fo_syn_drops ([0-9./]sec ago)") 92 | 93 | cookie := reFOc.FindStringSubmatch(out.String()) 94 | mss := reFOmss.FindStringSubmatch(out.String()) 95 | drop := reFOdrop.FindStringSubmatch(out.String()) 96 | 97 | success = len(cookie) > 0 98 | 99 | if len(cookie) > 0 { 100 | cached = append(cached, "cookie: "+cookie[1]) 101 | } 102 | 103 | if len(mss) > 0 { 104 | cached = append(cached, "mss: "+mss[1]) 105 | } 106 | 107 | if len(drop) > 0 { 108 | cached = append(cached, "syn_drops: "+drop[1]) 109 | } 110 | 111 | return 112 | 113 | } 114 | --------------------------------------------------------------------------------