├── CODE_OF_CONDUCT.md ├── TARGETS ├── util.go ├── LICENSE ├── CONTRIBUTING.md ├── README.md ├── tcp.go └── main.go /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | 3 | Facebook has adopted a Code of Conduct that we expect project participants to adhere to. Please [read the full text](https://code.fb.com/codeofconduct/) so that you can understand what actions will and will not be tolerated. 4 | -------------------------------------------------------------------------------- /TARGETS: -------------------------------------------------------------------------------- 1 | go_binary( 2 | name = "fbtracert", 3 | srcs = util.files("**/*.go"), 4 | go_external_deps = [ 5 | ("github.com/golang/glog", 6 | "d1c4472bf2efd3826f2b5bdcc02d8416798d678c"), 7 | ("github.com/olekukonko/tablewriter", 8 | "bc39950e081b457853031334b3c8b95cdfe428ba"), 9 | ], 10 | go_version = "1.5.1", 11 | ) 12 | -------------------------------------------------------------------------------- /util.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2016-present, Facebook, Inc. and its affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the BSD-style license found in the 6 | * LICENSE file in the root directory of this source tree. 7 | */ 8 | 9 | package main 10 | 11 | import ( 12 | "sync" 13 | ) 14 | 15 | // 16 | // Filter data on input channel 17 | // 18 | func filter(f func(interface{}) bool, in chan interface{}) chan interface{} { 19 | out := make(chan interface{}) 20 | 21 | go func() { 22 | for val := range in { 23 | if f(val) { 24 | out <- val 25 | } 26 | } 27 | }() 28 | 29 | return out 30 | } 31 | 32 | // 33 | // fork input channel into two, copy data 34 | // 35 | func fork(in <-chan interface{}) (out1, out2 chan interface{}) { 36 | out1, out2 = make(chan interface{}), make(chan interface{}) 37 | 38 | go func() { 39 | for val := range in { 40 | out1 <- val 41 | out2 <- val 42 | } 43 | }() 44 | 45 | return 46 | } 47 | 48 | // 49 | // Merge data from multiple channels into one 50 | // 51 | func merge(cs ...chan interface{}) chan interface{} { 52 | var wg sync.WaitGroup 53 | out := make(chan interface{}) 54 | 55 | output := func(c <-chan interface{}) { 56 | defer wg.Done() 57 | for val := range c { 58 | out <- val 59 | } 60 | } 61 | 62 | wg.Add(len(cs)) 63 | for _, ch := range cs { 64 | go output(ch) 65 | } 66 | 67 | go func() { 68 | wg.Wait() 69 | close(out) 70 | }() 71 | 72 | return out 73 | } 74 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD License 2 | 3 | For fbtracert software 4 | 5 | Copyright (c) Facebook, Inc. and its affiliates. All rights reserved. 6 | 7 | Redistribution and use in source and binary forms, with or without modification, 8 | are permitted provided that the following conditions are met: 9 | 10 | * Redistributions of source code must retain the above copyright notice, this 11 | list of conditions and the following disclaimer. 12 | 13 | * Redistributions in binary form must reproduce the above copyright notice, 14 | this list of conditions and the following disclaimer in the documentation 15 | and/or other materials provided with the distribution. 16 | 17 | * Neither the name Facebook nor the names of its contributors may be used to 18 | endorse or promote products derived from this software without specific 19 | prior written permission. 20 | 21 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 22 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 23 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR 25 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 26 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 27 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 28 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 30 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to fbtracert 2 | We want to make contributing to this project as easy and transparent as 3 | possible. 4 | 5 | ## Pull Requests 6 | We actively welcome your pull requests. 7 | 8 | 1. Fork the repo and create your branch from `master`. 9 | 2. If you've added code that should be tested, add tests. 10 | 3. If you've changed APIs, update the documentation. 11 | 4. Ensure the test suite passes. 12 | 5. Make sure your code lints. 13 | 6. If you haven't already, complete the Contributor License Agreement ("CLA"). 14 | 15 | ## Contributor License Agreement ("CLA") 16 | In order to accept your pull request, we need you to submit a CLA. You only need 17 | to do this once to work on any of Facebook's open source projects. 18 | 19 | Complete your CLA here: 20 | 21 | ## Issues 22 | We use GitHub issues to track public bugs. Please ensure your description is 23 | clear and has sufficient instructions to be able to reproduce the issue. 24 | 25 | Facebook has a [bounty program](https://www.facebook.com/whitehat/) for the safe 26 | disclosure of security bugs. In those cases, please go through the process 27 | outlined on that page and do not file a public issue. 28 | 29 | ## Coding Style 30 | * gofmt 31 | 32 | ## Code of Conduct 33 | Facebook has adopted a Code of Conduct that we expect project participants to adhere to. Please [read the full text](https://code.facebook.com/pages/876921332402685/open-source-code-of-conduct) so that you can understand what actions will and will not be tolerated. 34 | 35 | ## License 36 | By contributing to fbtracert, you agree that your contributions will be licensed 37 | under its BSD license. 38 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # fbtracert 2 | > pronounced: ef-BEE-tracerTEE 3 | 4 | ## Installing 5 | 6 | Requires golang >= 1.5.1: 7 | 8 | ```bash 9 | go get -d github.com/facebook/fbtracert 10 | go install github.com/facebook/fbtracert 11 | ``` 12 | 13 | ```bash 14 | $GOPATH/bin/fbtracert --help 15 | ``` 16 | 17 | ## Full documentation 18 | 19 | ### Fault isolation in ECMP networks via multi-port traceroute 20 | 21 | This tool attempts to identify the network component that drops packets by employing the traceroute logic 22 | that explores multiple parallel paths. The following describes the main goroutines and their logic. 23 | 24 | ### Sender 25 | 26 | We start this goroutine for every TTL that we expect on path to the destination. We start with some max TTL 27 | value, and then stop all senders that have TTL above the distance to the target. For every TTL, the sender 28 | loops over a range of source ports, and emits a TCP SYN packet towards the destination with the set target port. 29 | The Sender also emits "Probe" objects on a special channel so that the analysis part may know what packets 30 | have been injected in the network (srcPort and Ttl). 31 | 32 | Notice how encode the sending time-stamp and the ttl in the ISN of the TCP SYN packet. This allows for measuring 33 | the probe RTT, and recovering the TTL of the response. Just like regular traceroute, we expect the network to return 34 | us either ICMP Unreachable message (TTL exceeded) or TCP RST message (when we hit the ultimate hop). 35 | 36 | The Sender thread stops once it completes the requested number of iterations over the source port range. 37 | 38 | ### ICMP Receiver 39 | 40 | We run only one ICMP receiver goroutine: it is responsible for receiving the ICMP Unreachable messages and recovering 41 | the original probe information from them. We only use the first 8 bytes of the TCP packet embedded into ICMP Unreachable 42 | message, though in IPv6 case we could have more. This is sufficient anyways to recover the TTL and the timestamp of the 43 | original probe. 44 | 45 | Upon reception of an ICMP message, we build IcmpResponse struct and forward it to the input work queue of the Resolver 46 | goroutine ensemble. This is needed to resolve the IP address of the node that sent us the response into its DNS name. 47 | 48 | ### TCP Receiver 49 | 50 | Similar to IcmpReceiver in logic, but this goroutine intercepts TCP RST/ACK packets sent by the ultimate destinations of 51 | our probes. These responses are processed and have TTL extracted, and then forwarded to the Resolver thread. We can 52 | work both with close ports (RST expect) and open ports (ACK expected). Be careful and make sure there are no open 53 | connections from your machine to the target on the port you are probing - this may confuse the hell out of TcpReceiver. 54 | 55 | ### Resolver 56 | 57 | This goroutine listens to the incoming Icmp/Tcp Response messages and resolves the names embedded into the Icmp responses. 58 | We start lots of those so we can handle concurrent name resolution. The resolver is effectively a transformation function 59 | on the stream of messages. 60 | 61 | ### Main goroutine 62 | 63 | This one is responsible for starting all other goroutines, and then assembling their output. It is also responsible for 64 | terminating the unnecessary Senders. This is done by seeing what TTL hops actually return TCP RST messages; once we receive 65 | TCP RST for TTL x, we can safely stop all senders for TTL > x 66 | 67 | The main loop expects to receive all "Probes" from the channels fed by the Sender goroutines. The Sender will close its 68 | output channels once its done sending. This serves as an indicator that all sending has completed. After that, we 69 | wait a few more seconds all tell the TcpReceiver and IcmpReceiver to stop by closing their "signal" channel. 70 | 71 | After that, we process all data that the Receivers have fed to the main thread. We need to find the source ports 72 | whos' paths show consistent packet loss after a given hop N. We then output these paths as the "suspects" along with the 73 | counts of sent/received packets per hop. 74 | 75 | ## License 76 | fbtracert is BSD licensed, as found in the LICENSE file. 77 | -------------------------------------------------------------------------------- /tcp.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2016-present, Facebook, Inc. and its affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the BSD-style license found in the 6 | * LICENSE file in the root directory of this source tree. 7 | */ 8 | 9 | package main 10 | 11 | import ( 12 | "bytes" 13 | "encoding/binary" 14 | "net" 15 | ) 16 | 17 | // 18 | // TCP flags 19 | // 20 | const ( 21 | FIN = 1 << 0 22 | SYN = 1 << 1 23 | RST = 1 << 2 24 | PSH = 1 << 3 25 | ACK = 1 << 4 26 | URG = 1 << 5 27 | ) 28 | 29 | // TCPHeader defines the TCP header struct 30 | type TCPHeader struct { 31 | Source uint16 32 | Destination uint16 33 | SeqNum uint32 34 | AckNum uint32 35 | DataOffset uint8 // 4 bits 36 | Reserved uint8 // 6 bits 37 | Flags uint8 // 6 bits 38 | Window uint16 39 | Checksum uint16 40 | Urgent uint16 41 | } 42 | 43 | // 44 | // create & serialize a TCP header, compute and fill in the checksum (v4/v6) 45 | // 46 | func makeTCPHeader(af string, srcAddr, dstAddr net.IP, srcPort, dstPort int, ts uint32) []byte { 47 | tcpHeader := TCPHeader{ 48 | Source: uint16(srcPort), // Random ephemeral port 49 | Destination: uint16(dstPort), 50 | SeqNum: ts, 51 | AckNum: 0, 52 | DataOffset: 5, // 4 bits 53 | Reserved: 0, // 6 bits 54 | Flags: SYN, // 6 bits (000010, SYN bit set) 55 | Window: 0xffff, // max window 56 | Checksum: 0, 57 | Urgent: 0, 58 | } 59 | 60 | // temporary bytes for checksum 61 | bytes := tcpHeader.Serialize() 62 | tcpHeader.Checksum = tcpChecksum(af, bytes, srcAddr, dstAddr) 63 | 64 | return tcpHeader.Serialize() 65 | } 66 | 67 | // Parse packet into TCPHeader structure 68 | func parseTCPHeader(data []byte) *TCPHeader { 69 | var tcp TCPHeader 70 | 71 | r := bytes.NewReader(data) 72 | 73 | binary.Read(r, binary.BigEndian, &tcp.Source) 74 | binary.Read(r, binary.BigEndian, &tcp.Destination) 75 | binary.Read(r, binary.BigEndian, &tcp.SeqNum) 76 | binary.Read(r, binary.BigEndian, &tcp.AckNum) 77 | 78 | // read the flags from a 16-bit field 79 | var field uint16 80 | 81 | binary.Read(r, binary.BigEndian, &field) 82 | // most significant 4 bits 83 | tcp.DataOffset = byte(field >> 12) 84 | // reserved part - 6 bits 85 | tcp.Reserved = byte(field >> 6 & 0x3f) 86 | // flags - 6 bits 87 | tcp.Flags = byte(field & 0x3f) 88 | 89 | binary.Read(r, binary.BigEndian, &tcp.Window) 90 | binary.Read(r, binary.BigEndian, &tcp.Checksum) 91 | binary.Read(r, binary.BigEndian, &tcp.Urgent) 92 | 93 | return &tcp 94 | } 95 | 96 | // Serialize emits raw bytes for the header 97 | func (tcp *TCPHeader) Serialize() []byte { 98 | 99 | buf := new(bytes.Buffer) 100 | binary.Write(buf, binary.BigEndian, tcp.Source) 101 | binary.Write(buf, binary.BigEndian, tcp.Destination) 102 | binary.Write(buf, binary.BigEndian, tcp.SeqNum) 103 | binary.Write(buf, binary.BigEndian, tcp.AckNum) 104 | 105 | var mix uint16 106 | mix = uint16(tcp.DataOffset)<<12 | 107 | uint16(tcp.Reserved&0x3f)<<9 | 108 | uint16(tcp.Flags&0x3f) 109 | binary.Write(buf, binary.BigEndian, mix) 110 | 111 | binary.Write(buf, binary.BigEndian, tcp.Window) 112 | binary.Write(buf, binary.BigEndian, tcp.Checksum) 113 | binary.Write(buf, binary.BigEndian, tcp.Urgent) 114 | 115 | out := buf.Bytes() 116 | 117 | return out 118 | } 119 | 120 | // 121 | // TCP Checksum, works for both v4 and v6 IP addresses 122 | // 123 | func tcpChecksum(af string, data []byte, srcip, dstip net.IP) uint16 { 124 | 125 | // the pseudo header used for TCP c-sum computation 126 | var pseudoHeader []byte 127 | 128 | pseudoHeader = append(pseudoHeader, srcip...) 129 | pseudoHeader = append(pseudoHeader, dstip...) 130 | switch { 131 | case af == "ip4": 132 | pseudoHeader = append(pseudoHeader, []byte{ 133 | 0, 134 | 6, // protocol number for TCP 135 | 0, byte(len(data)), // TCP length (16 bits), w/o pseudoheader 136 | }...) 137 | case af == "ip6": 138 | pseudoHeader = append(pseudoHeader, []byte{ 139 | 0, 0, 0, byte(len(data)), // TCP length (32 bits), w/0 pseudoheader 140 | 0, 0, 0, 141 | 6, // protocol number for TCP 142 | }...) 143 | } 144 | 145 | body := make([]byte, 0, len(pseudoHeader)+len(data)) 146 | body = append(body, pseudoHeader...) 147 | body = append(body, data...) 148 | 149 | bodyLen := len(body) 150 | 151 | var word uint16 152 | var csum uint32 153 | 154 | for i := 0; i+1 < bodyLen; i += 2 { 155 | word = uint16(body[i])<<8 | uint16(body[i+1]) 156 | csum += uint32(word) 157 | } 158 | 159 | if bodyLen%2 != 0 { 160 | csum += uint32(body[len(body)-1]) 161 | } 162 | 163 | csum = (csum >> 16) + (csum & 0xffff) 164 | csum = csum + (csum >> 16) 165 | 166 | return uint16(^csum) 167 | } 168 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2016-present, Facebook, Inc. and its affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the BSD-style license found in the 6 | * LICENSE file in the root directory of this source tree. 7 | */ 8 | 9 | package main 10 | 11 | import ( 12 | "encoding/json" 13 | "flag" 14 | "fmt" 15 | "math/rand" 16 | "net" 17 | "os" 18 | "runtime" 19 | "time" 20 | 21 | "golang.org/x/net/icmp" 22 | "golang.org/x/net/ipv4" 23 | "golang.org/x/net/ipv6" 24 | 25 | "github.com/golang/glog" 26 | "github.com/olekukonko/tablewriter" 27 | ) 28 | 29 | const ( 30 | icmpHdrSize int = 8 31 | minTCPHdrSize int = 20 32 | maxTCPHdrSize int = 60 33 | minIP4HeaderSize int = 20 34 | maxIP4HeaderSize int = 60 35 | ip6HeaderSize int = 40 36 | ) 37 | 38 | // 39 | // Command line flags 40 | // 41 | var maxTTL = flag.Int("maxTTL", 30, "The maximum ttl to use") 42 | var minTTL = flag.Int("minTTL", 1, "The ttl to start at") 43 | var maxSrcPorts = flag.Int("maxSrcPorts", 256, "The maximum number of source ports to use") 44 | var maxTime = flag.Int("maxTime", 60, "The time to run the process for") 45 | var targetPort = flag.Int("targetPort", 22, "The target port to trace to") 46 | var probeRate = flag.Int("probeRate", 96, "The probe rate per ttl layer") 47 | var tosValue = flag.Int("tosValue", 140, "The TOS/TC to use in probes") 48 | var numResolvers = flag.Int("numResolvers", 32, "The number of DNS resolver goroutines") 49 | var addrFamily = flag.String("addrFamily", "ip4", "The address family (ip4/ip6) to use for testing") 50 | var maxColumns = flag.Int("maxColumns", 4, "Maximum number of columns in report tables") 51 | var showAll = flag.Bool("showAll", false, "Show all paths, regardless of loss detection") 52 | var srcAddr = flag.String("srcAddr", "", "The source address for pings, default to auto-discover") 53 | var jsonOutput = flag.Bool("jsonOutput", false, "Output raw JSON data") 54 | var baseSrcPort = flag.Int("baseSrcPort", 32768, "The base source port to start probing from") 55 | 56 | // 57 | // Discover the source address for pinging 58 | // 59 | func getSourceAddr(af string, srcAddr string) (net.IP, error) { 60 | 61 | if srcAddr != "" { 62 | addr, err := net.ResolveIPAddr(*addrFamily, srcAddr) 63 | if err != nil { 64 | return nil, err 65 | } 66 | return addr.IP, nil 67 | } 68 | 69 | addrs, err := net.InterfaceAddrs() 70 | if err != nil { 71 | return nil, err 72 | } 73 | for _, a := range addrs { 74 | if ipnet, ok := a.(*net.IPNet); ok && !ipnet.IP.IsLoopback() && !ipnet.IP.IsLinkLocalUnicast() { 75 | if (ipnet.IP.To4() != nil && af == "ip4") || (ipnet.IP.To4() == nil && af == "ip6") { 76 | return ipnet.IP, nil 77 | } 78 | } 79 | } 80 | return nil, fmt.Errorf("Could not find a source address in af %s", af) 81 | } 82 | 83 | // Resolve given hostname/address in the given address family 84 | func resolveName(dest string, af string) (net.IP, error) { 85 | addr, err := net.ResolveIPAddr(af, dest) 86 | if err != nil { 87 | return nil, err 88 | } 89 | return addr.IP, nil 90 | } 91 | 92 | // Probe is emitted by sender 93 | type Probe struct { 94 | srcPort int 95 | ttl int 96 | } 97 | 98 | // ICMPResponse is emitted by ICMPReceiver 99 | type ICMPResponse struct { 100 | Probe 101 | fromAddr net.IP 102 | fromName string 103 | rtt uint32 104 | } 105 | 106 | // TCPResponse is emitted by TCPReceiver 107 | type TCPResponse struct { 108 | Probe 109 | rtt uint32 110 | } 111 | 112 | // TCPReceiver Feeds on TCP RST messages we receive from the end host; we use lots of parameters to check if the incoming packet 113 | // is actually a response to our probe. We create TCPResponse structs and emit them on the output channel 114 | func TCPReceiver(done <-chan struct{}, af string, srcAddr net.IP, targetAddr string, probePortStart, probePortEnd, targetPort, maxTTL int) (chan interface{}, error) { 115 | glog.V(2).Infoln("TCPReceiver starting...") 116 | 117 | conn, err := net.ListenPacket(af+":tcp", srcAddr.String()) 118 | if err != nil { 119 | return nil, err 120 | } 121 | 122 | // we'll be writing the TCPResponse structs to this channel 123 | out := make(chan interface{}) 124 | 125 | // IP + TCP header, this channel is fed from the socket 126 | recv := make(chan TCPResponse) 127 | go func() { 128 | ipHdrSize := 0 // no IPv6 header present on TCP packets received on the raw socket 129 | if af == "ip4" { 130 | // IPv4 header is always included with the ipv4 raw socket receive 131 | ipHdrSize = minIP4HeaderSize 132 | } 133 | packet := make([]byte, ipHdrSize+maxTCPHdrSize) 134 | 135 | for { 136 | n, from, err := conn.ReadFrom(packet) 137 | if err != nil { 138 | break // parent has closed the socket likely 139 | } 140 | 141 | // IP + TCP header size 142 | if n < ipHdrSize+minTCPHdrSize { 143 | continue 144 | } 145 | 146 | // is that from the target port we expect? 147 | tcpHdr := parseTCPHeader(packet[ipHdrSize:n]) 148 | if int(tcpHdr.Source) != targetPort { 149 | continue 150 | } 151 | 152 | // is that TCP RST TCP ACK? 153 | if tcpHdr.Flags&RST != RST && tcpHdr.Flags&ACK != ACK { 154 | continue 155 | } 156 | 157 | // is that from our target? 158 | if from.String() != targetAddr { 159 | continue 160 | } 161 | 162 | glog.V(4).Infof("Received TCP response message %d: %x\n", n, packet[:n]) 163 | 164 | // we extract the original TTL and timestamp from the ack number 165 | ackNum := tcpHdr.AckNum - 1 166 | ttl := int(ackNum >> 24) 167 | 168 | if ttl > maxTTL || ttl < 1 { 169 | continue 170 | } 171 | 172 | // recover the time-stamp from the ack # 173 | ts := ackNum & 0x00ffffff 174 | now := uint32(time.Now().UnixNano()/(1000*1000)) & 0x00ffffff 175 | 176 | // received timestamp is higher than local time; it is possible 177 | // that ts == now, since our clock resolution is coarse 178 | if ts > now { 179 | continue 180 | } 181 | 182 | recv <- TCPResponse{Probe: Probe{srcPort: int(tcpHdr.Destination), ttl: ttl}, rtt: now - ts} 183 | } 184 | }() 185 | 186 | go func() { 187 | defer conn.Close() 188 | defer close(out) 189 | for { 190 | select { 191 | case response := <-recv: 192 | out <- response 193 | case <-done: 194 | glog.V(2).Infoln("TCPReceiver terminating...") 195 | return 196 | } 197 | } 198 | }() 199 | 200 | return out, nil 201 | } 202 | 203 | // ICMPReceiver runs on its own collecting ICMP responses until its explicitly told to stop 204 | func ICMPReceiver(done <-chan struct{}, af string, srcAddr net.IP) (chan interface{}, error) { 205 | var ( 206 | minInnerIPHdrSize int 207 | icmpMsgType byte 208 | listenNet string 209 | ) 210 | 211 | switch af { 212 | case "ip4": 213 | minInnerIPHdrSize = minIP4HeaderSize // the size of the original IPv4 header that was on the TCP packet sent out 214 | icmpMsgType = 11 // time to live exceeded 215 | listenNet = "ip4:1" // IPv4 ICMP proto number 216 | case "ip6": 217 | minInnerIPHdrSize = ip6HeaderSize // the size of the original IPv4 header that was on the TCP packet sent out 218 | icmpMsgType = 3 // time to live exceeded 219 | listenNet = "ip6:58" // IPv6 ICMP proto number 220 | default: 221 | return nil, fmt.Errorf("sender: unsupported network %q", af) 222 | } 223 | 224 | conn, err := icmp.ListenPacket(listenNet, srcAddr.String()) 225 | if err != nil { 226 | return nil, err 227 | } 228 | 229 | glog.V(2).Infoln("ICMPReceiver is starting...") 230 | 231 | recv := make(chan interface{}) 232 | 233 | go func() { 234 | // TODO: remove hardcode; 20 bytes for IP header, 8 bytes for ICMP header, 8 bytes for TCP header 235 | packet := make([]byte, icmpHdrSize+maxIP4HeaderSize+maxTCPHdrSize) 236 | for { 237 | n, from, err := conn.ReadFrom(packet) 238 | if err != nil { 239 | break 240 | } 241 | // extract the 8 bytes of the original TCP header 242 | if n < icmpHdrSize+minInnerIPHdrSize+minTCPHdrSize { 243 | continue 244 | } 245 | // not ttl exceeded 246 | if packet[0] != icmpMsgType || packet[1] != 0 { 247 | continue 248 | } 249 | glog.V(4).Infof("Received ICMP response message %d: %x\n", n, packet[:n]) 250 | tcpHdr := parseTCPHeader(packet[icmpHdrSize+minInnerIPHdrSize : n]) 251 | 252 | // extract ttl bits from the ISN 253 | ttl := int(tcpHdr.SeqNum) >> 24 254 | 255 | // extract the timestamp from the ISN 256 | ts := tcpHdr.SeqNum & 0x00ffffff 257 | // scale the current time 258 | now := uint32(time.Now().UnixNano()/(1000*1000)) & 0x00ffffff 259 | recv <- ICMPResponse{Probe: Probe{srcPort: int(tcpHdr.Source), ttl: ttl}, fromAddr: net.ParseIP(from.String()), rtt: now - ts} 260 | } 261 | }() 262 | 263 | out := make(chan interface{}) 264 | go func() { 265 | defer conn.Close() 266 | defer close(out) 267 | for { 268 | select { 269 | // read ICMP struct 270 | case response := <-recv: 271 | out <- response 272 | case <-done: 273 | glog.V(2).Infoln("ICMPReceiver done") 274 | return 275 | } 276 | } 277 | }() 278 | 279 | return out, nil 280 | } 281 | 282 | // Resolver resolves names in incoming ICMPResponse messages 283 | // Everything else is passed through as is 284 | func Resolver(input chan interface{}) (chan interface{}, error) { 285 | out := make(chan interface{}) 286 | go func() { 287 | defer close(out) 288 | 289 | for val := range input { 290 | switch val.(type) { 291 | case ICMPResponse: 292 | resp := val.(ICMPResponse) 293 | names, err := net.LookupAddr(resp.fromAddr.String()) 294 | if err != nil { 295 | resp.fromName = resp.fromAddr.String() 296 | } else { 297 | resp.fromName = names[0] 298 | } 299 | out <- resp 300 | default: 301 | out <- val 302 | } 303 | } 304 | }() 305 | return out, nil 306 | } 307 | 308 | // Sender generates TCP SYN packet probes with given TTL at given packet per second rate 309 | // The packet descriptions are published to the output channel as Probe messages 310 | // As a side effect, the packets are injected into raw socket 311 | func Sender(done <-chan struct{}, srcAddr net.IP, af, dest string, dstPort, baseSrcPort, maxSrcPorts, maxIters, ttl, pps, tos int) (chan interface{}, error) { 312 | var err error 313 | 314 | out := make(chan interface{}) 315 | 316 | glog.V(2).Infof("Sender for ttl %d starting\n", ttl) 317 | 318 | dstAddr, err := resolveName(dest, af) 319 | if err != nil { 320 | return nil, err 321 | } 322 | 323 | conn, err := net.ListenPacket(af+":tcp", srcAddr.String()) 324 | if err != nil { 325 | return nil, err 326 | } 327 | 328 | switch af { 329 | case "ip4": 330 | conn := ipv4.NewPacketConn(conn) 331 | if err := conn.SetTTL(ttl); err != nil { 332 | return nil, err 333 | } 334 | if err := conn.SetTOS(tos); err != nil { 335 | return nil, err 336 | } 337 | case "ip6": 338 | conn := ipv6.NewPacketConn(conn) 339 | if err := conn.SetHopLimit(ttl); err != nil { 340 | return nil, err 341 | } 342 | if runtime.GOOS == "windows" { 343 | glog.Infoln("Setting IPv6 traffic class not supported on Windows") 344 | break 345 | } 346 | if err := conn.SetTrafficClass(tos); err != nil { 347 | return nil, err 348 | } 349 | default: 350 | return nil, fmt.Errorf("sender: unsupported network %q", af) 351 | } 352 | 353 | // spawn a new goroutine and return the channel to be used for reading 354 | go func() { 355 | defer conn.Close() 356 | defer close(out) 357 | 358 | delay := time.Duration(1000/pps) * time.Millisecond 359 | 360 | for i := 0; i < maxSrcPorts*maxIters; i++ { 361 | srcPort := baseSrcPort + i%maxSrcPorts 362 | now := uint32(time.Now().UnixNano()/(1000*1000)) & 0x00ffffff 363 | seqNum := ((uint32(ttl) & 0xff) << 24) | (now & 0x00ffffff) 364 | packet := makeTCPHeader(af, srcAddr, dstAddr, srcPort, dstPort, seqNum) 365 | 366 | if _, err := conn.WriteTo(packet, &net.IPAddr{IP: dstAddr}); err != nil { 367 | glog.Errorf("Error sending packet %s\n", err) 368 | break 369 | } 370 | 371 | probe := Probe{srcPort: srcPort, ttl: ttl} 372 | start := time.Now() // grab time before blocking on send channel 373 | select { 374 | case out <- probe: 375 | end := time.Now() 376 | jitter := time.Duration(((rand.Float64()-0.5)/20)*1000/float64(pps)) * time.Millisecond 377 | if end.Sub(start) < delay+jitter { 378 | time.Sleep(delay + jitter - (end.Sub(start))) 379 | } 380 | case <-done: 381 | glog.V(2).Infof("Sender for ttl %d exiting prematurely\n", ttl) 382 | return 383 | } 384 | } 385 | glog.V(2).Infoln("Sender done") 386 | }() 387 | 388 | return out, nil 389 | } 390 | 391 | // 392 | // Normalize rcvd by send count to get the hit rate 393 | // 394 | func normalizeRcvd(sent, rcvd []int) ([]float64, error) { 395 | if len(rcvd) != len(sent) { 396 | return nil, fmt.Errorf("Length mismatch for sent/rcvd") 397 | } 398 | 399 | result := make([]float64, len(rcvd)) 400 | for i := range sent { 401 | result[i] = float64(rcvd[i]) / float64(sent[i]) 402 | } 403 | 404 | return result, nil 405 | } 406 | 407 | // 408 | // Detect a pattern where all samples after 409 | // a sample [i] have lower hit rate than [i] 410 | // this normally indicates a breaking point after [i] 411 | // 412 | func isLossy(hitRates []float64) bool { 413 | var found bool 414 | var segLen int 415 | for i := 0; i < len(hitRates)-1 && !found; i++ { 416 | found = true 417 | segLen = len(hitRates) - i 418 | for j := i + 1; j < len(hitRates); j++ { 419 | if hitRates[j] >= hitRates[i] { 420 | found = false 421 | break 422 | } 423 | } 424 | } 425 | // do not alarm on single-hop segment 426 | if segLen > 2 { 427 | return found 428 | } 429 | return false 430 | } 431 | 432 | // 433 | // print the paths reported as having losses 434 | // 435 | func printLossyPaths(sent, rcvd map[int] /* src port */ []int, hops map[int] /* src port */ []string, maxColumns, maxTTL int) { 436 | var allPorts []int 437 | 438 | for srcPort := range hops { 439 | allPorts = append(allPorts, srcPort) 440 | } 441 | 442 | // split in multiple tables to fit the columns on the screen 443 | for i := 0; i < len(allPorts)/maxColumns; i++ { 444 | data := make([][]string, maxTTL) 445 | table := tablewriter.NewWriter(os.Stdout) 446 | header := []string{"TTL"} 447 | 448 | maxOffset := (i + 1) * maxColumns 449 | if maxOffset > len(allPorts) { 450 | maxOffset = len(allPorts) 451 | } 452 | 453 | for _, srcPort := range allPorts[i*maxColumns : maxOffset] { 454 | header = append(header, fmt.Sprintf("port: %d", srcPort), fmt.Sprintf("sent/rcvd")) 455 | } 456 | 457 | table.SetHeader(header) 458 | 459 | for ttl := 0; ttl < maxTTL-1; ttl++ { 460 | data[ttl] = make([]string, 2*(maxOffset-i*maxColumns)+1) 461 | data[ttl][0] = fmt.Sprintf("%d", ttl+1) 462 | for j, srcPort := range allPorts[i*maxColumns : maxOffset] { 463 | data[ttl][2*j+1] = hops[srcPort][ttl] 464 | data[ttl][2*j+2] = fmt.Sprintf("%02d/%02d", sent[srcPort][ttl], rcvd[srcPort][ttl]) 465 | } 466 | } 467 | 468 | for _, v := range data { 469 | table.Append(v) 470 | } 471 | 472 | table.Render() 473 | fmt.Fprintf(os.Stdout, "\n") 474 | } 475 | } 476 | 477 | // Report defines a JSON report from go/fbtracert 478 | type Report struct { 479 | // The path map 480 | Paths map[string] /* srcPort */ []string /* path hops */ 481 | // Probe count sent per source port/hop name 482 | Sent map[string][]int 483 | // Probe count received per source port/hop name 484 | Rcvd map[string][]int 485 | } 486 | 487 | func newReport() (report Report) { 488 | report.Paths = make(map[string][]string) 489 | report.Sent = make(map[string][]int) 490 | report.Rcvd = make(map[string][]int) 491 | 492 | return report 493 | } 494 | 495 | // 496 | // Raw Json output for external program to analyze 497 | // 498 | func printLossyPathsJSON(sent, rcvd map[int] /* src port */ []int, hops map[int] /* src port */ []string, maxTTL int) { 499 | var report = newReport() 500 | 501 | for srcPort, path := range hops { 502 | report.Paths[fmt.Sprintf("%d", srcPort)] = path 503 | report.Sent[fmt.Sprintf("%d", srcPort)] = sent[srcPort] 504 | report.Rcvd[fmt.Sprintf("%d", srcPort)] = rcvd[srcPort] 505 | } 506 | 507 | b, err := json.MarshalIndent(report, "", "\t") 508 | if err != nil { 509 | glog.Errorf("Could not generate JSON %s", err) 510 | return 511 | } 512 | fmt.Fprintf(os.Stdout, "%s\n", b) 513 | } 514 | 515 | func main() { 516 | flag.Parse() 517 | if flag.Arg(0) == "" { 518 | fmt.Fprintf(os.Stderr, "Must specify a target\n") 519 | return 520 | } 521 | target := flag.Arg(0) 522 | 523 | var probes []chan interface{} 524 | 525 | numIters := int(*maxTime * *probeRate / *maxSrcPorts) 526 | 527 | if numIters <= 1 { 528 | fmt.Fprintf(os.Stderr, "Number of iterations too low, increase probe rate / run time or decrease src port range...\n") 529 | return 530 | } 531 | 532 | source, err := getSourceAddr(*addrFamily, *srcAddr) 533 | 534 | if err != nil { 535 | fmt.Fprintf(os.Stderr, "Could not identify a source address to trace from\n") 536 | return 537 | } 538 | fmt.Fprintf(os.Stderr, "Using as source address: %s\n", source.String()) 539 | 540 | fmt.Fprintf(os.Stderr, "Starting fbtracert with %d probes per second/ttl, base src port %d and with the port span of %d\n", *probeRate, *baseSrcPort, *maxSrcPorts) 541 | if flag.Lookup("logtostderr").Value.String() != "true" { 542 | fmt.Fprintf(os.Stderr, "Use '-logtostderr=true' cmd line option to see GLOG output\n") 543 | } 544 | 545 | // this will catch senders quitting - we have one sender per ttl 546 | senderDone := make([]chan struct{}, *maxTTL) 547 | for ttl := *minTTL; ttl <= *maxTTL; ttl++ { 548 | senderDone[ttl-1] = make(chan struct{}) 549 | c, err := Sender(senderDone[ttl-1], source, *addrFamily, target, *targetPort, *baseSrcPort, *maxSrcPorts, numIters, ttl, *probeRate, *tosValue) 550 | if err != nil { 551 | glog.Errorf("Failed to start sender for ttl %d, %s", ttl, err) 552 | if err.Error() == "operation not permitted" { 553 | glog.Error(" -- are you running with the correct privileges?") 554 | } 555 | return 556 | } 557 | probes = append(probes, c) 558 | } 559 | 560 | // channel to tell receivers to stop 561 | recvDone := make(chan struct{}) 562 | 563 | // collect ICMP unreachable messages for our probes 564 | icmpResp, err := ICMPReceiver(recvDone, *addrFamily, source) 565 | if err != nil { 566 | return 567 | } 568 | 569 | // collect TCP RST's from the target 570 | targetAddr, err := resolveName(target, *addrFamily) 571 | tcpResp, err := TCPReceiver(recvDone, *addrFamily, source, targetAddr.String(), *baseSrcPort, *baseSrcPort+*maxSrcPorts, *targetPort, *maxTTL) 572 | if err != nil { 573 | return 574 | } 575 | 576 | // add DNS name resolvers to the mix 577 | var resolved []chan interface{} 578 | unresolved := merge(tcpResp, icmpResp) 579 | 580 | for i := 0; i < *numResolvers; i++ { 581 | c, err := Resolver(unresolved) 582 | if err != nil { 583 | return 584 | } 585 | resolved = append(resolved, c) 586 | } 587 | 588 | // maps that store various counters per source port/ttl 589 | // e..g sent, for every soruce port, contains vector 590 | // of sent packets for each TTL 591 | sent := make(map[int] /*src Port */ []int /* pkts sent */) 592 | rcvd := make(map[int] /*src Port */ []int /* pkts rcvd */) 593 | hops := make(map[int] /*src Port */ []string /* hop name */) 594 | 595 | for srcPort := *baseSrcPort; srcPort < *baseSrcPort+*maxSrcPorts; srcPort++ { 596 | sent[srcPort] = make([]int, *maxTTL) 597 | rcvd[srcPort] = make([]int, *maxTTL) 598 | hops[srcPort] = make([]string, *maxTTL) 599 | //hops[srcPort][*maxTTL-1] = target 600 | 601 | for i := 0; i < *maxTTL; i++ { 602 | hops[srcPort][i] = "?" 603 | } 604 | } 605 | 606 | // collect all probe specs emitted by senders 607 | // once all senders terminate, tell receivers to quit too 608 | go func() { 609 | for val := range merge(probes...) { 610 | probe := val.(Probe) 611 | sent[probe.srcPort][probe.ttl-1]++ 612 | } 613 | glog.V(2).Infoln("All senders finished!") 614 | // give receivers time to catch up on in-flight data 615 | time.Sleep(2 * time.Second) 616 | // tell receivers to stop receiving 617 | close(recvDone) 618 | }() 619 | 620 | // this store DNS names of all nodes that ever replied to us 621 | var names []string 622 | 623 | // src ports that changed their paths in process of tracing 624 | var flappedPorts = make(map[int]bool) 625 | 626 | lastClosed := *maxTTL 627 | for val := range merge(resolved...) { 628 | switch val.(type) { 629 | case ICMPResponse: 630 | resp := val.(ICMPResponse) 631 | rcvd[resp.srcPort][resp.ttl-1]++ 632 | currName := hops[resp.srcPort][resp.ttl-1] 633 | if currName != "?" && currName != resp.fromName { 634 | glog.V(2).Infof("%d: Source port %d flapped at ttl %d from: %s to %s\n", time.Now().UnixNano()/(1000*1000), resp.srcPort, resp.ttl, currName, resp.fromName) 635 | flappedPorts[resp.srcPort] = true 636 | } 637 | hops[resp.srcPort][resp.ttl-1] = resp.fromName 638 | // accumulate all names for processing later 639 | // XXX: we may have duplicates, which is OK, 640 | // but not very efficient 641 | names = append(names, resp.fromName) 642 | case TCPResponse: 643 | resp := val.(TCPResponse) 644 | // stop all senders sending above this ttl, since they are not needed 645 | // XXX: this is not always optimal, i.e. we may receive TCP RST for 646 | // a port mapped to a short WAN path, and it would tell us to terminate 647 | // probing at higher TTL, thus cutting visibility on "long" paths 648 | // however, this mostly concerned that last few hops... 649 | for i := resp.ttl; i < lastClosed; i++ { 650 | close(senderDone[i]) 651 | } 652 | // update the last closed ttl, so we don't double-close the channels 653 | if resp.ttl < lastClosed { 654 | lastClosed = resp.ttl 655 | } 656 | rcvd[resp.srcPort][resp.ttl-1]++ 657 | hops[resp.srcPort][resp.ttl-1] = target 658 | } 659 | } 660 | 661 | for srcPort, hopVector := range hops { 662 | for i := range hopVector { 663 | // truncate lists once we hit the target name 664 | if hopVector[i] == target && i < *maxTTL-1 { 665 | sent[srcPort] = sent[srcPort][:i+1] 666 | rcvd[srcPort] = rcvd[srcPort][:i+1] 667 | hopVector = hopVector[:i+1] 668 | break 669 | } 670 | } 671 | } 672 | 673 | if len(flappedPorts) > 0 { 674 | glog.Infof("A total of %d ports out of %d changed their paths while tracing\n", len(flappedPorts), *maxSrcPorts) 675 | } 676 | 677 | lossyPathSent := make(map[int] /*src port */ []int) 678 | lossyPathRcvd := make(map[int] /* src port */ []int) 679 | lossyPathHops := make(map[int] /*src port*/ []string) 680 | 681 | // process the accumulated data, find and output lossy paths 682 | for port, sentVector := range sent { 683 | if flappedPorts[port] { 684 | continue 685 | } 686 | if rcvdVector, ok := rcvd[port]; ok { 687 | norm, err := normalizeRcvd(sentVector, rcvdVector) 688 | 689 | if err != nil { 690 | glog.Errorf("Could not normalize %v / %v", rcvdVector, sentVector) 691 | continue 692 | } 693 | 694 | if isLossy(norm) || *showAll { 695 | hosts := make([]string, len(norm)) 696 | for i := range norm { 697 | hosts[i] = hops[port][i] 698 | } 699 | lossyPathSent[port] = sentVector 700 | lossyPathRcvd[port] = rcvdVector 701 | lossyPathHops[port] = hosts 702 | } 703 | } else { 704 | glog.Errorf("No responses received for port %d", port) 705 | } 706 | } 707 | 708 | if len(lossyPathHops) > 0 { 709 | if *jsonOutput { 710 | printLossyPathsJSON(lossyPathSent, lossyPathRcvd, lossyPathHops, lastClosed+1) 711 | } else { 712 | printLossyPaths(lossyPathSent, lossyPathRcvd, lossyPathHops, *maxColumns, lastClosed+1) 713 | } 714 | return 715 | } 716 | glog.Infof("Did not find any faulty paths\n") 717 | } 718 | --------------------------------------------------------------------------------