├── .gitignore ├── .travis.yml ├── LICENSE.md ├── README.md ├── embed_html.py ├── ng.go ├── ngnet ├── dump.pcapng ├── httpstream.go ├── httpstreamfactory.go ├── httpstreampair.go ├── ngnet_test.go └── streamreader.go ├── ngserver.go ├── screenshot.png ├── test.sh └── web ├── index.html ├── lib ├── angular-websocket.js ├── angular.min.js ├── base64.js └── jquery-1.9.1.min.js ├── main.css ├── main.js ├── web.go └── web_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | netgraph 2 | test.pcap 3 | .vscode/ 4 | .idea/ 5 | debug 6 | .DS_Store -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go: 3 | - 1.10.x 4 | 5 | before_install: 6 | - sudo apt-get install -y libpcap-dev 7 | 8 | script: 9 | - ./test.sh 10 | 11 | after_success: 12 | - bash <(curl -s https://codecov.io/bash) -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 ga0 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |  2 | [](https://goreportcard.com/report/github.com/ga0/netgraph) 3 | [](https://codecov.io/gh/ga0/netgraph) 4 |  5 | 6 | # Netgraph 7 | 8 | Netgraph is a packet sniffer tool that captures all HTTP requests/responses, and display them in web page. 9 | 10 | 11 |  12 | 13 | You can run Netgraph in your linux server without desktop environment installed, and monitor http requests/responses in your laptop's browser. 14 | 15 | ## Compile, Install, Run 16 | 17 | 1. go get github.com/ga0/netgraph 18 | 2. run $GOPATH/bin/netgraph -i INTERFACE -p PORT 19 | 3. open the netgraph web page in your browser (for example: http://localhost:9000, 9000 is the PORT set in step 2) 20 | 21 | Windows user need to install winpcap library first. 22 | 23 | ## Options 24 | 25 | -bpf string 26 | Set berkeley packet filter (default "tcp port 80") 27 | -i string 28 | Listen on interface, auto select one if no interface is provided 29 | -input-pcap string 30 | Open a pcap file 31 | -o string 32 | Write HTTP requests/responses to file, set value "stdout" to print to console 33 | -output-pcap string 34 | Write captured packet to a pcap file 35 | -output-request-only 36 | Write only HTTP request to file, drop response. Only used when option "-o" is present. (default true) 37 | -p int 38 | Web server port. If the port is set to '0', the server will not run. (default 9000) 39 | -s Save HTTP event in server 40 | -v Show verbose message (default true) 41 | 42 | 43 | Example: print captured requests to stdout: 44 | 45 | $ ./netgraph -i en0 -o=stdout 46 | 2018/07/26 10:33:24 open live on device "en0", bpf "tcp port 80" 47 | [2018-07-26 10:33:34.873] #0 Request 192.168.1.50:60448->93.184.216.34:80 48 | GET / HTTP/1.1 49 | Host: www.example.com 50 | Connection: keep-alive 51 | Pragma: no-cache 52 | Cache-Control: no-cache 53 | Upgrade-Insecure-Requests: 1 54 | User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36 55 | Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 56 | Accept-Encoding: gzip, deflate 57 | Accept-Language: zh-CN,zh;q=0.9,en;q=0.8,zh-TW;q=0.7 58 | 59 | content(0) 60 | 61 | ## License 62 | 63 | [MIT](https://opensource.org/licenses/MIT) 64 | 65 | 66 | -------------------------------------------------------------------------------- /embed_html.py: -------------------------------------------------------------------------------- 1 | 2 | """ 3 | This python script will combine all the frontend files(.js, .html, .css) in ./web 4 | to one go source file (web.go) 5 | """ 6 | import os 7 | valid_files = [".js", ".html", ".css"] 8 | content = "" 9 | index = {} 10 | def generate_go(nouse, dir, files): 11 | def valid(f): 12 | for v in valid_files: 13 | if f.endswith(v): 14 | return True 15 | return False 16 | 17 | def getOneFile(f): 18 | global content 19 | ff = open(dir+os.sep+f) 20 | i0 = len(content) 21 | fcontent = ff.read() 22 | i1 = len(fcontent) + i0 23 | print(f) 24 | content += fcontent 25 | index[dir[len("web"):]+"/"+f] = (i0, i1) 26 | 27 | for f in filter(valid, files): 28 | getOneFile(f) 29 | 30 | if __name__ == "__main__": 31 | outf = open("web/web.go", "w") 32 | outf.write("//auto generated - don't edit it\n") 33 | outf.write("package web\n") 34 | outf.write('import "errors"\n') 35 | os.path.walk("web", generate_go, 0) 36 | outf.write("var content = []byte(`" + content.replace('`', '`+"`"+`') + "`)\n") 37 | outf.write("""type contentIndexStruct struct { 38 | begin int 39 | end int 40 | } 41 | """) 42 | outf.write("var contentIndex = map[string]contentIndexStruct{") 43 | for k in index: 44 | v = index[k] 45 | outf.write('"' + k + '":{' + str(v[0]) + ',' + str(v[1]) + '},\n') 46 | outf.write("}\n") 47 | 48 | outf.write("""func GetContent(uri string) ([]byte, error) { 49 | if val, ok := contentIndex[uri]; ok { 50 | return content[val.begin:val.end], nil 51 | } 52 | return []byte{}, errors.New("not found") 53 | }""") 54 | 55 | 56 | 57 | 58 | 59 | -------------------------------------------------------------------------------- /ng.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "io/ioutil" 7 | "log" 8 | "os" 9 | "time" 10 | 11 | "github.com/ga0/netgraph/ngnet" 12 | "github.com/google/gopacket" 13 | "github.com/google/gopacket/layers" 14 | "github.com/google/gopacket/pcap" 15 | "github.com/google/gopacket/pcapgo" 16 | "github.com/google/gopacket/tcpassembly" 17 | ) 18 | 19 | var device = flag.String("i", "", "Device to capture, auto select one if no device provided") 20 | var bpf = flag.String("bpf", "tcp port 80", "Set berkeley packet filter") 21 | 22 | var outputHTTP = flag.String("o", "", "Write HTTP request/response to file") 23 | var inputPcap = flag.String("input-pcap", "", "Open pcap file") 24 | var outputPcap = flag.String("output-pcap", "", "Write captured packet to a pcap file") 25 | var requestOnly = flag.Bool("output-request-only", true, "Write HTTP request only, drop response") 26 | 27 | var bindingPort = flag.Int("p", 9000, "Web server port. If the port is set to '0', the server will not run.") 28 | var saveEvent = flag.Bool("s", false, "Save HTTP event in server") 29 | 30 | var verbose = flag.Bool("v", true, "Show more message") 31 | 32 | // NGHTTPEventHandler handle HTTP events 33 | type NGHTTPEventHandler interface { 34 | PushEvent(interface{}) 35 | Wait() 36 | } 37 | 38 | var handlers []NGHTTPEventHandler 39 | 40 | func init() { 41 | flag.Parse() 42 | if *inputPcap != "" && *outputPcap != "" { 43 | log.Fatalln("ERROR: set -input-pcap and -output-pcap at the same time") 44 | } 45 | if *inputPcap != "" && *device != "" { 46 | log.Fatalln("ERROR: set -input-pcap and -i at the same time") 47 | } 48 | if !*verbose { 49 | log.SetOutput(ioutil.Discard) 50 | } 51 | if *inputPcap != "" { 52 | *saveEvent = true 53 | } 54 | } 55 | 56 | func initEventHandlers() { 57 | if *bindingPort != 0 { 58 | addr := fmt.Sprintf(":%d", *bindingPort) 59 | ngserver := NewNGServer(addr, *saveEvent) 60 | ngserver.Serve() 61 | handlers = append(handlers, ngserver) 62 | } 63 | 64 | if *outputHTTP != "" { 65 | p := NewEventPrinter(*outputHTTP) 66 | handlers = append(handlers, p) 67 | } 68 | } 69 | 70 | func autoSelectDev() string { 71 | ifs, err := pcap.FindAllDevs() 72 | if err != nil { 73 | log.Fatalln(err) 74 | } 75 | var available []string 76 | for _, i := range ifs { 77 | addrFound := false 78 | var addrs []string 79 | for _, addr := range i.Addresses { 80 | if addr.IP.IsLoopback() || 81 | addr.IP.IsMulticast() || 82 | addr.IP.IsUnspecified() || 83 | addr.IP.IsLinkLocalUnicast() { 84 | continue 85 | } 86 | addrFound = true 87 | addrs = append(addrs, addr.IP.String()) 88 | } 89 | if addrFound { 90 | available = append(available, i.Name) 91 | } 92 | } 93 | if len(available) > 0 { 94 | return available[0] 95 | } 96 | return "" 97 | } 98 | 99 | func packetSource() *gopacket.PacketSource { 100 | if *inputPcap != "" { 101 | handle, err := pcap.OpenOffline(*inputPcap) 102 | if err != nil { 103 | log.Fatalln(err) 104 | } 105 | log.Printf("open pcap file \"%s\"\n", *inputPcap) 106 | return gopacket.NewPacketSource(handle, handle.LinkType()) 107 | } 108 | 109 | if *device == "" { 110 | *device = autoSelectDev() 111 | if *device == "" { 112 | log.Fatalln("no device to capture") 113 | } 114 | } 115 | 116 | handle, err := pcap.OpenLive(*device, 1024*1024, true, pcap.BlockForever) 117 | if err != nil { 118 | log.Fatalln(err) 119 | } 120 | if *bpf != "" { 121 | if err = handle.SetBPFFilter(*bpf); err != nil { 122 | log.Fatalln("Failed to set BPF filter:", err) 123 | } 124 | } 125 | log.Printf("open live on device \"%s\", bpf \"%s\"\n", *device, *bpf) 126 | return gopacket.NewPacketSource(handle, handle.LinkType()) 127 | } 128 | 129 | func runNGNet(packetSource *gopacket.PacketSource, eventChan chan<- interface{}) { 130 | streamFactory := ngnet.NewHTTPStreamFactory(eventChan) 131 | pool := tcpassembly.NewStreamPool(streamFactory) 132 | assembler := tcpassembly.NewAssembler(pool) 133 | 134 | var pcapWriter *pcapgo.Writer 135 | if *outputPcap != "" { 136 | outPcapFile, err := os.Create(*outputPcap) 137 | if err != nil { 138 | log.Fatalln(err) 139 | } 140 | defer outPcapFile.Close() 141 | pcapWriter = pcapgo.NewWriter(outPcapFile) 142 | pcapWriter.WriteFileHeader(65536, layers.LinkTypeEthernet) 143 | } 144 | 145 | var count uint 146 | ticker := time.Tick(time.Minute) 147 | var lastPacketTimestamp time.Time 148 | 149 | LOOP: 150 | for { 151 | select { 152 | case packet := <-packetSource.Packets(): 153 | if packet == nil { 154 | break LOOP 155 | } 156 | 157 | count++ 158 | netLayer := packet.NetworkLayer() 159 | if netLayer == nil { 160 | continue 161 | } 162 | transLayer := packet.TransportLayer() 163 | if transLayer == nil { 164 | continue 165 | } 166 | tcp, _ := transLayer.(*layers.TCP) 167 | if tcp == nil { 168 | continue 169 | } 170 | 171 | if pcapWriter != nil { 172 | pcapWriter.WritePacket(packet.Metadata().CaptureInfo, packet.Data()) 173 | } 174 | 175 | assembler.AssembleWithTimestamp( 176 | netLayer.NetworkFlow(), 177 | tcp, 178 | packet.Metadata().CaptureInfo.Timestamp) 179 | 180 | lastPacketTimestamp = packet.Metadata().CaptureInfo.Timestamp 181 | case <-ticker: 182 | assembler.FlushOlderThan(lastPacketTimestamp.Add(time.Minute * -2)) 183 | } 184 | } 185 | 186 | assembler.FlushAll() 187 | log.Println("Read pcap file complete") 188 | streamFactory.Wait() 189 | log.Println("Parse complete, packet count: ", count) 190 | 191 | close(eventChan) 192 | } 193 | 194 | // EventPrinter print HTTP events to file or stdout 195 | type EventPrinter struct { 196 | file *os.File 197 | } 198 | 199 | // NewEventPrinter creates EventPrinter 200 | func NewEventPrinter(name string) *EventPrinter { 201 | p := new(EventPrinter) 202 | var err error 203 | if name == "stdout" { 204 | p.file = os.Stdout 205 | } else { 206 | p.file, err = os.OpenFile(name, os.O_CREATE|os.O_WRONLY, 0755) 207 | if err != nil { 208 | log.Fatalln("Cannot open file ", name) 209 | } 210 | } 211 | 212 | return p 213 | } 214 | 215 | func (p *EventPrinter) printHTTPRequestEvent(req ngnet.HTTPRequestEvent) { 216 | fmt.Fprintf(p.file, "[%s] #%d Request %s->%s\r\n", 217 | req.Start.Format("2006-01-02 15:04:05.000"), req.StreamSeq, req.ClientAddr, req.ServerAddr) 218 | fmt.Fprintf(p.file, "%s %s %s\r\n", req.Method, req.URI, req.Version) 219 | for _, h := range req.Headers { 220 | fmt.Fprintf(p.file, "%s: %s\r\n", h.Name, h.Value) 221 | } 222 | 223 | fmt.Fprintf(p.file, "\r\ncontent(%d)", len(req.Body)) 224 | if len(req.Body) > 0 { 225 | fmt.Fprintf(p.file, "%s", req.Body) 226 | } 227 | fmt.Fprintf(p.file, "\r\n\r\n") 228 | } 229 | 230 | func (p *EventPrinter) printHTTPResponseEvent(resp ngnet.HTTPResponseEvent) { 231 | fmt.Fprintf(p.file, "[%s] #%d Response %s<-%s\r\n", 232 | resp.Start.Format("2006-01-02 15:04:05.000"), resp.StreamSeq, resp.ClientAddr, resp.ServerAddr) 233 | fmt.Fprintf(p.file, "%s %d %s\r\n", resp.Version, resp.Code, resp.Reason) 234 | for _, h := range resp.Headers { 235 | fmt.Fprintf(p.file, "%s: %s\r\n", h.Name, h.Value) 236 | } 237 | 238 | fmt.Fprintf(p.file, "\r\ncontent(%d)", len(resp.Body)) 239 | if len(resp.Body) > 0 { 240 | fmt.Fprintf(p.file, "%s", resp.Body) 241 | } 242 | fmt.Fprintf(p.file, "\r\n\r\n") 243 | } 244 | 245 | // PushEvent implements the function of interface NGHTTPEventHandler 246 | func (p *EventPrinter) PushEvent(e interface{}) { 247 | switch v := e.(type) { 248 | case ngnet.HTTPRequestEvent: 249 | p.printHTTPRequestEvent(v) 250 | case ngnet.HTTPResponseEvent: 251 | if !*requestOnly { 252 | p.printHTTPResponseEvent(v) 253 | } 254 | default: 255 | log.Printf("Unknown event: %v", e) 256 | } 257 | } 258 | 259 | // Wait implements the function of interface NGHTTPEventHandler 260 | func (p *EventPrinter) Wait() {} 261 | 262 | func runEventHandler(eventChan <-chan interface{}) { 263 | for e := range eventChan { 264 | if e == nil { 265 | break 266 | } 267 | for _, h := range handlers { 268 | h.PushEvent(e) 269 | } 270 | } 271 | 272 | for _, h := range handlers { 273 | h.Wait() 274 | } 275 | } 276 | 277 | /* 278 | create client.go 279 | */ 280 | //go:generate python embed_html.py 281 | 282 | func main() { 283 | initEventHandlers() 284 | eventChan := make(chan interface{}, 1024) 285 | go runNGNet(packetSource(), eventChan) 286 | runEventHandler(eventChan) 287 | } 288 | -------------------------------------------------------------------------------- /ngnet/dump.pcapng: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ga0/netgraph/69b72fb7588df950f28bcbb94fc696eb8b3d68e4/ngnet/dump.pcapng -------------------------------------------------------------------------------- /ngnet/httpstream.go: -------------------------------------------------------------------------------- 1 | package ngnet 2 | 3 | import ( 4 | "bytes" 5 | "compress/gzip" 6 | "fmt" 7 | "io/ioutil" 8 | "regexp" 9 | "strconv" 10 | "strings" 11 | "time" 12 | 13 | "github.com/google/gopacket" 14 | "github.com/google/gopacket/tcpassembly" 15 | ) 16 | 17 | var ( 18 | httpRequestFirtLine *regexp.Regexp 19 | httpResponseFirtLine *regexp.Regexp 20 | ) 21 | 22 | func init() { 23 | httpRequestFirtLine = regexp.MustCompile(`([A-Z]+) (.+) (HTTP/.+)\r\n`) 24 | httpResponseFirtLine = regexp.MustCompile(`(HTTP/.+) (\d{3}) (.+)\r\n`) 25 | } 26 | 27 | type streamKey struct { 28 | net, tcp gopacket.Flow 29 | } 30 | 31 | func (k streamKey) String() string { 32 | return fmt.Sprintf("{%v:%v} -> {%v:%v}", k.net.Src(), k.tcp.Src(), k.net.Dst(), k.tcp.Dst()) 33 | } 34 | 35 | type httpStream struct { 36 | reader *StreamReader 37 | bytes *uint64 38 | key streamKey 39 | bad *bool 40 | } 41 | 42 | func newHTTPStream(key streamKey) httpStream { 43 | var s httpStream 44 | s.reader = NewStreamReader() 45 | s.bytes = new(uint64) 46 | s.key = key 47 | s.bad = new(bool) 48 | return s 49 | } 50 | 51 | // Reassembled is called by tcpassembly 52 | func (s httpStream) Reassembled(rs []tcpassembly.Reassembly) { 53 | if *s.bad { 54 | return 55 | } 56 | 57 | for _, r := range rs { 58 | if r.Skip != 0 { 59 | *s.bad = true 60 | return 61 | } 62 | 63 | if len(r.Bytes) == 0 { 64 | continue 65 | } 66 | 67 | *s.bytes += uint64(len(r.Bytes)) 68 | ticker := time.Tick(time.Second) 69 | 70 | select { 71 | case <-s.reader.stopCh: 72 | *s.bad = true 73 | return 74 | case s.reader.src <- NewStreamDataBlock(r.Bytes, r.Seen): 75 | case <-ticker: 76 | // Sometimes pcap only captured HTTP response with no request! 77 | // Let's wait few seconds to avoid dead lock. 78 | *s.bad = true 79 | return 80 | } 81 | } 82 | } 83 | 84 | // ReassemblyComplete is called by tcpassembly 85 | func (s httpStream) ReassemblyComplete() { 86 | close(s.reader.src) 87 | } 88 | 89 | func (s *httpStream) getRequestLine() (method string, uri string, version string) { 90 | bytes, err := s.reader.ReadUntil([]byte("\r\n")) 91 | if err != nil { 92 | panic("Cannot read request line, err=" + err.Error()) 93 | } 94 | line := string(bytes) 95 | r := httpRequestFirtLine.FindStringSubmatch(line) 96 | if len(r) != 4 { 97 | panic("Bad HTTP Request: " + line) 98 | } 99 | 100 | method = r[1] 101 | uri = r[2] 102 | version = r[3] 103 | return 104 | } 105 | 106 | func (s *httpStream) getResponseLine() (version string, code uint, reason string) { 107 | bytes, err := s.reader.ReadUntil([]byte("\r\n")) 108 | if err != nil { 109 | panic("Cannot read response line, err=" + err.Error()) 110 | } 111 | line := string(bytes) 112 | r := httpResponseFirtLine.FindStringSubmatch(line) 113 | if len(r) != 4 { 114 | panic("Bad HTTP Response: " + line) 115 | } 116 | 117 | version = r[1] 118 | var code64 uint64 119 | code64, err = strconv.ParseUint(r[2], 10, 32) 120 | if err != nil { 121 | panic("Bad HTTP Response: " + line + ", err=" + err.Error()) 122 | } 123 | code = uint(code64) 124 | reason = r[3] 125 | return 126 | } 127 | 128 | func (s *httpStream) getHeaders() (headers []HTTPHeaderItem) { 129 | d, err := s.reader.ReadUntil([]byte("\r\n\r\n")) 130 | if err != nil { 131 | panic("Cannot read headers, err=" + err.Error()) 132 | } 133 | data := string(d[:len(d)-4]) 134 | for i, line := range strings.Split(data, "\r\n") { 135 | p := strings.Index(line, ":") 136 | if p == -1 { 137 | panic(fmt.Sprintf("Bad http header (line %d): %s", i, data)) 138 | } 139 | var h HTTPHeaderItem 140 | h.Name = line[:p] 141 | h.Value = strings.Trim(line[p+1:], " ") 142 | headers = append(headers, h) 143 | } 144 | return 145 | } 146 | 147 | func (s *httpStream) getChunked() []byte { 148 | var body []byte 149 | for { 150 | buf, err := s.reader.ReadUntil([]byte("\r\n")) 151 | if err != nil { 152 | panic("Cannot read chuncked content, err=" + err.Error()) 153 | } 154 | l := string(buf) 155 | l = strings.Trim(l[:len(l)-2], " ") 156 | blockSize, err := strconv.ParseInt(l, 16, 32) 157 | if err != nil { 158 | panic("bad chunked block length: " + l + ", err=" + err.Error()) 159 | } 160 | 161 | buf, err = s.reader.Next(int(blockSize)) 162 | body = append(body, buf...) 163 | if err != nil { 164 | panic("Cannot read chuncked content, err=" + err.Error()) 165 | } 166 | buf, err = s.reader.Next(2) 167 | if err != nil { 168 | panic("Cannot read chuncked content, err=" + err.Error()) 169 | } 170 | CRLF := string(buf) 171 | if CRLF != "\r\n" { 172 | panic("Bad chunked block data") 173 | } 174 | 175 | if blockSize == 0 { 176 | break 177 | } 178 | } 179 | return body 180 | } 181 | 182 | func (s *httpStream) getFixedLengthContent(contentLength int) []byte { 183 | body, err := s.reader.Next(contentLength) 184 | if err != nil { 185 | panic("Cannot read content, err=" + err.Error()) 186 | } 187 | return body 188 | } 189 | 190 | func getContentInfo(hs []HTTPHeaderItem) (contentLength int, contentEncoding string, contentType string, chunked bool) { 191 | for _, h := range hs { 192 | lowerName := strings.ToLower(h.Name) 193 | if lowerName == "content-length" { 194 | var err error 195 | contentLength, err = strconv.Atoi(h.Value) 196 | if err != nil { 197 | panic("Content-Length error: " + h.Value + ", err=" + err.Error()) 198 | } 199 | } else if lowerName == "transfer-encoding" && h.Value == "chunked" { 200 | chunked = true 201 | } else if lowerName == "content-encoding" { 202 | contentEncoding = h.Value 203 | } else if lowerName == "content-type" { 204 | contentType = h.Value 205 | } 206 | } 207 | return 208 | } 209 | 210 | func (s *httpStream) getBody(method string, headers []HTTPHeaderItem, isRequest bool) (body []byte) { 211 | contentLength, contentEncoding, _, chunked := getContentInfo(headers) 212 | if (contentLength == 0 && !chunked) || (!isRequest && method == "HEAD") { 213 | return 214 | } 215 | 216 | if chunked { 217 | body = s.getChunked() 218 | } else { 219 | body = s.getFixedLengthContent(contentLength) 220 | } 221 | 222 | var uncompressedBody []byte 223 | var err error 224 | // TODO: more compress type should be supported 225 | if contentEncoding == "gzip" { 226 | buffer := bytes.NewBuffer(body) 227 | zipReader, _ := gzip.NewReader(buffer) 228 | uncompressedBody, err = ioutil.ReadAll(zipReader) 229 | defer zipReader.Close() 230 | if err != nil { 231 | body = []byte("(gzip data uncompress error)") 232 | } else { 233 | body = uncompressedBody 234 | } 235 | } 236 | return 237 | } 238 | -------------------------------------------------------------------------------- /ngnet/httpstreamfactory.go: -------------------------------------------------------------------------------- 1 | package ngnet 2 | 3 | import ( 4 | "sync" 5 | "sync/atomic" 6 | 7 | "github.com/google/gopacket" 8 | "github.com/google/gopacket/tcpassembly" 9 | ) 10 | 11 | // HTTPStreamFactory implements StreamFactory interface for tcpassembly 12 | type HTTPStreamFactory struct { 13 | runningStream *int32 14 | wg *sync.WaitGroup 15 | seq *uint 16 | uniStreams *map[streamKey]*httpStreamPair 17 | eventChan chan<- interface{} 18 | } 19 | 20 | // NewHTTPStreamFactory create a NewHTTPStreamFactory 21 | func NewHTTPStreamFactory(out chan<- interface{}) HTTPStreamFactory { 22 | var f HTTPStreamFactory 23 | f.seq = new(uint) 24 | *f.seq = 0 25 | f.wg = new(sync.WaitGroup) 26 | f.uniStreams = new(map[streamKey]*httpStreamPair) 27 | *f.uniStreams = make(map[streamKey]*httpStreamPair) 28 | f.eventChan = out 29 | f.runningStream = new(int32) 30 | return f 31 | } 32 | 33 | // Wait for all stream exit 34 | func (f HTTPStreamFactory) Wait() { 35 | f.wg.Wait() 36 | } 37 | 38 | // RunningStreamCount get the running stream count 39 | func (f *HTTPStreamFactory) RunningStreamCount() int32 { 40 | return atomic.LoadInt32(f.runningStream) 41 | } 42 | 43 | func (f *HTTPStreamFactory) runStreamPair(streamPair *httpStreamPair) { 44 | atomic.AddInt32(f.runningStream, 1) 45 | 46 | defer f.wg.Done() 47 | defer func() { atomic.AddInt32(f.runningStream, -1) }() 48 | streamPair.run() 49 | } 50 | 51 | // New creates a HTTPStreamFactory 52 | func (f HTTPStreamFactory) New(netFlow, tcpFlow gopacket.Flow) (ret tcpassembly.Stream) { 53 | revkey := streamKey{netFlow.Reverse(), tcpFlow.Reverse()} 54 | streamPair, ok := (*f.uniStreams)[revkey] 55 | if ok { 56 | if streamPair.upStream == nil { 57 | panic("unbelievable!?") 58 | } 59 | delete(*f.uniStreams, revkey) 60 | key := streamKey{netFlow, tcpFlow} 61 | s := newHTTPStream(key) 62 | streamPair.downStream = &s 63 | ret = s 64 | } else { 65 | streamPair = newHTTPStreamPair(*f.seq, f.eventChan) 66 | key := streamKey{netFlow, tcpFlow} 67 | s := newHTTPStream(key) 68 | streamPair.upStream = &s 69 | (*f.uniStreams)[key] = streamPair 70 | *f.seq++ 71 | f.wg.Add(1) 72 | go f.runStreamPair(streamPair) 73 | ret = s 74 | } 75 | return 76 | } 77 | -------------------------------------------------------------------------------- /ngnet/httpstreampair.go: -------------------------------------------------------------------------------- 1 | package ngnet 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | // HTTPHeaderItem is HTTP header key-value pair 8 | type HTTPHeaderItem struct { 9 | Name string 10 | Value string 11 | } 12 | 13 | // HTTPEvent is HTTP request or response 14 | type HTTPEvent struct { 15 | Type string 16 | Start time.Time 17 | End time.Time 18 | StreamSeq uint 19 | } 20 | 21 | // HTTPRequestEvent is HTTP request 22 | type HTTPRequestEvent struct { 23 | HTTPEvent 24 | ClientAddr string 25 | ServerAddr string 26 | Method string 27 | URI string 28 | Version string 29 | Headers []HTTPHeaderItem 30 | Body []byte 31 | } 32 | 33 | // HTTPResponseEvent is HTTP response 34 | type HTTPResponseEvent struct { 35 | HTTPEvent 36 | ClientAddr string 37 | ServerAddr string 38 | Version string 39 | Code uint 40 | Reason string 41 | Headers []HTTPHeaderItem 42 | Body []byte 43 | } 44 | 45 | // httpStreamPair is Bi-direction HTTP stream pair 46 | type httpStreamPair struct { 47 | upStream *httpStream 48 | downStream *httpStream 49 | 50 | requestSeq uint 51 | connSeq uint 52 | eventChan chan<- interface{} 53 | } 54 | 55 | func newHTTPStreamPair(seq uint, eventChan chan<- interface{}) *httpStreamPair { 56 | pair := new(httpStreamPair) 57 | pair.connSeq = seq 58 | pair.eventChan = eventChan 59 | 60 | return pair 61 | } 62 | 63 | func (pair *httpStreamPair) run() { 64 | defer func() { 65 | if r := recover(); r != nil { 66 | if pair.upStream != nil { 67 | close(pair.upStream.reader.stopCh) 68 | } 69 | if pair.downStream != nil { 70 | close(pair.downStream.reader.stopCh) 71 | } 72 | //fmt.Printf("HTTPStream (#%d %v) error: %v\n", pair.connSeq, pair.upStream.key, r) 73 | } 74 | }() 75 | 76 | for { 77 | pair.handleTransaction() 78 | pair.requestSeq++ 79 | } 80 | } 81 | 82 | func (pair *httpStreamPair) handleTransaction() { 83 | upStream := pair.upStream 84 | method, uri, version := upStream.getRequestLine() 85 | reqStart := upStream.reader.lastSeen 86 | reqHeaders := upStream.getHeaders() 87 | reqBody := upStream.getBody(method, reqHeaders, true) 88 | 89 | var req HTTPRequestEvent 90 | req.ClientAddr = pair.upStream.key.net.Src().String() + ":" + pair.upStream.key.tcp.Src().String() 91 | req.ServerAddr = pair.upStream.key.net.Dst().String() + ":" + pair.upStream.key.tcp.Dst().String() 92 | req.Type = "HTTPRequest" 93 | req.Method = method 94 | req.URI = uri 95 | req.Version = version 96 | req.Headers = reqHeaders 97 | req.Body = reqBody 98 | req.StreamSeq = pair.connSeq 99 | req.Start = reqStart 100 | req.End = upStream.reader.lastSeen 101 | pair.eventChan <- req 102 | 103 | downStream := pair.downStream 104 | respVersion, code, reason := downStream.getResponseLine() 105 | respStart := downStream.reader.lastSeen 106 | respHeaders := downStream.getHeaders() 107 | respBody := downStream.getBody(method, respHeaders, false) 108 | 109 | var resp HTTPResponseEvent 110 | resp.ClientAddr = pair.upStream.key.net.Src().String() + ":" + pair.upStream.key.tcp.Src().String() 111 | resp.ServerAddr = pair.upStream.key.net.Dst().String() + ":" + pair.upStream.key.tcp.Dst().String() 112 | resp.Type = "HTTPResponse" 113 | resp.Version = respVersion 114 | resp.Code = uint(code) 115 | resp.Reason = reason 116 | resp.Headers = respHeaders 117 | resp.Body = respBody 118 | resp.StreamSeq = pair.connSeq 119 | resp.Start = respStart 120 | resp.End = downStream.reader.lastSeen 121 | pair.eventChan <- resp 122 | } 123 | -------------------------------------------------------------------------------- /ngnet/ngnet_test.go: -------------------------------------------------------------------------------- 1 | package ngnet 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/google/gopacket" 8 | "github.com/google/gopacket/layers" 9 | "github.com/google/gopacket/pcap" 10 | "github.com/google/gopacket/tcpassembly" 11 | ) 12 | 13 | func TestNgnet(t *testing.T) { 14 | eventChan := make(chan interface{}, 1024) 15 | f := NewHTTPStreamFactory(eventChan) 16 | pool := tcpassembly.NewStreamPool(f) 17 | assembler := tcpassembly.NewAssembler(pool) 18 | packetCount := 0 19 | fmt.Println("Run") 20 | if handle, err := pcap.OpenOffline("dump.pcapng"); err != nil { 21 | panic(err) 22 | } else { 23 | packetSource := gopacket.NewPacketSource(handle, handle.LinkType()) 24 | for packet := range packetSource.Packets() { 25 | netLayer := packet.NetworkLayer() 26 | transLayer := packet.TransportLayer() 27 | 28 | if netLayer == nil { 29 | continue 30 | } 31 | if transLayer == nil { 32 | continue 33 | } 34 | packetCount++ 35 | tcp, _ := transLayer.(*layers.TCP) 36 | assembler.AssembleWithTimestamp(netLayer.NetworkFlow(), tcp, packet.Metadata().CaptureInfo.Timestamp) 37 | } 38 | } 39 | assembler.FlushAll() 40 | f.Wait() 41 | fmt.Println("packet:", packetCount, "http:", len(eventChan)) 42 | } 43 | -------------------------------------------------------------------------------- /ngnet/streamreader.go: -------------------------------------------------------------------------------- 1 | package ngnet 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "time" 7 | ) 8 | 9 | // StreamDataBlock is copied from tcpassembly.Reassembly 10 | type StreamDataBlock struct { 11 | Bytes []byte 12 | Seen time.Time 13 | } 14 | 15 | // NewStreamDataBlock create a new StreamDataBlock 16 | func NewStreamDataBlock(bytes []byte, seen time.Time) *StreamDataBlock { 17 | b := new(StreamDataBlock) 18 | b.Bytes = make([]byte, len(bytes)) 19 | copy(b.Bytes, bytes[:]) 20 | b.Seen = seen 21 | return b 22 | } 23 | 24 | // StreamReader read data from tcp stream 25 | type StreamReader struct { 26 | src chan *StreamDataBlock 27 | stopCh chan interface{} 28 | buffer *bytes.Buffer 29 | lastSeen time.Time 30 | } 31 | 32 | // NewStreamReader create a new StreamReader 33 | func NewStreamReader() *StreamReader { 34 | r := new(StreamReader) 35 | r.stopCh = make(chan interface{}) 36 | r.buffer = bytes.NewBuffer([]byte("")) 37 | r.src = make(chan *StreamDataBlock, 32) 38 | return r 39 | } 40 | 41 | func (s *StreamReader) fillBuffer() error { 42 | if dataBlock, ok := <-s.src; ok { 43 | s.buffer.Write(dataBlock.Bytes) 44 | s.lastSeen = dataBlock.Seen 45 | return nil 46 | } 47 | return errors.New("EOF") 48 | } 49 | 50 | // ReadUntil read bytes until delim 51 | func (s *StreamReader) ReadUntil(delim []byte) ([]byte, error) { 52 | var p int 53 | for { 54 | if p = bytes.Index(s.buffer.Bytes(), delim); p == -1 { 55 | if err := s.fillBuffer(); err != nil { 56 | return nil, err 57 | } 58 | } else { 59 | break 60 | } 61 | } 62 | return s.buffer.Next(p + len(delim)), nil 63 | } 64 | 65 | // Next read n bytes from stream 66 | func (s *StreamReader) Next(n int) ([]byte, error) { 67 | for s.buffer.Len() < n { 68 | if err := s.fillBuffer(); err != nil { 69 | return nil, err 70 | } 71 | } 72 | dst := make([]byte, n) 73 | copy(dst, s.buffer.Next(n)) 74 | return dst, nil 75 | } 76 | -------------------------------------------------------------------------------- /ngserver.go: -------------------------------------------------------------------------------- 1 | /*Package ngserver get the captured http data from ngnet, 2 | and send these data to frontend by websocket. 3 | 4 | chan +-----NGClient 5 | ngnet----------NGServer---------+-----NGClient 6 | +-----NGClient 7 | */ 8 | package main 9 | 10 | import ( 11 | "encoding/json" 12 | "log" 13 | "net/http" 14 | "os" 15 | "sync" 16 | 17 | "github.com/ga0/netgraph/web" 18 | "golang.org/x/net/websocket" 19 | ) 20 | 21 | // NGClient is the websocket client 22 | type NGClient struct { 23 | eventChan chan interface{} 24 | server *NGServer 25 | ws *websocket.Conn 26 | } 27 | 28 | func (c *NGClient) recvAndProcessCommand() { 29 | for { 30 | var msg string 31 | err := websocket.Message.Receive(c.ws, &msg) 32 | if err != nil { 33 | return 34 | } 35 | if len(msg) > 0 { 36 | if msg == "sync" { 37 | c.server.sync(c) 38 | } 39 | } else { 40 | panic("empty command") 41 | } 42 | } 43 | } 44 | 45 | func (c *NGClient) transmitEvents() { 46 | for ev := range c.eventChan { 47 | json, err := json.Marshal(ev) 48 | if err == nil { 49 | websocket.Message.Send(c.ws, string(json)) 50 | } 51 | } 52 | } 53 | 54 | func (c *NGClient) close() { 55 | close(c.eventChan) 56 | } 57 | 58 | // NewNGClient creates NGClient 59 | func NewNGClient(ws *websocket.Conn, server *NGServer) *NGClient { 60 | c := new(NGClient) 61 | c.server = server 62 | c.ws = ws 63 | c.eventChan = make(chan interface{}, 16) 64 | return c 65 | } 66 | 67 | // NGServer is a http server which push captured HTTPEvent to the front end 68 | type NGServer struct { 69 | addr string 70 | staticFileDir string 71 | connectedClient map[*websocket.Conn]*NGClient 72 | connectedClientMutex *sync.Mutex 73 | eventBuffer []interface{} 74 | saveEvent bool 75 | wg sync.WaitGroup 76 | } 77 | 78 | func (s *NGServer) websocketHandler(ws *websocket.Conn) { 79 | c := NewNGClient(ws, s) 80 | 81 | s.connectedClientMutex.Lock() 82 | s.connectedClient[ws] = c 83 | s.connectedClientMutex.Unlock() 84 | 85 | go c.transmitEvents() 86 | c.recvAndProcessCommand() 87 | c.close() 88 | 89 | s.connectedClientMutex.Lock() 90 | delete(s.connectedClient, ws) 91 | s.connectedClientMutex.Unlock() 92 | } 93 | 94 | // PushEvent dispatches the event received from ngnet to all clients connected with websocket. 95 | func (s *NGServer) PushEvent(e interface{}) { 96 | if s.saveEvent { 97 | s.eventBuffer = append(s.eventBuffer, e) 98 | } 99 | s.connectedClientMutex.Lock() 100 | for _, c := range s.connectedClient { 101 | c.eventChan <- e 102 | } 103 | s.connectedClientMutex.Unlock() 104 | } 105 | 106 | // Wait waits for serving 107 | func (s *NGServer) Wait() { 108 | s.wg.Wait() 109 | } 110 | 111 | /* 112 | If the flag '-s' is set and the browser sent a 'sync' command, 113 | the NGServer will push all the http message buffered in eventBuffer to 114 | the client. 115 | */ 116 | func (s *NGServer) sync(c *NGClient) { 117 | for _, ev := range s.eventBuffer { 118 | c.eventChan <- ev 119 | } 120 | } 121 | 122 | /* 123 | Handle static files (.html, .js, .css). 124 | */ 125 | func (s *NGServer) handleStaticFile(w http.ResponseWriter, r *http.Request) { 126 | uri := r.RequestURI 127 | if uri == "/" { 128 | uri = "/index.html" 129 | } 130 | c, err := web.GetContent(uri) 131 | if err != nil { 132 | log.Println(r.RequestURI) 133 | http.NotFound(w, r) 134 | return 135 | } 136 | w.Write([]byte(c)) 137 | } 138 | 139 | func (s *NGServer) listenAndServe() { 140 | defer s.wg.Done() 141 | err := http.ListenAndServe(s.addr, nil) 142 | if err != nil { 143 | log.Fatalln(err) 144 | } 145 | } 146 | 147 | // Serve the web page 148 | func (s *NGServer) Serve() { 149 | http.Handle("/data", websocket.Handler(s.websocketHandler)) 150 | 151 | /* 152 | If './client' directory exists, create a FileServer with it, 153 | otherwise we use package client. 154 | */ 155 | _, err := os.Stat("client") 156 | if err == nil { 157 | fs := http.FileServer(http.Dir("client")) 158 | http.Handle("/", fs) 159 | } else { 160 | http.HandleFunc("/", s.handleStaticFile) 161 | } 162 | s.wg.Add(1) 163 | go s.listenAndServe() 164 | } 165 | 166 | // NewNGServer creates NGServer 167 | func NewNGServer(addr string, saveEvent bool) *NGServer { 168 | s := new(NGServer) 169 | s.addr = addr 170 | s.connectedClient = make(map[*websocket.Conn]*NGClient) 171 | s.connectedClientMutex = &sync.Mutex{} 172 | s.saveEvent = saveEvent 173 | return s 174 | } 175 | -------------------------------------------------------------------------------- /screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ga0/netgraph/69b72fb7588df950f28bcbb94fc696eb8b3d68e4/screenshot.png -------------------------------------------------------------------------------- /test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | echo "" > coverage.txt 5 | 6 | for d in $(go list ./... | grep -v vendor); do 7 | go test -race -coverprofile=profile.out -covermode=atomic $d 8 | if [ -f profile.out ]; then 9 | cat profile.out >> coverage.txt 10 | rm profile.out 11 | fi 12 | done -------------------------------------------------------------------------------- /web/index.html: -------------------------------------------------------------------------------- 1 | 2 |
3 |Method | 38 |Host | 39 |URI | 40 |Code | 41 |Start | 42 |Duration | 43 |Stream# | 44 |
---|---|---|---|---|---|---|
{{ req.Method }} | 48 |{{ req.Host }} | 49 |{{ req.URI }} | 50 |{{ req.Response.Code }} | 51 |{{ req.Start | date : 'HH:mm:ss.sss' }} | 52 |{{ req.Duration }} ms | 53 |{{ req.StreamSeq }} | 54 |
{{ h.Name }} | 66 |{{ h.Value }} |
67 |
{{ selectedReq.Body }}
71 |{{ h.Name }} | 80 |{{ h.Value }} |
81 |
{{ selectedReq.Response.Body }}
85 |t |