├── .gitattributes ├── Makefile ├── go.mod ├── sniffer.json ├── LICENSE ├── config.go ├── README.md └── sniffer.go /.gitattributes: -------------------------------------------------------------------------------- 1 | *.* linguist-language=go 2 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | sniffer: *.go 2 | go build sniffer.go config.go 3 | 4 | clean: 5 | rm -f sniffer httpdump.db 6 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module httpdump 2 | 3 | go 1.13 4 | 5 | require ( 6 | github.com/gin-gonic/gin v1.6.3 // indirect 7 | github.com/google/gopacket v1.1.19 // indirect 8 | github.com/jinzhu/gorm v1.9.16 // indirect 9 | ) 10 | -------------------------------------------------------------------------------- /sniffer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "sniffer", 3 | "device": "en0", 4 | "port":80, 5 | 6 | 7 | "NOTESERVER":"提交数据的到服务器时，为了防止提交垃圾数据，需要配置clientDeviceid，clientDevicekey，只有正确的key服务器才会存储，否则服务器会丢弃", 8 | "serverurl":"http://127.0.0.1:1337/", 9 | "clientDeviceid":"2", 10 | "clientDevicekey":"43ffweqr3", 11 | 12 | 13 | "NOTE1":"jslength 是指response 返回数据为 javascript 格式时，用来调试 显示内容的长度", 14 | "jslength":1000, 15 | "htmllength":0, 16 | 17 | "NOTE2":"该字段用来配置response返回的任何格式时，显示的长度,目的也是调试", 18 | "dumpanystr":"text/plain", 19 | "dumpanylen":200 20 | 21 | } 22 | 23 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2018 Asmcos 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /config.go: -------------------------------------------------------------------------------- 1 | /* 2 | * the config for main 3 | * Author:asmcos 4 | * Date:2021.2 5 | */ 6 | 7 | 8 | package main 9 | 10 | import ( 11 | 12 | "encoding/json" 13 | "io/ioutil" 14 | "log" 15 | ) 16 | 17 | /***************************** 18 | config value 19 | ******************************/ 20 | var colorReset = "\033[0m" 21 | 22 | var colorRed = "\033[31m" 23 | 24 | 25 | 26 | 27 | func loadConfig(){ 28 | 29 | f, err := ioutil.ReadFile("sniffer.json") 30 | if err != nil { 31 | log.Println("**********************************************") 32 | log.Println(colorRed,"Warning: Read config sniffer.json fail,use default config value.", err,colorReset) 33 | log.Println("**********************************************") 34 | return 35 | } 36 | 37 | var configData map[string]interface{} 38 | err = json.Unmarshal([]byte(f), &configData) 39 | if err != nil { 40 | log.Println("sniffer.json err,user default config",err) 41 | return 42 | } 43 | 44 | log.Println("Load config file :sniffer.json success!\n") 45 | 46 | *iface = configData["device"].(string) 47 | *port = int(configData["port"].(float64)) 48 | 49 | *djslen = int(configData["jslength"].(float64)) 50 | *dhtmllen = int(configData["htmllength"].(float64)) 51 | 52 | *danystr = configData["dumpanystr"].(string) 53 | *danylen = int(configData["dumpanylen"].(float64)) 54 | 55 | *serverurl = configData["serverurl"].(string) 56 | clientDeviceid = configData["clientDeviceid"].(string) 57 | clientDevicekey = configData["clientDevicekey"].(string) 58 | } 59 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | #sniffer 2 | 3 | sniffer capture http packet in gopacket(libpcap). 4 | 5 | The sniffer project captures packets through pcap and parses the http protocol. 6 | The fetched results will be stored in the database. 7 | He also provides a webserver interface to view packet capture results. 8 | 9 | # install libpcap 10 | for ubuntu/debian: 11 | 12 | sudo aptitude install libcap-dev 13 | 14 | for centos/redhat/fedora: 15 | 16 | sudo yum install libpcap-devel 17 | 18 | 19 | #depend 20 | 21 | ``` 22 | go get github.com/asmcos/requests 23 | ``` 24 | 25 | # make 26 | ``` 27 | make 28 | ``` 29 | 30 | # run 31 | 32 | ``` 33 | nohup ./sniffer -i eth0 & 34 | ``` 35 | 36 | 37 | #Support Config Json file 38 | 39 | sniffer.json 40 | 41 | ``` 42 | { 43 | "name": "sniffer", 44 | "device": "en0", 45 | "port":80 46 | } 47 | ``` 48 | 49 | 50 | # Save data to webserver 51 | 52 | The sniffer can store data to a remote server or not save the data. 53 | You can submit data through serverurl. 54 | 55 | How to config serverurl? 56 | 57 | ``` 58 | vim sniffer.json 59 | "serverurl":"http://127.0.0.1:1337/", //Save data to server 60 | "serverurl":"" //Don't save data 61 | ``` 62 | 63 | How to build data server ? 64 | 65 | ``` 66 | See https://github.com/asmcos/AIDatas 67 | 68 | ``` 69 | 70 | 71 | # capture example: 72 | ``` 73 | 192.168.10.110->175.27.0.201:54893->80 74 | 75 | extshort.weixin.qq.com 76 | POST /mmtls/1d70cf00 HTTP/1.1 77 | Connection :[close] 78 | Content-Length :[542] 79 | Content-Type :[application/octet-stream] 80 | Upgrade :[mmtls] 81 | User-Agent :[MicroMessenger Client] 82 | Accept :[*/*] 83 | Cache-Control :[no-cache] 84 | 85 | 175.27.0.201->192.168.10.110:80->54893 86 | 87 | HTTP/1.1 200 OK 88 | Content-Type :[application/octet-stream] 89 | Content-Length :[229] 90 | 91 | ``` 92 | 93 | # View results through webserver 94 | ``` 95 | http://zhanluejia.net.cn:8080/httpdata/ 96 | ``` 97 | ![image](http://www.zhanluejia.net.cn/static/uploads/8f74446537e233fb2af932355cd927f0.png) 98 | ![image](http://www.zhanluejia.net.cn/static/uploads/f5ef64bf4874bd2103945975a1db5d4a.png) 99 | 100 | 101 | 102 | # Use AIDatas and sniffer for linkage 103 | https://note.youdao.com/s/BdhDLrwb 104 | 105 | -------------------------------------------------------------------------------- /sniffer.go: -------------------------------------------------------------------------------- 1 | // Copyright 2012 Google, Inc. All rights reserved. 2 | // 3 | // Use of this source code is governed by a BSD-style license 4 | // that can be found in the LICENSE file in the root of the source 5 | // tree. 6 | 7 | // The pcapdump binary implements a tcpdump-like command line tool with gopacket 8 | // using pcap as a backend data collection mechanism. 9 | // 10 | // Use tcpassembly.go in the gpacket reassembly directory instead of the tcpassembly directory. 11 | // HTTP protocol analysis, the request and response start flags need to be detected to prevent packet leakage. 12 | // Author: asmcos 13 | // Date: 2018 14 | // 15 | package main 16 | 17 | import ( 18 | "bufio" 19 | "bytes" 20 | "compress/gzip" 21 | "compress/zlib" 22 | "encoding/hex" 23 | "flag" 24 | "fmt" 25 | "io" 26 | "io/ioutil" 27 | "log" 28 | "net/http" 29 | "net/url" 30 | "net/textproto" 31 | "os" 32 | "os/signal" 33 | "runtime/pprof" 34 | "strings" 35 | "sync" 36 | "time" 37 | "encoding/json" 38 | 39 | "github.com/google/gopacket" 40 | "github.com/google/gopacket/examples/util" 41 | "github.com/google/gopacket/ip4defrag" 42 | "github.com/google/gopacket/layers" // pulls in all layers decoders 43 | "github.com/google/gopacket/pcap" 44 | "github.com/google/gopacket/reassembly" 45 | 46 | "github.com/asmcos/requests" 47 | 48 | //_ "net/http/pprof" 49 | ) 50 | 51 | var maxcount = flag.Int("c", -1, "Only grab this many packets, then exit") 52 | var statsevery = flag.Int("stats", 1000, "Output statistics every N packets") 53 | var lazy = flag.Bool("lazy", false, "If true, do lazy decoding") 54 | var nodefrag = flag.Bool("nodefrag", false, "If true, do not do IPv4 defrag") 55 | var checksum = flag.Bool("checksum", false, "Check TCP checksum") 56 | var nooptcheck = flag.Bool("nooptcheck", false, "Do not check TCP options (useful to ignore MSS on captures with TSO)") 57 | var ignorefsmerr = flag.Bool("ignorefsmerr", false, "Ignore TCP FSM errors") 58 | var allowmissinginit = flag.Bool("allowmissinginit", true, "Support streams without SYN/SYN+ACK/ACK sequence") 59 | var verbose = flag.Bool("verbose", false, "Be verbose") 60 | var debug = flag.Bool("debug", false, "Display debug information") 61 | var quiet = flag.Bool("quiet", false, "Be quiet regarding errors") 62 | 63 | // http 64 | var nohttp = flag.Bool("nohttp", false, "Disable HTTP parsing") 65 | var output = flag.String("output", "", "Path to create file for HTTP 200 OK responses") 66 | var writeincomplete = flag.Bool("writeincomplete", false, "Write incomplete response") 67 | 68 | var hexdump = flag.Bool("dump", false, "Dump HTTP request/response as hex") 69 | var hexdumppkt = flag.Bool("dumppkt", false, "Dump packet as hex") 70 | 71 | var djslen = flag.Int("dumpjs",0,"Display response javascript format length") 72 | var dhtmllen = flag.Int("dumphtml",0,"Display response html format length") 73 | 74 | var danystr = flag.String("dumpanystr","text/plain","Display response ContentType,e.g. text/html") 75 | var danylen = flag.Int("dumpanylen",0,"Display response dumpanystr format length") 76 | 77 | // capture 78 | var iface = flag.String("i", "eth0", "Interface to read packets from") 79 | var port = flag.Int("p", 80, "Interface to read packets from") 80 | var fname = flag.String("r", "", "Filename to read from, overrides -i") 81 | var snaplen = flag.Int("s", 65536, "Snap length (number of bytes max to read per packet") 82 | var tstype = flag.String("timestamp_type", "", "Type of timestamps to use") 83 | var promisc = flag.Bool("promisc", true, "Set promiscuous mode") 84 | 85 | var memprofile = flag.String("memprofile", "", "Write memory profile") 86 | var serverurl = flag.String("serverurl", "", "save data to remote server: http://www.cpython.org/httpdata/") 87 | 88 | var signalChan chan os.Signal 89 | var sysexit bool = false 90 | 91 | var clientDeviceid string = "" 92 | var clientDevicekey string = "" 93 | 94 | const ( 95 | defaultMaxMemory = 32 << 20 // 32 MB 96 | ) 97 | 98 | //var db *gorm.DB 99 | 100 | // t is type 1:request,2:response 101 | func HeaderToDB(session * requests.Request,h http.Header,t string) ([]requests.Datas){ 102 | 103 | var d []requests.Datas 104 | 105 | for n,v :=range h{ 106 | val := strings.Join(v,", ") 107 | 108 | d = append(d,requests.Datas{"type":t, 109 | "name":n, 110 | "values":val, 111 | }) 112 | } 113 | return d 114 | } 115 | 116 | func FormToDB(session *requests.Request,val url.Values,t string)([]requests.Datas){ 117 | 118 | 119 | var d []requests.Datas 120 | for n,v :=range val{ 121 | content := strings.Join(v,", ") 122 | 123 | d = append(d,requests.Datas{"type":t, 124 | "name":n, 125 | "values":content, 126 | }) 127 | } 128 | return d 129 | } 130 | 131 | 132 | var stats struct { 133 | ipdefrag int 134 | missedBytes int 135 | pkt int 136 | sz int 137 | totalsz int 138 | rejectFsm int 139 | rejectOpt int 140 | rejectConnFsm int 141 | reassembled int 142 | outOfOrderBytes int 143 | outOfOrderPackets int 144 | biggestChunkBytes int 145 | biggestChunkPackets int 146 | overlapBytes int 147 | overlapPackets int 148 | } 149 | 150 | //const closeTimeout time.Duration = time.Hour * 24 // Closing inactive: TODO: from CLI 151 | const closeTimeout time.Duration = time.Minute * 5 // Closing inactive: TODO: from CLI 152 | const timeout time.Duration = time.Minute * 3 // Pending bytes: TODO: from CLI 153 | 154 | /* 155 | * HTTP part 156 | */ 157 | 158 | type httpGroup struct { 159 | req *http.Request 160 | reqFirstLine string 161 | reqTimeStamp int64 162 | reqFlag int //0=new,1=found,2=finish 163 | 164 | resp *http.Response 165 | respFirstLine string 166 | respTimeStamp int64 167 | respFlag int //0=new,1=found,2=finish 168 | } 169 | 170 | 171 | type httpReader struct { 172 | ident string 173 | isClient bool 174 | bytes chan []byte 175 | timeStamp chan int64 176 | data []byte 177 | hexdump bool 178 | parent *tcpStream 179 | logbuf string 180 | srcip string 181 | dstip string 182 | srcport string 183 | dstport string 184 | httpstart int // 0 = new,1=find, 2 = old and find new 185 | } 186 | 187 | func (h *httpReader) Read(p []byte) (int, error) { 188 | ok := true 189 | for ok && len(h.data) == 0 { 190 | h.data, ok = <-h.bytes 191 | } 192 | if !ok || len(h.data) == 0 { 193 | return 0, io.EOF 194 | } 195 | 196 | ishttp,_ := detectHttp(h.data) 197 | 198 | if ishttp { 199 | switch h.httpstart { 200 | case 0: // run read,only copy 201 | h.httpstart = 1 202 | l := copy(p, h.data) 203 | return l,nil 204 | 205 | case 1: //http read 206 | h.httpstart = 2 207 | l := copy(p, h.data) 208 | h.data = h.data[l:] 209 | return l, nil 210 | 211 | case 2: //http read 212 | h.httpstart = 0 213 | return 0, io.EOF 214 | } 215 | } 216 | 217 | 218 | l := copy(p, h.data) 219 | h.data = h.data[l:] 220 | return l, nil 221 | } 222 | 223 | func (h *httpReader) Print() { 224 | 225 | fmt.Println(h.logbuf) 226 | 227 | } 228 | 229 | var outputLevel int 230 | var errorsMap map[string]uint 231 | var errorsMapMutex sync.Mutex 232 | var errors uint 233 | 234 | // Too bad for perf that a... is evaluated 235 | func Error(t string, s string, a ...interface{}) { 236 | errorsMapMutex.Lock() 237 | errors++ 238 | nb, _ := errorsMap[t] 239 | errorsMap[t] = nb + 1 240 | errorsMapMutex.Unlock() 241 | if outputLevel >= 0 { 242 | fmt.Printf(s, a...) 243 | } 244 | } 245 | func Info(s string, a ...interface{}) { 246 | if outputLevel >= 1 { 247 | fmt.Printf(s, a...) 248 | } 249 | } 250 | func Debug(s string, a ...interface{}) { 251 | if outputLevel >= 2 { 252 | fmt.Printf(s, a...) 253 | } 254 | } 255 | 256 | 257 | func printHeader(h http.Header)string{ 258 | var logbuf string 259 | 260 | for k,v := range h{ 261 | logbuf += fmt.Sprintf("%s :%s\n",k,v) 262 | } 263 | return logbuf 264 | } 265 | 266 | // url.Values map[string][]string 267 | 268 | func printForm(v url.Values)string{ 269 | var logbuf string 270 | 271 | logbuf += fmt.Sprint("\n**************\n") 272 | for k,data := range v{ 273 | logbuf += fmt.Sprint(k,":") 274 | for _,d := range data{ 275 | logbuf += fmt.Sprintf("%s",d) 276 | } 277 | logbuf += "\n" 278 | } 279 | logbuf += fmt.Sprint("**************\n") 280 | 281 | return logbuf 282 | } 283 | 284 | func printRequest(req *http.Request)string{ 285 | 286 | logbuf := fmt.Sprintf("\n") 287 | logbuf += fmt.Sprintf("%s\n",req.Host) 288 | logbuf += fmt.Sprintf("%s %s %s \n",req.Method, req.RequestURI, req.Proto) 289 | logbuf += printHeader(req.Header) 290 | logbuf += printForm(req.Form) 291 | logbuf += printForm(req.PostForm) 292 | if req.MultipartForm != nil { 293 | logbuf += printForm(url.Values(req.MultipartForm.Value)) 294 | } 295 | logbuf += fmt.Sprintf("\n") 296 | return logbuf 297 | } 298 | 299 | func printResponse(resp *http.Response)string{ 300 | 301 | logbuf := fmt.Sprintf("\n") 302 | logbuf += fmt.Sprintf("%s %s\n",resp.Proto, resp.Status) 303 | logbuf += printHeader(resp.Header) 304 | logbuf += fmt.Sprintf("\n") 305 | return logbuf 306 | } 307 | 308 | // detect http infomation 309 | // isRequest isResponse 310 | 311 | func isRequest(data []byte) (bool,string) { 312 | buf := bytes.NewBuffer(data) 313 | reader := bufio.NewReader(buf) 314 | tp := textproto.NewReader(reader) 315 | 316 | firstLine, _ := tp.ReadLine() 317 | arr := strings.Split(firstLine, " ") 318 | 319 | switch strings.TrimSpace(arr[0]) { 320 | case "GET", "HEAD", "POST", "PUT", "DELETE", "TRACE", "CONNECT": 321 | return true,firstLine 322 | default: 323 | return false,firstLine 324 | } 325 | } 326 | 327 | 328 | //server to client 329 | func isResponse(data []byte) (bool,string) { 330 | buf := bytes.NewBuffer(data) 331 | reader := bufio.NewReader(buf) 332 | tp := textproto.NewReader(reader) 333 | 334 | firstLine, _:= tp.ReadLine() 335 | return strings.HasPrefix(strings.TrimSpace(firstLine), "HTTP/"),firstLine 336 | } 337 | 338 | 339 | // 0 = response 340 | // 1 = request 341 | 342 | func detectHttp(data []byte) (bool ,int){ 343 | 344 | ishttp,_ := isResponse(data) 345 | if ishttp{ 346 | return true,0 347 | } 348 | 349 | ishttp,_ = isRequest(data) 350 | if ishttp{ 351 | return true,1 //request 352 | } 353 | 354 | return false,2 355 | } 356 | 357 | 358 | 359 | func (h *httpReader) DecompressBody(header http.Header, reader io.ReadCloser) (io.ReadCloser, bool) { 360 | contentEncoding := header.Get("Content-Encoding") 361 | var nr io.ReadCloser 362 | var err error 363 | if contentEncoding == "" { 364 | // do nothing 365 | return reader, false 366 | } else if strings.Contains(contentEncoding, "gzip") { 367 | nr, err = gzip.NewReader(reader) 368 | if err != nil { 369 | return reader, false 370 | } 371 | return nr, true 372 | } else if strings.Contains(contentEncoding, "deflate") { 373 | nr, err = zlib.NewReader(reader) 374 | if err != nil { 375 | return reader, false 376 | } 377 | return nr, true 378 | } else { 379 | return reader, false 380 | } 381 | } 382 | 383 | 384 | 385 | 386 | 387 | 388 | func (h * httpReader) HandleRequest (timeStamp int64,firstline string) { 389 | 390 | b := bufio.NewReader(h) 391 | 392 | req, err := http.ReadRequest(b) 393 | h.parent.UpdateReq(req,timeStamp,firstline) 394 | 395 | if err == io.EOF || err == io.ErrUnexpectedEOF { 396 | return 397 | } else if err != nil { 398 | Error("HTTP-request", "HTTP Request error: %s (%v,%+v)\n", err, err, err) 399 | 400 | } else { 401 | 402 | req.ParseMultipartForm(defaultMaxMemory) 403 | 404 | r,ok := h.DecompressBody(req.Header,req.Body) 405 | if ok { 406 | defer r.Close() 407 | } 408 | contentType := req.Header.Get("Content-Type") 409 | logbuf := fmt.Sprintf("%v->%v:%v->%v\n",h.srcip,h.dstip,h.srcport,h.dstport) 410 | logbuf += printRequest(req) 411 | 412 | if strings.Contains(contentType,"application/json"){ 413 | 414 | bodydata, err := ioutil.ReadAll(r) 415 | if err == nil { 416 | var jsonValue interface{} 417 | err = json.Unmarshal([]byte(bodydata), &jsonValue) 418 | if err == nil { 419 | logbuf += fmt.Sprintf("%#v\n",jsonValue) 420 | } 421 | } 422 | } 423 | 424 | fmt.Printf("%s",logbuf) 425 | } 426 | 427 | } 428 | 429 | func (h *httpReader) runClient(wg *sync.WaitGroup) { 430 | defer wg.Done() 431 | 432 | var p = make([]byte,1900) 433 | 434 | for { 435 | 436 | h.httpstart = 0 437 | l,err := h.Read(p) 438 | if (err == io.EOF){ 439 | return 440 | } 441 | if( l > 8 ){ 442 | isReq,firstLine := isRequest(p) 443 | if(isReq){ //start new request 444 | timeStamp := <-h.timeStamp 445 | 446 | h.HandleRequest(timeStamp,firstLine) 447 | } 448 | } 449 | } 450 | 451 | } 452 | 453 | func (h * httpReader) HandleResponse (timeStamp int64,firstline string) { 454 | 455 | 456 | b := bufio.NewReader(h) 457 | 458 | resp, err := http.ReadResponse(b, nil) 459 | h.parent.UpdateResp(resp,timeStamp,firstline) 460 | 461 | if err == io.EOF || err == io.ErrUnexpectedEOF { 462 | return 463 | } else if err != nil { 464 | Error("HTTP-reponse", "HTTP Response error: %s (%v,%+v)\n", err, err, err) 465 | 466 | } else { 467 | 468 | r,ok := h.DecompressBody(resp.Header,resp.Body) 469 | if ok { 470 | defer r.Close() 471 | } 472 | contentType := resp.Header.Get("Content-Type") 473 | logbuf := fmt.Sprintf("%v->%v:%v->%v\n",h.srcip,h.dstip,h.srcport,h.dstport) 474 | logbuf += printResponse(resp) 475 | 476 | if strings.Contains(contentType,"application/json"){ 477 | 478 | bodydata, err := ioutil.ReadAll(r) 479 | if err == nil { 480 | var jsonValue interface{} 481 | err = json.Unmarshal([]byte(bodydata), &jsonValue) 482 | if err == nil { 483 | logbuf += fmt.Sprintf("%#v\n",jsonValue) 484 | } 485 | } 486 | } else if strings.Contains(contentType,"application/javascript"){ 487 | bodydata, err := ioutil.ReadAll(r) 488 | bodylen := len(bodydata) 489 | 490 | if bodylen < *djslen{ 491 | *djslen = bodylen 492 | } 493 | if err == nil { 494 | 495 | logbuf += fmt.Sprintf("%s\n",string(bodydata[:*djslen])) 496 | } 497 | } else if strings.Contains(contentType,"text/html"){ 498 | bodydata, err := ioutil.ReadAll(r) 499 | bodylen := len(bodydata) 500 | 501 | if bodylen < *dhtmllen{ 502 | *dhtmllen = bodylen 503 | } 504 | if err == nil { 505 | 506 | logbuf += fmt.Sprintf("%s\n",string(bodydata[:*dhtmllen])) 507 | } 508 | } else if strings.Contains(contentType,*danystr){ //default text/plain 509 | bodydata, err := ioutil.ReadAll(r) 510 | bodylen := len(bodydata) 511 | 512 | if bodylen < *danylen{ 513 | *danylen = bodylen 514 | } 515 | if err == nil { 516 | 517 | logbuf += fmt.Sprintf("%s\n",string(bodydata[:*danylen])) 518 | } 519 | } 520 | 521 | 522 | 523 | fmt.Printf("%s",logbuf) 524 | } 525 | 526 | 527 | } 528 | 529 | 530 | 531 | // response 532 | func (h *httpReader) runServer(wg *sync.WaitGroup) { 533 | defer wg.Done() 534 | 535 | var p = make([]byte,1900) 536 | 537 | for { 538 | h.httpstart = 0 539 | l,err := h.Read(p) 540 | if (err == io.EOF){ 541 | return 542 | } 543 | if( l > 8 ){ 544 | isResp,firstLine := isResponse(p) 545 | if(isResp){ //start new response 546 | timeStamp := <-h.timeStamp 547 | 548 | h.HandleResponse(timeStamp,firstLine) 549 | 550 | } 551 | } 552 | } 553 | 554 | 555 | 556 | } 557 | 558 | 559 | 560 | /* 561 | * The TCP factory: returns a new Stream 562 | */ 563 | type tcpStreamFactory struct { 564 | wg sync.WaitGroup 565 | doHTTP bool 566 | } 567 | 568 | func (factory *tcpStreamFactory) New(net, transport gopacket.Flow, tcp *layers.TCP, ac reassembly.AssemblerContext) reassembly.Stream { 569 | Debug("* NEW: %s %s\n", net, transport) 570 | sip,dip := net.Endpoints() 571 | srcip := fmt.Sprintf("%s",sip) 572 | dstip := fmt.Sprintf("%s",dip) 573 | 574 | fsmOptions := reassembly.TCPSimpleFSMOptions{ 575 | SupportMissingEstablishment: *allowmissinginit, 576 | } 577 | stream := &tcpStream{ 578 | net: net, 579 | transport: transport, 580 | isHTTP: (tcp.SrcPort == layers.TCPPort(*port) || tcp.DstPort == layers.TCPPort(*port)) && factory.doHTTP, 581 | reversed: tcp.SrcPort == layers.TCPPort(*port), 582 | tcpstate: reassembly.NewTCPSimpleFSM(fsmOptions), 583 | ident: fmt.Sprintf("%s:%s", net, transport), 584 | optchecker: reassembly.NewTCPOptionCheck(), 585 | } 586 | if stream.isHTTP { 587 | stream.client = httpReader{ 588 | bytes: make(chan []byte), 589 | timeStamp: make(chan int64), 590 | ident: fmt.Sprintf("%s %s", net, transport), 591 | hexdump: *hexdump, 592 | parent: stream, 593 | isClient: true, 594 | srcport: fmt.Sprintf("%d",tcp.SrcPort), 595 | dstport: fmt.Sprintf("%d",tcp.DstPort), 596 | srcip: srcip, 597 | dstip: dstip, 598 | httpstart:0, 599 | } 600 | stream.server = httpReader{ 601 | bytes: make(chan []byte), 602 | timeStamp: make(chan int64), 603 | ident: fmt.Sprintf("%s %s", net.Reverse(), transport.Reverse()), 604 | hexdump: *hexdump, 605 | parent: stream, 606 | dstport: fmt.Sprintf("%d",tcp.SrcPort), 607 | srcport: fmt.Sprintf("%d",tcp.DstPort), 608 | dstip: srcip, 609 | srcip: dstip, 610 | httpstart:0, 611 | } 612 | factory.wg.Add(2) 613 | go stream.client.runClient(&factory.wg) 614 | go stream.server.runServer(&factory.wg) 615 | } 616 | return stream 617 | } 618 | 619 | func (factory *tcpStreamFactory) WaitGoRoutines() { 620 | factory.wg.Wait() 621 | } 622 | 623 | /* 624 | * The assembler context 625 | */ 626 | type Context struct { 627 | CaptureInfo gopacket.CaptureInfo 628 | } 629 | 630 | func (c *Context) GetCaptureInfo() gopacket.CaptureInfo { 631 | return c.CaptureInfo 632 | } 633 | 634 | /* 635 | * TCP stream 636 | */ 637 | 638 | /* It's a connection (bidirectional) */ 639 | type tcpStream struct { 640 | tcpstate *reassembly.TCPSimpleFSM 641 | fsmerr bool 642 | optchecker reassembly.TCPOptionCheck 643 | net, transport gopacket.Flow 644 | isHTTP bool 645 | reversed bool 646 | client httpReader 647 | server httpReader 648 | urls []string 649 | ident string 650 | all []httpGroup 651 | hg sync.Mutex 652 | sync.Mutex 653 | } 654 | 655 | //req = 1 is request 656 | //req = 0 is response 657 | 658 | func (t * tcpStream) NewhttpGroup(req int,timestamp int64) { 659 | 660 | t.hg.Lock() 661 | for _, hg := range t.all { 662 | //exist same req 663 | if hg.reqTimeStamp == timestamp||hg.respTimeStamp == timestamp{ 664 | fmt.Println("Have same ",req,timestamp) 665 | t.hg.Unlock() 666 | return 667 | } 668 | 669 | } 670 | 671 | if req == 1 { 672 | //try find response 673 | for i, hg := range t.all { 674 | if hg.respFlag > 0 && hg.reqFlag == 0{ 675 | t.all[i].respFlag = 1 676 | t.all[i].respTimeStamp = timestamp 677 | t.hg.Unlock() 678 | return 679 | } 680 | } 681 | 682 | hg := httpGroup{ 683 | reqFlag : 1, 684 | reqTimeStamp : timestamp, 685 | respFlag : 0, 686 | } 687 | t.all = append(t.all,hg) 688 | t.hg.Unlock() 689 | 690 | } else { 691 | //try find request 692 | for i, hg := range t.all { 693 | if hg.reqFlag > 0 && hg.respFlag == 0{ 694 | t.all[i].respFlag = 1 695 | t.all[i].respTimeStamp = timestamp 696 | t.hg.Unlock() 697 | return 698 | } 699 | } 700 | hg := httpGroup{ 701 | respFlag :1, 702 | respTimeStamp :timestamp, 703 | reqFlag :0, 704 | } 705 | t.all = append(t.all,hg) 706 | t.hg.Unlock() 707 | } 708 | 709 | } 710 | 711 | func (t * tcpStream) UpdateReq(req * http.Request,timestamp int64,firstLine string) { 712 | 713 | t.hg.Lock() 714 | for i, hg := range t.all { 715 | if hg.reqTimeStamp == timestamp { 716 | t.all[i].req = req 717 | t.all[i].reqFlag = 2 718 | t.all[i].reqFirstLine = firstLine 719 | if hg.respFlag == 2{ 720 | t.Save(&t.all[i]) 721 | if i < len(t.all){ 722 | t.all = append(t.all[:i],t.all[i+1:]...) 723 | } else { 724 | t.all = t.all[:i] 725 | } 726 | } 727 | } //if timestramp 728 | 729 | }//for 730 | 731 | t.hg.Unlock() 732 | } 733 | 734 | func (t * tcpStream) UpdateResp(resp * http.Response,timestamp int64,firstLine string) { 735 | 736 | 737 | t.hg.Lock() 738 | for i, hg := range t.all { 739 | if hg.respTimeStamp == timestamp { 740 | t.all[i].resp = resp 741 | t.all[i].respFlag = 2 742 | t.all[i].respFirstLine = firstLine 743 | if hg.reqFlag == 2{ 744 | t.Save(&t.all[i]) 745 | if i < len(t.all){ 746 | t.all = append(t.all[:i],t.all[i+1:]...) 747 | } else { 748 | t.all = t.all[:i] 749 | } 750 | } 751 | } //if timestramp 752 | 753 | }//for 754 | 755 | t.hg.Unlock() 756 | } 757 | 758 | 759 | // save to database 760 | func (t * tcpStream)Save(hg * httpGroup){ 761 | 762 | 763 | req := hg.req 764 | resp := hg.resp 765 | if (req == nil || resp == nil || *serverurl==""){ 766 | return 767 | } 768 | 769 | var h []requests.Datas 770 | var f []requests.Datas 771 | 772 | var postdata map[string]interface{} 773 | 774 | 775 | StatusCode := fmt.Sprintf("%d",resp.StatusCode) 776 | 777 | postdata = make(map[string]interface{}) 778 | reqdata := requests.Datas{ 779 | "Host":req.Host, 780 | "Method":req.Method, 781 | "RequestURI":req.RequestURI, 782 | "Proto":req.Proto, 783 | "StatusCode":StatusCode, 784 | "SrcIp":t.client.srcip, 785 | "SrcPort":t.client.srcport, 786 | "DstIp":t.client.dstip, 787 | "DstPort":t.client.dstport, 788 | } 789 | 790 | 791 | session := requests.Requests() 792 | 793 | postdata["request"] = reqdata 794 | 795 | respdata := requests.Datas{ 796 | "Proto":resp.Proto, 797 | "StatusCode": StatusCode, 798 | "SrcIp":t.server.srcip, 799 | "SrcPort":t.server.srcport, 800 | "DstIp":t.server.dstip, 801 | "DstPort":t.server.dstport, 802 | } 803 | 804 | postdata["response"] = respdata 805 | 806 | h = append(h,HeaderToDB(session,req.Header,"1")...) 807 | 808 | f = append(f,FormToDB(session,req.Form,"1")...) 809 | f = append(f,FormToDB(session,req.PostForm,"1")...) 810 | if req.MultipartForm != nil { 811 | f = append(f,FormToDB(session,url.Values(req.MultipartForm.Value),"1")...) 812 | } 813 | 814 | h = append(h,HeaderToDB(session,resp.Header,"2")...) 815 | 816 | postdata["header"] = h 817 | postdata["form"] = f 818 | 819 | postdata["clientDevice"] = map[string]string{"clientid":clientDeviceid, 820 | "clientkey":clientDevicekey} 821 | 822 | session.PostJson(*serverurl+"requests?alldata=1",postdata) 823 | 824 | } 825 | 826 | func (t *tcpStream) Accept(tcp *layers.TCP, ci gopacket.CaptureInfo, dir reassembly.TCPFlowDirection, nextSeq reassembly.Sequence, start *bool, ac reassembly.AssemblerContext) bool { 827 | // FSM 828 | 829 | var isReq int 830 | 831 | *start,isReq = detectHttp(tcp.Payload) 832 | 833 | 834 | if !t.tcpstate.CheckState(tcp, dir) { 835 | Error("FSM", "%s: Packet rejected by FSM (state:%s)\n", t.ident, t.tcpstate.String()) 836 | stats.rejectFsm++ 837 | if !t.fsmerr { 838 | t.fsmerr = true 839 | stats.rejectConnFsm++ 840 | } 841 | if !*ignorefsmerr { 842 | return false 843 | } 844 | } 845 | // Options //skip mss check 846 | err := t.optchecker.Accept(tcp, ci, dir, nextSeq, start) 847 | if err != nil { 848 | // 重复的包，丢弃 drop 849 | // 调试发现此包为以前序号的包，并且出现过。 850 | // mss BUG,server mss通过路由拆解成mss要求的包尺寸， 851 | // 因此不能判断包大小大于mss为错包 852 | if strings.Contains(fmt.Sprintf("%s",err)," > mss "){ 853 | // > mss 包 不丢弃 854 | } else { 855 | 856 | Error("OptionChecker", "%v ->%v : Packet rejected by OptionChecker: %s\n", t.net, t.transport, err) 857 | stats.rejectOpt++ 858 | if !*nooptcheck { 859 | return false 860 | } 861 | } 862 | } 863 | 864 | // Checksum 865 | accept := true 866 | if *checksum { 867 | c, err := tcp.ComputeChecksum() 868 | if err != nil { 869 | Error("ChecksumCompute", "%s: Got error computing checksum: %s\n", t.ident, err) 870 | accept = false 871 | } else if c != 0x0 { 872 | Error("Checksum", "%s: Invalid checksum: 0x%x\n", t.ident, c) 873 | accept = false 874 | } 875 | } 876 | if !accept { 877 | stats.rejectOpt++ 878 | } 879 | 880 | // create new httpgroup,wait request+response 881 | if *start { 882 | t.NewhttpGroup(isReq,ci.Timestamp.UnixNano()) 883 | } 884 | 885 | return accept 886 | } 887 | 888 | func (t *tcpStream) ReassembledSG(sg reassembly.ScatterGather, ac reassembly.AssemblerContext) { 889 | dir, start, end, skip := sg.Info() 890 | length, saved := sg.Lengths() 891 | 892 | // update stats 893 | sgStats := sg.Stats() 894 | if skip > 0 { 895 | stats.missedBytes += skip 896 | } 897 | stats.sz += length - saved 898 | stats.pkt += sgStats.Packets 899 | if sgStats.Chunks > 1 { 900 | stats.reassembled++ 901 | } 902 | stats.outOfOrderPackets += sgStats.QueuedPackets 903 | stats.outOfOrderBytes += sgStats.QueuedBytes 904 | if length > stats.biggestChunkBytes { 905 | stats.biggestChunkBytes = length 906 | } 907 | if sgStats.Packets > stats.biggestChunkPackets { 908 | stats.biggestChunkPackets = sgStats.Packets 909 | } 910 | if sgStats.OverlapBytes != 0 && sgStats.OverlapPackets == 0 { 911 | fmt.Printf("bytes:%d, pkts:%d\n", sgStats.OverlapBytes, sgStats.OverlapPackets) 912 | panic("Invalid overlap") 913 | } 914 | stats.overlapBytes += sgStats.OverlapBytes 915 | stats.overlapPackets += sgStats.OverlapPackets 916 | 917 | var ident string 918 | if dir == reassembly.TCPDirClientToServer { 919 | ident = fmt.Sprintf("%v %v(%s): ", t.net, t.transport, dir) 920 | } else { 921 | ident = fmt.Sprintf("%v %v(%s): ", t.net.Reverse(), t.transport.Reverse(), dir) 922 | } 923 | Debug("%s: SG reassembled packet with %d bytes (start:%v,end:%v,skip:%d,saved:%d,nb:%d,%d,overlap:%d,%d)\n", ident, length, start, end, skip, saved, sgStats.Packets, sgStats.Chunks, sgStats.OverlapBytes, sgStats.OverlapPackets) 924 | if skip == -1 && *allowmissinginit { 925 | // this is allowed 926 | } else if skip != 0 { 927 | // Missing bytes in stream: do not even try to parse it 928 | return 929 | } 930 | 931 | //use timeStamp as match flag 932 | timeStamp :=sg.CaptureInfo(0).Timestamp.UnixNano() 933 | data := sg.Fetch(length) 934 | if t.isHTTP { 935 | if length > 0 { 936 | if *hexdump { 937 | Debug("Feeding http with:\n%s", hex.Dump(data)) 938 | } 939 | 940 | ok,_:=detectHttp(data) 941 | 942 | //if dir == reassembly.TCPDirClientToServer && !t.reversed { 943 | if dir == reassembly.TCPDirClientToServer { 944 | t.client.bytes <- data 945 | if ok { 946 | t.client.timeStamp <- timeStamp 947 | } 948 | } else { 949 | t.server.bytes <- data 950 | if ok { 951 | t.server.timeStamp <- timeStamp 952 | } 953 | } 954 | } 955 | } 956 | } 957 | 958 | func (t *tcpStream) ReassemblyComplete(ac reassembly.AssemblerContext) bool { 959 | Debug("%s: Connection closed\n", t.ident) 960 | if t.isHTTP { 961 | close(t.client.bytes) 962 | close(t.server.bytes) 963 | } 964 | // do not remove the connection to allow last ACK 965 | return false 966 | } 967 | 968 | func HandlerSig (){ 969 | 970 | for { 971 | select { 972 | case <-signalChan: 973 | fmt.Fprintf(os.Stderr, "\nCaught SIGINT: aborting %v\n",sysexit) 974 | if sysexit == false{ 975 | sysexit = true 976 | } else { 977 | os.Exit(1) //Second ctrl+c system exit 978 | } 979 | } 980 | } 981 | } 982 | 983 | func main() { 984 | defer util.Run()() 985 | var handle *pcap.Handle 986 | var err error 987 | if *debug { 988 | outputLevel = 2 989 | } else if *verbose { 990 | outputLevel = 1 991 | } else if *quiet { 992 | outputLevel = -1 993 | } 994 | log.SetOutput(os.Stdout) 995 | errorsMap = make(map[string]uint) 996 | 997 | loadConfig() 998 | 999 | 1000 | if *fname != "" { 1001 | if handle, err = pcap.OpenOffline(*fname); err != nil { 1002 | log.Fatal("PCAP OpenOffline error:", err) 1003 | } 1004 | } else { 1005 | // This is a little complicated because we want to allow all possible options 1006 | // for creating the packet capture handle... instead of all this you can 1007 | // just call pcap.OpenLive if you want a simple handle. 1008 | inactive, err := pcap.NewInactiveHandle(*iface) 1009 | if err != nil { 1010 | log.Fatalf("could not create: %v", err) 1011 | } 1012 | defer inactive.CleanUp() 1013 | if err = inactive.SetSnapLen(*snaplen); err != nil { 1014 | log.Fatalf("could not set snap length: %v", err) 1015 | } else if err = inactive.SetPromisc(*promisc); err != nil { 1016 | log.Fatalf("could not set promisc mode: %v", err) 1017 | } else if err = inactive.SetTimeout(time.Second); err != nil { 1018 | log.Fatalf("could not set timeout: %v", err) 1019 | } 1020 | if *tstype != "" { 1021 | if t, err := pcap.TimestampSourceFromString(*tstype); err != nil { 1022 | log.Fatalf("Supported timestamp types: %v", inactive.SupportedTimestamps()) 1023 | } else if err := inactive.SetTimestampSource(t); err != nil { 1024 | log.Fatalf("Supported timestamp types: %v", inactive.SupportedTimestamps()) 1025 | } 1026 | } 1027 | if handle, err = inactive.Activate(); err != nil { 1028 | log.Fatal("PCAP Activate error:", err) 1029 | } 1030 | defer handle.Close() 1031 | } 1032 | if len(flag.Args()) > 0 { 1033 | bpffilter := strings.Join(flag.Args(), " ") 1034 | Info("Using BPF filter %q\n", bpffilter) 1035 | if err = handle.SetBPFFilter(bpffilter); err != nil { 1036 | log.Fatal("BPF filter error:", err) 1037 | } 1038 | } else { 1039 | 1040 | bpffilter := fmt.Sprintf("tcp and port %d",*port) 1041 | if err = handle.SetBPFFilter(bpffilter); err != nil { 1042 | log.Fatal("BPF filter error:", err) 1043 | } 1044 | 1045 | } 1046 | 1047 | source := gopacket.NewPacketSource(handle, handle.LinkType()) 1048 | source.Lazy = *lazy 1049 | source.NoCopy = true 1050 | Info("Starting to read packets\n") 1051 | count := 0 1052 | bytes := int64(0) 1053 | start := time.Now() 1054 | defragger := ip4defrag.NewIPv4Defragmenter() 1055 | 1056 | streamFactory := &tcpStreamFactory{doHTTP: !*nohttp} 1057 | streamPool := reassembly.NewStreamPool(streamFactory) 1058 | assembler := reassembly.NewAssembler(streamPool) 1059 | 1060 | signalChan = make(chan os.Signal, 1) 1061 | signal.Notify(signalChan, os.Interrupt) 1062 | go HandlerSig() 1063 | 1064 | for packet := range source.Packets() { 1065 | count++ 1066 | Debug("PACKET #%d\n", count) 1067 | data := packet.Data() 1068 | bytes += int64(len(data)) 1069 | if *hexdumppkt { 1070 | Debug("Packet content (%d/0x%x)\n%s\n", len(data), len(data), hex.Dump(data)) 1071 | } 1072 | 1073 | // defrag the IPv4 packet if required 1074 | if !*nodefrag { 1075 | ip4Layer := packet.Layer(layers.LayerTypeIPv4) 1076 | if ip4Layer == nil { 1077 | continue 1078 | } 1079 | ip4 := ip4Layer.(*layers.IPv4) 1080 | l := ip4.Length 1081 | newip4, err := defragger.DefragIPv4(ip4) 1082 | if err != nil { 1083 | log.Fatalln("Error while de-fragmenting", err) 1084 | } else if newip4 == nil { 1085 | Debug("Fragment...\n") 1086 | continue // packet fragment, we don't have whole packet yet. 1087 | } 1088 | if newip4.Length != l { 1089 | stats.ipdefrag++ 1090 | Debug("Decoding re-assembled packet: %s\n", newip4.NextLayerType()) 1091 | pb, ok := packet.(gopacket.PacketBuilder) 1092 | if !ok { 1093 | panic("Not a PacketBuilder") 1094 | } 1095 | nextDecoder := newip4.NextLayerType() 1096 | nextDecoder.Decode(newip4.Payload, pb) 1097 | } 1098 | } 1099 | 1100 | tcp := packet.Layer(layers.LayerTypeTCP) 1101 | if tcp != nil { 1102 | tcp := tcp.(*layers.TCP) 1103 | if *checksum { 1104 | err := tcp.SetNetworkLayerForChecksum(packet.NetworkLayer()) 1105 | if err != nil { 1106 | log.Fatalf("Failed to set network layer for checksum: %s\n", err) 1107 | } 1108 | } 1109 | c := Context{ 1110 | CaptureInfo: packet.Metadata().CaptureInfo, 1111 | } 1112 | stats.totalsz += len(tcp.Payload) 1113 | 1114 | assembler.AssembleWithContext(packet.NetworkLayer().NetworkFlow(), tcp, &c) 1115 | } 1116 | if count%*statsevery == 0 { 1117 | ref := packet.Metadata().CaptureInfo.Timestamp 1118 | flushed, closed := assembler.FlushWithOptions(reassembly.FlushOptions{T: ref.Add(-timeout), TC: ref.Add(-closeTimeout)}) 1119 | Debug("Forced flush: %d flushed, %d closed (%s)", flushed, closed, ref) 1120 | } 1121 | 1122 | done := *maxcount > 0 && count >= *maxcount 1123 | if count%*statsevery == 0 || done || sysexit { 1124 | errorsMapMutex.Lock() 1125 | errorMapLen := len(errorsMap) 1126 | errorsMapMutex.Unlock() 1127 | fmt.Fprintf(os.Stderr, "Processed %v packets (%v bytes) in %v (errors: %v, errTypes:%v)\n", count, bytes, time.Since(start), errors, errorMapLen) 1128 | } 1129 | if sysexit { 1130 | break 1131 | } 1132 | } 1133 | 1134 | closed := assembler.FlushAll() 1135 | Debug("Final flush: %d closed", closed) 1136 | if outputLevel >= 2 { 1137 | streamPool.Dump() 1138 | } 1139 | 1140 | if *memprofile != "" { 1141 | f, err := os.Create(*memprofile) 1142 | if err != nil { 1143 | log.Fatal(err) 1144 | } 1145 | pprof.WriteHeapProfile(f) 1146 | f.Close() 1147 | } 1148 | 1149 | streamFactory.WaitGoRoutines() 1150 | Debug("%s\n", assembler.Dump()) 1151 | if !*nodefrag { 1152 | fmt.Printf("IPdefrag:\t\t%d\n", stats.ipdefrag) 1153 | } 1154 | fmt.Printf("TCP stats:\n") 1155 | fmt.Printf(" missed bytes:\t\t%d\n", stats.missedBytes) 1156 | fmt.Printf(" total packets:\t\t%d\n", stats.pkt) 1157 | fmt.Printf(" rejected FSM:\t\t%d\n", stats.rejectFsm) 1158 | fmt.Printf(" rejected Options:\t%d\n", stats.rejectOpt) 1159 | fmt.Printf(" reassembled bytes:\t%d\n", stats.sz) 1160 | fmt.Printf(" total TCP bytes:\t%d\n", stats.totalsz) 1161 | fmt.Printf(" conn rejected FSM:\t%d\n", stats.rejectConnFsm) 1162 | fmt.Printf(" reassembled chunks:\t%d\n", stats.reassembled) 1163 | fmt.Printf(" out-of-order packets:\t%d\n", stats.outOfOrderPackets) 1164 | fmt.Printf(" out-of-order bytes:\t%d\n", stats.outOfOrderBytes) 1165 | fmt.Printf(" biggest-chunk packets:\t%d\n", stats.biggestChunkPackets) 1166 | fmt.Printf(" biggest-chunk bytes:\t%d\n", stats.biggestChunkBytes) 1167 | fmt.Printf(" overlap packets:\t%d\n", stats.overlapPackets) 1168 | fmt.Printf(" overlap bytes:\t\t%d\n", stats.overlapBytes) 1169 | fmt.Printf("Errors: %d\n", errors) 1170 | for e, _ := range errorsMap { 1171 | fmt.Printf(" %s:\t\t%d\n", e, errorsMap[e]) 1172 | } 1173 | } 1174 | 1175 | --------------------------------------------------------------------------------