├── go.mod ├── packet ├── handle.go ├── index.go ├── streamreader.go ├── listen.go ├── httpstreamfactory.go ├── httpstreampair.go └── httpstream.go ├── examples ├── custom │ └── main.go └── default │ └── main.go ├── .gitignore ├── go.sum ├── README.md └── LICENSE /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/shixiaofeia/gopacket-http 2 | 3 | go 1.19 4 | 5 | require github.com/google/gopacket v1.1.19 6 | 7 | require golang.org/x/sys v0.20.0 // indirect 8 | -------------------------------------------------------------------------------- /packet/handle.go: -------------------------------------------------------------------------------- 1 | package packet 2 | 3 | // EventHandle 设置goroutine进行消息处理. 4 | func (slf *Handle) EventHandle() { 5 | if slf.goroutineNum <= 0 { 6 | return 7 | } 8 | 9 | for i := 0; i < slf.goroutineNum; i++ { 10 | go slf.handle() 11 | } 12 | 13 | select { 14 | case <-slf.ctx.Done(): 15 | close(slf.eventCh) 16 | return 17 | } 18 | 19 | } 20 | 21 | // handle 消息处理. 22 | func (slf *Handle) handle() { 23 | for val := range slf.eventCh { 24 | data := val.(Event) 25 | slf.eventHandle(data.Req, data.Resp) 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /examples/custom/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "github.com/shixiaofeia/gopacket-http/packet" 6 | "log" 7 | ) 8 | 9 | var eventCh = make(chan interface{}, 1024) 10 | 11 | func main() { 12 | go handle() 13 | if err := packet.NewPacketHandle(context.Background(), "en0", eventCh).Listen(); err != nil { 14 | log.Println(err.Error()) 15 | } 16 | } 17 | 18 | func handle() { 19 | for i := range eventCh { 20 | data := i.(packet.Event) 21 | log.Printf("request uri: %s, response status: %v", data.Req.RequestURI, data.Resp.Status) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # If you prefer the allow list template instead of the deny list, see community template: 2 | # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore 3 | # 4 | # Binaries for programs and plugins 5 | *.exe 6 | *.exe~ 7 | *.dll 8 | *.so 9 | *.dylib 10 | 11 | # Test binary, built with `go test -c` 12 | *.test 13 | 14 | # Output of the go coverage tool, specifically when used with LiteIDE 15 | *.out 16 | 17 | # Dependency directories (remove the comment below to include it) 18 | # vendor/ 19 | 20 | # Go workspace file 21 | go.work 22 | go.work.sum 23 | .idea -------------------------------------------------------------------------------- /examples/default/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "github.com/shixiaofeia/gopacket-http/packet" 6 | "log" 7 | "net/http" 8 | "time" 9 | ) 10 | 11 | var ( 12 | eventCh = make(chan interface{}, 1024) 13 | ctx, cancel = context.WithCancel(context.Background()) 14 | ) 15 | 16 | func main() { 17 | go shutdown() 18 | srv := packet.NewPacketHandle(ctx, "en0", eventCh) 19 | srv.SetBpf("tcp port 80") // 设置BPF过滤规则 20 | srv.SetEventHandle(5, handle) // 设置多协程事件处理, 21 | srv.SetPromisc(true) // 设置混杂模式开启状态, 22 | srv.SetFlushTime(time.Minute) // 设置清理缓存时间 23 | if err := srv.Listen(); err != nil { 24 | log.Println(err.Error()) 25 | } 26 | } 27 | 28 | func handle(req *http.Request, resp *http.Response) { 29 | log.Printf("request uri: %s, response status: %v", req.RequestURI, resp.Status) 30 | } 31 | 32 | func shutdown() { 33 | time.Sleep(time.Second * 10) 34 | cancel() 35 | } 36 | -------------------------------------------------------------------------------- /packet/index.go: -------------------------------------------------------------------------------- 1 | package packet 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | "time" 7 | ) 8 | 9 | type ( 10 | Event struct { 11 | Req *http.Request 12 | Resp *http.Response 13 | } 14 | EventFunc func(req *http.Request, resp *http.Response) 15 | Handle struct { 16 | ctx context.Context // 上下文. 17 | cardName string // 网卡名称. 18 | bpf string // 过滤器规则. 19 | promisc bool // 是否混杂模式. 20 | eventCh chan interface{} // 事件通道. 21 | goroutineNum int // 协程数量. 22 | eventHandle EventFunc // 事件处理. 23 | flushTime time.Duration // 清理缓存时间. 24 | } 25 | ) 26 | 27 | func NewPacketHandle(ctx context.Context, cardName string, eventCh chan interface{}) *Handle { 28 | return &Handle{ 29 | ctx: ctx, 30 | cardName: cardName, 31 | bpf: "tcp", 32 | promisc: false, 33 | eventCh: eventCh, 34 | flushTime: time.Minute * -2, 35 | } 36 | } 37 | 38 | // SetBpf 设置过滤器规则. 39 | // 设置后仅处理过滤后的数据包. 40 | func (slf *Handle) SetBpf(bpf string) *Handle { 41 | slf.bpf = bpf 42 | return slf 43 | } 44 | 45 | // SetPromisc 设置混杂模式. 46 | // 开启后接收和处理所有经过的网络流量,而不仅仅是发送到该接口的流量. 47 | func (slf *Handle) SetPromisc(promise bool) *Handle { 48 | slf.promisc = promise 49 | return slf 50 | } 51 | 52 | // SetEventHandle 设置多协程事件处理. 53 | // 设置后启动自定义数量goroutine去处理逻辑. 54 | func (slf *Handle) SetEventHandle(goroutineNum int, handle EventFunc) *Handle { 55 | slf.goroutineNum = goroutineNum 56 | slf.eventHandle = handle 57 | return slf 58 | } 59 | 60 | // SetFlushTime 设置清理缓存时间, 61 | // 清除收到的最后一个数据包时间加上此时间之前的所有的数据包. 62 | func (slf *Handle) SetFlushTime(timer time.Duration) *Handle { 63 | slf.flushTime = timer.Abs() * -1 64 | return slf 65 | } 66 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= 2 | github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= 3 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 4 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 5 | golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= 6 | golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= 7 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 8 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= 9 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 10 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 11 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 12 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 13 | golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= 14 | golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 15 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 16 | golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 17 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 18 | -------------------------------------------------------------------------------- /packet/streamreader.go: -------------------------------------------------------------------------------- 1 | package packet 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "time" 7 | ) 8 | 9 | // StreamDataBlock is copied from tcpassembly.Reassembly 10 | type StreamDataBlock struct { 11 | Bytes []byte 12 | Seen time.Time 13 | } 14 | 15 | // NewStreamDataBlock create a new StreamDataBlock. 16 | func NewStreamDataBlock(bytes []byte, seen time.Time) *StreamDataBlock { 17 | b := new(StreamDataBlock) 18 | b.Bytes = make([]byte, len(bytes)) 19 | copy(b.Bytes, bytes[:]) 20 | b.Seen = seen 21 | return b 22 | } 23 | 24 | // StreamReader read data from tcp stream 25 | type StreamReader struct { 26 | src chan *StreamDataBlock 27 | stopCh chan interface{} 28 | buffer *bytes.Buffer 29 | lastSeen time.Time 30 | } 31 | 32 | // NewStreamReader create a new StreamReader. 33 | func NewStreamReader() *StreamReader { 34 | r := new(StreamReader) 35 | r.stopCh = make(chan interface{}) 36 | r.buffer = bytes.NewBuffer([]byte("")) 37 | r.src = make(chan *StreamDataBlock, 32) 38 | return r 39 | } 40 | 41 | // fillBuffer 填充缓冲区. 42 | func (s *StreamReader) fillBuffer() error { 43 | if dataBlock, ok := <-s.src; ok { 44 | s.buffer.Write(dataBlock.Bytes) 45 | s.lastSeen = dataBlock.Seen 46 | return nil 47 | } 48 | return errors.New("EOF") 49 | } 50 | 51 | // ReadUntil 读取到指定分隔符. 52 | func (s *StreamReader) ReadUntil(delim []byte) ([]byte, error) { 53 | var p int 54 | for { 55 | if p = bytes.Index(s.buffer.Bytes(), delim); p == -1 { 56 | if err := s.fillBuffer(); err != nil { 57 | return nil, err 58 | } 59 | } else { 60 | break 61 | } 62 | } 63 | return s.buffer.Next(p + len(delim)), nil 64 | } 65 | 66 | // Next 往后获取指定长度的数据. 67 | func (s *StreamReader) Next(n int) ([]byte, error) { 68 | for s.buffer.Len() < n { 69 | if err := s.fillBuffer(); err != nil { 70 | return nil, err 71 | } 72 | } 73 | dst := make([]byte, n) 74 | copy(dst, s.buffer.Next(n)) 75 | return dst, nil 76 | } 77 | -------------------------------------------------------------------------------- /packet/listen.go: -------------------------------------------------------------------------------- 1 | package packet 2 | 3 | import ( 4 | "fmt" 5 | "github.com/google/gopacket" 6 | "github.com/google/gopacket/layers" 7 | "github.com/google/gopacket/pcap" 8 | "github.com/google/gopacket/tcpassembly" 9 | "log" 10 | "net" 11 | "time" 12 | ) 13 | 14 | // Listen 监听网卡. 15 | func (slf *Handle) Listen() error { 16 | // 获取网卡信息 17 | iface, err := net.InterfaceByName(slf.cardName) 18 | if err != nil { 19 | return fmt.Errorf("cardName %s not found, err: %v", slf.cardName, err) 20 | } 21 | log.Printf("cardName: %s, MTU: %d", slf.cardName, iface.MTU) 22 | 23 | // 打开设备监听 24 | handle, err := pcap.OpenLive(slf.cardName, 1024*1024, slf.promisc, pcap.BlockForever) 25 | if err != nil { 26 | return fmt.Errorf("openLive %s err: %v", slf.cardName, err) 27 | } 28 | defer handle.Close() 29 | 30 | // 设置过滤器 31 | if err := handle.SetBPFFilter(slf.bpf); err != nil { 32 | return fmt.Errorf("set bpf filter: %v", err) 33 | } 34 | 35 | go slf.EventHandle() 36 | packetSource := gopacket.NewPacketSource(handle, handle.LinkType()) 37 | streamFactory := NewHTTPStreamFactory(slf.eventCh) 38 | pool := tcpassembly.NewStreamPool(streamFactory) 39 | assembler := tcpassembly.NewAssembler(pool) 40 | 41 | ticker := time.Tick(time.Minute) 42 | var lastPacketTimestamp time.Time 43 | 44 | for { 45 | select { 46 | case <-slf.ctx.Done(): 47 | return nil 48 | case packet := <-packetSource.Packets(): 49 | netLayer := packet.NetworkLayer() 50 | if netLayer == nil { 51 | continue 52 | } 53 | transLayer := packet.TransportLayer() 54 | if transLayer == nil { 55 | continue 56 | } 57 | tcp, _ := transLayer.(*layers.TCP) 58 | if tcp == nil { 59 | continue 60 | } 61 | assembler.AssembleWithTimestamp( 62 | netLayer.NetworkFlow(), 63 | tcp, 64 | packet.Metadata().CaptureInfo.Timestamp) 65 | 66 | lastPacketTimestamp = packet.Metadata().CaptureInfo.Timestamp 67 | case <-ticker: 68 | assembler.FlushOlderThan(lastPacketTimestamp.Add(slf.flushTime)) 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /packet/httpstreamfactory.go: -------------------------------------------------------------------------------- 1 | package packet 2 | 3 | import ( 4 | "sync" 5 | "sync/atomic" 6 | 7 | "github.com/google/gopacket" 8 | "github.com/google/gopacket/tcpassembly" 9 | ) 10 | 11 | // HTTPStreamFactory implements StreamFactory interface for tcpassembly 12 | type HTTPStreamFactory struct { 13 | runningStream *int32 14 | wg *sync.WaitGroup 15 | seq *uint 16 | uniStreams *map[streamKey]*httpStreamPair 17 | eventChan chan<- interface{} 18 | } 19 | 20 | // NewHTTPStreamFactory create a NewHTTPStreamFactory. 21 | func NewHTTPStreamFactory(out chan<- interface{}) HTTPStreamFactory { 22 | var f HTTPStreamFactory 23 | f.seq = new(uint) 24 | *f.seq = 0 25 | f.wg = new(sync.WaitGroup) 26 | f.uniStreams = new(map[streamKey]*httpStreamPair) 27 | *f.uniStreams = make(map[streamKey]*httpStreamPair) 28 | f.eventChan = out 29 | f.runningStream = new(int32) 30 | return f 31 | } 32 | 33 | // Wait 等待所有流完成. 34 | func (f HTTPStreamFactory) Wait() { 35 | f.wg.Wait() 36 | } 37 | 38 | // RunningStreamCount 当前流计数. 39 | func (f *HTTPStreamFactory) RunningStreamCount() int32 { 40 | return atomic.LoadInt32(f.runningStream) 41 | } 42 | 43 | // runStreamPair 运行一个httpStreamPair. 44 | func (f *HTTPStreamFactory) runStreamPair(streamPair *httpStreamPair) { 45 | atomic.AddInt32(f.runningStream, 1) 46 | 47 | defer f.wg.Done() 48 | defer func() { atomic.AddInt32(f.runningStream, -1) }() 49 | streamPair.run() 50 | } 51 | 52 | // New creates a HTTPStreamFactory. 53 | func (f HTTPStreamFactory) New(netFlow, tcpFlow gopacket.Flow) (ret tcpassembly.Stream) { 54 | reverseKey := streamKey{netFlow.Reverse(), tcpFlow.Reverse()} 55 | streamPair, ok := (*f.uniStreams)[reverseKey] 56 | if ok { 57 | if streamPair.upStream == nil { 58 | panic("unbelievable!?") 59 | } 60 | delete(*f.uniStreams, reverseKey) 61 | key := streamKey{netFlow, tcpFlow} 62 | s := newHTTPStream(key) 63 | streamPair.downStream = &s 64 | ret = s 65 | } else { 66 | streamPair = newHTTPStreamPair(*f.seq, f.eventChan) 67 | key := streamKey{netFlow, tcpFlow} 68 | s := newHTTPStream(key) 69 | streamPair.upStream = &s 70 | (*f.uniStreams)[key] = streamPair 71 | *f.seq++ 72 | f.wg.Add(1) 73 | go f.runStreamPair(streamPair) 74 | ret = s 75 | } 76 | return 77 | } 78 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # gopacket-http 2 | 3 |

4 | 5 | 6 | 7 | 8 | GitHub 9 | 10 | 11 | GitHub last commit 12 | GitHub Repo stars 13 |

14 | 15 | 监听网卡流量, 过滤并组装HTTP请求和响应, 供旁路分析, 抓包等用途 16 | 17 | 参考项目 [netgraph](https://github.com/ga0/netgraph) 18 | 19 | ## 使用 20 | 21 | 1. 安装libpcap-dev 和 gcc 22 | 23 | ```sh 24 | # Ubuntu 25 | sudo apt install -y libpcap-dev gcc 26 | 27 | # CentOS 28 | sudo yum install -y libpcap-devel gcc 29 | 30 | # MacOS(Homebrew) 31 | brew install libpcap 32 | 33 | ``` 34 | 35 | 2. 安装gopacket-http 36 | 37 | ```sh 38 | go get -u github.com/shixiaofeia/gopacket-http 39 | ``` 40 | 41 | 3. 在代码中导入 42 | 43 | ```go 44 | import "github.com/shixiaofeia/gopacket-http/packet" 45 | ``` 46 | 47 | ## 快速开始 48 | 49 | ```go 50 | package main 51 | 52 | import ( 53 | "context" 54 | "github.com/shixiaofeia/gopacket-http/packet" 55 | "log" 56 | ) 57 | 58 | var eventCh = make(chan interface{}, 1024) 59 | 60 | func main() { 61 | go handle() 62 | if err := packet.NewPacketHandle(context.Background(), "en0", eventCh).Listen(); err != nil { 63 | log.Println(err.Error()) 64 | } 65 | } 66 | 67 | func handle() { 68 | for i := range eventCh { 69 | data := i.(packet.Event) 70 | log.Printf("request uri: %s, response status: %v", data.Req.RequestURI, data.Resp.Status) 71 | } 72 | } 73 | 74 | ``` 75 | 76 | 77 | ## 可配置的参数 78 | 79 | ```go 80 | package main 81 | 82 | import ( 83 | "context" 84 | "github.com/shixiaofeia/gopacket-http/packet" 85 | "log" 86 | "net/http" 87 | "time" 88 | ) 89 | 90 | var ( 91 | eventCh = make(chan interface{}, 1024) 92 | ctx, cancel = context.WithCancel(context.Background()) 93 | ) 94 | 95 | func main() { 96 | go shutdown() 97 | srv := packet.NewPacketHandle(ctx, "en0", eventCh) 98 | srv.SetBpf("tcp port 80") // 设置BPF过滤规则 99 | srv.SetEventHandle(5, handle) // 设置多协程事件处理, 100 | srv.SetPromisc(true) // 设置混杂模式开启状态, 101 | srv.SetFlushTime(time.Minute) // 设置清理缓存时间 102 | if err := srv.Listen(); err != nil { 103 | log.Println(err.Error()) 104 | } 105 | } 106 | 107 | func handle(req *http.Request, resp *http.Response) { 108 | log.Printf("request uri: %s, response status: %v", req.RequestURI, resp.Status) 109 | } 110 | 111 | func shutdown() { 112 | time.Sleep(time.Second * 10) 113 | cancel() 114 | } 115 | 116 | ``` 117 | 118 | 119 | -------------------------------------------------------------------------------- /packet/httpstreampair.go: -------------------------------------------------------------------------------- 1 | package packet 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "log" 7 | "net/http" 8 | "time" 9 | ) 10 | 11 | // HTTPHeaderItem is HTTP header key-value pair 12 | type HTTPHeaderItem struct { 13 | Name string 14 | Value string 15 | } 16 | 17 | // HTTPEvent is HTTP request or response 18 | type HTTPEvent struct { 19 | Type string 20 | Start time.Time 21 | End time.Time 22 | StreamSeq uint 23 | } 24 | 25 | // HTTPRequestEvent is HTTP request 26 | type HTTPRequestEvent struct { 27 | HTTPEvent 28 | ClientAddr string 29 | ServerAddr string 30 | Method string 31 | URI string 32 | Version string 33 | Headers []HTTPHeaderItem 34 | Body []byte 35 | } 36 | 37 | // HTTPResponseEvent is HTTP response 38 | type HTTPResponseEvent struct { 39 | HTTPEvent 40 | ClientAddr string 41 | ServerAddr string 42 | Version string 43 | Code uint 44 | Reason string 45 | Headers []HTTPHeaderItem 46 | Body []byte 47 | } 48 | 49 | // httpStreamPair is Bi-direction HTTP stream pair 50 | type httpStreamPair struct { 51 | upStream *httpStream 52 | downStream *httpStream 53 | 54 | requestSeq uint 55 | connSeq uint 56 | eventChan chan<- interface{} 57 | } 58 | 59 | // newHTTPStreamPair 实例化httpStreamPair. 60 | func newHTTPStreamPair(seq uint, eventChan chan<- interface{}) *httpStreamPair { 61 | pair := new(httpStreamPair) 62 | pair.connSeq = seq 63 | pair.eventChan = eventChan 64 | 65 | return pair 66 | } 67 | 68 | // run 循环处理. 69 | func (pair *httpStreamPair) run() { 70 | defer func() { 71 | if r := recover(); r != nil { 72 | if pair.upStream != nil { 73 | close(pair.upStream.reader.stopCh) 74 | } 75 | if pair.downStream != nil { 76 | close(pair.downStream.reader.stopCh) 77 | } 78 | } 79 | }() 80 | 81 | for { 82 | pair.handleTransaction() 83 | pair.requestSeq++ 84 | } 85 | } 86 | 87 | // handleTransaction 处理HTTP事务. 88 | func (pair *httpStreamPair) handleTransaction() { 89 | upStream := pair.upStream 90 | reqBytes, method, reqHeaders := upStream.GetRequestBytes() 91 | reqBody := upStream.getBody(method, reqHeaders, true) 92 | reqBytes = append(reqBytes, reqBody...) 93 | 94 | downStream := pair.downStream 95 | resBytes, resHeaders := downStream.GetResponseBytes() 96 | respBody := downStream.getBody(method, resHeaders, false) 97 | resBytes = append(resBytes, respBody...) 98 | 99 | req, err := http.ReadRequest(bufio.NewReader(bytes.NewReader(reqBytes))) 100 | if err != nil { 101 | log.Printf("read request err: %v\n", err) 102 | } 103 | 104 | // chunked response不可读取, 需替换过滤. 105 | var emptyByte []byte 106 | resBytes = bytes.Replace(resBytes, []byte("Transfer-Encoding: chunked\r\n"), emptyByte, -1) 107 | resp, err := http.ReadResponse(bufio.NewReader(bytes.NewReader(resBytes)), req) 108 | if err != nil { 109 | log.Printf("read response err: %v\n", err) 110 | } 111 | 112 | req.RemoteAddr = upStream.key.net.Src().String() 113 | pair.eventChan <- Event{ 114 | Req: req, 115 | Resp: resp, 116 | } 117 | 118 | } 119 | -------------------------------------------------------------------------------- /packet/httpstream.go: -------------------------------------------------------------------------------- 1 | package packet 2 | 3 | import ( 4 | "bytes" 5 | "compress/flate" 6 | "compress/gzip" 7 | "fmt" 8 | "io" 9 | "regexp" 10 | "strconv" 11 | "strings" 12 | "time" 13 | 14 | "github.com/google/gopacket" 15 | "github.com/google/gopacket/tcpassembly" 16 | ) 17 | 18 | var ( 19 | httpRequestFirstLine *regexp.Regexp 20 | ) 21 | 22 | func init() { 23 | httpRequestFirstLine = regexp.MustCompile(`([A-Z]+) (.+) (HTTP/.+)\r\n`) 24 | } 25 | 26 | type streamKey struct { 27 | net, tcp gopacket.Flow 28 | } 29 | 30 | func (k streamKey) String() string { 31 | return fmt.Sprintf("{%v:%v} -> {%v:%v}", k.net.Src(), k.tcp.Src(), k.net.Dst(), k.tcp.Dst()) 32 | } 33 | 34 | type httpStream struct { 35 | reader *StreamReader 36 | bytes *uint64 37 | key streamKey 38 | bad *bool 39 | } 40 | 41 | func newHTTPStream(key streamKey) httpStream { 42 | var s httpStream 43 | s.reader = NewStreamReader() 44 | s.bytes = new(uint64) 45 | s.key = key 46 | s.bad = new(bool) 47 | return s 48 | } 49 | 50 | // Reassembled 由tcpassembly调用. 51 | func (s httpStream) Reassembled(rs []tcpassembly.Reassembly) { 52 | if *s.bad { 53 | return 54 | } 55 | 56 | for _, r := range rs { 57 | if r.Skip != 0 { 58 | *s.bad = true 59 | return 60 | } 61 | 62 | if len(r.Bytes) == 0 { 63 | continue 64 | } 65 | 66 | *s.bytes += uint64(len(r.Bytes)) 67 | ticker := time.Tick(time.Second) 68 | 69 | select { 70 | case <-s.reader.stopCh: 71 | *s.bad = true 72 | return 73 | case s.reader.src <- NewStreamDataBlock(r.Bytes, r.Seen): 74 | case <-ticker: 75 | // Sometimes pcap only captured HTTP response with no request! 76 | // Let's wait few seconds to avoid dead lock. 77 | *s.bad = true 78 | return 79 | } 80 | } 81 | } 82 | 83 | // ReassemblyComplete 由tcpassembly调用. 84 | func (s httpStream) ReassemblyComplete() { 85 | close(s.reader.src) 86 | } 87 | 88 | // getChunked 获取chunked内容. 89 | func (s *httpStream) getChunked() []byte { 90 | var body []byte 91 | for { 92 | buf, err := s.reader.ReadUntil([]byte("\r\n")) 93 | if err != nil { 94 | panic("Cannot read chuncked content, err=" + err.Error()) 95 | } 96 | l := string(buf) 97 | l = strings.Trim(l[:len(l)-2], " ") 98 | blockSize, err := strconv.ParseInt(l, 16, 32) 99 | if err != nil { 100 | panic("bad chunked block length: " + l + ", err=" + err.Error()) 101 | } 102 | 103 | buf, err = s.reader.Next(int(blockSize)) 104 | body = append(body, buf...) 105 | if err != nil { 106 | panic("Cannot read chuncked content, err=" + err.Error()) 107 | } 108 | buf, err = s.reader.Next(2) 109 | if err != nil { 110 | panic("Cannot read chuncked content, err=" + err.Error()) 111 | } 112 | CRLF := string(buf) 113 | if CRLF != "\r\n" { 114 | panic("Bad chunked block data") 115 | } 116 | 117 | if blockSize == 0 { 118 | break 119 | } 120 | } 121 | return body 122 | } 123 | 124 | // getFixedLengthContent 获取固定长度内容. 125 | func (s *httpStream) getFixedLengthContent(contentLength int) []byte { 126 | body, err := s.reader.Next(contentLength) 127 | if err != nil { 128 | panic("Cannot read content, err=" + err.Error()) 129 | } 130 | return body 131 | } 132 | 133 | // getContentInfo 获取请求/响应数据长度和编码. 134 | func getContentInfo(hs []HTTPHeaderItem) (contentLength int, contentEncoding string, contentType string, chunked bool) { 135 | for _, h := range hs { 136 | lowerName := strings.ToLower(h.Name) 137 | if lowerName == "content-length" { 138 | var err error 139 | contentLength, err = strconv.Atoi(h.Value) 140 | if err != nil { 141 | panic("Content-Length error: " + h.Value + ", err=" + err.Error()) 142 | } 143 | } else if lowerName == "transfer-encoding" && h.Value == "chunked" { 144 | chunked = true 145 | } else if lowerName == "content-encoding" { 146 | contentEncoding = h.Value 147 | } else if lowerName == "content-type" { 148 | contentType = h.Value 149 | } 150 | } 151 | return 152 | } 153 | 154 | // getBody 获取请求/响应数据. 155 | func (s *httpStream) getBody(method string, headers []HTTPHeaderItem, isRequest bool) (body []byte) { 156 | contentLength, contentEncoding, _, chunked := getContentInfo(headers) 157 | if (contentLength == 0 && !chunked) || (!isRequest && method == "HEAD") { 158 | return 159 | } 160 | 161 | if chunked { 162 | body = s.getChunked() 163 | } else { 164 | body = s.getFixedLengthContent(contentLength) 165 | } 166 | 167 | var err error 168 | // TODO: 支持更多压缩格式的处理 169 | switch contentEncoding { 170 | case "gzip": 171 | body, err = unGzip(body) 172 | case "deflate": 173 | body, err = unDeflate(body) 174 | default: 175 | } 176 | if err != nil { 177 | body = []byte("(decompression failed)") 178 | } 179 | 180 | return 181 | } 182 | 183 | // GetRequestBytes 获取请求数据. 184 | func (s *httpStream) GetRequestBytes() (bytes []byte, method string, headers []HTTPHeaderItem) { 185 | lineBytes, err := s.reader.ReadUntil([]byte("\r\n")) 186 | if err != nil { 187 | panic("Cannot read request line, err=" + err.Error()) 188 | } 189 | line := string(lineBytes) 190 | r := httpRequestFirstLine.FindStringSubmatch(line) 191 | if len(r) != 4 { 192 | panic("Bad HTTP Request: " + line) 193 | } 194 | method = r[1] 195 | 196 | headerByte, _ := s.reader.ReadUntil([]byte("\r\n\r\n")) 197 | bytes1 := make([]byte, len(lineBytes)) 198 | bytes2 := make([]byte, len(headerByte)) 199 | copy(bytes1, lineBytes) 200 | copy(bytes2, headerByte) 201 | bytes = append(bytes1, bytes2...) 202 | headers = s.ByteToHeader(headerByte) 203 | 204 | return 205 | } 206 | 207 | // GetResponseBytes 获取响应数据. 208 | func (s *httpStream) GetResponseBytes() (bytes []byte, headers []HTTPHeaderItem) { 209 | lineBytes, _ := s.reader.ReadUntil([]byte("\r\n")) 210 | headerByte, _ := s.reader.ReadUntil([]byte("\r\n\r\n")) 211 | bytes1 := make([]byte, len(lineBytes)) 212 | bytes2 := make([]byte, len(headerByte)) 213 | copy(bytes1, lineBytes) 214 | copy(bytes2, headerByte) 215 | bytes = append(bytes1, bytes2...) 216 | headers = s.ByteToHeader(headerByte) 217 | 218 | return 219 | } 220 | 221 | // ByteToHeader header转换. 222 | func (s *httpStream) ByteToHeader(headerByte []byte) (headers []HTTPHeaderItem) { 223 | data := string(headerByte[:len(headerByte)-4]) 224 | for i, line := range strings.Split(data, "\r\n") { 225 | p := strings.Index(line, ":") 226 | if p == -1 { 227 | panic(fmt.Sprintf("Bad http header (line %d): %s", i, data)) 228 | } 229 | var h HTTPHeaderItem 230 | h.Name = line[:p] 231 | h.Value = strings.Trim(line[p+1:], " ") 232 | headers = append(headers, h) 233 | } 234 | 235 | return 236 | } 237 | 238 | // unGzip gzip解压缩. 239 | func unGzip(data []byte) ([]byte, error) { 240 | reader, err := gzip.NewReader(bytes.NewReader(data)) 241 | if err != nil { 242 | return nil, err 243 | } 244 | defer reader.Close() 245 | return io.ReadAll(reader) 246 | } 247 | 248 | // unDeflate deflate解压缩. 249 | func unDeflate(data []byte) ([]byte, error) { 250 | reader := flate.NewReader(bytes.NewReader(data)) 251 | defer reader.Close() 252 | return io.ReadAll(reader) 253 | } 254 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | --------------------------------------------------------------------------------