├── .github ├── FUNDING.yml ├── build │ ├── linux.yml │ ├── mac.yml │ ├── mac_arm64.yml │ └── windows.yml └── workflows │ └── build.yml ├── .gitignore ├── DoFile.go ├── PipelineHttp.go ├── README.md ├── cmd ├── main.go ├── testUrl.txt └── testUrls.txt ├── go.mod ├── go.sum ├── http2Imp.go ├── http3client.go ├── portscan └── Main.go └── test2 ├── main.go ├── testUrl.txt └── testUrls.txt /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: [hktalent] 4 | -------------------------------------------------------------------------------- /.github/build/linux.yml: -------------------------------------------------------------------------------- 1 | env: 2 | - GO111MODULE=on 3 | #before: 4 | # hooks: 5 | # - go mod tidy 6 | project_name: PipelineHttp 7 | builds: 8 | - id: PipelineHttp-linux 9 | ldflags: 10 | - -s -w 11 | binary: ppHttp 12 | env: 13 | - CGO_ENABLED=1 14 | # 0 你再重新编译链接，那么 Go 链接器会使用 Go 版本的实现，这样你将得到一个没有动态链接的纯静态二进制程序 15 | main: cmd/main.go 16 | goos: 17 | - linux 18 | goarch: 19 | - amd64 20 | archives: 21 | - format: zip 22 | 23 | checksum: 24 | name_template: "{{ .ProjectName }}-linux-checksums.txt" 25 | -------------------------------------------------------------------------------- /.github/build/mac.yml: -------------------------------------------------------------------------------- 1 | env: 2 | - GO111MODULE=on 3 | #before: 4 | # hooks: 5 | # - go mod tidy 6 | project_name: PipelineHttp 7 | builds: 8 | - id: PipelineHttp-darwin 9 | ldflags: 10 | - -s -w 11 | binary: ppHttp 12 | env: 13 | - CGO_ENABLED=1 14 | main: cmd/main.go 15 | goos: 16 | - darwin 17 | goarch: 18 | - amd64 19 | 20 | archives: 21 | - format: zip 22 | replacements: 23 | darwin: macOS 24 | 25 | checksum: 26 | name_template: "{{ .ProjectName }}-mac-checksums.txt" 27 | -------------------------------------------------------------------------------- /.github/build/mac_arm64.yml: -------------------------------------------------------------------------------- 1 | env: 2 | - GO111MODULE=on 3 | #before: 4 | # hooks: 5 | # - go mod tidy 6 | project_name: PipelineHttp 7 | builds: 8 | - id: PipelineHttp-darwin 9 | ldflags: 10 | - -s -w 11 | binary: ppHttp 12 | env: 13 | - CGO_ENABLED=1 14 | main: cmd/main.go 15 | goos: 16 | - darwin 17 | goarch: 18 | - arm64 19 | 20 | archives: 21 | - format: zip 22 | replacements: 23 | darwin: macOS 24 | 25 | checksum: 26 | name_template: "{{ .ProjectName }}-mac-arm64-checksums.txt" 27 | -------------------------------------------------------------------------------- /.github/build/windows.yml: -------------------------------------------------------------------------------- 1 | env: 2 | - GO111MODULE=on 3 | #before: 4 | # hooks: 5 | # - go mod tidy 6 | project_name: PipelineHttp 7 | builds: 8 | - id: PipelineHttp-windows 9 | ldflags: 10 | - -s -w 11 | binary: ppHttp 12 | env: 13 | - CGO_ENABLED=1 14 | - CC=x86_64-w64-mingw32-gcc 15 | - CXX=x86_64-w64-mingw32-g++ 16 | main: cmd/main.go 17 | goos: 18 | - windows 19 | goarch: 20 | - amd64 21 | 22 | archives: 23 | - format: zip 24 | 25 | checksum: 26 | name_template: "{{ .ProjectName }}-windows-checksums.txt" 27 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: 🎉 Release Binary 2 | on: 3 | create: 4 | tags: 5 | - v* 6 | workflow_dispatch: 7 | 8 | jobs: 9 | build-mac: 10 | runs-on: macos-latest 11 | steps: 12 | - name: Code checkout 13 | uses: actions/checkout@v2 14 | with: 15 | submodules: recursive 16 | fetch-depth: 0 17 | - name: Checkout submodules 18 | run: git submodule update --init --recursive 19 | - name: Set up Go 20 | uses: actions/setup-go@v2 21 | with: 22 | go-version: 1.19 23 | - name: Install Dependences 24 | run: | 25 | brew install libpcap upx 26 | # git submodule update --init --recursive --remote 27 | - name: Run GoReleaser 28 | uses: goreleaser/goreleaser-action@v2 29 | with: 30 | version: latest 31 | args: release -f .github/build/mac.yml --rm-dist 32 | env: 33 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 34 | # https://github.com/marketplace/actions/run-on-architecture#supported-platforms 35 | build-linux: 36 | runs-on: ubuntu-latest 37 | steps: 38 | - name: Code checkout 39 | uses: actions/checkout@v2 40 | with: 41 | submodules: recursive 42 | fetch-depth: 0 43 | - name: Checkout submodules 44 | run: git submodule update --init --recursive 45 | - name: Set up Go 46 | uses: actions/setup-go@v2 47 | with: 48 | go-version: 1.19 49 | - name: Install Dependences 50 | run: | 51 | sudo apt install -yy --fix-missing libpcap-dev upx 52 | # git submodule update --init --recursive --remote 53 | - name: Run GoReleaser 54 | uses: goreleaser/goreleaser-action@v2 55 | with: 56 | version: latest 57 | args: release -f .github/build/linux.yml --rm-dist 58 | env: 59 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 60 | 61 | build-windows: 62 | runs-on: windows-latest 63 | steps: 64 | - name: Code checkout 65 | uses: actions/checkout@v2 66 | with: 67 | submodules: recursive 68 | fetch-depth: 0 69 | - name: Checkout submodules 70 | run: git submodule update --init --recursive 71 | - name: Set up Go 72 | uses: actions/setup-go@v2 73 | with: 74 | go-version: 1.19 75 | - name: Run GoReleaser 76 | uses: goreleaser/goreleaser-action@v2 77 | with: 78 | version: latest 79 | args: release -f .github/build/windows.yml --rm-dist 80 | env: 81 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | .idea 3 | *.iml 4 | *.exe 5 | *.exe~ 6 | *.dll 7 | *.so 8 | *.dylib 9 | ppHttp 10 | scan4all 11 | release 12 | .DbCache 13 | scan4all_linux 14 | .DS_Store 15 | 16 | # Test binary, built with `go cmd -c` 17 | *.test 18 | 19 | # Output of the go coverage tool, specifically when used with LiteIDE 20 | *.out 21 | 22 | # Dependency directories (remove the comment below to include it) 23 | 24 | dist/ 25 | -------------------------------------------------------------------------------- /DoFile.go: -------------------------------------------------------------------------------- 1 | package PipelineHttp 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "log" 9 | "mime/multipart" 10 | "net/http" 11 | "net/textproto" 12 | ) 13 | 14 | // post 发送的多文件 15 | type PostFileData struct { 16 | ContentType string `json:"content_type"` 17 | Name string `json:"name"` 18 | FileName string `json:"file_name"` 19 | FileData io.Reader `json:"file_data"` 20 | } 21 | 22 | // 增加、支持：多文件上传 23 | func (r *PipelineHttp) SendFiles(c *http.Client, szUrl string, parms *map[string]interface{}, files *[]PostFileData, fnCbk func(resp *http.Response, err error, szU string), setHeader func() map[string]string) { 24 | body_buf := bytes.NewBufferString("") 25 | // reader, writer := io.Pipe()// 这里不适合这样的场景，无法提前预知发送数据的总长度 26 | // head start 27 | body_writer := multipart.NewWriter(body_buf) 28 | // head end 29 | var err error 30 | var f01 io.Writer 31 | if nil != parms { 32 | var data []byte 33 | for k, v := range *parms { 34 | if f01, err = body_writer.CreateFormField(k); nil == err { 35 | if data, err = json.Marshal(v); nil == err { 36 | f01.Write(data) 37 | continue 38 | } 39 | } 40 | if nil != err { 41 | log.Println(err) 42 | } 43 | } 44 | } 45 | if nil != files { 46 | for _, x := range *files { 47 | mh := textproto.MIMEHeader{} // make(textproto.MIMEHeader) 48 | szC := x.ContentType 49 | if "" == szC { 50 | szC = "text/plain; charset=UTF-8" 51 | } 52 | mh.Set("Content-Type", szC) 53 | mh.Set("Content-Disposition", fmt.Sprintf("form-data; name=\"%s\"; filename=\"%s\"", x.Name, x.FileName)) 54 | f01, err = body_writer.CreatePart(mh) 55 | if nil == err { 56 | io.Copy(f01, x.FileData) 57 | continue 58 | } 59 | if nil != err { 60 | log.Println(err) 61 | } 62 | } 63 | } 64 | body_writer.Close() 65 | bbData := body_buf.Bytes() 66 | r.DoGetWithClient4SetHd(c, szUrl, "POST", bytes.NewReader(bbData), fnCbk, func() map[string]string { 67 | m10 := setHeader() 68 | if nil == m10 { 69 | m10 = make(map[string]string) 70 | } 71 | m10["Content-Type"] = fmt.Sprintf("multipart/related; boundary=%s", body_writer.Boundary()) 72 | m10["Content-Length"] = fmt.Sprintf("%d", len(bbData)) 73 | //m10["Authorization"] = fmt.Sprintf("Bearer %s", accessToken) 74 | return m10 75 | }, true) 76 | } 77 | -------------------------------------------------------------------------------- /PipelineHttp.go: -------------------------------------------------------------------------------- 1 | package PipelineHttp 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "crypto/tls" 7 | "encoding/json" 8 | "fmt" 9 | "github.com/quic-go/quic-go/http3" 10 | "io" 11 | "log" 12 | "math/rand" 13 | "net" 14 | "net/http" 15 | "net/url" 16 | "regexp" 17 | "strings" 18 | "sync" 19 | "time" 20 | ) 21 | 22 | /* 23 | MaxConnsPerHost 控制单个Host的最大连接总数,该值默认是0，也就是不限制，连接池里的连接能用就用，不能用创建新连接 24 | MaxIdleConnsPerHost：优先设置这个，决定了对于单个Host需要维持的连接池大小。该值的合理确定，应该根据性能测试的结果调整。 25 | MaxIdleConns：客户端连接单个Host，不少于MaxIdleConnsPerHost大小，不然影响MaxIdleConnsPerHost控制的连接池；客户端连接 n 个Host，少于 n X MaxIdleConnsPerHost 会影响MaxIdleConnsPerHost控制的连接池（导致连接重建）。嫌麻烦，建议设置为0，不限制。 26 | MaxConnsPerHost：对于单个Host允许的最大连接数，包含IdleConns，所以一般大于等于MaxIdleConnsPerHost。设置为等于MaxIdleConnsPerHost，也就是尽可能复用连接池中的连接。另外设置过小，可能会导致并发下降，超过这个值会 block 请求，直到有空闲连接。（所以默认值是不限制的） 27 | */ 28 | type PipelineHttp struct { 29 | Timeout time.Duration `json:"timeout"` 30 | KeepAlive time.Duration `json:"keep_alive"` 31 | MaxIdleConns int `json:"max_idle_conns"` 32 | MaxIdleConnsPerHost int `json:"max_idle_conns_per_host"` 33 | MaxConnsPerHost int `json:"max_conns_per_host"` 34 | IdleConnTimeout time.Duration `json:"idle_conn_timeout"` 35 | TLSHandshakeTimeout time.Duration `json:"tls_handshake_timeout"` 36 | ExpectContinueTimeout time.Duration `json:"expect_continue_timeout"` 37 | ResponseHeaderTimeout time.Duration `json:"response_header_timeout"` 38 | Client *http.Client `json:"client"` 39 | Ctx context.Context `json:"ctx"` 40 | StopAll context.CancelFunc `json:"stop_all"` 41 | IsClosed bool `json:"is_closed"` 42 | ErrLimit int `json:"err_limit"` // 错误次数统计，失败就停止 43 | ErrCount int `json:"err_count"` // 错误次数统计，失败就停止 44 | SetHeader func() map[string]string `json:"set_header"` 45 | Buf *bytes.Buffer `json:"buf"` // http2 client framer message 46 | UseHttp2 bool `json:"use_http_2"` 47 | TestHttp bool `json:"test_http"` 48 | ReTry int `json:"re_try"` // 连接超时重试 49 | NoLimit bool `json:"no_limit"` // 不限制 50 | ver int 51 | } 52 | 53 | func NewPipelineHttp(args ...map[string]interface{}) *PipelineHttp { 54 | nTimeout := 5 * 60 * time.Second 55 | nIdle := 1000 56 | // Timeout 这个超时是设置整个请求的超时时间,一般应该设置比IdleConnTimeout更长 57 | // Timeout包含了拨号、TLS握手等时间,所以应该设置长一些。 58 | x1 := &PipelineHttp{ 59 | ver: 1, 60 | UseHttp2: false, 61 | NoLimit: false, 62 | TestHttp: false, 63 | Buf: &bytes.Buffer{}, 64 | Timeout: time.Duration(nTimeout*2) * time.Second, // 拨号、连接 65 | KeepAlive: time.Duration(nTimeout) * time.Second, // 默认值（当前为 15 秒）发送保持活动探测。 66 | MaxIdleConns: nIdle, // MaxIdleConns controls the maximum number of idle (keep-alive) connections across all hosts. Zero means no limit. 67 | IdleConnTimeout: nTimeout, // 不限制，秒 68 | ResponseHeaderTimeout: time.Duration(nTimeout) * time.Second, // response 69 | TLSHandshakeTimeout: time.Duration(nTimeout) * time.Second, // TLSHandshakeTimeout specifies the maximum amount of time waiting to wait for a TLS handshake. Zero means no timeout. 70 | ExpectContinueTimeout: 0, // 零表示没有超时，并导致正文立即发送，无需等待服务器批准 71 | MaxIdleConnsPerHost: nIdle, // MaxIdleConnsPerHost, if non-zero, controls the maximum idle (keep-alive) connections to keep per-host. If zero, DefaultMaxIdleConnsPerHost is used. 72 | MaxConnsPerHost: 0, // 控制单个Host的最大连接总数,该值默认是0，也就是不限制，连接池里的连接能用就用，不能用创建新连接 73 | ErrLimit: 10, // 相同目标，累计错误10次就退出 74 | ErrCount: 0, 75 | IsClosed: false, 76 | SetHeader: nil, 77 | ReTry: 3, 78 | } 79 | if x1.UseHttp2 { 80 | x1.Client = x1.GetClient4Http2() 81 | } else { 82 | x1.Client = x1.GetClient(nil) 83 | } 84 | x1.SetCtx(context.Background()) 85 | if nil != args && 0 < len(args) { 86 | for _, x := range args { 87 | if data, err := json.Marshal(x); nil == err { 88 | json.Unmarshal(data, x1) 89 | } 90 | } 91 | } 92 | //http.DefaultTransport.(*http.Transport).MaxIdleConns = x1.MaxIdleConns 93 | //http.DefaultTransport.(*http.Transport).MaxIdleConnsPerHost = x1.MaxIdleConnsPerHost 94 | return x1 95 | } 96 | func (r *PipelineHttp) SetNoLimit() { 97 | r.NoLimit = true 98 | } 99 | 100 | // https://cloud.tencent.com/developer/article/1529840 101 | // https://zhuanlan.zhihu.com/p/451642373 102 | func (r *PipelineHttp) Dial(ctx context.Context, network, addr string) (conn net.Conn, err error) { 103 | for i := 0; i < r.ReTry; i++ { 104 | conn, err = (&net.Dialer{ 105 | //Timeout: r.Timeout, // 不能打开，否则： dial tcp 127.0.0.1:1389: i/o timeout 106 | KeepAlive: r.KeepAlive, 107 | //Control: r.Control, 108 | DualStack: true, 109 | }).DialContext(ctx, network, addr) 110 | 111 | if err == nil { 112 | //conn.SetReadDeadline(time.Now().Add(r.Timeout))// 不能打开，否则： dial tcp 127.0.0.1:5900: i/o timeout 113 | //one := make([]byte, 0) 114 | //conn.SetReadDeadline(time.Now()) 115 | //if _, err := conn.Read(one); err != io.EOF { 116 | break 117 | //}else{ 118 | // conn.SetReadDeadline(time.Now().Add(r.Timeout * 10)) 119 | //} 120 | } 121 | } 122 | return conn, err 123 | } 124 | func (r *PipelineHttp) SetCtx(ctx context.Context) { 125 | r.Ctx, r.StopAll = context.WithCancel(ctx) 126 | } 127 | 128 | // https://github.com/golang/go/issues/23427 129 | // https://cloud.tencent.com/developer/article/1529840 130 | // https://romatic.net/post/go_net_errors/ 131 | // https://www.jianshu.com/p/2e5a7317be38 132 | func (r *PipelineHttp) GetTransport() http.RoundTripper { 133 | var tr http.RoundTripper = &http.Transport{ 134 | Proxy: http.ProxyFromEnvironment, 135 | DialContext: r.Dial, 136 | TLSClientConfig: &tls.Config{InsecureSkipVerify: true, MinVersion: tls.VersionTLS10, Renegotiation: tls.RenegotiateOnceAsClient}, 137 | //ForceAttemptHTTP2: true, // 不能加 138 | //MaxResponseHeaderBytes: 4096, //net/http default is 10Mb 139 | DisableKeepAlives: false, // false 才会复用连接 https://blog.csdn.net/qq_21514303/article/details/87794750 140 | MaxIdleConns: r.MaxIdleConns, // 是长连接在关闭之前，连接池对所有host的最大链接数量 141 | IdleConnTimeout: r.IdleConnTimeout, // 连接最大空闲时间，超过这个时间就会被关闭 142 | TLSHandshakeTimeout: r.TLSHandshakeTimeout, // 限制TLS握手使用的时间 143 | ExpectContinueTimeout: r.ExpectContinueTimeout, // 限制客户端在发送一个包含：100-continue的http报文头后，等待收到一个go-ahead响应报文所用的时间。在1.6中，此设置对HTTP/2无效。（在1.6.2中提供了一个特定的封装DefaultTransport） 144 | MaxIdleConnsPerHost: r.MaxIdleConnsPerHost, // 连接池对每个host的最大链接数量(MaxIdleConnsPerHost <= MaxIdleConns,如果客户端只需要访问一个host，那么最好将MaxIdleConnsPerHost与MaxIdleConns设置为相同，这样逻辑更加清晰) 145 | MaxConnsPerHost: r.MaxConnsPerHost, 146 | ResponseHeaderTimeout: r.ResponseHeaderTimeout, // 限制读取响应报文头使用的时间 147 | } 148 | return tr 149 | } 150 | 151 | func (r *PipelineHttp) GetClient(tr http.RoundTripper) *http.Client { 152 | if nil == tr { 153 | tr = r.GetTransport() 154 | } 155 | //c := &fasthttp.Client{ 156 | // ReadTimeout: readTimeout, 157 | // WriteTimeout: writeTimeout, 158 | // MaxIdleConnDuration: maxIdleConnDuration, 159 | // NoDefaultUserAgentHeader: true, // Don't send: User-Agent: fasthttp 160 | // DisableHeaderNamesNormalizing: true, // If you set the case on your headers correctly you can enable this 161 | // DisablePathNormalizing: true, 162 | // // increase DNS cache time to an hour instead of default minute 163 | // Dial: (&fasthttp.TCPDialer{ 164 | // Concurrency: 4096, 165 | // DNSCacheDuration: time.Hour, 166 | // }).Dial, 167 | //} 168 | c := &http.Client{ 169 | Transport: tr, 170 | //Timeout: r.Timeout, // 超时为零表示没有超时, context canceled (Client.Timeout exceeded while awaiting headers) 171 | CheckRedirect: func(req *http.Request, via []*http.Request) error { 172 | return http.ErrUseLastResponse /* 不进入重定向 */ 173 | }, 174 | } 175 | if o, ok := c.Transport.(*http.Transport); ok { 176 | o.Proxy = http.ProxyFromEnvironment 177 | } 178 | return c 179 | } 180 | 181 | func (r *PipelineHttp) DoGet(szUrl string, fnCbk func(resp *http.Response, err error, szU string)) { 182 | r.DoGetWithClient(nil, szUrl, "GET", nil, fnCbk) 183 | } 184 | 185 | /* 186 | */ 187 | func byPass403(szU string) string { 188 | a := []string{"/", "//", "/*", "/*/", "/.", "/./", "/./.", "?", "??", "???", "...;/", "/...;/", "%20/", "%09/"} 189 | //a1 := []string{"%2e/", "%2f/", "/%20"} // 最后一个/ 随机替换为 190 | //oU, _ := url.Parse(szU) 191 | switch szU[len(szU)-1:] { 192 | case "/": 193 | return szU + a[rand.Int()%len(a)] 194 | case "?": 195 | return fmt.Sprintf("%s%05f", szU, rand.Float64()) 196 | default: 197 | if -1 < strings.Index(szU, "?") { 198 | return szU + strings.Repeat("&", rand.Int()%10) + fmt.Sprintf("%05f", rand.Float64()) 199 | } 200 | //if rand.Int()%2 == 0 && -1 < strings.Index(oU.Path, "/") { 201 | // x := strings.Split(oU.Path, "/") 202 | // return oU.Scheme + "://" + oU.Host + strings.Join(x[0:len(x)-1]) 203 | //} 204 | return szU 205 | } 206 | } 207 | 208 | func (r *PipelineHttp) DoGetWithClient(client *http.Client, szUrl string, method string, postBody io.Reader, fnCbk func(resp *http.Response, err error, szU string)) { 209 | szUrl = byPass403(szUrl) 210 | oU, _ := url.Parse(szUrl) 211 | if nil == oU { 212 | return 213 | } 214 | szLip := "127.0.0.1" 215 | r.DoGetWithClient4SetHd(client, szUrl, method, postBody, fnCbk, func() map[string]string { 216 | // 403 bypass, https://zhuanlan.zhihu.com/p/642297652 217 | return map[string]string{ 218 | "Host": szLip, 219 | "Referer": szUrl, 220 | "X-Original-URL": oU.Path, 221 | "X-Rewrite-URL": oU.Path, 222 | "X-Originating-IP": szLip, 223 | "X-Remote-IP": szLip, 224 | "X-Client-IP": szLip, 225 | "X-Forwarded-For": szLip, 226 | "X-Forwared-Host": szLip, 227 | "X-Host": szLip, 228 | "X-Custom-IP-Authorization": szLip, 229 | } 230 | }, true) 231 | } 232 | 233 | func (r *PipelineHttp) DoGetWithClient4SetHdNoCloseBody(client *http.Client, szUrl string, method string, postBody io.Reader, fnCbk func(resp *http.Response, err error, szU string), setHd func() map[string]string) { 234 | r.DoGetWithClient4SetHd(client, szUrl, method, postBody, fnCbk, setHd, false) 235 | } 236 | 237 | func (r *PipelineHttp) CloseResponse(resp *http.Response) { 238 | if nil != resp { 239 | defer resp.Body.Close() 240 | io.Copy(io.Discard, resp.Body) 241 | } 242 | } 243 | 244 | // application/x-www-form-urlencoded 245 | // multipart/form-data 246 | // text/plain 247 | func (r *PipelineHttp) DoGetWithClient4SetHd(client *http.Client, szUrl string, method string, postBody io.Reader, fnCbk func(resp *http.Response, err error, szU string), setHd func() map[string]string, bCloseBody bool) { 248 | //r.testHttp2(szUrl) 249 | if client == nil { 250 | if nil != r.Client { 251 | client = r.Client 252 | } else { 253 | client = r.GetClient(nil) 254 | } 255 | } 256 | req, err := http.NewRequest(method, szUrl, postBody) 257 | if nil == err { 258 | if 1 == r.ver && !r.UseHttp2 && !r.TestHttp && strings.HasPrefix(szUrl, "https://") { 259 | req.Header.Set("Connection", "Upgrade, HTTP2-Settings") 260 | req.Header.Set("Upgrade", "h2c") 261 | req.Header.Set("HTTP2-Settings", "AAMAAABkAARAAAAAAAIAAAAA") 262 | } else { 263 | req.Header.Set("Connection", "keep-alive") 264 | } 265 | //req.Close = true // 避免 Read返回EOF error 266 | var fnShk func() map[string]string 267 | if nil != setHd { 268 | fnShk = setHd 269 | } else { 270 | fnShk = r.SetHeader 271 | } 272 | if nil != fnShk { 273 | m1 := fnShk() 274 | for k09, v09 := range m1 { 275 | req.Header.Set(k09, v09) 276 | } 277 | } 278 | } else { 279 | log.Println("http.NewRequest is error ", err) 280 | return 281 | } 282 | n1 := client.Timeout 283 | if 0 == n1 { 284 | n1 = 50 285 | } 286 | if req.Header.Get("User-Agent") == "" { 287 | req.Header.Set("User-Agent", fmt.Sprintf("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.%05f.159 Safari/537.36", rand.Float64())) 288 | } 289 | //if 0 < r.Timeout { 290 | // ctx, cc := context.WithTimeout(r.Ctx, n1*r.Timeout) 291 | // defer cc() 292 | // req = req.WithContext(ctx) 293 | //} else { 294 | // req = req.WithContext(r.Ctx) // context canceled 295 | //} 296 | 297 | resp, err := client.Do(req) 298 | if bCloseBody && resp != nil { 299 | defer r.CloseResponse(resp) // resp 可能为 nil，不能读取 Body 300 | } 301 | if nil != err { 302 | r.ErrCount++ 303 | } 304 | if (!NoLimit && !r.NoLimit) && r.ErrCount >= r.ErrLimit { 305 | log.Printf("PipelineHttp %d >= %d not close\n", r.ErrCount, r.ErrLimit) 306 | r.Close() 307 | return 308 | } 309 | if nil != err && rNohost.MatchString(err.Error()) { 310 | log.Println(szUrl, err) 311 | r.Close() 312 | return 313 | } 314 | if !r.UseHttp2 && strings.HasPrefix(szUrl, "https://") && nil != resp && 200 != resp.StatusCode { 315 | r.UseHttp2 = true 316 | if a1 := resp.Header["Alt-Svc"]; 0 < len(a1) && strings.Contains(a1[0], "h3=\"") || strings.HasPrefix(resp.Proto, "HTTP/3") { 317 | r.Client = r.GetClient4Http3() 318 | } else if resp.StatusCode == http.StatusSwitchingProtocols { 319 | r.Client = r.GetRawClient4Http2() 320 | } 321 | oU7, _ := url.Parse(szUrl) 322 | szUrl09 := "https://" + oU7.Host + strings.Split(szUrl, oU7.Host)[1] 323 | r.ErrLimit = 99999999 324 | r.CloseResponse(resp) 325 | r.DoGetWithClient4SetHd(r.Client, szUrl09, method, postBody, fnCbk, setHd, bCloseBody) 326 | return 327 | } 328 | fnCbk(resp, err, szUrl) 329 | } 330 | 331 | var ( 332 | rNohost = regexp.MustCompile(`.*dial tcp: [^:]+: no such host.*`) 333 | NoLimit = false 334 | ) 335 | 336 | func (r *PipelineHttp) Close() { 337 | r.IsClosed = true 338 | if r.Client != nil && r.Client.Transport != nil { 339 | if tr, ok := r.Client.Transport.(*http3.RoundTripper); ok { 340 | tr.Close() 341 | } else if tr, ok := r.Client.Transport.(*http.Transport); ok { 342 | tr.CloseIdleConnections() 343 | } 344 | } 345 | r.StopAll() 346 | r.Client = nil 347 | } 348 | 349 | func (r *PipelineHttp) DoDirs4Http2(szUrl string, dirs []string, nThread int, fnCbk func(resp *http.Response, err error, szU string)) { 350 | r.UseHttp2 = true 351 | r.doDirsPrivate(szUrl, dirs, nThread, fnCbk) 352 | } 353 | 354 | func (r *PipelineHttp) DoDirs(szUrl string, dirs []string, nThread int, fnCbk func(resp *http.Response, err error, szU string)) { 355 | r.doDirsPrivate(szUrl, dirs, nThread, fnCbk) 356 | } 357 | 358 | func (r *PipelineHttp) testHttp2(szUrl001 string) { 359 | if !r.UseHttp2 || !r.TestHttp { 360 | r.TestHttp = true 361 | r.UseHttp2 = true 362 | c1 := r.GetRawClient4Http2() 363 | oU7, _ := url.Parse(szUrl001) 364 | if "" == oU7.Path { 365 | oU7.Path = "/" 366 | } 367 | szUrl09 := "https://" + oU7.Host + oU7.Path 368 | r.DoGetWithClient(c1, szUrl09, "GET", nil, func(resp *http.Response, err error, szU string) { 369 | if nil != resp { 370 | if resp.StatusCode == http.StatusSwitchingProtocols { 371 | r.CloseResponse(resp) 372 | if nil != r.Client { 373 | r.Client.CloseIdleConnections() 374 | } 375 | if strings.HasPrefix(resp.Proto, "HTTP/2") { 376 | r.Client = c1 377 | } 378 | } else if a1 := resp.Header["Alt-Svc"]; 0 < len(a1) && strings.Contains(a1[0], "h3=\"") { 379 | r.Client = r.GetClient4Http3() 380 | } else if resp.Proto == "HTTP/2.0" { 381 | r.UseHttp2 = true 382 | r.Client = c1 383 | } 384 | r.ErrLimit = 99999999 385 | } else { 386 | r.UseHttp2 = false 387 | } 388 | }) 389 | } 390 | } 391 | 392 | // more see cmd/main.go 393 | func (r *PipelineHttp) doDirsPrivate(szUrl string, dirs []string, nThread int, fnCbk func(resp *http.Response, err error, szU string)) { 394 | c02 := make(chan struct{}, nThread) 395 | defer close(c02) 396 | oUrl, err := url.Parse(szUrl) 397 | if nil != err { 398 | log.Printf("url.Parse is error: %v %s", err, szUrl) 399 | return 400 | } 401 | if "" == oUrl.Scheme { 402 | oUrl.Scheme = "http" 403 | } 404 | szUrl = oUrl.Scheme + "://" + oUrl.Host + oUrl.Path 405 | var wg sync.WaitGroup 406 | var client *http.Client 407 | r.testHttp2(szUrl) 408 | if r.UseHttp2 { 409 | client = r.GetClient4Http2() 410 | } else { 411 | client = r.GetClient(nil) 412 | client = r.Client 413 | } 414 | for _, j := range dirs { 415 | if r.IsClosed { 416 | return 417 | } 418 | select { 419 | case <-r.Ctx.Done(): 420 | return 421 | default: 422 | { 423 | c02 <- struct{}{} 424 | wg.Add(1) 425 | go func(s2 string) { 426 | defer func() { 427 | <-c02 428 | wg.Done() 429 | }() 430 | select { 431 | case <-r.Ctx.Done(): 432 | return 433 | default: 434 | { 435 | s2 = strings.TrimSpace(s2) 436 | if !strings.HasPrefix(s2, "/") { 437 | s2 = "/" + s2 438 | } 439 | szUrl001 := szUrl + s2 440 | //fmt.Printf("%s\033[2K\r", szUrl001) 441 | //fmt.Printf(".") 442 | r.DoGetWithClient(client, szUrl001, "GET", nil, fnCbk) 443 | //r.DoGet(szUrl001, fnCbk) 444 | return 445 | } 446 | } 447 | }(j) 448 | continue 449 | } 450 | } 451 | } 452 | wg.Wait() 453 | } 454 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PipelineHttp 2 | 3 | # What features 4 | - auto support HTTP/2.0 5 | - support HTTP/3.0 6 | 7 | # How install cmd 8 | ``` 9 | go get -u ./... 10 | go build -o ppHttp cmd/main.go 11 | ln -s $PWD/ppHttp $HOME/go/bin/ppHttp 12 | # go install github.com/hktalent/PipelineHttp/cmd/@latest 13 | ``` 14 | # How use 15 | ``` 16 | ppHttp https://xx1.com https://b1.xx2.com 17 | ``` 18 | 19 | # Test speed 20 | - http 2.0 18s req 30612 * 2(host) times 21 | -------------------------------------------------------------------------------- /cmd/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | _ "embed" 5 | xxx "github.com/hktalent/PipelineHttp" 6 | "log" 7 | "net/http" 8 | "net/url" 9 | "os" 10 | "strings" 11 | "sync" 12 | "time" 13 | ) 14 | 15 | //go:embed testUrl.txt 16 | var szPath string 17 | 18 | //go:embed testUrls.txt 19 | var szUrls string 20 | 21 | /* 22 | 1、遵循不重复造轮子的原则 23 | 为什么有了 gobuster 还要有 speeDir（speed dir） 24 | 25 | go build -o ppHttp cmd/main.go 26 | cp $HOME/MyWork/scan4all/brute/dicts/filedic.txt $HOME/MyWork/PipelineHttp/test2/testUrl.txt 27 | cat $HOME/MyWork/mybugbounty/bak/fingerprint.json|grep -v '"delete":true'|jq ".probeList[].url"|sort -u|sed 's/"//g'>$HOME/MyWork/PipelineHttp/cmd/testUrl.txt 28 | */ 29 | func main() { 30 | os.Args = []string{"", "/Users/51pwn/MyWork/scan4all/brute/dicts/filedic.txt"} 31 | var a []string 32 | if 1 < len(os.Args) { 33 | if data, err := os.ReadFile(os.Args[1]); nil == err { 34 | a = strings.Split(string(data), "\n") 35 | } else { 36 | a = os.Args[1:] 37 | } 38 | } else { 39 | a = strings.Split(szUrls, "\n") 40 | } 41 | c01 := make(chan struct{}, 4) 42 | x := strings.Split(szPath, "\n") 43 | var wg sync.WaitGroup 44 | n009 := time.Now().UnixMilli() 45 | for _, i := range a { 46 | if "" == i { 47 | continue 48 | } 49 | c01 <- struct{}{} 50 | oUrl, err := url.Parse(i) 51 | if nil != err { 52 | continue 53 | } 54 | if "" == oUrl.Scheme { 55 | oUrl.Scheme = "http" 56 | } 57 | i = oUrl.Scheme + "://" + oUrl.Host 58 | wg.Add(1) 59 | go func(s1 string) { 60 | defer func() { 61 | <-c01 62 | wg.Done() 63 | }() 64 | x1 := xxx.NewPipelineHttp() 65 | x1.ErrLimit = 9999999 66 | defer x1.Close() 67 | 68 | log.Println("start ", s1) 69 | x1.Client = x1.GetClient4Http2() 70 | //x1.DoDirs4Http2(s1, x, 128, func(resp *http.Response, err error, szU string) { 71 | x1.DoDirs(s1, x, 16, func(resp *http.Response, err error, szU string) { // auto cmd http2.0 and use http2.0 72 | if nil != err { 73 | log.Println(err) 74 | return 75 | } 76 | 77 | if nil != resp && 200 == resp.StatusCode { 78 | log.Printf("\r%d %s %s\n", resp.StatusCode, resp.Proto, szU) 79 | } else if nil != resp { 80 | //log.Printf("%d %s %s", resp.StatusCode, resp.Proto, szU) 81 | } 82 | }) 83 | //time.Sleep(time.Second * 5) 84 | x1.Close() 85 | }(i) 86 | } 87 | wg.Wait() 88 | 89 | log.Printf("use time: %d/%d sec\n", len(x), (time.Now().UnixMilli()-n009)/1000) 90 | } 91 | -------------------------------------------------------------------------------- /cmd/testUrls.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GhostTroops/PipelineHttp/371c458fea698af618ac9cc2745422bd55ed0a9b/cmd/testUrls.txt -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/hktalent/PipelineHttp 2 | 3 | go 1.21 4 | 5 | require ( 6 | github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf 7 | github.com/json-iterator/go v1.1.12 8 | github.com/quic-go/quic-go v0.44.0 9 | golang.org/x/net v0.26.0 10 | ) 11 | 12 | require ( 13 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect 14 | github.com/go-task/slim-sprig/v3 v3.0.0 // indirect 15 | github.com/google/pprof v0.0.0-20240528025155-186aa0362fba // indirect 16 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 17 | github.com/modern-go/reflect2 v1.0.2 // indirect 18 | github.com/onsi/ginkgo/v2 v2.19.0 // indirect 19 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect 20 | github.com/quic-go/qpack v0.4.0 // indirect 21 | go.uber.org/mock v0.4.0 // indirect 22 | golang.org/x/crypto v0.24.0 // indirect 23 | golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 // indirect 24 | golang.org/x/mod v0.18.0 // indirect 25 | golang.org/x/sys v0.21.0 // indirect 26 | golang.org/x/text v0.16.0 // indirect 27 | golang.org/x/tools v0.22.0 // indirect 28 | ) 29 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 2 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= 4 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 5 | github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= 6 | github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= 7 | github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= 8 | github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= 9 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 10 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 11 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 12 | github.com/google/pprof v0.0.0-20240528025155-186aa0362fba h1:ql1qNgCyOB7iAEk8JTNM+zJrgIbnyCKX/wdlyPufP5g= 13 | github.com/google/pprof v0.0.0-20240528025155-186aa0362fba/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= 14 | github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf h1:umfGUaWdFP2s6457fz1+xXYIWDxdGc7HdkLS9aJ1skk= 15 | github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf/go.mod h1:V99KdStnMHZsvVOwIvhfcUzYgYkRZeQWUtumtL+SKxA= 16 | github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= 17 | github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= 18 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 19 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 20 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 21 | github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= 22 | github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= 23 | github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA= 24 | github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= 25 | github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= 26 | github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= 27 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 28 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= 29 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 30 | github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= 31 | github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= 32 | github.com/quic-go/quic-go v0.44.0 h1:So5wOr7jyO4vzL2sd8/pD9Kesciv91zSk8BoFngItQ0= 33 | github.com/quic-go/quic-go v0.44.0/go.mod h1:z4cx/9Ny9UtGITIPzmPTXh1ULfOyWh4qGQlpnPcWmek= 34 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 35 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 36 | github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= 37 | github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= 38 | go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= 39 | go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= 40 | golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= 41 | golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= 42 | golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 h1:LoYXNGAShUG3m/ehNk4iFctuhGX/+R1ZpfJ4/ia80JM= 43 | golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= 44 | golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= 45 | golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= 46 | golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= 47 | golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= 48 | golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= 49 | golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 50 | golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= 51 | golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 52 | golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= 53 | golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= 54 | golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= 55 | golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= 56 | golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= 57 | golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= 58 | google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= 59 | google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= 60 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 61 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 62 | -------------------------------------------------------------------------------- /http2Imp.go: -------------------------------------------------------------------------------- 1 | package PipelineHttp 2 | 3 | import ( 4 | "context" 5 | "crypto/tls" 6 | "fmt" 7 | "golang.org/x/net/http2" 8 | "io" 9 | "net" 10 | "net/http" 11 | "net/url" 12 | "os" 13 | "time" 14 | ) 15 | 16 | // for http2 17 | type tConn struct { 18 | net.Conn 19 | T io.Writer // receives everything that is read from Conn 20 | } 21 | 22 | func (w *tConn) Read(b []byte) (n int, err error) { 23 | n, err = w.Conn.Read(b) 24 | w.T.Write(b) 25 | return 26 | } 27 | 28 | func (r *PipelineHttp) GetRawClient4Http2() *http.Client { 29 | return r.GetClient(r.GetTransport4http2()) 30 | } 31 | 32 | // get http2 client 33 | func (r *PipelineHttp) GetClient4Http2() *http.Client { 34 | if nil == r.Client { 35 | r.Client = r.GetRawClient4Http2() 36 | r.UseHttp2 = r.Client != nil 37 | r.ver = 2 38 | } 39 | if o, ok := r.Client.Transport.(*http.Transport); ok { 40 | o.Proxy = http.ProxyFromEnvironment 41 | } 42 | return r.Client 43 | } 44 | 45 | /* 46 | Upgrade: h2c 47 | */ 48 | func (r *PipelineHttp) DoUrl4Http24Frame(szUrl, szMethod string, framerCbk func(*http2.Frame, *http.Response, *error) bool, w io.Writer) { 49 | r.UseHttp2 = true 50 | client := r.GetClient4Http2() 51 | res, err := client.Get(szUrl) 52 | if err != nil { 53 | framerCbk(nil, res, &err) 54 | return 55 | } 56 | //io.Copy(ioutil.Discard, res.Body) 57 | defer res.Body.Close() // res.Write(os.Stdout) 58 | r.httpDataIO(func(frame *http2.Frame, e *error) bool { 59 | return framerCbk(frame, res, e) 60 | }, w) 61 | } 62 | 63 | //func (r *PipelineHttp) hdec() { 64 | // hpack.NewDecoder(uint32(4<<10), func(hf hpack.HeaderField) { 65 | // if hf.Name == name && hf.Value == value { 66 | // matched = true 67 | // } 68 | // }) 69 | //} 70 | 71 | // dialT returns a connection that writes everything that is read to w. 72 | func (r *PipelineHttp) dialT(w io.Writer) func(ctx context.Context, network, addr string, cfg *tls.Config) (net.Conn, error) { 73 | return func(ctx context.Context, network, addr string, cfg *tls.Config) (net.Conn, error) { 74 | if s := os.Getenv("HTTPS_PROXY"); "" != s { 75 | if proxyURL, err := url.Parse(s); err == nil { 76 | transport := &http.Transport{ 77 | Proxy: http.ProxyURL(proxyURL), 78 | TLSClientConfig: cfg, 79 | } 80 | client := &http.Client{Transport: transport} 81 | req := &http.Request{ 82 | URL: &url.URL{Scheme: "https", Host: addr}, 83 | Method: "CONNECT", 84 | } 85 | resp, err := client.Do(req) 86 | if err != nil { 87 | return nil, err 88 | } 89 | if resp.StatusCode != 200 { 90 | return nil, fmt.Errorf("non-200 status code: %d", resp.StatusCode) 91 | } 92 | hijacker, ok := resp.Body.(http.Hijacker) 93 | if !ok { 94 | return nil, fmt.Errorf("response body is not a http.Hijacker") 95 | } 96 | conn, _, err := hijacker.Hijack() 97 | return &tConn{conn, w}, err 98 | } 99 | } 100 | 101 | conn, err := tls.Dial(network, addr, cfg) 102 | return &tConn{conn, w}, err 103 | } 104 | } 105 | func (r *PipelineHttp) httpDataIO(framerCbk func(*http2.Frame, *error) bool, w io.Writer) { 106 | framer := http2.NewFramer(w, r.Buf) 107 | for { 108 | f, err := framer.ReadFrame() 109 | if framerCbk(&f, &err) { 110 | continue 111 | } else { 112 | break 113 | } 114 | //if err == io.EOF || err == io.ErrUnexpectedEOF { 115 | // break 116 | //} 117 | //if nil != err { 118 | // log.Println(err, framer.ErrorDetail()) 119 | //} 120 | //switch err.(type) { 121 | //case nil: 122 | // log.Println(f) 123 | //case http2.ConnectionError: 124 | // // Ignore. There will be many errors of type "PROTOCOL_ERROR, DATA 125 | // // frame with stream ID 0". Presumably we are abusing the framer. 126 | //default: 127 | // log.Println(err, framer.ErrorDetail()) 128 | //} 129 | } 130 | } 131 | 132 | // 传输对象配置 133 | func (r *PipelineHttp) GetTransport4http2() http.RoundTripper { 134 | if r.UseHttp2 { 135 | var tr http.RoundTripper = &http2.Transport{ 136 | //TLSClientConfig: r.tlsConfig(), 137 | TLSClientConfig: &tls.Config{ 138 | InsecureSkipVerify: true, //跳过证书验证 139 | }, 140 | DialTLSContext: r.dialT(r.Buf), 141 | //DialTLS: r.dialT(r.Buf), 142 | DisableCompression: false, 143 | AllowHTTP: true, 144 | ReadIdleTimeout: 50 * time.Second, 145 | PingTimeout: 15 * time.Second, 146 | WriteByteTimeout: 600 * time.Second, 147 | StrictMaxConcurrentStreams: true, // true 则全局复用，client时建议全局复用，false则为每一个创建一个链接 148 | //DialTLS: func(netw, addr string, cfg *tls.Config) (net.Conn, error) { 149 | // return net.Dial(netw, addr) 150 | //}, 151 | } 152 | if x, ok := tr.(*http.Transport); ok { 153 | http2.ConfigureTransport(x) 154 | } 155 | return tr 156 | } 157 | return nil 158 | } 159 | 160 | //func (r *PipelineHttp) tlsConfig() *tls.Config { 161 | // crt, err := ioutil.ReadFile("./cert/public.crt") 162 | // if err != nil { 163 | // log.Fatal(err) 164 | // } 165 | // 166 | // rootCAs := x509.NewCertPool() 167 | // rootCAs.AppendCertsFromPEM(crt) 168 | // 169 | // return &tls.Config{ 170 | // RootCAs: rootCAs, 171 | // InsecureSkipVerify: false, 172 | // ServerName: "localhost", 173 | // } 174 | //} 175 | -------------------------------------------------------------------------------- /http3client.go: -------------------------------------------------------------------------------- 1 | package PipelineHttp 2 | 3 | import ( 4 | "crypto/tls" 5 | "crypto/x509" 6 | "github.com/quic-go/quic-go/http3" 7 | "log" 8 | "net/http" 9 | ) 10 | 11 | // 在线测试http3 https://geekflare.com/tools/http3-test 12 | func (r *PipelineHttp) GetTransport4Http3() http.RoundTripper { 13 | pool, err := x509.SystemCertPool() 14 | if nil != err { 15 | log.Println(err) 16 | return nil 17 | } 18 | var tr = &http3.RoundTripper{ 19 | DisableCompression: false, 20 | TLSClientConfig: &tls.Config{ 21 | RootCAs: pool, 22 | InsecureSkipVerify: true, 23 | }, 24 | } 25 | return tr 26 | } 27 | 28 | // get http3 client 29 | func (r *PipelineHttp) GetClient4Http3() *http.Client { 30 | r.Client = r.GetClient(r.GetTransport4Http3()) 31 | r.ver = 3 32 | return r.Client 33 | } 34 | -------------------------------------------------------------------------------- /portscan/Main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | xxx "github.com/hktalent/PipelineHttp" 6 | "net/http" 7 | ) 8 | 9 | var ppe = xxx.NewPipelineHttp() 10 | 11 | func main() { 12 | c2 := ppe.GetClient4Http2() 13 | var c1 = make(chan struct{}, 8) 14 | for i := 0; i < 65536; i++ { 15 | c1 <- struct{}{} 16 | fmt.Printf("start %d\r", i) 17 | go func(n int) { 18 | defer func() { 19 | <-c1 20 | ppe.ErrCount = 0 21 | ppe.ErrLimit = 100000000 22 | }() 23 | ppe.DoGetWithClient(c2, fmt.Sprintf("https://www.baidu.com:%d", n), "GET", nil, func(resp *http.Response, err error, szU string) { 24 | if nil != err { 25 | return 26 | } 27 | if nil != resp { 28 | fmt.Printf("%d %s %s\n", resp.StatusCode, resp.Proto, szU) 29 | } 30 | }) 31 | }(i) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /test2/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "crypto/sha1" 5 | _ "embed" 6 | "encoding/hex" 7 | "fmt" 8 | "github.com/hbakhtiyor/strsim" 9 | xxx "github.com/hktalent/PipelineHttp" 10 | jsoniter "github.com/json-iterator/go" 11 | "io" 12 | "log" 13 | "net/http" 14 | "net/url" 15 | "os" 16 | "strings" 17 | "sync" 18 | ) 19 | 20 | //go:embed testUrl.txt 21 | var szPath string 22 | 23 | //go:embed testUrls.txt 24 | var szUrls string 25 | 26 | // 获取 Sha1 27 | func GetSha1(a ...interface{}) string { 28 | h := sha1.New() 29 | //if isMap(a[0]) { // map嵌套map 确保顺序，相同数据map得到相同的sha1 30 | if data, err := jsoniter.ConfigCompatibleWithStandardLibrary.Marshal(a); nil == err { 31 | h.Write(data) 32 | } else { 33 | for _, x := range a { 34 | h.Write([]byte(fmt.Sprintf("%v", x))) 35 | } 36 | } 37 | bs := h.Sum(nil) 38 | return hex.EncodeToString(bs) // fmt.Sprintf("%x", bs) 39 | } 40 | 41 | /* 42 | cp -f $HOME/MyWork/scan4all/brute/dicts/filedic.txt $HOME/MyWork/PipelineHttp/test2/testUrl.txt 43 | cat $HOME/MyWork/mybugbounty/bak/fingerprint.json|grep -v '"delete":true'|jq ".probeList[].url"|sort -u|sed 's/"//g'>$HOME/MyWork/PipelineHttp/cmd/testUrl.txt 44 | */ 45 | func main() { 46 | a := strings.Split(szUrls, "\n") 47 | c01 := make(chan struct{}, 8) 48 | x := strings.Split(szPath, "\n") 49 | var wg sync.WaitGroup 50 | //opts := retryablehttp.DefaultOptionsSpraying 51 | // opts := retryablehttp.DefaultOptionsSingle // use single options for single host 52 | //client := retryablehttp.NewClient(opts) 53 | fok, err := os.OpenFile("url_ok.txt", os.O_APPEND|os.O_CREATE, os.ModePerm) 54 | if nil != err { 55 | return 56 | } 57 | defer fok.Close() 58 | for _, i := range a { 59 | c01 <- struct{}{} 60 | oUrl, err := url.Parse(i) 61 | if nil != err { 62 | continue 63 | } 64 | if "" == oUrl.Scheme { 65 | oUrl.Scheme = "http" 66 | } 67 | i = oUrl.Scheme + "://" + oUrl.Host + oUrl.Path 68 | wg.Add(1) 69 | go func(s1 string) { 70 | defer func() { 71 | <-c01 72 | wg.Done() 73 | }() 74 | x1 := xxx.NewPipelineHttp() 75 | defer x1.Close() 76 | x1.ErrCount = 0 77 | x1.ErrLimit = len(x) + 1 78 | //x1.Client = x1.GetClient4Http2() 79 | //x1.UseHttp2 = false 80 | //x1.TestHttp = false 81 | var szLastData string 82 | x1.DoDirs4Http2(s1, x, 8, func(resp *http.Response, err error, szU string) { 83 | if err == nil { 84 | defer resp.Body.Close() 85 | if nil != resp && 200 == resp.StatusCode { 86 | if data, err := io.ReadAll(resp.Body); nil == err { 87 | szCur := string(data) 88 | var nF float64 = 0 89 | if 0 < len(szLastData) { 90 | nF = strsim.Compare(szLastData, szCur) 91 | if 0.89 < nF { 92 | return 93 | } 94 | } 95 | szLastData = szCur 96 | szT := fmt.Sprintf("%d %s %d:%s %02f %s\n", resp.StatusCode, resp.Proto, len(data), GetSha1(data), nF, szU) 97 | fok.WriteString(szT) 98 | log.Printf("\n%s", szT) 99 | } 100 | } else { 101 | fmt.Printf("%d %s %s\r", resp.StatusCode, resp.Proto, szU) 102 | } 103 | } 104 | }) 105 | //log.Println("start ", s1) 106 | //for _, j := range x { 107 | // resp, err := client.Get(s1 + j) 108 | // if err == nil { 109 | // defer resp.Body.Close() 110 | // if nil != resp && 200 == resp.StatusCode { 111 | // log.Printf("%d %s %s\n", resp.StatusCode, resp.Proto, s1+j) 112 | // } 113 | // } 114 | //} 115 | // 116 | ////x1.DoDirs4Http2(s1, x, 128, func(resp *http.Response, err error, szU string) { 117 | //x1.DoDirs(s1, x, 128, func(resp *http.Response, err error, szU string) { 118 | // if nil != resp && 200 == resp.StatusCode { 119 | // log.Printf("%d %s %s", resp.StatusCode, resp.Proto, szU) 120 | // } else { 121 | // //log.Printf("%d %s", resp.StatusCode, szU) 122 | // } 123 | //}) 124 | //time.Sleep(time.Second * 5) 125 | //x1.Close() 126 | 127 | }(i) 128 | } 129 | wg.Wait() 130 | } 131 | -------------------------------------------------------------------------------- /test2/testUrls.txt: -------------------------------------------------------------------------------- 1 | https://www.paypal.com/webapps/mpp 2 | https://www.paypal.com/webapps 3 | https://www.paypal.com/us/cgi-bin 4 | https://www.paypal.com/cn/webapps 5 | https://www.paypal.com/c2/webapps/mpp 6 | https://cms.paypal.com/cgi-bin 7 | https://www.paypal.com/c2 --------------------------------------------------------------------------------