├── 123 ├── README.md ├── .DS_Store └── httpx.patch /123: -------------------------------------------------------------------------------- 1 | 1 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # httpx_patch 2 | ## 3 | -------------------------------------------------------------------------------- /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/githubmaidou/httpx_patch/main/.DS_Store -------------------------------------------------------------------------------- /httpx.patch: -------------------------------------------------------------------------------- 1 | diff -uNr -x .idea httpx-1.2.1/common/fingerprint/data/all.json httpx-1.2.1-m/common/fingerprint/data/all.json 2 | --- httpx-1.2.1/common/fingerprint/data/all.json 1970-01-01 08:00:00.000000000 +0800 3 | +++ httpx-1.2.1-m/common/fingerprint/data/all.json 2022-06-08 16:23:31.000000000 +0800 4 | @@ -0,0 +1 @@ 5 | +[] 6 | \ No newline at end of file 7 | diff -uNr -x .idea httpx-1.2.1/common/fingerprint/fingerprint.go httpx-1.2.1-m/common/fingerprint/fingerprint.go 8 | --- httpx-1.2.1/common/fingerprint/fingerprint.go 1970-01-01 08:00:00.000000000 +0800 9 | +++ httpx-1.2.1-m/common/fingerprint/fingerprint.go 2022-06-08 15:27:43.000000000 +0800 10 | @@ -0,0 +1,66 @@ 11 | +package fingerprint 12 | + 13 | +import ( 14 | + _ "embed" 15 | + "encoding/json" 16 | + "github.com/boy-hack/govaluate" 17 | + "github.com/projectdiscovery/httpx/common/httpx" 18 | +) 19 | + 20 | +type FingerPrint struct { 21 | + RuleId int64 `json:"rule_id"` 22 | + Product string `json:"product"` 23 | + Company string `json:"company"` 24 | + Condition string `json:"condition"` 25 | +} 26 | +type FingerPrints []FingerPrint 27 | + 28 | +//go:embed data/all.json 29 | +var FofaJson []byte 30 | + 31 | +func InitFingerPrints() (FingerPrints, error) { 32 | + //data, err := ioutil.ReadFile(filename) 33 | + //if err != nil { 34 | + // return nil, err 35 | + //} 36 | + var fingers FingerPrints 37 | + err := json.Unmarshal(FofaJson, &fingers) 38 | + if err != nil { 39 | + return nil, err 40 | + } 41 | + return fingers, err 42 | +} 43 | + 44 | +func (f *FingerPrint) Matcher(response *httpx.Response) (bool, error) { 45 | + expString := f.Condition 46 | + expression, err := govaluate.NewEvaluableExpressionWithFunctions(expString, HelperFunctions(response)) 47 | + if err != nil { 48 | + return false, err 49 | + } 50 | + paramters := make(map[string]interface{}) 51 | + paramters["title"] = response.Title 52 | + paramters["server"] = response.GetHeader("server") 53 | + paramters["protocol"] = "http" 54 | + 55 | + result, err := expression.Evaluate(paramters) 56 | + if err != nil { 57 | + return false, err 58 | + } 59 | + t := result.(bool) 60 | + return t, err 61 | +} 62 | + 63 | +func (f *FingerPrints) Matcher(response *httpx.Response) ([]string, error) { 64 | + ret := make([]string, 0) 65 | + for _, item := range *f { 66 | + v, err := item.Matcher(response) 67 | + if err != nil { 68 | + return nil, err 69 | + } 70 | + if v { 71 | + n := item.Product 72 | + ret = append(ret, n) 73 | + } 74 | + } 75 | + return ret, nil 76 | +} 77 | diff -uNr -x .idea httpx-1.2.1/common/fingerprint/generators.go httpx-1.2.1-m/common/fingerprint/generators.go 78 | --- httpx-1.2.1/common/fingerprint/generators.go 1970-01-01 08:00:00.000000000 +0800 79 | +++ httpx-1.2.1-m/common/fingerprint/generators.go 2022-06-08 15:27:58.000000000 +0800 80 | @@ -0,0 +1,62 @@ 81 | +package fingerprint 82 | + 83 | +import ( 84 | + "fmt" 85 | + "github.com/boy-hack/govaluate" 86 | + "github.com/projectdiscovery/httpx/common/httpx" 87 | + "strings" 88 | +) 89 | + 90 | +func toString(v interface{}) string { 91 | + return fmt.Sprint(v) 92 | +} 93 | +func toInt(v interface{}) int { 94 | + return int(v.(float64)) 95 | +} 96 | + 97 | +// HelperFunctions contains the dsl functions 98 | +func HelperFunctions(resp *httpx.Response) (functions map[string]govaluate.ExpressionFunction) { 99 | + functions = make(map[string]govaluate.ExpressionFunction) 100 | + 101 | + functions["title_contains"] = func(args ...interface{}) (interface{}, error) { 102 | + pattern := strings.ToLower(toString(args[0])) 103 | + title := strings.ToLower(resp.Title) 104 | + return strings.Index(title, pattern) != -1, nil 105 | + } 106 | + 107 | + functions["body_contains"] = func(args ...interface{}) (interface{}, error) { 108 | + pattern := strings.ToLower(toString(args[0])) 109 | + data := strings.ToLower(resp.DataStr) 110 | + return strings.Index(data, pattern) != -1, nil 111 | + } 112 | + 113 | + functions["protocol_contains"] = func(args ...interface{}) (interface{}, error) { 114 | + return false, nil 115 | + } 116 | + 117 | + functions["banner_contains"] = func(args ...interface{}) (interface{}, error) { 118 | + return false, nil 119 | + } 120 | + 121 | + functions["header_contains"] = func(args ...interface{}) (interface{}, error) { 122 | + pattern := strings.ToLower(toString(args[0])) 123 | + data := strings.ToLower(resp.HeaderStr) 124 | + return strings.Index(data, pattern) != -1, nil 125 | + } 126 | + 127 | + functions["server_contains"] = func(args ...interface{}) (interface{}, error) { 128 | + pattern := strings.ToLower(toString(args[0])) 129 | + server := resp.GetHeader("server") 130 | + return strings.Index(server, pattern) != -1, nil 131 | + } 132 | + 133 | + functions["cert_contains"] = func(args ...interface{}) (interface{}, error) { 134 | + return false, nil 135 | + } 136 | + 137 | + functions["port_contains"] = func(args ...interface{}) (interface{}, error) { 138 | + return false, nil 139 | + } 140 | + 141 | + return functions 142 | +} 143 | diff -uNr -x .idea httpx-1.2.1/common/httpx/httpx.go httpx-1.2.1-m/common/httpx/httpx.go 144 | --- httpx-1.2.1/common/httpx/httpx.go 2022-04-08 21:58:49.000000000 +0800 145 | +++ httpx-1.2.1-m/common/httpx/httpx.go 2022-06-08 15:36:18.000000000 +0800 146 | @@ -170,6 +170,12 @@ 147 | 148 | resp.Headers = httpresp.Header.Clone() 149 | 150 | + // HeaderStr 151 | + resp.HeaderStr = "" 152 | + for h, v := range resp.Headers { 153 | + resp.HeaderStr += fmt.Sprintf("%s: %s\n", h, strings.Join(v, " ")) 154 | + } 155 | + 156 | // httputil.DumpResponse does not handle websockets 157 | headers, rawResp, err := pdhttputil.DumpResponseHeadersAndRaw(httpresp) 158 | if err != nil { 159 | @@ -224,6 +230,10 @@ 160 | } 161 | 162 | resp.Data = respbody 163 | + resp.DataStr = respbodystr 164 | + 165 | + // Get Title 166 | + resp.Title = ExtractTitle(&resp) 167 | 168 | // fill metrics 169 | resp.StatusCode = httpresp.StatusCode 170 | diff -uNr -x .idea httpx-1.2.1/common/httpx/response.go httpx-1.2.1-m/common/httpx/response.go 171 | --- httpx-1.2.1/common/httpx/response.go 2022-04-08 21:58:49.000000000 +0800 172 | +++ httpx-1.2.1-m/common/httpx/response.go 2022-06-08 15:34:22.000000000 +0800 173 | @@ -12,8 +12,11 @@ 174 | type Response struct { 175 | StatusCode int 176 | Headers map[string][]string 177 | + HeaderStr string 178 | Data []byte 179 | + DataStr string 180 | ContentLength int 181 | + Title string 182 | Raw string 183 | RawHeaders string 184 | Words int 185 | diff -uNr -x .idea httpx-1.2.1/go.mod httpx-1.2.1-m/go.mod 186 | --- httpx-1.2.1/go.mod 2022-04-08 21:58:49.000000000 +0800 187 | +++ httpx-1.2.1-m/go.mod 2022-06-08 15:54:00.000000000 +0800 188 | @@ -5,6 +5,7 @@ 189 | require ( 190 | github.com/akrylysov/pogreb v0.10.1 // indirect 191 | github.com/bluele/gcache v0.0.2 192 | + github.com/boy-hack/govaluate v3.1.0+incompatible 193 | github.com/corpix/uarand v0.1.1 194 | github.com/golang/snappy v0.0.4 // indirect 195 | github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf 196 | @@ -47,12 +48,12 @@ 197 | 198 | require ( 199 | github.com/PuerkitoBio/goquery v1.8.0 200 | + github.com/RumbleDiscovery/jarm-go v0.0.6 201 | + github.com/ammario/ipisp/v2 v2.0.0 202 | github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 203 | ) 204 | 205 | require ( 206 | - github.com/RumbleDiscovery/jarm-go v0.0.6 // indirect 207 | - github.com/ammario/ipisp/v2 v2.0.0 // indirect 208 | github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect 209 | github.com/andybalholm/cascadia v1.3.1 // indirect 210 | github.com/aymerick/douceur v0.2.0 // indirect 211 | diff -uNr -x .idea httpx-1.2.1/go.sum httpx-1.2.1-m/go.sum 212 | --- httpx-1.2.1/go.sum 2022-04-08 21:58:49.000000000 +0800 213 | +++ httpx-1.2.1-m/go.sum 2022-06-08 15:54:00.000000000 +0800 214 | @@ -9,8 +9,6 @@ 215 | github.com/akrylysov/pogreb v0.10.0/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= 216 | github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w= 217 | github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= 218 | -github.com/ammario/ipisp v1.0.0 h1:U4xdVMBFWm0/4sHrQ3hVMC+ygg/Ynm4/vdFdkVAex1o= 219 | -github.com/ammario/ipisp v1.0.0/go.mod h1:HM60VFpmEWyU+FisnTTHCeswaU3RW0dCVHihgIGUEGM= 220 | github.com/ammario/ipisp/v2 v2.0.0 h1:/aRMp5srZViiBfOUGzl/Esqae4s0MDDzm9buhGcZ0XU= 221 | github.com/ammario/ipisp/v2 v2.0.0/go.mod h1:bQ6KAL5LnYYEj6olUn+Bzv/im/4Esa5oGkbv9b+uOjo= 222 | github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= 223 | @@ -21,6 +19,8 @@ 224 | github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= 225 | github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= 226 | github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= 227 | +github.com/boy-hack/govaluate v3.1.0+incompatible h1:Ui9xUi7jayj2Sft3rZeaToB5UOC9OfyMggc+B+Y67gE= 228 | +github.com/boy-hack/govaluate v3.1.0+incompatible/go.mod h1:qMekcE/D3ipGRM3KCm+ihzioAl/oGuRpt55GizYo0uU= 229 | github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ= 230 | github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4= 231 | github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA= 232 | @@ -118,7 +118,6 @@ 233 | github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= 234 | github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= 235 | github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= 236 | -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 237 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 238 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 239 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 240 | @@ -200,7 +199,6 @@ 241 | github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= 242 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 243 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 244 | -github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 245 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 246 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 247 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 248 | diff -uNr -x .idea httpx-1.2.1/runner/options.go httpx-1.2.1-m/runner/options.go 249 | --- httpx-1.2.1/runner/options.go 2022-04-08 21:58:49.000000000 +0800 250 | +++ httpx-1.2.1-m/runner/options.go 2022-06-08 15:41:31.000000000 +0800 251 | @@ -1,11 +1,6 @@ 252 | package runner 253 | 254 | import ( 255 | - "math" 256 | - "os" 257 | - "regexp" 258 | - "strings" 259 | - "github.com/projectdiscovery/httpx/common/slice" 260 | "github.com/projectdiscovery/fileutil" 261 | "github.com/projectdiscovery/goconfig" 262 | "github.com/projectdiscovery/goflags" 263 | @@ -16,7 +11,12 @@ 264 | "github.com/projectdiscovery/httpx/common/customlist" 265 | customport "github.com/projectdiscovery/httpx/common/customports" 266 | fileutilz "github.com/projectdiscovery/httpx/common/fileutil" 267 | + "github.com/projectdiscovery/httpx/common/slice" 268 | "github.com/projectdiscovery/httpx/common/stringz" 269 | + "math" 270 | + "os" 271 | + "regexp" 272 | + "strings" 273 | ) 274 | 275 | const ( 276 | @@ -72,6 +72,7 @@ 277 | OutputLinesCount bool 278 | OutputWordsCount bool 279 | Hashes string 280 | + CustomFingerprint bool 281 | } 282 | 283 | func (s *scanOptions) Clone() *scanOptions { 284 | @@ -116,6 +117,7 @@ 285 | OutputLinesCount: s.OutputLinesCount, 286 | OutputWordsCount: s.OutputWordsCount, 287 | Hashes: s.Hashes, 288 | + CustomFingerprint: s.CustomFingerprint, 289 | } 290 | } 291 | 292 | @@ -229,6 +231,7 @@ 293 | Hashes string 294 | Jarm bool 295 | Asn bool 296 | + CustomFingerprint bool 297 | } 298 | 299 | // ParseOptions parses the command line options for application 300 | @@ -264,6 +267,7 @@ 301 | flagSet.BoolVar(&options.Asn, "asn", false, "display host asn information"), 302 | flagSet.BoolVar(&options.OutputCDN, "cdn", false, "display cdn in use"), 303 | flagSet.BoolVar(&options.Probe, "probe", false, "display probe status"), 304 | + flagSet.BoolVarP(&options.CustomFingerprint, "custom-fingerprint", "cf", false, "display custom fingerprint detected"), 305 | ) 306 | 307 | createGroup(flagSet, "matchers", "Matchers", 308 | diff -uNr -x .idea httpx-1.2.1/runner/runner.go httpx-1.2.1-m/runner/runner.go 309 | --- httpx-1.2.1/runner/runner.go 2022-04-08 21:58:49.000000000 +0800 310 | +++ httpx-1.2.1-m/runner/runner.go 2022-06-08 15:45:49.000000000 +0800 311 | @@ -7,6 +7,7 @@ 312 | "encoding/csv" 313 | "encoding/json" 314 | "fmt" 315 | + "github.com/projectdiscovery/httpx/common/fingerprint" 316 | "io/ioutil" 317 | "net" 318 | "net/http" 319 | @@ -60,6 +61,7 @@ 320 | hp *httpx.HTTPX 321 | wappalyzer *wappalyzer.Wappalyze 322 | scanopts scanOptions 323 | + fingerprints *fingerprint.FingerPrints 324 | hm *hybrid.HybridMap 325 | stats clistats.StatisticsClient 326 | ratelimiter ratelimit.Limiter 327 | @@ -131,6 +133,13 @@ 328 | gologger.Fatal().Msgf("Could not create httpx instance: %s\n", err) 329 | } 330 | 331 | + // FingerPrints 332 | + fingerprints, err := fingerprint.InitFingerPrints() 333 | + if err != nil { 334 | + return nil, err 335 | + } 336 | + runner.fingerprints = &fingerprints 337 | + 338 | var scanopts scanOptions 339 | 340 | if options.InputRawRequest != "" { 341 | @@ -230,6 +239,7 @@ 342 | scanopts.OutputLinesCount = options.OutputLinesCount 343 | scanopts.OutputWordsCount = options.OutputWordsCount 344 | scanopts.Hashes = options.Hashes 345 | + scanopts.CustomFingerprint = options.CustomFingerprint 346 | runner.scanopts = scanopts 347 | 348 | if options.ShowStatistics { 349 | @@ -886,6 +896,22 @@ 350 | req.Header.Add("Connection", "close") 351 | } 352 | resp, err := hp.Do(req, httpx.UnsafeOptions{URIPath: reqURI}) 353 | + 354 | + // Status Code 200 & window.location 355 | + if resp != nil { 356 | + if len(resp.DataStr) < 200 && strings.Contains(resp.DataStr, "window.location=\"") { 357 | + re, _ := regexp.Compile("window\\.location=\"([0-9a-zA-Z/.]+)\";") 358 | + if len(re.FindStringSubmatch(resp.DataStr)) > 0 { 359 | + uri := re.FindStringSubmatch(resp.DataStr)[1] 360 | + if !strings.HasPrefix(uri, "/") { 361 | + uri = "/" + uri 362 | + } 363 | + req.URL.Path = uri 364 | + resp, _ = hp.Do(req, httpx.UnsafeOptions{URIPath: uri}) 365 | + } 366 | + } 367 | + } 368 | + 369 | if r.options.ShowStatistics { 370 | r.stats.IncrementCounter("requests", 1) 371 | } 372 | @@ -1160,6 +1186,29 @@ 373 | builder.WriteString(fmt.Sprintf(" [%s]", ip)) 374 | } 375 | 376 | + // 指纹处理 377 | + fingerStr := "" 378 | + var fingerPrints []string 379 | + if scanopts.CustomFingerprint { 380 | + fingerResults, err := r.fingerprints.Matcher(resp) 381 | + if err != nil { 382 | + gologger.Error().Msgf("FingerPrint error: %s", err) 383 | + } 384 | + if fingerResults != nil { 385 | + 386 | + fingerPrints = fingerResults 387 | + fingerStr = strings.Join(fingerPrints, ",") 388 | + 389 | + builder.WriteString(" [") 390 | + if !scanopts.OutputWithNoColor { 391 | + builder.WriteString(aurora.Red(fingerStr).String()) 392 | + } else { 393 | + builder.WriteString(fingerStr) 394 | + } 395 | + builder.WriteRune(']') 396 | + } 397 | + } 398 | + 399 | ips, cnames, err := getDNSData(hp, domain) 400 | if err != nil { 401 | ips = append(ips, ip) 402 | @@ -1293,7 +1342,7 @@ 403 | } 404 | jarmhash := "" 405 | if r.options.Jarm { 406 | - jarmhash = hashes.Jarm(fullURL,r.options.Timeout) 407 | + jarmhash = hashes.Jarm(fullURL, r.options.Timeout) 408 | builder.WriteString(" [") 409 | if !scanopts.OutputWithNoColor { 410 | builder.WriteString(aurora.Magenta(jarmhash).String()) 411 | @@ -1411,6 +1460,7 @@ 412 | Lines: resp.Lines, 413 | Words: resp.Words, 414 | ASN: asnResponse, 415 | + Fingerprint: fingerStr, 416 | } 417 | } 418 | 419 | @@ -1478,6 +1528,7 @@ 420 | Lines int `json:"lines" csv:"lines"` 421 | Words int `json:"words" csv:"words"` 422 | Jarm string `json:"jarm,omitempty" csv:"jarm"` 423 | + Fingerprint string `json:"fingerprint,omitempty" csv:"fingerprint"` 424 | } 425 | 426 | // JSON the result 427 | --------------------------------------------------------------------------------