├── .gitignore ├── utils ├── title_test.go ├── useragent_test.go ├── validater.go ├── simhash.go ├── simhash_test.go ├── cidr.go ├── convert.go ├── http.go ├── encodings.go ├── fileutil.go ├── title.go └── useragent.go ├── brute ├── dict_test.go ├── dir_test.go ├── dict.go └── dir.go ├── finger ├── finger_test.go └── finger.go ├── runner ├── runner_test.go ├── icon_test.go ├── icon.go ├── banner.go ├── requests_test.go ├── csp.go ├── request.go └── runner.go ├── main.go ├── config ├── options.go └── update.go ├── go.mod ├── README.md ├── cli └── cli.go ├── LICENSE └── go.sum /.gitignore: -------------------------------------------------------------------------------- 1 | out 2 | tmp 3 | .idea 4 | .gitignore 5 | blackJack-* -------------------------------------------------------------------------------- /utils/title_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import "testing" 4 | 5 | func TestExtractTitle(t *testing.T) { 6 | 7 | } -------------------------------------------------------------------------------- /utils/useragent_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestGetUserAgent(t *testing.T) { 8 | ua := GetUserAgent() 9 | if ua == ""{ 10 | t.Errorf("useragent error") 11 | } 12 | } -------------------------------------------------------------------------------- /brute/dict_test.go: -------------------------------------------------------------------------------- 1 | package brute 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestPrepareDict(t *testing.T) { 8 | dicts, err := PrepareDict() 9 | if err != nil && len(dicts) > 0{ 10 | t.Error("PrepareDict Error") 11 | } 12 | } -------------------------------------------------------------------------------- /finger/finger_test.go: -------------------------------------------------------------------------------- 1 | package finger 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestLoadFinger(t *testing.T){ 8 | finger, err := LoadFinger() 9 | if err != nil && len(finger.Rules) == 0{ 10 | t.Errorf("LoadFinger test error") 11 | } 12 | } -------------------------------------------------------------------------------- /runner/runner_test.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "github.com/Athena1337/blackJack/config" 5 | "testing" 6 | ) 7 | 8 | func TestRunner(t *testing.T) { 9 | options := config.DefaultOption 10 | r, _ := New(&options) 11 | r.CreateRunner() 12 | } 13 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/Athena1337/blackJack/cli" 5 | "github.com/Athena1337/blackJack/runner" 6 | log "github.com/t43Wiu6/tlog" 7 | ) 8 | 9 | func main() { 10 | runner.ShowBanner() 11 | cli.Parse() 12 | log.Infof("Done, Good Luck !") 13 | } 14 | -------------------------------------------------------------------------------- /runner/icon_test.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "github.com/Athena1337/blackJack/config" 5 | "testing" 6 | ) 7 | 8 | func TestGetFaviconHash(t *testing.T) { 9 | options := config.DefaultOption 10 | r, _ := New(&options) 11 | hash, err := r.GetFaviconHash("https://google.com/favicon.ico") 12 | if err != nil && hash != "1693998826" { 13 | t.Errorf("Analyze test error") 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /utils/validater.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "github.com/t43Wiu6/tlog" 5 | "net/url" 6 | "strings" 7 | ) 8 | 9 | func ValidateUrl(targetUrl string) string{ 10 | if strings.HasPrefix(targetUrl, "https") { 11 | _, err := url.Parse(targetUrl) 12 | if err != nil { 13 | log.Errorf("could not parse request URL: %s", targetUrl) 14 | return "" 15 | } 16 | return "https" 17 | } 18 | 19 | if strings.HasPrefix(targetUrl, "http") { 20 | _, err := url.Parse(targetUrl) 21 | if err != nil { 22 | log.Errorf("could not parse request URL: %s", targetUrl) 23 | return "" 24 | } 25 | return "http" 26 | } 27 | return "none" 28 | } -------------------------------------------------------------------------------- /brute/dir_test.go: -------------------------------------------------------------------------------- 1 | package brute 2 | 3 | import ( 4 | "github.com/Athena1337/blackJack/config" 5 | "github.com/pterm/pterm" 6 | . "github.com/t43Wiu6/tlog" 7 | "testing" 8 | ) 9 | 10 | func TestStart(t *testing.T) { 11 | DEBUG = false 12 | d := &DirBrute{ 13 | IndexUrl: "https://os.alipayobjects.com", 14 | ErrorUrl: "https://os.alipayobjects.com/ljaisdkhfkjashdfkjahsdjkf", 15 | Options: &config.DefaultOption, 16 | } 17 | var output chan []string 18 | spinnerLiveText, _ := pterm.DefaultSpinner.Start("[DirBrute] Waiting to Brute Force") 19 | ds := DirStatus{ 20 | AllJob: 0, 21 | DoneJob: 0, 22 | } 23 | d.Start(output, spinnerLiveText, &ds) 24 | } -------------------------------------------------------------------------------- /runner/icon.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "github.com/Athena1337/blackJack/utils" 5 | "io" 6 | "io/ioutil" 7 | ) 8 | 9 | func (r *Runner) GetFaviconHash(url string) (hash string, err error) { 10 | request, err := newRequest("GET", url) 11 | if err != nil { 12 | return 13 | } 14 | resp, err := r.client.Do(request) 15 | if err != nil { 16 | return 17 | } 18 | body, err := ioutil.ReadAll(resp.Body) 19 | if err != nil { 20 | return 21 | } 22 | defer func(Body io.ReadCloser) { 23 | err := Body.Close() 24 | if err != nil { 25 | return 26 | } 27 | }(resp.Body) //nolint 28 | 29 | hash = utils.Mmh3Hash32(utils.StandBase64(body)) 30 | return 31 | } 32 | -------------------------------------------------------------------------------- /utils/simhash.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "gopkg.in/go-dedup/simhash.v2" 5 | ) 6 | 7 | func GetHash(resp []byte) (hash uint64){ 8 | sh := simhash.NewSimhash() 9 | data := TrimHtml(string(resp)) 10 | hash = sh.GetSimhash(sh.NewWordFeatureSet([]byte(data))) 11 | return 12 | } 13 | 14 | func IsSimilarHash(hash1 uint64, hash2 uint64) bool{ 15 | rs := simhash.Compare(hash1, hash2) 16 | // 差异大于8，认定为不同页面 17 | if rs > 8 { 18 | return false 19 | } 20 | return true 21 | } 22 | 23 | // IsEqualHash 判断页面hash是否相同 24 | func IsEqualHash(hash1 uint64, hash2 uint64) bool{ 25 | rs := simhash.Compare(hash1, hash2) 26 | if rs == 0 { 27 | return true 28 | } 29 | return false 30 | 31 | } 32 | -------------------------------------------------------------------------------- /runner/banner.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import "fmt" 4 | 5 | const banner = ` 6 | ██████╗ ██╗ █████╗ ██████╗██╗ ██╗ ██╗ █████╗ ██████╗██╗ ██╗ 7 | ██╔══██╗██║ ██╔══██╗██╔════╝██║ ██╔╝ ██║██╔══██╗██╔════╝██║ ██╔╝ 8 | ██████╔╝██║ ███████║██║ █████╔╝ ██║███████║██║ █████╔╝ 9 | ██╔══██╗██║ ██╔══██║██║ ██╔═██╗ ██ ██║██╔══██║██║ ██╔═██╗ 10 | ██████╔╝███████╗██║ ██║╚██████╗██║ ██╗╚█████╔╝██║ ██║╚██████╗██║ ██╗ 11 | ╚═════╝ ╚══════╝╚═╝ ╚═╝ ╚═════╝╚═╝ ╚═╝ ╚════╝ ╚═╝ ╚═╝ ╚═════╝╚═╝ ╚═╝ v1.0.0 12 | ` 13 | 14 | // Version is the current version of httpx 15 | const Version = `v1.0.0` 16 | 17 | // ShowBanner used to show the banner to the user 18 | func ShowBanner() { 19 | fmt.Println(banner) 20 | } 21 | -------------------------------------------------------------------------------- /runner/requests_test.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "github.com/Athena1337/blackJack/config" 5 | "testing" 6 | ) 7 | 8 | func TestHttpReqWithNoRedirect(t *testing.T) { 9 | options := config.DefaultOption 10 | r, _ := New(&options) 11 | resp, err := r.Request("GET","https://bing.com", false) 12 | if err != nil { 13 | t.Errorf("HttpReqWithNoRedirect test error") 14 | } 15 | if err != nil || resp.StatusCode != 301{ 16 | t.Errorf("HttpReqWithNoRedirect test error") 17 | } 18 | } 19 | 20 | 21 | func TestHttpReq(t *testing.T) { 22 | options := config.DefaultOption 23 | r, _ := New(&options) 24 | resp, err := r.Request("GET","https://bing.com", true) 25 | if err != nil { 26 | t.Errorf("HttpReq test error") 27 | } 28 | if err != nil || resp.StatusCode != 200 || resp.Title != "必应"{ 29 | t.Errorf("HttpReq test error") 30 | } 31 | } -------------------------------------------------------------------------------- /config/options.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "github.com/t43Wiu6/tlog" 5 | "os" 6 | "sync" 7 | "time" 8 | ) 9 | 10 | func SetEnv(isDebug bool) { 11 | log.DEBUG = isDebug 12 | } 13 | 14 | type OutputFile struct { 15 | sync.Mutex 16 | File *os.File 17 | } 18 | 19 | type Options struct { 20 | TargetUrl string 21 | IndexUrl string 22 | FaviconUrl string 23 | ErrorUrl string 24 | OrigProtocol string 25 | Urls []string 26 | UrlFile string 27 | IsDebug bool 28 | EnableDirBrute bool 29 | TimeOut time.Duration 30 | Threads int 31 | RetryMax int 32 | Output string 33 | JSONOutput bool 34 | Proxy string 35 | OutputFile OutputFile 36 | } 37 | 38 | var DefaultOption = Options{ 39 | TargetUrl: "https://google.com", 40 | IsDebug: true, 41 | TimeOut: 30 * time.Second, 42 | Threads: 50, 43 | RetryMax: 5, 44 | } 45 | -------------------------------------------------------------------------------- /utils/simhash_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | "github.com/hashicorp/go-retryablehttp" 6 | "testing" 7 | ) 8 | 9 | func TestIsSimilarHash0(t *testing.T) { 10 | if IsSimilarHash(9832410273512798190, 12133804658672123316) { 11 | fmt.Println("yes") 12 | } else { 13 | t.Fatal("") 14 | } 15 | } 16 | 17 | func TestIsSimilarHash2(t *testing.T) { 18 | var rs []uint64 19 | test := []string{ 20 | "?wsdl", 21 | "asdkfjaksjdfkajsdkfjkajsdkfj", 22 | "", 23 | } 24 | retryClient := retryablehttp.NewClient() 25 | client := retryClient.StandardClient() 26 | 27 | for _, t := range test { 28 | resp, err := client.Get("http://192.168.22.176:8080/" + t) 29 | if err != nil { 30 | return 31 | } 32 | data := DumpHttpResponse(resp) 33 | hash1 := GetHash(data) 34 | fmt.Println(t, " : ", hash1) 35 | for _, k := range rs { 36 | if IsSimilarHash(hash1, k) { 37 | fmt.Println("yes") 38 | fmt.Println(k) 39 | } 40 | } 41 | rs = append(rs, hash1) 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/Athena1337/blackJack 2 | 3 | go 1.16 4 | 5 | require ( 6 | github.com/chromedp/cdproto v0.0.0-20220228194516-ead8cf59b1fa 7 | github.com/chromedp/chromedp v0.7.8 8 | github.com/go-dedup/megophone v0.0.0-20170830025436-f01be21026f5 // indirect 9 | github.com/go-dedup/simhash v0.0.0-20170904020510-9ecaca7b509c // indirect 10 | github.com/go-dedup/text v0.0.0-20170907015346-8bb1b95e3cb7 // indirect 11 | github.com/hashicorp/go-retryablehttp v0.7.0 12 | github.com/projectdiscovery/cdncheck v0.0.3 13 | github.com/projectdiscovery/fastdialer v0.0.14 14 | github.com/projectdiscovery/httputil v0.0.0-20210913064028-815c8db933ff 15 | github.com/projectdiscovery/httpx v1.2.0 16 | github.com/pterm/pterm v0.12.37 17 | github.com/remeh/sizedwaitgroup v1.0.0 18 | github.com/t43Wiu6/tlog v0.0.0-20220117055205-3d75ddb8e805 19 | github.com/twmb/murmur3 v1.1.6 20 | github.com/urfave/cli/v2 v2.3.0 21 | golang.org/x/net v0.0.0-20220225172249-27dd8689420f 22 | golang.org/x/text v0.3.7 23 | gopkg.in/go-dedup/simhash.v2 v2.0.0-20170703161212-581a1061a875 24 | ) 25 | -------------------------------------------------------------------------------- /brute/dict.go: -------------------------------------------------------------------------------- 1 | package brute 2 | 3 | import ( 4 | "github.com/Athena1337/blackJack/config" 5 | "github.com/Athena1337/blackJack/utils" 6 | "github.com/t43Wiu6/tlog" 7 | "os" 8 | "path/filepath" 9 | ) 10 | 11 | func PrepareDict() (dicts []string, err error){ 12 | dict, err := loadDict(config.Dict_mini) 13 | if err != nil { 14 | return 15 | } 16 | dicts = append(dicts, dict...) 17 | 18 | dict, err = loadDict(config.Api) 19 | if err != nil { 20 | return 21 | } 22 | dicts = append(dicts, dict...) 23 | 24 | dict, err = loadDict(config.File) 25 | if err != nil { 26 | return 27 | } 28 | dicts = append(dicts, dict...) 29 | 30 | log.Debugf("Generate dict %d 's", len(dicts)) 31 | return 32 | } 33 | 34 | func loadDict(name string)(dict []string, err error) { 35 | filePath, err := utils.GetCurrentAbPathByCaller() 36 | if err == nil{ 37 | home, _ := os.UserHomeDir() 38 | filePath = filepath.Join(home, ".blackJack", name) 39 | dict, err = utils.ReadFile(filePath) 40 | if err != nil { 41 | log.Warnf("dict not found, unable to read dict file: %s", filePath) 42 | return 43 | } 44 | } 45 | return 46 | } -------------------------------------------------------------------------------- /finger/finger.go: -------------------------------------------------------------------------------- 1 | package finger 2 | 3 | import ( 4 | "github.com/Athena1337/blackJack/utils" 5 | "encoding/json" 6 | "github.com/t43Wiu6/tlog" 7 | "io/ioutil" 8 | "os" 9 | "path/filepath" 10 | ) 11 | 12 | type Config struct { 13 | Rules []Finger 14 | } 15 | 16 | type MetaFinger struct { 17 | Method string 18 | Location string 19 | StatusCode int 20 | Keyword []string 21 | } 22 | 23 | type Finger struct { 24 | Name string 25 | Fingerprint []MetaFinger 26 | } 27 | 28 | func LoadFinger() (configs Config, err error) { 29 | filePath, err := utils.GetCurrentAbPathByCaller() 30 | if err == nil{ 31 | home, _ := os.UserHomeDir() 32 | filePath = filepath.Join(home, ".blackJack", "finger.json") 33 | dat, errs := ioutil.ReadFile(filePath) 34 | if errs != nil { 35 | log.Warnf("finger.json not found, unable to read config file: %s", filePath) 36 | return configs, errs 37 | } 38 | err = json.Unmarshal(dat, &configs) 39 | if err != nil { 40 | log.Errorf("%s", err) 41 | return 42 | } 43 | } 44 | a := 0 45 | for _, k := range configs.Rules { 46 | a = a + len(k.Fingerprint) 47 | } 48 | log.Infof("Totally load finger %d 's", a) 49 | return 50 | } 51 | -------------------------------------------------------------------------------- /utils/cidr.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | "github.com/t43Wiu6/tlog" 6 | "net" 7 | "sort" 8 | "strings" 9 | ) 10 | 11 | // CheckCIDR 计算给出的一组ip或域名列表的CIDR重合值 12 | func CheckCIDR(D []string) string{ 13 | type tmp struct { 14 | CIDR string 15 | Count int 16 | } 17 | 18 | var ( 19 | lst []tmp 20 | rs string 21 | ) 22 | /* 创建集合 */ 23 | cidrMap := make(map[string]int) 24 | for _, name := range D { 25 | addr, err := net.ResolveIPAddr("ip", name) 26 | if err != nil { 27 | log.Debugf("Resolvtion error, %v" , err.Error()) 28 | continue 29 | } 30 | 31 | n := strings.Split(addr.String(), ".") 32 | n = n[:len(n)-1] 33 | p := strings.Join(n, ".") + ".0/24" 34 | 35 | /* 查看元素在集合中是否存在 */ 36 | _, ok := cidrMap[p] 37 | /* 如果 ok 是 true, 则存在，否则不存在 */ 38 | if ok { 39 | cidrMap[p] = cidrMap[p] + 1 40 | //fmt.Println("the cidr", p, " count is :", cidr) 41 | } else { 42 | cidrMap[p] = 1 43 | } 44 | } 45 | 46 | for k, v := range cidrMap { 47 | lst = append(lst, tmp{k, v}) 48 | } 49 | 50 | sort.Slice(lst, func(i, j int) bool { 51 | return lst[i].Count > lst[j].Count // 降序 52 | }) 53 | 54 | for _, vv := range lst { 55 | rs += fmt.Sprintf("%s : %d", vv.CIDR, vv.Count) 56 | } 57 | return rs 58 | } 59 | -------------------------------------------------------------------------------- /utils/convert.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "bytes" 5 | "math/rand" 6 | "strings" 7 | "unsafe" 8 | ) 9 | 10 | const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" 11 | 12 | // SplitChar76 按照 76 字符切分 13 | func SplitChar76(braw []byte) []byte { 14 | // 去掉 data:image/vnd.microsoft.icon;base64 15 | if strings.HasPrefix(string(braw), "data:image/vnd.microsoft.icon;base64,") { 16 | braw = braw[37:] 17 | } 18 | 19 | var buffer bytes.Buffer 20 | for i := 0; i < len(braw); i++ { 21 | ch := braw[i] 22 | buffer.WriteByte(ch) 23 | if (i+1)%76 == 0 { 24 | buffer.WriteByte('\n') 25 | } 26 | } 27 | buffer.WriteByte('\n') 28 | 29 | return buffer.Bytes() 30 | } 31 | 32 | // ByteArrayToString convert char array to string 33 | func ByteArrayToString(b []byte) string { 34 | return *(*string)(unsafe.Pointer(&b)) 35 | } 36 | 37 | func StringArrayToString(s []string) string{ 38 | return strings.Join(s , " ") 39 | } 40 | 41 | func StringArrayContains(s []string, str string) bool { 42 | if len(s) == 0{ 43 | return false 44 | } 45 | for _, v := range s { 46 | if strings.ToLower(v) == strings.ToLower(str) { 47 | return true 48 | } 49 | } 50 | return false 51 | } 52 | 53 | func RandStringBytes(n int) string { 54 | b := make([]byte, n) 55 | for i := range b { 56 | b[i] = letterBytes[rand.Intn(len(letterBytes))] 57 | } 58 | return string(b) 59 | } -------------------------------------------------------------------------------- /runner/csp.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "net/http" 5 | "strings" 6 | 7 | "github.com/projectdiscovery/httpx/common/slice" 8 | ) 9 | 10 | // CSPHeaders is an incomplete list of most common CSP headers 11 | var CSPHeaders = []string{ 12 | "Content-Security-Policy", // standard 13 | "Content-Security-Policy-Report-Only", // standard 14 | "X-Content-Security-Policy-Report-Only", // non - standard 15 | "X-Webkit-Csp-Report-Only", // non - standard 16 | } 17 | 18 | // CSPData contains the Content-Security-Policy domain list 19 | type CSPData struct { 20 | Domains []string `json:"domains,omitempty"` 21 | } 22 | 23 | // CSPGrab fills the CSPData 24 | func CSPGrab(r *http.Response) *CSPData { 25 | domains := make(map[string]struct{}) 26 | for _, cspHeader := range CSPHeaders { 27 | cspRaw := r.Header.Get(cspHeader) 28 | if cspRaw != "" { 29 | rules := strings.Split(cspRaw, ";") 30 | for _, rule := range rules { 31 | // rule is like aa bb domain1 domain2 domain3 32 | tokens := strings.Split(rule, " ") 33 | // we extracts only potential domains 34 | for _, t := range tokens { 35 | if isPotentialDomain(t) { 36 | domains[t] = struct{}{} 37 | } 38 | } 39 | } 40 | } 41 | } 42 | 43 | if len(domains) > 0 { 44 | return &CSPData{Domains: slice.ToSlice(domains)} 45 | } 46 | return nil 47 | } 48 | 49 | func isPotentialDomain(s string) bool { 50 | return strings.Contains(s, ".") || strings.HasPrefix(s, "http") 51 | } 52 | -------------------------------------------------------------------------------- /utils/http.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "compress/flate" 7 | "compress/gzip" 8 | "github.com/t43Wiu6/tlog" 9 | "io/ioutil" 10 | "net/http" 11 | "regexp" 12 | "strings" 13 | ) 14 | 15 | func TrimHtml(src string) string { 16 | //将HTML标签全转换成小写 17 | re, _ := regexp.Compile("\\<[\\S\\s]+?\\>") 18 | src = re.ReplaceAllStringFunc(src, strings.ToLower) 19 | //去除STYLE 20 | re, _ = regexp.Compile("\\") 21 | src = re.ReplaceAllString(src, "") 22 | //去除SCRIPT 23 | re, _ = regexp.Compile("\\") 24 | src = re.ReplaceAllString(src, "") 25 | //去除所有尖括号内的HTML代码，并换成换行符 26 | re, _ = regexp.Compile("\\<[\\S\\s]+?\\>") 27 | src = re.ReplaceAllString(src, "\n") 28 | //去除连续的换行符 29 | re, _ = regexp.Compile("\\s{2,}") 30 | src = re.ReplaceAllString(src, "\n") 31 | return strings.TrimSpace(src) 32 | } 33 | 34 | func DumpHttpResponse(resp *http.Response) []byte{ 35 | respBody, err := ioutil.ReadAll(resp.Body) 36 | if err != nil { 37 | log.Debugf("func httpDump read resp body err: %v", err) 38 | } else { 39 | acceptEncode := resp.Header["Content-Encoding"] 40 | var respBodyBin bytes.Buffer 41 | w := bufio.NewWriter(&respBodyBin) 42 | w.Write(respBody) 43 | w.Flush() 44 | for _, compress := range acceptEncode { 45 | switch compress { 46 | case "gzip": 47 | r, err := gzip.NewReader(&respBodyBin) 48 | if err != nil { 49 | log.Debugf("gzip reader err: %v", err) 50 | } else { 51 | defer r.Close() 52 | respBody, _ = ioutil.ReadAll(r) 53 | } 54 | break 55 | case "deflate": 56 | r := flate.NewReader(&respBodyBin) 57 | defer r.Close() 58 | respBody, _ = ioutil.ReadAll(r) 59 | break 60 | } 61 | } 62 | // log.Debugf("DumpContent %s : %s", resp.Request.URL.String() ,string(respBody)) 63 | } 64 | return respBody 65 | } -------------------------------------------------------------------------------- /utils/encodings.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "bytes" 5 | "encoding/base64" 6 | "fmt" 7 | "github.com/twmb/murmur3" 8 | "golang.org/x/text/encoding/simplifiedchinese" 9 | "golang.org/x/text/encoding/traditionalchinese" 10 | "golang.org/x/text/transform" 11 | "hash" 12 | "io/ioutil" 13 | ) 14 | 15 | 16 | // Mmh3Hash32 计算 mmh3 hash 17 | func Mmh3Hash32(raw []byte) string { 18 | var h32 hash.Hash32 = murmur3.New32() 19 | h32.Write(raw) 20 | // if IsUint32 { 21 | // return fmt.Sprintf("%d", h32.Sum32()) 22 | // } 23 | return fmt.Sprintf("%d", int32(h32.Sum32())) 24 | } 25 | 26 | // StandBase64 计算 base64 的值 27 | func StandBase64(braw []byte) []byte { 28 | bckd := base64.StdEncoding.EncodeToString(braw) 29 | var buffer bytes.Buffer 30 | for i := 0; i < len(bckd); i++ { 31 | ch := bckd[i] 32 | buffer.WriteByte(ch) 33 | if (i+1)%76 == 0 { 34 | buffer.WriteByte('\n') 35 | } 36 | } 37 | buffer.WriteByte('\n') 38 | return buffer.Bytes() 39 | } 40 | 41 | // Credits: https://gist.github.com/zhangbaohe/c691e1da5bbdc7f41ca5 42 | 43 | // Decodegbk converts GBK to UTF-8 44 | func Decodegbk(s []byte) ([]byte, error) { 45 | I := bytes.NewReader(s) 46 | O := transform.NewReader(I, simplifiedchinese.GBK.NewDecoder()) 47 | d, e := ioutil.ReadAll(O) 48 | if e != nil { 49 | return nil, e 50 | } 51 | return d, nil 52 | } 53 | 54 | // Decodebig5 converts BIG5 to UTF-8 55 | func Decodebig5(s []byte) ([]byte, error) { 56 | I := bytes.NewReader(s) 57 | O := transform.NewReader(I, traditionalchinese.Big5.NewDecoder()) 58 | d, e := ioutil.ReadAll(O) 59 | if e != nil { 60 | return nil, e 61 | } 62 | return d, nil 63 | } 64 | 65 | // Encodebig5 converts UTF-8 to BIG5 66 | func Encodebig5(s []byte) ([]byte, error) { 67 | I := bytes.NewReader(s) 68 | O := transform.NewReader(I, traditionalchinese.Big5.NewEncoder()) 69 | d, e := ioutil.ReadAll(O) 70 | if e != nil { 71 | return nil, e 72 | } 73 | return d, nil 74 | } 75 | -------------------------------------------------------------------------------- /utils/fileutil.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "bufio" 5 | "errors" 6 | "os" 7 | "path/filepath" 8 | "regexp" 9 | ) 10 | 11 | // ReadFile read content 12 | func ReadFile(path string) ([]string, error){ 13 | var ret []string 14 | buf, err := os.Open(path) 15 | if err != nil { 16 | return nil, err 17 | } 18 | 19 | defer func() { 20 | if err = buf.Close(); err != nil { 21 | } 22 | }() 23 | 24 | snl := bufio.NewScanner(buf) 25 | for snl.Scan() { 26 | ret = append(ret, snl.Text()) 27 | } 28 | err = snl.Err() 29 | if err != nil { 30 | return nil, err 31 | } 32 | return ret ,nil 33 | } 34 | 35 | // FileExists checks if a file exists and is not a directory 36 | func FileExists(filename string) bool { 37 | info, err := os.Stat(filename) 38 | if os.IsNotExist(err) || err != nil || info == nil { 39 | return false 40 | } 41 | return !info.IsDir() 42 | } 43 | 44 | // FolderExists checks if a folder exists 45 | func FolderExists(folderpath string) bool { 46 | _, err := os.Stat(folderpath) 47 | return !os.IsNotExist(err) 48 | } 49 | 50 | // HasStdin determines if the user has piped input 51 | func HasStdin() bool { 52 | stat, err := os.Stdin.Stat() 53 | if err != nil { 54 | return false 55 | } 56 | 57 | mode := stat.Mode() 58 | 59 | isPipedFromChrDev := (mode & os.ModeCharDevice) == 0 60 | isPipedFromFIFO := (mode & os.ModeNamedPipe) != 0 61 | 62 | return isPipedFromChrDev || isPipedFromFIFO 63 | } 64 | 65 | // LoadFile content to slice 66 | func LoadFile(filename string) (lines []string) { 67 | f, err := os.Open(filename) 68 | if err != nil { 69 | return 70 | } 71 | defer f.Close() //nolint 72 | s := bufio.NewScanner(f) 73 | for s.Scan() { 74 | lines = append(lines, s.Text()) 75 | } 76 | return 77 | } 78 | 79 | // ListFilesWithPattern in a root folder 80 | func ListFilesWithPattern(rootpattern string) ([]string, error) { 81 | files, err := filepath.Glob(rootpattern) 82 | if err != nil { 83 | return nil, err 84 | } 85 | if len(files) == 0 { 86 | return nil, errors.New("no files found") 87 | } 88 | return files, err 89 | } 90 | 91 | // FileNameIsGlob check if the filename is a pattern 92 | func FileNameIsGlob(pattern string) bool { 93 | _, err := regexp.Compile(pattern) 94 | return err == nil 95 | } 96 | 97 | // GetCurrentAbPathByCaller 获取当前执行文件绝对路径 98 | func GetCurrentAbPathByCaller() (exPath string, err error) { 99 | ex, err := os.Executable() 100 | if err != nil { 101 | return 102 | } 103 | exPath = filepath.Dir(ex) 104 | return 105 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # blackJack 2 | 3 | `blackJack`是由 [Lumos框架](https://github.com/Athena1337/Lumos) 的核心侦查功能独立出来的小工具 4 | 5 | 用于从大量的资产中进行Web指纹探测，提取`有用`的系统，并能对探测后的目标进行目录扫描和备份文件扫描 6 | 7 | ## Usage 8 | 9 | ### help 10 | 11 | ```bash 12 | λ blackJack -h 13 | 14 | ██████╗ ██╗ █████╗ ██████╗██╗ ██╗ ██╗ █████╗ ██████╗██╗ ██╗ 15 | ██╔══██╗██║ ██╔══██╗██╔════╝██║ ██╔╝ ██║██╔══██╗██╔════╝██║ ██╔╝ 16 | ██████╔╝██║ ███████║██║ █████╔╝ ██║███████║██║ █████╔╝ 17 | ██╔══██╗██║ ██╔══██║██║ ██╔═██╗ ██ ██║██╔══██║██║ ██╔═██╗ 18 | ██████╔╝███████╗██║ ██║╚██████╗██║ ██╗╚█████╔╝██║ ██║╚██████╗██║ ██╗ 19 | ╚═════╝ ╚══════╝╚═╝ ╚═╝ ╚═════╝╚═╝ ╚═╝ ╚════╝ ╚═╝ ╚═╝ ╚═════╝╚═╝ ╚═╝ v1.0.0 20 | 21 | NAME: 22 | blackJack - Usage Menu 23 | 24 | USAGE: 25 | blackJack [global options] command [command options] [arguments...] 26 | 27 | COMMANDS: 28 | help, h Shows a list of commands or help for one command 29 | 30 | GLOBAL OPTIONS: 31 | -d Enable debug mode (default: false) 32 | -u value Single target url 33 | -l value The list file contain mutilple target url 34 | -t value Request thread (default: 50) 35 | --time value Request timeout (default: 30s) 36 | -r value Max Retry attempts (default: 5) 37 | -o value Output file 38 | -p value http proxy ,Ex: http://127.0.0.1:8080 39 | -i value Analyse target favicon fingerprint 40 | -b Enable DirBrute for analyse target (default: false) 41 | --help, -h show help (default: false) 42 | ``` 43 | 44 | ### Running with file input 45 | 46 | ```bash 47 | λ blackJack -l urls.txt 48 | ``` 49 | 50 | ### Running with single url 51 | 52 | ```bash 53 | λ blackJack -u https://google.com 54 | ``` 55 | 56 | ### Dir Brute 57 | 58 | 对指纹探测后的目标进行目录扫描和备份文件扫描 59 | 60 | 使用`simhash`算法识别页面，必须与伪404页面、主页不相似的页面才被打印 61 | 62 | 并能动态识别不同路径高相似页面然后进行过滤 ，实测0误报 63 | ```bash 64 | λ blackJack -l urls.txt -b 65 | ``` 66 | 字典默认使用[blackJack-Dicts](https://github.com/t43Wiu6/blackJack-Dicts) 67 | 68 | 首次使用时字典默认下载到当前用户的`home/.blackJack`目录, 具体分类请看该项目, 欢迎pr和自定义修改 69 | 70 | 由于终端打印重用覆盖会导致刷大量换行符，虽不影响使用但不建议用`win`，推荐`Linux` 71 | 72 | ### More 73 | ```bash 74 | λ blackJack -l urls.txt -d # 开启Debug模式 75 | λ blackJack -i google.com # 检测提供域名的favicon hash，用于fofa等测绘引擎 76 | ``` 77 | 78 | ## Update Logs 79 | 80 | ### V1.0 81 | + 新增目录扫描和备份文件扫描功能 82 | + 新增已识别页面相等性计算，避免冗余输出 83 | 84 | ### V1.0-beta3 85 | + 更换底层HTTP请求库 86 | + 重构并发模块，减少开销 87 | + 提高稳定性 88 | + 速度提升500% 89 | 90 | ### V1.0-beta2 91 | + 深度去重指纹 92 | + 重构指纹结构 93 | + 重构优化部分处理模块 94 | + 更新指纹1833条，合计2581条 95 | 96 | ### v1.0-beta1 97 | + 自动协议识别 98 | + WAF、CDN识别 99 | + 指纹覆盖优化，避免302跳转、CDN、均衡负载导致识别失效 100 | + 集成`icon hash`生成 101 | + 集成指纹748条 102 | 103 | ## Thanks 104 | 105 | 探测功能的灵感和原基本指纹库来自[EHole](https://github.com/EdgeSecurityTeam/EHole) 106 | 一些细节参考了[httpx](https://github.com/projectdiscovery/httpx) -------------------------------------------------------------------------------- /config/update.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "github.com/Athena1337/blackJack/utils" 5 | "github.com/t43Wiu6/tlog" 6 | "io/ioutil" 7 | "net/http" 8 | "os" 9 | "path/filepath" 10 | ) 11 | 12 | var ( 13 | Dir_bigger = "dir_bigger.txt" // 15000 14 | Dict_mini = "dir_mini.txt" // 2000 15 | Dir = "dir.txt" // 5000 16 | Api = "api/per_root_api.txt" 17 | File = "file/per_root_file.txt" 18 | Rfile = "file/per_dir_file_replace.txt" 19 | Asp_dir = "asp/asp_dir.txt" 20 | Asp_file = "asp/asp_file.txt" 21 | Aspx_dir = "aspx/aspx_dir.txt" 22 | Aspx_file = "aspx/aspx_file.txt" 23 | Java_dir = "java/action_dir.txt" 24 | Java_file = "java/action_file.txt" 25 | Jsp_dir = "jsp/jsp_dir.txt" 26 | Jsp_file = "jsp/jsp_file.txt" 27 | Php_dir = "php/php_dir.txt" 28 | Php_file = "php/php_file.txt" 29 | ) 30 | 31 | func DownloadAll() (err error){ 32 | log.Warn("try to download from github...") 33 | err = DownloadDictFile() 34 | if err != nil { 35 | return 36 | } 37 | 38 | err = DownloadFinger() 39 | return 40 | } 41 | 42 | func DownloadFinger()(err error){ 43 | fingerUrl := "https://raw.githubusercontent.com/Athena1337/blackJack/main/finger.json" 44 | home, _ := os.UserHomeDir() 45 | fingerPath := filepath.Join(home, ".blackJack", "finger.json") 46 | if !utils.FolderExists(filepath.Join(home, ".blackJack")){ 47 | err = os.Mkdir( filepath.Join(home, ".blackJack"), os.ModePerm) 48 | if err != nil { 49 | return 50 | } 51 | } 52 | resp, err := http.Get(fingerUrl) 53 | if err != nil { 54 | log.Error("download failed, please check you network...") 55 | return 56 | } 57 | data, err := ioutil.ReadAll(resp.Body) 58 | if err != nil { 59 | return 60 | } 61 | if utils.FileExists(fingerPath){ 62 | err := os.Remove(fingerPath) 63 | if err != nil { 64 | return err 65 | } 66 | } 67 | err = ioutil.WriteFile(fingerPath, data, 0666) 68 | if err != nil { 69 | log.Warn("unable to write config file") 70 | } 71 | return 72 | } 73 | 74 | func DownloadDictFile() (err error){ 75 | err = download(Dict_mini) 76 | if err != nil { 77 | log.Warnf("Download dict from github error: %s", err) 78 | } 79 | err = download(Api) 80 | if err != nil { 81 | log.Warnf("Download dict from github error: %s", err) 82 | } 83 | err = download(File) 84 | if err != nil { 85 | log.Warnf("Download dict from github error: %s", err) 86 | } 87 | return 88 | } 89 | 90 | func Check() bool{ 91 | home, _ := os.UserHomeDir() 92 | if !utils.FolderExists(filepath.Join(home, ".blackJack")){ 93 | return false 94 | } 95 | return true 96 | } 97 | 98 | func download(name string) (err error){ 99 | baseUrl := "https://raw.githubusercontent.com/t43Wiu6/blackJack-Dicts/main/" 100 | home, _ := os.UserHomeDir() 101 | filePath, err := utils.GetCurrentAbPathByCaller() 102 | filePath = filepath.Join(home, ".blackJack", name) 103 | 104 | if !utils.FolderExists(filepath.Join(home, ".blackJack")){ 105 | err = os.Mkdir( filepath.Join(home, ".blackJack"), os.ModePerm) 106 | if err != nil { 107 | return 108 | } 109 | } 110 | 111 | if !utils.FolderExists(filepath.Join(home, ".blackJack", "api")){ 112 | err = os.Mkdir( filepath.Join(home, ".blackJack", "api"), os.ModePerm) 113 | if err != nil { 114 | return 115 | } 116 | } 117 | 118 | if !utils.FolderExists(filepath.Join(home, ".blackJack", "file")){ 119 | err = os.Mkdir( filepath.Join(home, ".blackJack", "file"), os.ModePerm) 120 | if err != nil { 121 | return 122 | } 123 | } 124 | 125 | resp, err := http.Get(baseUrl + name) 126 | if err != nil { 127 | log.Error("download failed, please check you network...") 128 | return 129 | } 130 | data, err := ioutil.ReadAll(resp.Body) 131 | if err != nil { 132 | return 133 | } 134 | if utils.FileExists(filePath){ 135 | err := os.Remove(filePath) 136 | if err != nil { 137 | return err 138 | } 139 | } 140 | err = ioutil.WriteFile(filePath, data, 0666) 141 | if err != nil { 142 | log.Warnf("unable to write dict file, %s", err) 143 | } 144 | return 145 | } -------------------------------------------------------------------------------- /utils/title.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "github.com/t43Wiu6/tlog" 7 | "golang.org/x/net/html" 8 | "io" 9 | "io/ioutil" 10 | . "net/http" 11 | "regexp" 12 | "strings" 13 | ) 14 | 15 | var ( 16 | cutset = "\n\t\v\f\r" 17 | reTitle *regexp.Regexp = regexp.MustCompile(`(?im)<\s*title.*>(.*?)<\s*/\s*title>`) 18 | reTitle2 *regexp.Regexp = regexp.MustCompile(`document\.title=(.+);`) 19 | reContentType *regexp.Regexp = regexp.MustCompile(`(?im)\s*charset="(.*?)"|charset=(.*?)"\s*`) 20 | ) 21 | 22 | // ExtractTitle from a response 23 | func ExtractTitle(r *Response) (title string, body_content string) { 24 | bodyBak, err := ioutil.ReadAll(r.Body) 25 | // Try to parse the DOM 26 | titleDom, err, body := getTitleWithDom(r) 27 | // fix body for re extract title 28 | r.Body = ioutil.NopCloser(bytes.NewBuffer(bodyBak)) 29 | // convert io.ReadCloser to string 30 | buf := new(bytes.Buffer) 31 | buf.ReadFrom(r.Body) 32 | newStr := buf.String() 33 | body_content = ByteArrayToString(body) 34 | // In case of error fallback to regex 35 | if err != nil { 36 | for _, match := range reTitle.FindAllString(newStr, -1) { 37 | title = match 38 | break 39 | } 40 | } else { 41 | title = renderNode(titleDom) 42 | } 43 | 44 | title = html.UnescapeString(trimTitleTags(title)) 45 | // remove unwanted chars 46 | title = strings.TrimSpace(strings.Trim(title, cutset)) 47 | 48 | // support overwrite title with js 49 | if title == ""{ 50 | for _, match := range reTitle2.FindAllString(newStr, -1) { 51 | title = match 52 | break 53 | } 54 | title = strings.Replace(title,"document.title","",-1) 55 | title = strings.Replace(title,"=","",-1) 56 | title = strings.Replace(title,"'","",-1) 57 | title = strings.Replace(title,";","",-1) 58 | title = strings.Replace(title,"\"","",-1) 59 | } 60 | 61 | // Non UTF-8 62 | contentTypes := r.Header.Values("Content-Type") 63 | if len(contentTypes) > 0 { 64 | contentType := strings.Join(contentTypes, ";") 65 | 66 | // special cases 67 | if strings.Contains(strings.ToLower(contentType), "charset=gb2312") || 68 | strings.Contains(strings.ToLower(contentType), "charset=gbk") { 69 | titleUtf8, err := Decodegbk([]byte(title)) 70 | if err != nil { 71 | return 72 | } 73 | return string(titleUtf8), body_content 74 | } 75 | 76 | // Content-Type from head tag 77 | var match = reContentType.FindSubmatch(body) 78 | var mcontentType = "" 79 | if len(match) != 0 { 80 | for i, v := range match { 81 | if string(v) != "" && i != 0 { 82 | mcontentType = string(v) 83 | } 84 | } 85 | mcontentType = strings.ToLower(mcontentType) 86 | } 87 | if strings.Contains(mcontentType, "gb2312") || strings.Contains(mcontentType, "gbk") { 88 | titleUtf8, err := Decodegbk([]byte(title)) 89 | if err != nil { 90 | return 91 | } 92 | return string(titleUtf8), body_content 93 | } 94 | } 95 | return //nolint 96 | } 97 | 98 | func getTitleWithDom(r *Response) (*html.Node, error, []byte) { 99 | var title *html.Node 100 | var crawler func(*html.Node) 101 | crawler = func(node *html.Node) { 102 | if node.Type == html.ElementNode && node.Data == "title" { 103 | title = node 104 | return 105 | } 106 | for child := node.FirstChild; child != nil; child = child.NextSibling { 107 | crawler(child) 108 | } 109 | } 110 | body, err := ioutil.ReadAll(r.Body) 111 | 112 | if err != nil { 113 | log.Errorf("Error reading body: %v", err) 114 | } 115 | htmlDoc, err := html.Parse(bytes.NewReader(body)) 116 | if err != nil { 117 | return nil, err, body 118 | } 119 | crawler(htmlDoc) 120 | if title != nil { 121 | return title, nil, body 122 | } 123 | return nil, fmt.Errorf("title not found"), body 124 | } 125 | 126 | func renderNode(n *html.Node) string { 127 | var buf bytes.Buffer 128 | w := io.Writer(&buf) 129 | html.Render(w, n) //nolint 130 | return buf.String() 131 | } 132 | 133 | func trimTitleTags(title string) string { 134 | // trim * 135 | titleBegin := strings.Index(title, ">") 136 | titleEnd := strings.Index(title, " 399 && resp.StatusCode != 500 && resp.StatusCode != 403 { 130 | log.Debugf("[DirBrute] Detected url : %s, StatusCode : %d", dictUrl, resp.StatusCode) 131 | return 132 | } 133 | if len(data) == 0 { 134 | log.Debugf("[DirBrute] Detected url : %s, Body Data 0 size", dictUrl) 135 | return 136 | } 137 | 138 | // 不相似 139 | h := utils.GetHash(data) 140 | if dir.isInBlackList(h) { 141 | log.Debugf("The similarity of the page %s to BlackList is less than 8, StatusCode : %d, hash is %d", dictUrl, resp.StatusCode, h) 142 | return 143 | } 144 | 145 | // 与已知页面完全相同的页面，不再输出 146 | for _, hash := range dir.ExistPageSimHash { 147 | if utils.IsEqualHash(hash, h) { 148 | log.Debugf("The page %s same as the known page, StatusCode : %d, hash is %d", dictUrl, resp.StatusCode, h) 149 | return 150 | } 151 | } 152 | 153 | // 保存新页面hash 154 | dir.Lock() 155 | dir.ExistPageSimHash = append(dir.ExistPageSimHash, h) 156 | dir.Unlock() 157 | 158 | template := fmt.Sprintf("[!] [%d] [%d] - %s", resp.StatusCode, len(data), dictUrl) 159 | log.Debugf("[DirBrute] Detected url %s hash is %d, %d", dictUrl, h, resp.StatusCode) 160 | dir.list = append(dir.list, template) 161 | } 162 | 163 | // isInBlackList 根据simhash算法计算相似性，不相似即返回false 164 | func (dir *DirBrute) isInBlackList(h uint64) bool { 165 | // TODO 166 | for _, hash := range dir.Simhash { 167 | // hash类似， 认为是两个近似页面 168 | if utils.IsSimilarHash(hash, h) { 169 | return true 170 | } 171 | } 172 | return false 173 | } 174 | 175 | // detectPseudo 计算主页和不存在页面的simhash 用于检测 176 | func (dir *DirBrute) detectPseudo() (err error) { 177 | dir.Lock() 178 | defer dir.Unlock() 179 | resp, err := dir.client.Get(dir.ErrorUrl) 180 | if err == nil { 181 | data := utils.DumpHttpResponse(resp) 182 | hash := utils.GetHash(data) 183 | dir.Simhash = append(dir.Simhash, hash) 184 | log.Debugf("[DirBrute] %s hash is : %d", dir.ErrorUrl, hash) 185 | } 186 | 187 | resp, err = dir.client.Get(dir.IndexUrl) 188 | if err == nil { 189 | data := utils.DumpHttpResponse(resp) 190 | hash := utils.GetHash(data) 191 | dir.Simhash = append(dir.Simhash, hash) 192 | log.Debugf("[DirBrute] %s hash is : %d", dir.IndexUrl, hash) 193 | } 194 | return 195 | } 196 | -------------------------------------------------------------------------------- /runner/request.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "github.com/Athena1337/blackJack/utils" 5 | "bytes" 6 | "context" 7 | "fmt" 8 | "github.com/chromedp/cdproto/page" 9 | "github.com/chromedp/chromedp" 10 | pdhttputil "github.com/projectdiscovery/httputil" 11 | "github.com/t43Wiu6/tlog" 12 | "io/ioutil" 13 | "net/http" 14 | "net/url" 15 | "strconv" 16 | "strings" 17 | "time" 18 | "unicode/utf8" 19 | ) 20 | 21 | // Response contains the response to a server 22 | type Response struct { 23 | StatusCode int 24 | Headers map[string][]string 25 | Host string 26 | URL *url.URL 27 | Title string 28 | Data []byte 29 | ContentLength int 30 | Raw string 31 | RawHeaders string 32 | Words int 33 | Lines int 34 | CSPData *CSPData 35 | HTTP2 bool 36 | Pipeline bool 37 | Duration time.Duration 38 | } 39 | 40 | func (r *Runner) Request(method string, url string, redirect bool) (resp Response, err error) { 41 | timeStart := time.Now() 42 | var gzipRetry bool 43 | request, err := newRequest(method, url) 44 | get_response: 45 | if err != nil { 46 | return 47 | } 48 | var do *http.Response 49 | if redirect { 50 | do, err = r.client.Do(request) 51 | } else { 52 | do, err = r.noRedirectClient.Do(request) 53 | } 54 | 55 | if err != nil { 56 | return 57 | } 58 | 59 | bodyBak, err := ioutil.ReadAll(do.Body) 60 | do.Body = ioutil.NopCloser(bytes.NewBuffer(bodyBak)) 61 | 62 | resp.Title, _ = utils.ExtractTitle(do) 63 | // 拒绝one-shot, 还原Body, 后面还要ExtractTitle 64 | do.Body = ioutil.NopCloser(bytes.NewBuffer(bodyBak)) 65 | 66 | var respbody []byte 67 | // websockets don't have a readable body 68 | if do.StatusCode != http.StatusSwitchingProtocols { 69 | respbody, err = ioutil.ReadAll(do.Body) 70 | if err != nil { 71 | return 72 | } 73 | } 74 | // 后面还要DumpRaw 75 | do.Body = ioutil.NopCloser(bytes.NewBuffer(bodyBak)) 76 | 77 | rawHeader, rawResp, err := pdhttputil.DumpResponseHeadersAndRaw(do) 78 | if err != nil { 79 | // Edge case - some servers respond with gzip encoding header but uncompressed body, in this case the standard library configures the reader as gzip, triggering an error when read. 80 | // The bytes slice is not accessible because of abstraction, therefore we need to perform the request again tampering the Accept-Encoding header 81 | if !gzipRetry && strings.Contains(err.Error(), "gzip: invalid header") { 82 | gzipRetry = true 83 | request.Header.Set("Accept-Encoding", "identity") 84 | goto get_response 85 | } 86 | } 87 | resp.Raw = string(rawResp) 88 | resp.RawHeaders = string(rawHeader) 89 | resp.Headers = do.Header.Clone() 90 | resp.Host = request.Host 91 | resp.URL = request.URL 92 | 93 | err = do.Body.Close() 94 | if err != nil { 95 | return 96 | } 97 | 98 | respbodystr := string(respbody) 99 | 100 | // if content length is not defined 101 | if resp.ContentLength <= 0 { 102 | // check if it's in the header and convert to int 103 | if contentLength, ok := resp.Headers["Content-Length"]; ok { 104 | contentLengthInt, _ := strconv.Atoi(strings.Join(contentLength, "")) 105 | resp.ContentLength = contentLengthInt 106 | } 107 | 108 | // if we have a body, then use the number of bytes in the body if the length is still zero 109 | if resp.ContentLength <= 0 && len(respbodystr) > 0 { 110 | resp.ContentLength = utf8.RuneCountInString(respbodystr) 111 | } 112 | } 113 | 114 | resp.Data = respbody 115 | 116 | // fill metrics 117 | resp.StatusCode = do.StatusCode 118 | // number of words 119 | resp.Words = len(strings.Split(respbodystr, " ")) 120 | // number of lines 121 | resp.Lines = len(strings.Split(respbodystr, "\n")) 122 | 123 | resp.Duration = time.Since(timeStart) 124 | 125 | return 126 | } 127 | 128 | // newRequest from url 129 | func newRequest(method, targetURL string) (req *http.Request, err error) { 130 | req, err = http.NewRequest(method, targetURL, nil) 131 | if err != nil { 132 | return 133 | } 134 | 135 | // set default user agent 136 | req.Header.Add("User-Agent", utils.GetUserAgent()) 137 | // 检测shiro指纹 138 | req.Header.Add("Cookie", fmt.Sprintf("rememberMe=%s", utils.RandStringBytes(67))) 139 | // set default encoding to accept utf8 140 | req.Header.Add("Accept-Charset", "utf-8") 141 | return 142 | } 143 | 144 | func RequestByChrome(url string, screenPath string) (title []string, err error) { 145 | // create context 146 | opts := append(chromedp.DefaultExecAllocatorOptions[:], 147 | chromedp.Flag("headless", true), 148 | chromedp.WindowSize(1980, 1080), 149 | chromedp.Flag("ignore-certificate-errors", "1"), 150 | chromedp.Flag("enable-features", "NetworkService"), 151 | chromedp.UserAgent(`Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36`), 152 | ) 153 | allocCtx, cancel := chromedp.NewExecAllocator(context.Background(), opts...) 154 | defer cancel() 155 | ctx, cancel := chromedp.NewContext(allocCtx) 156 | defer cancel() 157 | 158 | // 创建超时时间 159 | ctx, cancel = context.WithTimeout(ctx, 15*time.Second) 160 | defer cancel() 161 | // 缓存对象 162 | var buf []byte 163 | // 运行截屏 164 | if err = chromedp.Run(ctx, fullScreenshot(url, 100, &buf)); err != nil { 165 | return 166 | } 167 | // 保存文件 168 | if err = ioutil.WriteFile(screenPath, buf, 0644); err != nil { 169 | log.Error(err.Error()) 170 | } 171 | return 172 | } 173 | 174 | // fullScreenshot 全屏截图 175 | func fullScreenshot(url string, quality int64, res *[]byte) chromedp.Tasks { 176 | return chromedp.Tasks{ 177 | chromedp.Navigate(url), 178 | ////延时：等待有些页面有js自动跳转，待js跳转后再执行截图操作 179 | chromedp.Sleep(5 * time.Second), 180 | chromedp.ActionFunc(func(ctx context.Context) (err error) { 181 | *res, err = page.CaptureScreenshot().WithQuality(quality).WithClip(&page.Viewport{ 182 | X: 0, 183 | Y: 0, 184 | Width: 1980, 185 | Height: 1080, 186 | Scale: 1, 187 | }).Do(ctx) 188 | if err != nil { 189 | return err 190 | } 191 | return nil 192 | }), 193 | } 194 | } 195 | -------------------------------------------------------------------------------- /utils/useragent.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "math/rand" 5 | "time" 6 | ) 7 | 8 | var UA = [...] string{ 9 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11", 10 | "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 OPR/26.0.1656.60", 11 | "Opera/8.0 (Windows NT 5.1; U; en)", 12 | "Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera 9.50", 13 | "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 9.50", 14 | "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11", 15 | "Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11", 16 | "Opera/9.80 (Android 2.3.4; Linux; Opera Mobi/build-1107180945; U; en-GB) Presto/2.8.149 Version/11.10", 17 | "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0", 18 | "Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10", 19 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv,2.0.1) Gecko/20100101 Firefox/4.0.1", 20 | "Mozilla/5.0 (Windows NT 6.1; rv,2.0.1) Gecko/20100101 Firefox/4.0.1", 21 | "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2", 22 | "MAC：Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.122 Safari/537.36", 23 | "Windows：Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50", 24 | "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5", 25 | "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5", 26 | "Mozilla/5.0 (iPad; U; CPU OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5", 27 | "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36", 28 | "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11", 29 | "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16", 30 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11", 31 | "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36", 32 | "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko", 33 | "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)", 34 | "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.11 TaoBrowser/2.0 Safari/536.11", 35 | "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER", 36 | "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)", 37 | "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)", 38 | "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0", 39 | "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; SE 2.X MetaSr 1.0)", 40 | "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)", 41 | "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Maxthon/4.4.3.4000 Chrome/30.0.1599.101 Safari/537.36", 42 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11", 43 | "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 UBrowser/4.0.3214.0 Safari/537.36", 44 | "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 UBrowser/6.2.4094.1 Safari/537.36", 45 | "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5", 46 | "Mozilla/5.0 (iPod; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5", 47 | "Mozilla/5.0 (iPad; U; CPU OS 4_2_1 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148 Safari/6533.18.5", 48 | "Mozilla/5.0 (iPad; U; CPU OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5", 49 | "Mozilla/5.0 (Linux; U; Android 2.2.1; zh-cn; HTC_Wildfire_A3333 Build/FRG83D) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", 50 | "Mozilla/5.0 (Linux; U; Android 2.3.7; en-us; Nexus One Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", 51 | "MQQBrowser/26 Mozilla/5.0 (Linux; U; Android 2.3.7; zh-cn; MB200 Build/GRJ22; CyanogenMod-7) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", 52 | "Opera/9.80 (Android 2.3.4; Linux; Opera Mobi/build-1107180945; U; en-GB) Presto/2.8.149 Version/11.10", 53 | "Mozilla/5.0 (Linux; U; Android 3.0; en-us; Xoom Build/HRI39) AppleWebKit/534.13 (KHTML, like Gecko) Version/4.0 Safari/534.13", 54 | "Mozilla/5.0 (BlackBerry; U; BlackBerry 9800; en) AppleWebKit/534.1+ (KHTML, like Gecko) Version/6.0.0.337 Mobile Safari/534.1+", 55 | "Mozilla/5.0 (hp-tablet; Linux; hpwOS/3.0.0; U; en-US) AppleWebKit/534.6 (KHTML, like Gecko) wOSBrowser/233.70 Safari/534.6 TouchPad/1.0", 56 | "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0;", 57 | "Mozilla/5.0 (Linux; U; Android 2.3.7; en-us; Nexus One Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", 58 | "Mozilla/5.0 (SymbianOS/9.4; Series60/5.0 NokiaN97-1/20.0.019; Profile/MIDP-2.1 Configuration/CLDC-1.1) AppleWebKit/525 (KHTML, like Gecko) BrowserNG/7.1.18124", 59 | "Mozilla/5.0 (compatible; MSIE 9.0; Windows Phone OS 7.5; Trident/5.0; IEMobile/9.0; HTC; Titan)", 60 | "UCWEB7.0.2.37/28/999", 61 | "NOKIA5700/ UCWEB7.0.2.37/28/999", 62 | "Openwave/ UCWEB7.0.2.37/28/999", 63 | "Openwave/ UCWEB7.0.2.37/28/999", 64 | } 65 | 66 | func GetUserAgent() (userAgent string) { 67 | s := rand.NewSource(time.Now().Unix()) 68 | r := rand.New(s) // initialize local pseudorandom generator 69 | userAgent = UA[r.Intn(len(UA))] 70 | return userAgent 71 | } 72 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /runner/runner.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "github.com/Athena1337/blackJack/brute" 5 | "github.com/Athena1337/blackJack/config" 6 | "github.com/Athena1337/blackJack/finger" 7 | "github.com/Athena1337/blackJack/utils" 8 | "crypto/tls" 9 | "errors" 10 | "fmt" 11 | "github.com/hashicorp/go-retryablehttp" 12 | "github.com/projectdiscovery/cdncheck" 13 | "github.com/projectdiscovery/fastdialer/fastdialer" 14 | pdhttputil "github.com/projectdiscovery/httputil" 15 | "github.com/pterm/pterm" 16 | "github.com/remeh/sizedwaitgroup" 17 | "github.com/t43Wiu6/tlog" 18 | "io/ioutil" 19 | "net" 20 | "net/http" 21 | "net/url" 22 | "strconv" 23 | "strings" 24 | "time" 25 | "unicode/utf8" 26 | ) 27 | 28 | var ( 29 | focusOn []string 30 | CONFIG finger.Config 31 | dirWg = sizedwaitgroup.New(10) 32 | ) 33 | 34 | // Runner A user options 35 | type Runner struct { 36 | noRedirectClient *http.Client 37 | client *http.Client 38 | Dialer *fastdialer.Dialer 39 | options *config.Options 40 | printer *pterm.SpinnerPrinter 41 | DirStatus brute.DirStatus 42 | DirBrutes []brute.DirBrute 43 | } 44 | 45 | // Result of a scan 46 | type Result struct { 47 | Raw string 48 | URL string `json:"url,omitempty"` 49 | Location string `json:"location,omitempty"` 50 | Title string `json:"title,omitempty"` 51 | Host string `json:"host,omitempty"` 52 | ContentLength int64 `json:"content-length,omitempty"` 53 | StatusCode int `json:"status-code,omitempty"` 54 | VHost string `json:"vhost,omitempty"` 55 | CDN string `json:"cdn,omitempty"` 56 | Finger []string `json:"finger,omitempty"` 57 | DirBruteRs []string `json:"dir"` 58 | Technologies []string `json:"technologies,omitempty"` 59 | } 60 | 61 | func New(options *config.Options) (*Runner, error) { 62 | runner := &Runner{ 63 | options: options, 64 | } 65 | if options.EnableDirBrute { 66 | spinnerLiveText, _ := pterm.DefaultSpinner.Start("[DirBrute] Waiting Target to Brute Force from Alive Module") 67 | runner.printer = spinnerLiveText 68 | } 69 | 70 | // 创建不允许重定向的http client 71 | var redirectFunc = func(_ *http.Request, _ []*http.Request) error { 72 | // Tell the http client to not follow redirect 73 | return http.ErrUseLastResponse 74 | } 75 | retryClient := retryablehttp.NewClient() 76 | retryClient.RetryMax = runner.options.RetryMax 77 | 78 | transport := &http.Transport{ 79 | DialContext: (&net.Dialer{ 80 | Timeout: 10 * time.Second, 81 | KeepAlive: 10 * time.Second, 82 | DualStack: true, 83 | }).DialContext, 84 | MaxIdleConnsPerHost: -1, 85 | TLSClientConfig: &tls.Config{ 86 | InsecureSkipVerify: true, // don't check cert 87 | }, 88 | DisableKeepAlives: true, 89 | } 90 | 91 | if runner.options.Proxy != "" { 92 | proxyUrl, err := url.Parse(runner.options.Proxy) 93 | if err != nil { 94 | log.Error("Proxy Url Can Not Identify, Droped") 95 | } else { 96 | transport.Proxy = http.ProxyURL(proxyUrl) 97 | } 98 | } 99 | 100 | runner.client = retryClient.StandardClient() // *http.Client 101 | runner.client.Timeout = runner.options.TimeOut 102 | runner.client.Transport = transport 103 | 104 | runner.noRedirectClient = retryClient.StandardClient() // *http.Client 105 | runner.noRedirectClient.CheckRedirect = redirectFunc 106 | runner.noRedirectClient.Timeout = runner.options.TimeOut 107 | runner.noRedirectClient.Transport = transport 108 | return runner, nil 109 | } 110 | 111 | // CreateRunner 创建扫描 112 | func (r *Runner) CreateRunner() { 113 | var err error 114 | if !config.Check() { 115 | log.Error("unable to found finger & dict file") 116 | errs := config.DownloadAll() 117 | if errs != nil { 118 | log.Fatal("unable to download automatically") 119 | } 120 | } 121 | CONFIG, err = finger.LoadFinger() 122 | if err != nil { 123 | log.Fatalf("LoadFinger failed: %v", err) 124 | } 125 | config.SetEnv(r.options.IsDebug) 126 | 127 | // runner 128 | log.Warnf("Default threads: %d", r.options.Threads) 129 | wg := sizedwaitgroup.New(r.options.Threads) // set threads , 50 by default 130 | r.generate(&wg) 131 | wg.Wait() 132 | 133 | // 调度目录爆破 134 | if r.options.EnableDirBrute { 135 | // 计算超时时间大小 136 | dicts, _ := brute.PrepareDict() 137 | var timeout int 138 | if len(dicts) < 4000 { 139 | // 按每秒13个请求计算超时时间 140 | timeout = 300 141 | } else { 142 | // 按每秒220个请求计算超时时间 143 | timeout = len(dicts) / 13 144 | } 145 | 146 | r.DirStatus.AllJob = len(r.DirBrutes) 147 | for _, d := range r.DirBrutes { 148 | ch := make(chan []string) 149 | d := d 150 | dirWg.Add() 151 | go func() { 152 | defer dirWg.Done() 153 | r.options.OutputFile.Lock() 154 | defer r.options.OutputFile.Unlock() 155 | if r.options.Output != "" { 156 | select { 157 | case dirbResult := <-ch: 158 | r.DirStatus.DoneJob = r.DirStatus.DoneJob + 1 159 | if len(dirbResult) < 1 { 160 | return 161 | } 162 | // 后写入 目录爆破结果 163 | for _, raw := range dirbResult { 164 | _, err = r.options.OutputFile.File.WriteString(raw + "\n") 165 | pterm.DefaultBasicText.Print(raw + "\n") 166 | if err != nil { 167 | log.Errorf("Could not write output file '%s': %s", r.options.Output, err) 168 | } 169 | } 170 | case <-time.After(time.Duration(timeout) * time.Second): 171 | log.Error("a dirBrute task timed out") 172 | return 173 | } 174 | } else if r.options.Output == "" { 175 | select { 176 | case dirbResult := <-ch: 177 | r.DirStatus.DoneJob = r.DirStatus.DoneJob + 1 178 | if len(dirbResult) < 1 { 179 | return 180 | } 181 | // 没有输出文件 直接打印目录爆破结果 182 | for _, raw := range dirbResult { 183 | pterm.DefaultBasicText.Print(raw + "\n") 184 | } 185 | case <-time.After(time.Duration(timeout) * time.Second): 186 | log.Error("a dirBrute task timed out") 187 | return 188 | } 189 | } 190 | }() 191 | go d.Start(ch, r.printer, &r.DirStatus) 192 | } 193 | dirWg.Wait() 194 | } 195 | 196 | // 扫描结束 追加重点资产列表 197 | if r.options.Output != "" && len(focusOn) != 0 { 198 | r.options.OutputFile.Lock() 199 | defer r.options.OutputFile.Unlock() 200 | _, err = r.options.OutputFile.File.WriteString(fmt.Sprintf("%s", "\n重点资产: \n")) 201 | if err != nil { 202 | log.Fatalf("Could not write output file '%s': %s", r.options.Output, err) 203 | } 204 | 205 | for _, fo := range focusOn { 206 | _, err := r.options.OutputFile.File.WriteString(fo + "\n") 207 | if err != nil { 208 | continue 209 | } 210 | } 211 | } 212 | } 213 | 214 | // generate 读取文件或直接处理url并开始调度 215 | func (r *Runner) generate(wg *sizedwaitgroup.SizedWaitGroup) { 216 | if r.options.TargetUrl != "" { 217 | log.Infof("single target: %s", r.options.TargetUrl) 218 | wg.Add() 219 | go r.process(r.options.TargetUrl, wg) 220 | } else { 221 | urls, err := utils.ReadFile(r.options.UrlFile) 222 | if err != nil { 223 | log.Fatal("Can't read url file") 224 | } else { 225 | log.Infof("Read %d's url", len(urls)) 226 | for _, u := range urls { 227 | wg.Add() 228 | go r.process(u, wg) 229 | } 230 | } 231 | } 232 | } 233 | 234 | // process 请求获取每个url内容并分析后输出 235 | func (r *Runner) process(url string, wg *sizedwaitgroup.SizedWaitGroup) () { 236 | defer wg.Done() 237 | options := &config.Options{} 238 | origProtocol := options.OrigProtocol 239 | log.Debug(url) 240 | faviconHash, headerContent, urlContent, resultContent, err := r.scan(url, origProtocol) 241 | if err != nil { 242 | return 243 | } 244 | 245 | raw := makeOutput(analyze(faviconHash, headerContent, urlContent, resultContent)) 246 | if r.options.Output != "" { 247 | r.options.OutputFile.Lock() 248 | defer r.options.OutputFile.Unlock() 249 | _, err = r.options.OutputFile.File.WriteString(raw + "\n") 250 | if err != nil { 251 | log.Fatalf("Could not open output file '%s': %s", r.options.Output, err) 252 | } 253 | } 254 | } 255 | 256 | // scan 扫描单个url 257 | func (r *Runner) scan(url string, origProtocol string) (faviconHash string, headerContent []http.Header, urlContent []string, resultContent *Result, err error) { 258 | var indexUrl string 259 | var faviconUrl string 260 | var errorUrl string 261 | var urls []string 262 | 263 | // 适配协议 264 | if utils.ValidateUrl(url) == "" { 265 | log.Fatal("unable to identify url") 266 | } 267 | if utils.ValidateUrl(url) == "http" { 268 | url = strings.Split(url, "://")[1] 269 | log.Debug("validate rs is http") 270 | origProtocol = "http" 271 | } 272 | if utils.ValidateUrl(url) == "https" { 273 | url = strings.Split(url, "://")[1] 274 | log.Debug("validate rs is https") 275 | origProtocol = "https" 276 | } 277 | if strings.HasSuffix(url, "/") { 278 | url = strings.Split(url, "/")[0] 279 | } 280 | log.Debugf("got target: %s", url) 281 | if !strings.Contains(url, ".") { 282 | log.Error("no a valid domain or ip") 283 | err = errors.New("no a valid domain or ip") 284 | return 285 | } 286 | targetUrl := url 287 | prot := "https" // have no protocol, use https default 288 | retried := false 289 | retry: 290 | if origProtocol == "https" { 291 | prot = "https" 292 | } 293 | if origProtocol == "http" { 294 | prot = "http" 295 | } 296 | 297 | if retried && origProtocol == "https" { 298 | prot = "http" 299 | } 300 | 301 | indexUrl = fmt.Sprintf("%s://%s", prot, targetUrl) 302 | faviconUrl = fmt.Sprintf("%s://%s/%s", prot, targetUrl, "favicon.ico") 303 | errorUrl = fmt.Sprintf("%s://%s/%s", prot, targetUrl, utils.RandStringBytes(20)) 304 | 305 | if utils.ValidateUrl(indexUrl) != "" && !retried { 306 | urls = append(urls, indexUrl) 307 | } 308 | if utils.ValidateUrl(errorUrl) != "" && !retried { 309 | urls = append(urls, errorUrl) 310 | } 311 | 312 | // 获得网站主页内容和不存在页面的内容 313 | for k, v := range urls { //range returns both the index and valu 314 | resp, errs := r.Request("GET", v, true) 315 | log.Debugf("GetContent: %v , %d", v, resp.StatusCode) 316 | // 如果是https，则拥有一次重试机会，避免协议不匹配问题 317 | if errs != nil && !retried && origProtocol == "https" { 318 | log.Debugf("request url %s error", v) 319 | retried = true 320 | goto retry 321 | } 322 | if errs != nil && retried || err != nil && origProtocol == "http" || resp.ContentLength == 0 { 323 | err = errors.New("i/o timeout") 324 | log.Debugf("request url %s error: %s", v, err) 325 | return 326 | } else { 327 | urlContent = append(urlContent, string(resp.Data)) 328 | headerContent = append(headerContent, resp.Headers) 329 | if k == 0 { 330 | resultContent = makeResultTemplate(resp) 331 | } 332 | } 333 | } 334 | 335 | // 以非重定向的方式获得网站内容 336 | // 同时分析两种情况，避免重定向跳转导致获得失败，避免反向代理和CDN导致收集面缩小 337 | log.Debugf("GetContent with NoRedirect: %s", indexUrl) 338 | resp, err := r.Request("GET", indexUrl, false) 339 | 340 | if err != nil { 341 | log.Debugf("%v", err) 342 | return 343 | } else { 344 | urlContent = append(urlContent, string(resp.Data)) 345 | headerContent = append(headerContent, resp.Headers) 346 | } 347 | 348 | // 获取icon指纹 349 | log.Debugf("GetIconHash: %s", faviconUrl) 350 | faviconHash, err = r.GetFaviconHash(faviconUrl) 351 | if err == nil && faviconHash != "" { 352 | log.Debugf("GetIconHash: %s %s success", faviconUrl, faviconHash) 353 | } else if err != nil { 354 | log.Debugf("GetIconHash Error %s", err) 355 | } 356 | 357 | // CDN检测 358 | cdn, err := cdncheck.NewWithCache() 359 | if err != nil{ 360 | log.Debugf("Cdn Initialize Failed %v", err) 361 | }else{ 362 | if found, _, err := cdn.Check(net.ParseIP(targetUrl)); found && err == nil { 363 | resultContent.CDN = "isCDN" 364 | } else if err != nil{ 365 | log.Debugf("Check CDN Error, %v", err) 366 | } 367 | } 368 | 369 | if r.options.EnableDirBrute{ 370 | d := brute.DirBrute{ 371 | IndexUrl: indexUrl, 372 | ErrorUrl: errorUrl, 373 | Options: r.options, 374 | } 375 | r.DirBrutes = append(r.DirBrutes, d) 376 | } 377 | return 378 | } 379 | 380 | func makeResultTemplate(resp Response) (r *Result) { 381 | r = &Result{ 382 | Raw: resp.Raw, 383 | URL: resp.URL.String(), 384 | Location: "None", // 385 | Title: resp.Title, 386 | Host: resp.Host, // 387 | ContentLength: int64(resp.ContentLength), 388 | StatusCode: resp.StatusCode, 389 | VHost: "noVhost", // 390 | CDN: "noCDN", // 391 | Finger: []string{}, 392 | Technologies: []string{}, 393 | } 394 | if resp.Title == "" && resp.ContentLength < 120 { 395 | r.Title = strings.Replace(string(resp.Data), "\n", "", -1) 396 | r.Title = strings.Replace(r.Title, " ", "", -1) 397 | } 398 | return 399 | } 400 | 401 | // makeOutput 分析指纹并生成输出 402 | func makeOutput(resp Result) string { 403 | var finger string 404 | var technology string 405 | for k, v := range resp.Finger { 406 | if k == 0 { 407 | finger = v 408 | } else { 409 | finger = finger + "," + v 410 | } 411 | } 412 | 413 | for k, v := range resp.Technologies { 414 | if k == 0 { 415 | technology = v 416 | } else { 417 | technology = technology + "," + v 418 | } 419 | } 420 | row := pterm.NewStyle(pterm.Bold).Sprintf("%s ", resp.URL) 421 | row += fmt.Sprintf("[%s] ", pterm.NewStyle(pterm.FgMagenta, pterm.Bold).Sprintf("%d", resp.StatusCode)) 422 | row += fmt.Sprintf("[%s] ", pterm.NewStyle(pterm.FgCyan).Sprintf("%d", resp.ContentLength)) 423 | row += fmt.Sprintf("[%s] ", pterm.NewStyle(pterm.Bold).Sprintf("%s", resp.Title)) 424 | raw := fmt.Sprintf("%s [%d] [%d] [%s] ", resp.URL, resp.StatusCode, resp.ContentLength, resp.Title) 425 | 426 | if technology != "" { 427 | row += fmt.Sprintf("[%s] ", technology) 428 | raw += fmt.Sprintf("[%s] ", technology) 429 | } 430 | 431 | if finger != "" { 432 | row += fmt.Sprintf("[%s] ", pterm.NewStyle(pterm.FgRed, pterm.Bold).Sprintf("%s", finger)) 433 | raw += fmt.Sprintf("[%s] ", finger) 434 | focusOn = append(focusOn, row) 435 | } 436 | 437 | if resp.VHost != "noVhost" { 438 | row += fmt.Sprintf("[%s] ", resp.VHost) 439 | raw += fmt.Sprintf("[%s] ", resp.VHost) 440 | } 441 | if resp.CDN != "noCDN" { 442 | row += fmt.Sprintf("[%s] ", resp.CDN) 443 | raw += fmt.Sprintf("[%s] ", resp.CDN) 444 | } 445 | pterm.DefaultBasicText.Print(row + "\n") 446 | return raw 447 | } 448 | 449 | // analyze content by fingerprint 450 | func analyze(faviconHash string, headerContent []http.Header, indexContent []string, resultContent *Result) Result { 451 | configs := CONFIG.Rules 452 | result := resultContent 453 | var f finger.Finger 454 | 455 | for _, f = range configs { 456 | for _, p := range f.Fingerprint { 457 | result.Finger = append(result.Finger, detect(f.Name, faviconHash, headerContent, indexContent, p)...) 458 | } 459 | } 460 | 461 | // range all header extract `X-Powered-By` and `Header` value 462 | for _, h := range headerContent { 463 | if h.Get("X-Powered-By") != "" { 464 | result.Technologies = append(result.Technologies, utils.StringArrayToString(h.Values("X-Powered-By"))) 465 | } 466 | if h.Get("Server") != "" { 467 | result.Technologies = append(result.Technologies, utils.StringArrayToString(h.Values("Server"))) 468 | } 469 | break 470 | } 471 | return *result 472 | } 473 | 474 | func detect(name string, faviconHash string, headerContent []http.Header, indexContent []string, mf finger.MetaFinger) (rs []string) { 475 | if len(headerContent) == 0 && len(indexContent) == 0 { 476 | return 477 | } 478 | if mf.Method == "keyword" { 479 | flag := detectKeywords(headerContent, indexContent, mf) 480 | if flag && !utils.StringArrayContains(rs, name) { 481 | rs = append(rs, name) 482 | } 483 | } 484 | 485 | if mf.Method == "faviconhash" && faviconHash != "" { 486 | if mf.Keyword[0] == faviconHash && !utils.StringArrayContains(rs, name) { 487 | rs = append(rs, name) 488 | } 489 | } 490 | return 491 | } 492 | 493 | func detectKeywords(headerContent []http.Header, indexContent []string, mf finger.MetaFinger) bool { 494 | // if keyword in body 495 | if mf.Location == "body" { 496 | for _, k := range mf.Keyword { 497 | for _, c := range indexContent { 498 | // make sure Both conditions are satisfied 499 | // TODO && mf.StatusCode == header 500 | if !strings.Contains(c, k) { 501 | return false 502 | } 503 | } 504 | } 505 | } 506 | 507 | if mf.Location == "header" { 508 | for _, k := range mf.Keyword { 509 | for _, h := range headerContent { 510 | for _, v := range h { 511 | if !strings.Contains(utils.StringArrayToString(v), k) { 512 | return false 513 | } 514 | } 515 | } 516 | } 517 | } 518 | return true 519 | } 520 | 521 | func (r *Runner) Do(req *http.Request) (*Response, error) { 522 | timeStart := time.Now() 523 | 524 | var gzipRetry bool 525 | getResponse: 526 | httpresp, err := r.client.Do(req) 527 | if err != nil { 528 | return nil, err 529 | } 530 | 531 | var resp Response 532 | 533 | resp.Headers = httpresp.Header.Clone() 534 | 535 | // httputil.DumpResponse does not handle websockets 536 | headers, rawResp, err := pdhttputil.DumpResponseHeadersAndRaw(httpresp) 537 | if err != nil { 538 | // Edge case - some servers respond with gzip encoding header but uncompressed body, in this case the standard library configures the reader as gzip, triggering an error when read. 539 | // The bytes slice is not accessible because of abstraction, therefore we need to perform the request again tampering the Accept-Encoding header 540 | if !gzipRetry && strings.Contains(err.Error(), "gzip: invalid header") { 541 | gzipRetry = true 542 | req.Header.Set("Accept-Encoding", "identity") 543 | goto getResponse 544 | } 545 | } 546 | resp.Raw = string(rawResp) 547 | resp.RawHeaders = string(headers) 548 | 549 | var respbody []byte 550 | // websockets don't have a readable body 551 | if httpresp.StatusCode != http.StatusSwitchingProtocols { 552 | var err error 553 | respbody, err = ioutil.ReadAll(httpresp.Body) 554 | if err != nil { 555 | return nil, err 556 | } 557 | } 558 | 559 | closeErr := httpresp.Body.Close() 560 | if closeErr != nil { 561 | return nil, closeErr 562 | } 563 | 564 | respbodystr := string(respbody) 565 | // if content length is not defined 566 | if resp.ContentLength <= 0 { 567 | // check if it's in the header and convert to int 568 | if contentLength, ok := resp.Headers["Content-Length"]; ok { 569 | contentLengthInt, _ := strconv.Atoi(strings.Join(contentLength, "")) 570 | resp.ContentLength = contentLengthInt 571 | } 572 | 573 | // if we have a body, then use the number of bytes in the body if the length is still zero 574 | if resp.ContentLength <= 0 && len(respbodystr) > 0 { 575 | resp.ContentLength = utf8.RuneCountInString(respbodystr) 576 | } 577 | } 578 | 579 | resp.Data = respbody 580 | 581 | // fill metrics 582 | resp.StatusCode = httpresp.StatusCode 583 | // number of words 584 | resp.Words = len(strings.Split(respbodystr, " ")) 585 | // number of lines 586 | resp.Lines = len(strings.Split(respbodystr, "\n")) 587 | 588 | resp.Duration = time.Since(timeStart) 589 | return &resp, nil 590 | } 591 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 2 | github.com/MarvinJWendt/testza v0.1.0/go.mod h1:7AxNvlfeHP7Z/hDQ5JtE3OKYT3XFUeLCDE2DQninSqs= 3 | github.com/MarvinJWendt/testza v0.2.1/go.mod h1:God7bhG8n6uQxwdScay+gjm9/LnO4D3kkcZX4hv9Rp8= 4 | github.com/MarvinJWendt/testza v0.2.8/go.mod h1:nwIcjmr0Zz+Rcwfh3/4UhBp7ePKVhuBExvZqnKYWlII= 5 | github.com/MarvinJWendt/testza v0.2.10/go.mod h1:pd+VWsoGUiFtq+hRKSU1Bktnn+DMCSrDrXDpX2bG66k= 6 | github.com/MarvinJWendt/testza v0.2.12 h1:/PRp/BF+27t2ZxynTiqj0nyND5PbOtfJS0SuTuxmgeg= 7 | github.com/MarvinJWendt/testza v0.2.12/go.mod h1:JOIegYyV7rX+7VZ9r77L/eH6CfJHHzXjB69adAhzZkI= 8 | github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic= 9 | github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= 10 | github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= 11 | github.com/akrylysov/pogreb v0.10.0/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= 12 | github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w= 13 | github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= 14 | github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= 15 | github.com/atomicgo/cursor v0.0.1 h1:xdogsqa6YYlLfM+GyClC/Lchf7aiMerFiZQn7soTOoU= 16 | github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk= 17 | github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= 18 | github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= 19 | github.com/chromedp/cdproto v0.0.0-20220217222649-d8c14a5c6edf/go.mod h1:At5TxYYdxkbQL0TSefRjhLE3Q0lgvqKKMSFUglJ7i1U= 20 | github.com/chromedp/cdproto v0.0.0-20220228194516-ead8cf59b1fa h1:7vxtzxV0Kxcw5JfGHqbBqtPimYynqd3GX1gu+xuXrfM= 21 | github.com/chromedp/cdproto v0.0.0-20220228194516-ead8cf59b1fa/go.mod h1:At5TxYYdxkbQL0TSefRjhLE3Q0lgvqKKMSFUglJ7i1U= 22 | github.com/chromedp/chromedp v0.7.8 h1:JFPIFb28LPjcx6l6mUUzLOTD/TgswcTtg7KrDn8S/2I= 23 | github.com/chromedp/chromedp v0.7.8/go.mod h1:HcIUFBa5vA+u2QI3+xljiU59llUQ8lgGoLzYSCBfmUA= 24 | github.com/chromedp/sysutil v1.0.0 h1:+ZxhTpfpZlmchB58ih/LBHX52ky7w2VhQVKQMucy3Ic= 25 | github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww= 26 | github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4= 27 | github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA= 28 | github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ0/FNU= 29 | github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= 30 | github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= 31 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 32 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 33 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 34 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 35 | github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= 36 | github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= 37 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 38 | github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= 39 | github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= 40 | github.com/go-dedup/megophone v0.0.0-20170830025436-f01be21026f5 h1:4U+x+EB1P66zwYgTjxWXSOT8vF+651Ksr1lojiCZnT8= 41 | github.com/go-dedup/megophone v0.0.0-20170830025436-f01be21026f5/go.mod h1:poR/Cp00iqtqu9ltFwl6C00sKC0HY13u/Gh05ZBmP54= 42 | github.com/go-dedup/simhash v0.0.0-20170904020510-9ecaca7b509c h1:mucYYQn+sMGNSxidhleonzAdwL203RxhjJGnxQU4NWU= 43 | github.com/go-dedup/simhash v0.0.0-20170904020510-9ecaca7b509c/go.mod h1:gO3u2bjRAgUaLdQd2XK+3oooxrheOAx1BzS7WmPzw1s= 44 | github.com/go-dedup/text v0.0.0-20170907015346-8bb1b95e3cb7 h1:11wFcswN+37U+ByjxdKzsRY5KzNqqq5Uk5ztxnLOc7w= 45 | github.com/go-dedup/text v0.0.0-20170907015346-8bb1b95e3cb7/go.mod h1:wSsK4VOECOSfSYTzkBFw+iGY7wj59e7X96ABtNj9aCQ= 46 | github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= 47 | github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= 48 | github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= 49 | github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= 50 | github.com/gobwas/ws v1.1.0 h1:7RFti/xnNkMJnrK7D1yQ/iCIB5OrrY/54/H930kIbHA= 51 | github.com/gobwas/ws v1.1.0/go.mod h1:nzvNcVha5eUziGrbxFCo6qFIojQHjJV5cLYIbezhfL0= 52 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 53 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 54 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 55 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 56 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 57 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 58 | github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 59 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= 60 | github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= 61 | github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 62 | github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 63 | github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 64 | github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= 65 | github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 66 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 67 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 68 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 69 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 70 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 71 | github.com/gookit/color v1.4.2 h1:tXy44JFSFkKnELV6WaMo/lLfu/meqITX3iAV52do7lk= 72 | github.com/gookit/color v1.4.2/go.mod h1:fqRyamkC1W8uxl+lxCQxOT09l/vYfZ+QeiX3rKQHCoQ= 73 | github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= 74 | github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= 75 | github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= 76 | github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= 77 | github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= 78 | github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= 79 | github.com/hashicorp/go-retryablehttp v0.7.0 h1:eu1EI/mbirUgP5C8hVsTNaGZreBDlYiwC1FZWkvQPQ4= 80 | github.com/hashicorp/go-retryablehttp v0.7.0/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= 81 | github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf/go.mod h1:V99KdStnMHZsvVOwIvhfcUzYgYkRZeQWUtumtL+SKxA= 82 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 83 | github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= 84 | github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= 85 | github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 86 | github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 87 | github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= 88 | github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= 89 | github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= 90 | github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= 91 | github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= 92 | github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= 93 | github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= 94 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 95 | github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 96 | github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= 97 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 98 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 99 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 100 | github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= 101 | github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= 102 | github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= 103 | github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= 104 | github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= 105 | github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= 106 | github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6/go.mod h1:WVJJvUw/pIOcwu2O8ZzHEhmigq2jzwRNfJVRMJB7bR8= 107 | github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= 108 | github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= 109 | github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg= 110 | github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= 111 | github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= 112 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 113 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 114 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 115 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 116 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 117 | github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= 118 | github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= 119 | github.com/ngdinhtoan/glide-cleanup v0.2.0/go.mod h1:UQzsmiDOb8YV3nOsCxK/c9zPpCZVNoHScRE3EO9pVMM= 120 | github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= 121 | github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= 122 | github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= 123 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 124 | github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 125 | github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= 126 | github.com/onsi/ginkgo v1.15.0 h1:1V1NfVQR87RtWAgp1lv9JZJ5Jap+XFGKPi00andXGi4= 127 | github.com/onsi/ginkgo v1.15.0/go.mod h1:hF8qUzuuC8DJGygJH3726JnCZX4MYbRB8yFfISqnKUg= 128 | github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 129 | github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= 130 | github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= 131 | github.com/onsi/gomega v1.10.5 h1:7n6FEkpFmfCoo2t+YYqXH0evK+a9ICQz0xcAy9dYcaQ= 132 | github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= 133 | github.com/orisano/pixelmatch v0.0.0-20210112091706-4fa4c7ba91d5 h1:1SoBaSPudixRecmlHXb/GxmaD3fLMtHIDN13QujwQuc= 134 | github.com/orisano/pixelmatch v0.0.0-20210112091706-4fa4c7ba91d5/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= 135 | github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= 136 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 137 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 138 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 139 | github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e h1:7bwaFH1jvtOo5ndhTQgoA349ozhX+1dc4b6tbaPnBOA= 140 | github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e/go.mod h1:/IsapnEYiWG+yEDPXp0e8NWj3npzB9Ccy9lXEUJwMZs= 141 | github.com/projectdiscovery/cdncheck v0.0.3 h1:li2/rUJmhVXSqRFyhJMqi6pdBX6ZxMnwzBfE0Kifj/g= 142 | github.com/projectdiscovery/cdncheck v0.0.3/go.mod h1:EevMeCG1ogBoUJYaa0Mv9R1VUboDm/DiynId7DboKy0= 143 | github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg= 144 | github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 h1:jT6f/cdOpLkp9GAfRrxk57BUjYfIrR8E+AjMv5H5U4U= 145 | github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345/go.mod h1:clhQmPnt35ziJW1AhJRKyu8aygXCSoyWj6dtmZBRjjc= 146 | github.com/projectdiscovery/fastdialer v0.0.14-0.20211117222717-6599e7bc586e/go.mod h1:Mex24omi3RxrmhA8Ote7rw+6LWMiaBvbJq8CNp0ksII= 147 | github.com/projectdiscovery/fastdialer v0.0.14 h1:xTcU8c8wTp+AE92TVLINSCvgXsbF0ITera8HfbU1dok= 148 | github.com/projectdiscovery/fastdialer v0.0.14/go.mod h1:Mex24omi3RxrmhA8Ote7rw+6LWMiaBvbJq8CNp0ksII= 149 | github.com/projectdiscovery/fdmax v0.0.3/go.mod h1:NWRcaR7JTO7fC27H4jCl9n7Z+KIredwpgw1fV+4KrKI= 150 | github.com/projectdiscovery/fileutil v0.0.0-20210926044607-04f32490aa21/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= 151 | github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c/go.mod h1:mBv7GRD5n3WNbFE9blG8ynzXTM5eh9MmwaK6EOyn6Pk= 152 | github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= 153 | github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= 154 | github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY= 155 | github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0= 156 | github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa h1:9sZWFUAshIa/ea0RKjGRuuZiS5PzYXAFjTRUnSbezr0= 157 | github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa/go.mod h1:lV5f/PNPmCCjCN/dR317/chN9s7VG5h/xcbFfXOz8Fo= 158 | github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5/go.mod h1:BueJPSPWAX11IFS6bdAqTkekiIz5Fgco5LVc1kqO9L4= 159 | github.com/projectdiscovery/httputil v0.0.0-20210913064028-815c8db933ff h1:7PH2QB1tyQHyg+Nw8WpRz/POfDl4yvB/MVD4KBiDr/E= 160 | github.com/projectdiscovery/httputil v0.0.0-20210913064028-815c8db933ff/go.mod h1:BueJPSPWAX11IFS6bdAqTkekiIz5Fgco5LVc1kqO9L4= 161 | github.com/projectdiscovery/httpx v1.2.0 h1:UzwO5FoEq4IKe6cPymhZmU6srv9Fkx8OtN0tXky74rw= 162 | github.com/projectdiscovery/httpx v1.2.0/go.mod h1:xgpP6S2OXHwz9LtQOllcQO5obTaoP7vq0nSV33RS6L4= 163 | github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+dKwYooEOHGVPP/RWE= 164 | github.com/projectdiscovery/iputil v0.0.0-20210414194613-4b4d2517acf0/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A= 165 | github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3 h1:VZ9H51A7tI7R/9I5W5l960Nkq7eMJqGd3y1wsuwzdjE= 166 | github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3/go.mod h1:blmYJkS8lSrrx3QcmcgS2tZIxlojeVmoGeA9twslCBU= 167 | github.com/projectdiscovery/mapcidr v0.0.4/go.mod h1:ALOIj6ptkWujNoX8RdQwB2mZ+kAmKuLJBq9T5gR5wG0= 168 | github.com/projectdiscovery/mapcidr v0.0.6/go.mod h1:ZEBhMmBU3laUl3g9QGTrzJku1VJOzjdFwW01f/zVVzM= 169 | github.com/projectdiscovery/mapcidr v0.0.7/go.mod h1:7CzdUdjuLVI0s33dQ33lWgjg3vPuLFw2rQzZ0RxkT00= 170 | github.com/projectdiscovery/mapcidr v0.0.8 h1:16U05F2x3o/jSTsxSCY2hCuCs9xOSwVxjo2zlsL4L4E= 171 | github.com/projectdiscovery/mapcidr v0.0.8/go.mod h1:7CzdUdjuLVI0s33dQ33lWgjg3vPuLFw2rQzZ0RxkT00= 172 | github.com/projectdiscovery/networkpolicy v0.0.1 h1:RGRuPlxE8WLFF9tdKSjTsYiTIKHNHW20Kl0nGGiRb1I= 173 | github.com/projectdiscovery/networkpolicy v0.0.1/go.mod h1:asvdg5wMy3LPVMGALatebKeOYH5n5fV5RCTv6DbxpIs= 174 | github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= 175 | github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f/go.mod h1:3L0WfNIcVWXIDur8k+gKDLZLWY2F+rs0SQXtcn/3AYU= 176 | github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a h1:WJQjr9qi/VjWhdNiGyNqcFi0967Gp0W3I769bCpHOJE= 177 | github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a/go.mod h1:tXaLDs4n3pRZHwfa8mdXpUWe/AYDNK3HlWDjldhRbjI= 178 | github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek= 179 | github.com/projectdiscovery/retryablehttp-go v1.0.2 h1:LV1/KAQU+yeWhNVlvveaYFsjBYRwXlNEq0PvrezMV0U= 180 | github.com/projectdiscovery/retryablehttp-go v1.0.2/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI= 181 | github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea/go.mod h1:QHXvznfPfA5f0AZUIBkbLapoUJJlsIDgUlkKva6dOr4= 182 | github.com/projectdiscovery/stringsutil v0.0.0-20210524051937-51dabe3b72c0/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4= 183 | github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= 184 | github.com/projectdiscovery/stringsutil v0.0.0-20210823090203-2f5f137e8e1d/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= 185 | github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xbL1/7h0k6HE3RzPdYk9W/8pUxESrGWewTaZdIB5Pes= 186 | github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= 187 | github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo= 188 | github.com/projectdiscovery/wappalyzergo v0.0.30/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= 189 | github.com/pterm/pterm v0.12.27/go.mod h1:PhQ89w4i95rhgE+xedAoqous6K9X+r6aSOI2eFF7DZI= 190 | github.com/pterm/pterm v0.12.29/go.mod h1:WI3qxgvoQFFGKGjGnJR849gU0TsEOvKn5Q8LlY1U7lg= 191 | github.com/pterm/pterm v0.12.30/go.mod h1:MOqLIyMOgmTDz9yorcYbcw+HsgoZo3BQfg2wtl3HEFE= 192 | github.com/pterm/pterm v0.12.31/go.mod h1:32ZAWZVXD7ZfG0s8qqHXePte42kdz8ECtRyEejaWgXU= 193 | github.com/pterm/pterm v0.12.33/go.mod h1:x+h2uL+n7CP/rel9+bImHD5lF3nM9vJj80k9ybiiTTE= 194 | github.com/pterm/pterm v0.12.37 h1:QGOyuaDUmY3yTbP0k6i0uPNqNHA9YofEBQDy0tIyKTA= 195 | github.com/pterm/pterm v0.12.37/go.mod h1:NjiL09hFhT/vWjQHSj1athJpx6H8cjpHXNAK5bUw8T8= 196 | github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= 197 | github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= 198 | github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= 199 | github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= 200 | github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= 201 | github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= 202 | github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= 203 | github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= 204 | github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 205 | github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= 206 | github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= 207 | github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= 208 | github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= 209 | github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= 210 | github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= 211 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 212 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 213 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 214 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 215 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= 216 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 217 | github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= 218 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 219 | github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= 220 | github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= 221 | github.com/t43Wiu6/tlog v0.0.0-20220117055205-3d75ddb8e805 h1:W6MFpKbKP9UUVItVETzgAjVUAQg9R0spY+eo2dPnDSA= 222 | github.com/t43Wiu6/tlog v0.0.0-20220117055205-3d75ddb8e805/go.mod h1:X8wyAqj5684vBDbaxgtfxWvsDmFyNgshFyB0RoWUy4E= 223 | github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= 224 | github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= 225 | github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= 226 | github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= 227 | github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 h1:QldyIu/L63oPpyvQmHgvgickp1Yw510KJOqX7H24mg8= 228 | github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs= 229 | github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU= 230 | github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g= 231 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 232 | go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= 233 | go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= 234 | go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= 235 | go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= 236 | go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= 237 | go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg= 238 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 239 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 240 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 241 | golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 242 | golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= 243 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 244 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 245 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 246 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 247 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 248 | golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 249 | golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 250 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 251 | golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 252 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 253 | golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= 254 | golang.org/x/net v0.0.0-20210414194228-064579744ee0/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= 255 | golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 256 | golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 257 | golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 258 | golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc= 259 | golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= 260 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 261 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 262 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 263 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= 264 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 265 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 266 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 267 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 268 | golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 269 | golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 270 | golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 271 | golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 272 | golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 273 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 274 | golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 275 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 276 | golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 277 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 278 | golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 279 | golang.org/x/sys v0.0.0-20201207223542-d4d67f95c62d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 280 | golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 281 | golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 282 | golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 283 | golang.org/x/sys v0.0.0-20210415045647-66c3f260301c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 284 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 285 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 286 | golang.org/x/sys v0.0.0-20210915083310-ed5796bab164/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 287 | golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 288 | golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 289 | golang.org/x/sys v0.0.0-20220209214540-3681064d5158 h1:rm+CHSpPEEW2IsXUib1ThaHIjuBVZjxNgSKmBLFfD4c= 290 | golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 291 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 292 | golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 293 | golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 294 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= 295 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 296 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 297 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 298 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 299 | golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= 300 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 301 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 302 | golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 303 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 304 | golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 305 | golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 306 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 307 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 308 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 309 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 310 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 311 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 312 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 313 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 314 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 315 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 316 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 317 | google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= 318 | google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= 319 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 320 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 321 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 322 | gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= 323 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 324 | gopkg.in/go-dedup/simhash.v2 v2.0.0-20170703161212-581a1061a875 h1:Z8j66KbsIEdPvHdYZBhdAgSh7OI4+PHJN41R7wgMho8= 325 | gopkg.in/go-dedup/simhash.v2 v2.0.0-20170703161212-581a1061a875/go.mod h1:Vo3UnmzfDGF6dHa7wJX6xVE1iXdaYjcCAj30FbzE7pA= 326 | gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= 327 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= 328 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 329 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 330 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 331 | gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 332 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 333 | gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 334 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 335 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 336 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 337 | gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= 338 | gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 339 | --------------------------------------------------------------------------------