├── main.go ├── go.mod ├── config ├── config.yaml └── config.go ├── internal ├── crawler │ ├── linkfind.go │ └── crawler.go ├── analyze │ └── ai.go ├── utils │ └── utils.go ├── output │ └── output.go ├── parser │ └── parser.go └── matcher │ └── matcher.go ├── README.md ├── cmd └── root.go └── go.sum /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "runtime/debug" 7 | 8 | "SecureJS/cmd" 9 | ) 10 | 11 | func main() { 12 | debug.SetTraceback("none") 13 | 14 | // 2) 在最顶层拦截 panic 15 | defer func() { 16 | if r := recover(); r != nil { 17 | fmt.Fprintf(os.Stderr, "[!] An error occurred: %v\n", r) 18 | os.Exit(1) 19 | } 20 | }() 21 | 22 | cmd.Execute() 23 | } -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module SecureJS 2 | 3 | go 1.23 4 | 5 | require ( 6 | github.com/go-rod/rod v0.116.2 7 | github.com/spf13/cobra v1.8.1 8 | github.com/volcengine/volcengine-go-sdk v1.0.181 9 | gopkg.in/yaml.v3 v3.0.1 10 | ) 11 | 12 | require ( 13 | github.com/google/uuid v1.3.0 // indirect 14 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 15 | github.com/jmespath/go-jmespath v0.4.0 // indirect 16 | github.com/spf13/pflag v1.0.5 // indirect 17 | github.com/volcengine/volc-sdk-golang v1.0.23 // indirect 18 | github.com/ysmood/fetchup v0.2.4 // indirect 19 | github.com/ysmood/goob v0.4.0 // indirect 20 | github.com/ysmood/got v0.40.0 // indirect 21 | github.com/ysmood/gson v0.7.3 // indirect 22 | github.com/ysmood/leakless v0.9.0 // indirect 23 | gopkg.in/yaml.v2 v2.2.8 // indirect 24 | ) 25 | -------------------------------------------------------------------------------- /config/config.yaml: -------------------------------------------------------------------------------- 1 | rules: 2 | - name: Extended Sensitive Field 3 | f_regex: "(?i)([\"']?[\\w-]{0,15}(?:key|secret|token|config|auth|access|admin|ticket|api_key|client_secret|private_key|public_key|bearer|session|cookie|license|cert|ssh|salt|pepper)[\\w-]{0,15}[\"']?)\\s*(?:=|:|\\)\\.val\\()\\s*\\[?\\{?(?:'([^']{8,500})'|\"([^\\\"]{8,500})\")(?:[:;,\\}\\]]?)?" 4 | 5 | - name: Extended Password Field 6 | f_regex: "(?i)([\"']?[\\w-]{0,10}(p(?:ass|wd|asswd|assword))[\\w-]{0,10}[\"']?)\\s*[:=]\\s*[\"']([^'\"\\\\]+)[\"']" 7 | 8 | - name: Extended JSON Web Token 9 | f_regex: "(?i)(eyJ[A-Za-z0-9_-]{5,}\\.[A-Za-z0-9._-]{5,}\\.[A-Za-z0-9._-]{5,})" 10 | 11 | - name: Extended Cloud Key 12 | f_regex: "(?i)(AWSAccessKeyId=[A-Z0-9]{16,32}|access[-_]?key[-_]?(?:id|secret)|LTAI[a-z0-9]{12,20}|(?:AKIA|ABIA|ACCA|ASIA)[A-Z0-9]{16}|aws_secret_access_key\\s*=\\s*[\"'][^\"']{8,100}[\"'])" 13 | 14 | - name: Azure Key 15 | f_regex: "(?i)(AZURE_STORAGE[_-]?ACCOUNT[_-]?KEY|AZURE_STORAGE_KEY|AZURE_KEY_VAULT|azure_tenant_id)\\s*=\\s*[\"']([^\"']{8,100})[\"']" 16 | 17 | - name: GCP Service Account 18 | f_regex: "(?s)(\"type\"\\s*:\\s*\"service_account\".*?\"private_key_id\"\\s*:\\s*\"([a-z0-9]{10,})\".*?\"private_key\"\\s*:\\s*\"-----BEGIN PRIVATE KEY-----.*?-----END PRIVATE KEY-----\")" 19 | 20 | - name: Private Key 21 | f_regex: "(?s)-----BEGIN\\s+(?:RSA|EC|DSA|OPENSSH)?\\s*PRIVATE\\s+KEY-----.*?-----END\\s+(?:RSA|EC|DSA|OPENSSH)?\\s*PRIVATE\\s+KEY-----" -------------------------------------------------------------------------------- /internal/crawler/linkfind.go: -------------------------------------------------------------------------------- 1 | package crawler 2 | 3 | import ( 4 | "SecureJS/internal/parser" 5 | "SecureJS/internal/utils" 6 | "regexp" 7 | ) 8 | 9 | func CollectLinksFromBody(urls []string, threads int, uniqueLinks map[string]struct{}, toParse *[]string, customHeaders []string, proxy string) error { 10 | // 解析所有 URL 的内容 11 | parsedResult, err := parser.ParseAll(urls, threads, customHeaders, proxy) 12 | if err != nil { 13 | //return fmt.Errorf("解析失败: %v", err) 14 | } 15 | 16 | // 正则表达式匹配 http 或 https 链接 17 | urlRegex := regexp.MustCompile(`https?://[^\s"']+`) 18 | 19 | for _, parsed := range parsedResult { 20 | if parsed.Error != nil { 21 | //fmt.Printf("[!] 解析 URL: %s, 错误: %v\n", parsed.URL, parsed.Error) 22 | continue 23 | } 24 | 25 | // 查找所有匹配的链接 26 | foundURLs := urlRegex.FindAllString(parsed.Body, -1) 27 | 28 | for _, extractedURL := range foundURLs { 29 | // 调用 HasSkipExtension 函数判断是否过滤该链接 30 | if utils.HasSkipExtension(extractedURL) { 31 | continue 32 | } 33 | // 调用 Skip 函数判断是否过滤该链接 34 | if utils.Skip(extractedURL) { 35 | continue 36 | } 37 | 38 | if _, exists := uniqueLinks[extractedURL]; !exists { 39 | uniqueLinks[extractedURL] = struct{}{} 40 | *toParse = append(*toParse, extractedURL) 41 | } 42 | } 43 | } 44 | 45 | return nil 46 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SecureJS 2 | 3 | SecureJS 是一个强大的工具，旨在从目标网站收集所有相关链接，对这些链接（主要是 JavaScript 文件）执行请求，并扫描敏感信息，如令牌、密钥、密码、AKSK 等，后引入 DeepSeek 对结果进行分析。 4 | 5 | ## 目录 6 | 7 | - [SecureJS](#securejs) 8 | - [目录](#目录) 9 | - [使用方法](#使用方法) 10 | - [帮助信息](#帮助信息) 11 | - [示例](#示例) 12 | - [配置](#配置) 13 | - [项目结构](#项目结构) 14 | - [免责声明](#免责声明) 15 | 16 | ## 使用方法 17 | 18 | ### 帮助信息 19 | 20 | ``` 21 | Usage: 22 | SecureJS [flags] 23 | 24 | Flags: 25 | -a, --ai string true/false. Enable AI Analytics. If not set, will use false (default "false") 26 | -b, --browser string Path to Chrome/Chromium executable (optional). If not set, will use Rod's default. 27 | -c, --config string Path to config file (e.g. config.yaml) (default "config/config.yaml") 28 | -H, --header stringArray Add custom request headers. (e.g. -H 'Key: Value') 29 | -h, --help help for SecureJS 30 | -i, --id string YOUR_ENDPOINT_ID 31 | -k, --key string ARK_API_KEY 32 | -l, --list string File containing target URLs (one per line) 33 | -o, --output string Output file (supports .txt, .csv, .json) 34 | -p, --proxy string Proxy to use (e.g. http://127.0.0.1:8080) 35 | -t, --threads int Number of concurrent threads for scanning (default 20) 36 | -u, --url string Single target URL to scan (e.g. https://example.com) 37 | ``` 38 | 39 | ### 示例 40 | 41 | 42 | ## 配置 43 | 44 | SecureJS 使用 `config/config.yaml` 文件来定义自定义匹配规则和其他项目级配置。如不存在，首次运行后将自动生成该文件。另外，规则将进行尽可能的匹配，因为后续可进行AI分析，但这也会导致AI分析前误报结果高。 45 | 46 | ## 项目结构 47 | 48 | ``` 49 | SecureJS/ 50 | ├── cmd/ 51 | │ └── root.go # 处理命令行参数（-u、-l、-t 等）的入口点 52 | │ 53 | ├── internal/ 54 | │ ├── crawler/ 55 | │ │ └── ai.go # 引入 DeepSeek 对结果二次分析 56 | │ │ 57 | │ ├── crawler/ 58 | │ │ ├── crawler.go # 爬虫逻辑，模拟浏览器访问，收集所有链接和 JS 文件 59 | │ │ └── linkfind.go # 从目标页面的响应体中提取所有链接和 JS 60 | │ │ 61 | │ ├── parser/ 62 | │ │ └── parser.go # 对所有收集的链接和 JS 文件执行二次请求 63 | │ │ 64 | │ ├── matcher/ 65 | │ │ └── matcher.go # 从 config.yaml 中读取并解析自定义规则，并与响应体匹配 66 | │ │ 67 | │ └── output/ 68 | │ └── output.go # 将结果输出为 CSV、JSON 或文本格式的文件 69 | │ 70 | ├── config/ 71 | │ ├── config.go # 处理配置文件（config.yaml）的加载和解析 72 | │ └── config.yaml # 自定义规则和其他项目级配置 73 | │ 74 | ├── go.mod # Go Modules 管理文件 75 | ├── go.sum # Go Modules 校验文件 76 | └── main.go # 主程序入口点，初始化并启动应用程序 77 | ``` 78 | 79 | ## 免责声明 80 | 81 | 本工具仅用于安全研究与合法测试目的。请确保遵守相关法律法规，不得将本工具用于任何非法或未经授权的行为。作者及项目维护者对因使用或滥用本工具而导致的任何损失或损害，不承担任何责任。 -------------------------------------------------------------------------------- /internal/analyze/ai.go: -------------------------------------------------------------------------------- 1 | package analyze 2 | 3 | import ( 4 | "SecureJS/internal/matcher" 5 | "context" 6 | "fmt" 7 | "time" 8 | 9 | "github.com/volcengine/volcengine-go-sdk/service/arkruntime" 10 | "github.com/volcengine/volcengine-go-sdk/service/arkruntime/model" 11 | "github.com/volcengine/volcengine-go-sdk/volcengine" 12 | ) 13 | 14 | func FormatResultsToString(results []*matcher.MatchResult) string { 15 | var resultString string 16 | for _, mr := range results { 17 | if mr.Error != nil { 18 | resultString += fmt.Sprintf("\n[!] Parse error on %s: %v\n", mr.URL, mr.Error) 19 | continue 20 | } 21 | if len(mr.Items) == 0 { 22 | continue 23 | } 24 | resultString += fmt.Sprintf("\n[+] %s: found %d item(s)\n", mr.URL, len(mr.Items)) 25 | for _, item := range mr.Items { 26 | resultString += fmt.Sprintf(" - Rule: %s, Matched: %s\n", item.RuleName, item.MatchedText) 27 | } 28 | } 29 | return resultString 30 | } 31 | 32 | func Analyze(resultString string, key string, id string) { 33 | var instructionString = ` 34 | 您是资深网络安全专家，擅长识别代码中的敏感信息泄露。请保持专业严谨，区分测试数据与实际风险。 35 | 请按以下步骤处理输入内容（主要目标是从JS中寻找一些硬编码的”有具体数值的”敏感信息，不需要乱七八遭的代码如某token、key等值为+t.access_token+这样形式的）： 36 | 1. 提取条目：识别所有符合 字段名 + 赋值符 + 值 结构的条目 37 | 2. 过滤规则： 38 | （1）排除项：无明确值的字段（例如只包含一个AccessKeyId而没有具体的值）；公开/测试密钥；通用配置；没有具体的敏感信息数值（有的只是代码？乱糟糟的？）； 39 | 3. 输出格式（严格遵循，不要多余的输出比如总结什么的，就只输出我下面三行内容）： 40 | URL：{文件URL} 41 | 敏感信息：{字段名} = {值} 42 | 分析：{风险说明，包含用途、泄露后果、是否生产环境} 43 | 4. 如果此次分析整体并没有任何敏感信息，就直接表明 “无敏感信息”` 44 | 45 | client := arkruntime.NewClientWithApiKey( 46 | //通过 os.Getenv 从环境变量中获取 ARK_API_KEY 47 | key, 48 | //深度推理模型耗费时间会较长，请您设置较大的超时时间，避免超时导致任务失败。推荐30分钟以上 49 | arkruntime.WithTimeout(30*time.Minute), 50 | ) 51 | // 创建一个上下文，通常用于传递请求的上下文信息，如超时、取消等 52 | ctx := context.Background() 53 | // 构建聊天完成请求，设置请求的模型和消息内容 54 | req := model.ChatCompletionRequest{ 55 | // 需要替换 为您的推理接入点 ID 56 | Model: id, 57 | Messages: []*model.ChatCompletionMessage{ 58 | { 59 | // 消息的角色为用户 60 | Role: model.ChatMessageRoleUser, 61 | Content: &model.ChatCompletionMessageContent{ 62 | StringValue: volcengine.String(instructionString + resultString), 63 | }, 64 | }, 65 | }, 66 | } 67 | 68 | // 发送聊天完成请求，并将结果存储在 resp 中，将可能出现的错误存储在 err 中 69 | resp, err := client.CreateChatCompletion(ctx, req) 70 | if err != nil { 71 | // 若出现错误，打印错误信息并终止程序 72 | fmt.Printf("standard chat error: %v\n", err) 73 | return 74 | } 75 | // 检查是否触发深度推理，触发则打印思维链内容 76 | // if resp.Choices[0].Message.ReasoningContent != nil { 77 | // fmt.Println(*resp.Choices[0].Message.ReasoningContent) 78 | // } 79 | // 打印聊天完成请求的响应结果 80 | fmt.Println(*resp.Choices[0].Message.Content.StringValue) 81 | } -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | 8 | "gopkg.in/yaml.v3" 9 | ) 10 | 11 | // Rule 表示一条匹配规则，对应 config.yaml 里的每个条目 12 | type Rule struct { 13 | Name string `yaml:"name"` 14 | FRegex string `yaml:"f_regex"` 15 | } 16 | 17 | // Config 表示整个配置文件内容，里面是若干 Rule 18 | type Config struct { 19 | Rules []Rule `yaml:"rules"` 20 | } 21 | 22 | // LoadConfig 从指定路径的 YAML 文件中加载配置，若文件不存在则创建并写入默认配置，返回 *Config 23 | func LoadConfig(path string) (*Config, error) { 24 | // 1. 尝试读取文件 25 | data, err := os.ReadFile(path) 26 | if err != nil { 27 | if os.IsNotExist(err) { 28 | // 文件不存在，创建并写入默认配置 29 | 30 | // 获取文件的目录路径 31 | dir := filepath.Dir(path) 32 | 33 | // 创建目录（如果不存在） 34 | if err := os.MkdirAll(dir, 0755); err != nil { 35 | return nil, fmt.Errorf("无法创建目录 '%s': %w", dir, err) 36 | } 37 | 38 | // 定义默认配置内容 39 | defaultContent := ` 40 | rules: 41 | - name: Extended Sensitive Field 42 | f_regex: "(?i)([\"']?[\\w-]{0,15}(?:key|secret|token|config|auth|access|admin|ticket|api_key|client_secret|private_key|public_key|bearer|session|cookie|license|cert|ssh|salt|pepper)[\\w-]{0,15}[\"']?)\\s*(?:=|:|\\)\\.val\\()\\s*\\[?\\{?(?:'([^']{8,500})'|\"([^\\\"]{8,500})\")(?:[:;,\\}\\]]?)?" 43 | 44 | - name: Extended Password Field 45 | f_regex: "(?i)((|\\\\)(?:'|\")[\\w-]{0,10}(?:p(?:ass|wd|asswd|assword|asscode|assphrase)|secret)[\\w-]{0,10}(|\\\\)(?:'|\"))\\s*(?:=|:|\\)\\.val\\()(|)(|\\\\)(?:'|\")([^'\"]+?)(|\\\\)(?:'|\")(?:|,|\\)|;)?" 46 | 47 | - name: Extended JSON Web Token 48 | f_regex: "(?i)(eyJ[A-Za-z0-9_-]{5,}\\.[A-Za-z0-9._-]{5,}\\.[A-Za-z0-9._-]{5,})" 49 | 50 | - name: Extended Cloud Key 51 | f_regex: "(?i)(AWSAccessKeyId=[A-Z0-9]{16,32}|access[-_]?key[-_]?(?:id|secret)|LTAI[a-z0-9]{12,20}|(?:AKIA|ABIA|ACCA|ASIA)[A-Z0-9]{16}|aws_secret_access_key\\s*=\\s*[\"'][^\"']{8,100}[\"'])" 52 | 53 | - name: Azure Key 54 | f_regex: "(?i)(AZURE_STORAGE[_-]?ACCOUNT[_-]?KEY|AZURE_STORAGE_KEY|AZURE_KEY_VAULT|azure_tenant_id)\\s*=\\s*[\"']([^\"']{8,100})[\"']" 55 | 56 | - name: GCP Service Account 57 | f_regex: "(?s)(\"type\"\\s*:\\s*\"service_account\".*?\"private_key_id\"\\s*:\\s*\"([a-z0-9]{10,})\".*?\"private_key\"\\s*:\\s*\"-----BEGIN PRIVATE KEY-----.*?-----END PRIVATE KEY-----\")" 58 | 59 | - name: Private Key 60 | f_regex: "(?s)-----BEGIN\\s+(?:RSA|EC|DSA|OPENSSH)?\\s*PRIVATE\\s+KEY-----.*?-----END\\s+(?:RSA|EC|DSA|OPENSSH)?\\s*PRIVATE\\s+KEY-----" 61 | ` 62 | 63 | // 创建文件并写入默认内容 64 | err = os.WriteFile(path, []byte(defaultContent), 0644) 65 | if err != nil { 66 | return nil, fmt.Errorf("无法创建默认配置文件 '%s': %w", path, err) 67 | } 68 | 69 | // 重新读取刚创建的文件 70 | data, err = os.ReadFile(path) 71 | if err != nil { 72 | return nil, fmt.Errorf("创建后无法读取配置文件 '%s': %w", path, err) 73 | } 74 | } else { 75 | // 其他读取错误 76 | return nil, fmt.Errorf("读取配置文件 '%s' 失败: %w", path, err) 77 | } 78 | } 79 | 80 | // 2. 解析 YAML 81 | var cfg Config 82 | if err := yaml.Unmarshal(data, &cfg); err != nil { 83 | return nil, fmt.Errorf("解析 YAML 失败: %w", err) 84 | } 85 | 86 | return &cfg, nil 87 | } 88 | -------------------------------------------------------------------------------- /internal/utils/utils.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "bufio" 5 | "os" 6 | "strings" 7 | ) 8 | 9 | // ReadURLs 从指定文件中读取非空行并将其添加到传入的 urls 切片中 10 | func ReadURLs(filePath string, urls []string) ([]string, error) { 11 | // 打开文件 12 | file, err := os.Open(filePath) 13 | if err != nil { 14 | return urls, err 15 | } 16 | defer file.Close() 17 | 18 | // 使用 bufio.Scanner 逐行读取文件 19 | scanner := bufio.NewScanner(file) 20 | for scanner.Scan() { 21 | line := strings.TrimSpace(scanner.Text()) 22 | if line != "" { 23 | urls = append(urls, line) 24 | } 25 | } 26 | 27 | // 检查扫描过程中是否有错误 28 | if err := scanner.Err(); err != nil { 29 | return urls, err 30 | } 31 | 32 | return urls, nil 33 | } 34 | 35 | 36 | //定义一个包含所有需要过滤的子字符串的切片，Skip 检查 URL 是否包含任何需要跳过的子字符串 37 | var skipSubstrings = []string{ 38 | "static.ocecdn.oraclecloud.com", 39 | "google", 40 | "baidu", 41 | "data:image", 42 | "www.youtube.com", 43 | "www.facebook.com", 44 | "www.w3.org", 45 | "twitter.com", 46 | } 47 | func Skip(url string) bool { 48 | for _, substr := range skipSubstrings { 49 | if strings.Contains(url, substr) { 50 | return true 51 | } 52 | } 53 | return false 54 | } 55 | 56 | // skipExtensions 是需要过滤掉的所有后缀(小写)，HasSkipExtension 函数判断请求 URL 是否包含需要过滤的后缀 57 | var skipExtensions = map[string]bool{ 58 | ".3g2": true, ".3gp": true, ".7z": true, ".aac": true, 59 | ".abw": true, ".aif": true, ".aifc": true, ".aiff": true, 60 | ".apk": true, ".arc": true, ".au": true, ".avi": true, 61 | ".azw": true, ".bat": true, ".bin": true, ".bmp": true, 62 | ".bz": true, ".bz2": true, ".cmd": true, ".cmx": true, 63 | ".cod": true, ".com": true, ".csh": true, ".css": true, 64 | ".csv": true, ".dll": true, ".doc": true, ".docx": true, 65 | ".ear": true, ".eot": true, ".epub": true, ".exe": true, 66 | ".flac": true, ".flv": true, ".gif": true, ".gz": true, 67 | ".ico": true, ".ics": true, ".ief": true, ".jar": true, 68 | ".jfif": true, ".jpe": true, ".jpeg": true, ".jpg": true, 69 | ".less": true, ".m3u": true, ".mid": true, ".midi": true, 70 | ".mjs": true, ".mkv": true, ".mov": true, ".mp2": true, 71 | ".mp3": true, ".mp4": true, ".mpa": true, ".mpe": true, 72 | ".mpeg": true, ".mpg": true, ".mpkg": true, ".mpp": true, 73 | ".mpv2": true, ".odp": true, ".ods": true, ".odt": true, 74 | ".oga": true, ".ogg": true, ".ogv": true, ".ogx": true, 75 | ".otf": true, ".pbm": true, ".pdf": true, ".pgm": true, 76 | ".png": true, ".pnm": true, ".ppm": true, ".ppt": true, 77 | ".pptx": true, ".ra": true, ".ram": true, ".rar": true, 78 | ".ras": true, ".rgb": true, ".rmi": true, ".rtf": true, 79 | ".scss": true, ".sh": true, ".snd": true, ".svg": true, 80 | ".swf": true, ".tar": true, ".tif": true, ".tiff": true, 81 | ".ttf": true, ".vsd": true, ".war": true, ".wav": true, 82 | ".weba": true, ".webm": true, ".webp": true, ".wmv": true, 83 | ".woff": true, ".woff2": true, ".xbm": true, ".xls": true, 84 | ".xlsx": true, ".xpm": true, ".xul": true, ".xwd": true, 85 | ".zip": true, 86 | } 87 | func HasSkipExtension(reqURL string) bool { 88 | beforeQuery := strings.Split(reqURL, "?")[0] 89 | beforeHash := strings.Split(beforeQuery, "#")[0] 90 | 91 | slashIndex := strings.LastIndex(beforeHash, "/") 92 | filename := beforeHash 93 | if slashIndex != -1 { 94 | filename = beforeHash[slashIndex+1:] 95 | } 96 | 97 | for ext := range skipExtensions { 98 | if strings.HasSuffix(filename, ext) { 99 | return true 100 | } 101 | } 102 | return false 103 | } -------------------------------------------------------------------------------- /internal/output/output.go: -------------------------------------------------------------------------------- 1 | package output 2 | 3 | import ( 4 | "encoding/csv" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "os" 9 | "path/filepath" 10 | "strings" 11 | 12 | "SecureJS/internal/matcher" 13 | ) 14 | 15 | // PrintResultsToConsole 在控制台打印结果。 16 | // 如果某个URL没有命中任何信息，仍然会显示"No sensitive info found." 17 | func PrintResultsToConsole(results []*matcher.MatchResult) { 18 | for _, mr := range results { 19 | if mr.Error != nil { 20 | fmt.Printf("\n[!] Parse error on %s: %v\n", mr.URL, mr.Error) 21 | continue 22 | } 23 | if len(mr.Items) == 0 { 24 | //fmt.Printf("\n[+] %s: no sensitive info found.\n", mr.URL) 25 | continue 26 | } 27 | fmt.Printf("\n[+] %s: found %d item(s)\n", mr.URL, len(mr.Items)) 28 | for _, item := range mr.Items { 29 | fmt.Printf(" - Rule: %s, Matched: %s\n", item.RuleName, item.MatchedText) 30 | } 31 | } 32 | } 33 | 34 | // WriteResultsToFile 将匹配结果写入指定文件；如果没有敏感信息则跳过该URL，不写入。 35 | // ext 可以是 ".txt" / ".csv" / ".json"，否则视为 ".txt"。 36 | func WriteResultsToFile(results []*matcher.MatchResult, outPath string) error { 37 | ext := strings.ToLower(filepath.Ext(outPath)) 38 | if ext == "" { 39 | ext = ".txt" 40 | } 41 | 42 | f, err := os.Create(outPath) 43 | if err != nil { 44 | return fmt.Errorf("create file error: %w", err) 45 | } 46 | defer f.Close() 47 | 48 | switch ext { 49 | case ".txt": 50 | return writeTxt(results, f) 51 | case ".csv": 52 | return writeCSV(results, f) 53 | case ".json": 54 | return writeJSON(results, f) 55 | default: 56 | // 如果后缀不是这三个，默认按 txt 处理 57 | return writeTxt(results, f) 58 | } 59 | } 60 | 61 | // writeTxt：只写有敏感信息的记录 62 | func writeTxt(results []*matcher.MatchResult, w io.Writer) error { 63 | for _, mr := range results { 64 | // 如果解析出错，可以考虑还是写出来提示一下 65 | if mr.Error != nil { 66 | _, _ = fmt.Fprintf(w, "[!] Parse error on %s: %v\n\n", mr.URL, mr.Error) 67 | continue 68 | } 69 | // 如果没有任何命中，直接跳过，不写 70 | if len(mr.Items) == 0 { 71 | continue 72 | } 73 | // 写有敏感信息的 74 | _, _ = fmt.Fprintf(w, "[+] %s: found %d item(s)\n", mr.URL, len(mr.Items)) 75 | for _, item := range mr.Items { 76 | _, _ = fmt.Fprintf(w, " - Rule: %s, Matched: %s\n", item.RuleName, item.MatchedText) 77 | } 78 | _, _ = fmt.Fprintln(w) // 空行分隔 79 | } 80 | return nil 81 | } 82 | 83 | // writeCSV：只写有敏感信息的记录 84 | func writeCSV(results []*matcher.MatchResult, w io.Writer) error { 85 | csvWriter := csv.NewWriter(w) 86 | defer csvWriter.Flush() 87 | 88 | // 写表头 89 | _ = csvWriter.Write([]string{"URL", "Rule", "MatchedText", "Error"}) 90 | 91 | for _, mr := range results { 92 | if mr.Error != nil { 93 | // 如果整个页面解析出错，也写一行记录 94 | _ = csvWriter.Write([]string{mr.URL, "", "", mr.Error.Error()}) 95 | continue 96 | } 97 | // 如果没报错但也没有任何命中 -> 跳过 98 | if len(mr.Items) == 0 { 99 | continue 100 | } 101 | // 写出匹配的条目 102 | for _, item := range mr.Items { 103 | _ = csvWriter.Write([]string{mr.URL, item.RuleName, item.MatchedText, ""}) 104 | } 105 | } 106 | return nil 107 | } 108 | 109 | // writeJSON：只写有敏感信息的记录 110 | func writeJSON(results []*matcher.MatchResult, w io.Writer) error { 111 | // 先构造一个新的 slice，仅存有匹配的结果 112 | filtered := make([]*matcher.MatchResult, 0, len(results)) 113 | for _, mr := range results { 114 | if mr.Error != nil { 115 | // 即便出错，也可以把它保留，供排查 116 | filtered = append(filtered, mr) 117 | continue 118 | } 119 | if len(mr.Items) > 0 { 120 | filtered = append(filtered, mr) 121 | } 122 | } 123 | // 如果全都没有敏感信息，也就会是个空数组 [] 124 | 125 | data, err := json.MarshalIndent(filtered, "", " ") 126 | if err != nil { 127 | return fmt.Errorf("json marshal error: %w", err) 128 | } 129 | _, _ = w.Write(data) 130 | return nil 131 | } 132 | -------------------------------------------------------------------------------- /internal/parser/parser.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "crypto/tls" 5 | "fmt" 6 | "io" 7 | "log" 8 | "net/http" 9 | "net/url" 10 | "strings" 11 | "sync" 12 | "time" 13 | ) 14 | 15 | // ParseResult 用于存储对单个 URL 做二次请求的结果 16 | type ParseResult struct { 17 | URL string // 当前目标 URL 18 | StatusCode int // HTTP 状态码 19 | Body string // 响应内容（纯文本/HTML/JSON 等） 20 | Error error // 如果请求失败或解析失败，则记录错误 21 | } 22 | 23 | // ParseAll 并发请求一批 URLs，并返回每个 URL 的响应内容。 24 | // concurrency 用于控制并发线程数。 25 | func ParseAll(urls []string, concurrency int, customHeaders []string, proxy string) ([]*ParseResult, error) { 26 | if len(urls) == 0 { 27 | return nil, fmt.Errorf("no URLs to parse") 28 | } 29 | if concurrency <= 0 { 30 | concurrency = 1 31 | } 32 | 33 | // 准备结果通道 34 | resultChan := make(chan *ParseResult, len(urls)) 35 | 36 | // 并发控制 - 使用有缓冲的通道作为信号量 37 | sem := make(chan struct{}, concurrency) 38 | 39 | // 初始化 WaitGroup 40 | var wg sync.WaitGroup 41 | 42 | // 启动 goroutine 处理每个 URL 43 | for _, targetURL := range urls { 44 | wg.Add(1) 45 | 46 | go func(url string) { 47 | defer wg.Done() 48 | 49 | // 获取一个信号，限制并发数量 50 | sem <- struct{}{} 51 | defer func() { <-sem }() 52 | 53 | // 执行 URL 处理 54 | res, err := parseOneURL(url, customHeaders, proxy) 55 | if err != nil { 56 | resultChan <- &ParseResult{ 57 | URL: url, 58 | Error: err, 59 | } 60 | return 61 | } 62 | resultChan <- res 63 | }(targetURL) 64 | } 65 | 66 | // 等待所有 goroutine 完成 67 | wg.Wait() 68 | close(resultChan) 69 | 70 | // 收集结果 71 | var results []*ParseResult 72 | for r := range resultChan { 73 | results = append(results, r) 74 | } 75 | 76 | return results, nil 77 | } 78 | 79 | // parseOneURL 对单个 URL 发起请求，获取响应内容。 80 | func parseOneURL(urlStr string, customHeaders []string, proxy string) (*ParseResult, error) { 81 | // 自定义 Transport，忽略证书错误 82 | tr := &http.Transport{ 83 | TLSClientConfig: &tls.Config{ 84 | InsecureSkipVerify: true, // 忽略证书错误 85 | }, 86 | } 87 | 88 | // 如果 proxy != "" 就设置代理 89 | if proxy != "" { 90 | proxyURL, err := url.Parse(proxy) 91 | if err == nil { 92 | tr.Proxy = http.ProxyURL(proxyURL) 93 | } 94 | } 95 | 96 | // 使用自定义 Transport 97 | client := &http.Client{ 98 | Timeout: 10 * time.Second, 99 | Transport: tr, 100 | } 101 | 102 | // 手动创建请求，以便设置 UA 和其他伪装头 103 | req, err := http.NewRequest("GET", urlStr, nil) 104 | if err != nil { 105 | return nil, fmt.Errorf("failed to create request for %s: %w", urlStr, err) 106 | } 107 | 108 | // 设置伪装头 109 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "+ 110 | "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36") 111 | 112 | // 自定义请求头 113 | for _, h := range customHeaders { 114 | parts := strings.SplitN(h, ":", 2) 115 | if len(parts) == 2 { 116 | key := strings.TrimSpace(parts[0]) 117 | val := strings.TrimSpace(parts[1]) 118 | req.Header.Set(key, val) 119 | } 120 | } 121 | 122 | // 发起请求 123 | resp, err := client.Do(req) 124 | if err != nil { 125 | return nil, fmt.Errorf("failed to GET %s: %w", urlStr, err) 126 | } 127 | defer func() { 128 | if cerr := resp.Body.Close(); cerr != nil { 129 | log.Printf("[!] failed to close response body for %s: %v", urlStr, cerr) 130 | } 131 | }() 132 | 133 | // 读取响应体 134 | bodyBytes, err := io.ReadAll(resp.Body) 135 | if err != nil { 136 | return nil, fmt.Errorf("failed to read body from %s: %w", urlStr, err) 137 | } 138 | 139 | body := strings.TrimSpace(string(bodyBytes)) 140 | 141 | return &ParseResult{ 142 | URL: urlStr, 143 | StatusCode: resp.StatusCode, 144 | Body: body, 145 | }, nil 146 | } -------------------------------------------------------------------------------- /cmd/root.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os" 7 | 8 | "SecureJS/config" 9 | "SecureJS/internal/analyze" 10 | "SecureJS/internal/crawler" 11 | "SecureJS/internal/matcher" 12 | "SecureJS/internal/output" 13 | "SecureJS/internal/parser" 14 | "SecureJS/internal/utils" 15 | 16 | "github.com/spf13/cobra" 17 | ) 18 | 19 | var ( 20 | singleURL string 21 | listFile string 22 | threads int 23 | configPath string 24 | outputFile string 25 | browserPath string 26 | customHeaders []string 27 | proxy string 28 | ai string 29 | ARK_API_KEY string 30 | Model_ENDPOINT_ID string 31 | ) 32 | 33 | func init() { 34 | rootCmd.Flags().StringVarP(&singleURL, "url", "u", "", "Single target URL to scan (e.g. https://example.com)") 35 | rootCmd.Flags().StringVarP(&listFile, "list", "l", "", "File containing target URLs (one per line)") 36 | rootCmd.Flags().IntVarP(&threads, "threads", "t", 20, "Number of concurrent threads for scanning") 37 | rootCmd.Flags().StringVarP(&configPath, "config", "c", "config/config.yaml", "Path to config file (e.g. config.yaml)") 38 | rootCmd.Flags().StringVarP(&outputFile, "output", "o", "", "Output file (supports .txt, .csv, .json)") 39 | rootCmd.Flags().StringVarP(&browserPath, "browser", "b", "", "Path to Chrome/Chromium executable (optional). If not set, will use Rod's default.") 40 | rootCmd.Flags().StringArrayVarP(&customHeaders, "header", "H", nil, "Add custom request headers. (e.g. -H 'Key: Value')") 41 | rootCmd.Flags().StringVarP(&proxy, "proxy", "p", "", "Proxy to use (e.g. http://127.0.0.1:8080)") 42 | rootCmd.Flags().StringVarP(&ai, "ai", "a", "false", "true/false. Enable AI Analytics. If not set, will use false") 43 | rootCmd.Flags().StringVarP(&Model_ENDPOINT_ID, "id", "i", "", "YOUR_ENDPOINT_ID") 44 | rootCmd.Flags().StringVarP(&ARK_API_KEY, "key", "k", "", "ARK_API_KEY") 45 | } 46 | 47 | var rootCmd = &cobra.Command{ 48 | Use: "SecureJS", 49 | Short: "A tool to crawl websites, parse links/JS, and match sensitive patterns based on custom config.", 50 | Long: `...`, 51 | 52 | Run: func(cmd *cobra.Command, args []string) { 53 | // 1) 收集目标 URL 54 | var urls []string 55 | if singleURL != "" { // 使用 -u 参数 56 | urls = append(urls, singleURL) 57 | } else if listFile != "" { //使用 -l 参数 58 | var err error 59 | urls, err = utils.ReadURLs(listFile, urls) 60 | if err != nil { 61 | log.Fatalf("[!] Failed to read URLs from file %s: %v\n", listFile, err) 62 | } 63 | } else { 64 | fmt.Println("[!] Please provide either a single -u or a -l containing URLs.") 65 | os.Exit(1) 66 | } 67 | 68 | // 2) 爬取链接（这里两个思路）2.1 && 2.2 69 | uniqueLinks := make(map[string]struct{}) // 使用 map 来跟踪已添加的链接，实现去重 70 | var toParse []string // 所有捕获的链接放入 toParse 71 | 72 | // 2.1 收集加载某个目标 url 后（使用无头浏览器），默认加载的所有其他链接（js等）并放入 toParse 73 | err := crawler.CollectLinks(urls, threads, uniqueLinks, &toParse, browserPath, customHeaders, proxy) 74 | if err != nil { 75 | log.Fatalf("[!] Error collecting links: %v", err) 76 | } 77 | 78 | // 2.2 收集加载某个目标 url 后其 body 中的链接（js等）并放入 toParse 79 | err = crawler.CollectLinksFromBody(urls, threads, uniqueLinks, &toParse, customHeaders, proxy) 80 | if err != nil { 81 | log.Fatalf("[!] Error collecting links from body: %v", err) 82 | } 83 | 84 | // for _, jsurl := range toParse { 85 | // fmt.Println(jsurl) 86 | // } 87 | 88 | // 3) 对所有收集到的链接进行二次请求 89 | parseResults, err := parser.ParseAll(toParse, threads, customHeaders, proxy) 90 | if err != nil { 91 | log.Fatalf("[!] Failed to parseAll: %v\n", err) 92 | } 93 | 94 | // 4) 加载 config.yaml 中的敏感信息正则匹配规则 95 | cfg, err := config.LoadConfig(configPath) 96 | if err != nil { 97 | log.Fatalf("[!] Failed to load config: %v\n", err) 98 | } 99 | 100 | // 5) 对所有收集到的链接进行二次请求后的 body 中进行敏感信息的匹配 101 | matchResults, err := matcher.MatchAll(cfg.Rules, parseResults) 102 | if err != nil { 103 | log.Fatalf("[!] Failed to matchAll: %v\n", err) 104 | } 105 | 106 | // 6) 输出 107 | if outputFile == "" { 108 | if ai == "true" { 109 | resultString := analyze.FormatResultsToString(matchResults) 110 | analyze.Analyze(resultString, ARK_API_KEY, Model_ENDPOINT_ID) 111 | } else { 112 | output.PrintResultsToConsole(matchResults) 113 | } 114 | } else { 115 | err := output.WriteResultsToFile(matchResults, outputFile) 116 | if err != nil { 117 | log.Fatalf("[!] Failed to write results to file: %v\n", err) 118 | } 119 | } 120 | }, 121 | } 122 | 123 | func Execute() error { 124 | return rootCmd.Execute() 125 | } 126 | -------------------------------------------------------------------------------- /internal/matcher/matcher.go: -------------------------------------------------------------------------------- 1 | package matcher 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | "strings" 7 | 8 | "SecureJS/config" 9 | "SecureJS/internal/parser" 10 | ) 11 | 12 | // MatchItem 表示单条命中结果 13 | type MatchItem struct { 14 | RuleName string // 命中的规则名称 15 | MatchedText string // 实际匹配到的敏感信息片段 16 | } 17 | 18 | // MatchResult 表示对某个 URL 的匹配结果 19 | type MatchResult struct { 20 | URL string // 目标URL 21 | Items []MatchItem // 命中的所有结果 22 | Error error // 如果在匹配过程中有什么错误，可记录在这里（一般不会有） 23 | } 24 | 25 | // compiledRule 用于保存编译后的正则，避免重复编译 26 | type compiledRule struct { 27 | Name string 28 | Regex *regexp.Regexp 29 | } 30 | 31 | // MatchAll 对从 parser 获得的一组响应内容进行匹配， 32 | // 返回每个 URL 对应的匹配情况。 33 | func MatchAll(rules []config.Rule, parseResults []*parser.ParseResult) ([]*MatchResult, error) { 34 | // 1) 先编译所有规则（减少重复编译） 35 | compiledRules := make([]compiledRule, 0, len(rules)) 36 | for _, r := range rules { 37 | // 如果 f_regex 为空或无效，可以跳过 38 | if r.FRegex == "" { 39 | continue 40 | } 41 | re, err := regexp.Compile(r.FRegex) 42 | if err != nil { 43 | return nil, fmt.Errorf("failed to compile regex for rule '%s': %w", r.Name, err) 44 | } 45 | compiledRules = append(compiledRules, compiledRule{ 46 | Name: r.Name, 47 | Regex: re, 48 | }) 49 | } 50 | 51 | // 2) 对每个 parseResult 的 Body 做匹配 52 | results := make([]*MatchResult, 0, len(parseResults)) 53 | for _, pr := range parseResults { 54 | if pr.Error != nil { 55 | // 如果 parse 出错了，这里就直接记录错误 56 | results = append(results, &MatchResult{ 57 | URL: pr.URL, 58 | Error: pr.Error, 59 | }) 60 | continue 61 | } 62 | 63 | //去重 64 | uniqueMatches := make(map[string]bool) 65 | // 准备收集此 URL 下所有命中项 66 | var matchedItems []MatchItem 67 | body := pr.Body 68 | 69 | // 对所有规则匹配 70 | for _, cr := range compiledRules { 71 | allMatches := cr.Regex.FindAllString(body, -1) 72 | for _, matchStr := range allMatches { 73 | if _, exists := uniqueMatches[matchStr]; !exists { 74 | uniqueMatches[matchStr] = true 75 | matchedItems = append(matchedItems, MatchItem{ 76 | RuleName: cr.Name, 77 | MatchedText: matchStr, 78 | }) 79 | } 80 | } 81 | } 82 | // 过滤匹配项 83 | matchedItems = filterMatchedItems(matchedItems) 84 | 85 | // 如果有匹配项，记录结果 86 | if len(matchedItems) > 0 { 87 | results = append(results, &MatchResult{ 88 | URL: pr.URL, 89 | Items: matchedItems, 90 | Error: nil, 91 | }) 92 | } 93 | } 94 | // 输出匹配结果 95 | // var lastURL string 96 | // for _, result := range results { 97 | // for _, item := range result.Items { 98 | // if item.MatchedText != "" { 99 | // if lastURL != result.URL { 100 | // if lastURL != "" { 101 | // fmt.Println() // 为了分隔不同的URL输出 102 | // } 103 | // fmt.Printf("URL: %s\n", result.URL) 104 | // lastURL = result.URL 105 | // } 106 | // fmt.Printf(" Rule: %s, Matched: %s\n", item.RuleName, item.MatchedText) 107 | // } 108 | // } 109 | // } 110 | return results, nil 111 | } 112 | 113 | // filterMatchedItems 只会针对“找到的第一个 : 或 =”进行拆分。 114 | // 拆分后的 key/value 如果包含了指定的关键词或中文字符，就过滤掉。 115 | func filterMatchedItems(matchedItems []MatchItem) []MatchItem { 116 | // 1) 设定普通过滤关键词（子串匹配、忽略大小写） 117 | filterKeywords := []string{ 118 | "xml", 119 | } 120 | 121 | // 2) 前置过滤关键词（只要 key 包含这些，就过滤） 122 | preFilterKeywords := []string{ 123 | "passive", 124 | } 125 | 126 | // 3) 编译对应的正则，用于子串匹配和忽略大小写 127 | keywordRegex := regexp.MustCompile(`(?i)(` + strings.Join(filterKeywords, "|") + `)`) 128 | preKeywordRegex := regexp.MustCompile(`(?i)(` + strings.Join(preFilterKeywords, "|") + `)`) 129 | 130 | // 4) 检查是否有中文字符 131 | chineseRegex := regexp.MustCompile(`[\p{Han}]`) 132 | 133 | var filteredItems []MatchItem 134 | 135 | for _, item := range matchedItems { 136 | // 默认不过滤 137 | shouldFilter := false 138 | 139 | // 取出整条文本 140 | text := strings.TrimSpace(item.MatchedText) 141 | 142 | // 找到第一个 ':' 或 '=' 143 | idxColon := strings.IndexRune(text, ':') 144 | idxEqual := strings.IndexRune(text, '=') 145 | 146 | var splitPos int 147 | switch { 148 | case idxColon < 0 && idxEqual < 0: 149 | // 连一个 ':' 或 '=' 都没有，说明无法拆分 150 | // 你可以选择：直接保留 (shouldFilter=false)，也可以选择直接过滤 151 | splitPos = -1 152 | case idxColon < 0: 153 | // 没有冒号，只找到等号 154 | splitPos = idxEqual 155 | case idxEqual < 0: 156 | // 没有等号，只找到冒号 157 | splitPos = idxColon 158 | default: 159 | // 同时存在 ':' 和 '='，取最小位置 => 谁先出现 160 | if idxColon < idxEqual { 161 | splitPos = idxColon 162 | } else { 163 | splitPos = idxEqual 164 | } 165 | } 166 | 167 | if splitPos != -1 { 168 | // 说明找到了分隔符，进行拆分 169 | key := text[:splitPos] 170 | value := text[splitPos+1:] // 从分隔符的下一个字符开始到末尾都算 value 171 | 172 | // 转小写并去除首尾的引号、空格 173 | key = strings.ToLower(strings.Trim(key, `"' `)) 174 | value = strings.ToLower(strings.Trim(value, `"' `)) 175 | 176 | // ① 如果 key 命中“前置过滤关键词”，则过滤 177 | if preKeywordRegex.MatchString(key) { 178 | shouldFilter = true 179 | } 180 | 181 | // ② value 中出现“普通过滤关键词”，则过滤 182 | if keywordRegex.MatchString(value) { 183 | shouldFilter = true 184 | } 185 | 186 | // ③ value 中包含中文字符，也过滤 187 | if chineseRegex.MatchString(value) { 188 | shouldFilter = true 189 | } 190 | } 191 | 192 | if !shouldFilter { 193 | filteredItems = append(filteredItems, item) 194 | } 195 | } 196 | 197 | return filteredItems 198 | } 199 | -------------------------------------------------------------------------------- /internal/crawler/crawler.go: -------------------------------------------------------------------------------- 1 | package crawler 2 | 3 | import ( 4 | "SecureJS/internal/utils" 5 | "fmt" 6 | "log" 7 | "strings" 8 | "sync" 9 | "time" 10 | 11 | "github.com/go-rod/rod" 12 | "github.com/go-rod/rod/lib/launcher" 13 | "github.com/go-rod/rod/lib/proto" 14 | 15 | ) 16 | 17 | type CrawlResult struct { 18 | URL string 19 | AllRequests []string 20 | Error error 21 | } 22 | 23 | // ----------------------------------------------------------- 24 | // 并发爬取多个链接 25 | // ----------------------------------------------------------- 26 | func crawlAll(urls []string, concurrency int, browserPath string, customHeaders []string, proxy string) ([]*CrawlResult, error) { 27 | if len(urls) == 0 { 28 | return nil, fmt.Errorf("no URLs provided") 29 | } 30 | if concurrency <= 0 { 31 | concurrency = 1 32 | } 33 | 34 | var chromePath string 35 | if browserPath != "" { 36 | chromePath = browserPath 37 | } else { 38 | chromePath = launcher.NewBrowser().MustGet() 39 | } 40 | 41 | launch := launcher.New(). 42 | Bin(chromePath). 43 | Headless(true). 44 | NoSandbox(true). 45 | Set("ignore-certificate-errors"). 46 | Set("disable-blink-features", "AutomationControlled"). 47 | Set("disable-infobars") 48 | 49 | if proxy != "" { 50 | launch = launch.Proxy(proxy) 51 | } 52 | 53 | u := launch.MustLaunch() 54 | 55 | browser := rod.New().ControlURL(u).MustConnect() 56 | defer func() { 57 | if err := browser.Close(); err != nil { 58 | log.Printf("[!] failed to close browser: %v\n", err) 59 | } 60 | }() 61 | 62 | resultChan := make(chan *CrawlResult, len(urls)) 63 | sem := make(chan struct{}, concurrency) 64 | var wg sync.WaitGroup 65 | 66 | for _, targetURL := range urls { 67 | wg.Add(1) 68 | go func(url string) { 69 | defer wg.Done() 70 | sem <- struct{}{} 71 | defer func() { <-sem }() 72 | 73 | // 最大重试次数，可自行调整 74 | const maxRetry = 3 75 | res, err := fetchOneURLWithRetry(browser, url, maxRetry, customHeaders) 76 | if err != nil { 77 | resultChan <- &CrawlResult{URL: url, Error: err} 78 | return 79 | } 80 | resultChan <- res 81 | }(targetURL) 82 | } 83 | 84 | wg.Wait() 85 | close(resultChan) 86 | 87 | var results []*CrawlResult 88 | for r := range resultChan { 89 | results = append(results, r) 90 | } 91 | 92 | return results, nil 93 | } 94 | 95 | // ----------------------------------------------------------- 96 | // 带重试的抓取逻辑 97 | // ----------------------------------------------------------- 98 | func fetchOneURLWithRetry(browser *rod.Browser, url string, maxAttempts int, customHeaders []string) (*CrawlResult, error) { 99 | var lastErr error 100 | baseTime := 20 * time.Second 101 | 102 | for attempt := 1; attempt <= maxAttempts; attempt++ { 103 | currentTimeout := time.Duration(attempt) * baseTime 104 | 105 | result, err := tryFetchOneURL(browser, url, currentTimeout, customHeaders) 106 | if err == nil { 107 | return result, nil 108 | } 109 | 110 | lastErr = err 111 | log.Printf("[Attempt %d/%d] Failed to fetch '%s' (timeout=%v) error: %v", 112 | attempt, maxAttempts, url, currentTimeout, err) 113 | 114 | if attempt < maxAttempts { 115 | time.Sleep(2 * time.Second) 116 | } 117 | } 118 | 119 | return nil, lastErr 120 | } 121 | 122 | // ----------------------------------------------------------- 123 | // 单次访问逻辑：在已有 page 上使用 stealth.Inject(page) 124 | // ----------------------------------------------------------- 125 | func tryFetchOneURL(browser *rod.Browser, url string, timeout time.Duration, customHeaders []string) (*CrawlResult, error) { 126 | page := browser.MustPage("") 127 | defer page.Close() 128 | 129 | // // 注入 stealth 130 | // if err := stealth.Inject(page); err != nil { 131 | // return nil, fmt.Errorf("failed to inject stealth: %w", err) 132 | // } 133 | 134 | // 设置 User-Agent 135 | err := page.SetUserAgent(&proto.NetworkSetUserAgentOverride{ 136 | UserAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + 137 | "AppleWebKit/537.36 (KHTML, like Gecko) " + 138 | "Chrome/95.0.4638.69 Safari/537.36", 139 | }) 140 | if err != nil { 141 | return nil, fmt.Errorf("failed to set user agent: %w", err) 142 | } 143 | 144 | // 设置自定义请求头 145 | if len(customHeaders) > 0 { 146 | // 1) 准备一个 []string 来存储 "Key", "Value" 这种键值对 147 | var headerPairs []string 148 | 149 | // 2) 遍历 -H 参数里传来的 "Key: Value" 格式字符串 150 | for _, h := range customHeaders { 151 | parts := strings.SplitN(h, ":", 2) 152 | if len(parts) == 2 { 153 | key := strings.TrimSpace(parts[0]) 154 | val := strings.TrimSpace(parts[1]) 155 | // 3) 依次将键和值 append 到同一个切片中 156 | headerPairs = append(headerPairs, key, val) 157 | } 158 | } 159 | 160 | // 4) 调用 page.SetExtraHeaders(...) 161 | // 注意要用变长参数传进去，所以是 headerPairs... 162 | page.SetExtraHeaders(headerPairs) 163 | } 164 | 165 | // 设置超时 166 | page = page.Timeout(timeout) 167 | 168 | loadedMap := make(map[string]bool) 169 | loadedMap[url] = true 170 | 171 | stop := page.EachEvent(func(e *proto.NetworkRequestWillBeSent) { 172 | reqURL := e.Request.URL 173 | lowerURL := strings.ToLower(reqURL) 174 | 175 | // 过滤 176 | if utils.Skip(lowerURL) { 177 | return 178 | } 179 | if utils.HasSkipExtension(lowerURL) { 180 | return 181 | } 182 | 183 | normalized := strings.TrimSuffix(reqURL, "/") 184 | loadedMap[normalized] = true 185 | }) 186 | 187 | // 导航 188 | if err := page.Navigate(url); err != nil { 189 | stop() 190 | return nil, fmt.Errorf("failed to navigate %s: %w", url, err) 191 | } 192 | 193 | // 等待页面空闲 194 | if err := page.WaitIdle(15 * time.Second); err != nil { 195 | stop() 196 | return nil, fmt.Errorf("failed to wait idle %s: %w", url, err) 197 | } 198 | 199 | stop() 200 | 201 | allRequests := make([]string, 0, len(loadedMap)) 202 | for r := range loadedMap { 203 | allRequests = append(allRequests, r) 204 | } 205 | 206 | return &CrawlResult{ 207 | URL: url, 208 | AllRequests: allRequests, 209 | }, nil 210 | } 211 | 212 | // ----------------------------------------------------------- 213 | // 对外的接口，用于收集 214 | // ----------------------------------------------------------- 215 | func CollectLinks(urls []string, threads int, uniqueLinks map[string]struct{}, toParse *[]string, browserPath string, customHeaders []string, proxy string) error { 216 | results, err := crawlAll(urls, threads, browserPath, customHeaders, proxy) 217 | if err != nil { 218 | return fmt.Errorf("failed to crawl: %v", err) 219 | } 220 | 221 | for _, result := range results { 222 | if result.Error != nil { 223 | fmt.Printf("[!] URL: %s, Error: %v\n", result.URL, result.Error) 224 | continue 225 | } 226 | for _, reqURL := range result.AllRequests { 227 | if _, exists := uniqueLinks[reqURL]; !exists { 228 | uniqueLinks[reqURL] = struct{}{} 229 | *toParse = append(*toParse, reqURL) 230 | } 231 | } 232 | } 233 | return nil 234 | } -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 3 | github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= 4 | github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= 5 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 6 | github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= 7 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 8 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 9 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 10 | github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 11 | github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= 12 | github.com/go-rod/rod v0.116.2 h1:A5t2Ky2A+5eD/ZJQr1EfsQSe5rms5Xof/qj296e+ZqA= 13 | github.com/go-rod/rod v0.116.2/go.mod h1:H+CMO9SCNc2TJ2WfrG+pKhITz57uGNYU43qYHh438Mg= 14 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 15 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 16 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 17 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 18 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 19 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 20 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 21 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 22 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 23 | github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= 24 | github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 25 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 26 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 27 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 28 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 29 | github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 30 | github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= 31 | github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 32 | github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= 33 | github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= 34 | github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= 35 | github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= 36 | github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= 37 | github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= 38 | github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= 39 | github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 40 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 41 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 42 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 43 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 44 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 45 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 46 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 47 | github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 48 | github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= 49 | github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= 50 | github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= 51 | github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 52 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 53 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= 54 | github.com/volcengine/volc-sdk-golang v1.0.23 h1:anOslb2Qp6ywnsbyq9jqR0ljuO63kg9PY+4OehIk5R8= 55 | github.com/volcengine/volc-sdk-golang v1.0.23/go.mod h1:AfG/PZRUkHJ9inETvbjNifTDgut25Wbkm2QoYBTbvyU= 56 | github.com/volcengine/volcengine-go-sdk v1.0.181 h1:/3PB4M1N4fjMqiSKTJwX43EZ5Nn1HUOtQrSCk+22+wI= 57 | github.com/volcengine/volcengine-go-sdk v1.0.181/go.mod h1:gfEDc1s7SYaGoY+WH2dRrS3qiuDJMkwqyfXWCa7+7oA= 58 | github.com/ysmood/fetchup v0.2.4 h1:2kfWr/UrdiHg4KYRrxL2Jcrqx4DZYD+OtWu7WPBZl5o= 59 | github.com/ysmood/fetchup v0.2.4/go.mod h1:hbysoq65PXL0NQeNzUczNYIKpwpkwFL4LXMDEvIQq9A= 60 | github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ= 61 | github.com/ysmood/goob v0.4.0/go.mod h1:u6yx7ZhS4Exf2MwciFr6nIM8knHQIE22lFpWHnfql18= 62 | github.com/ysmood/gop v0.2.0 h1:+tFrG0TWPxT6p9ZaZs+VY+opCvHU8/3Fk6BaNv6kqKg= 63 | github.com/ysmood/gop v0.2.0/go.mod h1:rr5z2z27oGEbyB787hpEcx4ab8cCiPnKxn0SUHt6xzk= 64 | github.com/ysmood/got v0.40.0 h1:ZQk1B55zIvS7zflRrkGfPDrPG3d7+JOza1ZkNxcc74Q= 65 | github.com/ysmood/got v0.40.0/go.mod h1:W7DdpuX6skL3NszLmAsC5hT7JAhuLZhByVzHTq874Qg= 66 | github.com/ysmood/gotrace v0.6.0 h1:SyI1d4jclswLhg7SWTL6os3L1WOKeNn/ZtzVQF8QmdY= 67 | github.com/ysmood/gotrace v0.6.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM= 68 | github.com/ysmood/gson v0.7.3 h1:QFkWbTH8MxyUTKPkVWAENJhxqdBa4lYTQWqZCiLG6kE= 69 | github.com/ysmood/gson v0.7.3/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg= 70 | github.com/ysmood/leakless v0.9.0 h1:qxCG5VirSBvmi3uynXFkcnLMzkphdh3xx5FtrORwDCU= 71 | github.com/ysmood/leakless v0.9.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ= 72 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 73 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 74 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 75 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 76 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 77 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 78 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 79 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 80 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 81 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 82 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 83 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 84 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 85 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 86 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 87 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 88 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 89 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 90 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 91 | golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 92 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 93 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 94 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 95 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 96 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 97 | google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= 98 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 99 | google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 100 | google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 101 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 102 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 103 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 104 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 105 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 106 | google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 107 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 108 | google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 109 | google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= 110 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 111 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= 112 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 113 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 114 | gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= 115 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 116 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 117 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 118 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 119 | honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 120 | --------------------------------------------------------------------------------