├── .github └── workflows │ └── release.yaml ├── .goreleaser.yaml ├── LICENSE ├── README.md ├── cmd └── cmd.go ├── config └── config.go ├── crawler ├── crawler.go ├── filter.go ├── find.go ├── jsFuzz.go ├── run.go ├── state.go └── urlFuzz.go ├── go.mod ├── go.sum ├── img ├── 0.jpg ├── 1.jpg ├── 2.jpg ├── 3.jpg ├── 4.jpg ├── 5.jpg └── process.png ├── main.go ├── mode └── mode.go ├── result ├── report.html └── result.go └── util └── utils.go /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | name: goreleaser 2 | 3 | on: 4 | push: 5 | # run only against tags 6 | tags: 7 | - '*' 8 | 9 | permissions: 10 | contents: write 11 | # packages: write 12 | # issues: write 13 | 14 | jobs: 15 | goreleaser: 16 | runs-on: ubuntu-latest 17 | steps: 18 | - uses: actions/checkout@v3 19 | with: 20 | fetch-depth: 0 21 | - run: git fetch --force --tags 22 | - uses: actions/setup-go@v3 23 | with: 24 | go-version: '>=1.20.2' 25 | cache: true 26 | # More assembly might be required: Docker logins, GPG, etc. It all depends 27 | # on your needs. 28 | - uses: goreleaser/goreleaser-action@v4 29 | with: 30 | # either 'goreleaser' (default) or 'goreleaser-pro': 31 | distribution: goreleaser 32 | version: latest 33 | args: release --clean 34 | env: 35 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 36 | # Your GoReleaser Pro key, if you are using the 'goreleaser-pro' 37 | # distribution: 38 | # GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }} -------------------------------------------------------------------------------- /.goreleaser.yaml: -------------------------------------------------------------------------------- 1 | # This is an example .goreleaser.yml file with some sensible defaults. 2 | # Make sure to check the documentation at https://goreleaser.com 3 | before: 4 | hooks: 5 | # You may remove this if you don't use go modules. 6 | - go mod tidy 7 | # you may remove this if you don't need go generate 8 | - go generate ./... 9 | builds: 10 | - env: 11 | - CGO_ENABLED=0 12 | goos: 13 | - linux 14 | - windows 15 | - darwin 16 | 17 | archives: 18 | - format: tar.gz 19 | # this name template makes the OS and Arch compatible with the results of uname. 20 | name_template: >- 21 | {{ .ProjectName }}_ 22 | {{- title .Os }}_ 23 | {{- if eq .Arch "amd64" }}x86_64 24 | {{- else if eq .Arch "386" }}i386 25 | {{- else }}{{ .Arch }}{{ end }} 26 | {{- if .Arm }}v{{ .Arm }}{{ end }} 27 | # use zip for windows archives 28 | format_overrides: 29 | - goos: windows 30 | format: zip 31 | checksum: 32 | name_template: 'checksums.txt' 33 | snapshot: 34 | name_template: "{{ incpatch .Version }}-next" 35 | changelog: 36 | sort: asc 37 | filters: 38 | exclude: 39 | - '^docs:' 40 | - '^test:' 41 | 42 | # The lines beneath this are called `modelines`. See `:help modeline` 43 | # Feel free to remove those if you don't want/use them. 44 | # yaml-language-server: $schema=https://goreleaser.com/static/schema.json 45 | # vim: set ts=2 sw=2 tw=0 fo=cnqoj 46 | 47 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 pingc0y 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## URLFinder 2 | 3 | URLFinder是一款快速、全面、易用的页面信息提取工具 4 | 5 | 用于分析页面中的js与url,查找隐藏在其中的敏感信息或未授权api接口 6 | 7 | 大致执行流程: 8 | 9 | 10 | 11 | 12 | 13 | 有什么需求或bug欢迎各位师傅提交lssues 14 | 15 | ## 快速使用 16 | 单url 17 | ``` 18 | 显示全部状态码 19 | URLFinder.exe -u http://www.baidu.com -s all -m 3 20 | 21 | 显示200和403状态码 22 | URLFinder.exe -u http://www.baidu.com -s 200,403 -m 3 23 | ``` 24 | 批量url 25 | ``` 26 | 结果分开保存 27 | 导出全部 28 | URLFinder.exe -s all -m 3 -f url.txt -o . 29 | 只导出html 30 | URLFinder.exe -s all -m 3 -f url.txt -o res.html 31 | 32 | 结果统一保存 33 | URLFinder.exe -s all -m 3 -ff url.txt -o . 34 | ``` 35 | 参数（更多参数使用 -i 配置）： 36 | ``` 37 | -a 自定义user-agent请求头 38 | -b 自定义baseurl路径 39 | -c 请求添加cookie 40 | -d 指定获取的域名,支持正则表达式 41 | -f 批量url抓取,需指定url文本路径 42 | -ff 与-f区别：全部抓取的数据,视为同一个url的结果来处理（只打印一份结果 | 只会输出一份结果） 43 | -h 帮助信息 44 | -i 加载yaml配置文件,可自定义请求头、抓取规则等（不存在时,会在当前目录创建一个默认yaml配置文件） 45 | -m 抓取模式： 46 | 1 正常抓取（默认） 47 | 2 深入抓取 （URL深入一层 JS深入三层 防止抓偏） 48 | 3 安全深入抓取（过滤delete,remove等敏感路由） 49 | -max 最大抓取数 50 | -o 结果导出到csv、json、html文件,需指定导出文件目录（.代表当前目录） 51 | -s 显示指定状态码,all为显示全部 52 | -t 设置线程数（默认50） 53 | -time 设置超时时间（默认5,单位秒） 54 | -u 目标URL 55 | -x 设置代理,格式: http://username:password@127.0.0.1:8877 56 | -z 提取所有目录对404链接进行fuzz(只对主域名下的链接生效,需要与 -s 一起使用） 57 | 1 目录递减fuzz 58 | 2 2级目录组合fuzz 59 | 3 3级目录组合fuzz（适合少量链接使用） 60 | ``` 61 | ## 使用截图 62 | 63 | [![0.jpg](https://github.com/pingc0y/URLFinder/raw/master/img/0.jpg)](https://github.com/pingc0y/URLFinder/raw/master/img/0.jpg) 64 | [![1.jpg](https://github.com/pingc0y/URLFinder/raw/master/img/1.jpg)](https://github.com/pingc0y/URLFinder/raw/master/img/1.jpg) 65 | [![2.jpg](https://github.com/pingc0y/URLFinder/raw/master/img/2.jpg)](https://github.com/pingc0y/URLFinder/raw/master/img/2.jpg) 66 | [![3.jpg](https://github.com/pingc0y/URLFinder/raw/master/img/3.jpg)](https://github.com/pingc0y/URLFinder/raw/master/img/3.jpg) 67 | [![4.jpg](https://github.com/pingc0y/URLFinder/raw/master/img/4.jpg)](https://github.com/pingc0y/URLFinder/raw/master/img/4.jpg) 68 | [![5.jpg](https://github.com/pingc0y/URLFinder/raw/master/img/5.jpg)](https://github.com/pingc0y/URLFinder/raw/master/img/5.jpg) 69 | 70 | ## 部分说明 71 | 72 | fuzz功能是基于抓到的404目录和路径。将其当作字典,随机组合并碰撞出有效路径,从而解决路径拼接错误的问题 73 | 74 | 结果会优先显示输入的url顶级域名,其他域名不做区分显示在 other 75 | 76 | 结果会优先显示200,按从小到大排序（输入的域名最优先,就算是404也会排序在其他子域名的200前面） 77 | 78 | 为了更好的兼容和防止漏抓链接,放弃了低误报率,错误的链接会变多但漏抓概率变低,可通过 ‘-s 200’ 筛选状态码过滤无效的链接（但不推荐只看200状态码） 79 | ## 编译 80 | 以下是在windows环境下,编译出各平台可执行文件的命令 81 | 82 | ``` 83 | SET CGO_ENABLED=0 84 | SET GOOS=windows 85 | SET GOARCH=amd64 86 | go build -ldflags "-s -w" -o ./URLFinder-windows-amd64.exe 87 | 88 | SET CGO_ENABLED=0 89 | SET GOOS=windows 90 | SET GOARCH=386 91 | go build -ldflags "-s -w" -o ./URLFinder-windows-386.exe 92 | 93 | SET CGO_ENABLED=0 94 | SET GOOS=linux 95 | SET GOARCH=amd64 96 | go build -ldflags "-s -w" -o ./URLFinder-linux-amd64 97 | 98 | SET CGO_ENABLED=0 99 | SET GOOS=linux 100 | SET GOARCH=arm64 101 | go build -ldflags "-s -w" -o ./URLFinder-linux-arm64 102 | 103 | SET CGO_ENABLED=0 104 | SET GOOS=linux 105 | SET GOARCH=386 106 | go build -ldflags "-s -w" -o ./URLFinder-linux-386 107 | 108 | SET CGO_ENABLED=0 109 | SET GOOS=darwin 110 | SET GOARCH=amd64 111 | go build -ldflags "-s -w" -o ./URLFinder-macos-amd64 112 | 113 | SET CGO_ENABLED=0 114 | SET GOOS=darwin 115 | SET GOARCH=arm64 116 | go build -ldflags "-s -w" -o ./URLFinder-macos-arm64 117 | ``` 118 | 119 | 120 | ## 更新说明 121 | 2023/9/9 122 | 修复 -ff 重复验证问题 123 | 修复 自动识别协议bug 124 | 125 | 2023/9/2 126 | 修复 子目录定位bug 127 | 128 | 2023/8/30 129 | 修复 -i 配置请求头错误问题 130 | 变化 支持自动识别http/https 131 | 变化 -o 输入完整文件名可只导出指定类型 132 | 变化 无 -s 参数时，链接改为无颜色方便使用管道符 133 | 134 | 2023/5/11 135 | 变化 -i 配置文件可自定义：线程数、抓取深度、敏感路由、超时时间、最大抓取数 136 | 新增 -time 设置超时时间 137 | 新增 -max 设置最大抓取数 138 | 新增 添加版本更新提示 139 | 修复 已知bug 140 | 141 | 2023/5/5 142 | 修复 多个任务时html结果混乱 143 | 新增 结果添加302跳转信息 144 | 变化 未获取到数据时不打印与输出结果 145 | 146 | 2023/4/22 147 | 修复 已知bug 148 | 变化 -d 改为正则表达式 149 | 变化 打印显示抓取来源 150 | 新增 敏感信息增加Other 151 | 新增 -ff 全部抓取的数据,视为同一个url的结果来处理（只打印一份结果 | 只会输出一份结果） 152 | 153 | 2023/2/21 154 | 修复 已知bug 155 | 156 | 2023/2/3 157 | 新增 域名信息展示 158 | 变化 -i配置文件可配置抓取规则等 159 | 160 | 2023/1/29 161 | 新增 -b 设置baseurl路径 162 | 新增 -o json、html格式导出 163 | 新增 部分敏感信息获取 164 | 新增 默认会进行简单的js爆破 165 | 变化 能抓到更多链接,但垃圾数据变多 166 | 变化 代理设置方式变更 167 | 168 | 2022/10/25 169 | 新增 -t 设置线程数(默认50) 170 | 新增 -z 对主域名的404链接fuzz测试 171 | 优化 部分细节 172 | 173 | 2022/10/6 174 | 新增 -x http代理设置 175 | 修改 多个相同域名导出时覆盖问题处理 176 | 177 | 2022/9/23 178 | 新增 对base标签的兼容 179 | 修复 正则bug 180 | 181 | 2022/9/16 182 | 新增 -m 3 安全的深入抓取,过滤delete、remove等危险URL 183 | 新增 -d 获取指定域名资源 184 | 新增 -o 导出到文件显示获取来源source 185 | 修复 已知bug 186 | 187 | 2022/9/15 188 | 修复 某种情况下的数组越界 189 | 190 | 2022/9/12 191 | 修复 linux与mac下的配置文件生成错误 192 | 修复 已知逻辑bug 193 | 194 | 2022/9/5 195 | 新增 链接存在标题时,显示标题 196 | 新增 -i 参数,加载yaml配置文件（目前只支持配置请求头headers） 197 | 修改 部分代码逻辑 198 | 修复 当ip存在端口时,导出会去除端口 199 | 200 | 2022/8/29 201 | 新增 抓取url数量显示 202 | 优化 部分代码 203 | 新增 提供各平台可执行文件 204 | 205 | 2022/8/27 206 | 新增 -o 改为自定义文件目录 207 | 新增 导出文件改为csv后缀,表格查看更方便 208 | 修复 已知正则bug 209 | 210 | 2022/8/19 211 | 优化 加长超时时间避免误判 212 | 213 | 2022/8/5 214 | 新增 状态码过滤 215 | 新增 状态码验证显示进度 216 | 修复 域名带端口输出本地错误问题 217 | 218 | 2022/7/25 219 | 优化 js规则 220 | 优化 排序 221 | 新增 根据状态码显示彩色字体 222 | 223 | 2022/7/6 224 | 完善 规则 225 | 226 | 2022/6/27 227 | 优化 规则 228 | 新增 提供linux成品程序 229 | 230 | 2022/6/21 231 | 修改 获取状态码从自动改为手动（-s） 232 | 新增 显示响应内容大小 233 | 234 | 2022/6/16 235 | 优化 提取规则增强兼容性 236 | 修复 数组越界错误处理 237 | 238 | 2022/6/14 239 | 修复 部分网站返回空值的问题 240 | 241 | 2022/6/13 242 | 新增 自定义user-agent请求头功能 243 | 新增 批量url抓取功能 244 | 新增 结果导出功能 245 | 优化 过滤规则 246 | 优化 结果排版 247 | 248 | 2022/6/8 249 | 修复 忽略ssl证书错误 250 | 251 | # 开发由来 252 | 致敬JSFinder！开发此工具的初衷是因为经常使用 JSFinder 时会返回空或链接不完整,而且作者已经很久没有更新修复 bug 了。因此,萌生了自己开发一款类似工具的想法。 253 | 254 | -------------------------------------------------------------------------------- /cmd/cmd.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "github.com/gookit/color" 7 | "os" 8 | ) 9 | 10 | var Update = "2023.9.9" 11 | var XUpdate string 12 | 13 | var ( 14 | H bool 15 | I bool 16 | M int 17 | S string 18 | U string 19 | D string 20 | C string 21 | A string 22 | B string 23 | F string 24 | FF string 25 | O string 26 | X string 27 | T = 50 28 | TI = 5 29 | MA = 99999 30 | Z int 31 | ) 32 | 33 | func init() { 34 | flag.StringVar(&A, "a", "", "set user-agent\n设置user-agent请求头") 35 | flag.StringVar(&B, "b", "", "set baseurl\n设置baseurl路径") 36 | flag.StringVar(&C, "c", "", "set cookie\n设置cookie") 37 | flag.StringVar(&D, "d", "", "set domainName\n指定获取的域名,支持正则表达式") 38 | flag.StringVar(&F, "f", "", "set urlFile\n批量抓取url,指定文件路径") 39 | flag.StringVar(&FF, "ff", "", "set urlFile one\n与-f区别：全部抓取的数据,视为同一个url的结果来处理（只打印一份结果 | 只会输出一份结果）") 40 | flag.BoolVar(&H, "h", false, "this help\n帮助信息") 41 | flag.BoolVar(&I, "i", false, "set configFile\n加载yaml配置文件（不存在时,会在当前目录创建一个默认yaml配置文件）") 42 | flag.IntVar(&M, "m", 1, "set mode\n抓取模式 \n 1 normal\n 正常抓取（默认） \n 2 thorough\n 深入抓取（默认url深入一层,js深入三层,-i可以自定义） \n 3 security\n 安全深入抓取（过滤delete,remove等敏感路由.-i可自定义） ") 43 | flag.IntVar(&MA, "max", 99999, "set maximum\n最大抓取链接数") 44 | flag.StringVar(&O, "o", "", "set outFile\n结果导出到csv、json、html文件,需指定导出文件目录,可填写完整文件名只导出一种类型（.代表当前目录）") 45 | flag.StringVar(&S, "s", "", "set Status\n显示指定状态码,all为显示全部（多个状态码用,隔开）") 46 | flag.IntVar(&T, "t", 50, "set Thread\n设置线程数（默认50）") 47 | flag.IntVar(&TI, "time", 5, "set Timeout\n设置超时时间（默认5,单位秒）") 48 | flag.StringVar(&U, "u", "", "set Url\n目标URL") 49 | flag.StringVar(&X, "x", "", "set Proxy\n设置代理,格式: http://username:password@127.0.0.1:8809") 50 | flag.IntVar(&Z, "z", 0, "set Fuzz\n对404链接进行fuzz(只对主域名下的链接生效,需要与 -s 一起使用） \n 1 decreasing\n 目录递减fuzz \n 2 2combination\n 2级目录组合fuzz（适合少量链接使用） \n 3 3combination\n 3级目录组合fuzz（适合少量链接使用） ") 51 | 52 | // 改变默认的 Usage 53 | flag.Usage = usage 54 | } 55 | func usage() { 56 | fmt.Fprintf(os.Stderr, `Usage: URLFinder [-a user-agent] [-b baseurl] [-c cookie] [-d domainName] [-f urlFile] [-ff urlFile one] [-h help] [-i configFile] [-m mode] [-max maximum] [-o outFile] [-s Status] [-t thread] [-time timeout] [-u url] [-x proxy] [-z fuzz] 57 | 58 | Options: 59 | `) 60 | flag.PrintDefaults() 61 | } 62 | 63 | func Parse() { 64 | color.LightCyan.Printf(" __ __ ___ _ _ \n /\\ /\\ /__\\ / / / __(_)_ __ __| | ___ _ __ \n/ / \\ \\/ \\/// / / _\\ | | '_ \\ / _` |/ _ \\ '__|\n\\ \\_/ / _ \\ /___ / | | | | | (_| | __/ | \n \\___/\\/ \\_\\____\\/ |_|_| |_|\\__,_|\\___|_| \n\nBy: pingc0y\nUpdate: %s | %s\nGithub: https://github.com/pingc0y/URLFinder \n\n", Update, XUpdate) 65 | flag.Parse() 66 | } 67 | -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "fmt" 5 | "github.com/pingc0y/URLFinder/cmd" 6 | "github.com/pingc0y/URLFinder/mode" 7 | "gopkg.in/yaml.v3" 8 | "os" 9 | "strings" 10 | "sync" 11 | ) 12 | 13 | var Conf mode.Config 14 | var Progress = 1 15 | var FuzzNum int 16 | 17 | var ( 18 | Risks = []string{"remove", "delete", "insert", "update", "logout"} 19 | 20 | JsFuzzPath = []string{ 21 | "login.js", 22 | "app.js", 23 | "main.js", 24 | "config.js", 25 | "admin.js", 26 | "info.js", 27 | "open.js", 28 | "user.js", 29 | "input.js", 30 | "list.js", 31 | "upload.js", 32 | } 33 | JsFind = []string{ 34 | "(https{0,1}:[-a-zA-Z0-9（）@:%_\\+.~#?&//=]{2,250}?[-a-zA-Z0-9（）@:%_\\+.~#?&//=]{3}[.]js)", 35 | "[\"'‘“`]\\s{0,6}(/{0,1}[-a-zA-Z0-9（）@:%_\\+.~#?&//=]{2,250}?[-a-zA-Z0-9（）@:%_\\+.~#?&//=]{3}[.]js)", 36 | "=\\s{0,6}[\",',’,”]{0,1}\\s{0,6}(/{0,1}[-a-zA-Z0-9（）@:%_\\+.~#?&//=]{2,250}?[-a-zA-Z0-9（）@:%_\\+.~#?&//=]{3}[.]js)", 37 | } 38 | UrlFind = []string{ 39 | "[\"'‘“`]\\s{0,6}(https{0,1}:[-a-zA-Z0-9()@:%_\\+.~#?&//={}]{2,250}?)\\s{0,6}[\"'‘“`]", 40 | "=\\s{0,6}(https{0,1}:[-a-zA-Z0-9()@:%_\\+.~#?&//={}]{2,250})", 41 | "[\"'‘“`]\\s{0,6}([#,.]{0,2}/[-a-zA-Z0-9()@:%_\\+.~#?&//={}]{2,250}?)\\s{0,6}[\"'‘“`]", 42 | "\"([-a-zA-Z0-9()@:%_\\+.~#?&//={}]+?[/]{1}[-a-zA-Z0-9()@:%_\\+.~#?&//={}]+?)\"", 43 | "href\\s{0,6}=\\s{0,6}[\"'‘“`]{0,1}\\s{0,6}([-a-zA-Z0-9()@:%_\\+.~#?&//={}]{2,250})|action\\s{0,6}=\\s{0,6}[\"'‘“`]{0,1}\\s{0,6}([-a-zA-Z0-9()@:%_\\+.~#?&//={}]{2,250})", 44 | } 45 | 46 | JsFiler = []string{ 47 | "www\\.w3\\.org", 48 | "example\\.com", 49 | } 50 | UrlFiler = []string{ 51 | "\\.js\\?|\\.css\\?|\\.jpeg\\?|\\.jpg\\?|\\.png\\?|.gif\\?|www\\.w3\\.org|example\\.com|\\<|\\>|\\{|\\}|\\[|\\]|\\||\\^|;|/js/|\\.src|\\.replace|\\.url|\\.att|\\.href|location\\.href|javascript:|location:|application/x-www-form-urlencoded|\\.createObject|:location|\\.path|\\*#__PURE__\\*|\\*\\$0\\*|\\n", 52 | ".*\\.js$|.*\\.css$|.*\\.scss$|.*,$|.*\\.jpeg$|.*\\.jpg$|.*\\.png$|.*\\.gif$|.*\\.ico$|.*\\.svg$|.*\\.vue$|.*\\.ts$", 53 | } 54 | 55 | Phone = []string{"['\"](1(3([0-35-9]\\d|4[1-8])|4[14-9]\\d|5([\\d]\\d|7[1-79])|66\\d|7[2-35-8]\\d|8\\d{2}|9[89]\\d)\\d{7})['\"]"} 56 | Email = []string{"['\"]([\\w!#$%&'*+=?^_`{|}~-]+(?:\\.[\\w!#$%&'*+=?^_`{|}~-]+)*@(?:[\\w](?:[\\w-]*[\\w])?\\.)+[\\w](?:[\\w-]*[\\w])?)['\"]"} 57 | IDcard = []string{"['\"]((\\d{8}(0\\d|10|11|12)([0-2]\\d|30|31)\\d{3}$)|(\\d{6}(18|19|20)\\d{2}(0[1-9]|10|11|12)([0-2]\\d|30|31)\\d{3}(\\d|X|x)))['\"]"} 58 | Jwt = []string{"['\"](ey[A-Za-z0-9_-]{10,}\\.[A-Za-z0-9._-]{10,}|ey[A-Za-z0-9_\\/+-]{10,}\\.[A-Za-z0-9._\\/+-]{10,})['\"]"} 59 | Other = []string{"(access.{0,1}key|access.{0,1}Key|access.{0,1}Id|access.{0,1}id|.{0,5}密码|.{0,5}账号|默认.{0,5}|加密|解密|password:.{0,10}|username:.{0,10})"} 60 | ) 61 | 62 | var ( 63 | UrlSteps = 1 64 | JsSteps = 3 65 | ) 66 | 67 | var ( 68 | Lock sync.Mutex 69 | Wg sync.WaitGroup 70 | Mux sync.Mutex 71 | Ch = make(chan int, 50) 72 | Jsch = make(chan int, 50/10*3) 73 | Urlch = make(chan int, 50/10*7) 74 | ) 75 | 76 | // 读取配置文件 77 | func GetConfig(path string) { 78 | if f, err := os.Open(path); err != nil { 79 | if strings.Contains(err.Error(), "The system cannot find the file specified") || strings.Contains(err.Error(), "no such file or directory") { 80 | Conf.Headers = map[string]string{"Cookie": cmd.C, "User-Agent": `Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.87 Safari/537.36 SE 2.X MetaSr 1.0`, "Accept": "*/*"} 81 | Conf.Proxy = "" 82 | Conf.JsFind = JsFind 83 | Conf.UrlFind = UrlFind 84 | Conf.JsFiler = JsFiler 85 | Conf.UrlFiler = UrlFiler 86 | Conf.JsFuzzPath = JsFuzzPath 87 | Conf.JsSteps = JsSteps 88 | Conf.UrlSteps = UrlSteps 89 | Conf.Risks = Risks 90 | Conf.Timeout = cmd.TI 91 | Conf.Thread = cmd.T 92 | Conf.Max = cmd.MA 93 | Conf.InfoFind = map[string][]string{"Phone": Phone, "Email": Email, "IDcard": IDcard, "Jwt": Jwt, "Other": Other} 94 | data, err2 := yaml.Marshal(Conf) 95 | err2 = os.WriteFile(path, data, 0644) 96 | if err2 != nil { 97 | fmt.Println(err) 98 | } else { 99 | fmt.Println("未找到配置文件,已在当面目录下创建配置文件: config.yaml") 100 | } 101 | } else { 102 | fmt.Println("配置文件错误,请尝试重新生成配置文件") 103 | fmt.Println(err) 104 | } 105 | os.Exit(1) 106 | } else { 107 | yaml.NewDecoder(f).Decode(&Conf) 108 | JsFind = Conf.JsFind 109 | UrlFind = Conf.UrlFind 110 | JsFiler = Conf.JsFiler 111 | UrlFiler = Conf.UrlFiler 112 | JsFuzzPath = Conf.JsFuzzPath 113 | Phone = Conf.InfoFind["Phone"] 114 | Email = Conf.InfoFind["Email"] 115 | IDcard = Conf.InfoFind["IDcard"] 116 | Jwt = Conf.InfoFind["Jwt"] 117 | Other = Conf.InfoFind["Other"] 118 | JsSteps = Conf.JsSteps 119 | UrlSteps = Conf.UrlSteps 120 | Risks = Conf.Risks 121 | cmd.T = Conf.Thread 122 | cmd.MA = Conf.Max 123 | cmd.TI = Conf.Timeout 124 | } 125 | 126 | } 127 | -------------------------------------------------------------------------------- /crawler/crawler.go: -------------------------------------------------------------------------------- 1 | package crawler 2 | 3 | import ( 4 | "compress/gzip" 5 | "fmt" 6 | "github.com/pingc0y/URLFinder/cmd" 7 | "github.com/pingc0y/URLFinder/config" 8 | "github.com/pingc0y/URLFinder/result" 9 | "github.com/pingc0y/URLFinder/util" 10 | "io" 11 | "net/http" 12 | "net/url" 13 | "regexp" 14 | "strings" 15 | ) 16 | 17 | // 蜘蛛抓取页面内容 18 | func Spider(u string, num int) { 19 | is := true 20 | defer func() { 21 | config.Wg.Done() 22 | if is { 23 | <-config.Ch 24 | } 25 | 26 | }() 27 | config.Mux.Lock() 28 | fmt.Printf("\rStart %d Spider...", config.Progress) 29 | config.Progress++ 30 | config.Mux.Unlock() 31 | //标记完成 32 | 33 | u, _ = url.QueryUnescape(u) 34 | if num > 1 && cmd.D != "" && !regexp.MustCompile(cmd.D).MatchString(u) { 35 | return 36 | } 37 | if GetEndUrl(u) { 38 | return 39 | } 40 | if cmd.M == 3 { 41 | for _, v := range config.Risks { 42 | if strings.Contains(u, v) { 43 | return 44 | } 45 | } 46 | } 47 | AppendEndUrl(u) 48 | request, err := http.NewRequest("GET", u, nil) 49 | if err != nil { 50 | return 51 | } 52 | 53 | request.Header.Set("Accept-Encoding", "gzip") //使用gzip压缩传输数据让访问更快 54 | request.Header.Set("User-Agent", util.GetUserAgent()) 55 | request.Header.Set("Accept", "*/*") 56 | //增加header选项 57 | if cmd.C != "" { 58 | request.Header.Set("Cookie", cmd.C) 59 | } 60 | //加载yaml配置(headers) 61 | if cmd.I { 62 | util.SetHeadersConfig(&request.Header) 63 | } 64 | 65 | //处理返回结果 66 | //tr := &http.Transport{ 67 | // TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, 68 | //} 69 | //client = &http.Client{Timeout: time.Duration(cmd.TI) * time.Second, 70 | // Transport: tr, 71 | // CheckRedirect: func(req *http.Request, via []*http.Request) error { 72 | // if len(via) >= 10 { 73 | // return fmt.Errorf("Too many redirects") 74 | // } 75 | // if len(via) > 0 { 76 | // if via[0] != nil && via[0].URL != nil { 77 | // result.Redirect[via[0].URL.String()] = true 78 | // } else { 79 | // result.Redirect[req.URL.String()] = true 80 | // } 81 | // 82 | // } 83 | // return nil 84 | // }, 85 | //} 86 | response, err := client.Do(request) 87 | if err != nil { 88 | return 89 | } 90 | defer response.Body.Close() 91 | 92 | result := "" 93 | //解压 94 | if response.Header.Get("Content-Encoding") == "gzip" { 95 | reader, err := gzip.NewReader(response.Body) // gzip解压缩 96 | if err != nil { 97 | return 98 | } 99 | defer reader.Close() 100 | con, err := io.ReadAll(reader) 101 | if err != nil { 102 | return 103 | } 104 | result = string(con) 105 | } else { 106 | //提取url用于拼接其他url或js 107 | dataBytes, err := io.ReadAll(response.Body) 108 | if err != nil { 109 | return 110 | } 111 | //字节数组 转换成 字符串 112 | result = string(dataBytes) 113 | } 114 | path := response.Request.URL.Path 115 | host := response.Request.URL.Host 116 | scheme := response.Request.URL.Scheme 117 | source := scheme + "://" + host + path 118 | //处理base标签 119 | re := regexp.MustCompile("base.{1,5}href.{1,5}(http.+?//[^\\s]+?)[\"'‘“]") 120 | base := re.FindAllStringSubmatch(result, -1) 121 | if len(base) > 0 { 122 | host = regexp.MustCompile("http.*?//([^/]+)").FindAllStringSubmatch(base[0][1], -1)[0][1] 123 | scheme = regexp.MustCompile("(http.*?)://").FindAllStringSubmatch(base[0][1], -1)[0][1] 124 | paths := regexp.MustCompile("http.*?//.*?(/.*)").FindAllStringSubmatch(base[0][1], -1) 125 | if len(paths) > 0 { 126 | path = paths[0][1] 127 | } else { 128 | path = "/" 129 | } 130 | } 131 | is = false 132 | <-config.Ch 133 | //提取js 134 | jsFind(result, host, scheme, path, u, num) 135 | //提取url 136 | urlFind(result, host, scheme, path, u, num) 137 | //提取信息 138 | infoFind(result, source) 139 | 140 | } 141 | 142 | // 打印Validate进度 143 | func PrintProgress() { 144 | config.Mux.Lock() 145 | num := len(result.ResultJs) + len(result.ResultUrl) 146 | fmt.Printf("\rValidate %.0f%%", float64(config.Progress+1)/float64(num+1)*100) 147 | config.Progress++ 148 | config.Mux.Unlock() 149 | } 150 | -------------------------------------------------------------------------------- /crawler/filter.go: -------------------------------------------------------------------------------- 1 | package crawler 2 | 3 | import ( 4 | "github.com/pingc0y/URLFinder/config" 5 | "net/url" 6 | "regexp" 7 | "strings" 8 | ) 9 | 10 | // 过滤JS 11 | func jsFilter(str [][]string) [][]string { 12 | 13 | //对不需要的数据过滤 14 | for i := range str { 15 | str[i][0], _ = url.QueryUnescape(str[i][1]) 16 | str[i][0] = strings.TrimSpace(str[i][0]) 17 | str[i][0] = strings.Replace(str[i][0], " ", "", -1) 18 | str[i][0] = strings.Replace(str[i][0], "\\/", "/", -1) 19 | str[i][0] = strings.Replace(str[i][0], "%3A", ":", -1) 20 | str[i][0] = strings.Replace(str[i][0], "%2F", "/", -1) 21 | //去除不是.js的链接 22 | if !strings.HasSuffix(str[i][0], ".js") && !strings.Contains(str[i][0], ".js?") { 23 | str[i][0] = "" 24 | continue 25 | } 26 | 27 | //过滤配置的黑名单 28 | for i2 := range config.JsFiler { 29 | re := regexp.MustCompile(config.JsFiler[i2]) 30 | is := re.MatchString(str[i][0]) 31 | if is { 32 | str[i][0] = "" 33 | break 34 | } 35 | } 36 | 37 | } 38 | return str 39 | 40 | } 41 | 42 | // 过滤URL 43 | func urlFilter(str [][]string) [][]string { 44 | 45 | //对不需要的数据过滤 46 | for i := range str { 47 | str[i][0], _ = url.QueryUnescape(str[i][1]) 48 | str[i][0] = strings.TrimSpace(str[i][0]) 49 | str[i][0] = strings.Replace(str[i][0], " ", "", -1) 50 | str[i][0] = strings.Replace(str[i][0], "\\/", "/", -1) 51 | str[i][0] = strings.Replace(str[i][0], "%3A", ":", -1) 52 | str[i][0] = strings.Replace(str[i][0], "%2F", "/", -1) 53 | //去除不存在字符串和数字的url,判断为错误数据 54 | match, _ := regexp.MatchString("[a-zA-Z]+|[0-9]+", str[i][0]) 55 | if !match { 56 | str[i][0] = "" 57 | continue 58 | } 59 | 60 | //对抓到的域名做处理 61 | re := regexp.MustCompile("([a-z0-9\\-]+\\.)+([a-z0-9\\-]+\\.[a-z0-9\\-]+)(:[0-9]+)?").FindAllString(str[i][0], 1) 62 | if len(re) != 0 && !strings.HasPrefix(str[i][0], "http") && !strings.HasPrefix(str[i][0], "/") { 63 | str[i][0] = "http://" + str[i][0] 64 | } 65 | 66 | //过滤配置的黑名单 67 | for i2 := range config.UrlFiler { 68 | re := regexp.MustCompile(config.UrlFiler[i2]) 69 | is := re.MatchString(str[i][0]) 70 | if is { 71 | str[i][0] = "" 72 | break 73 | } 74 | } 75 | 76 | } 77 | return str 78 | } 79 | -------------------------------------------------------------------------------- /crawler/find.go: -------------------------------------------------------------------------------- 1 | package crawler 2 | 3 | import ( 4 | "github.com/pingc0y/URLFinder/cmd" 5 | "github.com/pingc0y/URLFinder/config" 6 | "github.com/pingc0y/URLFinder/mode" 7 | "github.com/pingc0y/URLFinder/result" 8 | "regexp" 9 | "strings" 10 | ) 11 | 12 | // 分析内容中的js 13 | func jsFind(cont, host, scheme, path, source string, num int) { 14 | var cata string 15 | care := regexp.MustCompile("/.*/{1}|/") 16 | catae := care.FindAllString(path, -1) 17 | if len(catae) == 0 { 18 | cata = "/" 19 | } else { 20 | cata = catae[0] 21 | } 22 | //js匹配正则 23 | host = scheme + "://" + host 24 | for _, re := range config.JsFind { 25 | reg := regexp.MustCompile(re) 26 | jss := reg.FindAllStringSubmatch(cont, -1) 27 | //return 28 | jss = jsFilter(jss) 29 | //循环提取js放到结果中 30 | for _, js := range jss { 31 | if js[0] == "" { 32 | continue 33 | } 34 | if strings.HasPrefix(js[0], "https:") || strings.HasPrefix(js[0], "http:") { 35 | switch AppendJs(js[0], source) { 36 | case 0: 37 | if num <= config.JsSteps && (cmd.M == 2 || cmd.M == 3) { 38 | config.Wg.Add(1) 39 | config.Ch <- 1 40 | go Spider(js[0], num+1) 41 | } 42 | case 1: 43 | return 44 | case 2: 45 | continue 46 | } 47 | 48 | } else if strings.HasPrefix(js[0], "//") { 49 | switch AppendJs(scheme+":"+js[0], source) { 50 | case 0: 51 | if num <= config.JsSteps && (cmd.M == 2 || cmd.M == 3) { 52 | config.Wg.Add(1) 53 | config.Ch <- 1 54 | go Spider(scheme+":"+js[0], num+1) 55 | } 56 | case 1: 57 | return 58 | case 2: 59 | continue 60 | } 61 | 62 | } else if strings.HasPrefix(js[0], "/") { 63 | switch AppendJs(host+js[0], source) { 64 | case 0: 65 | if num <= config.JsSteps && (cmd.M == 2 || cmd.M == 3) { 66 | config.Wg.Add(1) 67 | config.Ch <- 1 68 | go Spider(host+js[0], num+1) 69 | } 70 | case 1: 71 | return 72 | case 2: 73 | continue 74 | } 75 | 76 | } else { 77 | switch AppendJs(host+cata+js[0], source) { 78 | case 0: 79 | if num <= config.JsSteps && (cmd.M == 2 || cmd.M == 3) { 80 | config.Wg.Add(1) 81 | config.Ch <- 1 82 | go Spider(host+cata+js[0], num+1) 83 | } 84 | case 1: 85 | return 86 | case 2: 87 | continue 88 | } 89 | 90 | } 91 | } 92 | 93 | } 94 | 95 | } 96 | 97 | // 分析内容中的url 98 | func urlFind(cont, host, scheme, path, source string, num int) { 99 | var cata string 100 | care := regexp.MustCompile("/.*/{1}|/") 101 | catae := care.FindAllString(path, -1) 102 | if len(catae) == 0 { 103 | cata = "/" 104 | } else { 105 | cata = catae[0] 106 | } 107 | host = scheme + "://" + host 108 | 109 | //url匹配正则 110 | 111 | for _, re := range config.UrlFind { 112 | reg := regexp.MustCompile(re) 113 | urls := reg.FindAllStringSubmatch(cont, -1) 114 | //fmt.Println(urls) 115 | urls = urlFilter(urls) 116 | 117 | //循环提取url放到结果中 118 | for _, url := range urls { 119 | if url[0] == "" { 120 | continue 121 | } 122 | if strings.HasPrefix(url[0], "https:") || strings.HasPrefix(url[0], "http:") { 123 | switch AppendUrl(url[0], source) { 124 | case 0: 125 | if num <= config.UrlSteps && (cmd.M == 2 || cmd.M == 3) { 126 | config.Wg.Add(1) 127 | config.Ch <- 1 128 | go Spider(url[0], num+1) 129 | } 130 | case 1: 131 | return 132 | case 2: 133 | continue 134 | } 135 | } else if strings.HasPrefix(url[0], "//") { 136 | switch AppendUrl(scheme+":"+url[0], source) { 137 | case 0: 138 | if num <= config.UrlSteps && (cmd.M == 2 || cmd.M == 3) { 139 | config.Wg.Add(1) 140 | config.Ch <- 1 141 | go Spider(scheme+":"+url[0], num+1) 142 | } 143 | case 1: 144 | return 145 | case 2: 146 | continue 147 | } 148 | 149 | } else if strings.HasPrefix(url[0], "/") { 150 | urlz := "" 151 | if cmd.B != "" { 152 | urlz = cmd.B + url[0] 153 | } else { 154 | urlz = host + url[0] 155 | } 156 | switch AppendUrl(urlz, source) { 157 | case 0: 158 | if num <= config.UrlSteps && (cmd.M == 2 || cmd.M == 3) { 159 | config.Wg.Add(1) 160 | config.Ch <- 1 161 | go Spider(urlz, num+1) 162 | } 163 | case 1: 164 | return 165 | case 2: 166 | continue 167 | } 168 | } else if !strings.HasSuffix(source, ".js") { 169 | urlz := "" 170 | if cmd.B != "" { 171 | if strings.HasSuffix(cmd.B, "/") { 172 | urlz = cmd.B + url[0] 173 | } else { 174 | urlz = cmd.B + "/" + url[0] 175 | } 176 | } else { 177 | urlz = host + cata + url[0] 178 | } 179 | switch AppendUrl(urlz, source) { 180 | case 0: 181 | if num <= config.UrlSteps && (cmd.M == 2 || cmd.M == 3) { 182 | config.Wg.Add(1) 183 | config.Ch <- 1 184 | go Spider(urlz, num+1) 185 | } 186 | case 1: 187 | return 188 | case 2: 189 | continue 190 | } 191 | 192 | } else if strings.HasSuffix(source, ".js") { 193 | urlz := "" 194 | if cmd.B != "" { 195 | if strings.HasSuffix(cmd.B, "/") { 196 | urlz = cmd.B + url[0] 197 | } else { 198 | urlz = cmd.B + "/" + url[0] 199 | } 200 | } else { 201 | config.Lock.Lock() 202 | su := result.Jsinurl[source] 203 | config.Lock.Unlock() 204 | if strings.HasSuffix(su, "/") { 205 | urlz = su + url[0] 206 | } else { 207 | urlz = su + "/" + url[0] 208 | } 209 | } 210 | switch AppendUrl(urlz, source) { 211 | case 0: 212 | if num <= config.UrlSteps && (cmd.M == 2 || cmd.M == 3) { 213 | config.Wg.Add(1) 214 | config.Ch <- 1 215 | go Spider(urlz, num+1) 216 | } 217 | case 1: 218 | return 219 | case 2: 220 | continue 221 | } 222 | 223 | } 224 | } 225 | } 226 | } 227 | 228 | // 分析内容中的敏感信息 229 | func infoFind(cont, source string) { 230 | info := mode.Info{} 231 | //手机号码 232 | for i := range config.Phone { 233 | phones := regexp.MustCompile(config.Phone[i]).FindAllStringSubmatch(cont, -1) 234 | for i := range phones { 235 | info.Phone = append(info.Phone, phones[i][1]) 236 | } 237 | } 238 | 239 | for i := range config.Email { 240 | emails := regexp.MustCompile(config.Email[i]).FindAllStringSubmatch(cont, -1) 241 | for i := range emails { 242 | info.Email = append(info.Email, emails[i][1]) 243 | } 244 | } 245 | 246 | for i := range config.IDcard { 247 | IDcards := regexp.MustCompile(config.IDcard[i]).FindAllStringSubmatch(cont, -1) 248 | for i := range IDcards { 249 | info.IDcard = append(info.IDcard, IDcards[i][1]) 250 | } 251 | } 252 | 253 | for i := range config.Jwt { 254 | Jwts := regexp.MustCompile(config.Jwt[i]).FindAllStringSubmatch(cont, -1) 255 | for i := range Jwts { 256 | info.JWT = append(info.JWT, Jwts[i][1]) 257 | } 258 | } 259 | for i := range config.Other { 260 | Others := regexp.MustCompile(config.Other[i]).FindAllStringSubmatch(cont, -1) 261 | for i := range Others { 262 | info.Other = append(info.Other, Others[i][1]) 263 | } 264 | } 265 | 266 | info.Source = source 267 | if len(info.Phone) != 0 || len(info.IDcard) != 0 || len(info.JWT) != 0 || len(info.Email) != 0 || len(info.Other) != 0 { 268 | AppendInfo(info) 269 | } 270 | 271 | } 272 | -------------------------------------------------------------------------------- /crawler/jsFuzz.go: -------------------------------------------------------------------------------- 1 | package crawler 2 | 3 | import ( 4 | "github.com/pingc0y/URLFinder/config" 5 | "github.com/pingc0y/URLFinder/mode" 6 | "github.com/pingc0y/URLFinder/result" 7 | "github.com/pingc0y/URLFinder/util" 8 | "regexp" 9 | ) 10 | 11 | func JsFuzz() { 12 | 13 | paths := []string{} 14 | for i := range result.ResultJs { 15 | re := regexp.MustCompile("(.+/)[^/]+.js").FindAllStringSubmatch(result.ResultJs[i].Url, -1) 16 | if len(re) != 0 { 17 | paths = append(paths, re[0][1]) 18 | } 19 | re2 := regexp.MustCompile("(https{0,1}://([a-z0-9\\-]+\\.)*([a-z0-9\\-]+\\.[a-z0-9\\-]+)(:[0-9]+)?/)").FindAllStringSubmatch(result.ResultJs[i].Url, -1) 20 | if len(re2) != 0 { 21 | paths = append(paths, re2[0][1]) 22 | } 23 | } 24 | paths = util.UniqueArr(paths) 25 | for i := range paths { 26 | for i2 := range config.JsFuzzPath { 27 | result.ResultJs = append(result.ResultJs, mode.Link{ 28 | Url: paths[i] + config.JsFuzzPath[i2], 29 | Source: "Fuzz", 30 | }) 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /crawler/run.go: -------------------------------------------------------------------------------- 1 | package crawler 2 | 3 | import ( 4 | "bufio" 5 | "crypto/tls" 6 | "flag" 7 | "fmt" 8 | "github.com/pingc0y/URLFinder/cmd" 9 | "github.com/pingc0y/URLFinder/config" 10 | "github.com/pingc0y/URLFinder/mode" 11 | "github.com/pingc0y/URLFinder/result" 12 | "github.com/pingc0y/URLFinder/util" 13 | "io" 14 | "net" 15 | "net/http" 16 | "net/url" 17 | "os" 18 | "regexp" 19 | "strings" 20 | "time" 21 | ) 22 | 23 | var client *http.Client 24 | 25 | func load() { 26 | 27 | if cmd.I { 28 | config.GetConfig("config.yaml") 29 | } 30 | if cmd.H { 31 | flag.Usage() 32 | os.Exit(0) 33 | } 34 | if cmd.U == "" && cmd.F == "" && cmd.FF == "" { 35 | fmt.Println("至少使用 -u -f -ff 指定一个url") 36 | os.Exit(0) 37 | } 38 | u, ok := url.Parse(cmd.U) 39 | if cmd.U != "" && ok != nil { 40 | fmt.Println("url格式错误,请填写正确url") 41 | os.Exit(0) 42 | } 43 | cmd.U = u.String() 44 | 45 | if cmd.T != 50 { 46 | config.Ch = make(chan int, cmd.T) 47 | config.Jsch = make(chan int, cmd.T/10*3) 48 | config.Urlch = make(chan int, cmd.T/10*7) 49 | } 50 | 51 | tr := &http.Transport{ 52 | TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, 53 | Proxy: http.ProxyFromEnvironment, 54 | DialContext: (&net.Dialer{ 55 | Timeout: time.Second * 30, 56 | KeepAlive: time.Second * 30, 57 | }).DialContext, 58 | MaxIdleConns: cmd.T / 2, 59 | MaxIdleConnsPerHost: cmd.T + 10, 60 | IdleConnTimeout: time.Second * 90, 61 | TLSHandshakeTimeout: time.Second * 90, 62 | ExpectContinueTimeout: time.Second * 10, 63 | } 64 | 65 | if cmd.X != "" { 66 | tr.DisableKeepAlives = true 67 | proxyUrl, parseErr := url.Parse(cmd.X) 68 | if parseErr != nil { 69 | fmt.Println("代理地址错误: \n" + parseErr.Error()) 70 | os.Exit(1) 71 | } 72 | tr.Proxy = http.ProxyURL(proxyUrl) 73 | } 74 | if cmd.I { 75 | util.SetProxyConfig(tr) 76 | } 77 | client = &http.Client{Timeout: time.Duration(cmd.TI) * time.Second, 78 | Transport: tr, 79 | CheckRedirect: func(req *http.Request, via []*http.Request) error { 80 | if len(via) >= 10 { 81 | return fmt.Errorf("Too many redirects") 82 | } 83 | if len(via) > 0 { 84 | if via[0] != nil && via[0].URL != nil { 85 | AddRedirect(via[0].URL.String()) 86 | } else { 87 | AddRedirect(req.URL.String()) 88 | } 89 | 90 | } 91 | return nil 92 | }, 93 | } 94 | 95 | } 96 | 97 | func Run() { 98 | load() 99 | if cmd.F != "" { 100 | // 创建句柄 101 | fi, err := os.Open(cmd.F) 102 | if err != nil { 103 | fmt.Println(err) 104 | os.Exit(0) 105 | } 106 | r := bufio.NewReader(fi) // 创建 Reader 107 | for { 108 | lineBytes, err := r.ReadBytes('\n') 109 | //去掉字符串首尾空白字符,返回字符串 110 | if len(lineBytes) > 5 { 111 | line := util.GetProtocol(strings.TrimSpace(string(lineBytes))) 112 | cmd.U = line 113 | Initialization() 114 | start(cmd.U) 115 | Res() 116 | fmt.Println("----------------------------------------") 117 | } 118 | if err == io.EOF { 119 | break 120 | } 121 | 122 | } 123 | return 124 | } 125 | if cmd.FF != "" { 126 | // 创建句柄 127 | fi, err := os.Open(cmd.FF) 128 | if err != nil { 129 | fmt.Println(err) 130 | os.Exit(0) 131 | } 132 | r := bufio.NewReader(fi) // 创建 Reader 133 | Initialization() 134 | for { 135 | lineBytes, err := r.ReadBytes('\n') 136 | //去掉字符串首尾空白字符,返回字符串 137 | if len(lineBytes) > 5 { 138 | line := util.GetProtocol(strings.TrimSpace(string(lineBytes))) 139 | if cmd.U == "" { 140 | cmd.U = line 141 | } 142 | startFF(line) 143 | fmt.Println("----------------------------------------") 144 | } 145 | 146 | if err == io.EOF { 147 | break 148 | } 149 | } 150 | ValidateFF() 151 | Res() 152 | return 153 | } 154 | Initialization() 155 | cmd.U = util.GetProtocol(cmd.U) 156 | start(cmd.U) 157 | Res() 158 | } 159 | func startFF(u string) { 160 | fmt.Println("Target URL: " + u) 161 | config.Wg.Add(1) 162 | config.Ch <- 1 163 | go Spider(u, 1) 164 | config.Wg.Wait() 165 | config.Progress = 1 166 | fmt.Printf("\r\nSpider OK \n") 167 | } 168 | 169 | func ValidateFF() { 170 | result.ResultUrl = util.RemoveRepeatElement(result.ResultUrl) 171 | result.ResultJs = util.RemoveRepeatElement(result.ResultJs) 172 | if cmd.S != "" { 173 | fmt.Printf("Start %d Validate...\n", len(result.ResultUrl)+len(result.ResultJs)) 174 | fmt.Printf("\r ") 175 | JsFuzz() 176 | //验证JS状态 177 | for i, s := range result.ResultJs { 178 | config.Wg.Add(1) 179 | config.Jsch <- 1 180 | go JsState(s.Url, i, result.ResultJs[i].Source) 181 | } 182 | //验证URL状态 183 | for i, s := range result.ResultUrl { 184 | config.Wg.Add(1) 185 | config.Urlch <- 1 186 | go UrlState(s.Url, i) 187 | } 188 | config.Wg.Wait() 189 | 190 | time.Sleep(1 * time.Second) 191 | fmt.Printf("\r ") 192 | fmt.Printf("\rValidate OK \n\n") 193 | 194 | if cmd.Z != 0 { 195 | UrlFuzz() 196 | time.Sleep(1 * time.Second) 197 | } 198 | } 199 | AddSource() 200 | } 201 | 202 | func start(u string) { 203 | fmt.Println("Target URL: " + u) 204 | config.Wg.Add(1) 205 | config.Ch <- 1 206 | go Spider(u, 1) 207 | config.Wg.Wait() 208 | config.Progress = 1 209 | fmt.Printf("\r\nSpider OK \n") 210 | result.ResultUrl = util.RemoveRepeatElement(result.ResultUrl) 211 | result.ResultJs = util.RemoveRepeatElement(result.ResultJs) 212 | if cmd.S != "" { 213 | fmt.Printf("Start %d Validate...\n", len(result.ResultUrl)+len(result.ResultJs)) 214 | fmt.Printf("\r ") 215 | JsFuzz() 216 | //验证JS状态 217 | for i, s := range result.ResultJs { 218 | config.Wg.Add(1) 219 | config.Jsch <- 1 220 | go JsState(s.Url, i, result.ResultJs[i].Source) 221 | } 222 | //验证URL状态 223 | for i, s := range result.ResultUrl { 224 | config.Wg.Add(1) 225 | config.Urlch <- 1 226 | go UrlState(s.Url, i) 227 | } 228 | config.Wg.Wait() 229 | 230 | time.Sleep(1 * time.Second) 231 | fmt.Printf("\r ") 232 | fmt.Printf("\rValidate OK \n\n") 233 | 234 | if cmd.Z != 0 { 235 | UrlFuzz() 236 | time.Sleep(1 * time.Second) 237 | } 238 | } 239 | AddSource() 240 | } 241 | 242 | func Res() { 243 | if len(result.ResultJs) == 0 && len(result.ResultUrl) == 0 { 244 | fmt.Println("未获取到数据") 245 | return 246 | } 247 | //打印还是输出 248 | if len(cmd.O) > 0 { 249 | if strings.HasSuffix(cmd.O, ".json") { 250 | result.OutFileJson(cmd.O) 251 | } else if strings.HasSuffix(cmd.O, ".html") { 252 | result.OutFileHtml(cmd.O) 253 | } else if strings.HasSuffix(cmd.O, ".csv") { 254 | result.OutFileCsv(cmd.O) 255 | } else { 256 | result.OutFileJson("") 257 | result.OutFileCsv("") 258 | result.OutFileHtml("") 259 | } 260 | } else { 261 | UrlToRedirect() 262 | result.Print() 263 | } 264 | } 265 | 266 | func AppendJs(ur string, urltjs string) int { 267 | config.Lock.Lock() 268 | defer config.Lock.Unlock() 269 | if len(result.ResultUrl)+len(result.ResultJs) >= cmd.MA { 270 | return 1 271 | } 272 | _, err := url.Parse(ur) 273 | if err != nil { 274 | return 2 275 | } 276 | for _, eachItem := range result.ResultJs { 277 | if eachItem.Url == ur { 278 | return 0 279 | } 280 | } 281 | result.ResultJs = append(result.ResultJs, mode.Link{Url: ur}) 282 | if strings.HasSuffix(urltjs, ".js") { 283 | result.Jsinurl[ur] = result.Jsinurl[urltjs] 284 | } else { 285 | re := regexp.MustCompile("[a-zA-z]+://[^\\s]*/|[a-zA-z]+://[^\\s]*") 286 | u := re.FindAllStringSubmatch(urltjs, -1) 287 | result.Jsinurl[ur] = u[0][0] 288 | } 289 | result.Jstourl[ur] = urltjs 290 | return 0 291 | 292 | } 293 | 294 | func AppendUrl(ur string, urlturl string) int { 295 | config.Lock.Lock() 296 | defer config.Lock.Unlock() 297 | if len(result.ResultUrl)+len(result.ResultJs) >= cmd.MA { 298 | return 1 299 | } 300 | _, err := url.Parse(ur) 301 | if err != nil { 302 | return 2 303 | } 304 | for _, eachItem := range result.ResultUrl { 305 | if eachItem.Url == ur { 306 | return 0 307 | } 308 | } 309 | url.Parse(ur) 310 | result.ResultUrl = append(result.ResultUrl, mode.Link{Url: ur}) 311 | result.Urltourl[ur] = urlturl 312 | return 0 313 | } 314 | 315 | func AppendInfo(info mode.Info) { 316 | config.Lock.Lock() 317 | defer config.Lock.Unlock() 318 | result.Infos = append(result.Infos, info) 319 | } 320 | 321 | func AppendEndUrl(url string) { 322 | config.Lock.Lock() 323 | defer config.Lock.Unlock() 324 | for _, eachItem := range result.EndUrl { 325 | if eachItem == url { 326 | return 327 | } 328 | } 329 | result.EndUrl = append(result.EndUrl, url) 330 | 331 | } 332 | 333 | func GetEndUrl(url string) bool { 334 | config.Lock.Lock() 335 | defer config.Lock.Unlock() 336 | for _, eachItem := range result.EndUrl { 337 | if eachItem == url { 338 | return true 339 | } 340 | } 341 | return false 342 | 343 | } 344 | 345 | func AddRedirect(url string) { 346 | config.Lock.Lock() 347 | defer config.Lock.Unlock() 348 | result.Redirect[url] = true 349 | } 350 | 351 | func AddSource() { 352 | for i := range result.ResultJs { 353 | result.ResultJs[i].Source = result.Jstourl[result.ResultJs[i].Url] 354 | } 355 | for i := range result.ResultUrl { 356 | result.ResultUrl[i].Source = result.Urltourl[result.ResultUrl[i].Url] 357 | } 358 | 359 | } 360 | 361 | func UrlToRedirect() { 362 | for i := range result.ResultJs { 363 | if result.ResultJs[i].Status == "302" { 364 | result.ResultJs[i].Url = result.ResultJs[i].Url + " -> " + result.ResultJs[i].Redirect 365 | } 366 | } 367 | for i := range result.ResultUrl { 368 | if result.ResultUrl[i].Status == "302" { 369 | result.ResultUrl[i].Url = result.ResultUrl[i].Url + " -> " + result.ResultUrl[i].Redirect 370 | } 371 | } 372 | 373 | } 374 | 375 | func Initialization() { 376 | result.ResultJs = []mode.Link{} 377 | result.ResultUrl = []mode.Link{} 378 | result.Fuzzs = []mode.Link{} 379 | result.Infos = []mode.Info{} 380 | result.EndUrl = []string{} 381 | result.Domains = []string{} 382 | result.Jsinurl = make(map[string]string) 383 | result.Jstourl = make(map[string]string) 384 | result.Urltourl = make(map[string]string) 385 | result.Redirect = make(map[string]bool) 386 | 387 | } 388 | -------------------------------------------------------------------------------- /crawler/state.go: -------------------------------------------------------------------------------- 1 | package crawler 2 | 3 | import ( 4 | "github.com/pingc0y/URLFinder/cmd" 5 | "github.com/pingc0y/URLFinder/config" 6 | "github.com/pingc0y/URLFinder/mode" 7 | "github.com/pingc0y/URLFinder/result" 8 | "github.com/pingc0y/URLFinder/util" 9 | "io" 10 | "net/http" 11 | "net/url" 12 | "regexp" 13 | "strconv" 14 | "strings" 15 | ) 16 | 17 | // 检测js访问状态码 18 | func JsState(u string, i int, sou string) { 19 | 20 | defer func() { 21 | config.Wg.Done() 22 | <-config.Jsch 23 | PrintProgress() 24 | }() 25 | if cmd.S == "" { 26 | result.ResultJs[i].Url = u 27 | return 28 | } 29 | if cmd.M == 3 { 30 | for _, v := range config.Risks { 31 | if strings.Contains(u, v) { 32 | result.ResultJs[i] = mode.Link{Url: u, Status: "疑似危险路由"} 33 | return 34 | } 35 | } 36 | } 37 | 38 | //加载yaml配置(proxy) 39 | //配置代理 40 | var redirect string 41 | ur, err2 := url.Parse(u) 42 | if err2 != nil { 43 | return 44 | } 45 | request, err := http.NewRequest("GET", ur.String(), nil) 46 | if err != nil { 47 | result.ResultJs[i].Url = "" 48 | return 49 | } 50 | if cmd.C != "" { 51 | request.Header.Set("Cookie", cmd.C) 52 | } 53 | //增加header选项 54 | request.Header.Set("User-Agent", util.GetUserAgent()) 55 | request.Header.Set("Accept", "*/*") 56 | //加载yaml配置 57 | if cmd.I { 58 | util.SetHeadersConfig(&request.Header) 59 | } 60 | //tr := &http.Transport{ 61 | // TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, 62 | //} 63 | //client = &http.Client{Timeout: time.Duration(cmd.TI) * time.Second, 64 | // Transport: tr, 65 | // CheckRedirect: func(req *http.Request, via []*http.Request) error { 66 | // if len(via) >= 10 { 67 | // return fmt.Errorf("Too many redirects") 68 | // } 69 | // if len(via) > 0 { 70 | // if via[0] != nil && via[0].URL != nil { 71 | // result.Redirect[via[0].URL.String()] = true 72 | // } else { 73 | // result.Redirect[req.URL.String()] = true 74 | // } 75 | // 76 | // } 77 | // return nil 78 | // }, 79 | //} 80 | //处理返回结果 81 | response, err := client.Do(request) 82 | if err != nil { 83 | if strings.Contains(err.Error(), "Client.Timeout") && cmd.S == "" { 84 | result.ResultJs[i] = mode.Link{Url: u, Status: "timeout", Size: "0"} 85 | 86 | } else { 87 | result.ResultJs[i].Url = "" 88 | } 89 | return 90 | } 91 | defer response.Body.Close() 92 | 93 | code := response.StatusCode 94 | if strings.Contains(cmd.S, strconv.Itoa(code)) || cmd.S == "all" && (sou != "Fuzz" && code == 200) { 95 | var length int 96 | dataBytes, err := io.ReadAll(response.Body) 97 | if err != nil { 98 | length = 0 99 | } else { 100 | length = len(dataBytes) 101 | } 102 | config.Lock.Lock() 103 | if result.Redirect[ur.String()] { 104 | code = 302 105 | redirect = response.Request.URL.String() 106 | } 107 | config.Lock.Unlock() 108 | result.ResultJs[i] = mode.Link{Url: u, Status: strconv.Itoa(code), Size: strconv.Itoa(length), Redirect: redirect} 109 | } else { 110 | result.ResultJs[i].Url = "" 111 | } 112 | } 113 | 114 | // 检测url访问状态码 115 | func UrlState(u string, i int) { 116 | defer func() { 117 | config.Wg.Done() 118 | <-config.Urlch 119 | PrintProgress() 120 | }() 121 | if cmd.S == "" { 122 | result.ResultUrl[i].Url = u 123 | return 124 | } 125 | if cmd.M == 3 { 126 | for _, v := range config.Risks { 127 | if strings.Contains(u, v) { 128 | result.ResultUrl[i] = mode.Link{Url: u, Status: "0", Size: "0", Title: "疑似危险路由,已跳过验证"} 129 | return 130 | } 131 | } 132 | } 133 | 134 | var redirect string 135 | ur, err2 := url.Parse(u) 136 | if err2 != nil { 137 | return 138 | } 139 | request, err := http.NewRequest("GET", ur.String(), nil) 140 | if err != nil { 141 | result.ResultUrl[i].Url = "" 142 | return 143 | } 144 | 145 | if cmd.C != "" { 146 | request.Header.Set("Cookie", cmd.C) 147 | } 148 | //增加header选项 149 | request.Header.Set("User-Agent", util.GetUserAgent()) 150 | request.Header.Set("Accept", "*/*") 151 | 152 | //加载yaml配置 153 | if cmd.I { 154 | util.SetHeadersConfig(&request.Header) 155 | } 156 | //tr := &http.Transport{ 157 | // TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, 158 | //} 159 | //client = &http.Client{Timeout: time.Duration(cmd.TI) * time.Second, 160 | // Transport: tr, 161 | // CheckRedirect: func(req *http.Request, via []*http.Request) error { 162 | // if len(via) >= 10 { 163 | // return fmt.Errorf("Too many redirects") 164 | // } 165 | // if len(via) > 0 { 166 | // if via[0] != nil && via[0].URL != nil { 167 | // result.Redirect[via[0].URL.String()] = true 168 | // } else { 169 | // result.Redirect[req.URL.String()] = true 170 | // } 171 | // 172 | // } 173 | // return nil 174 | // }, 175 | //} 176 | //处理返回结果 177 | response, err := client.Do(request) 178 | if err != nil { 179 | if strings.Contains(err.Error(), "Client.Timeout") && cmd.S == "all" { 180 | result.ResultUrl[i] = mode.Link{Url: u, Status: "timeout", Size: "0"} 181 | } else { 182 | result.ResultUrl[i].Url = "" 183 | } 184 | return 185 | } 186 | defer response.Body.Close() 187 | 188 | code := response.StatusCode 189 | if strings.Contains(cmd.S, strconv.Itoa(code)) || cmd.S == "all" { 190 | var length int 191 | dataBytes, err := io.ReadAll(response.Body) 192 | if err != nil { 193 | length = 0 194 | } else { 195 | length = len(dataBytes) 196 | } 197 | body := string(dataBytes) 198 | re := regexp.MustCompile("<[tT]itle>(.*?)") 199 | title := re.FindAllStringSubmatch(body, -1) 200 | config.Lock.Lock() 201 | if result.Redirect[ur.String()] { 202 | code = 302 203 | redirect = response.Request.URL.String() 204 | } 205 | config.Lock.Unlock() 206 | 207 | if len(title) != 0 { 208 | result.ResultUrl[i] = mode.Link{Url: u, Status: strconv.Itoa(code), Size: strconv.Itoa(length), Title: title[0][1], Redirect: redirect} 209 | } else { 210 | result.ResultUrl[i] = mode.Link{Url: u, Status: strconv.Itoa(code), Size: strconv.Itoa(length), Redirect: redirect} 211 | } 212 | } else { 213 | result.ResultUrl[i].Url = "" 214 | } 215 | } 216 | -------------------------------------------------------------------------------- /crawler/urlFuzz.go: -------------------------------------------------------------------------------- 1 | package crawler 2 | 3 | import ( 4 | "fmt" 5 | "github.com/pingc0y/URLFinder/cmd" 6 | "github.com/pingc0y/URLFinder/config" 7 | "github.com/pingc0y/URLFinder/mode" 8 | "github.com/pingc0y/URLFinder/result" 9 | "github.com/pingc0y/URLFinder/util" 10 | "io" 11 | "net/http" 12 | "regexp" 13 | "strconv" 14 | "strings" 15 | ) 16 | 17 | // Fuzz 18 | func UrlFuzz() { 19 | var host string 20 | re := regexp.MustCompile("([a-z0-9\\-]+\\.)*([a-z0-9\\-]+\\.[a-z0-9\\-]+)(:[0-9]+)?") 21 | hosts := re.FindAllString(cmd.U, 1) 22 | if len(hosts) == 0 { 23 | host = cmd.U 24 | } else { 25 | host = hosts[0] 26 | } 27 | if cmd.D != "" { 28 | host = cmd.D 29 | } 30 | disposes, _ := util.UrlDispose(append(result.ResultUrl, mode.Link{Url: cmd.U, Status: "200", Size: "0"}), host, "") 31 | if cmd.Z == 2 || cmd.Z == 3 { 32 | fuzz2(disposes) 33 | } else if cmd.Z != 0 { 34 | fuzz1(disposes) 35 | } 36 | fmt.Println("\rFuzz OK ") 37 | } 38 | 39 | // fuzz请求 40 | func fuzzGet(u string) { 41 | defer func() { 42 | config.Wg.Done() 43 | <-config.Ch 44 | util.PrintFuzz() 45 | }() 46 | if cmd.M == 3 { 47 | for _, v := range config.Risks { 48 | if strings.Contains(u, v) { 49 | return 50 | } 51 | } 52 | } 53 | request, err := http.NewRequest("GET", u, nil) 54 | if err != nil { 55 | return 56 | } 57 | if cmd.C != "" { 58 | request.Header.Set("Cookie", cmd.C) 59 | } 60 | //增加header选项 61 | request.Header.Set("User-Agent", util.GetUserAgent()) 62 | request.Header.Set("Accept", "*/*") 63 | //加载yaml配置 64 | if cmd.I { 65 | util.SetHeadersConfig(&request.Header) 66 | } 67 | //处理返回结果 68 | response, err := client.Do(request) 69 | if err != nil { 70 | return 71 | } else { 72 | defer response.Body.Close() 73 | } 74 | code := response.StatusCode 75 | if strings.Contains(cmd.S, strconv.Itoa(code)) || cmd.S == "all" { 76 | var length int 77 | dataBytes, err := io.ReadAll(response.Body) 78 | if err != nil { 79 | length = 0 80 | } else { 81 | length = len(dataBytes) 82 | } 83 | body := string(dataBytes) 84 | re := regexp.MustCompile("(.*?)") 85 | title := re.FindAllStringSubmatch(body, -1) 86 | config.Lock.Lock() 87 | defer config.Lock.Unlock() 88 | if len(title) != 0 { 89 | result.Fuzzs = append(result.Fuzzs, mode.Link{Url: u, Status: strconv.Itoa(code), Size: strconv.Itoa(length), Title: title[0][1], Source: "Fuzz"}) 90 | } else { 91 | result.Fuzzs = append(result.Fuzzs, mode.Link{Url: u, Status: strconv.Itoa(code), Size: strconv.Itoa(length), Title: "", Source: "Fuzz"}) 92 | } 93 | 94 | } 95 | 96 | } 97 | func fuzz1(disposes []mode.Link) { 98 | dispose404 := []string{} 99 | for _, v := range disposes { 100 | if v.Status == "404" { 101 | dispose404 = append(dispose404, v.Url) 102 | } 103 | } 104 | fuzz1s := []string{} 105 | host := "" 106 | if len(dispose404) != 0 { 107 | host = regexp.MustCompile("(http.{0,1}://.+?)/").FindAllStringSubmatch(dispose404[0]+"/", -1)[0][1] 108 | } 109 | 110 | for _, v := range dispose404 { 111 | submatch := regexp.MustCompile("http.{0,1}://.+?(/.*)").FindAllStringSubmatch(v, -1) 112 | if len(submatch) != 0 { 113 | v = submatch[0][1] 114 | } else { 115 | continue 116 | } 117 | v1 := v 118 | v2 := v 119 | reh2 := "" 120 | if !strings.HasSuffix(v, "/") { 121 | _submatch := regexp.MustCompile("/.+(/[^/]+)").FindAllStringSubmatch(v, -1) 122 | if len(_submatch) != 0 { 123 | reh2 = _submatch[0][1] 124 | } else { 125 | continue 126 | } 127 | } 128 | for { 129 | re1 := regexp.MustCompile("/.+?(/.+)").FindAllStringSubmatch(v1, -1) 130 | re2 := regexp.MustCompile("(/.+)/[^/]+").FindAllStringSubmatch(v2, -1) 131 | if len(re1) == 0 && len(re2) == 0 { 132 | break 133 | } 134 | if len(re1) > 0 { 135 | v1 = re1[0][1] 136 | fuzz1s = append(fuzz1s, host+v1) 137 | } 138 | if len(re2) > 0 { 139 | v2 = re2[0][1] 140 | fuzz1s = append(fuzz1s, host+v2+reh2) 141 | } 142 | } 143 | } 144 | fuzz1s = util.UniqueArr(fuzz1s) 145 | config.FuzzNum = len(fuzz1s) 146 | config.Progress = 1 147 | fmt.Printf("Start %d Fuzz...\n", config.FuzzNum) 148 | fmt.Printf("\r ") 149 | for _, v := range fuzz1s { 150 | config.Wg.Add(1) 151 | config.Ch <- 1 152 | go fuzzGet(v) 153 | } 154 | config.Wg.Wait() 155 | result.Fuzzs = util.Del404(result.Fuzzs) 156 | } 157 | 158 | func fuzz2(disposes []mode.Link) { 159 | disposex := []string{} 160 | dispose404 := []string{} 161 | for _, v := range disposes { 162 | if v.Status == "404" { 163 | dispose404 = append(dispose404, v.Url) 164 | } 165 | //防止太多跑不完 166 | if len(dispose404) > 20 { 167 | if v.Status != "timeout" && v.Status != "404" { 168 | disposex = append(disposex, v.Url) 169 | } 170 | } else { 171 | if v.Status != "timeout" { 172 | disposex = append(disposex, v.Url) 173 | } 174 | } 175 | 176 | } 177 | dispose, _ := util.PathExtract(disposex) 178 | _, targets := util.PathExtract(dispose404) 179 | 180 | config.FuzzNum = len(dispose) * len(targets) 181 | config.Progress = 1 182 | fmt.Printf("Start %d Fuzz...\n", len(dispose)*len(targets)) 183 | fmt.Printf("\r ") 184 | for _, v := range dispose { 185 | for _, vv := range targets { 186 | config.Wg.Add(1) 187 | config.Ch <- 1 188 | go fuzzGet(v + vv) 189 | } 190 | } 191 | config.Wg.Wait() 192 | result.Fuzzs = util.Del404(result.Fuzzs) 193 | } 194 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/pingc0y/URLFinder 2 | 3 | go 1.19 4 | 5 | require ( 6 | github.com/gookit/color v1.5.2 7 | gopkg.in/yaml.v3 v3.0.1 8 | ) 9 | 10 | require ( 11 | github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect 12 | golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44 // indirect 13 | ) 14 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 2 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 3 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/gookit/color v1.5.2 h1:uLnfXcaFjlrDnQDT+NCBcfhrXqYTx/rcCa6xn01Y8yI= 5 | github.com/gookit/color v1.5.2/go.mod h1:w8h4bGiHeeBpvQVePTutdbERIUf3oJE5lZ8HM0UgXyg= 6 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 7 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 8 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 9 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 10 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 11 | github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= 12 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 13 | github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 h1:QldyIu/L63oPpyvQmHgvgickp1Yw510KJOqX7H24mg8= 14 | github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs= 15 | golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44 h1:Bli41pIlzTzf3KEY06n+xnzK/BESIg2ze4Pgfh/aI8c= 16 | golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 17 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 18 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 19 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 20 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 21 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 22 | /.github/ 23 | -------------------------------------------------------------------------------- /img/0.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pingc0y/URLFinder/e1c0334f72e00f5eed98af985da3d0879210e260/img/0.jpg -------------------------------------------------------------------------------- /img/1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pingc0y/URLFinder/e1c0334f72e00f5eed98af985da3d0879210e260/img/1.jpg -------------------------------------------------------------------------------- /img/2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pingc0y/URLFinder/e1c0334f72e00f5eed98af985da3d0879210e260/img/2.jpg -------------------------------------------------------------------------------- /img/3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pingc0y/URLFinder/e1c0334f72e00f5eed98af985da3d0879210e260/img/3.jpg -------------------------------------------------------------------------------- /img/4.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pingc0y/URLFinder/e1c0334f72e00f5eed98af985da3d0879210e260/img/4.jpg -------------------------------------------------------------------------------- /img/5.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pingc0y/URLFinder/e1c0334f72e00f5eed98af985da3d0879210e260/img/5.jpg -------------------------------------------------------------------------------- /img/process.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pingc0y/URLFinder/e1c0334f72e00f5eed98af985da3d0879210e260/img/process.png -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/pingc0y/URLFinder/cmd" 5 | "github.com/pingc0y/URLFinder/crawler" 6 | "github.com/pingc0y/URLFinder/util" 7 | "io" 8 | "log" 9 | ) 10 | 11 | func main() { 12 | log.SetOutput(io.Discard) 13 | util.GetUpdate() 14 | cmd.Parse() 15 | crawler.Run() 16 | } 17 | -------------------------------------------------------------------------------- /mode/mode.go: -------------------------------------------------------------------------------- 1 | package mode 2 | 3 | type Config struct { 4 | Proxy string `yaml:"proxy"` 5 | Timeout int `yaml:"timeout"` 6 | Thread int `yaml:"thread"` 7 | UrlSteps int `yaml:"urlSteps"` 8 | JsSteps int `yaml:"jsSteps"` 9 | Max int `yaml:"max"` 10 | Headers map[string]string `yaml:"headers"` 11 | JsFind []string `yaml:"jsFind"` 12 | UrlFind []string `yaml:"urlFind"` 13 | InfoFind map[string][]string `yaml:"infoFiler"` 14 | Risks []string `yaml:"risks"` 15 | JsFiler []string `yaml:"jsFiler"` 16 | UrlFiler []string `yaml:"urlFiler"` 17 | JsFuzzPath []string `yaml:"jsFuzzPath"` 18 | } 19 | 20 | type Link struct { 21 | Url string 22 | Status string 23 | Size string 24 | Title string 25 | Redirect string 26 | Source string 27 | } 28 | 29 | type Info struct { 30 | Phone []string 31 | Email []string 32 | IDcard []string 33 | JWT []string 34 | Other []string 35 | Source string 36 | } 37 | -------------------------------------------------------------------------------- /result/report.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | URLFinder 8 | 9 | 402 | 403 | 404 |
405 |
406 |
407 |
413 |

415 | URLFinder Report 416 | 417 |

418 |
419 |
422 |
423 | 424 |
425 |
426 |
427 |
428 |

JS to {urlHost}

429 |
430 |
431 |
432 |
433 |
434 |
435 |
436 |
437 |
438 | 439 |
440 | 441 | 442 | 443 | 452 | 461 | 470 | 479 | 488 | 497 | 498 | 499 | 500 | {JS} 501 | 502 | 503 |
446 | 447 |
448 | URL 449 |
450 | 451 | 		455 | 456 |
457 | Status 458 |
459 | 460 | 		464 | 465 |
466 | Size 467 |
468 | 469 | 		473 | 474 |
475 | Title 476 |
477 | 478 | 		482 | 483 |
484 | Redirect 485 |
486 | 487 | 		491 | 492 |
493 | Source 494 |
495 | 496 |
504 |
505 |
506 |
507 |
508 |
509 |
510 |
511 |
512 |
513 |
514 | 515 |
516 |
517 |
518 |
519 |

JS to Other

520 |
521 |
522 |
523 |
524 |
525 |
526 |
527 |
528 |
529 | 530 |
531 | 532 | 533 | 534 | 543 | 552 | 561 | 570 | 579 | 588 | 589 | 590 | 591 | {JSOther} 592 | 593 | 594 |
537 | 538 |
539 | URL 540 |
541 | 542 | 		546 | 547 |
548 | Status 549 |
550 | 551 | 		555 | 556 |
557 | Size 558 |
559 | 560 | 		564 | 565 |
566 | Title 567 |
568 | 569 | 		573 | 574 |
575 | Redirect 576 |
577 | 578 | 		582 | 583 |
584 | Source 585 |
586 | 587 |
595 |
596 |
597 |
598 |
599 |
600 |
601 |
602 |
603 |
604 |
605 | 606 |
607 |
608 |
609 |
610 |

URL to {urlHost}

611 |
612 |
613 |
614 |
615 |
616 |
617 |
618 |
619 |
620 | 621 |
622 | 623 | 624 | 625 | 634 | 643 | 652 | 661 | 670 | 679 | 680 | 681 | 682 | {URL} 683 | 684 | 685 |
628 | 629 |
630 | URL 631 |
632 | 633 | 		637 | 638 |
639 | Status 640 |
641 | 642 | 		646 | 647 |
648 | Size 649 |
650 | 651 | 		655 | 656 |
657 | Title 658 |
659 | 660 | 		664 | 665 |
666 | Redirect 667 |
668 | 669 | 		673 | 674 |
675 | Source 676 |
677 | 678 |
686 |
687 |
688 |
689 |
690 |
691 |
692 |
693 |
694 |
695 |
696 | 697 |
698 |
699 |
700 |
701 |

URL to Other

702 |
703 |
704 |
705 |
706 |
707 |
708 |
709 |
710 |
711 | 712 |
713 | 714 | 715 | 716 | 725 | 734 | 743 | 752 | 761 | 770 | 771 | 772 | 773 | {URLOther} 774 | 775 | 776 |
719 | 720 |
721 | URL 722 |
723 | 724 | 		728 | 729 |
730 | Status 731 |
732 | 733 | 		737 | 738 |
739 | Size 740 |
741 | 742 | 		746 | 747 |
748 | Title 749 |
750 | 751 | 		755 | 756 |
757 | Redirect 758 |
759 | 760 | 		764 | 765 |
766 | Source 767 |
768 | 769 |
777 |
778 |
779 |
780 |
781 |
782 |
783 |
784 |
785 |
786 |
787 | 788 |
789 |
790 |
791 |
792 |

Fuzz

793 |
794 |
795 |
796 |
797 |
798 |
799 |
800 |
801 |
802 | 803 |
804 | 805 | 806 | 807 | 816 | 825 | 834 | 843 | 852 | 853 | 862 | 863 | 864 | 865 | {Fuzz} 866 | 867 | 868 |
810 | 811 |
812 | URL 813 |
814 | 815 | 		819 | 820 |
821 | Status 822 |
823 | 824 | 		828 | 829 |
830 | Size 831 |
832 | 833 | 		837 | 838 |
839 | Title 840 |
841 | 842 | 		846 | 847 |
848 | Redirect 849 |
850 | 851 | 		856 | 857 |
858 | Source 859 |
860 | 861 |
869 |
870 |
871 |
872 |
873 |
874 |
875 |
876 |
877 |
878 |
879 | 880 |
881 |
882 |
883 |
884 |

Domains

885 |
886 |
887 |
888 |
889 |
890 |
891 |
892 |
893 |
894 | 895 |
896 | 897 | 898 | 899 | 908 | 909 | 910 | 911 | {Domains} 912 | 913 |
902 | 903 |
904 | Domain 905 |
906 | 907 |
914 |
915 |
916 |
917 |
918 |
919 |
920 |
921 |
922 |
923 |
924 | 925 |
926 |
927 |
928 |
929 |

Info

930 |
931 |
932 |
933 |
934 |
935 |
936 |
937 |
938 |
939 | 940 |
941 | 942 | 943 | 944 | 953 | 962 | 971 | 972 | 973 | 974 | {Info} 975 | 976 |
947 | 948 |
949 | Type 950 |
951 | 952 | 		956 | 957 |
958 | Value 959 |
960 | 961 | 		965 | 966 |
967 | Source 968 |
969 | 970 |
977 |
978 |
979 |
980 |
981 |
982 |
983 |
984 |
985 |
986 | 987 | 988 |
989 | 990 |
991 | 992 |
993 |
994 | 995 | 996 | 997 | -------------------------------------------------------------------------------- /result/result.go: -------------------------------------------------------------------------------- 1 | package result 2 | 3 | import ( 4 | "bufio" 5 | _ "embed" 6 | "encoding/csv" 7 | "encoding/json" 8 | "fmt" 9 | "github.com/gookit/color" 10 | "github.com/pingc0y/URLFinder/cmd" 11 | "github.com/pingc0y/URLFinder/mode" 12 | "github.com/pingc0y/URLFinder/util" 13 | "net/url" 14 | "os" 15 | "regexp" 16 | "strconv" 17 | "strings" 18 | ) 19 | 20 | //go:embed report.html 21 | var html string 22 | 23 | var ( 24 | ResultJs []mode.Link 25 | ResultUrl []mode.Link 26 | Fuzzs []mode.Link 27 | Infos []mode.Info 28 | 29 | EndUrl []string 30 | Jsinurl map[string]string 31 | Jstourl map[string]string 32 | Urltourl map[string]string 33 | Domains []string 34 | Redirect map[string]bool 35 | ) 36 | 37 | func outHtmlString(link mode.Link) string { 38 | ht := ` 39 | 40 | 41 | ` + link.Url + ` 42 | 43 | 44 | ` + link.Status + ` 45 | 46 | 47 | ` + link.Size + ` 48 | 49 | 50 | ` + link.Title + ` 51 | 52 | 53 | 54 | ` + link.Redirect + ` 55 | 56 | 57 | 58 | ` + link.Source + ` 59 | 60 | ` 61 | return ht 62 | } 63 | 64 | func outHtmlInfoString(ty, val, sou string) string { 65 | ht := ` 66 | 67 | ` + ty + ` 68 | 69 | 70 | ` + val + ` 71 | 72 | 73 | 74 | ` + sou + ` 75 | 76 | ` 77 | return ht 78 | } 79 | 80 | func outHtmlDomainString(domain string) string { 81 | ht := ` 82 | 83 | ` + domain + ` 84 | 85 | ` 86 | return ht 87 | } 88 | 89 | // 导出csv 90 | func OutFileCsv(out string) { 91 | //获取域名 92 | var host string 93 | re := regexp.MustCompile("([a-z0-9\\-]+\\.)*([a-z0-9\\-]+\\.[a-z0-9\\-]+)(:[0-9]+)?") 94 | hosts := re.FindAllString(cmd.U, 1) 95 | if len(hosts) == 0 { 96 | host = cmd.U 97 | } else { 98 | host = hosts[0] 99 | } 100 | 101 | //抓取的域名优先排序 102 | if cmd.S != "" { 103 | ResultUrl = util.SelectSort(ResultUrl) 104 | ResultJs = util.SelectSort(ResultJs) 105 | } 106 | ResultJsHost, ResultJsOther := util.UrlDispose(ResultJs, host, util.GetHost(cmd.U)) 107 | ResultUrlHost, ResultUrlOther := util.UrlDispose(ResultUrl, host, util.GetHost(cmd.U)) 108 | Domains = util.GetDomains(util.MergeArray(ResultJs, ResultUrl)) 109 | var fileName string 110 | if out != "" { 111 | fileName = out 112 | } else { 113 | //输出到文件 114 | if strings.Contains(host, ":") { 115 | host = strings.Replace(host, ":", "：", -1) 116 | } 117 | //在当前文件夹创建文件夹 118 | err := os.MkdirAll(cmd.O+"/"+host, 0755) 119 | if err != nil { 120 | fmt.Printf(cmd.O+"/"+host+" 目录创建失败 ：%s", err) 121 | return 122 | } 123 | //多相同url处理 124 | fileName = cmd.O + "/" + host + "/" + host + ".csv" 125 | for fileNum := 1; util.Exists(fileName); fileNum++ { 126 | fileName = cmd.O + "/" + host + "/" + host + "(" + strconv.Itoa(fileNum) + ").csv" 127 | } 128 | } 129 | file, err := os.OpenFile(fileName, os.O_CREATE|os.O_WRONLY, 0644) 130 | 131 | resultWriter := csv.NewWriter(file) 132 | // 写数据到文件 133 | if err != nil { 134 | fmt.Println("open file error:", err) 135 | return 136 | } 137 | if cmd.S == "" { 138 | resultWriter.Write([]string{"url", "Source"}) 139 | } else { 140 | resultWriter.Write([]string{"url", "Status", "Size", "Title", "Redirect", "Source"}) 141 | } 142 | if cmd.D == "" { 143 | resultWriter.Write([]string{strconv.Itoa(len(ResultJsHost)) + " JS to " + util.GetHost(cmd.U)}) 144 | } else { 145 | resultWriter.Write([]string{strconv.Itoa(len(ResultJsHost)+len(ResultJsOther)) + " JS to " + cmd.D}) 146 | } 147 | 148 | for _, j := range ResultJsHost { 149 | if cmd.S != "" { 150 | resultWriter.Write([]string{j.Url, j.Status, j.Size, "", j.Redirect, j.Source}) 151 | } else { 152 | resultWriter.Write([]string{j.Url, j.Source}) 153 | } 154 | } 155 | 156 | if cmd.D == "" { 157 | resultWriter.Write([]string{""}) 158 | resultWriter.Write([]string{strconv.Itoa(len(ResultJsOther)) + " JS to Other"}) 159 | } 160 | for _, j := range ResultJsOther { 161 | if cmd.S != "" { 162 | resultWriter.Write([]string{j.Url, j.Status, j.Size, "", j.Redirect, j.Source}) 163 | } else { 164 | resultWriter.Write([]string{j.Url, j.Source}) 165 | } 166 | } 167 | 168 | resultWriter.Write([]string{""}) 169 | if cmd.D == "" { 170 | resultWriter.Write([]string{strconv.Itoa(len(ResultUrlHost)) + " URL to " + util.GetHost(cmd.U)}) 171 | } else { 172 | resultWriter.Write([]string{strconv.Itoa(len(ResultUrlHost)+len(ResultUrlOther)) + " URL to " + cmd.D}) 173 | } 174 | 175 | for _, u := range ResultUrlHost { 176 | if cmd.S != "" { 177 | resultWriter.Write([]string{u.Url, u.Status, u.Size, u.Title, u.Redirect, u.Source}) 178 | } else { 179 | resultWriter.Write([]string{u.Url, u.Source}) 180 | } 181 | } 182 | if cmd.D == "" { 183 | resultWriter.Write([]string{""}) 184 | resultWriter.Write([]string{strconv.Itoa(len(ResultUrlOther)) + " URL to Other"}) 185 | } 186 | for _, u := range ResultUrlOther { 187 | if cmd.S != "" { 188 | resultWriter.Write([]string{u.Url, u.Status, u.Size, u.Title, u.Redirect, u.Source}) 189 | } else { 190 | resultWriter.Write([]string{u.Url, u.Source}) 191 | } 192 | } 193 | if cmd.S != "" && cmd.Z != 0 { 194 | resultWriter.Write([]string{""}) 195 | resultWriter.Write([]string{strconv.Itoa(len(Fuzzs)) + " URL to Fuzz"}) 196 | Fuzzs = util.SelectSort(Fuzzs) 197 | for _, u := range Fuzzs { 198 | resultWriter.Write([]string{u.Url, u.Status, u.Size, u.Title, u.Redirect, "Fuzz"}) 199 | } 200 | } 201 | resultWriter.Write([]string{""}) 202 | resultWriter.Write([]string{strconv.Itoa(len(Domains)) + " Domain"}) 203 | for _, u := range Domains { 204 | resultWriter.Write([]string{u}) 205 | } 206 | resultWriter.Write([]string{""}) 207 | resultWriter.Write([]string{"Phone"}) 208 | for i := range Infos { 209 | for i2 := range Infos[i].Phone { 210 | resultWriter.Write([]string{Infos[i].Phone[i2], "", "", "", Infos[i].Source}) 211 | } 212 | } 213 | resultWriter.Write([]string{""}) 214 | resultWriter.Write([]string{"Email"}) 215 | for i := range Infos { 216 | for i2 := range Infos[i].Email { 217 | resultWriter.Write([]string{Infos[i].Email[i2], "", "", "", Infos[i].Source}) 218 | } 219 | } 220 | resultWriter.Write([]string{""}) 221 | resultWriter.Write([]string{"Email"}) 222 | for i := range Infos { 223 | for i2 := range Infos[i].IDcard { 224 | resultWriter.Write([]string{Infos[i].IDcard[i2], "", "", "", Infos[i].Source}) 225 | } 226 | } 227 | resultWriter.Write([]string{""}) 228 | resultWriter.Write([]string{"JWT"}) 229 | for i := range Infos { 230 | for i2 := range Infos[i].JWT { 231 | resultWriter.Write([]string{Infos[i].JWT[i2], "", "", "", Infos[i].Source}) 232 | } 233 | } 234 | resultWriter.Write([]string{""}) 235 | resultWriter.Write([]string{"Other"}) 236 | for i := range Infos { 237 | for i2 := range Infos[i].Other { 238 | resultWriter.Write([]string{Infos[i].Other[i2], "", "", "", Infos[i].Source}) 239 | } 240 | } 241 | 242 | resultWriter.Flush() 243 | 244 | fmt.Println(strconv.Itoa(len(ResultJsHost)+len(ResultJsOther))+"JS + "+strconv.Itoa(len(ResultUrlHost)+len(ResultUrlOther))+"URL --> ", file.Name()) 245 | 246 | return 247 | } 248 | 249 | // 导出json 250 | func OutFileJson(out string) { 251 | jsons := make(map[string]interface{}) 252 | var info map[string][]map[string]string 253 | //获取域名 254 | var host string 255 | re := regexp.MustCompile("([a-z0-9\\-]+\\.)*([a-z0-9\\-]+\\.[a-z0-9\\-]+)(:[0-9]+)?") 256 | hosts := re.FindAllString(cmd.U, 1) 257 | if len(hosts) == 0 { 258 | host = cmd.U 259 | } else { 260 | host = hosts[0] 261 | } 262 | //抓取的域名优先排序 263 | if cmd.S != "" { 264 | ResultUrl = util.SelectSort(ResultUrl) 265 | ResultJs = util.SelectSort(ResultJs) 266 | } 267 | ResultJsHost, ResultJsOther := util.UrlDispose(ResultJs, host, util.GetHost(cmd.U)) 268 | ResultUrlHost, ResultUrlOther := util.UrlDispose(ResultUrl, host, util.GetHost(cmd.U)) 269 | Domains = util.GetDomains(util.MergeArray(ResultJs, ResultUrl)) 270 | 271 | if len(Infos) > 0 { 272 | info = make(map[string][]map[string]string) 273 | info["IDcard"] = nil 274 | info["JWT"] = nil 275 | info["Email"] = nil 276 | info["Phone"] = nil 277 | info["Other"] = nil 278 | } 279 | 280 | for i := range Infos { 281 | for i2 := range Infos[i].IDcard { 282 | info["IDcard"] = append(info["IDcard"], map[string]string{"IDcard": Infos[i].IDcard[i2], "Source": Infos[i].Source}) 283 | } 284 | for i2 := range Infos[i].JWT { 285 | info["JWT"] = append(info["JWT"], map[string]string{"JWT": Infos[i].JWT[i2], "Source": Infos[i].Source}) 286 | } 287 | for i2 := range Infos[i].Email { 288 | info["Email"] = append(info["Email"], map[string]string{"Email": Infos[i].Email[i2], "Source": Infos[i].Source}) 289 | } 290 | for i2 := range Infos[i].Phone { 291 | info["Phone"] = append(info["Phone"], map[string]string{"Phone": Infos[i].Phone[i2], "Source": Infos[i].Source}) 292 | } 293 | for i2 := range Infos[i].Other { 294 | info["Other"] = append(info["Other"], map[string]string{"Other": Infos[i].Other[i2], "Source": Infos[i].Source}) 295 | } 296 | } 297 | 298 | var fileName string 299 | if out != "" { 300 | fileName = out 301 | } else { 302 | //输出到文件 303 | if strings.Contains(host, ":") { 304 | host = strings.Replace(host, ":", "：", -1) 305 | } 306 | //在当前文件夹创建文件夹 307 | err := os.MkdirAll(cmd.O+"/"+host, 0755) 308 | if err != nil { 309 | fmt.Printf(cmd.O+"/"+host+" 目录创建失败 ：%s", err) 310 | return 311 | } 312 | //多相同url处理 313 | fileName = cmd.O + "/" + host + "/" + host + ".json" 314 | for fileNum := 1; util.Exists(fileName); fileNum++ { 315 | fileName = cmd.O + "/" + host + "/" + host + "(" + strconv.Itoa(fileNum) + ").json" 316 | } 317 | } 318 | file, err := os.OpenFile(fileName, os.O_CREATE|os.O_WRONLY, 0644) 319 | if err != nil { 320 | fmt.Printf("创建失败：%s", err) 321 | return 322 | } 323 | if cmd.D == "" { 324 | jsons["jsOther"] = ResultJsOther 325 | jsons["urlOther"] = ResultUrlOther 326 | } 327 | jsons["js"] = ResultJsHost 328 | jsons["url"] = ResultUrlHost 329 | jsons["info"] = info 330 | jsons["fuzz"] = Fuzzs 331 | jsons["domain"] = Domains 332 | if cmd.S != "" && cmd.Z != 0 { 333 | Fuzzs = util.SelectSort(Fuzzs) 334 | if len(Fuzzs) > 0 { 335 | jsons["fuzz"] = Fuzzs 336 | } else { 337 | jsons["fuzz"] = nil 338 | } 339 | 340 | } 341 | 342 | defer file.Close() 343 | 344 | data, err := json.Marshal(jsons) 345 | if err != nil { 346 | fmt.Printf("json化失败：%s", err) 347 | return 348 | } 349 | buf := bufio.NewWriter(file) 350 | // 字节写入 351 | buf.Write(data) 352 | // 将缓冲中的数据写入 353 | err = buf.Flush() 354 | if err != nil { 355 | fmt.Println("json保存失败:", err) 356 | } 357 | fmt.Println(strconv.Itoa(len(ResultJsHost)+len(ResultJsOther))+"JS + "+strconv.Itoa(len(ResultUrlHost)+len(ResultUrlOther))+"URL --> ", file.Name()) 358 | return 359 | } 360 | 361 | // 导出html 362 | func OutFileHtml(out string) { 363 | htmlTemp := html 364 | //获取域名 365 | var host string 366 | re := regexp.MustCompile("([a-z0-9\\-]+\\.)*([a-z0-9\\-]+\\.[a-z0-9\\-]+)(:[0-9]+)?") 367 | hosts := re.FindAllString(cmd.U, 1) 368 | if len(hosts) == 0 { 369 | host = cmd.U 370 | } else { 371 | host = hosts[0] 372 | } 373 | 374 | //抓取的域名优先排序 375 | if cmd.S != "" { 376 | ResultUrl = util.SelectSort(ResultUrl) 377 | ResultJs = util.SelectSort(ResultJs) 378 | } 379 | ResultJsHost, ResultJsOther := util.UrlDispose(ResultJs, host, util.GetHost(cmd.U)) 380 | ResultUrlHost, ResultUrlOther := util.UrlDispose(ResultUrl, host, util.GetHost(cmd.U)) 381 | Domains = util.GetDomains(util.MergeArray(ResultJs, ResultUrl)) 382 | 383 | var fileName string 384 | if out != "" { 385 | fileName = out 386 | } else { 387 | //输出到文件 388 | if strings.Contains(host, ":") { 389 | host = strings.Replace(host, ":", "：", -1) 390 | } 391 | //在当前文件夹创建文件夹 392 | err := os.MkdirAll(cmd.O+"/"+host, 0755) 393 | if err != nil { 394 | fmt.Printf(cmd.O+"/"+host+" 目录创建失败 ：%s", err) 395 | return 396 | } 397 | //多相同url处理 398 | fileName = cmd.O + "/" + host + "/" + host + ".html" 399 | for fileNum := 1; util.Exists(fileName); fileNum++ { 400 | fileName = cmd.O + "/" + host + "/" + host + "(" + strconv.Itoa(fileNum) + ").html" 401 | } 402 | } 403 | file, err := os.OpenFile(fileName, os.O_CREATE|os.O_WRONLY, 0644) 404 | 405 | file.WriteString("\xEF\xBB\xBF") // 写入UTF-8 BOM,防止中文乱码 406 | // 写数据到文件 407 | if err != nil { 408 | fmt.Println("open file error:", err) 409 | return 410 | } 411 | defer file.Close() 412 | 413 | writer := bufio.NewWriter(file) 414 | 415 | if cmd.D == "" { 416 | htmlTemp = strings.Replace(htmlTemp, "{urlHost}", util.GetHost(cmd.U), -1) 417 | } else { 418 | htmlTemp = strings.Replace(htmlTemp, "{urlHost}", cmd.D, -1) 419 | } 420 | var ResultJsHostStr string 421 | for _, j := range ResultJsHost { 422 | ResultJsHostStr += outHtmlString(j) 423 | } 424 | htmlTemp = strings.Replace(htmlTemp, "{JS}", ResultJsHostStr, -1) 425 | 426 | var ResultJsOtherStr string 427 | for _, j := range ResultJsOther { 428 | ResultJsOtherStr += outHtmlString(j) 429 | } 430 | htmlTemp = strings.Replace(htmlTemp, "{JSOther}", ResultJsOtherStr, -1) 431 | 432 | var ResultUrlHostStr string 433 | for _, u := range ResultUrlHost { 434 | ResultUrlHostStr += outHtmlString(u) 435 | } 436 | htmlTemp = strings.Replace(htmlTemp, "{URL}", ResultUrlHostStr, -1) 437 | 438 | var ResultUrlOtherStr string 439 | for _, u := range ResultUrlOther { 440 | ResultUrlOtherStr += outHtmlString(u) 441 | } 442 | htmlTemp = strings.Replace(htmlTemp, "{URLOther}", ResultUrlOtherStr, -1) 443 | 444 | var FuzzsStr string 445 | if cmd.S != "" && cmd.Z != 0 { 446 | Fuzzs = util.SelectSort(Fuzzs) 447 | for _, u := range Fuzzs { 448 | FuzzsStr += outHtmlString(u) 449 | } 450 | } 451 | htmlTemp = strings.Replace(htmlTemp, "{Fuzz}", FuzzsStr, -1) 452 | 453 | var DomainsStr string 454 | for _, u := range Domains { 455 | DomainsStr += outHtmlDomainString(u) 456 | } 457 | htmlTemp = strings.Replace(htmlTemp, "{Domains}", DomainsStr, -1) 458 | 459 | var Infostr string 460 | for i := range Infos { 461 | for i2 := range Infos[i].Phone { 462 | Infostr += outHtmlInfoString("Phone", Infos[i].Phone[i2], Infos[i].Source) 463 | } 464 | } 465 | for i := range Infos { 466 | for i2 := range Infos[i].Email { 467 | Infostr += outHtmlInfoString("Email", Infos[i].Email[i2], Infos[i].Source) 468 | } 469 | } 470 | for i := range Infos { 471 | for i2 := range Infos[i].IDcard { 472 | Infostr += outHtmlInfoString("IDcard", Infos[i].IDcard[i2], Infos[i].Source) 473 | } 474 | } 475 | for i := range Infos { 476 | for i2 := range Infos[i].JWT { 477 | Infostr += outHtmlInfoString("JWT", Infos[i].JWT[i2], Infos[i].Source) 478 | } 479 | } 480 | for i := range Infos { 481 | for i2 := range Infos[i].Other { 482 | Infostr += outHtmlInfoString("Other", Infos[i].Other[i2], Infos[i].Source) 483 | } 484 | } 485 | htmlTemp = strings.Replace(htmlTemp, "{Info}", Infostr, -1) 486 | writer.WriteString(htmlTemp) 487 | writer.Flush() //内容是先写到缓存对,所以需要调用flush将缓存对数据真正写到文件中 488 | fmt.Println(strconv.Itoa(len(ResultJsHost)+len(ResultJsOther))+"JS + "+strconv.Itoa(len(ResultUrlHost)+len(ResultUrlOther))+"URL --> ", file.Name()) 489 | return 490 | } 491 | 492 | // 打印 493 | func Print() { 494 | //获取域名 495 | var host string 496 | re := regexp.MustCompile("([a-z0-9\\-]+\\.)*([a-z0-9\\-]+\\.[a-z0-9\\-]+)(:[0-9]+)?") 497 | hosts := re.FindAllString(cmd.U, 1) 498 | if len(hosts) == 0 { 499 | host = cmd.U 500 | } else { 501 | host = hosts[0] 502 | } 503 | //打印JS 504 | if cmd.S != "" { 505 | ResultJs = util.SelectSort(ResultJs) 506 | ResultUrl = util.SelectSort(ResultUrl) 507 | 508 | } 509 | //抓取的域名优先排序 510 | ResultJsHost, ResultJsOther := util.UrlDispose(ResultJs, host, util.GetHost(cmd.U)) 511 | ResultUrlHost, ResultUrlOther := util.UrlDispose(ResultUrl, host, util.GetHost(cmd.U)) 512 | Domains = util.GetDomains(util.MergeArray(ResultJs, ResultUrl)) 513 | var ulen string 514 | if len(ResultUrl) != 0 { 515 | uleni := 0 516 | for _, u := range ResultUrl { 517 | uleni += len(u.Url) 518 | } 519 | ulen = strconv.Itoa(uleni/len(ResultUrl) + 10) 520 | } 521 | var jlen string 522 | if len(ResultJs) != 0 { 523 | jleni := 0 524 | for _, j := range ResultJs { 525 | jleni += len(j.Url) 526 | } 527 | jlen = strconv.Itoa(jleni/len(ResultJs) + 10) 528 | } 529 | if cmd.D == "" { 530 | fmt.Println(strconv.Itoa(len(ResultJsHost)) + " JS to " + util.GetHost(cmd.U)) 531 | } else { 532 | fmt.Println(strconv.Itoa(len(ResultJsHost)+len(ResultJsOther)) + " JS to " + cmd.D) 533 | } 534 | for _, j := range ResultJsHost { 535 | if cmd.S != "" { 536 | if strings.HasPrefix(j.Status, "2") { 537 | fmt.Printf(color.LightBlue.Sprintf("%-"+jlen+"s", j.Url) + color.LightGreen.Sprintf(" [ Status: %s, Size: %s, Source: %s ]\n", j.Status, j.Size, j.Source)) 538 | } else if strings.HasPrefix(j.Status, "3") { 539 | fmt.Printf(color.LightBlue.Sprintf("%-"+jlen+"s", j.Url) + color.LightYellow.Sprintf(" [ Status: %s, Size: %s, Source: %s ]\n", j.Status, j.Size, j.Source)) 540 | } else { 541 | fmt.Printf(color.LightBlue.Sprintf("%-"+jlen+"s", j.Url) + color.LightRed.Sprintf(" [ Status: %s, Size: %s, Source: %s ]\n", j.Status, j.Size, j.Source)) 542 | } 543 | } else { 544 | fmt.Println(j.Url) 545 | } 546 | } 547 | if cmd.D == "" { 548 | fmt.Println("\n" + strconv.Itoa(len(ResultJsOther)) + " JS to Other") 549 | } 550 | for _, j := range ResultJsOther { 551 | if cmd.S != "" { 552 | if strings.HasPrefix(j.Status, "2") { 553 | fmt.Printf(color.LightBlue.Sprintf("%-"+jlen+"s", j.Url) + color.LightGreen.Sprintf(" [ Status: %s, Size: %s, Source: %s ]\n", j.Status, j.Size, j.Source)) 554 | } else if strings.HasPrefix(j.Status, "3") { 555 | fmt.Printf(color.LightBlue.Sprintf("%-"+jlen+"s", j.Url) + color.LightYellow.Sprintf(" [ Status: %s, Size: %s, Source: %s ]\n", j.Status, j.Size, j.Source)) 556 | } else { 557 | fmt.Printf(color.LightBlue.Sprintf("%-"+jlen+"s", j.Url) + color.LightRed.Sprintf(" [ Status: %s, Size: %s, Source: %s ]\n", j.Status, j.Size, j.Source)) 558 | } 559 | } else { 560 | fmt.Println(j.Url) 561 | } 562 | } 563 | 564 | fmt.Println("\n ") 565 | 566 | if cmd.D == "" { 567 | fmt.Println(strconv.Itoa(len(ResultUrlHost)) + " URL to " + util.GetHost(cmd.U)) 568 | } else { 569 | fmt.Println(strconv.Itoa(len(ResultUrlHost)+len(ResultUrlOther)) + " URL to " + cmd.D) 570 | } 571 | 572 | for _, u := range ResultUrlHost { 573 | urlx, err := url.QueryUnescape(u.Url) 574 | if err == nil { 575 | u.Url = urlx 576 | } 577 | if cmd.S != "" && len(u.Title) != 0 { 578 | if u.Status == "疑似危险路由" { 579 | fmt.Printf(color.LightBlue.Sprintf("%-"+ulen+"s", u.Url) + color.LightGreen.Sprintf(" [ Status: %s, Source: %s ]\n", u.Status, u.Source)) 580 | } else if strings.HasPrefix(u.Status, "2") { 581 | fmt.Printf(color.LightBlue.Sprintf("%-"+ulen+"s", u.Url) + color.LightGreen.Sprintf(" [ Status: %s, Size: %s, Title: %s, Source: %s ]\n", u.Status, u.Size, u.Title, u.Source)) 582 | } else if strings.HasPrefix(u.Status, "3") { 583 | fmt.Printf(color.LightBlue.Sprintf("%-"+ulen+"s", u.Url) + color.LightYellow.Sprintf(" [ Status: %s, Size: %s, Title: %s, Source: %s ]\n", u.Status, u.Size, u.Title, u.Source)) 584 | } else { 585 | fmt.Printf(color.LightBlue.Sprintf("%-"+ulen+"s", u.Url) + color.LightRed.Sprintf(" [ Status: %s, Size: %s, Title: %s, Source: %s ]\n", u.Status, u.Size, u.Title, u.Source)) 586 | } 587 | } else if cmd.S != "" { 588 | if strings.HasPrefix(u.Status, "2") { 589 | fmt.Printf(color.LightBlue.Sprintf("%-"+ulen+"s", u.Url) + color.LightGreen.Sprintf(" [ Status: %s, Size: %s, Source: %s ]\n", u.Status, u.Size, u.Source)) 590 | } else if strings.HasPrefix(u.Status, "3") { 591 | fmt.Printf(color.LightBlue.Sprintf("%-"+ulen+"s", u.Url) + color.LightYellow.Sprintf(" [ Status: %s, Size: %s, Source: %s ]\n", u.Status, u.Size, u.Source)) 592 | } else { 593 | fmt.Printf(color.LightBlue.Sprintf("%-"+ulen+"s", u.Url) + color.LightRed.Sprintf(" [ Status: %s, Size: %s, Source: %s ]\n", u.Status, u.Size, u.Source)) 594 | } 595 | } else { 596 | fmt.Println(u.Url) 597 | } 598 | } 599 | if cmd.D == "" { 600 | fmt.Println("\n" + strconv.Itoa(len(ResultUrlOther)) + " URL to Other") 601 | } 602 | for _, u := range ResultUrlOther { 603 | urlx, err := url.QueryUnescape(u.Url) 604 | if err == nil { 605 | u.Url = urlx 606 | } 607 | if cmd.S != "" && len(u.Title) != 0 { 608 | if u.Status == "疑似危险路由" { 609 | fmt.Printf(color.LightBlue.Sprintf("%-"+ulen+"s", u.Url) + color.LightGreen.Sprintf(" [ Status: %s, Source: %s ]\n", u.Status, u.Source)) 610 | } else if strings.HasPrefix(u.Status, "2") { 611 | fmt.Printf(color.LightBlue.Sprintf("%-"+ulen+"s", u.Url) + color.LightGreen.Sprintf(" [ Status: %s, Size: %s, Title: %s, Source: %s ]\n", u.Status, u.Size, u.Title, u.Source)) 612 | } else if strings.HasPrefix(u.Status, "3") { 613 | fmt.Printf(color.LightBlue.Sprintf("%-"+ulen+"s", u.Url) + color.LightYellow.Sprintf(" [ Status: %s, Size: %s, Title: %s, Source: %s ]\n", u.Status, u.Size, u.Title, u.Source)) 614 | } else { 615 | fmt.Printf(color.LightBlue.Sprintf("%-"+ulen+"s", u.Url) + color.LightRed.Sprintf(" [ Status: %s, Size: %s, Title: %s, Source: %s ]\n", u.Status, u.Size, u.Title, u.Source)) 616 | } 617 | } else if cmd.S != "" { 618 | if strings.HasPrefix(u.Status, "2") { 619 | fmt.Printf(color.LightBlue.Sprintf("%-"+ulen+"s", u.Url) + color.LightGreen.Sprintf(" [ Status: %s, Size: %s, Source: %s ]\n", u.Status, u.Size, u.Source)) 620 | } else if strings.HasPrefix(u.Status, "3") { 621 | fmt.Printf(color.LightBlue.Sprintf("%-"+ulen+"s", u.Url) + color.LightYellow.Sprintf(" [ Status: %s, Size: %s, Source: %s ]\n", u.Status, u.Size, u.Source)) 622 | } else { 623 | fmt.Printf(color.LightBlue.Sprintf("%-"+ulen+"s", u.Url) + color.LightRed.Sprintf(" [ Status: %s, Size: %s, Source: %s ]\n", u.Status, u.Size, u.Source)) 624 | } 625 | } else { 626 | fmt.Println(u.Url) 627 | } 628 | } 629 | 630 | if cmd.S != "" && cmd.Z != 0 { 631 | fmt.Println("\n" + strconv.Itoa(len(Fuzzs)) + " URL to Fuzz") 632 | Fuzzs = util.SelectSort(Fuzzs) 633 | for _, u := range Fuzzs { 634 | if len(u.Title) != 0 { 635 | if u.Status == "疑似危险路由" { 636 | fmt.Printf(color.LightBlue.Sprintf("%-"+ulen+"s", u.Url) + color.LightGreen.Sprintf(" [ Status: %s, Source: %s ]\n", u.Status, u.Source)) 637 | } else if strings.HasPrefix(u.Status, "2") { 638 | fmt.Printf(color.LightBlue.Sprintf("%-"+ulen+"s", u.Url) + color.LightGreen.Sprintf(" [ Status: %s, Size: %s, Title: %s, Source: %s ]\n", u.Status, u.Size, u.Title, u.Source)) 639 | } else if strings.HasPrefix(u.Status, "3") { 640 | fmt.Printf(color.LightBlue.Sprintf("%-"+ulen+"s", u.Url) + color.LightYellow.Sprintf(" [ Status: %s, Size: %s, Title: %s, Source: %s ]\n", u.Status, u.Size, u.Title, u.Source)) 641 | } else { 642 | fmt.Printf(color.LightBlue.Sprintf("%-"+ulen+"s", u.Url) + color.LightRed.Sprintf(" [ Status: %s, Size: %s, Title: %s, Source: %s ]\n", u.Status, u.Size, u.Title, u.Source)) 643 | } 644 | } else { 645 | if strings.HasPrefix(u.Status, "2") { 646 | fmt.Printf(color.LightBlue.Sprintf("%-"+ulen+"s", u.Url) + color.LightGreen.Sprintf(" [ Status: %s, Size: %s, Source: %s ]\n", u.Status, u.Size, u.Source)) 647 | } else if strings.HasPrefix(u.Status, "3") { 648 | fmt.Printf(color.LightBlue.Sprintf("%-"+ulen+"s", u.Url) + color.LightYellow.Sprintf(" [ Status: %s, Size: %s, Source: %s ]\n", u.Status, u.Size, u.Source)) 649 | } else { 650 | fmt.Printf(color.LightBlue.Sprintf("%-"+ulen+"s", u.Url) + color.LightRed.Sprintf(" [ Status: %s, Size: %s, Source: %s ]\n", u.Status, u.Size, u.Source)) 651 | } 652 | } 653 | } 654 | } 655 | fmt.Println("\n" + strconv.Itoa(len(Domains)) + " Domain") 656 | for _, u := range Domains { 657 | fmt.Printf(color.LightBlue.Sprintf("%s \n", u)) 658 | 659 | } 660 | 661 | if len(Infos) > 0 { 662 | fmt.Println("\n Phone ") 663 | for i := range Infos { 664 | for i2 := range Infos[i].Phone { 665 | fmt.Printf(color.LightBlue.Sprintf("%-10s", Infos[i].Phone[i2]) + color.LightGreen.Sprintf(" [ Source: %s ]\n", Infos[i].Source)) 666 | } 667 | } 668 | fmt.Println("\n Email ") 669 | for i := range Infos { 670 | for i2 := range Infos[i].Email { 671 | fmt.Printf(color.LightBlue.Sprintf("%-10s", Infos[i].Email[i2]) + color.LightGreen.Sprintf(" [ Source: %s ]\n", Infos[i].Source)) 672 | } 673 | } 674 | fmt.Println("\n IDcard ") 675 | for i := range Infos { 676 | for i2 := range Infos[i].IDcard { 677 | fmt.Printf(color.LightBlue.Sprintf("%-10s", Infos[i].IDcard[i2]) + color.LightGreen.Sprintf(" [ Source: %s ]\n", Infos[i].Source)) 678 | } 679 | } 680 | fmt.Println("\n JWT ") 681 | for i := range Infos { 682 | for i2 := range Infos[i].JWT { 683 | fmt.Printf(color.LightBlue.Sprintf("%-10s", Infos[i].JWT[i2]) + color.LightGreen.Sprintf(" [ Source: %s ]\n", Infos[i].Source)) 684 | } 685 | } 686 | 687 | fmt.Println("\n Other ") 688 | for i := range Infos { 689 | for i2 := range Infos[i].Other { 690 | fmt.Printf(color.LightBlue.Sprintf("%-10s", Infos[i].Other[i2]) + color.LightGreen.Sprintf(" [ Source: %s ]\n", Infos[i].Source)) 691 | } 692 | } 693 | 694 | } 695 | 696 | } 697 | -------------------------------------------------------------------------------- /util/utils.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "crypto/tls" 5 | "encoding/json" 6 | "fmt" 7 | "github.com/pingc0y/URLFinder/cmd" 8 | "github.com/pingc0y/URLFinder/config" 9 | "github.com/pingc0y/URLFinder/mode" 10 | "io" 11 | "math/rand" 12 | "net/http" 13 | "net/url" 14 | "os" 15 | "regexp" 16 | "strconv" 17 | "strings" 18 | "time" 19 | ) 20 | 21 | // MergeArray 合并数组 22 | func MergeArray(dest []mode.Link, src []mode.Link) (result []mode.Link) { 23 | result = make([]mode.Link, len(dest)+len(src)) 24 | //将第一个数组传入result 25 | copy(result, dest) 26 | //将第二个数组接在尾部,也就是 len(dest): 27 | copy(result[len(dest):], src) 28 | return 29 | } 30 | 31 | // 对结果进行状态码排序 32 | func SelectSort(arr []mode.Link) []mode.Link { 33 | length := len(arr) 34 | var sort []int 35 | for _, v := range arr { 36 | if v.Url == "" || len(v.Size) == 0 || v.Status == "timeout" { 37 | sort = append(sort, 999) 38 | } else { 39 | in, _ := strconv.Atoi(v.Status) 40 | sort = append(sort, in) 41 | } 42 | } 43 | if length <= 1 { 44 | return arr 45 | } else { 46 | for i := 0; i < length-1; i++ { //只剩一个元素不需要索引 47 | min := i //标记索引 48 | for j := i + 1; j < length; j++ { //每次选出一个极小值 49 | if sort[min] > sort[j] { 50 | min = j //保存极小值的索引 51 | } 52 | } 53 | if i != min { 54 | sort[i], sort[min] = sort[min], sort[i] //数据交换 55 | arr[i], arr[min] = arr[min], arr[i] //数据交换 56 | } 57 | } 58 | return arr 59 | } 60 | } 61 | 62 | // 对结果进行URL排序 63 | func UrlDispose(arr []mode.Link, url, host string) ([]mode.Link, []mode.Link) { 64 | var urls []mode.Link 65 | var urlts []mode.Link 66 | var other []mode.Link 67 | for _, v := range arr { 68 | if strings.Contains(v.Url, url) { 69 | urls = append(urls, v) 70 | } else { 71 | if host != "" && regexp.MustCompile(host).MatchString(v.Url) { 72 | urlts = append(urlts, v) 73 | } else { 74 | other = append(other, v) 75 | } 76 | } 77 | } 78 | 79 | for _, v := range urlts { 80 | urls = append(urls, v) 81 | } 82 | 83 | return RemoveRepeatElement(urls), RemoveRepeatElement(other) 84 | } 85 | 86 | // 处理Headers配置 87 | func SetHeadersConfig(header *http.Header) *http.Header { 88 | for k, v := range config.Conf.Headers { 89 | header.Set(k, v) 90 | } 91 | return header 92 | } 93 | 94 | // 设置proxy配置 95 | func SetProxyConfig(tr *http.Transport) *http.Transport { 96 | if len(config.Conf.Proxy) > 0 { 97 | tr.DisableKeepAlives = true 98 | proxyUrl, parseErr := url.Parse(config.Conf.Proxy) 99 | if parseErr != nil { 100 | fmt.Println("代理地址错误: \n" + parseErr.Error()) 101 | os.Exit(1) 102 | } 103 | tr.Proxy = http.ProxyURL(proxyUrl) 104 | } 105 | return tr 106 | } 107 | 108 | // 判断http协议 109 | func GetProtocol(domain string) string { 110 | if strings.HasPrefix(domain, "http") { 111 | return domain 112 | } 113 | 114 | response, err := http.Get("https://" + domain) 115 | if err == nil { 116 | return "https://" + domain 117 | } 118 | response, err = http.Get("http://" + domain) 119 | if err == nil { 120 | return "http://" + domain 121 | } 122 | defer response.Body.Close() 123 | if response.TLS == nil { 124 | return "http://" + domain 125 | } 126 | return "" 127 | } 128 | 129 | // 提取顶级域名 130 | func GetHost(u string) string { 131 | re := regexp.MustCompile("([a-z0-9\\-]+\\.)*([a-z0-9\\-]+\\.[a-z0-9\\-]+)(:[0-9]+)?") 132 | var host string 133 | hosts := re.FindAllString(u, 1) 134 | if len(hosts) == 0 { 135 | host = u 136 | } else { 137 | host = hosts[0] 138 | } 139 | re2 := regexp.MustCompile("[^.]*?\\.[^.,^:]*") 140 | host2 := re2.FindAllString(host, -1) 141 | re3 := regexp.MustCompile("(([01]?[0-9]{1,3}|2[0-4][0-9]|25[0-5])\\.){3}([01]?[0-9]{1,3}|2[0-4][0-9]|25[0-5])") 142 | hostIp := re3.FindAllString(u, -1) 143 | if len(hostIp) == 0 { 144 | if len(host2) == 1 { 145 | host = host2[0] 146 | } else { 147 | re3 := regexp.MustCompile("\\.[^.]*?\\.[^.,^:]*") 148 | var ho string 149 | hos := re3.FindAllString(host, -1) 150 | 151 | if len(hos) == 0 { 152 | ho = u 153 | } else { 154 | ho = hos[len(hos)-1] 155 | } 156 | host = strings.Replace(ho, ".", "", 1) 157 | } 158 | } else { 159 | return hostIp[0] 160 | } 161 | return host 162 | } 163 | 164 | // 去重+去除错误url 165 | func RemoveRepeatElement(list []mode.Link) []mode.Link { 166 | // 创建一个临时map用来存储数组元素 167 | temp := make(map[string]bool) 168 | var list2 []mode.Link 169 | index := 0 170 | for _, v := range list { 171 | 172 | //处理-d参数 173 | if cmd.D != "" { 174 | v.Url = domainNameFilter(v.Url) 175 | } 176 | if len(v.Url) > 10 { 177 | re := regexp.MustCompile("://([a-z0-9\\-]+\\.)*([a-z0-9\\-]+\\.[a-z0-9\\-]+)(:[0-9]+)?") 178 | hosts := re.FindAllString(v.Url, 1) 179 | if len(hosts) != 0 { 180 | // 遍历数组元素,判断此元素是否已经存在map中 181 | _, ok := temp[v.Url] 182 | if !ok { 183 | v.Url = strings.Replace(v.Url, "/./", "/", -1) 184 | list2 = append(list2, v) 185 | temp[v.Url] = true 186 | } 187 | } 188 | } 189 | index++ 190 | 191 | } 192 | return list2 193 | } 194 | 195 | // 打印Fuzz进度 196 | func PrintFuzz() { 197 | config.Mux.Lock() 198 | fmt.Printf("\rFuzz %.0f%%", float64(config.Progress+1)/float64(config.FuzzNum+1)*100) 199 | config.Progress++ 200 | config.Mux.Unlock() 201 | } 202 | 203 | // 处理-d 204 | func domainNameFilter(url string) string { 205 | re := regexp.MustCompile("://([a-z0-9\\-]+\\.)*([a-z0-9\\-]+\\.[a-z0-9\\-]+)(:[0-9]+)?") 206 | hosts := re.FindAllString(url, 1) 207 | if len(hosts) != 0 { 208 | if !regexp.MustCompile(cmd.D).MatchString(hosts[0]) { 209 | url = "" 210 | } 211 | } 212 | return url 213 | } 214 | 215 | // 文件是否存在 216 | func Exists(path string) bool { 217 | _, err := os.Stat(path) //os.Stat获取文件信息 218 | if err != nil { 219 | if os.IsExist(err) { 220 | return true 221 | } 222 | return false 223 | } 224 | return true 225 | } 226 | 227 | // 数组去重 228 | func UniqueArr(arr []string) []string { 229 | newArr := make([]string, 0) 230 | tempArr := make(map[string]bool, len(newArr)) 231 | for _, v := range arr { 232 | if tempArr[v] == false { 233 | tempArr[v] = true 234 | newArr = append(newArr, v) 235 | } 236 | } 237 | return newArr 238 | } 239 | 240 | func GetDomains(lis []mode.Link) []string { 241 | var urls []string 242 | for i := range lis { 243 | re := regexp.MustCompile("([a-z0-9\\-]+\\.)*([a-z0-9\\-]+\\.[a-z0-9\\-]+)(:[0-9]+)?") 244 | hosts := re.FindAllString(lis[i].Url, 1) 245 | if len(hosts) > 0 { 246 | urls = append(urls, hosts[0]) 247 | } 248 | } 249 | return UniqueArr(urls) 250 | } 251 | 252 | // 提取fuzz的目录结构 253 | func PathExtract(urls []string) ([]string, []string) { 254 | var catalogues []string 255 | var targets []string 256 | if len(urls) == 0 { 257 | return nil, nil 258 | } 259 | par, _ := url.Parse(urls[0]) 260 | host := par.Scheme + "://" + par.Host 261 | for _, v := range urls { 262 | parse, _ := url.Parse(v) 263 | catalogue := regexp.MustCompile("([^/]+?)/").FindAllStringSubmatch(parse.Path, -1) 264 | if !strings.HasSuffix(parse.Path, "/") { 265 | target := regexp.MustCompile(".*/([^/]+)").FindAllStringSubmatch(parse.Path, -1) 266 | if len(target) > 0 { 267 | targets = append(targets, target[0][1]) 268 | } 269 | } 270 | for _, v := range catalogue { 271 | if !strings.Contains(v[1], "..") { 272 | catalogues = append(catalogues, v[1]) 273 | } 274 | } 275 | 276 | } 277 | targets = append(targets, "upload") 278 | catalogues = UniqueArr(catalogues) 279 | targets = UniqueArr(targets) 280 | url1 := catalogues 281 | url2 := []string{} 282 | url3 := []string{} 283 | var path []string 284 | for _, v1 := range url1 { 285 | for _, v2 := range url1 { 286 | if !strings.Contains(v2, v1) { 287 | url2 = append(url2, "/"+v2+"/"+v1) 288 | } 289 | } 290 | } 291 | if cmd.Z == 3 { 292 | for _, v1 := range url1 { 293 | for _, v3 := range url2 { 294 | if !strings.Contains(v3, v1) { 295 | url3 = append(url3, v3+"/"+v1) 296 | } 297 | } 298 | } 299 | } 300 | for i := range url1 { 301 | url1[i] = "/" + url1[i] 302 | } 303 | if cmd.Z == 3 { 304 | path = make([]string, len(url1)+len(url2)+len(url3)) 305 | } else { 306 | path = make([]string, len(url1)+len(url2)) 307 | } 308 | copy(path, url1) 309 | copy(path[len(url1):], url2) 310 | if cmd.Z == 3 { 311 | copy(path[len(url1)+len(url2):], url3) 312 | } 313 | for i := range path { 314 | path[i] = host + path[i] + "/" 315 | } 316 | path = append(path, host+"/") 317 | return path, targets 318 | } 319 | 320 | // 去除状态码非404的404链接 321 | func Del404(urls []mode.Link) []mode.Link { 322 | is := make(map[int]int) 323 | //根据长度分别存放 324 | for _, v := range urls { 325 | arr, ok := is[len(v.Size)] 326 | if ok { 327 | is[len(v.Size)] = arr + 1 328 | } else { 329 | is[len(v.Size)] = 1 330 | } 331 | } 332 | res := []mode.Link{} 333 | //如果某个长度的数量大于全部的3分之2,那么就判定它是404页面 334 | for i, v := range is { 335 | if v > len(urls)/2 { 336 | for _, vv := range urls { 337 | if len(vv.Size) != i { 338 | res = append(res, vv) 339 | } 340 | } 341 | } 342 | } 343 | return res 344 | 345 | } 346 | 347 | var ( 348 | 349 | // for each request, a random UA will be selected from this list 350 | uas = [...]string{ 351 | "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.5304.68 Safari/537.36", 352 | "Mozilla/5.0 (Windows NT 6.3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.5249.61 Safari/537.36", 353 | "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.5359.71 Safari/537.36", 354 | "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.5359.71 Safari/537.36", 355 | "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.5304.62 Safari/537.36", 356 | "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.5304.107 Safari/537.36", 357 | "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.5304.121 Safari/537.36", 358 | "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.5304.88 Safari/537.36", 359 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.5359.71 Safari/537.36", 360 | "Mozilla/5.0 (Windows NT 6.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.5359.72 Safari/537.36", 361 | "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.5359.94 Safari/537.36", 362 | "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.5359.98 Safari/537.36", 363 | "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.5359.98 Safari/537.36", 364 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.5304.63 Safari/537.36", 365 | "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.5359.95 Safari/537.36", 366 | "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.5304.106 Safari/537.36", 367 | "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.5304.87 Safari/537.36", 368 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.82 Safari/537.36", 369 | "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36", 370 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.74 Safari/537.36 Edg/99.0.1150.46", 371 | "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.87 Safari/537.36 SE 2.X MetaSr 1.0", 372 | "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3883.400 QQBrowser/10.8.4559.400", 373 | } 374 | 375 | nuas = len(uas) 376 | ) 377 | 378 | func GetUserAgent() string { 379 | if cmd.A == "" { 380 | cmd.A = uas[rand.Intn(nuas)] 381 | } 382 | return cmd.A 383 | } 384 | 385 | func GetUpdate() { 386 | 387 | url := fmt.Sprintf("https://api.github.com/repos/pingc0y/URLFinder/releases/latest") 388 | client := &http.Client{ 389 | Timeout: time.Second * 2, 390 | Transport: &http.Transport{ 391 | TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, 392 | Proxy: http.ProxyFromEnvironment, 393 | }, 394 | } 395 | resp, err := client.Get(url) 396 | if err != nil { 397 | cmd.XUpdate = "更新检测失败" 398 | return 399 | } 400 | defer resp.Body.Close() 401 | body, err := io.ReadAll(resp.Body) 402 | if err != nil { 403 | cmd.XUpdate = "更新检测失败" 404 | return 405 | } 406 | var release struct { 407 | TagName string `json:"tag_name"` 408 | } 409 | err = json.Unmarshal(body, &release) 410 | if err != nil { 411 | cmd.XUpdate = "更新检测失败" 412 | return 413 | } 414 | if release.TagName == "" { 415 | cmd.XUpdate = "更新检测失败" 416 | return 417 | } 418 | if cmd.Update != release.TagName { 419 | cmd.XUpdate = "有新版本可用: " + release.TagName 420 | } else { 421 | cmd.XUpdate = "已是最新版本" 422 | } 423 | 424 | } 425 | --------------------------------------------------------------------------------