├── README.md ├── config.go ├── config.yml ├── go.mod ├── go_proxy_pool.README.md ├── main.go ├── spider.go ├── tunnelProxy.go ├── verify.go └── webApiServer.go /README.md: -------------------------------------------------------------------------------- 1 | # 构建前安装依赖 2 | ``` 3 | apk add go 4 | go get github.com/gin-gonic/gin 5 | go get gopkg.in/yaml.v3 6 | ``` 7 | # goProxyPool 8 | 9 | 一款无环境依赖开箱即用的免费代理IP池 10 | 11 | 内置14个免费代理源，均使用内置的简单正则获取 12 | 13 | 支持调用插件扩展代理源，返回的数据符合格式即可，无开发语言限制 14 | 15 | 支持webApi获取、删除、更新等代理池内的IP 16 | 17 | 支持 http，socket5 隧道代理模式，无需手动更换IP 18 | 19 | 遇到bug或有好的建议，欢迎提issue 20 | 21 | ## 隧道代理 22 | 隧道代理是代理IP存在的一种方式。 23 | 相对于传统固定代理IP，它的特点是自动地在代理服务器上改变IP，这样每个请求都使用一个不同的IP。 24 | 25 | ## 代理IP特征 26 | 这里提供一些代理IP的特征，师傅们可通过特征自己写代理源，api获取的话内置的正则方式就能写 27 | 360网络空间测绘_socket5： 28 | ```text 29 | protocol:"socks5" AND "Accepted Auth Method: 0x0" AND "connection: close" AND country: "China" 30 | ``` 31 | fofa_http: 32 | ```text 33 | "HTTP/1.1 403 Forbidden Server: nginx/1.12.1" && port="9091" 34 | 35 | port="3128" && title="ERROR: The requested URL could not be retrieved" 36 | 37 | "X-Cache: 'MISS from VideoCacheBox/CE8265A63696DECD7F0D17858B1BDADC37771805'" && "X-Squid-Error: ERR_ACCESS_DENIED 0" 38 | ``` 39 | hunter_http： 40 | ```text 41 | header.server="nginx/2.2.200603d"&&web.title="502 Bad Gateway" && ip.port="8085" 42 | ``` 43 | 44 | # 截图 45 | [![zuz6TU.png](https://s1.ax1x.com/2022/11/19/zuz6TU.png)](https://s1.ax1x.com/2022/11/19/zuz6TU.png) 46 | # 使用说明 47 | 下载 48 | ``` 49 | git clone https://github.com/pingc0y/go_proxy_pool.git 50 | ``` 51 | 编译（直接使用成品，就无需编译） 52 | 以下是在windows环境下，编译出各平台可执行文件的命令 53 | ``` 54 | SET CGO_ENABLED=0 55 | SET GOOS=windows 56 | SET GOARCH=amd64 57 | go build -ldflags "-s -w" -o ../goProxyPool-windows-amd64.exe 58 | 59 | SET CGO_ENABLED=0 60 | SET GOOS=windows 61 | SET GOARCH=386 62 | go build -ldflags "-s -w" -o ../goProxyPool-windows-386.exe 63 | 64 | SET CGO_ENABLED=0 65 | SET GOOS=linux 66 | SET GOARCH=amd64 67 | go build -ldflags "-s -w" -o ../goProxyPool-linux-amd64 68 | 69 | SET CGO_ENABLED=0 70 | SET GOOS=linux 71 | SET GOARCH=arm64 72 | go build -ldflags "-s -w" -o ../goProxyPool-linux-arm64 73 | 74 | SET CGO_ENABLED=0 75 | SET GOOS=linux 76 | SET GOARCH=386 77 | go build -ldflags "-s -w" -o ../goProxyPool-linux-386 78 | 79 | SET CGO_ENABLED=0 80 | SET GOOS=darwin 81 | SET GOARCH=amd64 82 | go build -ldflags "-s -w" -o ../goProxyPool-macos-amd64 83 | 84 | SET CGO_ENABLED=0 85 | SET GOOS=darwin 86 | SET GOARCH=arm64 87 | go build -ldflags "-s -w" -o ../goProxyPool-macos-arm64 88 | 89 | ``` 90 | 运行 91 | 需要与config.yml在同一目录 92 | 注意：抓取代理会进行类型地区等验证会比较缓慢，存活验证会快很多 93 | ``` 94 | .\goProxyPool.exe 95 | ``` 96 | 97 | 代理源中有部分需要翻墙才能访问，有条件就设置下config.yml的代理配置 98 | ```yml 99 | proxy: 100 | host: 127.0.0.1 101 | port: 10809 102 | ``` 103 | ## webAPi说明 104 | 查看代理池情况 105 | ``` 106 | http://127.0.0.1:8080/ 107 | ``` 108 | 109 | 获取代理 110 | ``` 111 | http://127.0.0.1:8080/get?type=HTTP&count=10&anonymity=all 112 | 可选参数： 113 | type 代理类型 114 | anonymity 匿名度 115 | country 国家 116 | source 代理源 117 | count 代理数量 118 | 获取所有：all 119 | ``` 120 | 121 | 删除代理 122 | ``` 123 | http://127.0.0.1:8080/delete?ip=127.0.0.1&port=8888 124 | 必须传参： 125 | ip 代理ip 126 | port 代理端口 127 | ``` 128 | 129 | 验证代理 130 | ``` 131 | http://127.0.0.1:8080/verify 132 | ``` 133 | 134 | 更换隧道代理IP 135 | ``` 136 | http://127.0.0.1:8080/tunnelUpdate 137 | ``` 138 | 139 | 抓取代理 140 | ``` 141 | http://127.0.0.1:8080/spider 142 | ``` 143 | ## 代理字段解读 144 | ```go 145 | type ProxyIp struct { 146 | Ip string //IP地址 147 | Port string //代理端口 148 | Country string //代理国家 149 | Province string //代理省份 150 | City string //代理城市 151 | Isp string //IP提供商 152 | Type string //代理类型 153 | Anonymity string //代理匿名度, 透明：显示真实IP, 普匿：显示假的IP, 高匿：无代理IP特征 154 | Time string //代理验证 155 | Speed string //代理响应速度 156 | SuccessNum int //验证请求成功的次数 157 | RequestNum int //验证请求的次数 158 | Source string //代理源 159 | } 160 | ``` 161 | ## 配置文件 162 | ```yaml 163 | #使用代理去获取代理IP 164 | proxy: 165 | host: 127.0.0.1 166 | port: 10809 167 | 168 | # 配置信息 169 | config: 170 | #监听IP 171 | ip: 0.0.0.0 172 | #web监听端口 173 | port: 8080 174 | #http隧道代理端口 175 | httpTunnelPort: 8111 176 | #socket隧道代理端口 177 | socketTunnelPort: 8112 178 | #隧道代理更换时间秒 179 | tunnelTime: 60 180 | #可用IP数量小于‘proxyNum’时就去抓取 181 | proxyNum: 50 182 | #代理IP验证间隔秒 183 | verifyTime: 1800 184 | #抓取/检测状态线程数 185 | threadNum: 200 186 | 187 | #ip源 188 | spider: 189 | #代理获取源1 190 | - name: '齐云代理' 191 | #请求方式 192 | method: 'GET' 193 | #POST传参用的请求体 194 | body: '' 195 | #urls请求间隔/秒，防止频率过快被限制 196 | interval: 0 197 | #使用的请求头 198 | Headers: 199 | User-Agent: 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' 200 | #获取的地址 201 | urls: 'https://proxy.ip3366.net/free/?action=china&page=1,https://proxy.ip3366.net/free/?action=china&page=2,https://proxy.ip3366.net/free/?action=china&page=3' 202 | #获取IP的正则表达式， 203 | ip: '\"IP\">(\d+?\.\d+?.\d+?\.\d+?)' 204 | #获取端口的正则表达式 205 | port: '\"PORT\">(\d+?)' 206 | #是否使用代理去请求 207 | proxy: false 208 | 209 | #通过插件，扩展ip源 210 | spiderPlugin: 211 | #插件名 212 | - name: test 213 | #运行命令，返回的结果要符合格式 214 | run: '.\test1.exe' 215 | 216 | #通过文件，导入IP 217 | spiderFile: 218 | #文件名 219 | - name: test1 220 | #文件路径 221 | path: 'ip.txt' 222 | ``` 223 | ### 扩展返回格式 224 | 通过,分割 225 | ```text 226 | 110.179.64.89:1080,111.2.155.180:1090,111.172.3.212:1090,111.196.186.95:6669 227 | ``` 228 | ### 文件导入格式 229 | 通过换行分割 230 | ```text 231 | 110.179.64.89:1080 232 | 111.2.155.180:1090 233 | 111.172.3.212:1090 234 | 111.196.186.95:6669 235 | 111.201.103.29:1080 236 | 113.12.200.66:1080 237 | 113.67.96.67:1090 238 | 113.104.217.45:1080 239 | 113.110.246.76:1080 240 | 113.116.9.18:1080 241 | 113.119.193.183:1090 242 | 113.119.193.187:1090 243 | 113.249.93.219:1080 244 | 114.95.200.164:1080 245 | 115.193.161.177:1080 246 | ``` 247 | 248 | 249 | ## 更新说明 250 | 2022/11/22 251 | 修复 ip归属地接口更换 252 | 优化 验证代理 253 | 254 | 2022/11/19 255 | 新增 socket5代理 256 | 新增 文件导入代理 257 | 新增 显示验证进度 258 | 新增 验证webApi 259 | 修改 扩展导入格式 260 | 优化 代理验证方式 261 | 优化 匿名度改为自动识别 262 | 修复 若干bug 263 | 264 | 265 | -------------------------------------------------------------------------------- /config.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "encoding/json" 6 | "gopkg.in/yaml.v3" 7 | "io" 8 | "log" 9 | "net/http" 10 | "os" 11 | ) 12 | 13 | var conf *Config 14 | 15 | type Config struct { 16 | Spider []Spider `yaml:"spider" json:"spider"` 17 | SpiderPlugin []SpiderPlugin `yaml:"spiderPlugin" json:"spiderPlugin"` 18 | SpiderFile []SpiderFile `yaml:"spiderFile" json:"spiderFile"` 19 | Proxy Proxy `yaml:"proxy" json:"proxy"` 20 | Config config `yaml:"config" json:"config"` 21 | } 22 | type config struct { 23 | Ip string `yaml:"ip" json:"ip"` 24 | Port string `yaml:"port" json:"port"` 25 | HttpTunnelPort string `yaml:"httpTunnelPort" json:"httpTunnelPort"` 26 | SocketTunnelPort string `yaml:"socketTunnelPort" json:"socketTunnelPort"` 27 | TunnelTime int `yaml:"tunnelTime" json:"tunnelTime"` 28 | ProxyNum int `yaml:"proxyNum" json:"proxyNum"` 29 | VerifyTime int `yaml:"verifyTime" json:"verifyTime"` 30 | ThreadNum int `yaml:"threadNum" json:"threadNum"` 31 | } 32 | type Spider struct { 33 | Name string `yaml:"name" json:"name"` 34 | Method string `yaml:"method" json:"method"` 35 | Interval int `yaml:"interval" json:"interval"` 36 | Body string `yaml:"body" json:"body"` 37 | ProxyIs bool `yaml:"proxy" json:"proxy"` 38 | Headers map[string]string `yaml:"headers" json:"headers"` 39 | Urls string `yaml:"urls" json:"urls"` 40 | Ip string `yaml:"ip" json:"ip"` 41 | Port string `yaml:"port" json:"port"` 42 | } 43 | type SpiderPlugin struct { 44 | Name string `yaml:"name" json:"name"` 45 | Run string `yaml:"run" json:"run"` 46 | } 47 | type SpiderFile struct { 48 | Name string `yaml:"name" json:"name"` 49 | Path string `yaml:"path" json:"path"` 50 | } 51 | 52 | type Proxy struct { 53 | Host string `yaml:"host" json:"host"` 54 | Port string `yaml:"port" json:"port"` 55 | } 56 | type ProxyIp struct { 57 | Ip string //IP地址 58 | Port string //代理端口 59 | Country string //代理国家 60 | Province string //代理省份 61 | City string //代理城市 62 | Isp string //IP提供商 63 | Type string //代理类型 64 | Anonymity string //代理匿名度, 透明：显示真实IP, 普匿：显示假的IP, 高匿：无代理IP特征 65 | Time string //代理验证 66 | Speed string //代理响应速度 67 | SuccessNum int //验证请求成功的次数 68 | RequestNum int //验证请求的次数 69 | Source string //代理源 70 | } 71 | 72 | // 数组去重 73 | func uniquePI(arr []ProxyIp) []ProxyIp { 74 | var pr []ProxyIp 75 | for _, v := range arr { 76 | is := true 77 | for _, vv := range pr { 78 | if v.Ip+v.Port == vv.Ip+vv.Port { 79 | is = false 80 | } 81 | } 82 | if is { 83 | pr = append(pr, v) 84 | } 85 | } 86 | return pr 87 | } 88 | 89 | // 读取配置文件 90 | func GetConfigData() { 91 | //导入配置文件 92 | yamlFile, err := os.ReadFile("config.yml") 93 | if err != nil { 94 | log.Println("配置文件打开错误：" + err.Error()) 95 | err.Error() 96 | return 97 | } 98 | //将配置文件读取到结构体中 99 | err = yaml.Unmarshal(yamlFile, &conf) 100 | if err != nil { 101 | log.Println("配置文件解析错误：" + err.Error()) 102 | err.Error() 103 | return 104 | } 105 | //导入代理缓存 106 | file, err := os.OpenFile("data.json", os.O_RDWR|os.O_CREATE, 0644) 107 | if err != nil { 108 | log.Println("代理json文件打开错误：" + err.Error()) 109 | err.Error() 110 | return 111 | } 112 | defer file.Close() 113 | all, err := io.ReadAll(file) 114 | if err != nil { 115 | log.Println("代理json解析错误：" + err.Error()) 116 | return 117 | } 118 | if len(all) == 0 { 119 | return 120 | } 121 | err = json.Unmarshal(all, &ProxyPool) 122 | if err != nil { 123 | log.Println("代理json解析错误：" + err.Error()) 124 | return 125 | } 126 | 127 | } 128 | 129 | // 处理Headers配置 130 | func SetHeadersConfig(he map[string]string, header *http.Header) *http.Header { 131 | for k, v := range he { 132 | header.Add(k, v) 133 | } 134 | return header 135 | } 136 | 137 | func export() { 138 | mux1.Lock() 139 | defer mux1.Unlock() 140 | //导出代理到文件 141 | err := os.Truncate("data.json", 0) 142 | if len(ProxyPool) == 0 { 143 | return 144 | } 145 | if err != nil { 146 | log.Printf("data.json清理失败：%s", err) 147 | return 148 | } 149 | file, err := os.OpenFile("data.json", os.O_RDWR|os.O_CREATE, 0644) 150 | if err != nil { 151 | log.Printf("data.json打开失败：%s", err) 152 | return 153 | } 154 | defer file.Close() 155 | 156 | data, err := json.Marshal(ProxyPool) 157 | if err != nil { 158 | log.Printf("代理json化失败：%s", err) 159 | return 160 | } 161 | buf := bufio.NewWriter(file) 162 | // 字节写入 163 | buf.Write(data) 164 | // 将缓冲中的数据写入 165 | err = buf.Flush() 166 | if err != nil { 167 | log.Println("代理json保存失败:", err) 168 | } 169 | } 170 | -------------------------------------------------------------------------------- /config.yml: -------------------------------------------------------------------------------- 1 | #使用代理去获取代理IP 2 | proxy: 3 | host: 127.0.0.1 4 | port: 10809 5 | 6 | #配置信息 7 | config: 8 | #监听IP 9 | ip: 0.0.0.0 10 | #webApi监听端口 11 | port: 8080 12 | #http隧道代理端口 13 | httpTunnelPort: 8111 14 | #socket隧道代理端口 15 | socketTunnelPort: 8112 16 | #隧道代理更换时间秒 17 | tunnelTime: 60 18 | #可用IP数量小于‘proxyNum’时就去抓取 19 | proxyNum: 30 20 | #代理IP验证间隔秒 21 | verifyTime: 1800 22 | #抓取/检测状态线程数 23 | threadNum: 200 24 | 25 | #ip源 26 | spider: 27 | - name: '齐云代理' 28 | #请求方式 29 | method: 'GET' 30 | #post传参用的请求体 31 | #body: '' 32 | #urls请求间隔/秒，防止频率过快被限制 33 | interval: 0 34 | #使用的请求头 35 | Headers: 36 | User-Agent: 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' 37 | #获取的地址 38 | urls: 'https://proxy.ip3366.net/free/?action=china&page=1,https://proxy.ip3366.net/free/?action=china&page=2,https://proxy.ip3366.net/free/?action=china&page=3' 39 | #获取IP的正则表达式， 40 | ip: '\"IP\">(\d+?\.\d+?.\d+?\.\d+?)' 41 | #获取端口的正则表达式 42 | port: '\"PORT\">(\d+?)' 43 | #是否使用代理去请求 44 | proxy: false 45 | 46 | - name: "89代理" 47 | method: 'GET' 48 | Headers: 49 | User-Agent: 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' 50 | urls: "https://www.89ip.cn/index_1.html,https://www.89ip.cn/index_2.html,https://www.89ip.cn/index_3.html" 51 | ip: '[\s]*?(\d+?\.\d+?.\d+?\.\d+?)[\s]*?' 52 | port: '[\s]*?\d+?\.\d+?.\d+?\.\d+?[\s]*?[\s]*?[\s]*?(\d+?)[\s]*?' 53 | proxy: false 54 | 55 | #每天需要跟换地址 56 | - name: "开心代理" 57 | method: 'GET' 58 | Headers: 59 | User-Agent: 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' 60 | urls: "http://www.kxdaili.com/daili/ip/7713.html" 61 | ip: '\[.+?\](\d+?\.\d+?.\d+?\.\d+?):\d+?@.+?#' 62 | port: '\[.+?\]\d+?\.\d+?.\d+?\.\d+?:(\d+?)@.+?#' 63 | proxy: false 64 | 65 | - name: "快代理" 66 | method: 'GET' 67 | Headers: 68 | User-Agent: 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' 69 | urls: "http://www.ip3366.net/?stype=1&page=1,http://www.ip3366.net/?stype=1&page=2,http://www.ip3366.net/?stype=3&page=1,http://www.ip3366.net/?stype=3&page=2" 70 | ip: '[\s]*?(\d+?\.\d+?.\d+?\.\d+?)[\s]*?' 71 | port: '[\s]*?\d+?\.\d+?.\d+?\.\d+?[\s]*?[\s]*?[\s]*?(\d+?)[\s]*?' 72 | proxy: false 73 | 74 | - name: "高可用代理" 75 | method: 'GET' 76 | Headers: 77 | User-Agent: 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' 78 | urls: "https://ip.jiangxianli.com/?page=1" 79 | ip: '[\s]*?(\d+?\.\d+?.\d+?\.\d+?)[\s]*?' 80 | port: '[\s]*?\d+?\.\d+?.\d+?\.\d+?[\s]*?[\s]*?[\s]*?(\d+?)[\s]*?' 81 | proxy: false 82 | 83 | 84 | - name: "小舒代理" 85 | method: 'GET' 86 | Headers: 87 | User-Agent: 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' 88 | urls: "https://xsdaili.cn/dayProxy/ip/1796.html" 89 | ip: '(\d+?\.\d+?.\d+?\.\d+?):\d+?@.+?#\[.+?\]' 90 | port: '\d+?\.\d+?.\d+?\.\d+?:(\d+?)@.+?#\[.+?\]' 91 | proxy: false 92 | 93 | - name: "命运零代理" 94 | method: 'GET' 95 | Headers: 96 | User-Agent: 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' 97 | urls: "http://proxylist.fatezero.org/proxy.list" 98 | ip: '\"host\": \"(.+?)\"' 99 | port: '\"port\": (\d+)' 100 | proxy: false 101 | 102 | 103 | - name: '自由代理' 104 | method: 'GET' 105 | Headers: 106 | #需要使用cookie,不然会有验证 107 | User-Agent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36' 108 | urls: "https://www.freeproxylists.net/zh/?c=CN&u=50&page=1" 109 | ip: '%3e(.{0,60}?)%3c%2f' 110 | port: 'center\">(\d+?)' 119 | port: 'href=\"/\d+?\.\d+?.\d+?\.\d+?/(\d+?)#http.{0,1}\">' 120 | proxy: false 121 | 122 | 123 | - name: "hidemy代理" 124 | method: 'GET' 125 | Headers: 126 | User-Agent: 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' 127 | urls: "https://hidemy.name/cn/proxy-list/?maxtime=1000&type=h#list,https://hidemy.name/cn/proxy-list/?maxtime=1000&type=h&start=64#list,https://hidemy.name/cn/proxy-list/?maxtime=1000&type=h&start=128#list,https://hidemy.name/cn/proxy-list/?maxtime=5000&type=5#list,https://hidemy.name/cn/proxy-list/?maxtime=5000&type=s#list" 128 | ip: '(\d+?\.\d+?.\d+?\.\d+?)\d+' 129 | port: '\d+?\.\d+?.\d+?\.\d+?(\d+)' 130 | proxy: true 131 | 132 | - name: "scrape代理" 133 | method: 'GET' 134 | Headers: 135 | User-Agent: 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' 136 | urls: "https://api.proxyscrape.com/v2/?request=getproxies&protocol=http&timeout=10000&country=CN&ssl=all&anonymity=all" 137 | ip: '(\d+?\.\d+?.\d+?\.\d+?):\d+' 138 | port: '\d+?\.\d+?.\d+?\.\d+?:(\d+)' 139 | anonymity: '透明' 140 | proxy: true 141 | 142 | - name: "my代理" 143 | method: 'GET' 144 | Headers: 145 | User-Agent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36' 146 | urls: "https://www.my-proxy.com/free-socks-5-proxy.html,https://www.my-proxy.com/free-elite-proxy.html,https://www.my-proxy.com/free-anonymous-proxy.html" 147 | ip: '>(\d+?\.\d+?.\d+?\.\d+?):\d+#' 148 | port: '>\d+?\.\d+?.\d+?\.\d+?:(\d+)#' 149 | proxy: true 150 | 151 | - name: "proxy代理" 152 | method: 'GET' 153 | Headers: 154 | User-Agent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36' 155 | urls: "https://free-proxy-list.net/,https://www.us-proxy.org/,https://www.socks-proxy.net/" 156 | ip: '(\d+?\.\d+?.\d+?\.\d+?):\d+' 157 | port: '\d+?\.\d+?.\d+?\.\d+?:(\d+)' 158 | proxy: true 159 | 160 | 161 | #通过插件，扩展ip源 162 | #spiderPlugin: 163 | # #插件名 164 | # - name: test 165 | # #运行命令，返回的结果要符合格式 166 | # run: 'text.exe' 167 | 168 | #通过文件导入ip 169 | #spiderFile: 170 | # #插件名 171 | # - name: test1 172 | # #运行命令，返回的结果要符合格式 173 | # path: 'ip.txt' 174 | # 175 | # 176 | 177 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module go_proxy_pool 2 | 3 | go 1.19 4 | 5 | require ( 6 | github.com/gin-gonic/gin v1.8.1 7 | gopkg.in/yaml.v3 v3.0.1 8 | ) 9 | 10 | require ( 11 | github.com/gin-contrib/sse v0.1.0 // indirect 12 | github.com/go-playground/locales v0.14.0 // indirect 13 | github.com/go-playground/universal-translator v0.18.0 // indirect 14 | github.com/go-playground/validator/v10 v10.11.1 // indirect 15 | github.com/goccy/go-json v0.9.11 // indirect 16 | github.com/google/go-cmp v0.5.9 // indirect 17 | github.com/json-iterator/go v1.1.12 // indirect 18 | github.com/leodido/go-urn v1.2.1 // indirect 19 | github.com/mattn/go-isatty v0.0.16 // indirect 20 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 21 | github.com/modern-go/reflect2 v1.0.2 // indirect 22 | github.com/pelletier/go-toml/v2 v2.0.5 // indirect 23 | github.com/stretchr/testify v1.8.1 // indirect 24 | github.com/ugorji/go/codec v1.2.7 // indirect 25 | golang.org/x/crypto v0.2.0 // indirect 26 | golang.org/x/net v0.7.0 // indirect 27 | golang.org/x/sys v0.5.0 // indirect 28 | golang.org/x/text v0.7.0 // indirect 29 | google.golang.org/protobuf v1.28.1 // indirect 30 | gopkg.in/yaml.v2 v2.4.0 // indirect 31 | ) 32 | -------------------------------------------------------------------------------- /go_proxy_pool.README.md: -------------------------------------------------------------------------------- 1 | # goProxyPool 2 | 3 | 一款无环境依赖开箱即用的免费代理IP池 4 | 5 | 内置14个免费代理源，均使用内置的简单正则获取 6 | 7 | 支持调用插件扩展代理源，返回的数据符合格式即可，无开发语言限制 8 | 9 | 支持webApi获取、删除、更新等代理池内的IP 10 | 11 | 支持 http，socket5 隧道代理模式，无需手动更换IP 12 | 13 | 遇到bug或有好的建议，欢迎提issue 14 | 15 | ## 隧道代理 16 | 隧道代理是代理IP存在的一种方式。 17 | 相对于传统固定代理IP，它的特点是自动地在代理服务器上改变IP，这样每个请求都使用一个不同的IP。 18 | 19 | ## 代理IP特征 20 | 这里提供一些代理IP的特征，师傅们可通过特征自己写代理源，api获取的话内置的正则方式就能写 21 | 360网络空间测绘_socket5： 22 | ```text 23 | protocol:"socks5" AND "Accepted Auth Method: 0x0" AND "connection: close" AND country: "China" 24 | ``` 25 | fofa_http: 26 | ```text 27 | "HTTP/1.1 403 Forbidden Server: nginx/1.12.1" && port="9091" 28 | 29 | port="3128" && title="ERROR: The requested URL could not be retrieved" 30 | 31 | "X-Cache: 'MISS from VideoCacheBox/CE8265A63696DECD7F0D17858B1BDADC37771805'" && "X-Squid-Error: ERR_ACCESS_DENIED 0" 32 | ``` 33 | hunter_http： 34 | ```text 35 | header.server="nginx/2.2.200603d"&&web.title="502 Bad Gateway" && ip.port="8085" 36 | ``` 37 | 38 | # 截图 39 | [![zuz6TU.png](https://s1.ax1x.com/2022/11/19/zuz6TU.png)](https://s1.ax1x.com/2022/11/19/zuz6TU.png) 40 | # 使用说明 41 | 下载 42 | ``` 43 | git clone https://github.com/pingc0y/go_proxy_pool.git 44 | ``` 45 | 编译（直接使用成品，就无需编译） 46 | 以下是在windows环境下，编译出各平台可执行文件的命令 47 | ``` 48 | SET CGO_ENABLED=0 49 | SET GOOS=windows 50 | SET GOARCH=amd64 51 | go build -ldflags "-s -w" -o ../goProxyPool-windows-amd64.exe 52 | 53 | SET CGO_ENABLED=0 54 | SET GOOS=windows 55 | SET GOARCH=386 56 | go build -ldflags "-s -w" -o ../goProxyPool-windows-386.exe 57 | 58 | SET CGO_ENABLED=0 59 | SET GOOS=linux 60 | SET GOARCH=amd64 61 | go build -ldflags "-s -w" -o ../goProxyPool-linux-amd64 62 | 63 | SET CGO_ENABLED=0 64 | SET GOOS=linux 65 | SET GOARCH=arm64 66 | go build -ldflags "-s -w" -o ../goProxyPool-linux-arm64 67 | 68 | SET CGO_ENABLED=0 69 | SET GOOS=linux 70 | SET GOARCH=386 71 | go build -ldflags "-s -w" -o ../goProxyPool-linux-386 72 | 73 | SET CGO_ENABLED=0 74 | SET GOOS=darwin 75 | SET GOARCH=amd64 76 | go build -ldflags "-s -w" -o ../goProxyPool-macos-amd64 77 | 78 | SET CGO_ENABLED=0 79 | SET GOOS=darwin 80 | SET GOARCH=arm64 81 | go build -ldflags "-s -w" -o ../goProxyPool-macos-arm64 82 | 83 | ``` 84 | 运行 85 | 需要与config.yml在同一目录 86 | 注意：抓取代理会进行类型地区等验证会比较缓慢，存活验证会快很多 87 | ``` 88 | .\goProxyPool.exe 89 | ``` 90 | 91 | 代理源中有部分需要翻墙才能访问，有条件就设置下config.yml的代理配置 92 | ```yml 93 | proxy: 94 | host: 127.0.0.1 95 | port: 10809 96 | ``` 97 | ## webAPi说明 98 | 查看代理池情况 99 | ``` 100 | http://127.0.0.1:8080/ 101 | ``` 102 | 103 | 获取代理 104 | ``` 105 | http://127.0.0.1:8080/get?type=HTTP&count=10&anonymity=all 106 | 可选参数： 107 | type 代理类型 108 | anonymity 匿名度 109 | country 国家 110 | source 代理源 111 | count 代理数量 112 | 获取所有：all 113 | ``` 114 | 115 | 删除代理 116 | ``` 117 | http://127.0.0.1:8080/delete?ip=127.0.0.1&port=8888 118 | 必须传参： 119 | ip 代理ip 120 | port 代理端口 121 | ``` 122 | 123 | 验证代理 124 | ``` 125 | http://127.0.0.1:8080/verify 126 | ``` 127 | 128 | 更换隧道代理IP 129 | ``` 130 | http://127.0.0.1:8080/tunnelUpdate 131 | ``` 132 | 133 | 抓取代理 134 | ``` 135 | http://127.0.0.1:8080/spider 136 | ``` 137 | ## 代理字段解读 138 | ```go 139 | type ProxyIp struct { 140 | Ip string //IP地址 141 | Port string //代理端口 142 | Country string //代理国家 143 | Province string //代理省份 144 | City string //代理城市 145 | Isp string //IP提供商 146 | Type string //代理类型 147 | Anonymity string //代理匿名度, 透明：显示真实IP, 普匿：显示假的IP, 高匿：无代理IP特征 148 | Time string //代理验证 149 | Speed string //代理响应速度 150 | SuccessNum int //验证请求成功的次数 151 | RequestNum int //验证请求的次数 152 | Source string //代理源 153 | } 154 | ``` 155 | ## 配置文件 156 | ```yaml 157 | #使用代理去获取代理IP 158 | proxy: 159 | host: 127.0.0.1 160 | port: 10809 161 | 162 | # 配置信息 163 | config: 164 | #监听IP 165 | ip: 0.0.0.0 166 | #web监听端口 167 | port: 8080 168 | #http隧道代理端口 169 | httpTunnelPort: 8111 170 | #socket隧道代理端口 171 | socketTunnelPort: 8112 172 | #隧道代理更换时间秒 173 | tunnelTime: 60 174 | #可用IP数量小于‘proxyNum’时就去抓取 175 | proxyNum: 50 176 | #代理IP验证间隔秒 177 | verifyTime: 1800 178 | #抓取/检测状态线程数 179 | threadNum: 200 180 | 181 | #ip源 182 | spider: 183 | #代理获取源1 184 | - name: '齐云代理' 185 | #请求方式 186 | method: 'GET' 187 | #POST传参用的请求体 188 | body: '' 189 | #urls请求间隔/秒，防止频率过快被限制 190 | interval: 0 191 | #使用的请求头 192 | Headers: 193 | User-Agent: 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' 194 | #获取的地址 195 | urls: 'https://proxy.ip3366.net/free/?action=china&page=1,https://proxy.ip3366.net/free/?action=china&page=2,https://proxy.ip3366.net/free/?action=china&page=3' 196 | #获取IP的正则表达式， 197 | ip: '\"IP\">(\d+?\.\d+?.\d+?\.\d+?)' 198 | #获取端口的正则表达式 199 | port: '\"PORT\">(\d+?)' 200 | #是否使用代理去请求 201 | proxy: false 202 | 203 | #通过插件，扩展ip源 204 | spiderPlugin: 205 | #插件名 206 | - name: test 207 | #运行命令，返回的结果要符合格式 208 | run: '.\test1.exe' 209 | 210 | #通过文件，导入IP 211 | spiderFile: 212 | #文件名 213 | - name: test1 214 | #文件路径 215 | path: 'ip.txt' 216 | ``` 217 | ### 扩展返回格式 218 | 通过,分割 219 | ```text 220 | 110.179.64.89:1080,111.2.155.180:1090,111.172.3.212:1090,111.196.186.95:6669 221 | ``` 222 | ### 文件导入格式 223 | 通过换行分割 224 | ```text 225 | 110.179.64.89:1080 226 | 111.2.155.180:1090 227 | 111.172.3.212:1090 228 | 111.196.186.95:6669 229 | 111.201.103.29:1080 230 | 113.12.200.66:1080 231 | 113.67.96.67:1090 232 | 113.104.217.45:1080 233 | 113.110.246.76:1080 234 | 113.116.9.18:1080 235 | 113.119.193.183:1090 236 | 113.119.193.187:1090 237 | 113.249.93.219:1080 238 | 114.95.200.164:1080 239 | 115.193.161.177:1080 240 | ``` 241 | 242 | 243 | ## 更新说明 244 | 2022/11/22 245 | 修复 ip归属地接口更换 246 | 优化 验证代理 247 | 248 | 2022/11/19 249 | 新增 socket5代理 250 | 新增 文件导入代理 251 | 新增 显示验证进度 252 | 新增 验证webApi 253 | 修改 扩展导入格式 254 | 优化 代理验证方式 255 | 优化 匿名度改为自动识别 256 | 修复 若干bug 257 | 258 | 259 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "sync" 7 | "time" 8 | ) 9 | 10 | var wg3 sync.WaitGroup 11 | var mux1 sync.Mutex 12 | var ch1 = make(chan int, 50) 13 | 14 | func main() { 15 | //VerifyHttp("127.0.0.1:10809") 16 | 17 | fmt.Println(" ___ ___ _ " + 18 | "\n ___ ___ | . " + "\\ _ _ ___ __ _ _ | . \\ ___ ___ | |" + 19 | "\n/ . |/ . \\| _/| '_>/ . \\\\ \\/| | || _// . \\/ . \\| |" + 20 | "\n\\_. |\\___/|_| |_| \\___//\\_\\`_. ||_| \\___/\\___/|_|" + 21 | "\n<___' <___' ") 22 | InitData() 23 | //开启隧道代理 24 | go httpSRunTunnelProxyServer() 25 | go socket5RunTunnelProxyServer() 26 | //启动webAPi 27 | Run() 28 | } 29 | 30 | // 初始化 31 | func InitData() { 32 | //获取配置文件 33 | GetConfigData() 34 | //设置线程数量 35 | ch1 = make(chan int, conf.Config.ThreadNum) 36 | ch2 = make(chan int, conf.Config.ThreadNum) 37 | //是否需要抓代理 38 | if len(ProxyPool) < conf.Config.ProxyNum { 39 | //抓取代理 40 | spiderRun() 41 | } 42 | //定时判断是否需要获取代理iP 43 | go func() { 44 | // 每 60 秒钟时执行一次 45 | ticker := time.NewTicker(60 * time.Second) 46 | for range ticker.C { 47 | if len(ProxyPool) < conf.Config.ProxyNum { 48 | if !run && !verifyIS { 49 | log.Printf("代理数量不足 %d\n", conf.Config.ProxyNum) 50 | //抓取代理 51 | spiderRun() 52 | } 53 | } else { 54 | //保存代理到本地 55 | export() 56 | } 57 | } 58 | }() 59 | 60 | //定时更换隧道IP 61 | go func() { 62 | tunnelTime := time.Duration(conf.Config.TunnelTime) 63 | ticker := time.NewTicker(tunnelTime * time.Second) 64 | for range ticker.C { 65 | if len(ProxyPool) != 0 { 66 | httpsIp = getHttpsIp() 67 | httpIp = gethttpIp() 68 | socket5Ip = getSocket5Ip() 69 | } 70 | } 71 | }() 72 | 73 | // 验证代理存活情况 74 | go func() { 75 | verifyTime := time.Duration(conf.Config.VerifyTime) 76 | ticker := time.NewTicker(verifyTime * time.Second) 77 | for range ticker.C { 78 | if !verifyIS && !run { 79 | VerifyProxy() 80 | } 81 | } 82 | }() 83 | } 84 | -------------------------------------------------------------------------------- /spider.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "crypto/tls" 6 | "io" 7 | "log" 8 | "net/http" 9 | "net/url" 10 | "os" 11 | "os/exec" 12 | "regexp" 13 | "strings" 14 | "sync" 15 | "time" 16 | ) 17 | 18 | var wg sync.WaitGroup 19 | var wg2 sync.WaitGroup 20 | var mux sync.Mutex 21 | var ch2 = make(chan int, 50) 22 | 23 | // 是否抓取中 24 | var run = false 25 | 26 | func spiderRun() { 27 | 28 | run = true 29 | defer func() { 30 | run = false 31 | }() 32 | 33 | count = 0 34 | log.Println("开始抓取代理...") 35 | for i := range conf.Spider { 36 | wg2.Add(1) 37 | go spider(&conf.Spider[i]) 38 | } 39 | wg2.Wait() 40 | log.Printf("\r%s 代理抓取结束 \n", time.Now().Format("2006-01-02 15:04:05")) 41 | 42 | count = 0 43 | log.Println("开始扩展抓取代理...") 44 | for i := range conf.SpiderPlugin { 45 | wg2.Add(1) 46 | go spiderPlugin(&conf.SpiderPlugin[i]) 47 | } 48 | wg2.Wait() 49 | log.Printf("\r%s 扩展代理抓取结束 \n", time.Now().Format("2006-01-02 15:04:05")) 50 | count = 0 51 | log.Println("开始文件抓取代理...") 52 | for i := range conf.SpiderFile { 53 | wg2.Add(1) 54 | go spiderFile(&conf.SpiderFile[i]) 55 | } 56 | wg2.Wait() 57 | log.Printf("\r%s 文件代理抓取结束 \n", time.Now().Format("2006-01-02 15:04:05")) 58 | 59 | //导出代理到文件 60 | export() 61 | 62 | } 63 | 64 | func spider(sp *Spider) { 65 | defer func() { 66 | wg2.Done() 67 | //log.Printf("%s 结束...",sp.Name) 68 | }() 69 | //log.Printf("%s 开始...", sp.Name) 70 | urls := strings.Split(sp.Urls, ",") 71 | var pis []ProxyIp 72 | for ui, v := range urls { 73 | if ui != 0 { 74 | time.Sleep(time.Duration(sp.Interval) * time.Second) 75 | } 76 | tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} 77 | if sp.ProxyIs { 78 | proxyUrl, parseErr := url.Parse("http://" + conf.Proxy.Host + ":" + conf.Proxy.Port) 79 | if parseErr != nil { 80 | log.Println("代理地址错误: \n" + parseErr.Error()) 81 | continue 82 | } 83 | tr.Proxy = http.ProxyURL(proxyUrl) 84 | } 85 | client := http.Client{Timeout: 20 * time.Second, Transport: tr} 86 | request, _ := http.NewRequest(sp.Method, v, strings.NewReader(sp.Body)) 87 | //设置请求头 88 | SetHeadersConfig(sp.Headers, &request.Header) 89 | //处理返回结果 90 | res, err := client.Do(request) 91 | if err != nil { 92 | continue 93 | } 94 | dataBytes, _ := io.ReadAll(res.Body) 95 | result := string(dataBytes) 96 | ip := regexp.MustCompile(sp.Ip).FindAllStringSubmatch(result, -1) 97 | port := regexp.MustCompile(sp.Port).FindAllStringSubmatch(result, -1) 98 | if len(ip) == 0 { 99 | continue 100 | } 101 | for i := range ip { 102 | var _ip string 103 | var _port string 104 | _ip, _ = url.QueryUnescape(ip[i][1]) 105 | _port, _ = url.QueryUnescape(port[i][1]) 106 | _is := true 107 | for pi := range ProxyPool { 108 | if ProxyPool[pi].Ip == _ip && ProxyPool[pi].Port == _port { 109 | _is = false 110 | break 111 | } 112 | } 113 | if _is { 114 | pis = append(pis, ProxyIp{Ip: _ip, Port: _port, Source: sp.Name}) 115 | } 116 | } 117 | } 118 | pis = uniquePI(pis) 119 | countAdd(len(pis)) 120 | for i := range pis { 121 | wg.Add(1) 122 | ch2 <- 1 123 | go Verify(&pis[i], &wg, ch2, true) 124 | } 125 | wg.Wait() 126 | 127 | } 128 | 129 | func spiderPlugin(spp *SpiderPlugin) { 130 | defer func() { 131 | wg2.Done() 132 | }() 133 | cmd := exec.Command("cmd.exe", "/c", spp.Run) 134 | //Start执行不会等待命令完成，Run会阻塞等待命令完成。 135 | //err := cmd.Start() 136 | //err := cmd.Run() 137 | //cmd.Output()函数的功能是运行命令并返回其标准输出。 138 | buf, err := cmd.Output() 139 | var pis []ProxyIp 140 | if err != nil { 141 | log.Println("失败", spp.Name, err) 142 | } else { 143 | _is := true 144 | line := strings.Split(string(buf), ",") 145 | for i := range line { 146 | split := strings.Split(line[i], ":") 147 | for pi := range ProxyPool { 148 | if ProxyPool[pi].Ip == split[0] && ProxyPool[pi].Port == split[1] { 149 | _is = false 150 | break 151 | } 152 | } 153 | if _is { 154 | pis = append(pis, ProxyIp{Ip: split[0], Port: split[1], Source: spp.Name}) 155 | } 156 | } 157 | //var _pis []ProxyIp 158 | //var pis []ProxyIp 159 | //var _is = true 160 | //err = json.Unmarshal(buf, &_pis) 161 | //if err != nil { 162 | // log.Printf("%s 返回值不符合规范\n", spp.Name) 163 | // return 164 | //} 165 | //for i := range _pis { 166 | // for pi := range ProxyPool { 167 | // if ProxyPool[pi].Ip == _pis[i].Ip && ProxyPool[pi].Port == _pis[i].Port { 168 | // _is = false 169 | // break 170 | // } 171 | // } 172 | // if _is { 173 | // pis = append(pis, ProxyIp{Ip: _pis[i].Ip, Port: _pis[i].Port, Source: spp.Name}) 174 | // } 175 | } 176 | pis = uniquePI(pis) 177 | countAdd(len(pis)) 178 | for i := range pis { 179 | wg.Add(1) 180 | ch2 <- 1 181 | go Verify(&pis[i], &wg, ch2, true) 182 | } 183 | wg.Wait() 184 | } 185 | 186 | func spiderFile(spp *SpiderFile) { 187 | defer func() { 188 | wg2.Done() 189 | }() 190 | var pis []ProxyIp 191 | fi, err := os.Open(spp.Path) 192 | if err != nil { 193 | log.Println(spp.Name, "失败", err) 194 | return 195 | } 196 | r := bufio.NewReader(fi) // 创建 Reader 197 | for { 198 | _is := true 199 | line, err := r.ReadBytes('\n') 200 | if len(line) > 0 { 201 | split := strings.Split(strings.TrimSpace(string(line)), ":") 202 | for pi := range ProxyPool { 203 | if ProxyPool[pi].Ip == split[0] && ProxyPool[pi].Port == split[1] { 204 | _is = false 205 | break 206 | } 207 | } 208 | if _is { 209 | pis = append(pis, ProxyIp{Ip: split[0], Port: split[1], Source: spp.Name}) 210 | } 211 | } 212 | if err != nil { 213 | break 214 | } 215 | } 216 | pis = uniquePI(pis) 217 | countAdd(len(pis)) 218 | for i := range pis { 219 | wg.Add(1) 220 | ch2 <- 1 221 | go Verify(&pis[i], &wg, ch2, true) 222 | } 223 | wg.Wait() 224 | 225 | } 226 | -------------------------------------------------------------------------------- /tunnelProxy.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "crypto/tls" 6 | "fmt" 7 | "io" 8 | "log" 9 | "net" 10 | "net/http" 11 | "net/url" 12 | "sync" 13 | "time" 14 | ) 15 | 16 | var lock2 sync.Mutex 17 | var httpI []ProxyIp 18 | var httpS []ProxyIp 19 | var socket5 []ProxyIp 20 | 21 | var httpIp string 22 | var httpsIp string 23 | var socket5Ip string 24 | 25 | func httpSRunTunnelProxyServer() { 26 | httpsIp = getHttpsIp() 27 | httpIp = gethttpIp() 28 | 29 | log.Println("HTTP 隧道代理启动 - 监听IP端口 -> ", conf.Config.Ip+":"+conf.Config.HttpTunnelPort) 30 | 31 | server := &http.Server{ 32 | Addr: conf.Config.Ip + ":" + conf.Config.HttpTunnelPort, 33 | TLSConfig: &tls.Config{InsecureSkipVerify: true}, 34 | Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 35 | 36 | if r.Method == http.MethodConnect { 37 | log.Printf("隧道代理 | HTTPS 请求：%s 使用代理: %s", r.URL.String(), httpsIp) 38 | destConn, err := net.DialTimeout("tcp", httpsIp, 20*time.Second) 39 | if err != nil { 40 | http.Error(w, err.Error(), http.StatusServiceUnavailable) 41 | return 42 | } 43 | destConn.SetReadDeadline(time.Now().Add(20 * time.Second)) 44 | var req []byte 45 | req = MergeArray([]byte(fmt.Sprintf("%s %s %s%s", r.Method, r.Host, r.Proto, []byte{13, 10})), []byte(fmt.Sprintf("Host: %s%s", r.Host, []byte{13, 10}))) 46 | for k, v := range r.Header { 47 | req = MergeArray(req, []byte(fmt.Sprintf( 48 | "%s: %s%s", k, v[0], []byte{13, 10}))) 49 | } 50 | req = MergeArray(req, []byte{13, 10}) 51 | io.ReadAll(r.Body) 52 | all, err := io.ReadAll(r.Body) 53 | if err == nil { 54 | req = MergeArray(req, all) 55 | } 56 | destConn.Write(req) 57 | w.WriteHeader(http.StatusOK) 58 | hijacker, ok := w.(http.Hijacker) 59 | if !ok { 60 | http.Error(w, "not supported", http.StatusInternalServerError) 61 | return 62 | } 63 | clientConn, _, err := hijacker.Hijack() 64 | if err != nil { 65 | return 66 | } 67 | clientConn.SetReadDeadline(time.Now().Add(20 * time.Second)) 68 | destConn.Read(make([]byte, 1024)) //先读取一次 69 | go io.Copy(destConn, clientConn) 70 | go io.Copy(clientConn, destConn) 71 | 72 | } else { 73 | log.Printf("隧道代理 | HTTP 请求：%s 使用代理: %s", r.URL.String(), httpIp) 74 | tr := &http.Transport{ 75 | TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, 76 | } 77 | //配置代理 78 | proxyUrl, parseErr := url.Parse("http://" + httpIp) 79 | if parseErr != nil { 80 | return 81 | } 82 | tr.Proxy = http.ProxyURL(proxyUrl) 83 | client := &http.Client{Timeout: 20 * time.Second, Transport: tr} 84 | request, err := http.NewRequest(r.Method, "", r.Body) 85 | //增加header选项 86 | request.URL = r.URL 87 | request.Header = r.Header 88 | //处理返回结果 89 | res, err := client.Do(request) 90 | if err != nil { 91 | http.Error(w, err.Error(), http.StatusServiceUnavailable) 92 | return 93 | } 94 | defer res.Body.Close() 95 | 96 | for k, vv := range res.Header { 97 | for _, v := range vv { 98 | w.Header().Add(k, v) 99 | } 100 | } 101 | var bodyBytes []byte 102 | bodyBytes, _ = io.ReadAll(res.Body) 103 | res.Body = io.NopCloser(bytes.NewBuffer(bodyBytes)) 104 | w.WriteHeader(res.StatusCode) 105 | io.Copy(w, res.Body) 106 | res.Body.Close() 107 | 108 | } 109 | }), 110 | } 111 | err := server.ListenAndServe() 112 | if err != nil { 113 | panic(err) 114 | } 115 | } 116 | 117 | func socket5RunTunnelProxyServer() { 118 | socket5Ip = getSocket5Ip() 119 | log.Println("SOCKET5 隧道代理启动 - 监听IP端口 -> ", conf.Config.Ip+":"+conf.Config.SocketTunnelPort) 120 | li, err := net.Listen("tcp", conf.Config.Ip+":"+conf.Config.SocketTunnelPort) 121 | if err != nil { 122 | log.Println(err) 123 | } 124 | for { 125 | clientConn, err := li.Accept() 126 | if err != nil { 127 | log.Panic(err) 128 | } 129 | go func() { 130 | log.Printf("隧道代理 | SOCKET5 请求 使用代理: %s", socket5Ip) 131 | if clientConn == nil { 132 | return 133 | } 134 | defer clientConn.Close() 135 | destConn, err := net.DialTimeout("tcp", socket5Ip, 30*time.Second) 136 | if err != nil { 137 | log.Println(err) 138 | return 139 | } 140 | defer destConn.Close() 141 | 142 | go io.Copy(destConn, clientConn) 143 | io.Copy(clientConn, destConn) 144 | }() 145 | } 146 | 147 | } 148 | 149 | // MergeArray 合并数组 150 | func MergeArray(dest []byte, src []byte) (result []byte) { 151 | result = make([]byte, len(dest)+len(src)) 152 | //将第一个数组传入result 153 | copy(result, dest) 154 | //将第二个数组接在尾部，也就是 len(dest): 155 | copy(result[len(dest):], src) 156 | return 157 | } 158 | 159 | func gethttpIp() string { 160 | lock2.Lock() 161 | defer lock2.Unlock() 162 | if len(ProxyPool) == 0 { 163 | return "" 164 | } 165 | for _, v := range ProxyPool { 166 | if v.Type == "HTTP" { 167 | is := true 168 | for _, vv := range httpI { 169 | if v.Ip == vv.Ip && v.Port == vv.Port { 170 | is = false 171 | } 172 | } 173 | if is { 174 | httpI = append(httpI, v) 175 | return v.Ip + ":" + v.Port 176 | } 177 | } 178 | } 179 | var addr string 180 | if len(httpI) != 0 { 181 | addr = httpI[0].Ip + ":" + httpI[0].Port 182 | } 183 | httpI = make([]ProxyIp, 0) 184 | if addr == "" { 185 | addr = httpsIp 186 | } 187 | return addr 188 | } 189 | 190 | func getHttpsIp() string { 191 | lock2.Lock() 192 | defer lock2.Unlock() 193 | if len(ProxyPool) == 0 { 194 | return "" 195 | } 196 | for _, v := range ProxyPool { 197 | if v.Type == "HTTPS" { 198 | is := true 199 | for _, vv := range httpS { 200 | if v.Ip == vv.Ip && v.Port == vv.Port { 201 | is = false 202 | } 203 | } 204 | if is { 205 | httpS = append(httpS, v) 206 | return v.Ip + ":" + v.Port 207 | } 208 | } 209 | } 210 | var addr string 211 | if len(httpS) != 0 { 212 | addr = httpS[0].Ip + ":" + httpS[0].Port 213 | } 214 | httpS = make([]ProxyIp, 0) 215 | return addr 216 | } 217 | func getSocket5Ip() string { 218 | lock2.Lock() 219 | defer lock2.Unlock() 220 | if len(ProxyPool) == 0 { 221 | return "" 222 | } 223 | for _, v := range ProxyPool { 224 | if v.Type == "SOCKET5" { 225 | is := true 226 | for _, vv := range socket5 { 227 | if v.Ip == vv.Ip && v.Port == vv.Port { 228 | is = false 229 | } 230 | } 231 | if is { 232 | socket5 = append(socket5, v) 233 | return v.Ip + ":" + v.Port 234 | } 235 | } 236 | } 237 | var addr string 238 | if len(socket5) != 0 { 239 | addr = socket5[0].Ip + ":" + socket5[0].Port 240 | } 241 | socket5 = make([]ProxyIp, 0) 242 | return addr 243 | } 244 | -------------------------------------------------------------------------------- /verify.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "crypto/tls" 5 | "fmt" 6 | "io" 7 | "log" 8 | "net" 9 | "net/http" 10 | "net/url" 11 | "regexp" 12 | "strings" 13 | "sync" 14 | "time" 15 | ) 16 | 17 | var verifyIS = false 18 | var ProxyPool []ProxyIp 19 | var lock sync.Mutex 20 | var mux2 sync.Mutex 21 | 22 | var count int 23 | 24 | func countAdd(i int) { 25 | mux2.Lock() 26 | count += i 27 | mux2.Unlock() 28 | 29 | } 30 | func countDel() { 31 | mux2.Lock() 32 | fmt.Printf("\r代理验证中: %d ", count) 33 | count-- 34 | mux2.Unlock() 35 | 36 | } 37 | func Verify(pi *ProxyIp, wg *sync.WaitGroup, ch chan int, first bool) { 38 | defer func() { 39 | wg.Done() 40 | countDel() 41 | <-ch 42 | }() 43 | pr := pi.Ip + ":" + pi.Port 44 | //是抓取验证，还是验证代理池内IP 45 | startT := time.Now() 46 | if first { 47 | if VerifyHttps(pr) { 48 | pi.Type = "HTTPS" 49 | } else if VerifyHttp(pr) { 50 | pi.Type = "HTTP" 51 | 52 | } else if VerifySocket5(pr) { 53 | pi.Type = "SOCKET5" 54 | } else { 55 | return 56 | } 57 | tc := time.Since(startT) 58 | pi.Time = time.Now().Format("2006-01-02 15:04:05") 59 | pi.Speed = fmt.Sprintf("%s", tc) 60 | anonymity := Anonymity(pi, 0) 61 | if anonymity == "" { 62 | return 63 | } 64 | pi.Anonymity = anonymity 65 | } else { 66 | pi.RequestNum++ 67 | if pi.Type == "HTTPS" { 68 | if VerifyHttps(pr) { 69 | pi.SuccessNum++ 70 | } 71 | } else if pi.Type == "HTTP" { 72 | if VerifyHttp(pr) { 73 | pi.SuccessNum++ 74 | } 75 | } else if pi.Type == "SOCKET5" { 76 | if VerifySocket5(pr) { 77 | pi.SuccessNum++ 78 | } 79 | } 80 | tc := time.Since(startT) 81 | pi.Time = time.Now().Format("2006-01-02 15:04:05") 82 | pi.Speed = fmt.Sprintf("%s", tc) 83 | return 84 | } 85 | tr := http.Transport{ 86 | TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, 87 | } 88 | client := http.Client{Timeout: 15 * time.Second, Transport: &tr} 89 | //处理返回结果 90 | res, err := client.Get("https://searchplugin.csdn.net/api/v1/ip/get?ip=" + pi.Ip) 91 | if err != nil { 92 | res, err = client.Get("https://searchplugin.csdn.net/api/v1/ip/get?ip=" + pi.Ip) 93 | if err != nil { 94 | return 95 | } 96 | } 97 | defer res.Body.Close() 98 | dataBytes, _ := io.ReadAll(res.Body) 99 | result := string(dataBytes) 100 | address := regexp.MustCompile("\"address\":\"(.+?)\",").FindAllStringSubmatch(result, -1) 101 | if len(address) != 0 { 102 | addresss := removeDuplication_map(strings.Split(address[0][1], " ")) 103 | le := len(addresss) 104 | pi.Isp = strings.Split(addresss[le-1], "/")[0] 105 | for i := range addresss { 106 | if i == le-1 { 107 | break 108 | } 109 | switch i { 110 | case 0: 111 | pi.Country = addresss[0] 112 | case 1: 113 | pi.Province = addresss[1] 114 | case 2: 115 | pi.City = addresss[2] 116 | } 117 | } 118 | } 119 | 120 | pi.RequestNum = 1 121 | pi.SuccessNum = 1 122 | PIAdd(pi) 123 | } 124 | func VerifyHttp(pr string) bool { 125 | proxyUrl, proxyErr := url.Parse("http://" + pr) 126 | if proxyErr != nil { 127 | return false 128 | } 129 | tr := http.Transport{ 130 | TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, 131 | } 132 | tr.Proxy = http.ProxyURL(proxyUrl) 133 | client := http.Client{Timeout: 10 * time.Second, Transport: &tr} 134 | request, err := http.NewRequest("GET", "http://baidu.com", nil) 135 | //处理返回结果 136 | res, err := client.Do(request) 137 | if err != nil { 138 | return false 139 | } 140 | defer res.Body.Close() 141 | dataBytes, _ := io.ReadAll(res.Body) 142 | result := string(dataBytes) 143 | if strings.Contains(result, "0;url=http://www.baidu.com") { 144 | return true 145 | } 146 | return false 147 | } 148 | func VerifyHttps(pr string) bool { 149 | destConn, err := net.DialTimeout("tcp", pr, 10*time.Second) 150 | if err != nil { 151 | return false 152 | } 153 | defer destConn.Close() 154 | req := []byte{67, 79, 78, 78, 69, 67, 84, 32, 119, 119, 119, 46, 98, 97, 105, 100, 117, 46, 99, 111, 109, 58, 52, 52, 51, 32, 72, 84, 84, 80, 47, 49, 46, 49, 13, 10, 72, 111, 115, 116, 58, 32, 119, 119, 119, 46, 98, 97, 105, 100, 117, 46, 99, 111, 109, 58, 52, 52, 51, 13, 10, 85, 115, 101, 114, 45, 65, 103, 101, 110, 116, 58, 32, 71, 111, 45, 104, 116, 116, 112, 45, 99, 108, 105, 101, 110, 116, 47, 49, 46, 49, 13, 10, 13, 10} 155 | destConn.Write(req) 156 | bytes := make([]byte, 1024) 157 | destConn.SetReadDeadline(time.Now().Add(10 * time.Second)) 158 | read, err := destConn.Read(bytes) 159 | if strings.Contains(string(bytes[:read]), "200 Connection established") { 160 | return true 161 | } 162 | return false 163 | } 164 | 165 | func VerifySocket5(pr string) bool { 166 | destConn, err := net.DialTimeout("tcp", pr, 10*time.Second) 167 | if err != nil { 168 | return false 169 | } 170 | defer destConn.Close() 171 | req := []byte{0x05, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00} 172 | destConn.Write(req) 173 | bytes := make([]byte, 1024) 174 | destConn.SetReadDeadline(time.Now().Add(10 * time.Second)) 175 | _, err = destConn.Read(bytes) 176 | if err != nil { 177 | return false 178 | } 179 | if bytes[0] == 5 && bytes[1] == 255 { 180 | return true 181 | } 182 | return false 183 | 184 | } 185 | func Anonymity(pr *ProxyIp, c int) string { 186 | c++ 187 | host := "http://httpbin.org/get" 188 | proxy := "" 189 | if pr.Type == "SOCKET5" { 190 | proxy = "socks5://" + pr.Ip + ":" + pr.Port 191 | } else { 192 | proxy = "http://" + pr.Ip + ":" + pr.Port 193 | } 194 | proxyUrl, proxyErr := url.Parse(proxy) 195 | if proxyErr != nil { 196 | if c >= 3 { 197 | return "" 198 | } 199 | return Anonymity(pr, c) 200 | } 201 | tr := http.Transport{ 202 | TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, 203 | } 204 | client := http.Client{Timeout: 15 * time.Second, Transport: &tr} 205 | tr.Proxy = http.ProxyURL(proxyUrl) 206 | request, err := http.NewRequest("GET", host, nil) 207 | request.Header.Add("Proxy-Connection", "keep-alive") 208 | //处理返回结果 209 | res, err := client.Do(request) 210 | if err != nil { 211 | if c >= 3 { 212 | return "" 213 | } 214 | return Anonymity(pr, c) 215 | } 216 | defer res.Body.Close() 217 | dataBytes, _ := io.ReadAll(res.Body) 218 | result := string(dataBytes) 219 | if !strings.Contains(result, `"url": "http://httpbin.org/`) { 220 | if c == 3 { 221 | return "" 222 | } 223 | c++ 224 | return Anonymity(pr, c) 225 | } 226 | origin := regexp.MustCompile("(\\d+?\\.\\d+?.\\d+?\\.\\d+?,.+\\d+?\\.\\d+?.\\d+?\\.\\d+?)").FindAllStringSubmatch(result, -1) 227 | if len(origin) != 0 { 228 | return "透明" 229 | } 230 | if strings.Contains(result, "keep-alive") { 231 | return "普匿" 232 | } 233 | return "高匿" 234 | } 235 | 236 | func PIAdd(pi *ProxyIp) { 237 | lock.Lock() 238 | defer lock.Unlock() 239 | for i := range ProxyPool { 240 | if ProxyPool[i].Ip == pi.Ip && ProxyPool[i].Port == pi.Port { 241 | return 242 | } 243 | } 244 | ProxyPool = append(ProxyPool, *pi) 245 | ProxyPool = uniquePI(ProxyPool) 246 | } 247 | 248 | func VerifyProxy() { 249 | if run { 250 | log.Println("代理抓取中, 无法进行代理验证") 251 | return 252 | } 253 | verifyIS = true 254 | 255 | log.Printf("开始验证代理存活情况, 验证次数是当前代理数的5倍: %d\n", len(ProxyPool)*5) 256 | for i, _ := range ProxyPool { 257 | ProxyPool[i].RequestNum = 0 258 | ProxyPool[i].SuccessNum = 0 259 | } 260 | count = len(ProxyPool) * 5 261 | 262 | for io := 0; io < 5; io++ { 263 | for i := range ProxyPool { 264 | wg3.Add(1) 265 | ch1 <- 1 266 | go Verify(&ProxyPool[i], &wg3, ch1, false) 267 | } 268 | time.Sleep(15 * time.Second) 269 | } 270 | wg3.Wait() 271 | lock.Lock() 272 | var pp []ProxyIp 273 | for i := range ProxyPool { 274 | if ProxyPool[i].SuccessNum != 0 { 275 | pp = append(pp, ProxyPool[i]) 276 | } 277 | } 278 | ProxyPool = pp 279 | export() 280 | lock.Unlock() 281 | log.Printf("\r%s 代理验证结束, 当前可用IP数: %d\n", time.Now().Format("2006-01-02 15:04:05"), len(ProxyPool)) 282 | verifyIS = false 283 | } 284 | 285 | func removeDuplication_map(arr []string) []string { 286 | set := make(map[string]struct{}, len(arr)) 287 | j := 0 288 | for _, v := range arr { 289 | _, ok := set[v] 290 | if ok { 291 | continue 292 | } 293 | set[v] = struct{}{} 294 | arr[j] = v 295 | j++ 296 | } 297 | 298 | return arr[:j] 299 | } 300 | -------------------------------------------------------------------------------- /webApiServer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "github.com/gin-gonic/gin" 7 | "log" 8 | "strconv" 9 | ) 10 | 11 | type Home struct { 12 | TunnelProxy map[string]string `yaml:"tunnelProxy" json:"tunnelProxy"` 13 | Sum int `yaml:"sum" json:"sum"` 14 | Type map[string]int `yaml:"type" json:"type"` 15 | Anonymity map[string]int `yaml:"anonymity" json:"anonymity"` 16 | Country map[string]int `yaml:"country" json:"country"` 17 | Source map[string]int `yaml:"source" json:"source"` 18 | } 19 | 20 | var record []ProxyIp 21 | 22 | func Run() { 23 | gin.SetMode(gin.ReleaseMode) 24 | r := gin.Default() 25 | //首页 26 | r.GET("/", index) 27 | 28 | //查询 29 | r.GET("/get", get) 30 | 31 | //删除 32 | r.GET("/delete", delete) 33 | 34 | //验证代理 35 | r.GET("/verify", verify) 36 | 37 | //抓取代理 38 | r.GET("/spider", spiderUp) 39 | 40 | //更换隧道代理IP 41 | r.GET("/tunnelUpdate", tunnelUpdate) 42 | 43 | log.Printf("webApi启动 - 监听IP端口 -> %s\n", conf.Config.Ip+":"+conf.Config.Port) 44 | r.Run(conf.Config.Ip + ":" + conf.Config.Port) 45 | 46 | } 47 | func index(c *gin.Context) { 48 | home := Home{Sum: len(ProxyPool), Type: make(map[string]int), Anonymity: make(map[string]int), Country: make(map[string]int), Source: make(map[string]int), TunnelProxy: make(map[string]string)} 49 | for i := range ProxyPool { 50 | home.Type[ProxyPool[i].Type] += 1 51 | home.Anonymity[ProxyPool[i].Anonymity] += 1 52 | home.Country[ProxyPool[i].Country] += 1 53 | home.Source[ProxyPool[i].Source] += 1 54 | } 55 | home.TunnelProxy["HTTP"] = httpIp 56 | home.TunnelProxy["HTTPS"] = httpsIp 57 | home.TunnelProxy["SOCKET5"] = socket5Ip 58 | jsonByte, _ := json.Marshal(&home) 59 | jsonStr := string(jsonByte) 60 | c.String(200, jsonStr) 61 | } 62 | func get(c *gin.Context) { 63 | if len(ProxyPool) == 0 { 64 | c.String(200, fmt.Sprintf("{\"code\": 200, \"msg\": \"代理池是空的\"}")) 65 | return 66 | } 67 | var prs []ProxyIp 68 | var jsonByte []byte 69 | ty := c.DefaultQuery("type", "all") 70 | an := c.DefaultQuery("anonymity", "all") 71 | re := c.DefaultQuery("country", "all") 72 | so := c.DefaultQuery("source", "all") 73 | co := c.DefaultQuery("count", "1") 74 | for _, v := range ProxyPool { 75 | if (v.Type == ty || ty == "all") && (v.Anonymity == an || an == "all") && (v.Country == re || re == "all") && (v.Source == so || so == "all") { 76 | prs = append(prs, v) 77 | } 78 | } 79 | if co == "all" { 80 | jsonByte, _ = json.Marshal(prs) 81 | } else if co == "1" { 82 | var _is bool 83 | for _, v := range prs { 84 | _is = true 85 | for _, vv := range record { 86 | if v.Ip+v.Port == vv.Ip+vv.Port { 87 | _is = false 88 | } 89 | } 90 | if _is { 91 | jsonByte, _ = json.Marshal(v) 92 | record = append(record, v) 93 | break 94 | } 95 | } 96 | if !_is { 97 | jsonByte, _ = json.Marshal(prs[0]) 98 | record = []ProxyIp{prs[0]} 99 | } 100 | } else { 101 | count, err := strconv.Atoi(co) 102 | if err != nil { 103 | c.String(500, fmt.Sprintf("{\"code\": 500, \"msg\": \"错误\"}")) 104 | } 105 | jsonByte, _ = json.Marshal(prs[:count]) 106 | } 107 | jsonStr := string(jsonByte) 108 | c.String(200, jsonStr) 109 | } 110 | func delete(c *gin.Context) { 111 | if len(ProxyPool) == 0 { 112 | c.String(200, fmt.Sprintf("{\"code\": 200, \"msg\": \"代理池是空的\"}")) 113 | return 114 | } 115 | ip := c.Query("ip") 116 | port := c.Query("port") 117 | i := delIp(ip + ":" + port) 118 | c.String(200, fmt.Sprintf("{\"code\": 200, \"count\": %d}", i)) 119 | } 120 | func verify(c *gin.Context) { 121 | if verifyIS { 122 | c.String(200, fmt.Sprintf("{\"code\": 200, \"msg\": \"验证中\"}")) 123 | } else if run { 124 | c.String(200, fmt.Sprintf("{\"code\": 200, \"msg\": \"代理抓取中，请稍后再来验证\"}")) 125 | } else { 126 | go VerifyProxy() 127 | c.String(200, fmt.Sprintf("{\"code\": 200, \"msg\": \"开始验证代理\"}")) 128 | } 129 | } 130 | 131 | func spiderUp(c *gin.Context) { 132 | if run { 133 | c.String(200, fmt.Sprintf("{\"code\": 200, \"msg\": \"抓取中\"}")) 134 | } else if verifyIS { 135 | c.String(200, fmt.Sprintf("{\"code\": 200, \"msg\": \"代理验证中，请稍后再来抓取\"}")) 136 | } else { 137 | go spiderRun() 138 | c.String(200, fmt.Sprintf("{\"code\": 200, \"msg\": \"开始抓取代理IP\"}")) 139 | } 140 | } 141 | 142 | func tunnelUpdate(c *gin.Context) { 143 | if len(ProxyPool) == 0 { 144 | c.String(200, fmt.Sprintf("{\"code\": 200, \"msg\": \"代理池是空的\"}")) 145 | } 146 | httpsIp = getHttpsIp() 147 | httpIp = gethttpIp() 148 | socket5Ip = getSocket5Ip() 149 | c.String(200, fmt.Sprintf("{\"code\": 200, \"HTTP\": \"%s\",\"HTTPS\": \"%s\",\"SOCKET5\": \"%s\" }", httpIp, httpsIp, socket5Ip)) 150 | } 151 | 152 | func delIp(addr string) int { 153 | lock.Lock() 154 | defer lock.Unlock() 155 | var in int 156 | for i, v := range ProxyPool { 157 | if v.Ip+":"+v.Port == addr { 158 | in++ 159 | if i+1 < len(ProxyPool) { 160 | ProxyPool = append(ProxyPool[:i], ProxyPool[i+1:]...) 161 | } else { 162 | ProxyPool = ProxyPool[:i] 163 | } 164 | } 165 | } 166 | return in 167 | } 168 | --------------------------------------------------------------------------------