├── .gitignore ├── Makefile ├── README.en.md ├── README.md ├── bootstrap ├── bootstrap.go ├── common.go ├── consts.go └── servers │ ├── api_server.go │ ├── job_server.go │ └── tunnel_server.go ├── cmd └── main.go ├── conf.yaml ├── config ├── conf.go ├── getter.go └── tunnel.go ├── consts ├── error_const.go ├── proxy.go └── setting.go ├── dao ├── dao.go ├── database │ ├── db.go │ └── proxy_dao.go ├── proxy_dao_interface.go └── redis │ ├── client.go │ └── proxy_dao.go ├── doc └── diagram.jpg ├── dto ├── ip_info_dto.go └── proxy_dto.go ├── global └── global.go ├── go.mod ├── go.sum ├── model └── proxy_model.go ├── scheduler ├── check_active_ip.go ├── check_fail_ip.go ├── get_ip.go ├── recheck_ip.go └── update_ip_info.go ├── service ├── common │ └── proxy.go ├── ip │ ├── local_ip_data.go │ ├── local_ip_service.go │ ├── on_line_ip_service.go │ └── service.go ├── proxy_getter │ ├── coderbusy.go │ ├── common_getter.go │ ├── data5u.go │ ├── fanqieip.go │ ├── geonode.go │ ├── getter.go │ ├── guobanjia.go │ ├── ip3366.go │ ├── jiangxianli.go │ ├── kuai.go │ ├── kxdaili.go │ ├── list_plus.go │ ├── nima.go │ ├── pachongdaili.go │ ├── proxy_66ip.go │ ├── proxy_7yip.go │ ├── proxy_89ip.go │ ├── seofangfa.go │ ├── xici.go │ ├── xila.go │ ├── xsdaili.go │ ├── yqie.go │ ├── zdaye.go │ └── zdaye_index.go ├── proxy_getter_interface.go ├── proxy_service.go ├── service.go └── tunnel │ ├── .ssl │ ├── README.md │ ├── localhost.crt │ ├── localhost.csr │ ├── localhost.key │ ├── rootCA.crt │ ├── rootCA.key │ └── rootCA.srl │ ├── cfg.go │ ├── gost.go │ └── route.go └── tests ├── check_ip_test.go ├── conf.yaml ├── config_test.go ├── get_ip_test.go ├── getter_test.go ├── local_ip_service_test.go └── test_check_active.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | # vendor/ 16 | .idea 17 | errors.log -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | checklint: 2 | ifeq (, $(shell which golangci-lint)) 3 | @echo 'error: golangci-lint is not installed, please exec `brew install golangci-lint`' 4 | @exit 1 5 | endif 6 | 7 | lint: checklint 8 | golangci-lint run --skip-dirs-use-default 9 | -------------------------------------------------------------------------------- /README.en.md: -------------------------------------------------------------------------------- 1 | ### proxy-collect - GO Proxy Collector 2 | 3 | ### A simple proxy pool written in Golang 4 | 5 | #### [中文文档](README.md) 6 | 7 | Features 8 | ------ 9 | 10 | * Support Redis or MySQL as storage 11 | * Automatically grab free proxy IP on the Internet 12 | * Regularly check IP availability and recheck failed IP 13 | * Get the list of available proxy IP through API 14 | * Tunnel proxy service: it supports opening multiple ports for monitoring, multi protocol, multi-level forwarding, user name and password verification, and user flow restriction 15 | 16 | ## Usage： 17 | 18 | #### install 19 | 20 | git clone https://github.com/tongsq/proxy-collect.git 21 | cd proxy-collect 22 | 23 | #### start service 24 | 1、Start script to collect free proxy IP 25 | 26 | go run ./cmd -S=job -C=conf.yaml 27 | 2、Start API service 28 | 29 | go run ./cmd -S=api -C=conf.yaml 30 | 2、Start Tunnel service 31 | 32 | go run ./cmd -S=tunnel -C=conf.yaml 33 | #### Get available IP through the API 34 | 35 | curl 127.0.0.1:8090/all?city=上海&duration=100 36 | 37 | #### Start multiple services 38 | 39 | go run ./cmd -S=job -S=api -C=conf.yaml 40 | #### One touch start 41 | go run ./cmd -S=all -C=conf.yaml 42 | 43 | ## yaml configure 44 | ### Set storage media 45 | A、 Set redis as storage media 46 | 47 | * Modify the configuration file conf.yaml as follows 48 | ``` 49 | dao: redis 50 | redis: 51 | MaxIdle: 10 52 | MaxActive: 20 53 | Network: tcp 54 | Address: 127.0.0.1:6379 55 | Password: your password 56 | ``` 57 | B、Set mysql as storage media 58 | 59 | * Modify the configuration file conf.yaml as follows 60 | ``` 61 | dao: database 62 | database: 63 | Dialect: mysql 64 | Args: user:password@(127.0.0.1:3306)/dbname?charset=utf8&loc=Local 65 | ``` 66 | * Create table 67 | ``` 68 | CREATE TABLE `proxy` ( 69 | `id` int(11) NOT NULL AUTO_INCREMENT, 70 | `host` varchar(255) NOT NULL DEFAULT '', 71 | `port` int(11) NOT NULL DEFAULT '0', 72 | `status` tinyint(4) NOT NULL DEFAULT '1' COMMENT '0:无效，1：有效', 73 | `create_time` int(11) NOT NULL DEFAULT '0', 74 | `update_time` int(11) NOT NULL DEFAULT '0', 75 | `active_time` int(11) NOT NULL DEFAULT '0', 76 | `country` varchar(100) NOT NULL DEFAULT '', 77 | `region` varchar(100) NOT NULL DEFAULT '', 78 | `city` varchar(100) NOT NULL DEFAULT '', 79 | `isp` varchar(255) NOT NULL DEFAULT '', 80 | `check_count` int(11) NOT NULL DEFAULT '10', 81 | `source` varchar(50) NOT NULL DEFAULT '', 82 | `proto` varchar(20) NOT NULL DEFAULT 'http', 83 | `user` varchar(50) NOT NULL DEFAULT '', 84 | `password` varchar(50) NOT NULL DEFAULT '', 85 | `ping` int(11) NOT NULL DEFAULT '0', 86 | PRIMARY KEY (`id`) USING BTREE, 87 | UNIQUE KEY `IDX_HOST_PORT_PROTO` (`host`,`port`, `proto`) USING BTREE, 88 | KEY `IDX_STATUS` (`status`) USING BTREE, 89 | KEY `IDX_ACTIVE_TIME` (`active_time`) USING BTREE 90 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8 ROW_FORMAT=COMPACT 91 | ``` 92 | # TODO list 93 | - [x] Supports proxy collection and comparison of other protocols such as socket5 94 | - [x] Support configuring log classification 95 | - [x] Support tunnel agent services 96 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | proxy-collect - GO Proxy Collector 2 | ------ 3 | ### GO语言实现的ip代理池，隧道代理 4 | 5 | #### [English README](README.en.md) 6 | ![图片](https://tongsq.github.io/proxy-collect/doc/diagram.jpg) 7 | 8 | 功能 9 | ------ 10 | 11 | * 通过api获取可用代理ip列表 12 | * 隧道代理服务：支持开启多个端口监听，支持多协议，多级转发，用户名密码较验，用户限流 13 | * 支持redis或mysql作为存储 14 | * 自动抓取互联网上的免费代理ip 15 | * 定时检测ip可用性，失效ip复检、记录ip存活时间 16 | * 获取ip归属地等信息 17 | * 配置代理超时时间 18 | 19 | ## 使用方法： 20 | 21 | #### 安装 22 | 23 | git clone https://github.com/tongsq/proxy-collect.git 24 | cd proxy-collect 25 | 26 | #### 启动服务 27 | 1、启动脚本收集免费代理ip 28 | 29 | go run ./cmd -S=job -C=conf.yaml 30 | 2、启动api服务 31 | 32 | go run ./cmd -S=api -C=conf.yaml 33 | 3、启动隧道代理服务 34 | 35 | go run ./cmd -S=tunnel -C=conf.yaml 36 | #### 通过接口获取可用ip 37 | 38 | curl 127.0.0.1:8090/all?city=上海&duration=100 39 | 40 | #### 启动多个服务 41 | 42 | go run ./cmd -S=job -S=api -C=conf.yaml 43 | #### 一键启动 44 | 45 | go run ./cmd -S=all -C=conf.yaml 46 | 47 | ## yaml配置 48 | ### 设置存储媒介 49 | A、数据存储到redis 50 | 51 | * 修改配置文件conf.yaml如下 52 | ``` 53 | dao: redis 54 | redis: 55 | MaxIdle: 10 56 | MaxActive: 20 57 | Network: tcp 58 | Address: 127.0.0.1:6379 59 | Password: your password 60 | ``` 61 | B、数据存储到mysql 62 | 63 | * 修改配置文件conf.yaml如下 64 | ``` 65 | dao: database 66 | database: 67 | Dialect: mysql 68 | Args: user:password@(127.0.0.1:3306)/dbname?charset=utf8&loc=Local&token=abc 69 | ``` 70 | * 创建表 71 | ``` 72 | CREATE TABLE `proxy` ( 73 | `id` int(11) NOT NULL AUTO_INCREMENT, 74 | `host` varchar(255) NOT NULL DEFAULT '', 75 | `port` int(11) NOT NULL DEFAULT '0', 76 | `status` tinyint(4) NOT NULL DEFAULT '1' COMMENT '0:无效，1：有效', 77 | `create_time` int(11) NOT NULL DEFAULT '0', 78 | `update_time` int(11) NOT NULL DEFAULT '0', 79 | `active_time` int(11) NOT NULL DEFAULT '0', 80 | `country` varchar(100) NOT NULL DEFAULT '', 81 | `region` varchar(100) NOT NULL DEFAULT '', 82 | `city` varchar(100) NOT NULL DEFAULT '', 83 | `isp` varchar(255) NOT NULL DEFAULT '', 84 | `check_count` int(11) NOT NULL DEFAULT '10', 85 | `source` varchar(50) NOT NULL DEFAULT '', 86 | `proto` varchar(20) NOT NULL DEFAULT 'http', 87 | `user` varchar(50) NOT NULL DEFAULT '', 88 | `password` varchar(50) NOT NULL DEFAULT '', 89 | `ping` int(11) NOT NULL DEFAULT '0', 90 | PRIMARY KEY (`id`) USING BTREE, 91 | UNIQUE KEY `IDX_HOST_PORT_PROTO` (`host`,`port`, `proto`) USING BTREE, 92 | KEY `IDX_STATUS` (`status`) USING BTREE, 93 | KEY `IDX_ACTIVE_TIME` (`active_time`) USING BTREE 94 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8 ROW_FORMAT=COMPACT 95 | ``` 96 | ### 隧道代理配置 97 | ``` 98 | tunnel: 99 | tunnelLevel: 1 //代理转发层级 100 | refresh: 10 //代理池自动刷新时间，单位秒 101 | debug: false //是否debug 102 | strategy: round //指定节点选择策略，round - 轮询，random - 随机, fifo - 自上而下。默认值为round。 103 | maxFails: 100 //指定节点连接的最大失败次数，当与一个节点建立连接失败次数超过此设定值时，此节点会被标记为死亡节点 104 | failTimeout: 10 //定死亡节点的超时时间，当一个节点被标记为死亡节点后，在此设定的时间间隔内不会被选择使用 105 | tunnels: //隧道代理监听的端口，可配多个 106 | - 107 | proto: http //协议：http、https、socks5等等 108 | host: 0.0.0.0 //host 109 | port: 8888 //监听端口 110 | users: //授权用户，可配多个, 不开启用户较验不要配这个 111 | - 112 | username: root //用户名 113 | password: 123 //密码 114 | limiter: 1,10,2 //限流规则,1:1秒内，10：可访问10次，2: 限制2个并发 115 | ``` 116 | # 待开发 117 | - [X] 支持socket5等其它协议代理采集、较验 118 | - [X] 支持配置日志分级 119 | - [X] 支持开启隧道代理服务：用户名密码验证、限流功能 120 | - [X] 记录代理响应时间 121 | 122 | # 感谢 123 | * 如果您觉得程序还不错，不妨点个星以鼓励一下，后续会更新更多ip源，谢谢！ 124 | * 如果您对程序有任何建议和意见或者有更好的免费ip源，欢迎提交issue。 125 | * 感谢大神[@ginuerzh](https://github.com/ginuerzh) 的开源项目提供隧道代理支持：[gost](https://github.com/ginuerzh/gost) 126 | -------------------------------------------------------------------------------- /bootstrap/bootstrap.go: -------------------------------------------------------------------------------- 1 | package bootstrap 2 | 3 | import ( 4 | "proxy-collect/config" 5 | "proxy-collect/dao" 6 | "proxy-collect/global" 7 | "proxy-collect/service" 8 | ) 9 | 10 | func Bootstrap() { 11 | config.StartLoadConfig() 12 | global.LoadGlobal() 13 | dao.LoadDao() 14 | service.LoadService() 15 | } 16 | -------------------------------------------------------------------------------- /bootstrap/common.go: -------------------------------------------------------------------------------- 1 | package bootstrap 2 | 3 | import "fmt" 4 | 5 | type StringList []string 6 | 7 | func (l *StringList) String() string { 8 | return fmt.Sprintf("%s", *l) 9 | } 10 | func (l *StringList) Set(value string) error { 11 | *l = append(*l, value) 12 | return nil 13 | } 14 | -------------------------------------------------------------------------------- /bootstrap/consts.go: -------------------------------------------------------------------------------- 1 | package bootstrap 2 | 3 | const ServerALl = "all" 4 | const ServerApi = "api" 5 | const ServerJob = "job" 6 | const ServerTunnel = "tunnel" 7 | -------------------------------------------------------------------------------- /bootstrap/servers/api_server.go: -------------------------------------------------------------------------------- 1 | package servers 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | "strings" 7 | "time" 8 | 9 | "github.com/gin-gonic/gin" 10 | "github.com/tongsq/go-lib/logger" 11 | "proxy-collect/config" 12 | "proxy-collect/dao" 13 | "proxy-collect/dto" 14 | ) 15 | 16 | func StartApiServer() { 17 | r := gin.Default() 18 | router := r.Use(TokenAuthMiddleware) 19 | router.GET("/ping", func(c *gin.Context) { 20 | c.JSON(200, gin.H{ 21 | "message": "pong", 22 | }) 23 | }) 24 | router.GET("/all", func(c *gin.Context) { 25 | if !config.SystemStart { 26 | c.JSON(403, gin.H{ 27 | "code": 403, 28 | "msg": "System not start", 29 | }) 30 | return 31 | } 32 | proxies, err := dao.ProxyDao.GetActiveList() 33 | if err != nil { 34 | logger.Error("get active proxy fail", logger.Fields{"err": err}) 35 | c.JSON(200, gin.H{ 36 | "data": []string{}, 37 | "code": 0, 38 | "count": 0, 39 | }) 40 | return 41 | } 42 | city := c.Query("city") 43 | var durationMin, durationMax int64 44 | duration := c.Query("duration") 45 | if duration != "" { 46 | if strings.Contains(duration, "-") { 47 | strs := strings.Split(duration, "-") 48 | if len(strs) >= 2 { 49 | durationMin, _ = strconv.ParseInt(strs[0], 10, 64) 50 | durationMax, _ = strconv.ParseInt(strs[1], 10, 64) 51 | } 52 | } else { 53 | durationMin, _ = strconv.ParseInt(duration, 10, 64) 54 | } 55 | } 56 | proto := c.Query("proto") 57 | count := c.Query("count") 58 | countNum, _ := strconv.Atoi(count) 59 | var list []dto.ProxyInfoDto 60 | nowTime := time.Now().Unix() 61 | for _, proxy := range proxies { 62 | if city != "" && !strings.Contains(proxy.City, city) { 63 | continue 64 | } 65 | if durationMin > 0 && ((nowTime - proxy.ActiveTime) < durationMin) { 66 | continue 67 | } 68 | if durationMax > 0 && ((nowTime - proxy.ActiveTime) > durationMax) { 69 | continue 70 | } 71 | if proto != "" && proto != proxy.Proto { 72 | continue 73 | } 74 | list = append(list, dto.NewProxyDto(proxy)) 75 | if countNum > 0 && len(list) >= countNum { 76 | break 77 | } 78 | } 79 | c.JSON(200, gin.H{ 80 | "data": list, 81 | "code": 0, 82 | "count": len(list), 83 | }) 84 | }) 85 | router.GET("/start", func(c *gin.Context) { 86 | config.SystemStart = true 87 | c.JSON(200, gin.H{ 88 | "message": "success", 89 | }) 90 | }) 91 | router.GET("/stop", func(c *gin.Context) { 92 | config.SystemStart = false 93 | c.JSON(200, gin.H{ 94 | "message": "success", 95 | }) 96 | }) 97 | err := r.Run(fmt.Sprintf("%s:%s", config.Get().Api.Host, config.Get().Api.Port)) 98 | if err != nil { 99 | panic("start api server fail:" + err.Error()) 100 | } 101 | } 102 | 103 | func TokenAuthMiddleware(c *gin.Context) { 104 | tokenSecret := config.Get().Api.Token 105 | if tokenSecret != "" { 106 | token := c.Query("token") 107 | if token == "" { 108 | token = c.PostForm("token") 109 | } 110 | if token != tokenSecret { 111 | c.JSON(401, gin.H{ 112 | "code": 401, 113 | "msg": "API token required", 114 | }) 115 | c.Abort() 116 | return 117 | } 118 | } 119 | c.Next() 120 | } 121 | -------------------------------------------------------------------------------- /bootstrap/servers/job_server.go: -------------------------------------------------------------------------------- 1 | package servers 2 | 3 | import ( 4 | "github.com/robfig/cron/v3" 5 | "github.com/tongsq/go-lib/logger" 6 | "proxy-collect/scheduler" 7 | "proxy-collect/service/ip" 8 | ) 9 | 10 | type JobItem struct { 11 | Spec string 12 | Job cron.Job 13 | } 14 | 15 | func StartJobServer() { 16 | c := cron.New() 17 | jobs := []JobItem{ 18 | {"*/3 * * * *", scheduler.GetIp{}}, 19 | {"*/2 * * * *", scheduler.CheckActiveIp{}}, 20 | {"@every 5h", scheduler.CheckFailIp{}}, 21 | {"@every 5m", scheduler.UpdateIpInfo{}}, 22 | {"@every 1m", scheduler.RecheckIp{}}, 23 | } 24 | for _, job := range jobs { 25 | _, err := c.AddJob(job.Spec, job.Job) 26 | if err != nil { 27 | logger.Error("start job fail", map[string]interface{}{"job": job}) 28 | } 29 | } 30 | 31 | //update local ip database 32 | _, err := c.AddFunc("2 2 * * *", func() { 33 | ip.UpdateLocalIpData() 34 | }) 35 | if err != nil { 36 | logger.Error("start job fail", map[string]interface{}{}) 37 | } 38 | c.Start() 39 | defer c.Stop() 40 | scheduler.GetIp{}.Run() 41 | select {} 42 | } 43 | -------------------------------------------------------------------------------- /bootstrap/servers/tunnel_server.go: -------------------------------------------------------------------------------- 1 | package servers 2 | 3 | import "proxy-collect/service/tunnel" 4 | 5 | func StartTunnelServer() { 6 | tunnel.StartTunnels() 7 | } 8 | -------------------------------------------------------------------------------- /cmd/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "os" 6 | 7 | "github.com/tongsq/go-lib/logger" 8 | "proxy-collect/bootstrap" 9 | "proxy-collect/bootstrap/servers" 10 | "proxy-collect/config" 11 | ) 12 | 13 | var ( 14 | configFile string 15 | serverList bootstrap.StringList 16 | ) 17 | 18 | func main() { 19 | flag.Var(&serverList, "S", "start server type") 20 | flag.StringVar(&configFile, "C", "conf.yaml", "") 21 | flag.Parse() 22 | if flag.NFlag() == 0 || len(serverList) == 0 { 23 | flag.PrintDefaults() 24 | os.Exit(0) 25 | } 26 | config.YamlPath = configFile 27 | //init 28 | bootstrap.Bootstrap() 29 | 30 | for _, server := range serverList { 31 | if server == bootstrap.ServerALl { 32 | go servers.StartApiServer() 33 | go servers.StartJobServer() 34 | go servers.StartTunnelServer() 35 | break 36 | } else if server == bootstrap.ServerJob { 37 | go servers.StartJobServer() 38 | } else if server == bootstrap.ServerApi { 39 | go servers.StartApiServer() 40 | } else if server == bootstrap.ServerTunnel { 41 | go servers.StartTunnelServer() 42 | } else { 43 | logger.Error("unknown server", nil) 44 | os.Exit(1) 45 | } 46 | } 47 | select {} 48 | } 49 | -------------------------------------------------------------------------------- /conf.yaml: -------------------------------------------------------------------------------- 1 | dao: redis #set storage type -> redis|database 2 | redis: #redis config 3 | MaxIdle: 0 4 | MaxActive: 10 5 | Network: tcp 6 | Address: 127.0.0.1:6379 7 | Password: 8 | database: #database config 9 | Dialect: mysql 10 | Args: python:123456@(127.0.0.1:3306)/py?charset=utf8&loc=Local 11 | MaxIdle: 5 #max idle connections count 12 | MaxOpen: 20 #max connections count 13 | api: 14 | host: 0.0.0.0 15 | port: 8090 16 | token: abc #调用api的token，不用较验可不配置 17 | poolSize: 1000 18 | localIpDataPath: /tmp/local_ip_data.dat 19 | recheckCount: 10 #recheck count if ip is invalid 20 | maxPing: 5000 #max timeout duration, Millisecond 21 | updateIpInfo: true #是否自动获取ip归属地等信息 22 | log: 23 | logLevel: 4 #0:Panic 1: Fatal 2:error 3:warning 4:info 5:debug 6:trace 24 | errorLogFile: errors.log 25 | tunnel: 26 | tunnelLevel: 1 27 | refresh: 20 28 | debug: false 29 | strategy: round 30 | maxFails: 100 31 | failTimeout: 10 32 | tunnels: 33 | - 34 | proto: http 35 | host: 0.0.0.0 36 | port: 8888 37 | users: 38 | - 39 | username: root 40 | password: 123 41 | limiter: 1,10,2 42 | getters: 43 | - 44 | method: GET 45 | regexp: (\d+\.\d+\.\d+\.\d+):(\d+) 46 | proto: http #获取到的代理类型 47 | proxy: false #是否通过代理获取 48 | agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36 49 | urls: 50 | - 51 | https://www.zdaye.com/dayProxy/1.html -------------------------------------------------------------------------------- /config/conf.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "fmt" 5 | "github.com/tongsq/go-lib/logger" 6 | "gopkg.in/yaml.v2" 7 | "io/ioutil" 8 | "time" 9 | ) 10 | 11 | var conf *ConfDto 12 | var YamlPath = "conf.yaml" 13 | var configRefreshHandlers []func(old, new *ConfDto) 14 | var SystemStart = true 15 | 16 | func Get() *ConfDto { 17 | if conf == nil { 18 | yamls, err := ioutil.ReadFile(YamlPath) 19 | if err != nil { 20 | ReadConfigError(err) 21 | } 22 | c := ConfDto{} 23 | err = yaml.Unmarshal(yamls, &c) 24 | if err != nil { 25 | ReadConfigError(err) 26 | } 27 | conf = &c 28 | logger.Info("load config success", logger.Fields{"conf": conf}) 29 | } 30 | return conf 31 | } 32 | 33 | func StartLoadConfig() { 34 | LoadConfig() 35 | go func() { 36 | t := time.NewTicker(time.Second * 10) 37 | for { 38 | <-t.C 39 | LoadConfig() 40 | } 41 | }() 42 | } 43 | 44 | func RegisterConfigRefreshHandler(f func(old, new *ConfDto)) { 45 | configRefreshHandlers = append(configRefreshHandlers, f) 46 | } 47 | 48 | func LoadConfig() { 49 | yamls, err := ioutil.ReadFile(YamlPath) 50 | if err != nil { 51 | ReadConfigError(err) 52 | } 53 | c := ConfDto{} 54 | err = yaml.Unmarshal(yamls, &c) 55 | if err != nil { 56 | ReadConfigError(err) 57 | } 58 | if conf != nil { 59 | for _, h := range configRefreshHandlers { 60 | h(conf, &c) 61 | } 62 | } 63 | conf = &c 64 | logger.Debug("load config success", logger.Fields{"conf": conf}) 65 | } 66 | 67 | func ReadConfigError(err error) { 68 | logger.Error("read config file error", logger.Fields{"err": err, "path": YamlPath}) 69 | panic("read config file error") 70 | } 71 | 72 | type ConfDto struct { 73 | Dao string `yaml:"dao"` 74 | Redis struct { 75 | MaxIdle int `yaml:"MaxIdle"` 76 | MaxActive int `yaml:"MaxActive"` 77 | Network string `yaml:"Network"` 78 | Address string `yaml:"Address"` 79 | Password string `yaml:"Password"` 80 | } 81 | Database struct { 82 | Dialect string `yaml:"Dialect"` 83 | Args string `yaml:"Args"` 84 | MaxIdle int `yaml:"MaxIdle"` 85 | MaxOpen int `yaml:"MaxOpen"` 86 | } 87 | Api struct { 88 | Host string `yaml:"host"` 89 | Port string `yaml:"port"` 90 | Token string `yaml:"token"` 91 | } 92 | PoolSize int `yaml:"poolSize"` 93 | LocalIpDataPath string `yaml:"localIpDataPath"` 94 | RecheckCount int64 `yaml:"recheckCount"` 95 | MaxPing int64 `yaml:"maxPing"` 96 | UpdateIpInfo bool `yaml:"updateIpInfo"` 97 | Log struct { 98 | LogLevel logger.Level `yaml:"logLevel"` 99 | ErrorLogFile string `yaml:"errorLogFile"` 100 | } 101 | Tunnel struct { 102 | TunnelLevel int `yaml:"tunnelLevel"` 103 | Refresh int64 `yaml:"refresh"` 104 | Debug bool `yaml:"debug"` 105 | Strategy string `yaml:"strategy"` 106 | MaxFails int `yaml:"maxFails"` 107 | FailTimeout int `yaml:"failTimeout"` 108 | } 109 | Tunnels []TunnelConfig `yaml:"tunnels"` 110 | Getters []Getter `yaml:"getters"` 111 | } 112 | 113 | func (c ConfDto) String() string { 114 | return fmt.Sprintf("%#v", c) 115 | } 116 | -------------------------------------------------------------------------------- /config/getter.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | type Getter struct { 4 | Method string `yaml:"method"` 5 | Agent string `yaml:"agent"` 6 | Regexp string `yaml:"regexp"` 7 | Proto string `yaml:"proto"` 8 | Urls []string `yaml:"urls"` 9 | Proxy bool `yaml:"proxy"` 10 | } 11 | -------------------------------------------------------------------------------- /config/tunnel.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | type TunnelConfig struct { 4 | Proto string `yaml:"proto"` 5 | Host string `yaml:"host"` 6 | Port string `yaml:"port"` 7 | Users []TunnelUserConfig `yaml:"users"` 8 | } 9 | 10 | type TunnelUserConfig struct { 11 | Username string `yaml:"username"` 12 | Password string `yaml:"password"` 13 | Limiter string `yaml:"limiter"` 14 | } 15 | -------------------------------------------------------------------------------- /consts/error_const.go: -------------------------------------------------------------------------------- 1 | package consts 2 | 3 | const PROXY_FORMAT_ERROR = "proxy ip format error" 4 | const GO_QUERY_READ_ERROR = "goquery parse html fail" 5 | -------------------------------------------------------------------------------- /consts/proxy.go: -------------------------------------------------------------------------------- 1 | package consts 2 | 3 | const STATUS_NO int8 = 0 4 | const STATUS_YES int8 = 1 5 | const STATUS_RECHECK int8 = 2 6 | 7 | const PROTO_HTTP = "http" 8 | const PROTO_HTTPS = "https" 9 | const PROTO_SOCKS4 = "socks4" 10 | const PROTO_SOCKS4A = "socks4a" 11 | const PROTO_SOCKS5 = "socks5" 12 | const PROTO_SS = "ss" 13 | 14 | // protocols 15 | var PROTO_LIST = []string{ 16 | PROTO_HTTP, 17 | PROTO_SOCKS5, 18 | PROTO_HTTPS, 19 | PROTO_SOCKS4, 20 | PROTO_SOCKS4A, 21 | PROTO_SS, 22 | } 23 | -------------------------------------------------------------------------------- /consts/setting.go: -------------------------------------------------------------------------------- 1 | package consts 2 | 3 | const USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36" 4 | -------------------------------------------------------------------------------- /dao/dao.go: -------------------------------------------------------------------------------- 1 | package dao 2 | 3 | import ( 4 | "proxy-collect/config" 5 | "proxy-collect/dao/database" 6 | "proxy-collect/dao/redis" 7 | ) 8 | 9 | var ProxyDao proxyDaoInterface 10 | 11 | func LoadDao() { 12 | c := config.Get().Dao 13 | if c == "redis" { 14 | ProxyDao = redis.NewRedisProxyDao() 15 | } else { 16 | ProxyDao = database.NewMysqlProxyDao() 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /dao/database/db.go: -------------------------------------------------------------------------------- 1 | package database 2 | 3 | import ( 4 | "github.com/jinzhu/gorm" 5 | "proxy-collect/config" 6 | ) 7 | import _ "github.com/jinzhu/gorm/dialects/mysql" 8 | 9 | var ( 10 | db *gorm.DB 11 | ) 12 | 13 | func DB() *gorm.DB { 14 | if db == nil { 15 | db = NewDB() 16 | } 17 | return db 18 | } 19 | 20 | func NewDB() (db *gorm.DB) { 21 | c := config.Get().Database 22 | db, err := gorm.Open(c.Dialect, c.Args) 23 | if err != nil { 24 | panic(err) 25 | } 26 | db.SingularTable(true) 27 | db.DB().SetMaxOpenConns(c.MaxOpen) 28 | db.DB().SetMaxIdleConns(c.MaxIdle) 29 | return db 30 | } 31 | -------------------------------------------------------------------------------- /dao/database/proxy_dao.go: -------------------------------------------------------------------------------- 1 | package database 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/jinzhu/gorm" 7 | "proxy-collect/consts" 8 | "proxy-collect/dto" 9 | "proxy-collect/model" 10 | ) 11 | 12 | func NewMysqlProxyDao() *proxyDao { 13 | return &proxyDao{} 14 | } 15 | 16 | type proxyDao struct { 17 | } 18 | 19 | func (d *proxyDao) GetFailList() ([]model.ProxyModel, error) { 20 | var proxies []model.ProxyModel 21 | DB().Where("status=?", consts.STATUS_NO).Find(&proxies) 22 | return proxies, nil 23 | } 24 | 25 | func (d *proxyDao) GetRecheckList() ([]model.ProxyModel, error) { 26 | var proxies []model.ProxyModel 27 | DB().Where("status=?", consts.STATUS_RECHECK).Where("check_count>0").Find(&proxies) 28 | return proxies, nil 29 | } 30 | 31 | func (d *proxyDao) GetOne(host string, port string, proto string) (*model.ProxyModel, error) { 32 | var m model.ProxyModel 33 | db := DB() 34 | err := db.Where("host = ? AND port = ? AND proto = ?", host, port, proto).First(&m).Error 35 | if err != nil { 36 | if gorm.IsRecordNotFoundError(err) { 37 | return nil, nil 38 | } 39 | return nil, err 40 | } 41 | return &m, nil 42 | } 43 | 44 | func (d *proxyDao) Create(proxy dto.ProxyDto, status int8) (*model.ProxyModel, error) { 45 | m := &model.ProxyModel{ 46 | Host: proxy.Host, 47 | Port: proxy.Port, 48 | Status: status, 49 | ActiveTime: time.Now().Unix(), 50 | CreateTime: time.Now().Unix(), 51 | UpdateTime: time.Now().Unix(), 52 | CheckCount: 1, 53 | Source: proxy.Source, 54 | Proto: proxy.Proto, 55 | User: proxy.User, 56 | Password: proxy.Password, 57 | Ping: proxy.Ping, 58 | } 59 | DB().Create(m) 60 | return m, nil 61 | } 62 | 63 | func (d *proxyDao) Save(m *model.ProxyModel) error { 64 | DB().Save(m) 65 | return nil 66 | } 67 | 68 | func (d *proxyDao) GetActiveList() ([]model.ProxyModel, error) { 69 | var proxies []model.ProxyModel 70 | DB().Where("status=?", consts.STATUS_YES).Find(&proxies) 71 | return proxies, nil 72 | } 73 | 74 | func (d *proxyDao) GetNeedUpdateInfoList() []model.ProxyModel { 75 | var proxies []model.ProxyModel 76 | DB().Where("status=? and country=?", consts.STATUS_YES, "").Find(&proxies) 77 | return proxies 78 | } 79 | 80 | func (d *proxyDao) Delete(host string, port string, proto string) error { 81 | DB().Where("host=? and port=? and proto=?", host, port, proto).Delete(model.ProxyModel{}) 82 | return nil 83 | } 84 | -------------------------------------------------------------------------------- /dao/proxy_dao_interface.go: -------------------------------------------------------------------------------- 1 | package dao 2 | 3 | import ( 4 | "proxy-collect/dto" 5 | "proxy-collect/model" 6 | ) 7 | 8 | type proxyDaoInterface interface { 9 | GetFailList() ([]model.ProxyModel, error) 10 | GetOne(host string, port string, proto string) (*model.ProxyModel, error) 11 | Create(proxy dto.ProxyDto, status int8) (*model.ProxyModel, error) 12 | Save(m *model.ProxyModel) error 13 | GetActiveList() ([]model.ProxyModel, error) 14 | GetRecheckList() ([]model.ProxyModel, error) 15 | GetNeedUpdateInfoList() []model.ProxyModel 16 | Delete(host string, port string, proto string) error 17 | } 18 | -------------------------------------------------------------------------------- /dao/redis/client.go: -------------------------------------------------------------------------------- 1 | package redis 2 | 3 | import ( 4 | redis_client "github.com/tongsq/go-lib/redis-client" 5 | "proxy-collect/config" 6 | ) 7 | 8 | var client *redis_client.RedisClient 9 | 10 | func Client() *redis_client.RedisClient { 11 | if client == nil { 12 | client = &redis_client.RedisClient{ 13 | MaxIdle: config.Get().Redis.MaxIdle, 14 | MaxActive: config.Get().Redis.MaxActive, 15 | Network: config.Get().Redis.Network, 16 | Address: config.Get().Redis.Address, 17 | Password: config.Get().Redis.Password, 18 | } 19 | } 20 | return client 21 | } 22 | -------------------------------------------------------------------------------- /dao/redis/proxy_dao.go: -------------------------------------------------------------------------------- 1 | package redis 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/tongsq/go-lib/logger" 9 | redis_client "github.com/tongsq/go-lib/redis-client" 10 | "proxy-collect/consts" 11 | "proxy-collect/dto" 12 | "proxy-collect/model" 13 | ) 14 | 15 | const PROXY_FAIL_SET = "proxy_fail_set" 16 | const PROXY_INFO_MAP = "proxy_info_map" 17 | const PROXY_SUCCESS_SET = "proxy_success_set" 18 | const PROXY_RECHECK_SET = "proxy_recheck_set" 19 | 20 | func NewRedisProxyDao() *proxyDao { 21 | return &proxyDao{} 22 | } 23 | 24 | type proxyDao struct { 25 | } 26 | 27 | func (d *proxyDao) GetFailList() ([]model.ProxyModel, error) { 28 | return d.getProxyList(PROXY_FAIL_SET) 29 | } 30 | 31 | func (d *proxyDao) GetRecheckList() ([]model.ProxyModel, error) { 32 | return d.getProxyList(PROXY_RECHECK_SET) 33 | } 34 | 35 | func (d *proxyDao) GetOne(host string, port string, proto string) (*model.ProxyModel, error) { 36 | var m model.ProxyModel 37 | info, err := Client().HMGetOne(PROXY_INFO_MAP, getProxyKey(host, port, proto)) 38 | if info == "" || err != nil { 39 | return nil, err 40 | } 41 | if err := json.Unmarshal([]byte(info), &m); err != nil { 42 | return nil, err 43 | } 44 | return &m, nil 45 | } 46 | 47 | func (d *proxyDao) Create(proxy dto.ProxyDto, status int8) (*model.ProxyModel, error) { 48 | m := &model.ProxyModel{ 49 | Host: proxy.Host, 50 | Port: proxy.Port, 51 | Status: status, 52 | CreateTime: time.Now().Unix(), 53 | UpdateTime: time.Now().Unix(), 54 | ActiveTime: time.Now().Unix(), 55 | CheckCount: 1, 56 | Source: proxy.Source, 57 | Proto: proxy.Proto, 58 | User: proxy.User, 59 | Password: proxy.Password, 60 | Ping: proxy.Ping, 61 | } 62 | key := getProxyKey(proxy.Host, proxy.Port, proxy.Proto) 63 | logger.FInfo("redis dao create new proxy") 64 | value, err := json.Marshal(m) 65 | if err != nil { 66 | return nil, err 67 | } 68 | result, err := Client().HMSet(PROXY_INFO_MAP, redis_client.HMDto{Field: key, Value: string(value)}) 69 | if !result || err != nil { 70 | return nil, err 71 | } 72 | updateProxySet(m) 73 | return m, err 74 | } 75 | 76 | func (d *proxyDao) Save(m *model.ProxyModel) error { 77 | key := getProxyKey(m.Host, m.Port, m.Proto) 78 | value, err := json.Marshal(m) 79 | if err != nil { 80 | return err 81 | } 82 | result, err := Client().HMSet(PROXY_INFO_MAP, redis_client.HMDto{Field: key, Value: string(value)}) 83 | if !result || err != nil { 84 | return err 85 | } 86 | updateProxySet(m) 87 | return nil 88 | } 89 | 90 | func (d *proxyDao) GetActiveList() ([]model.ProxyModel, error) { 91 | return d.getProxyList(PROXY_SUCCESS_SET) 92 | } 93 | 94 | func (d *proxyDao) GetNeedUpdateInfoList() []model.ProxyModel { 95 | proxies, err := d.GetActiveList() 96 | if err != nil || proxies == nil { 97 | return nil 98 | } 99 | var needUpdateList []model.ProxyModel 100 | for _, proxy := range proxies { 101 | if proxy.Country == "" { 102 | needUpdateList = append(needUpdateList, proxy) 103 | } 104 | } 105 | return needUpdateList 106 | } 107 | 108 | func (d *proxyDao) Delete(host string, port string, proto string) error { 109 | key := getProxyKey(host, port, proto) 110 | _, err := Client().HDel(PROXY_INFO_MAP, key) 111 | if err != nil { 112 | return err 113 | } 114 | deleteProxySet(host, port, proto) 115 | return nil 116 | } 117 | 118 | func ConvertInterface(args ...string) []interface{} { 119 | var result []interface{} 120 | for _, v := range args { 121 | result = append(result, v) 122 | } 123 | return result 124 | } 125 | 126 | func (d *proxyDao) getProxyList(key string) ([]model.ProxyModel, error) { 127 | proxys, err := Client().SMembers(key) 128 | if err != nil || proxys == nil { 129 | return nil, err 130 | } 131 | infoList, err := Client().HMGet(PROXY_INFO_MAP, ConvertInterface(proxys...)...) 132 | if err != nil || infoList == nil { 133 | return nil, err 134 | } 135 | var models []model.ProxyModel 136 | for _, v := range infoList { 137 | m := model.ProxyModel{} 138 | jsonErr := json.Unmarshal([]byte(v), &m) 139 | if jsonErr != nil { 140 | continue 141 | } 142 | models = append(models, m) 143 | } 144 | return models, nil 145 | } 146 | 147 | func getProxyKey(host string, port string, proto string) string { 148 | if proto == "" || proto == consts.PROTO_HTTP { 149 | return fmt.Sprintf("%s:%s", host, port) 150 | } else { 151 | return fmt.Sprintf("%s:%s:%s", host, port, proto) 152 | } 153 | } 154 | 155 | func updateProxySet(m *model.ProxyModel) { 156 | if m == nil { 157 | return 158 | } 159 | key := getProxyKey(m.Host, m.Port, m.Proto) 160 | if m.Status == consts.STATUS_YES { 161 | _, _ = Client().SAdd(PROXY_SUCCESS_SET, key) 162 | _, _ = Client().SRem(PROXY_FAIL_SET, key) 163 | _, _ = Client().SRem(PROXY_RECHECK_SET, key) 164 | } else if m.Status == consts.STATUS_RECHECK { 165 | _, _ = Client().SAdd(PROXY_RECHECK_SET, key) 166 | _, _ = Client().SRem(PROXY_SUCCESS_SET, key) 167 | _, _ = Client().SRem(PROXY_FAIL_SET, key) 168 | } else { 169 | _, _ = Client().SAdd(PROXY_FAIL_SET, key) 170 | _, _ = Client().SRem(PROXY_SUCCESS_SET, key) 171 | _, _ = Client().SRem(PROXY_RECHECK_SET, key) 172 | } 173 | } 174 | 175 | func deleteProxySet(host string, port string, proto string) { 176 | key := getProxyKey(host, port, proto) 177 | _, _ = Client().SRem(PROXY_SUCCESS_SET, key) 178 | _, _ = Client().SRem(PROXY_FAIL_SET, key) 179 | _, _ = Client().SRem(PROXY_RECHECK_SET, key) 180 | } 181 | -------------------------------------------------------------------------------- /doc/diagram.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tongsq/proxy-collect/327c996408336396db692c5198379335c9fdd343/doc/diagram.jpg -------------------------------------------------------------------------------- /dto/ip_info_dto.go: -------------------------------------------------------------------------------- 1 | package dto 2 | 3 | import "fmt" 4 | 5 | type IpInfoDto struct { 6 | Country string 7 | City string 8 | Region string 9 | Isp string 10 | } 11 | 12 | func (d *IpInfoDto) String() string { 13 | return fmt.Sprintf("country: %s, city: %s, region: %s, isp: %s", d.Country, d.City, d.Region, d.Isp) 14 | } 15 | -------------------------------------------------------------------------------- /dto/proxy_dto.go: -------------------------------------------------------------------------------- 1 | package dto 2 | 3 | import ( 4 | "proxy-collect/model" 5 | ) 6 | 7 | type ProxyInfoDto struct { 8 | ProxyDto 9 | Status int8 `json:"status"` 10 | CreateTime int64 `json:"create_time"` 11 | ActiveTime int64 `json:"active_time"` 12 | Country string `json:"country"` 13 | Region string `json:"region"` 14 | City string `json:"city"` 15 | Isp string `json:"isp"` 16 | } 17 | 18 | type ProxyDto struct { 19 | Host string `json:"host"` 20 | Port string `json:"port"` 21 | Proto string `json:"proto"` 22 | User string `json:"user"` 23 | Password string `json:"password"` 24 | Source string `json:"source"` 25 | Ping int64 `json:"ping"` 26 | } 27 | 28 | func NewProxyDto(m model.ProxyModel) ProxyInfoDto { 29 | return ProxyInfoDto{ 30 | ProxyDto: ProxyDto{ 31 | Host: m.Host, 32 | Port: m.Port, 33 | Proto: m.Proto, 34 | User: m.User, 35 | Password: m.Password, 36 | Source: m.Source, 37 | Ping: m.Ping, 38 | }, 39 | Status: m.Status, 40 | CreateTime: m.CreateTime, 41 | ActiveTime: m.ActiveTime, 42 | Country: m.Country, 43 | Region: m.Region, 44 | City: m.City, 45 | Isp: m.Isp, 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /global/global.go: -------------------------------------------------------------------------------- 1 | package global 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/tongsq/go-lib/component" 7 | "github.com/tongsq/go-lib/logger" 8 | "github.com/tongsq/go-lib/request" 9 | "proxy-collect/config" 10 | "proxy-collect/consts" 11 | ) 12 | 13 | var Pool *component.Pool 14 | var MaxPing time.Duration = request.DefaultTimeout 15 | 16 | var CommonHeader = &request.HeaderDto{ 17 | UserAgent: consts.USER_AGENT, 18 | } 19 | 20 | func LoadGlobal() { 21 | Pool = component.NewTaskPool(config.Get().PoolSize) 22 | logger.SetLogLevel(config.Get().Log.LogLevel) 23 | if config.Get().Log.ErrorLogFile != "" { 24 | logger.SetErrorFile(config.Get().Log.ErrorLogFile) 25 | } 26 | //set max proxy timeout 27 | MaxPing = time.Millisecond * time.Duration(config.Get().MaxPing) 28 | 29 | config.RegisterConfigRefreshHandler(func(old, new *config.ConfDto) { 30 | if old.Log.LogLevel != new.Log.LogLevel { 31 | logger.SetLogLevel(new.Log.LogLevel) 32 | } 33 | if old.Log.ErrorLogFile != new.Log.ErrorLogFile && new.Log.ErrorLogFile != "" { 34 | logger.SetErrorFile(new.Log.ErrorLogFile) 35 | } 36 | MaxPing = time.Millisecond * time.Duration(new.MaxPing) 37 | }) 38 | } 39 | 40 | func SimpleGet(requestUrl string) (*request.HttpResultDto, error) { 41 | return request.Get(requestUrl, request.NewOptions().WithHeader(CommonHeader)) 42 | } 43 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module proxy-collect 2 | 3 | go 1.14 4 | 5 | require ( 6 | github.com/PuerkitoBio/goquery v1.5.1 7 | github.com/gin-gonic/gin v1.6.3 8 | github.com/go-log/log v0.2.0 9 | github.com/jinzhu/gorm v1.9.16 10 | github.com/robfig/cron/v3 v3.0.0 11 | github.com/tongsq/go-lib v1.2.2 12 | github.com/tongsq/gost v1.0.1 13 | golang.org/x/text v0.3.7 14 | gopkg.in/yaml.v2 v2.4.0 15 | ) 16 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 3 | cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 4 | cloud.google.com/go v0.37.0/go.mod h1:TS1dMSSfndXH133OKGwekG838Om/cQT0BUHV3HcBgoo= 5 | dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU= 6 | dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU= 7 | dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4= 8 | dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU= 9 | git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= 10 | git.torproject.org/pluggable-transports/goptlib.git v1.0.0/go.mod h1:YT4XMSkuEXbtqlydr9+OxqFAyspUv0Gr9qhM3B++o/Q= 11 | git.torproject.org/pluggable-transports/goptlib.git v1.2.0 h1:0qRF7Dw5qXd0FtZkjWUiAh5GTutRtDGL4GXUDJ4qMHs= 12 | git.torproject.org/pluggable-transports/goptlib.git v1.2.0/go.mod h1:4PBMl1dg7/3vMWSoWb46eGWlrxkUyn/CAJmxhDLAlDs= 13 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 14 | github.com/LiamHaworth/go-tproxy v0.0.0-20190726054950-ef7efd7f24ed h1:eqa6queieK8SvoszxCu0WwH7lSVeL4/N/f1JwOMw1G4= 15 | github.com/LiamHaworth/go-tproxy v0.0.0-20190726054950-ef7efd7f24ed/go.mod h1:rA52xkgZwql9LRZXWb2arHEFP6qSR48KY2xOfWzEciQ= 16 | github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE= 17 | github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= 18 | github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da h1:KjTM2ks9d14ZYCvmHS9iAKVt9AyzRSqNU1qabPih5BY= 19 | github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da/go.mod h1:eHEWzANqSiWQsof+nXEI9bUVUyV6F53Fp89EuCh2EAA= 20 | github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo= 21 | github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= 22 | github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= 23 | github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= 24 | github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= 25 | github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= 26 | github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= 27 | github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= 28 | github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= 29 | github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE= 30 | github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= 31 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 32 | github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= 33 | github.com/coreos/go-iptables v0.6.0 h1:is9qnZMPYjLd8LYqmm/qlE+wwEgJIkTYdhV3rfZo4jk= 34 | github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q= 35 | github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= 36 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 37 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 38 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 39 | github.com/dchest/siphash v1.2.1/go.mod h1:q+IRvb2gOSrUnYoPqHiyHXS0FOBBOdl6tONBlVnOnt4= 40 | github.com/dchest/siphash v1.2.2 h1:9DFz8tQwl9pTVt5iok/9zKyzA1Q6bRGiF3HPiEEVr9I= 41 | github.com/dchest/siphash v1.2.2/go.mod h1:q+IRvb2gOSrUnYoPqHiyHXS0FOBBOdl6tONBlVnOnt4= 42 | github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd h1:83Wprp6ROGeiHFAP8WJdI2RoxALQYgdllERc3N5N2DM= 43 | github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= 44 | github.com/docker/libcontainer v2.2.1+incompatible h1:++SbbkCw+X8vAd4j2gOCzZ2Nn7s2xFALTf7LZKmM1/0= 45 | github.com/docker/libcontainer v2.2.1+incompatible/go.mod h1:osvj61pYsqhNCMLGX31xr7klUBhHb/ZBuXS0o1Fvwbw= 46 | github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= 47 | github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= 48 | github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= 49 | github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 50 | github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= 51 | github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= 52 | github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DPaBjB8zlTR87/ElzFsnQfuHnVUVqpZZIcV5Y= 53 | github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= 54 | github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= 55 | github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= 56 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 57 | github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= 58 | github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= 59 | github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= 60 | github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= 61 | github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= 62 | github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= 63 | github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14= 64 | github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= 65 | github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= 66 | github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= 67 | github.com/go-gost/gosocks4 v0.0.1 h1:+k1sec8HlELuQV7rWftIkmy8UijzUt2I6t+iMPlGB2s= 68 | github.com/go-gost/gosocks4 v0.0.1/go.mod h1:3B6L47HbU/qugDg4JnoFPHgJXE43Inz8Bah1QaN9qCc= 69 | github.com/go-gost/gosocks5 v0.3.0 h1:Hkmp9YDRBSCJd7xywW6dBPT6B9aQTkuWd+3WCheJiJA= 70 | github.com/go-gost/gosocks5 v0.3.0/go.mod h1:1G6I7HP7VFVxveGkoK8mnprnJqSqJjdcASKsdUn4Pp4= 71 | github.com/go-gost/relay v0.1.1-0.20211123134818-8ef7fd81ffd7 h1:itaaJhQJ19kUXEB4Igb0EbY8m+1Py2AaNNSBds/9gk4= 72 | github.com/go-gost/relay v0.1.1-0.20211123134818-8ef7fd81ffd7/go.mod h1:lcX+23LCQ3khIeASBo+tJ/WbwXFO32/N5YN6ucuYTG8= 73 | github.com/go-gost/tls-dissector v0.0.2-0.20211125135007-2b5d5bd9c07e h1:73NGqAs22ey3wJkIYVD/ACEoovuIuOlEzQTEoqrO5+U= 74 | github.com/go-gost/tls-dissector v0.0.2-0.20211125135007-2b5d5bd9c07e/go.mod h1:/9QfdewqmHdaE362Hv5nDaSWLx3pCmtD870d6GaquXs= 75 | github.com/go-log/log v0.2.0 h1:z8i91GBudxD5L3RmF0KVpetCbcGWAV7q1Tw1eRwQM9Q= 76 | github.com/go-log/log v0.2.0/go.mod h1:xzCnwajcues/6w7lne3yK2QU7DBPW7kqbgPGG5AF65U= 77 | github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= 78 | github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= 79 | github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= 80 | github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= 81 | github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= 82 | github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= 83 | github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY= 84 | github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= 85 | github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= 86 | github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= 87 | github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= 88 | github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= 89 | github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= 90 | github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= 91 | github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 92 | github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY= 93 | github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= 94 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 95 | github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= 96 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 97 | github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 98 | github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= 99 | github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= 100 | github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= 101 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 102 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 103 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 104 | github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= 105 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 106 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 107 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 108 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 109 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 110 | github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 111 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= 112 | github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= 113 | github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= 114 | github.com/gomodule/redigo v1.8.3 h1:HR0kYDX2RJZvAup8CsiJwxB4dTCSC0AaUq6S4SiLwUc= 115 | github.com/gomodule/redigo v1.8.3/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= 116 | github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 117 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 118 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 119 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 120 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 121 | github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= 122 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 123 | github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= 124 | github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= 125 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 126 | github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= 127 | github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= 128 | github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= 129 | github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= 130 | github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= 131 | github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= 132 | github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= 133 | github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= 134 | github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= 135 | github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= 136 | github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= 137 | github.com/h12w/go-socks5 v0.0.0-20200522160539-76189e178364 h1:5XxdakFhqd9dnXoAZy1Mb2R/DZ6D1e+0bGC/JhucGYI= 138 | github.com/h12w/go-socks5 v0.0.0-20200522160539-76189e178364/go.mod h1:eDJQioIyy4Yn3MVivT7rv/39gAJTrA7lgmYr8EW950c= 139 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 140 | github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= 141 | github.com/jinzhu/gorm v1.9.16 h1:+IyIjPEABKRpsu/F8OvDPy9fyQlgsg2luMV2ZIH5i5o= 142 | github.com/jinzhu/gorm v1.9.16/go.mod h1:G3LB3wezTOWM2ITLzPxEXgSkOXAntiLHS7UdBefADcs= 143 | github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= 144 | github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= 145 | github.com/jinzhu/now v1.0.1 h1:HjfetcXq097iXP0uoPCdnM4Efp5/9MsM0/M+XOTeR3M= 146 | github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= 147 | github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= 148 | github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= 149 | github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 150 | github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= 151 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 152 | github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= 153 | github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= 154 | github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= 155 | github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= 156 | github.com/klauspost/cpuid v1.2.4/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= 157 | github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= 158 | github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= 159 | github.com/klauspost/cpuid/v2 v2.0.6/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= 160 | github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= 161 | github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= 162 | github.com/klauspost/reedsolomon v1.9.9/go.mod h1:O7yFFHiQwDR6b2t63KPUpccPtNdp5ADgh1gg4fd12wo= 163 | github.com/klauspost/reedsolomon v1.9.15 h1:g2erWKD2M6rgnPf89fCji6jNlhMKMdXcuNHMW1SYCIo= 164 | github.com/klauspost/reedsolomon v1.9.15/go.mod h1:eqPAcE7xar5CIzcdfwydOEdcmchAKAP/qs14y4GCBOk= 165 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 166 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 167 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 168 | github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 169 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 170 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 171 | github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= 172 | github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= 173 | github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4= 174 | github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= 175 | github.com/lucas-clemente/quic-go v0.26.0 h1:ALBQXr9UJ8A1LyzvceX4jd9QFsHvlI0RR6BkV16o00A= 176 | github.com/lucas-clemente/quic-go v0.26.0/go.mod h1:AzgQoPda7N+3IqMMMkywBKggIFo2KT6pfnlrQ2QieeI= 177 | github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= 178 | github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= 179 | github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc= 180 | github.com/marten-seemann/qtls-go1-16 v0.1.5 h1:o9JrYPPco/Nukd/HpOHMHZoBDXQqoNtUCmny98/1uqQ= 181 | github.com/marten-seemann/qtls-go1-16 v0.1.5/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk= 182 | github.com/marten-seemann/qtls-go1-17 v0.1.1 h1:DQjHPq+aOzUeh9/lixAGunn6rIOQyWChPSI4+hgW7jc= 183 | github.com/marten-seemann/qtls-go1-17 v0.1.1/go.mod h1:C2ekUKcDdz9SDWxec1N/MvcXBpaX9l3Nx67XaR84L5s= 184 | github.com/marten-seemann/qtls-go1-18 v0.1.1 h1:qp7p7XXUFL7fpBvSS1sWD+uSqPvzNQK43DH+/qEkj0Y= 185 | github.com/marten-seemann/qtls-go1-18 v0.1.1/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4= 186 | github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= 187 | github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= 188 | github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/QA= 189 | github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= 190 | github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= 191 | github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= 192 | github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg= 193 | github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= 194 | github.com/milosgajdos/tenus v0.0.3 h1:jmaJzwaY1DUyYVD0lM4U+uvP2kkEg1VahDqRFxIkVBE= 195 | github.com/milosgajdos/tenus v0.0.3/go.mod h1:eIjx29vNeDOYWJuCnaHY2r4fq5egetV26ry3on7p8qY= 196 | github.com/mmcloughlin/avo v0.0.0-20200803215136-443f81d77104/go.mod h1:wqKykBG2QzQDJEzvRkcS8x6MiSJkF52hXZsXcjaB3ls= 197 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 198 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 199 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 200 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 201 | github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= 202 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 203 | github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= 204 | github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= 205 | github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= 206 | github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= 207 | github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= 208 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 209 | github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= 210 | github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= 211 | github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E= 212 | github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= 213 | github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= 214 | github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= 215 | github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= 216 | github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= 217 | github.com/onsi/gomega v1.13.0 h1:7lLHu94wT9Ij0o6EWWclhu0aOh32VxhkwEJvzuWPeak= 218 | github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= 219 | github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= 220 | github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc= 221 | github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= 222 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 223 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 224 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 225 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 226 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 227 | github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= 228 | github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= 229 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 230 | github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= 231 | github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 232 | github.com/riobard/go-bloom v0.0.0-20200614022211-cdc8013cb5b3 h1:f/FNXud6gA3MNr8meMVVGxhp+QBTqY91tM8HjEuMjGg= 233 | github.com/riobard/go-bloom v0.0.0-20200614022211-cdc8013cb5b3/go.mod h1:HgjTstvQsPGkxUsCd2KWxErBblirPizecHcpD3ffK+s= 234 | github.com/robfig/cron/v3 v3.0.0 h1:kQ6Cb7aHOHTSzNVNEhmp8EcWKLb4CbiMW9h9VyIhO4E= 235 | github.com/robfig/cron/v3 v3.0.0/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= 236 | github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= 237 | github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= 238 | github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= 239 | github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= 240 | github.com/shadowsocks/go-shadowsocks2 v0.1.5 h1:PDSQv9y2S85Fl7VBeOMF9StzeXZyK1HakRm86CUbr28= 241 | github.com/shadowsocks/go-shadowsocks2 v0.1.5/go.mod h1:AGGpIoek4HRno4xzyFiAtLHkOpcoznZEkAccaI/rplM= 242 | github.com/shadowsocks/shadowsocks-go v0.0.0-20200409064450-3e585ff90601 h1:XU9hik0exChEmY92ALW4l9WnDodxLVS9yOSNh2SizaQ= 243 | github.com/shadowsocks/shadowsocks-go v0.0.0-20200409064450-3e585ff90601/go.mod h1:mttDPaeLm87u74HMrP+n2tugXvIKWcwff/cqSX0lehY= 244 | github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= 245 | github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= 246 | github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0= 247 | github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= 248 | github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= 249 | github.com/shurcooL/gofontwoff v0.0.0-20180329035133-29b52fc0a18d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw= 250 | github.com/shurcooL/gopherjslib v0.0.0-20160914041154-feb6d3990c2c/go.mod h1:8d3azKNyqcHP1GaQE/c6dDgjkgSx2BZ4IoEi4F1reUI= 251 | github.com/shurcooL/highlight_diff v0.0.0-20170515013008-09bb4053de1b/go.mod h1:ZpfEhSmds4ytuByIcDnOLkTHGUI6KNqRNPDLHDk+mUU= 252 | github.com/shurcooL/highlight_go v0.0.0-20181028180052-98c3abbbae20/go.mod h1:UDKB5a1T23gOMUJrI+uSuH0VRDStOiUVSjBTRDVBVag= 253 | github.com/shurcooL/home v0.0.0-20181020052607-80b7ffcb30f9/go.mod h1:+rgNQw2P9ARFAs37qieuu7ohDNQ3gds9msbT2yn85sg= 254 | github.com/shurcooL/htmlg v0.0.0-20170918183704-d01228ac9e50/go.mod h1:zPn1wHpTIePGnXSHpsVPWEktKXHr6+SS6x/IKRb7cpw= 255 | github.com/shurcooL/httperror v0.0.0-20170206035902-86b7830d14cc/go.mod h1:aYMfkZ6DWSJPJ6c4Wwz3QtW22G7mf/PEgaB9k/ik5+Y= 256 | github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= 257 | github.com/shurcooL/httpgzip v0.0.0-20180522190206-b1c53ac65af9/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q= 258 | github.com/shurcooL/issues v0.0.0-20181008053335-6292fdc1e191/go.mod h1:e2qWDig5bLteJ4fwvDAc2NHzqFEthkqn7aOZAOpj+PQ= 259 | github.com/shurcooL/issuesapp v0.0.0-20180602232740-048589ce2241/go.mod h1:NPpHK2TI7iSaM0buivtFUc9offApnI0Alt/K8hcHy0I= 260 | github.com/shurcooL/notifications v0.0.0-20181007000457-627ab5aea122/go.mod h1:b5uSkrEVM1jQUspwbixRBhaIjIzL2xazXp6kntxYle0= 261 | github.com/shurcooL/octicon v0.0.0-20181028054416-fa4f57f9efb2/go.mod h1:eWdoE5JD4R5UVWDucdOPg1g2fqQRq78IQa9zlOV1vpQ= 262 | github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1lToEk4d2s07G3XGfz2QrgHXg4RJBvjrOozvoWfk= 263 | github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= 264 | github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4= 265 | github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw= 266 | github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= 267 | github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= 268 | github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 h1:TG/diQgUe0pntT/2D9tmUCz4VNwm9MfrtPr0SU2qSX8= 269 | github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8/go.mod h1:P5HUIBuIWKbyjl083/loAegFkfbFNx5i2qEP4CNbm7E= 270 | github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= 271 | github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= 272 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 273 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 274 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 275 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 276 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= 277 | github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= 278 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 279 | github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= 280 | github.com/templexxx/cpu v0.0.1/go.mod h1:w7Tb+7qgcAlIyX4NhLuDKt78AHA5SzPmq0Wj6HiEnnk= 281 | github.com/templexxx/cpu v0.0.7 h1:pUEZn8JBy/w5yzdYWgx+0m0xL9uk6j4K91C5kOViAzo= 282 | github.com/templexxx/cpu v0.0.7/go.mod h1:w7Tb+7qgcAlIyX4NhLuDKt78AHA5SzPmq0Wj6HiEnnk= 283 | github.com/templexxx/xorsimd v0.4.1 h1:iUZcywbOYDRAZUasAs2eSCUW8eobuZDy0I9FJiORkVg= 284 | github.com/templexxx/xorsimd v0.4.1/go.mod h1:W+ffZz8jJMH2SXwuKu9WhygqBMbFnp14G2fqEr8qaNo= 285 | github.com/tjfoc/gmsm v1.3.2/go.mod h1:HaUcFuY0auTiaHB9MHFGCPx5IaLhTUd2atbCFBQXn9w= 286 | github.com/tjfoc/gmsm v1.4.1 h1:aMe1GlZb+0bLjn+cKTPEvvn9oUEBlJitaZiiBwsbgho= 287 | github.com/tjfoc/gmsm v1.4.1/go.mod h1:j4INPkHWMrhJb38G+J6W4Tw0AbuN8Thu3PbdVYhVcTE= 288 | github.com/tongsq/go-lib v1.2.1 h1:4zKkvSbls4TjgvWqwoGfHV5sRDKJkme1rdNbnPOFs4s= 289 | github.com/tongsq/go-lib v1.2.1/go.mod h1:g1Ddvsmbq/i0YOAxGgsd4iImSQwUwpI+rOVb874/jvw= 290 | github.com/tongsq/go-lib v1.2.2 h1:1mvm/oBE24eKTtyQa/FMKCTqcTZC5+tTjui0UnflXpc= 291 | github.com/tongsq/go-lib v1.2.2/go.mod h1:g1Ddvsmbq/i0YOAxGgsd4iImSQwUwpI+rOVb874/jvw= 292 | github.com/tongsq/gost v1.0.1 h1:+H006ZyZEjbKc7n00GwvRWZSk6uDCHm4RzA9Silvefk= 293 | github.com/tongsq/gost v1.0.1/go.mod h1:CxT2BIvP+7he06PqNSsOy4g5cVCsIobIWR32yjIRZhY= 294 | github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= 295 | github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= 296 | github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= 297 | github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= 298 | github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= 299 | github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= 300 | github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= 301 | github.com/xtaci/kcp-go/v5 v5.6.1 h1:Pwn0aoeNSPF9dTS7IgiPXn0HEtaIlVb6y5UKWPsx8bI= 302 | github.com/xtaci/kcp-go/v5 v5.6.1/go.mod h1:W3kVPyNYwZ06p79dNwFWQOVFrdcBpDBsdyvK8moQrYo= 303 | github.com/xtaci/lossyconn v0.0.0-20190602105132-8df528c0c9ae/go.mod h1:gXtu8J62kEgmN++bm9BVICuT/e8yiLI2KFobd/TRFsE= 304 | github.com/xtaci/lossyconn v0.0.0-20200209145036-adba10fffc37 h1:EWU6Pktpas0n8lLQwDsRyZfmkPeRbdgPtW609es+/9E= 305 | github.com/xtaci/lossyconn v0.0.0-20200209145036-adba10fffc37/go.mod h1:HpMP7DB2CyokmAh4lp0EQnnWhmycP/TvwBGzvuie+H0= 306 | github.com/xtaci/smux v1.5.16 h1:FBPYOkW8ZTjLKUM4LI4xnnuuDC8CQ/dB04HD519WoEk= 307 | github.com/xtaci/smux v1.5.16/go.mod h1:OMlQbT5vcgl2gb49mFkYo6SMf+zP3rcjcwQz7ZU7IGY= 308 | github.com/xtaci/tcpraw v1.2.25 h1:VDlqo0op17JeXBM6e2G9ocCNLOJcw9mZbobMbJjo0vk= 309 | github.com/xtaci/tcpraw v1.2.25/go.mod h1:dKyZ2V75s0cZ7cbgJYdxPvms7af0joIeOyx1GgJQbLk= 310 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 311 | github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 312 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 313 | github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= 314 | github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= 315 | gitlab.com/yawning/bsaes.git v0.0.0-20190805113838-0a714cd429ec/go.mod h1:BZ1RAoRPbCxum9Grlv5aeksu2H8BiKehBYooU2LFiOQ= 316 | gitlab.com/yawning/obfs4.git v0.0.0-20210511220700-e330d1b7024b h1:w/f20IHUkUYEp+xYgpKz4Bs78zms0DbjPZCep5lc0xA= 317 | gitlab.com/yawning/obfs4.git v0.0.0-20210511220700-e330d1b7024b/go.mod h1:OM1ngEp5brdANPox+rqk2AGTLQvzobyB5Dwm3vu3CgM= 318 | gitlab.com/yawning/utls.git v0.0.12-1/go.mod h1:3ONKiSFR9Im/c3t5RKmMJTVdmZN496FNyk3mjrY1dyo= 319 | go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= 320 | go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= 321 | golang.org/x/arch v0.0.0-20190909030613-46d78d1859ac/go.mod h1:flIaEI6LNU6xOCD5PaJvn9wGP0agmIOqjrtsKGRguv4= 322 | golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= 323 | golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 324 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 325 | golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 326 | golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 327 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 328 | golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 329 | golang.org/x/crypto v0.0.0-20191219195013-becbf705a915/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 330 | golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 331 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 332 | golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 333 | golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 334 | golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= 335 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 336 | golang.org/x/crypto v0.0.0-20220321153916-2c7772ba3064/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= 337 | golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 h1:kUhD7nTDoI3fVd9G4ORWrbV5NY0liEs/Jg2pv5f+bBA= 338 | golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= 339 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 340 | golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 341 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 342 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 343 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 344 | golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= 345 | golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= 346 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 347 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 348 | golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 349 | golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o= 350 | golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= 351 | golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 352 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 353 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 354 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 355 | golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 356 | golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 357 | golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 358 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 359 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 360 | golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 361 | golang.org/x/net v0.0.0-20190328230028-74de082e2cca/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 362 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 363 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 364 | golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 365 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 366 | golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 367 | golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 368 | golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= 369 | golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= 370 | golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 371 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 372 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 373 | golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= 374 | golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= 375 | golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 376 | golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 377 | golang.org/x/net v0.0.0-20220325170049-de3da57026de h1:pZB1TWnKi+o4bENlbzAgLrEbY4RMYmUIRobMcSmfeYc= 378 | golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= 379 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 380 | golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 381 | golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 382 | golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 383 | golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= 384 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 385 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 386 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 387 | golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 388 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 389 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 390 | golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 391 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 392 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= 393 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 394 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 395 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 396 | golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 397 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 398 | golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 399 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 400 | golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 401 | golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 402 | golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 403 | golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 404 | golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 405 | golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 406 | golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 407 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 408 | golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 409 | golang.org/x/sys v0.0.0-20200808120158-1030fc2bf1d9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 410 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 411 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 412 | golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 413 | golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 414 | golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 415 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 416 | golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 417 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 418 | golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 419 | golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 420 | golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 421 | golang.org/x/sys v0.0.0-20220325203850-36772127a21f h1:TrmogKRsSOxRMJbLYGrB4SBbW+LJcEllYBLME5Zk5pU= 422 | golang.org/x/sys v0.0.0-20220325203850-36772127a21f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 423 | golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= 424 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 425 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= 426 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 427 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 428 | golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 429 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 430 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 431 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 432 | golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= 433 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 434 | golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 435 | golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 436 | golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 437 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 438 | golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 439 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 440 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 441 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 442 | golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 443 | golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 444 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 445 | golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 446 | golang.org/x/tools v0.0.0-20200425043458-8463f397d07c/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 447 | golang.org/x/tools v0.0.0-20200808161706-5bf02b21f123/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= 448 | golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 449 | golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= 450 | golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20= 451 | golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= 452 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 453 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 454 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 455 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= 456 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 457 | google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= 458 | google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= 459 | google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y= 460 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 461 | google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 462 | google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 463 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 464 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 465 | google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 466 | google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 467 | google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= 468 | google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 469 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 470 | google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= 471 | google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= 472 | google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= 473 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 474 | google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 475 | google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= 476 | google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= 477 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 478 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 479 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 480 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 481 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 482 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 483 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 484 | google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= 485 | google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= 486 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 487 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= 488 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 489 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 490 | gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= 491 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= 492 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 493 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 494 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 495 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 496 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 497 | gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 498 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 499 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 500 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= 501 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 502 | grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= 503 | h12.io/socks v1.0.3 h1:Ka3qaQewws4j4/eDQnOdpr4wXsC//dXtWvftlIcCQUo= 504 | h12.io/socks v1.0.3/go.mod h1:AIhxy1jOId/XCz9BO+EIgNL2rQiPTBNnOfnVnQ+3Eck= 505 | honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 506 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 507 | honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 508 | honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 509 | rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= 510 | sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= 511 | sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= 512 | -------------------------------------------------------------------------------- /model/proxy_model.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import "fmt" 4 | 5 | //import ( 6 | // "github.com/jinzhu/gorm" 7 | //) 8 | type ProxyModel struct { 9 | Id int `gorm:"column:id;AUTO_INCREMENT;PRIMARY_KEY"` 10 | Host string `gorm:"column:host"` 11 | Port string `gorm:"column:port"` 12 | Status int8 `gorm:"column:status"` 13 | CreateTime int64 `gorm:"column:create_time"` 14 | UpdateTime int64 `gorm:"column:update_time"` 15 | ActiveTime int64 `gorm:"column:active_time"` 16 | Country string `gorm:"column:country"` 17 | Region string `gorm:"column:region"` 18 | City string `gorm:"column:city"` 19 | Isp string `gorm:"column:isp"` 20 | CheckCount int64 `gorm:"column:check_count"` 21 | Source string `gorm:"column:source"` 22 | Proto string `gorm:"proto"` 23 | User string `gorm:"user"` 24 | Password string `gorm:"password"` 25 | Ping int64 `gorm:"ping"` 26 | } 27 | 28 | func (ProxyModel) TableName() string { 29 | return "proxy" 30 | } 31 | 32 | func (m ProxyModel) String() string { 33 | return fmt.Sprintf("%#v", m) 34 | } 35 | -------------------------------------------------------------------------------- /scheduler/check_active_ip.go: -------------------------------------------------------------------------------- 1 | package scheduler 2 | 3 | import ( 4 | "github.com/tongsq/go-lib/logger" 5 | "proxy-collect/dao" 6 | "proxy-collect/dto" 7 | "proxy-collect/global" 8 | "proxy-collect/service" 9 | ) 10 | 11 | type CheckActiveIp struct { 12 | } 13 | 14 | func (s CheckActiveIp) Run() { 15 | logger.FSuccess("check active ip start run") 16 | proxies, err := dao.ProxyDao.GetActiveList() 17 | if err != nil { 18 | logger.Error("get active ip fail", logger.Fields{"err": err}) 19 | } 20 | logger.FInfo("check active ip, len:%d, cap:%d", len(proxies), cap(proxies)) 21 | //pool := component.NewTaskPool(20) 22 | //defer pool.Close() 23 | for _, proxy := range proxies { 24 | var p = dto.NewProxyDto(proxy) 25 | global.Pool.RunTask(func() { service.ProxyService.CheckProxyAndSave(p.ProxyDto) }) 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /scheduler/check_fail_ip.go: -------------------------------------------------------------------------------- 1 | package scheduler 2 | 3 | import ( 4 | "github.com/tongsq/go-lib/logger" 5 | "proxy-collect/dao" 6 | "proxy-collect/dto" 7 | "proxy-collect/global" 8 | "proxy-collect/service" 9 | ) 10 | 11 | type CheckFailIp struct { 12 | } 13 | 14 | func (s CheckFailIp) Run() { 15 | logger.FSuccess("check fail ip start run") 16 | proxies, err := dao.ProxyDao.GetFailList() 17 | if err != nil { 18 | logger.Error("get need check ip list fail", logger.Fields{"err": err}) 19 | } 20 | logger.FInfo("check fail ip, len:%d, cap:%d", len(proxies), cap(proxies)) 21 | for _, proxy := range proxies { 22 | var p = dto.NewProxyDto(proxy) 23 | global.Pool.RunTask(func() { service.ProxyService.CheckProxyAndSave(p.ProxyDto) }) 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /scheduler/get_ip.go: -------------------------------------------------------------------------------- 1 | package scheduler 2 | 3 | import ( 4 | "encoding/json" 5 | "github.com/tongsq/go-lib/logger" 6 | "proxy-collect/config" 7 | "proxy-collect/global" 8 | "proxy-collect/service" 9 | "proxy-collect/service/proxy_getter" 10 | ) 11 | 12 | type GetIp struct { 13 | } 14 | 15 | func (s GetIp) Run() { 16 | logger.FSuccess("collect ip start run") 17 | if !config.SystemStart { 18 | logger.Info("system not start", map[string]interface{}{}) 19 | return 20 | } 21 | //pool := component.NewTaskPool(100) 22 | //defer pool.Close() 23 | serviceList := []service.ProxyGetterInterface{ 24 | service.GetProxyXila, 25 | service.GetProxyNima, 26 | service.GetProxyKuai, 27 | service.GetProxyData5u, 28 | service.GetProxy66ip, 29 | //service.GetProxyGuoBanjia, 30 | service.GetProxyCoderBusy, 31 | service.GetProxyIp3366, 32 | service.GetProxyIpJiangXianLi, 33 | service.GetProxy89Ip, 34 | service.GetProxy7Yip, 35 | service.GetProxyProxyList, 36 | service.GetProxyZdaye, 37 | service.GetProxyFanQie, 38 | //service.GetProxyZdayeIndex, 39 | service.GetProxySeofangfa, 40 | service.GetProxyXsdaili, 41 | service.GetProxyPaChong, 42 | service.KxDaili, 43 | service.Geonode, 44 | } 45 | var configGetterList []service.ProxyGetterInterface 46 | for _, conf := range config.Get().Getters { 47 | configGetterList = append(configGetterList, proxy_getter.NewGetter(&conf)) 48 | } 49 | config.RegisterConfigRefreshHandler(func(old, new *config.ConfDto) { 50 | var oldConfig, newConfig []byte 51 | oldConfig, _ = json.Marshal(old.Getters) 52 | newConfig, _ = json.Marshal(new.Getters) 53 | if string(oldConfig) != string(newConfig) { 54 | logger.Info("reload getter configs", map[string]interface{}{"old": string(oldConfig), "new": string(newConfig)}) 55 | configGetterList = []service.ProxyGetterInterface{} 56 | for _, conf := range new.Getters { 57 | configGetterList = append(configGetterList, proxy_getter.NewGetter(&conf)) 58 | } 59 | } 60 | }) 61 | for _, getProxyService := range serviceList { 62 | go service.ProxyService.DoGetProxy(getProxyService, global.Pool) 63 | } 64 | for _, configGetter := range configGetterList { 65 | go service.ProxyService.DoGetProxy(configGetter, global.Pool) 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /scheduler/recheck_ip.go: -------------------------------------------------------------------------------- 1 | package scheduler 2 | 3 | import ( 4 | "github.com/tongsq/go-lib/logger" 5 | "proxy-collect/dao" 6 | "proxy-collect/dto" 7 | "proxy-collect/global" 8 | "proxy-collect/service" 9 | ) 10 | 11 | type RecheckIp struct { 12 | } 13 | 14 | func (s RecheckIp) Run() { 15 | logger.FSuccess("recheck ip start run") 16 | proxies, err := dao.ProxyDao.GetRecheckList() 17 | if err != nil { 18 | logger.Error("get recheck ip list fail", logger.Fields{"err": err}) 19 | } 20 | logger.FInfo("recheck ip, len:%d, cap:%d", len(proxies), cap(proxies)) 21 | for _, proxy := range proxies { 22 | var p = dto.NewProxyDto(proxy) 23 | global.Pool.RunTask(func() { service.ProxyService.CheckProxyAndSave(p.ProxyDto) }) 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /scheduler/update_ip_info.go: -------------------------------------------------------------------------------- 1 | package scheduler 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/tongsq/go-lib/logger" 8 | "proxy-collect/dao" 9 | "proxy-collect/service/ip" 10 | ) 11 | 12 | type UpdateIpInfo struct { 13 | } 14 | 15 | func (s UpdateIpInfo) Run() { 16 | logger.FSuccess("update ip info start run") 17 | proxies := dao.ProxyDao.GetNeedUpdateInfoList() 18 | logger.FInfo(fmt.Sprintf("count:%d, cap: %d\n", len(proxies), cap(proxies))) 19 | for _, proxy := range proxies { 20 | //use online ip database 21 | ipInfoDto := ip.GetIpInfoByIp138(proxy.Host, proxy.Port) 22 | logger.Info("get ip info:", logger.Fields{"ipInfoDto": ipInfoDto}) 23 | time.Sleep(5 * time.Second) 24 | if ipInfoDto == nil { 25 | logger.FError("get ip info fail") 26 | continue 27 | } 28 | proxy.Country = ipInfoDto.Country 29 | proxy.Isp = ipInfoDto.Isp 30 | proxy.Region = ipInfoDto.Region 31 | proxy.City = ipInfoDto.City 32 | err := dao.ProxyDao.Save(&proxy) 33 | if err == nil { 34 | logger.Success("update ip detail success", logger.Fields{"host": proxy.Host, "port": proxy.Port}) 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /service/common/proxy.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | 7 | "proxy-collect/config" 8 | "proxy-collect/consts" 9 | "proxy-collect/dto" 10 | ) 11 | 12 | func CheckProxyFormat(host string, port string) bool { 13 | ok, _ := regexp.Match(`^[\d\.]+$`, []byte(host)) 14 | if !ok { 15 | return false 16 | } 17 | ok, _ = regexp.Match(`^\d+$`, []byte(port)) 18 | return ok 19 | } 20 | 21 | func GetProxyUrl(p *dto.ProxyDto) string { 22 | proto := p.Proto 23 | if proto == "" { 24 | proto = consts.PROTO_HTTP 25 | } 26 | if p.User == "" { 27 | return fmt.Sprintf("%s://%s:%s", proto, p.Host, p.Port) 28 | } else { 29 | return fmt.Sprintf("%s://%s:%s@%s:%s", proto, p.User, p.Password, p.Host, p.Port) 30 | } 31 | } 32 | 33 | func GetTunnelUrl(tunnel *config.TunnelConfig) string { 34 | return fmt.Sprintf("%s://%s:%s", tunnel.Proto, tunnel.Host, tunnel.Port) 35 | } 36 | -------------------------------------------------------------------------------- /service/ip/local_ip_data.go: -------------------------------------------------------------------------------- 1 | package ip 2 | 3 | import ( 4 | "bytes" 5 | "compress/zlib" 6 | "encoding/binary" 7 | "errors" 8 | "io/ioutil" 9 | "net/http" 10 | "os" 11 | 12 | "github.com/tongsq/go-lib/logger" 13 | ) 14 | 15 | type fileData struct { 16 | Data []byte 17 | FilePath string 18 | Path *os.File 19 | IPNum int64 20 | } 21 | 22 | func (f *fileData) UpdateLocalData() ([]byte, error) { 23 | defer func() { 24 | if err := recover(); err != nil { 25 | logger.Error("update local ip database fail", map[string]interface{}{"err": err}) 26 | } 27 | }() 28 | tmpData, err := f.getOnline() 29 | if err != nil { 30 | logger.Error("更新本地纯真ip库失败", map[string]interface{}{"err": err}) 31 | return nil, err 32 | } else { 33 | if err := ioutil.WriteFile(f.FilePath, tmpData, 0644); err == nil { 34 | logger.FDebug("已将最新的纯真 IP 库保存到本地 %s ", f.FilePath) 35 | } 36 | f.loadData(tmpData) 37 | return tmpData, nil 38 | } 39 | } 40 | 41 | // InitIPData 初始化ip库数据到内存中 42 | func (f *fileData) InitIPData() { 43 | var tmpData []byte 44 | // 判断文件是否存在 45 | _, err := os.Stat(f.FilePath) 46 | if err != nil && os.IsNotExist(err) { 47 | logger.Info("文件不存在，尝试从网络获取最新纯真 IP 库", nil) 48 | tmpData, err = f.UpdateLocalData() 49 | if err != nil { 50 | logger.Error("download local ip data fail", map[string]interface{}{"err": err}) 51 | return 52 | } 53 | } else { 54 | // 打开文件句柄 55 | logger.FDebug("从本地数据库文件 %s 打开\n", f.FilePath) 56 | f.Path, err = os.OpenFile(f.FilePath, os.O_RDONLY, 0400) 57 | if err != nil { 58 | return 59 | } 60 | defer f.Path.Close() 61 | 62 | tmpData, err = ioutil.ReadAll(f.Path) 63 | if err != nil { 64 | logger.Error("load local ip data fail", map[string]interface{}{"err": err}) 65 | return 66 | } 67 | } 68 | 69 | f.loadData(tmpData) 70 | } 71 | 72 | func (f *fileData) loadData(tmpData []byte) { 73 | if tmpData == nil || len(tmpData) <= 0 { 74 | return 75 | } 76 | f.Data = tmpData 77 | 78 | buf := f.Data[0:8] 79 | start := binary.LittleEndian.Uint32(buf[:4]) 80 | end := binary.LittleEndian.Uint32(buf[4:]) 81 | 82 | f.IPNum = int64((end-start)/IndexLen + 1) 83 | } 84 | 85 | func (f *fileData) getOnline() ([]byte, error) { 86 | resp, err := http.Get("http://update.cz88.net/ip/qqwry.rar") 87 | if err != nil { 88 | return nil, err 89 | } 90 | defer resp.Body.Close() 91 | if resp.StatusCode != http.StatusOK { 92 | return nil, errors.New("download ip data from cz88 fail") 93 | } 94 | if body, err := ioutil.ReadAll(resp.Body); err != nil { 95 | return nil, err 96 | } else { 97 | if key, err := f.getKey(body); err != nil { 98 | return nil, err 99 | } else { 100 | for i := 0; i < 0x200; i++ { 101 | key = key * 0x805 102 | key++ 103 | key = key & 0xff 104 | 105 | body[i] = byte(uint32(body[i]) ^ key) 106 | } 107 | reader, err := zlib.NewReader(bytes.NewReader(body)) 108 | if err != nil { 109 | return nil, err 110 | } 111 | 112 | return ioutil.ReadAll(reader) 113 | } 114 | } 115 | } 116 | 117 | func (f *fileData) getKey(body []byte) (uint32, error) { 118 | resp, err := http.Get("http://update.cz88.net/ip/copywrite.rar") 119 | if err != nil { 120 | return 0, err 121 | } 122 | defer resp.Body.Close() 123 | 124 | if body, err := ioutil.ReadAll(resp.Body); err != nil { 125 | return 0, err 126 | } else { 127 | // @see https://stackoverflow.com/questions/34078427/how-to-read-packed-binary-data-in-go 128 | return binary.LittleEndian.Uint32(body[5*4:]), nil 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /service/ip/local_ip_service.go: -------------------------------------------------------------------------------- 1 | package ip 2 | 3 | import ( 4 | "encoding/binary" 5 | "errors" 6 | "net" 7 | "strings" 8 | 9 | "github.com/tongsq/go-lib/logger" 10 | "golang.org/x/text/encoding/simplifiedchinese" 11 | "proxy-collect/dto" 12 | ) 13 | 14 | const ( 15 | // IndexLen 索引长度 16 | IndexLen = 7 17 | // RedirectMode1 国家的类型, 指向另一个指向 18 | RedirectMode1 = 0x01 19 | // RedirectMode2 国家的类型, 指向一个指向 20 | RedirectMode2 = 0x02 21 | ) 22 | 23 | type localIpService struct { 24 | Data *fileData 25 | Offset int64 26 | } 27 | 28 | func NewLocalIpService(ipData *fileData) *localIpService { 29 | return &localIpService{ 30 | Data: ipData, 31 | } 32 | } 33 | 34 | // ReadData 从文件中读取数据 35 | func (q *localIpService) ReadData(num int, offset ...int64) (rs []byte) { 36 | if len(offset) > 0 { 37 | q.SetOffset(offset[0]) 38 | } 39 | nums := int64(num) 40 | end := q.Offset + nums 41 | dataNum := int64(len(q.Data.Data)) 42 | if q.Offset > dataNum { 43 | return nil 44 | } 45 | 46 | if end > dataNum { 47 | end = dataNum 48 | } 49 | rs = q.Data.Data[q.Offset:end] 50 | q.Offset = end 51 | return 52 | } 53 | 54 | // SetOffset 设置偏移量 55 | func (q *localIpService) SetOffset(offset int64) { 56 | q.Offset = offset 57 | } 58 | 59 | // Find ip地址查询对应归属地信息 60 | func (q *localIpService) Find(ip string) (res *dto.IpInfoDto, err error) { 61 | defer func() { 62 | if re := recover(); re != nil { 63 | logger.Error("get ip info from local data fail", map[string]interface{}{"err": re}) 64 | err = errors.New("get ip info from local data fail") 65 | } 66 | }() 67 | res = &dto.IpInfoDto{} 68 | if strings.Count(ip, ".") != 3 { 69 | return res, errors.New("ip format error") 70 | } 71 | offset := q.searchIndex(binary.BigEndian.Uint32(net.ParseIP(ip).To4())) 72 | if offset <= 0 { 73 | return 74 | } 75 | 76 | var country []byte 77 | var area []byte 78 | 79 | mode := q.readMode(offset + 4) 80 | if mode == RedirectMode1 { 81 | countryOffset := q.readUInt24() 82 | mode = q.readMode(countryOffset) 83 | if mode == RedirectMode2 { 84 | c := q.readUInt24() 85 | country = q.readString(c) 86 | countryOffset += 4 87 | } else { 88 | country = q.readString(countryOffset) 89 | countryOffset += uint32(len(country) + 1) 90 | } 91 | area = q.readArea(countryOffset) 92 | } else if mode == RedirectMode2 { 93 | countryOffset := q.readUInt24() 94 | country = q.readString(countryOffset) 95 | area = q.readArea(offset + 8) 96 | } else { 97 | country = q.readString(offset + 4) 98 | area = q.readArea(offset + uint32(5+len(country))) 99 | } 100 | 101 | enc := simplifiedchinese.GBK.NewDecoder() 102 | city, err := enc.String(string(country)) 103 | if err != nil { 104 | return 105 | } 106 | if idx := strings.IndexRune(city, '省'); idx > 0 { 107 | res.Region = city[0 : idx+3] 108 | res.City = city[idx+3:] 109 | } else { 110 | res.City = city 111 | } 112 | res.Isp, err = enc.String(string(area)) 113 | 114 | return 115 | } 116 | 117 | // readMode 获取偏移值类型 118 | func (q *localIpService) readMode(offset uint32) byte { 119 | mode := q.ReadData(1, int64(offset)) 120 | return mode[0] 121 | } 122 | 123 | // readArea 读取区域 124 | func (q *localIpService) readArea(offset uint32) []byte { 125 | mode := q.readMode(offset) 126 | if mode == RedirectMode1 || mode == RedirectMode2 { 127 | areaOffset := q.readUInt24() 128 | if areaOffset == 0 { 129 | return []byte("") 130 | } 131 | return q.readString(areaOffset) 132 | } 133 | return q.readString(offset) 134 | } 135 | 136 | // readString 获取字符串 137 | func (q *localIpService) readString(offset uint32) []byte { 138 | q.SetOffset(int64(offset)) 139 | data := make([]byte, 0, 30) 140 | for { 141 | buf := q.ReadData(1) 142 | if len(buf) == 0 || buf[0] == 0 { 143 | break 144 | } 145 | data = append(data, buf[0]) 146 | } 147 | return data 148 | } 149 | 150 | // searchIndex 查找索引位置 151 | func (q *localIpService) searchIndex(ip uint32) uint32 { 152 | header := q.ReadData(8, 0) 153 | 154 | start := binary.LittleEndian.Uint32(header[:4]) 155 | end := binary.LittleEndian.Uint32(header[4:]) 156 | 157 | mid := uint32(0) 158 | _ip := uint32(0) 159 | 160 | for { 161 | mid = q.getMiddleOffset(start, end) 162 | buf := q.ReadData(IndexLen, int64(mid)) 163 | _ip = binary.LittleEndian.Uint32(buf[:4]) 164 | 165 | if end-start == IndexLen { 166 | offset := byteToUInt32(buf[4:]) 167 | buf = q.ReadData(IndexLen) 168 | if ip < binary.LittleEndian.Uint32(buf[:4]) { 169 | return offset 170 | } 171 | return 0 172 | } 173 | 174 | // 找到的比较大，向前移 175 | if _ip > ip { 176 | end = mid 177 | } else if _ip < ip { // 找到的比较小，向后移 178 | start = mid 179 | } else if _ip == ip { 180 | return byteToUInt32(buf[4:]) 181 | } 182 | } 183 | } 184 | 185 | // readUInt24 186 | func (q *localIpService) readUInt24() uint32 { 187 | buf := q.ReadData(3) 188 | return byteToUInt32(buf) 189 | } 190 | 191 | // getMiddleOffset 192 | func (q *localIpService) getMiddleOffset(start uint32, end uint32) uint32 { 193 | records := ((end - start) / IndexLen) >> 1 194 | return start + records*IndexLen 195 | } 196 | 197 | // byteToUInt32 将 byte 转换为uint32 198 | func byteToUInt32(data []byte) uint32 { 199 | i := uint32(data[0]) & 0xff 200 | i |= (uint32(data[1]) << 8) & 0xff00 201 | i |= (uint32(data[2]) << 16) & 0xff0000 202 | return i 203 | } 204 | -------------------------------------------------------------------------------- /service/ip/on_line_ip_service.go: -------------------------------------------------------------------------------- 1 | package ip 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "regexp" 7 | 8 | "github.com/tongsq/go-lib/logger" 9 | "github.com/tongsq/go-lib/request" 10 | "golang.org/x/text/encoding/simplifiedchinese" 11 | "proxy-collect/consts" 12 | "proxy-collect/dto" 13 | ) 14 | 15 | func GetIpInfoByIp138(host string, port string) *dto.IpInfoDto { 16 | requestUrl := fmt.Sprintf("https://www.ip138.com/iplookup.asp?ip=%s&action=2", host) 17 | h := &request.HeaderDto{ 18 | UserAgent: consts.USER_AGENT, 19 | UpgradeInsecureRequests: "1", 20 | Host: "www.ip138.com", 21 | Referer: "https://www.ip138.com/", 22 | AcceptEncoding: "gzip, deflate, br", 23 | Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", 24 | } 25 | 26 | logger.Info("get ip info from ip138", logger.Fields{"url": requestUrl}) 27 | data, err := request.Get(requestUrl, request.NewOptions().WithHeader(h).WithProxy(&request.ProxyDto{Host: host, Port: port})) 28 | if err != nil || data == nil { 29 | logger.Error("get from ip138 use proxy error", logger.Fields{"err": err, "data": data}) 30 | data, err = request.Get(requestUrl, request.NewOptions().WithHeader(h)) 31 | if err != nil || data == nil { 32 | logger.Error("get from ip138 no proxy error", logger.Fields{"err": err, "data": data}) 33 | return nil 34 | } 35 | } 36 | re := regexp.MustCompile(`var ip_result = (.+);`) 37 | matched := re.FindAllStringSubmatch(data.Body, -1) 38 | if len(matched) < 1 { 39 | return nil 40 | } 41 | jsonStr := matched[0][1] 42 | jsonStr, err = simplifiedchinese.GBK.NewDecoder().String(jsonStr) 43 | if err != nil { 44 | logger.FError("gb2313 decode error") 45 | } 46 | var result map[string][]map[string]string 47 | err = json.Unmarshal([]byte(jsonStr), &result) 48 | if err != nil { 49 | logger.Error("parse ip info fail", map[string]interface{}{"jsonStr": jsonStr}) 50 | } 51 | logger.Info("get ip info result", logger.Fields{"jsonStr": jsonStr, "result": result}) 52 | info := result["ip_c_list"][0] 53 | ipInfoDto := &dto.IpInfoDto{ 54 | Country: info["ct"], 55 | Region: info["prov"], 56 | City: info["city"], 57 | Isp: info["yunyin"], 58 | } 59 | if ipInfoDto.Country == "" { 60 | return nil 61 | } 62 | return ipInfoDto 63 | } 64 | -------------------------------------------------------------------------------- /service/ip/service.go: -------------------------------------------------------------------------------- 1 | package ip 2 | 3 | import ( 4 | "proxy-collect/config" 5 | "proxy-collect/dto" 6 | ) 7 | 8 | var localIpServiceInstance *localIpService 9 | 10 | func LocalIpService() *localIpService { 11 | if localIpServiceInstance == nil { 12 | LoadLocalIpData() 13 | } 14 | return localIpServiceInstance 15 | } 16 | 17 | func LoadLocalIpData() { 18 | // IPData IP库的数据 19 | var IPData = &fileData{ 20 | FilePath: config.Get().LocalIpDataPath, 21 | } 22 | IPData.InitIPData() 23 | localIpServiceInstance = NewLocalIpService(IPData) 24 | } 25 | 26 | func GetIpInfo(host string, port string) *dto.IpInfoDto { 27 | result, err := LocalIpService().Find(host) 28 | if err != nil { 29 | return GetIpInfoByIp138(host, port) 30 | } 31 | return result 32 | } 33 | 34 | func UpdateLocalIpData() { 35 | LocalIpService().Data.UpdateLocalData() 36 | } 37 | -------------------------------------------------------------------------------- /service/proxy_getter/coderbusy.go: -------------------------------------------------------------------------------- 1 | package proxy_getter 2 | 3 | import ( 4 | "github.com/PuerkitoBio/goquery" 5 | "github.com/tongsq/go-lib/logger" 6 | "proxy-collect/consts" 7 | "proxy-collect/global" 8 | "proxy-collect/service/common" 9 | 10 | "strings" 11 | ) 12 | 13 | func NewGetProxyCoderBusy() *getProxyCoderBusy { 14 | return &getProxyCoderBusy{} 15 | } 16 | 17 | type getProxyCoderBusy struct { 18 | } 19 | 20 | func (s *getProxyCoderBusy) GetUrlList() []string { 21 | list := []string{ 22 | "https://proxy.coderbusy.com/", 23 | } 24 | return list 25 | } 26 | func (s *getProxyCoderBusy) GetContentHtml(requestUrl string) string { 27 | logger.Info("get proxy from coderbusy", logger.Fields{"url": requestUrl}) 28 | data, err := global.SimpleGet(requestUrl) 29 | if err != nil || data == nil { 30 | logger.Error("get proxy from coderbusy fail", logger.Fields{"err": err, "data": data}) 31 | logger.Error("get proxy from coderbusy fail", logger.Fields{"err": err, "data": data}) 32 | return "" 33 | } 34 | return data.Body 35 | } 36 | 37 | func (s *getProxyCoderBusy) ParseHtml(body string) [][]string { 38 | 39 | doc, err := goquery.NewDocumentFromReader(strings.NewReader(body)) 40 | if err != nil { 41 | logger.Error(consts.GO_QUERY_READ_ERROR, logger.Fields{"err": err}) 42 | return nil 43 | } 44 | var proxyList [][]string 45 | doc.Find("tbody > tr").Each(func(i int, selection *goquery.Selection) { 46 | td := selection.ChildrenFiltered("td").First() 47 | host := strings.TrimSpace(td.Text()) 48 | td2 := selection.ChildrenFiltered("td").Eq(1) 49 | port := strings.TrimSpace(td2.Find("a").First().Text()) 50 | 51 | if !common.CheckProxyFormat(host, port) { 52 | logger.Error(consts.PROXY_FORMAT_ERROR, logger.Fields{"host": host, "port": port}) 53 | return 54 | } 55 | proxyArr := []string{host, port} 56 | proxyList = append(proxyList, proxyArr) 57 | }) 58 | return proxyList 59 | } 60 | -------------------------------------------------------------------------------- /service/proxy_getter/common_getter.go: -------------------------------------------------------------------------------- 1 | package proxy_getter 2 | 3 | import ( 4 | "regexp" 5 | 6 | "github.com/tongsq/go-lib/logger" 7 | "proxy-collect/consts" 8 | "proxy-collect/global" 9 | ) 10 | 11 | var urlList = map[string][]string{ 12 | consts.PROTO_SOCKS5: { 13 | "https://github.com/monosans/proxy-list/blob/main/proxies/socks5.txt", 14 | }, 15 | consts.PROTO_SOCKS4: { 16 | "https://github.com/monosans/proxy-list/blob/main/proxies/socks4.txt", 17 | }, 18 | consts.PROTO_HTTP: { 19 | "https://github.com/monosans/proxy-list/blob/main/proxies/http.txt", 20 | }, 21 | } 22 | 23 | func NewCommonGetter(proto string) *commonGetter { 24 | return &commonGetter{Proto: proto} 25 | } 26 | 27 | type commonGetter struct { 28 | Proto string 29 | } 30 | 31 | func (s *commonGetter) GetUrlList() []string { 32 | if list, ok := urlList[s.Proto]; ok { 33 | return list 34 | } 35 | return nil 36 | } 37 | func (s *commonGetter) GetContentHtml(requestUrl string) string { 38 | logger.Info("get proxy from common getter", logger.Fields{"url": requestUrl}) 39 | data, err := global.SimpleGet(requestUrl) 40 | if err != nil || data == nil { 41 | logger.Error("get proxy from common getter fail", logger.Fields{"err": err, "data": data, "url": requestUrl}) 42 | return "" 43 | } 44 | return data.Body 45 | } 46 | 47 | func (s *commonGetter) ParseHtml(body string) [][]string { 48 | var proxyList [][]string 49 | re := regexp.MustCompile(`(\d+\.\d+\.\d+\.\d+):(\d+)`) 50 | matched := re.FindAllStringSubmatch(body, -1) 51 | for _, match := range matched { 52 | proxyArr := []string{match[1], match[2], s.Proto} 53 | proxyList = append(proxyList, proxyArr) 54 | } 55 | return proxyList 56 | } 57 | -------------------------------------------------------------------------------- /service/proxy_getter/data5u.go: -------------------------------------------------------------------------------- 1 | package proxy_getter 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/PuerkitoBio/goquery" 7 | "github.com/tongsq/go-lib/logger" 8 | "github.com/tongsq/go-lib/request" 9 | "proxy-collect/consts" 10 | ) 11 | 12 | func NewGetProxyData5u() *getProxyData5u { 13 | return &getProxyData5u{} 14 | } 15 | 16 | type getProxyData5u struct { 17 | } 18 | 19 | func (s *getProxyData5u) GetUrlList() []string { 20 | list := []string{ 21 | "http://www.data5u.com/", 22 | "http://www.data5u.com/free/gngn/index.shtml", 23 | "http://www.data5u.com/free/gnpt/index.shtml", 24 | } 25 | return list 26 | } 27 | 28 | func (s *getProxyData5u) GetContentHtml(requestUrl string) string { 29 | h := &request.HeaderDto{ 30 | UserAgent: consts.USER_AGENT, 31 | Host: "www.data5u.com", 32 | UpgradeInsecureRequests: "1", 33 | } 34 | 35 | logger.Info("get proxy from data5u", logger.Fields{"url": requestUrl}) 36 | data, err := request.Get(requestUrl, request.NewOptions().WithHeader(h)) 37 | if err != nil || data == nil { 38 | logger.Error("get proxy from data5u fail", logger.Fields{"err": err, "data": data}) 39 | return "" 40 | } 41 | return data.Body 42 | } 43 | 44 | func (s *getProxyData5u) ParseHtml(body string) [][]string { 45 | 46 | doc, err := goquery.NewDocumentFromReader(strings.NewReader(body)) 47 | if err != nil { 48 | logger.Error(consts.GO_QUERY_READ_ERROR, logger.Fields{"err": err}) 49 | return nil 50 | } 51 | 52 | var proxyList [][]string 53 | doc.Find("ul.l2").Each(func(i int, selection *goquery.Selection) { 54 | td := selection.Find("span>li").First() 55 | proxyHost := td.Text() 56 | td2 := selection.Find("span>li").Eq(1) 57 | proxyPort := td2.Text() 58 | if proxyHost == "" || proxyPort == "" { 59 | logger.FError("parse html node fail") 60 | } 61 | proxyArr := []string{strings.TrimSpace(proxyHost), strings.TrimSpace(proxyPort)} 62 | proxyList = append(proxyList, proxyArr) 63 | }) 64 | 65 | return proxyList 66 | } 67 | -------------------------------------------------------------------------------- /service/proxy_getter/fanqieip.go: -------------------------------------------------------------------------------- 1 | package proxy_getter 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/PuerkitoBio/goquery" 8 | "github.com/tongsq/go-lib/logger" 9 | "github.com/tongsq/go-lib/request" 10 | "proxy-collect/consts" 11 | "proxy-collect/service/common" 12 | ) 13 | 14 | func NewGetProxyFanQie() *getProxyFanQie { 15 | return &getProxyFanQie{} 16 | } 17 | 18 | type getProxyFanQie struct { 19 | } 20 | 21 | func (s *getProxyFanQie) GetUrlList() []string { 22 | list := []string{ 23 | "https://www.fanqieip.com/free", 24 | } 25 | for i := 2; i < 6; i++ { 26 | list = append(list, fmt.Sprintf("https://www.fanqieip.com/free/%d", i)) 27 | } 28 | return list 29 | } 30 | 31 | func (s *getProxyFanQie) GetContentHtml(requestUrl string) string { 32 | h := &request.HeaderDto{ 33 | UserAgent: consts.USER_AGENT, 34 | Host: "www.fanqieip.com", 35 | UpgradeInsecureRequests: "1", 36 | Referer: "https://www.fanqieip.com/free", 37 | } 38 | logger.Info("get proxy from fanqieip", logger.Fields{"url": requestUrl}) 39 | data, err := request.Get(requestUrl, request.NewOptions().WithHeader(h)) 40 | if err != nil || data == nil { 41 | logger.Error("get proxy from fanqieip fail", logger.Fields{"err": err, "data": data}) 42 | return "" 43 | } 44 | return data.Body 45 | } 46 | 47 | func (s *getProxyFanQie) ParseHtml(body string) [][]string { 48 | 49 | doc, err := goquery.NewDocumentFromReader(strings.NewReader(body)) 50 | if err != nil { 51 | logger.Error(consts.GO_QUERY_READ_ERROR, logger.Fields{"err": err}) 52 | return nil 53 | } 54 | 55 | var proxyList [][]string 56 | doc.Find("tbody > tr").Each(func(i int, selection *goquery.Selection) { 57 | td := selection.ChildrenFiltered("td").First() 58 | host := strings.TrimSpace(td.ChildrenFiltered("div").First().Text()) 59 | td2 := selection.ChildrenFiltered("td").Eq(1) 60 | port := strings.TrimSpace(td2.ChildrenFiltered("div").First().Text()) 61 | 62 | if !common.CheckProxyFormat(host, port) { 63 | logger.Error(consts.PROXY_FORMAT_ERROR, logger.Fields{"host": host, "port": port}) 64 | return 65 | } 66 | proxyArr := []string{host, port} 67 | proxyList = append(proxyList, proxyArr) 68 | }) 69 | return proxyList 70 | } 71 | -------------------------------------------------------------------------------- /service/proxy_getter/geonode.go: -------------------------------------------------------------------------------- 1 | package proxy_getter 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/tongsq/go-lib/logger" 9 | "proxy-collect/consts" 10 | "proxy-collect/global" 11 | ) 12 | 13 | func NewGetProxyGeonode() *geonode { 14 | return &geonode{} 15 | } 16 | 17 | type geonode struct { 18 | } 19 | 20 | func (s *geonode) GetUrlList() []string { 21 | list := []string{ 22 | "https://proxylist.geonode.com/api/proxy-list?limit=50&page=1&sort_by=lastChecked&sort_type=desc", 23 | } 24 | for i := 2; i < 4; i++ { 25 | list = append(list, fmt.Sprintf("https://proxylist.geonode.com/api/proxy-list?limit=50&page=%d&sort_by=lastChecked&sort_type=desc", i)) 26 | } 27 | return list 28 | } 29 | func (s *geonode) GetContentHtml(requestUrl string) string { 30 | logger.Info("get proxy from geonode.com", logger.Fields{"url": requestUrl}) 31 | data, err := global.SimpleGet(requestUrl) 32 | if err != nil || data == nil { 33 | logger.Error("get proxy from geonode.com fail", logger.Fields{"err": err, "data": data}) 34 | return "" 35 | } 36 | return data.Body 37 | } 38 | 39 | func (s *geonode) ParseHtml(body string) [][]string { 40 | 41 | result := geonodeResult{} 42 | err := json.Unmarshal([]byte(body), &result) 43 | if err != nil { 44 | logger.Error("json parse fail", logger.Fields{"err": err}) 45 | return nil 46 | } 47 | var proxyList [][]string 48 | for _, item := range result.Data { 49 | proto := "" 50 | for _, protoConst := range consts.PROTO_LIST { 51 | for _, hasProto := range item.Protocols { 52 | if protoConst == hasProto { 53 | proto = protoConst 54 | break 55 | } 56 | } 57 | if proto != "" { 58 | break 59 | } 60 | } 61 | if proto != "" { 62 | proxyList = append(proxyList, []string{item.IP, item.Port, proto}) 63 | } 64 | } 65 | return proxyList 66 | } 67 | 68 | type geonodeResult struct { 69 | Data []struct { 70 | ID string `json:"_id"` 71 | IP string `json:"ip"` 72 | Port string `json:"port"` 73 | AnonymityLevel string `json:"anonymityLevel"` 74 | Asn string `json:"asn"` 75 | City string `json:"city"` 76 | Country string `json:"country"` 77 | CreatedAt time.Time `json:"created_at"` 78 | Google bool `json:"google"` 79 | Isp string `json:"isp"` 80 | LastChecked int `json:"lastChecked"` 81 | Latency float64 `json:"latency"` 82 | Org string `json:"org"` 83 | Protocols []string `json:"protocols"` 84 | Region interface{} `json:"region"` 85 | ResponseTime int `json:"responseTime"` 86 | Speed int `json:"speed"` 87 | UpdatedAt time.Time `json:"updated_at"` 88 | WorkingPercent interface{} `json:"workingPercent"` 89 | UpTime int `json:"upTime"` 90 | UpTimeSuccessCount int `json:"upTimeSuccessCount"` 91 | UpTimeTryCount int `json:"upTimeTryCount"` 92 | HostName interface{} `json:"hostName,omitempty"` 93 | } `json:"data"` 94 | Total int `json:"total"` 95 | Page string `json:"page"` 96 | Limit string `json:"limit"` 97 | } 98 | -------------------------------------------------------------------------------- /service/proxy_getter/getter.go: -------------------------------------------------------------------------------- 1 | package proxy_getter 2 | 3 | import ( 4 | "bufio" 5 | "github.com/tongsq/go-lib/request" 6 | "math/rand" 7 | "os" 8 | "proxy-collect/config" 9 | "proxy-collect/dao" 10 | "regexp" 11 | "strings" 12 | "time" 13 | 14 | "github.com/tongsq/go-lib/logger" 15 | ) 16 | 17 | func NewGetter(conf *config.Getter) *getter { 18 | return &getter{config: conf} 19 | } 20 | 21 | type getter struct { 22 | config *config.Getter 23 | } 24 | 25 | func (s *getter) GetUrlList() []string { 26 | var urls []string 27 | for _, url := range s.config.Urls { 28 | if strings.HasPrefix(url, "http") { 29 | urls = append(urls, url) 30 | } else { 31 | file, err := os.Open(url) 32 | if err != nil { 33 | logger.Warning("打开文件失败", map[string]interface{}{"err": err, "url": url}) 34 | continue 35 | } 36 | scanner := bufio.NewScanner(file) 37 | for scanner.Scan() { 38 | line := scanner.Text() 39 | urls = append(urls, strings.TrimSpace(line)) 40 | } 41 | _ = file.Close() 42 | } 43 | } 44 | return urls 45 | } 46 | func (s *getter) GetContentHtml(requestUrl string) string { 47 | logger.Info("get proxy from config getter", logger.Fields{"url": requestUrl}) 48 | h := &request.HeaderDto{ 49 | UserAgent: s.config.Agent, 50 | } 51 | var data *request.HttpResultDto 52 | var err error 53 | options := request.NewOptions().WithHeader(h) 54 | if s.config.Proxy { 55 | proxies, err := dao.ProxyDao.GetActiveList() 56 | if err == nil && len(proxies) > 0 { 57 | rand.Seed(time.Now().Unix()) 58 | i := rand.Intn(len(proxies)) 59 | proxy := proxies[i] 60 | options = options.WithProxy(&request.ProxyDto{Host: proxy.Host, Port: proxy.Port}) 61 | } 62 | } 63 | if s.config.Method == "GET" { 64 | data, err = request.Get(requestUrl, options) 65 | } else { 66 | data, err = request.Post(requestUrl, options) 67 | } 68 | if err != nil || data == nil { 69 | logger.Error("get proxy from config getter fail", logger.Fields{"err": err, "data": data, "url": requestUrl}) 70 | return "" 71 | } 72 | return data.Body 73 | } 74 | 75 | func (s *getter) ParseHtml(body string) [][]string { 76 | var proxyList [][]string 77 | re := regexp.MustCompile(s.config.Regexp) 78 | matched := re.FindAllStringSubmatch(body, -1) 79 | for _, match := range matched { 80 | proxyArr := []string{match[1], match[2], s.config.Proto} 81 | proxyList = append(proxyList, proxyArr) 82 | } 83 | return proxyList 84 | } 85 | -------------------------------------------------------------------------------- /service/proxy_getter/guobanjia.go: -------------------------------------------------------------------------------- 1 | package proxy_getter 2 | 3 | import ( 4 | "github.com/PuerkitoBio/goquery" 5 | "github.com/tongsq/go-lib/logger" 6 | "github.com/tongsq/go-lib/request" 7 | "proxy-collect/consts" 8 | "proxy-collect/service/common" 9 | 10 | "strings" 11 | ) 12 | 13 | func NewGetProxyGuoBanJia() *getProxyGuoBanJia { 14 | return &getProxyGuoBanJia{} 15 | } 16 | 17 | type getProxyGuoBanJia struct { 18 | } 19 | 20 | func (s *getProxyGuoBanJia) GetUrlList() []string { 21 | list := []string{ 22 | "http://www.goubanjia.com/", 23 | } 24 | return list 25 | } 26 | 27 | func (s *getProxyGuoBanJia) GetContentHtml(requestUrl string) string { 28 | h := &request.HeaderDto{ 29 | UserAgent: consts.USER_AGENT, 30 | Host: "www.goubanjia.com", 31 | UpgradeInsecureRequests: "1", 32 | } 33 | logger.Info("get proxy from guobanjia", logger.Fields{"url": requestUrl}) 34 | data, err := request.Get(requestUrl, request.NewOptions().WithHeader(h)) 35 | if err != nil || data == nil { 36 | logger.Error("get proxy from guobanjia fail", logger.Fields{"err": err, "data": data}) 37 | return "" 38 | } 39 | return data.Body 40 | } 41 | 42 | func (s *getProxyGuoBanJia) ParseHtml(body string) [][]string { 43 | doc, err := goquery.NewDocumentFromReader(strings.NewReader(body)) 44 | if err != nil { 45 | logger.Error("read fail", logger.Fields{"err": err}) 46 | return nil 47 | } 48 | var proxyList [][]string 49 | doc.Find("tbody > tr").Each(func(i int, selection *goquery.Selection) { 50 | td := selection.ChildrenFiltered("td").First() 51 | hostStr := "" 52 | len := td.Children().Size() 53 | td.Children().Each(func(i int, item *goquery.Selection) { 54 | style, _ := item.Attr("style") 55 | if !strings.Contains(style, "none") && i != (len-1) { 56 | hostStr = hostStr + item.Text() 57 | } 58 | }) 59 | port := td.Children().Last().Text() 60 | hostStr = strings.TrimSpace(hostStr) 61 | if !common.CheckProxyFormat(hostStr, port) { 62 | logger.Error(consts.PROXY_FORMAT_ERROR, logger.Fields{"host": hostStr, "port": port}) 63 | return 64 | } 65 | proxyArr := []string{hostStr, port} 66 | proxyList = append(proxyList, proxyArr) 67 | }) 68 | 69 | return proxyList 70 | } 71 | -------------------------------------------------------------------------------- /service/proxy_getter/ip3366.go: -------------------------------------------------------------------------------- 1 | package proxy_getter 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/PuerkitoBio/goquery" 8 | "github.com/tongsq/go-lib/logger" 9 | "proxy-collect/consts" 10 | "proxy-collect/global" 11 | "proxy-collect/service/common" 12 | ) 13 | 14 | func NewGetProxyIp3366() *getProxyIp3366 { 15 | return &getProxyIp3366{} 16 | } 17 | 18 | type getProxyIp3366 struct { 19 | } 20 | 21 | func (s *getProxyIp3366) GetUrlList() []string { 22 | list := []string{ 23 | "http://www.ip3366.net/free/?stype=1", 24 | "http://www.ip3366.net/free/?stype=2", 25 | } 26 | for i := 2; i < 4; i++ { 27 | list = append(list, fmt.Sprintf("http://www.ip3366.net/free/?stype=1&page=%d", i)) 28 | list = append(list, fmt.Sprintf("http://www.ip3366.net/free/?stype=2&page=%d", i)) 29 | } 30 | return list 31 | } 32 | func (s *getProxyIp3366) GetContentHtml(requestUrl string) string { 33 | logger.Info("get proxy from ip3366", logger.Fields{"url": requestUrl}) 34 | data, err := global.SimpleGet(requestUrl) 35 | if err != nil || data == nil { 36 | logger.Error("get proxy from ip3366 fail", logger.Fields{"err": err, "data": data}) 37 | return "" 38 | } 39 | return data.Body 40 | } 41 | 42 | func (s *getProxyIp3366) ParseHtml(body string) [][]string { 43 | 44 | doc, err := goquery.NewDocumentFromReader(strings.NewReader(body)) 45 | if err != nil { 46 | logger.Error("read fail", logger.Fields{"err": err}) 47 | return nil 48 | } 49 | var proxyList [][]string 50 | doc.Find("tbody > tr").Each(func(i int, selection *goquery.Selection) { 51 | td := selection.ChildrenFiltered("td").First() 52 | host := strings.TrimSpace(td.Text()) 53 | td2 := selection.ChildrenFiltered("td").Eq(1) 54 | port := strings.TrimSpace(td2.Text()) 55 | 56 | if !common.CheckProxyFormat(host, port) { 57 | logger.Error(consts.PROXY_FORMAT_ERROR, logger.Fields{"host": host, "port": port}) 58 | return 59 | } 60 | proxyArr := []string{host, port} 61 | proxyList = append(proxyList, proxyArr) 62 | }) 63 | return proxyList 64 | } 65 | -------------------------------------------------------------------------------- /service/proxy_getter/jiangxianli.go: -------------------------------------------------------------------------------- 1 | package proxy_getter 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/PuerkitoBio/goquery" 8 | "github.com/tongsq/go-lib/logger" 9 | "github.com/tongsq/go-lib/request" 10 | "proxy-collect/consts" 11 | "proxy-collect/service/common" 12 | ) 13 | 14 | func NewGetProxyIpJiangXianLi() *getProxyIpJiangXianLi { 15 | return &getProxyIpJiangXianLi{} 16 | } 17 | 18 | type getProxyIpJiangXianLi struct { 19 | } 20 | 21 | func (s *getProxyIpJiangXianLi) GetUrlList() []string { 22 | list := []string{ 23 | "https://ip.jiangxianli.com/", 24 | } 25 | for i := 2; i < 6; i++ { 26 | list = append(list, fmt.Sprintf("https://ip.jiangxianli.com/?page=%d", i)) 27 | } 28 | return list 29 | } 30 | func (s *getProxyIpJiangXianLi) GetContentHtml(requestUrl string) string { 31 | h := &request.HeaderDto{ 32 | UserAgent: consts.USER_AGENT, 33 | UpgradeInsecureRequests: "1", 34 | Host: "ip.jiangxianli.com", 35 | Referer: "https://ip.jiangxianli.com/", 36 | } 37 | logger.Info("get proxy from jangxianli", logger.Fields{"url": requestUrl}) 38 | data, err := request.Get(requestUrl, request.NewOptions().WithHeader(h)) 39 | if err != nil || data == nil { 40 | logger.Error("get proxy from jangxianli fail", logger.Fields{"err": err, "data": data}) 41 | return "" 42 | } 43 | return data.Body 44 | } 45 | 46 | func (s *getProxyIpJiangXianLi) ParseHtml(body string) [][]string { 47 | 48 | doc, err := goquery.NewDocumentFromReader(strings.NewReader(body)) 49 | if err != nil { 50 | logger.Error("read fail", logger.Fields{"err": err}) 51 | return nil 52 | } 53 | var proxyList [][]string 54 | doc.Find("tbody > tr").Each(func(i int, selection *goquery.Selection) { 55 | td := selection.ChildrenFiltered("td").First() 56 | host := strings.TrimSpace(td.Text()) 57 | td2 := selection.ChildrenFiltered("td").Eq(1) 58 | port := strings.TrimSpace(td2.Text()) 59 | 60 | if !common.CheckProxyFormat(host, port) { 61 | logger.Error(consts.PROXY_FORMAT_ERROR, logger.Fields{"host": host, "port": port}) 62 | return 63 | } 64 | proxyArr := []string{host, port} 65 | proxyList = append(proxyList, proxyArr) 66 | }) 67 | return proxyList 68 | } 69 | -------------------------------------------------------------------------------- /service/proxy_getter/kuai.go: -------------------------------------------------------------------------------- 1 | package proxy_getter 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/PuerkitoBio/goquery" 8 | "github.com/tongsq/go-lib/logger" 9 | "github.com/tongsq/go-lib/request" 10 | "proxy-collect/consts" 11 | ) 12 | 13 | func NewGetProxyKuai() *getProxyKuai { 14 | return &getProxyKuai{} 15 | } 16 | 17 | type getProxyKuai struct { 18 | } 19 | 20 | func (s *getProxyKuai) GetUrlList() []string { 21 | list := []string{ 22 | "https://www.kuaidaili.com/free/inha/", 23 | "https://www.kuaidaili.com/free/intr/", 24 | } 25 | for i := 2; i < 6; i++ { 26 | list = append(list, fmt.Sprintf("https://www.kuaidaili.com/free/inha/%d/", i)) 27 | list = append(list, fmt.Sprintf("https://www.kuaidaili.com/free/intr/%d/", i)) 28 | } 29 | return list 30 | } 31 | 32 | func (s *getProxyKuai) GetContentHtml(requestUrl string) string { 33 | 34 | h := &request.HeaderDto{ 35 | UserAgent: consts.USER_AGENT, 36 | Host: "www.kuaidaili.com", 37 | Referer: "https://www.kuaidaili.com/free/inha/", 38 | UpgradeInsecureRequests: "1", 39 | } 40 | logger.Info("get proxy from kuaidaili", logger.Fields{"url": requestUrl}) 41 | data, err := request.Get(requestUrl, request.NewOptions().WithHeader(h)) 42 | if err != nil || data == nil { 43 | logger.Error("ger proxy from kuaidaili fail", logger.Fields{"err": err, "data": data}) 44 | return "" 45 | } 46 | return data.Body 47 | } 48 | 49 | func (s *getProxyKuai) ParseHtml(body string) [][]string { 50 | 51 | doc, err := goquery.NewDocumentFromReader(strings.NewReader(body)) 52 | if err != nil { 53 | logger.Error(consts.GO_QUERY_READ_ERROR, logger.Fields{"err": err}) 54 | return nil 55 | } 56 | var proxyList [][]string 57 | doc.Find("tbody > tr").Each(func(i int, selection *goquery.Selection) { 58 | td := selection.ChildrenFiltered("td").First() 59 | proxyHost := td.Text() 60 | td2 := selection.ChildrenFiltered("td").Eq(1) 61 | proxyPort := td2.Text() 62 | if proxyHost == "" || proxyPort == "" { 63 | logger.FError("parse html node fail") 64 | } 65 | proxyArr := []string{strings.TrimSpace(proxyHost), strings.TrimSpace(proxyPort)} 66 | proxyList = append(proxyList, proxyArr) 67 | }) 68 | return proxyList 69 | } 70 | -------------------------------------------------------------------------------- /service/proxy_getter/kxdaili.go: -------------------------------------------------------------------------------- 1 | package proxy_getter 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/PuerkitoBio/goquery" 8 | "github.com/tongsq/go-lib/logger" 9 | "proxy-collect/consts" 10 | "proxy-collect/global" 11 | "proxy-collect/service/common" 12 | ) 13 | 14 | func NewGetProxyKxDaili() *getProxyKxDaili { 15 | return &getProxyKxDaili{} 16 | } 17 | 18 | type getProxyKxDaili struct { 19 | } 20 | 21 | func (s *getProxyKxDaili) GetUrlList() []string { 22 | list := []string{ 23 | "http://www.kxdaili.com/dailiip.html", 24 | "http://www.kxdaili.com/dailiip/2/1.html", 25 | } 26 | for i := 2; i < 5; i++ { 27 | list = append(list, fmt.Sprintf("http://www.kxdaili.com/dailiip/1/%d.html", i)) 28 | list = append(list, fmt.Sprintf("http://www.kxdaili.com/dailiip/2/%d.html", i)) 29 | } 30 | return list 31 | } 32 | func (s *getProxyKxDaili) GetContentHtml(requestUrl string) string { 33 | logger.Info("get proxy from kxdaili.com", logger.Fields{"url": requestUrl}) 34 | data, err := global.SimpleGet(requestUrl) 35 | if err != nil || data == nil { 36 | logger.Error("get proxy from kxdaili.com fail", logger.Fields{"err": err, "data": data}) 37 | return "" 38 | } 39 | return data.Body 40 | } 41 | 42 | func (s *getProxyKxDaili) ParseHtml(body string) [][]string { 43 | 44 | doc, err := goquery.NewDocumentFromReader(strings.NewReader(body)) 45 | if err != nil { 46 | logger.Error("read fail", logger.Fields{"err": err}) 47 | return nil 48 | } 49 | var proxyList [][]string 50 | doc.Find("tbody > tr").Each(func(i int, selection *goquery.Selection) { 51 | td := selection.ChildrenFiltered("td").First() 52 | host := strings.TrimSpace(td.Text()) 53 | td2 := selection.ChildrenFiltered("td").Eq(1) 54 | port := strings.TrimSpace(td2.Text()) 55 | 56 | if !common.CheckProxyFormat(host, port) { 57 | logger.Error(consts.PROXY_FORMAT_ERROR, logger.Fields{"host": host, "port": port}) 58 | return 59 | } 60 | proxyArr := []string{host, port} 61 | proxyList = append(proxyList, proxyArr) 62 | }) 63 | return proxyList 64 | } 65 | -------------------------------------------------------------------------------- /service/proxy_getter/list_plus.go: -------------------------------------------------------------------------------- 1 | package proxy_getter 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/PuerkitoBio/goquery" 8 | "github.com/tongsq/go-lib/logger" 9 | "github.com/tongsq/go-lib/request" 10 | "proxy-collect/consts" 11 | "proxy-collect/service/common" 12 | ) 13 | 14 | func NewGetProxyProxyList() *getProxyProxyList { 15 | return &getProxyProxyList{} 16 | } 17 | 18 | type getProxyProxyList struct { 19 | } 20 | 21 | func (s *getProxyProxyList) GetUrlList() []string { 22 | list := []string{ 23 | "https://list.proxylistplus.com/Fresh-HTTP-Proxy-List-1", 24 | } 25 | for i := 2; i < 6; i++ { 26 | list = append(list, fmt.Sprintf("https://list.proxylistplus.com/Fresh-HTTP-Proxy-List-%d", i)) 27 | } 28 | return list 29 | } 30 | func (s *getProxyProxyList) GetContentHtml(requestUrl string) string { 31 | h := &request.HeaderDto{ 32 | UserAgent: consts.USER_AGENT, 33 | UpgradeInsecureRequests: "1", 34 | Referer: "https://list.proxylistplus.com/update-2", 35 | } 36 | 37 | logger.Info("get proxy from list.proxylistplus.com", logger.Fields{"url": requestUrl}) 38 | data, err := request.Get(requestUrl, request.NewOptions().WithHeader(h)) 39 | if err != nil || data == nil { 40 | logger.Error("get proxy from list.proxylistplus.com fail", logger.Fields{"err": err, "data": data}) 41 | return "" 42 | } 43 | return data.Body 44 | } 45 | 46 | func (s *getProxyProxyList) ParseHtml(body string) [][]string { 47 | 48 | doc, err := goquery.NewDocumentFromReader(strings.NewReader(body)) 49 | if err != nil { 50 | logger.Error(consts.GO_QUERY_READ_ERROR, logger.Fields{"err": err}) 51 | return nil 52 | } 53 | 54 | var proxyList [][]string 55 | doc.Find("tr").Each(func(i int, selection *goquery.Selection) { 56 | td := selection.ChildrenFiltered("td").Eq(1) 57 | host := strings.TrimSpace(td.Text()) 58 | td2 := selection.ChildrenFiltered("td").Eq(2) 59 | port := strings.TrimSpace(td2.Text()) 60 | 61 | if !common.CheckProxyFormat(host, port) { 62 | return 63 | } 64 | proxyArr := []string{host, port} 65 | proxyList = append(proxyList, proxyArr) 66 | }) 67 | return proxyList 68 | } 69 | -------------------------------------------------------------------------------- /service/proxy_getter/nima.go: -------------------------------------------------------------------------------- 1 | package proxy_getter 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/PuerkitoBio/goquery" 8 | "github.com/tongsq/go-lib/logger" 9 | "github.com/tongsq/go-lib/request" 10 | "proxy-collect/consts" 11 | ) 12 | 13 | func NewGetProxyNima() *getProxyNima { 14 | return &getProxyNima{} 15 | } 16 | 17 | type getProxyNima struct { 18 | } 19 | 20 | func (s *getProxyNima) GetUrlList() []string { 21 | list := []string{ 22 | "http://www.nimadaili.com/https/", 23 | } 24 | for i := 2; i < 6; i++ { 25 | list = append(list, fmt.Sprintf("http://www.nimadaili.com/https/%d/", i)) 26 | } 27 | return list 28 | } 29 | func (s *getProxyNima) GetContentHtml(requestUrl string) string { 30 | h := &request.HeaderDto{ 31 | UserAgent: consts.USER_AGENT, 32 | Host: "www.nimadaili.com", 33 | Referer: "http://www.nimadaili.com/https/3/", 34 | UpgradeInsecureRequests: "1", 35 | } 36 | 37 | logger.Info("get proxy from nimadaili", logger.Fields{"url": requestUrl}) 38 | data, err := request.Get(requestUrl, request.NewOptions().WithHeader(h)) 39 | if err != nil || data == nil { 40 | logger.Error("get proxy from nimadaili fail", logger.Fields{"err": err, "data": data}) 41 | return "" 42 | } 43 | return data.Body 44 | } 45 | 46 | func (s *getProxyNima) ParseHtml(body string) [][]string { 47 | 48 | doc, err := goquery.NewDocumentFromReader(strings.NewReader(body)) 49 | if err != nil { 50 | logger.Error(consts.GO_QUERY_READ_ERROR, logger.Fields{"err": err}) 51 | return nil 52 | } 53 | var proxyList [][]string 54 | doc.Find("tbody > tr").Each(func(i int, selection *goquery.Selection) { 55 | td := selection.ChildrenFiltered("td").First() 56 | proxyStr := td.Text() 57 | proxyStr = strings.TrimSpace(proxyStr) 58 | proxyArr := strings.Split(proxyStr, ":") 59 | if len(proxyArr) != 2 { 60 | logger.Error(consts.PROXY_FORMAT_ERROR, logger.Fields{"proxyStr": proxyStr}) 61 | return 62 | } 63 | proxyList = append(proxyList, proxyArr) 64 | }) 65 | return proxyList 66 | } 67 | -------------------------------------------------------------------------------- /service/proxy_getter/pachongdaili.go: -------------------------------------------------------------------------------- 1 | package proxy_getter 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | "strings" 7 | 8 | "github.com/PuerkitoBio/goquery" 9 | "github.com/tongsq/go-lib/logger" 10 | "github.com/tongsq/go-lib/request" 11 | "proxy-collect/consts" 12 | ) 13 | 14 | func NewGetProxyPachong() *Pachong { 15 | return &Pachong{} 16 | } 17 | 18 | type Pachong struct { 19 | } 20 | 21 | func (s *Pachong) GetUrlList() []string { 22 | u := "http://www.pachongdaili.com/free/freelist1.html" 23 | body := s.GetContentHtml(u) 24 | doc, err := goquery.NewDocumentFromReader(strings.NewReader(body)) 25 | if err != nil { 26 | logger.Error(consts.GO_QUERY_READ_ERROR, logger.Fields{"err": err}) 27 | return nil 28 | } 29 | list := []string{} 30 | doc.Find("ul > li").Each(func(i int, selection *goquery.Selection) { 31 | a := selection.ChildrenFiltered("A").First() 32 | href, _ := a.Attr("href") 33 | list = append(list, fmt.Sprintf("http://www.pachongdaili.com%s", href)) 34 | }) 35 | if len(list) > 5 { 36 | return list[0:5] 37 | } 38 | return list 39 | } 40 | 41 | func (s *Pachong) GetContentHtml(requestUrl string) string { 42 | 43 | h := &request.HeaderDto{ 44 | UserAgent: consts.USER_AGENT, 45 | UpgradeInsecureRequests: "1", 46 | Host: "pachongdaili.com", 47 | Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", 48 | AcceptEncoding: "gzip, deflate, br", 49 | AcceptLanguage: "zh-CN,zh;q=0.9", 50 | SecFetchDest: "document", 51 | SecFetchMode: "navigate", 52 | } 53 | 54 | logger.Info("get proxy from pachongdaili.com", logger.Fields{"url": requestUrl}) 55 | data, err := request.Get(requestUrl, request.NewOptions().WithHeader(h)) 56 | if err != nil || data == nil { 57 | logger.Error("get proxy from pachongdaili.com fail", logger.Fields{"err": err, "data": data}) 58 | return "" 59 | } 60 | return data.Body 61 | } 62 | 63 | func (s *Pachong) ParseHtml(body string) [][]string { 64 | 65 | var proxyList [][]string 66 | re := regexp.MustCompile(`(\d+\.\d+\.\d+\.\d+):(\d+)`) 67 | matched := re.FindAllStringSubmatch(body, -1) 68 | for _, match := range matched { 69 | proxyArr := []string{match[1], match[2]} 70 | proxyList = append(proxyList, proxyArr) 71 | } 72 | return proxyList 73 | } 74 | -------------------------------------------------------------------------------- /service/proxy_getter/proxy_66ip.go: -------------------------------------------------------------------------------- 1 | package proxy_getter 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/PuerkitoBio/goquery" 8 | "github.com/tongsq/go-lib/logger" 9 | "github.com/tongsq/go-lib/request" 10 | "golang.org/x/text/encoding/simplifiedchinese" 11 | "proxy-collect/consts" 12 | "proxy-collect/service/common" 13 | ) 14 | 15 | func NewGetProxy66ip() *getProxy66ip { 16 | return &getProxy66ip{} 17 | } 18 | 19 | type getProxy66ip struct { 20 | } 21 | 22 | func (s *getProxy66ip) GetUrlList() []string { 23 | list := []string{ 24 | "http://www.66ip.cn/index.html", 25 | } 26 | for i := 2; i < 6; i++ { 27 | list = append(list, fmt.Sprintf("http://www.66ip.cn/%d.html", i)) 28 | } 29 | return list 30 | } 31 | 32 | func (s *getProxy66ip) GetContentHtml(requestUrl string) string { 33 | h := &request.HeaderDto{ 34 | UserAgent: consts.USER_AGENT, 35 | Host: "www.66ip.cn", 36 | Referer: "http://www.66ip.cn/2.html", 37 | UpgradeInsecureRequests: "1", 38 | } 39 | logger.Info("get proxy from 66ip", logger.Fields{"url": requestUrl}) 40 | data, err := request.Get(requestUrl, request.NewOptions().WithHeader(h)) 41 | if err != nil || data == nil { 42 | logger.Error("get proxy from 66ip fail", logger.Fields{"err": err, "data": data}) 43 | return "" 44 | } 45 | return data.Body 46 | } 47 | 48 | func (s *getProxy66ip) ParseHtml(body string) [][]string { 49 | body, err := simplifiedchinese.GBK.NewDecoder().String(body) 50 | if err != nil { 51 | logger.FError("chang charset to utf8 fail") 52 | return nil 53 | } 54 | doc, err := goquery.NewDocumentFromReader(strings.NewReader(body)) 55 | if err != nil { 56 | logger.Error(consts.GO_QUERY_READ_ERROR, logger.Fields{"err": err}) 57 | return nil 58 | } 59 | var proxyList [][]string 60 | 61 | doc.Find("tr").Each(func(i int, selection *goquery.Selection) { 62 | td := selection.ChildrenFiltered("td").First() 63 | td2 := selection.ChildrenFiltered("td").Eq(1) 64 | proxyHost := td.Text() 65 | proxyPort := td2.Text() 66 | if !common.CheckProxyFormat(proxyHost, proxyPort) { 67 | return 68 | } 69 | proxyArr := []string{strings.TrimSpace(proxyHost), strings.TrimSpace(proxyPort)} 70 | proxyList = append(proxyList, proxyArr) 71 | }) 72 | return proxyList 73 | } 74 | -------------------------------------------------------------------------------- /service/proxy_getter/proxy_7yip.go: -------------------------------------------------------------------------------- 1 | package proxy_getter 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/PuerkitoBio/goquery" 8 | "github.com/tongsq/go-lib/logger" 9 | "github.com/tongsq/go-lib/request" 10 | "proxy-collect/consts" 11 | "proxy-collect/service/common" 12 | ) 13 | 14 | func NewGetProxy7Yip() *getProxy7Yip { 15 | return &getProxy7Yip{} 16 | } 17 | 18 | type getProxy7Yip struct { 19 | } 20 | 21 | func (s *getProxy7Yip) GetUrlList() []string { 22 | list := []string{ 23 | "https://www.7yip.cn/free/", 24 | } 25 | for i := 2; i < 6; i++ { 26 | list = append(list, fmt.Sprintf("https://www.7yip.cn/free/?action=china&page=%d", i)) 27 | } 28 | return list 29 | } 30 | func (s *getProxy7Yip) GetContentHtml(requestUrl string) string { 31 | h := &request.HeaderDto{ 32 | UserAgent: consts.USER_AGENT, 33 | UpgradeInsecureRequests: "1", 34 | Host: "www.7yip.cn", 35 | Referer: "https://www.7yip.cn/", 36 | } 37 | logger.Info("get proxy from 7yip", logger.Fields{"url": requestUrl}) 38 | data, err := request.Get(requestUrl, request.NewOptions().WithHeader(h)) 39 | if err != nil || data == nil { 40 | logger.Error("get proxy from 7yip fail", logger.Fields{"err": err, "data": data}) 41 | } 42 | return data.Body 43 | } 44 | 45 | func (s *getProxy7Yip) ParseHtml(body string) [][]string { 46 | 47 | doc, err := goquery.NewDocumentFromReader(strings.NewReader(body)) 48 | if err != nil { 49 | logger.Error(consts.GO_QUERY_READ_ERROR, logger.Fields{"err": err}) 50 | return nil 51 | } 52 | var proxyList [][]string 53 | doc.Find("tbody > tr").Each(func(i int, selection *goquery.Selection) { 54 | td := selection.ChildrenFiltered("td").First() 55 | host := strings.TrimSpace(td.Text()) 56 | td2 := selection.ChildrenFiltered("td").Eq(1) 57 | port := strings.TrimSpace(td2.Text()) 58 | 59 | if !common.CheckProxyFormat(host, port) { 60 | logger.Error(consts.PROXY_FORMAT_ERROR, logger.Fields{"host": host, "port": port}) 61 | return 62 | } 63 | proxyArr := []string{host, port} 64 | proxyList = append(proxyList, proxyArr) 65 | }) 66 | 67 | return proxyList 68 | } 69 | 70 | //func (s *getProxy7Yip) GetSource() string { 71 | // _, file, _, _ := runtime.Caller(0) 72 | // arr := strings.Split(file, "/") 73 | // name := arr[len(arr)-1] 74 | // return name[0:len(name)-3] 75 | //} 76 | -------------------------------------------------------------------------------- /service/proxy_getter/proxy_89ip.go: -------------------------------------------------------------------------------- 1 | package proxy_getter 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/PuerkitoBio/goquery" 8 | "github.com/tongsq/go-lib/logger" 9 | "github.com/tongsq/go-lib/request" 10 | "proxy-collect/consts" 11 | "proxy-collect/service/common" 12 | ) 13 | 14 | func NewGetProxy89Ip() *getProxy89Ip { 15 | return &getProxy89Ip{} 16 | } 17 | 18 | type getProxy89Ip struct { 19 | } 20 | 21 | func (s *getProxy89Ip) GetUrlList() []string { 22 | list := []string{ 23 | "https://www.89ip.cn/", 24 | } 25 | for i := 2; i < 6; i++ { 26 | list = append(list, fmt.Sprintf("https://www.89ip.cn/index_%d.html", i)) 27 | } 28 | return list 29 | } 30 | func (s *getProxy89Ip) GetContentHtml(requestUrl string) string { 31 | h := &request.HeaderDto{ 32 | UserAgent: consts.USER_AGENT, 33 | UpgradeInsecureRequests: "1", 34 | Host: "www.89ip.cn", 35 | Referer: "https://www.89ip.cn/", 36 | } 37 | 38 | logger.Info("get proxy from 89ip", logger.Fields{"url": requestUrl}) 39 | data, err := request.Get(requestUrl, request.NewOptions().WithHeader(h)) 40 | if err != nil || data == nil { 41 | logger.Error("get proxy from 89ip fail", logger.Fields{"err": err, "data": data}) 42 | return "" 43 | } 44 | return data.Body 45 | } 46 | 47 | func (s *getProxy89Ip) ParseHtml(body string) [][]string { 48 | 49 | doc, err := goquery.NewDocumentFromReader(strings.NewReader(body)) 50 | if err != nil { 51 | logger.Error(consts.GO_QUERY_READ_ERROR, logger.Fields{"err": err}) 52 | return nil 53 | } 54 | var proxyList [][]string 55 | doc.Find("tbody > tr").Each(func(i int, selection *goquery.Selection) { 56 | td := selection.ChildrenFiltered("td").First() 57 | host := strings.TrimSpace(td.Text()) 58 | td2 := selection.ChildrenFiltered("td").Eq(1) 59 | port := strings.TrimSpace(td2.Text()) 60 | 61 | if !common.CheckProxyFormat(host, port) { 62 | logger.Error(consts.PROXY_FORMAT_ERROR, logger.Fields{"host": host, "port": port}) 63 | return 64 | } 65 | proxyArr := []string{host, port} 66 | proxyList = append(proxyList, proxyArr) 67 | }) 68 | return proxyList 69 | } 70 | -------------------------------------------------------------------------------- /service/proxy_getter/seofangfa.go: -------------------------------------------------------------------------------- 1 | package proxy_getter 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/PuerkitoBio/goquery" 7 | "github.com/tongsq/go-lib/logger" 8 | "github.com/tongsq/go-lib/request" 9 | "proxy-collect/consts" 10 | ) 11 | 12 | func NewGetProxySeofangfa() *Seofangfa { 13 | return &Seofangfa{} 14 | } 15 | 16 | type Seofangfa struct { 17 | } 18 | 19 | func (s *Seofangfa) GetUrlList() []string { 20 | return []string{"https://proxy.seofangfa.com/"} 21 | } 22 | 23 | func (s *Seofangfa) GetContentHtml(requestUrl string) string { 24 | 25 | h := &request.HeaderDto{ 26 | UserAgent: consts.USER_AGENT, 27 | UpgradeInsecureRequests: "1", 28 | Host: "proxy.seofangfa.com", 29 | Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", 30 | AcceptEncoding: "gzip, deflate, br", 31 | AcceptLanguage: "zh-CN,zh;q=0.9", 32 | SecFetchDest: "document", 33 | SecFetchMode: "navigate", 34 | } 35 | logger.Info("get proxy from proxy.seofangfa.com", logger.Fields{"url": requestUrl}) 36 | 37 | data, err := request.Get(requestUrl, request.NewOptions().WithHeader(h)) 38 | if err != nil || data == nil { 39 | logger.Error("get proxy from proxy.seofangfa.com fail", logger.Fields{"err": err, "data": data}) 40 | return "" 41 | } 42 | return data.Body 43 | } 44 | 45 | func (s *Seofangfa) ParseHtml(body string) [][]string { 46 | 47 | doc, err := goquery.NewDocumentFromReader(strings.NewReader(body)) 48 | if err != nil { 49 | logger.Error(consts.GO_QUERY_READ_ERROR, logger.Fields{"err": err}) 50 | return nil 51 | } 52 | var proxyList [][]string 53 | doc.Find("tbody > tr").Each(func(i int, selection *goquery.Selection) { 54 | td := selection.ChildrenFiltered("td").First() 55 | proxyHost := td.Text() 56 | td2 := selection.ChildrenFiltered("td").Eq(1) 57 | proxyPort := td2.Text() 58 | if proxyHost == "" || proxyPort == "" { 59 | logger.FError("parse html node fail") 60 | } 61 | proxyArr := []string{strings.TrimSpace(proxyHost), strings.TrimSpace(proxyPort)} 62 | proxyList = append(proxyList, proxyArr) 63 | }) 64 | return proxyList 65 | } 66 | -------------------------------------------------------------------------------- /service/proxy_getter/xici.go: -------------------------------------------------------------------------------- 1 | package proxy_getter 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/PuerkitoBio/goquery" 8 | "github.com/tongsq/go-lib/logger" 9 | "github.com/tongsq/go-lib/request" 10 | "proxy-collect/consts" 11 | ) 12 | 13 | func NewGetProxyXici() *getProxyXici { 14 | return &getProxyXici{} 15 | } 16 | 17 | type getProxyXici struct { 18 | } 19 | 20 | func (s *getProxyXici) GetUrlList() []string { 21 | list := []string{ 22 | "http://www.xicidaili.com/wn/", 23 | } 24 | for i := 2; i < 6; i++ { 25 | list = append(list, fmt.Sprintf("http://www.xicidaili.com/wn/%d", i)) 26 | } 27 | return list 28 | } 29 | 30 | func (s *getProxyXici) GetContentHtml(requestUrl string) string { 31 | 32 | h := &request.HeaderDto{ 33 | UserAgent: consts.USER_AGENT, 34 | Host: "www.xicidaili.com", 35 | Referer: "http://www.xicidaili.com", 36 | UpgradeInsecureRequests: "1", 37 | } 38 | 39 | logger.Info("get proxy from xicidaili", logger.Fields{"url": requestUrl}) 40 | data, err := request.Get(requestUrl, request.NewOptions().WithHeader(h)) 41 | if err != nil || data == nil { 42 | logger.Error("get proxy from xicidaili fail", logger.Fields{"err": err, "data": data}) 43 | return "" 44 | } 45 | return data.Body 46 | } 47 | 48 | func (s *getProxyXici) ParseHtml(body string) [][]string { 49 | doc, err := goquery.NewDocumentFromReader(strings.NewReader(body)) 50 | if err != nil { 51 | logger.Error(consts.GO_QUERY_READ_ERROR, logger.Fields{"err": err}) 52 | return nil 53 | } 54 | var proxyList [][]string 55 | doc.Find("tbody > tr").Each(func(i int, selection *goquery.Selection) { 56 | td := selection.ChildrenFiltered("td").First() 57 | proxyStr := td.Text() 58 | proxyStr = strings.TrimSpace(proxyStr) 59 | proxyArr := strings.Split(proxyStr, ":") 60 | if len(proxyArr) != 2 { 61 | logger.Error(consts.PROXY_FORMAT_ERROR, logger.Fields{"proxyStr": proxyStr}) 62 | return 63 | } 64 | proxyList = append(proxyList, proxyArr) 65 | }) 66 | return proxyList 67 | } 68 | -------------------------------------------------------------------------------- /service/proxy_getter/xila.go: -------------------------------------------------------------------------------- 1 | package proxy_getter 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/PuerkitoBio/goquery" 8 | "github.com/tongsq/go-lib/logger" 9 | "github.com/tongsq/go-lib/request" 10 | "proxy-collect/consts" 11 | ) 12 | 13 | func NewGetProxyXila() *getProxyXila { 14 | return &getProxyXila{} 15 | } 16 | 17 | type getProxyXila struct { 18 | } 19 | 20 | func (s *getProxyXila) GetUrlList() []string { 21 | list := []string{ 22 | "http://www.xiladaili.com/https/", 23 | } 24 | for i := 2; i < 6; i++ { 25 | list = append(list, fmt.Sprintf("http://www.xiladaili.com/https/%d/", i)) 26 | } 27 | return list 28 | } 29 | 30 | func (s *getProxyXila) GetContentHtml(requestUrl string) string { 31 | 32 | h := &request.HeaderDto{ 33 | UserAgent: consts.USER_AGENT, 34 | Host: "www.xiladaili.com", 35 | Referer: "http://www.xiladaili.com/https/", 36 | UpgradeInsecureRequests: "1", 37 | } 38 | logger.Info("get proxy from xiladaili", logger.Fields{"url": requestUrl}) 39 | data, err := request.Get(requestUrl, request.NewOptions().WithHeader(h)) 40 | if err != nil || data == nil { 41 | logger.Error("get proxy from xiladaili fail", logger.Fields{"err": err, "data": data}) 42 | return "" 43 | } 44 | return data.Body 45 | } 46 | 47 | func (s *getProxyXila) ParseHtml(body string) [][]string { 48 | doc, err := goquery.NewDocumentFromReader(strings.NewReader(body)) 49 | if err != nil { 50 | logger.Error(consts.GO_QUERY_READ_ERROR, logger.Fields{"err": err}) 51 | return nil 52 | } 53 | var proxyList [][]string 54 | doc.Find("tbody > tr").Each(func(i int, selection *goquery.Selection) { 55 | td := selection.ChildrenFiltered("td").First() 56 | proxyStr := td.Text() 57 | proxyStr = strings.TrimSpace(proxyStr) 58 | proxyArr := strings.Split(proxyStr, ":") 59 | if len(proxyArr) != 2 { 60 | logger.Error(consts.PROXY_FORMAT_ERROR, logger.Fields{"proxyStr": proxyStr}) 61 | return 62 | } 63 | proxyList = append(proxyList, proxyArr) 64 | }) 65 | return proxyList 66 | } 67 | -------------------------------------------------------------------------------- /service/proxy_getter/xsdaili.go: -------------------------------------------------------------------------------- 1 | package proxy_getter 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | "strings" 7 | 8 | "github.com/PuerkitoBio/goquery" 9 | "github.com/tongsq/go-lib/logger" 10 | "github.com/tongsq/go-lib/request" 11 | "proxy-collect/consts" 12 | ) 13 | 14 | func NewGetProxyXsdaili() *Xsdaili { 15 | return &Xsdaili{} 16 | } 17 | 18 | type Xsdaili struct { 19 | } 20 | 21 | func (s *Xsdaili) GetUrlList() []string { 22 | u := "http://www.xsdaili.cn/" 23 | body := s.GetContentHtml(u) 24 | doc, err := goquery.NewDocumentFromReader(strings.NewReader(body)) 25 | if err != nil { 26 | logger.Error(consts.GO_QUERY_READ_ERROR, logger.Fields{"err": err}) 27 | return nil 28 | } 29 | list := []string{} 30 | doc.Find("div.title").Each(func(i int, selection *goquery.Selection) { 31 | a := selection.ChildrenFiltered("A").First() 32 | href, _ := a.Attr("href") 33 | list = append(list, fmt.Sprintf("http://www.xsdaili.cn%s", href)) 34 | }) 35 | if len(list) > 5 { 36 | return list[0:5] 37 | } 38 | return list 39 | } 40 | 41 | func (s *Xsdaili) GetContentHtml(requestUrl string) string { 42 | 43 | h := &request.HeaderDto{ 44 | UserAgent: consts.USER_AGENT, 45 | Host: "www.xsdaili.cn", 46 | } 47 | 48 | logger.Info("get proxy from xsdaili.com", logger.Fields{"url": requestUrl}) 49 | data, err := request.Get(requestUrl, request.NewOptions().WithHeader(h)) 50 | if err != nil || data == nil { 51 | logger.Error("get proxy from zdaye.com fail", logger.Fields{"err": err, "data": data}) 52 | return "" 53 | } 54 | return data.Body 55 | } 56 | 57 | func (s *Xsdaili) ParseHtml(body string) [][]string { 58 | 59 | var proxyList [][]string 60 | re := regexp.MustCompile(`(\d+\.\d+\.\d+\.\d+):(\d+)`) 61 | matched := re.FindAllStringSubmatch(body, -1) 62 | for _, match := range matched { 63 | proxyArr := []string{match[1], match[2]} 64 | proxyList = append(proxyList, proxyArr) 65 | } 66 | return proxyList 67 | } 68 | -------------------------------------------------------------------------------- /service/proxy_getter/yqie.go: -------------------------------------------------------------------------------- 1 | package proxy_getter 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/PuerkitoBio/goquery" 7 | "github.com/tongsq/go-lib/logger" 8 | "github.com/tongsq/go-lib/request" 9 | "proxy-collect/consts" 10 | ) 11 | 12 | func NewGetProxyYqie() *Yqie { 13 | return &Yqie{} 14 | } 15 | 16 | type Yqie struct { 17 | } 18 | 19 | func (s *Yqie) GetUrlList() []string { 20 | return []string{"http://ip.yqie.com/ipproxy.htm"} 21 | } 22 | 23 | func (s *Yqie) GetContentHtml(requestUrl string) string { 24 | 25 | h := &request.HeaderDto{ 26 | UserAgent: consts.USER_AGENT, 27 | UpgradeInsecureRequests: "1", 28 | Host: "ip.yqie.com", 29 | Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", 30 | AcceptEncoding: "gzip, deflate, br", 31 | AcceptLanguage: "zh-CN,zh;q=0.9", 32 | SecFetchDest: "document", 33 | SecFetchMode: "navigate", 34 | } 35 | 36 | logger.Info("get proxy from ip.yqie.com", logger.Fields{"url": requestUrl}) 37 | 38 | data, err := request.Get(requestUrl, request.NewOptions().WithHeader(h)) 39 | 40 | if err != nil || data == nil { 41 | logger.Error("get proxy from ip.yqie.com fail", logger.Fields{"err": err, "data": data}) 42 | return "" 43 | } 44 | return data.Body 45 | } 46 | 47 | func (s *Yqie) ParseHtml(body string) [][]string { 48 | 49 | doc, err := goquery.NewDocumentFromReader(strings.NewReader(body)) 50 | if err != nil { 51 | logger.Error(consts.GO_QUERY_READ_ERROR, logger.Fields{"err": err}) 52 | return nil 53 | } 54 | var proxyList [][]string 55 | doc.Find("tbody > tr").Each(func(i int, selection *goquery.Selection) { 56 | td := selection.ChildrenFiltered("td").First() 57 | proxyHost := td.Text() 58 | td2 := selection.ChildrenFiltered("td").Eq(1) 59 | proxyPort := td2.Text() 60 | if proxyHost == "" || proxyPort == "" { 61 | logger.FError("parse html node fail") 62 | return 63 | } 64 | proxyArr := []string{strings.TrimSpace(proxyHost), strings.TrimSpace(proxyPort)} 65 | proxyList = append(proxyList, proxyArr) 66 | }) 67 | return proxyList 68 | } 69 | -------------------------------------------------------------------------------- /service/proxy_getter/zdaye.go: -------------------------------------------------------------------------------- 1 | package proxy_getter 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "regexp" 7 | "strings" 8 | "time" 9 | 10 | "github.com/PuerkitoBio/goquery" 11 | "github.com/tongsq/go-lib/logger" 12 | "github.com/tongsq/go-lib/request" 13 | "proxy-collect/consts" 14 | "proxy-collect/dao" 15 | ) 16 | 17 | func NewGetProxyZdaye() *Zdaye { 18 | return &Zdaye{} 19 | } 20 | 21 | type Zdaye struct { 22 | } 23 | 24 | func (s *Zdaye) GetUrlList() []string { 25 | u := "https://www.zdaye.com/dayProxy/1.html" 26 | body := s.GetContentHtml(u) 27 | doc, err := goquery.NewDocumentFromReader(strings.NewReader(body)) 28 | if err != nil { 29 | logger.Error(consts.GO_QUERY_READ_ERROR, logger.Fields{"err": err}) 30 | return nil 31 | } 32 | list := []string{} 33 | doc.Find("p.thread_tags").Each(func(i int, selection *goquery.Selection) { 34 | a := selection.ChildrenFiltered("A").First() 35 | href, _ := a.Attr("href") 36 | list = append(list, fmt.Sprintf("https://www.zdaye.com%s", href)) 37 | }) 38 | if len(list) > 5 { 39 | return list[0:5] 40 | } 41 | return list 42 | } 43 | 44 | func (s *Zdaye) GetContentHtml(requestUrl string) string { 45 | 46 | h := &request.HeaderDto{ 47 | UserAgent: consts.USER_AGENT, 48 | UpgradeInsecureRequests: "1", 49 | Host: "www.zdaye.com", 50 | Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", 51 | AcceptEncoding: "gzip, deflate, br", 52 | AcceptLanguage: "zh-CN,zh;q=0.9", 53 | SecFetchDest: "document", 54 | SecFetchMode: "navigate", 55 | } 56 | 57 | logger.Info("get proxy from zdaye.com", logger.Fields{"url": requestUrl}) 58 | proxies, err := dao.ProxyDao.GetActiveList() 59 | var data *request.HttpResultDto 60 | if err == nil && len(proxies) > 0 { 61 | rand.Seed(time.Now().Unix()) 62 | i := rand.Intn(len(proxies)) 63 | proxy := proxies[i] 64 | data, err = request.Get(requestUrl, request.NewOptions().WithHeader(h).WithProxy(&request.ProxyDto{Host: proxy.Host, Port: proxy.Port})) 65 | } else { 66 | data, err = request.Get(requestUrl, request.NewOptions().WithHeader(h)) 67 | } 68 | if err != nil || data == nil { 69 | logger.Error("get proxy from zdaye.com fail", logger.Fields{"err": err, "data": data}) 70 | return "" 71 | } 72 | return data.Body 73 | } 74 | 75 | func (s *Zdaye) ParseHtml(body string) [][]string { 76 | 77 | var proxyList [][]string 78 | re := regexp.MustCompile(`(\d+\.\d+\.\d+\.\d+):(\d+)`) 79 | matched := re.FindAllStringSubmatch(body, -1) 80 | for _, match := range matched { 81 | proxyArr := []string{match[1], match[2]} 82 | proxyList = append(proxyList, proxyArr) 83 | } 84 | return proxyList 85 | } 86 | -------------------------------------------------------------------------------- /service/proxy_getter/zdaye_index.go: -------------------------------------------------------------------------------- 1 | package proxy_getter 2 | 3 | import ( 4 | "regexp" 5 | 6 | "github.com/tongsq/go-lib/logger" 7 | "github.com/tongsq/go-lib/request" 8 | "proxy-collect/consts" 9 | ) 10 | 11 | func NewGetProxyZdayeIndex() *ZdayeIndex { 12 | return &ZdayeIndex{} 13 | } 14 | 15 | type ZdayeIndex struct { 16 | } 17 | 18 | func (s *ZdayeIndex) GetUrlList() []string { 19 | return []string{"https://www.zdaye.com/dayProxy/1.html"} 20 | } 21 | 22 | func (s *ZdayeIndex) GetContentHtml(requestUrl string) string { 23 | 24 | h := &request.HeaderDto{ 25 | UserAgent: consts.USER_AGENT, 26 | UpgradeInsecureRequests: "1", 27 | Host: "www.zdaye.com", 28 | Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", 29 | AcceptEncoding: "gzip, deflate, br", 30 | AcceptLanguage: "zh-CN,zh;q=0.9", 31 | SecFetchDest: "document", 32 | SecFetchMode: "navigate", 33 | } 34 | 35 | logger.Info("get proxy from zdaye.com index", logger.Fields{"url": requestUrl}) 36 | 37 | data, err := request.Get(requestUrl, request.NewOptions().WithHeader(h)) 38 | 39 | if err != nil || data == nil { 40 | logger.Error("get proxy from zdaye.com index fail", logger.Fields{"err": err, "data": data}) 41 | return "" 42 | } 43 | return data.Body 44 | } 45 | 46 | func (s *ZdayeIndex) ParseHtml(body string) [][]string { 47 | 48 | var proxyList [][]string 49 | re := regexp.MustCompile(`(\d+\.\d+\.\d+\.\d+):(\d+)`) 50 | matched := re.FindAllStringSubmatch(body, -1) 51 | for _, match := range matched { 52 | proxyArr := []string{match[1], match[2]} 53 | proxyList = append(proxyList, proxyArr) 54 | } 55 | return proxyList 56 | } 57 | -------------------------------------------------------------------------------- /service/proxy_getter_interface.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | type ProxyGetterInterface interface { 4 | GetContentHtml(requestUrl string) string 5 | /** 6 | result format: 7 | [ 8 | ["ip", "port", "protocol", "user", "password"] 9 | ] 10 | */ 11 | ParseHtml(body string) [][]string 12 | GetUrlList() []string 13 | } 14 | -------------------------------------------------------------------------------- /service/proxy_service.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "regexp" 7 | "sync" 8 | "time" 9 | 10 | "github.com/tongsq/go-lib/component" 11 | "github.com/tongsq/go-lib/logger" 12 | "github.com/tongsq/go-lib/request" 13 | "proxy-collect/config" 14 | "proxy-collect/consts" 15 | "proxy-collect/dao" 16 | "proxy-collect/dto" 17 | "proxy-collect/global" 18 | "proxy-collect/service/ip" 19 | ) 20 | 21 | func NewProxyService() *proxyService { 22 | return &proxyService{} 23 | } 24 | 25 | type proxyService struct { 26 | } 27 | 28 | func (s proxyService) TransferProxyDto(proxy *dto.ProxyDto) *request.ProxyDto { 29 | return &request.ProxyDto{ 30 | Host: proxy.Host, 31 | Port: proxy.Port, 32 | Proto: proxy.Proto, 33 | User: proxy.User, 34 | Password: proxy.Password, 35 | } 36 | } 37 | 38 | func (s *proxyService) CheckIpStatus(proxy *dto.ProxyDto) (bool, int64) { 39 | u := "https://www.baidu.com" 40 | h := &request.HeaderDto{ 41 | UserAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.106 Safari/537.36", 42 | } 43 | t := time.Now() 44 | _, err := request.Get(u, request.NewOptions().WithHeader(h).WithProxy(s.TransferProxyDto(proxy)).WithTimeout(global.MaxPing)) 45 | duration := time.Now().Sub(t) 46 | return err == nil, int64(duration / time.Millisecond) 47 | } 48 | 49 | func (s *proxyService) CheckProxyAndSave(p dto.ProxyDto) { 50 | result, ping := s.CheckIpStatus(&p) 51 | p.Ping = ping 52 | if result { 53 | logger.Success("ip is success", logger.Fields{"proxy": p}) 54 | } else { 55 | logger.Debug("ip is fail", logger.Fields{"proxy": p}) 56 | } 57 | var status int8 = consts.STATUS_YES 58 | if !result { 59 | status = consts.STATUS_NO 60 | } 61 | 62 | proxyModel, err := dao.ProxyDao.GetOne(p.Host, p.Port, p.Proto) 63 | if err != nil { 64 | logger.Error("get model fail:%s", logger.Fields{"err": err}) 65 | return 66 | } 67 | if proxyModel == nil { 68 | if status == consts.STATUS_NO { 69 | return 70 | } 71 | _, err = dao.ProxyDao.Create(p, status) 72 | if err != nil { 73 | logger.Error("create proxy model fail", map[string]interface{}{"proxy": p}) 74 | } 75 | return 76 | } 77 | if status == consts.STATUS_YES { 78 | if proxyModel.CheckCount <= 20 { 79 | proxyModel.CheckCount = proxyModel.CheckCount + 1 80 | } 81 | if proxyModel.ActiveTime == 0 || proxyModel.Status == consts.STATUS_NO { 82 | proxyModel.ActiveTime = time.Now().Unix() 83 | } 84 | } else { 85 | proxyModel.ActiveTime = 0 86 | //set yes to no, need recheck 87 | if proxyModel.Status == consts.STATUS_YES && config.Get().RecheckCount > 0 { 88 | proxyModel.CheckCount = config.Get().RecheckCount 89 | status = consts.STATUS_RECHECK 90 | } else if proxyModel.Status == consts.STATUS_RECHECK { 91 | proxyModel.CheckCount = proxyModel.CheckCount - 1 92 | if proxyModel.CheckCount >= 0 { 93 | status = consts.STATUS_RECHECK 94 | } 95 | } 96 | if proxyModel.CheckCount < 0 && status == consts.STATUS_NO { 97 | logger.Debug("start delete fail proxy", map[string]interface{}{"proxy": proxyModel}) 98 | if err := dao.ProxyDao.Delete(p.Host, p.Port, p.Proto); err != nil { 99 | logger.Error("delete proxy fail", logger.Fields{"host": p.Host, "port": p.Port}) 100 | } 101 | return 102 | } 103 | proxyModel.CheckCount = proxyModel.CheckCount - 1 104 | } 105 | if p.Source != "" { 106 | proxyModel.Source = p.Source 107 | } 108 | //if ip is ok, update ip info 109 | if result && proxyModel.City == "" && config.Get().UpdateIpInfo { 110 | if ipInfo := ip.GetIpInfo(p.Host, p.Port); ipInfo != nil { 111 | proxyModel.City = ipInfo.City 112 | proxyModel.Country = ipInfo.Country 113 | proxyModel.Isp = ipInfo.Isp 114 | proxyModel.Region = ipInfo.Region 115 | } 116 | } 117 | proxyModel.Status = status 118 | proxyModel.UpdateTime = time.Now().Unix() 119 | proxyModel.User = p.User 120 | proxyModel.Password = p.Password 121 | proxyModel.Ping = p.Ping 122 | _ = dao.ProxyDao.Save(proxyModel) 123 | } 124 | 125 | func (s *proxyService) DoGetProxy(getProxyService ProxyGetterInterface, pool *component.Pool) { 126 | for _, requestUrl := range getProxyService.GetUrlList() { 127 | contentBody := getProxyService.GetContentHtml(requestUrl) 128 | if contentBody == "" { 129 | time.Sleep(time.Second * 5) 130 | continue 131 | } 132 | proxyList := getProxyService.ParseHtml(contentBody) 133 | logger.Info("get ip list:", logger.Fields{"list": proxyList}) 134 | var wg sync.WaitGroup = sync.WaitGroup{} 135 | wg.Add(1) 136 | go func(wg *sync.WaitGroup) { 137 | defer wg.Done() 138 | logger.FDebug("wait 10s ...") 139 | time.Sleep(time.Second * 10) 140 | }(&wg) 141 | for _, proxyArr := range proxyList { 142 | p := s.ParseProxyArr(proxyArr) 143 | p.Source = reflect.TypeOf(getProxyService).String()[14:] 144 | pool.RunTask(func() { s.CheckProxyAndSave(p) }) 145 | } 146 | 147 | wg.Wait() 148 | } 149 | } 150 | 151 | func (s *proxyService) CheckProxyFormat(host string, port string) bool { 152 | ok, _ := regexp.Match(`^[\d\.]+$`, []byte(host)) 153 | if !ok { 154 | return false 155 | } 156 | ok, _ = regexp.Match(`^\d+$`, []byte(port)) 157 | return ok 158 | } 159 | 160 | func (s *proxyService) ParseProxyArr(proxyArr []string) dto.ProxyDto { 161 | p := dto.ProxyDto{ 162 | Proto: consts.PROTO_HTTP, 163 | } 164 | for i, val := range proxyArr { 165 | switch i { 166 | case 0: 167 | p.Host = val 168 | case 1: 169 | p.Port = val 170 | case 2: 171 | p.Proto = val 172 | case 3: 173 | p.User = val 174 | case 4: 175 | p.Password = val 176 | } 177 | } 178 | return p 179 | } 180 | 181 | func (s *proxyService) GetProxyUrl(p dto.ProxyDto) string { 182 | if p.Proto == "" { 183 | p.Proto = consts.PROTO_HTTP 184 | } 185 | if p.User == "" { 186 | return fmt.Sprintf("%s://%s:%s", p.Proto, p.Host, p.Port) 187 | } else { 188 | return fmt.Sprintf("%s://%s:%s@%s:%s", p.Proto, p.User, p.Password, p.Host, p.Port) 189 | } 190 | } 191 | -------------------------------------------------------------------------------- /service/service.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "proxy-collect/consts" 5 | "proxy-collect/service/ip" 6 | "proxy-collect/service/proxy_getter" 7 | ) 8 | 9 | var ProxyService = NewProxyService() 10 | var GetProxy66ip = proxy_getter.NewGetProxy66ip() 11 | var GetProxyData5u = proxy_getter.NewGetProxyData5u() 12 | var GetProxyKuai = proxy_getter.NewGetProxyKuai() 13 | var GetProxyXila = proxy_getter.NewGetProxyXila() 14 | var GetProxyNima = proxy_getter.NewGetProxyNima() 15 | var GetProxyGuoBanjia = proxy_getter.NewGetProxyGuoBanJia() 16 | var GetProxyCoderBusy = proxy_getter.NewGetProxyCoderBusy() 17 | var GetProxyIp3366 = proxy_getter.NewGetProxyIp3366() 18 | var GetProxyIpJiangXianLi = proxy_getter.NewGetProxyIpJiangXianLi() 19 | var GetProxy89Ip = proxy_getter.NewGetProxy89Ip() 20 | var GetProxy7Yip = proxy_getter.NewGetProxy7Yip() 21 | var GetProxyProxyList = proxy_getter.NewGetProxyProxyList() 22 | var GetProxyZdaye = proxy_getter.NewGetProxyZdaye() 23 | var GetProxyZdayeIndex = proxy_getter.NewGetProxyZdayeIndex() 24 | var GetProxyFanQie = proxy_getter.NewGetProxyFanQie() 25 | var GetProxySeofangfa = proxy_getter.NewGetProxySeofangfa() 26 | var GetProxyXsdaili = proxy_getter.NewGetProxyXsdaili() 27 | var GetProxyYqie = proxy_getter.NewGetProxyYqie() 28 | var GetProxyPaChong = proxy_getter.NewGetProxyPachong() 29 | var KxDaili = proxy_getter.NewGetProxyKxDaili() 30 | var Geonode = proxy_getter.NewGetProxyGeonode() 31 | var CommonGetterSocks5 = proxy_getter.NewCommonGetter(consts.PROTO_SOCKS5) 32 | var CommonGetterSocks4 = proxy_getter.NewCommonGetter(consts.PROTO_SOCKS4) 33 | var CommonGetterHttp = proxy_getter.NewCommonGetter(consts.PROTO_HTTP) 34 | 35 | func LoadService() { 36 | ip.LoadLocalIpData() 37 | } 38 | -------------------------------------------------------------------------------- /service/tunnel/.ssl/README.md: -------------------------------------------------------------------------------- 1 | [//]: <> (https://gist.github.com/fntlnz/cf14feb5a46b2eda428e000157447309) 2 | 3 | # Create Root CA (Done once) 4 | 5 | ## Create Root Key 6 | 7 | **Attention:** this is the key used to sign the certificate requests, anyone holding this can sign certificates on your behalf. So keep it in a safe place! 8 | 9 | ```bash 10 | openssl genrsa -des3 -out rootCA.key 4096 11 | ``` 12 | 13 | If you want a non password protected key just remove the `-des3` option 14 | 15 | 16 | ## Create and self sign the Root Certificate 17 | 18 | ```bash 19 | openssl req -x509 -new -nodes -key rootCA.key -sha256 -days 1024 -out rootCA.crt 20 | ``` 21 | 22 | Here we used our root key to create the root certificate that needs to be distributed in all the computers that have to trust us. 23 | 24 | 25 | # Create a certificate (Done for each server) 26 | 27 | This procedure needs to be followed for each server/appliance that needs a trusted certificate from our CA 28 | 29 | ## Create the certificate key 30 | 31 | ``` 32 | openssl genrsa -out mydomain.com.key 2048 33 | ``` 34 | 35 | ## Create the signing request 36 | 37 | **Important:** Please mind that while creating the signign request is important to specify the `Common Name` providing the IP address or URL for the service, otherwise the certificate 38 | cannot be verified 39 | 40 | ``` 41 | openssl req -new -key mydomain.com.key -out mydomain.com.csr 42 | ``` 43 | 44 | ## Generate the certificate using the `mydomain` csr and key along with the CA Root key 45 | 46 | ``` 47 | openssl x509 -req -in mydomain.com.csr -CA rootCA.crt -CAkey rootCA.key -CAcreateserial -out mydomain.com.crt -days 500 -sha256 48 | ``` 49 | -------------------------------------------------------------------------------- /service/tunnel/.ssl/localhost.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDvjCCAaYCCQC0XjV3wljvnjANBgkqhkiG9w0BAQsFADAUMRIwEAYDVQQDDAls 3 | b2NhbGhvc3QwHhcNMTgwNzA4MDQ1MzIyWhcNMTkxMTIwMDQ1MzIyWjAuMQswCQYD 4 | VQQGEwJDTjELMAkGA1UEBwwCU0gxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJ 5 | KoZIhvcNAQEBBQADggEPADCCAQoCggEBAMOKgzWil/KjyRy2Axb3XlLB1nMwLFJC 6 | pC6r8yb+1Kq/ldZghJZvymuFVjn+bihvJqvZiOv4KRtnM8gD55AhaQp6Ese5M9b+ 7 | 47HLB//SkfJsQREsmnrHfHxjmUQjhMy7jrpcf9OnDOXQ5zk3v6AWEIqMtAiZ99ku 8 | AQvyJJ07+VpwZrMuzbSGfFBCKEbbqP7yKHjSUm3QDTpTiK4AnBmzlVeThUIA68oa 9 | XZKQVXX/8U2i6H4eq5eNpyUsKSnnuK+cryHpAIK4vNMzw96vATTfEmuWASEzkHhW 10 | 3KtfXE0CIH0GsK5zueGDo9ygnO7hjtx60SWynlGf6c6edxPwNvEmTZcCAwEAATAN 11 | BgkqhkiG9w0BAQsFAAOCAgEApLkdhnDzErgBljY6qRaR0JlouTpqJXwi5BRi7F1P 12 | bx5ukrZAVSOsZ7ncEkZuxkIX+ktBVFBL8twkvMEl+sMQ24R+F+TrlHWN2xPR/pez 13 | 9V19hq26yMIlYLqSq3KZ0W9ZlT2ge+3sTvY+gAJhZ6nOz9WGRJ1mi+pN/ok678QX 14 | KdOJXcePzYr5iKqMq/5cJ2sA1xYwVl+0xrvfRVTFkp4yR6wzGODtjquB+scZ9S64 15 | GWnFTjHAJvUKYxpeoLAt9lZHsESDqGq7hA4z1uVjhNEDJHKnXW4OhXxMB8Gk2hY8 16 | 3k4zbnKsouNNW0a7jijCMpXOem/vgQF4GK5ecp0S+Ml/AunsPoi6rGgOCX8XXmti 17 | 6DfQhsxxgn1co/JKNxhgsnQftXFwKivh73JFctSh+bMLsewfXsvq+b0K3EuuV9bV 18 | EttVCgbUaCDYdA6IDkqD2PRx9tsotne76r+cX+ah+NjnA6XN+XY2bJgV1UaiKTrP 19 | moNHglw+xoUqOJ7FlGJcVC7uIFPhMviNkpSZh6WxX+OSS4fPO25kxxNpldql6I+3 20 | xb5XEHLpPCEI4PyK0rYnsjk764Loqff8YBMFRQSXIUz9ot5SgGs/FY1vsQap5OeD 21 | Hw2usWhCvkSzr7kiXI+30BvJKK2r9GOAM7mtO9dfkM9MMKKnMzd+O2XE4r6PNLrg 22 | Rds= 23 | -----END CERTIFICATE----- 24 | -------------------------------------------------------------------------------- /service/tunnel/.ssl/localhost.csr: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE REQUEST----- 2 | MIICczCCAVsCAQAwLjELMAkGA1UEBhMCQ04xCzAJBgNVBAcMAlNIMRIwEAYDVQQD 3 | DAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDDioM1 4 | opfyo8kctgMW915SwdZzMCxSQqQuq/Mm/tSqv5XWYISWb8prhVY5/m4obyar2Yjr 5 | +CkbZzPIA+eQIWkKehLHuTPW/uOxywf/0pHybEERLJp6x3x8Y5lEI4TMu466XH/T 6 | pwzl0Oc5N7+gFhCKjLQImffZLgEL8iSdO/lacGazLs20hnxQQihG26j+8ih40lJt 7 | 0A06U4iuAJwZs5VXk4VCAOvKGl2SkFV1//FNouh+HquXjaclLCkp57ivnK8h6QCC 8 | uLzTM8PerwE03xJrlgEhM5B4VtyrX1xNAiB9BrCuc7nhg6PcoJzu4Y7cetElsp5R 9 | n+nOnncT8DbxJk2XAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAQEAr+AkYRAPulBU 10 | B5HR3pAreYrf3Y2fvGLSNo4hvsJkmXJxDgMZnGsjVzW1IZLF8szn4v050y6Qm/Ne 11 | qupabYP5zpj0vKkACYGJ2zadnowmwlTzwlxEOv27uQykC/IuRcjdloAD7ZwhNwmO 12 | dLNjdiXn63GUeSL/JK0UHyXTqvpmiHq+6TAOdl3vmsRFCQDChRtViK2fwSeX2y87 13 | hLicSVQyNOe0gUx7IvE9B2QPNhdzaMVPYeN8I/cayNeUKhiWxEGKhwPAaievuSXJ 14 | fUsz11XYBYW+kjFsTqkV1OjkG0mxvwaiq5W3CRx8365w71IMdKV5t5xhc0n0TXp7 15 | cT27XN7cdw== 16 | -----END CERTIFICATE REQUEST----- 17 | -------------------------------------------------------------------------------- /service/tunnel/.ssl/localhost.key: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEowIBAAKCAQEAw4qDNaKX8qPJHLYDFvdeUsHWczAsUkKkLqvzJv7Uqr+V1mCE 3 | lm/Ka4VWOf5uKG8mq9mI6/gpG2czyAPnkCFpCnoSx7kz1v7jscsH/9KR8mxBESya 4 | esd8fGOZRCOEzLuOulx/06cM5dDnOTe/oBYQioy0CJn32S4BC/IknTv5WnBmsy7N 5 | tIZ8UEIoRtuo/vIoeNJSbdANOlOIrgCcGbOVV5OFQgDryhpdkpBVdf/xTaLofh6r 6 | l42nJSwpKee4r5yvIekAgri80zPD3q8BNN8Sa5YBITOQeFbcq19cTQIgfQawrnO5 7 | 4YOj3KCc7uGO3HrRJbKeUZ/pzp53E/A28SZNlwIDAQABAoIBAGx1pMeYMw3L2R5K 8 | urX/aVsf1xI3My5Bdo3IpGsJx+4ZrEOnb4N96FnxMF2kiXd2B44kb/TqxepEOQ2F 9 | VOi2D2xXP5l2WZGz+ZnBUuOL6ZX8g67B/cGCasMX/4gy51Mj6UvnSKOeMeI7GDW9 10 | fVWPR4eB+c4XkMju4ne8zKBGBs4pN4KoxTWSnZSM4p+q/Jb/DMa+kVhFfRjkqfWc 11 | vCpDgHs1uMcHvPBNYO9flDaC2Jgk4cvV9mX0TolXAvaNo8aN0joM7WH3fvw7NCD9 12 | LCkqCmpjOxJIqJQT1twIkSy42q7VaFi7ApyIaMfXlmnj4UTlVTe5bBO+2AgwLYtC 13 | cKgDMjECgYEA8JPm3Pc80EsYB6d4qp/Qmy2VrnlaxZwvaRwh63Pssqthg4SZkIp5 14 | yjXOT4MDlJdrEzMtATRZUXTCRxGFSs0tolNY2KQ2WvYRhISlN8UBkGuMEkRGLuct 15 | p++qpPcSZJcox25kT82CKin1nQYb48k33JAOMUOWIBJO56G35sfPj28CgYEA0BOE 16 | pa+FYj/WxZS79YT1ZbsajeuUKlNUtAIxKJ2cKSQyfFuPM+xZG3H+iroRfR8HCVai 17 | 2+Oz9/TlxZOPR7+P+2fpS8W2tT+Qkmiyz8QJAULd+Irw5XIdatkpVm343XxMx3Pa 18 | 2qtBmgj6RINvsWTWRotMqhcuDRirxqm1IIQhkFkCgYBLNmIhyOXpVODRW8k8xrQI 19 | H6tBHc2EJD0qRlJQczCX9z6ISIdeCfzjfAjhENuos+IU4ZX7X2thLPikEVUzuou+ 20 | yQHo0QXxUCbP4Exq8Bt6FDV5bIDonvvGGgamhlvouN1V5CxWSrCcD/wquEM15q2h 21 | NiRJwJCJvE+Q2R1OeD9q3wKBgFWDkAJf7luAjQ3KoKy4pfnXOYSWCuCSOr94Hyfo 22 | DmPCIpWFM4dNXRmwccIl0kYv2D54QppILJB9L2lRyZLdIZlbDUA802gN5aamLMbC 23 | dEj2aC9bOsGxcnGVKi4BKEQub4eRD6LKuz1I70H1GpQ3MvDvEuTcfeqX9xDAclYY 24 | t4qRAoGBAI6YSTs97DUe7Zwk7q+S3PBU5uct4Dwtmy2XWZdgHwl5aP8apSLciL5Y 25 | PMkpcTMzkuC+QFaPZ8wFcI7GLg0bOs91hkrqscDKEg4nGB9fJkU82iOQZNL4Hv1u 26 | wO7uIGa2kcpNtQOLNO88y45WFyrn5a+T6VhDmIuc+F+TU1ZzdYdH 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /service/tunnel/.ssl/rootCA.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIEpDCCAowCCQDwV08QFUCcSzANBgkqhkiG9w0BAQsFADAUMRIwEAYDVQQDDAls 3 | b2NhbGhvc3QwHhcNMTgwNzA4MDQ1MzE3WhcNMjEwNDI3MDQ1MzE3WjAUMRIwEAYD 4 | VQQDDAlsb2NhbGhvc3QwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDE 5 | +71yX9vQY3l52C31e04ACvm2oNMLSCrLOXGewOTFpv9yXjinMC0Ab6Xa4yB4MPtd 6 | ujWDSEq9gkKCkILoalVX7R4gtLDN+EdoVadBw/WHbrGB4sHnFOWwpUbjiiwwPSU5 7 | qXjcqIqR2sA1BgoAv6c1qHq7V4bgbEjtGL71KkDoQZSIgNyJlXe1mcUKqmqeAnro 8 | 0WfwrNXyYt66L3PmCy9MIpWRxf9sa8PDsgT3SQEN4BHb70Z8hNj6RXfVGDZcpfcI 9 | iwbrL//YnK7waRKaKD4LoLOodE0cx3fowSWYvlwUAoYx8wFKOcdtM1zaUp+QDAug 10 | xT5g5ghU150XuqhDU6Wq1An8dLgcDU1D4cxhLk/W8OXtIk4k7yny6eUJi2zHziMm 11 | 8jfHd3M6SwohUrE3LsQI5gpvu4sAVFLMkRxaWZ95XhsVMmIsE/L5FHwDfqid0dvx 12 | bafOKT+fI3N3BaUPJlVHCNqSzSZIW59+ufnDwBV7SmJj4KMlvixEU+EFfPFdGiCA 13 | Lr0dSG5+Scx1aClaMUeVccCljp2f99IEa9wI+xwMPDStkOmnhVuqG1aEogggQZkD 14 | /5yh04wrn8EwYCAiasNNUXTV7AoqIt2bgeFbGo2Qr7LdsYuUmaWEzTm0KsHogkkg 15 | Ibd3RPBLDr/WfWI13oHMdsz8jjbXG/D1AhrcdozYDQIDAQABMA0GCSqGSIb3DQEB 16 | CwUAA4ICAQC1EeQqn2AwrS+UVc5fKRpHzV9ZMiDpxFMRLsDWBP5kNr1nSA72yYcR 17 | WgxvdqG/rGRds6lvRbvIaWD0zeujPkR3iCpb1V5oRXQ6lWOlY44pZEwCdnDd2M8I 18 | yQ7BLZCHHmlCN7a51n2o0D78HeILIeeTCQlKFDc5r51qrZbZR5DZmrp9jaZ+3eCg 19 | LQ3Onfj0WEmQFuMFGQrbJ2oaCC1GvuZWEbRh+lrxjRKOyCaRQFTY4Efe8tIwMm6J 20 | 1iyMtqK7BxminQCfizQrstB67wMljydYeUf+wwbgkiKGYc9VGopckrO3lntzKycu 21 | 9l0BmlZYkmCFt3cv23BcqAbcLdyLXh3yASwVMXaLZ4iVSaslRm4uX+gbKFCBABLa 22 | vqu7JQHfAPOeYj7zCrN12EHejPxdCjSImBeAbe56vax4uFGAodXxDGcepRItSzax 23 | qPKJd8U/8e3JDn+wmZNKwD9UGLZPbiuYOg7X+EWhjki0J6ZjgLc8dMleeD2rO+j2 24 | P/Wgv1gMr6J1svUlqkNf1Ng9eSbl/nMhuOBVOGcPnK7+wCLxM7ByaR0QgeH6/9VO 25 | 4urq53/vspBC679BHsZx3gIhcg4VefmOM2cZnTRM4izPstq1JBQkbuvz+5XuT7Yj 26 | 5Fk1/xkapCUifntKYSoslkkbNHRYxAInqkc0txn3qNBI8GAQFksz5g== 27 | -----END CERTIFICATE----- 28 | -------------------------------------------------------------------------------- /service/tunnel/.ssl/rootCA.key: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIJKAIBAAKCAgEAxPu9cl/b0GN5edgt9XtOAAr5tqDTC0gqyzlxnsDkxab/cl44 3 | pzAtAG+l2uMgeDD7Xbo1g0hKvYJCgpCC6GpVV+0eILSwzfhHaFWnQcP1h26xgeLB 4 | 5xTlsKVG44osMD0lOal43KiKkdrANQYKAL+nNah6u1eG4GxI7Ri+9SpA6EGUiIDc 5 | iZV3tZnFCqpqngJ66NFn8KzV8mLeui9z5gsvTCKVkcX/bGvDw7IE90kBDeAR2+9G 6 | fITY+kV31Rg2XKX3CIsG6y//2Jyu8GkSmig+C6CzqHRNHMd36MElmL5cFAKGMfMB 7 | SjnHbTNc2lKfkAwLoMU+YOYIVNedF7qoQ1OlqtQJ/HS4HA1NQ+HMYS5P1vDl7SJO 8 | JO8p8unlCYtsx84jJvI3x3dzOksKIVKxNy7ECOYKb7uLAFRSzJEcWlmfeV4bFTJi 9 | LBPy+RR8A36ondHb8W2nzik/nyNzdwWlDyZVRwjaks0mSFuffrn5w8AVe0piY+Cj 10 | Jb4sRFPhBXzxXRoggC69HUhufknMdWgpWjFHlXHApY6dn/fSBGvcCPscDDw0rZDp 11 | p4VbqhtWhKIIIEGZA/+codOMK5/BMGAgImrDTVF01ewKKiLdm4HhWxqNkK+y3bGL 12 | lJmlhM05tCrB6IJJICG3d0TwSw6/1n1iNd6BzHbM/I421xvw9QIa3HaM2A0CAwEA 13 | AQKCAgBXbeSH/0PxGjWwfuLnMfNM0ZJEHN2PBFj6GmTzsWnY0GZQvMEoc5mFuAhF 14 | PsoKjrMCxsM5obyKoGYkzT9NKOT4QaY9nfVbdfc7t8ikx/USR29B1wN5LS1FWhY8 15 | p/c08e6zySR7y9K1KgJlhmiqLGZqynyu6gpTUbyMf49CAZ8Ndw4WCBvadRzM3ZM3 16 | SKxJtZAYBdm8WPocuwVgXe9zC0PS5wa7zMWxuaMKGNlbaGuvXOSQWYNPgSdM7chi 17 | LHz0YjVi9VH80TEdU23SBtDa20Gup4UWH4iaXW47QH8PbG4x82zcfp7z8vEw5rsv 18 | q7xmkvIWSXWGTJMmFQ0EmzRTray5fj38Oo2ZtwHvkJQ1WkiFNFiWFZXnoS3z6h5q 19 | 1lX6ZUhCoobUJRRlDYCwNDV6dMYKXK2NNeD1MPvzUoUIpoQnoxnNF+VYMMENax3e 20 | YuEiT6xbBXzB/WE0bFVAtSPzf1vPVw+8MP5BhaH3lQb6XA89FiEZg+u95rNpf2SA 21 | gFWvz0VZsGab+LwYhbYdicmKPRH+2Pzpt5MdWt8jyo066Lv0NP5xpH9IUv/u2RX7 22 | Ycw0Bu1HWKLoEzovoH6OEa0n1A7H+PNOhABzrLvbU8GMp4kEQpXACxR43KruxE7S 23 | QgotUAb7teCP54yEHTVAe06YIaq4JPk5xqnmMVvaeuy5rvssAQKCAQEA5d4NUONV 24 | /An+bAf31HicZfRH6Pf1N3JUdjYz2l1y60Pf7dzlDI2fjOWlccp24+efXLM1sMeK 25 | GXQZsAnYJevZktQxodM67CsgEFgdGhH2s5Ey3Dp5bt3uS/SaFv4sT1x9awYdYtKp 26 | 6fGovjB1Qp/eMuZNJVl9RwegFVzzrrSMxucNzUuL4v4L911ypR4wz4s1ptqI31/U 27 | 56B1VRKjwZntqJaNO2Plt/yY0s+ganhzdKBynoOKzTpYHCqZhHdqvqfc1qC0W1xI 28 | E/b3Nf0J+GqjjT7JDbWgqNty5ipDfCdIeems96U1Gu9oeKGDzvCgtImZNFMyHzLM 29 | MhO0v6GA6zkuVQKCAQEA22Cls2AAUuugi2tdZR/krHokrUMPaKvi9RIQpqHpoKqL 30 | E3rKX9aWyIMhktch7VsnDF52R8CMUhgc7PfL4wsWA5cCq26x4E57aJUH5mxc5va5 31 | n5Sxb3C99Ytr6GpCkG4Y3pzO93ihgfuW+mQREYLpFYd/c/1SH/e4Wx5nGKx6YocY 32 | 6b/AbxWcRMlihC2gK7JFgSZMqaL+wn68oKJ7j8RUN+ykZEzBXBWL2l4oKWm+qBDx 33 | pOFSQODeQ0CQhPWovD4dVNmyrh5TcDUlJQ3+iU2hRkXe13mLTkGpSM53kwkrfVn+ 34 | 4SmVLEm5YhNcHG14A1yDqs6SY//8l5xfUeZyrPBK2QKCAQEAqIsRLm8SG9RkFWge 35 | Qk8RNfxQQbSVu0r8PRTvHjyIx5Ij/e+KjpLFGvVDQtUWKXMquTi5tF4KlzE2qIn/ 36 | T4bIKE2n+qS7vnC8eN9yryvevLlJFotVgIH/ePfnh9ZkPOhvGWsJXu1iIqPLe3Bi 37 | ejBoJuAQTsN4BP3FVgSqtD20Px8pUo8DCbQGqCB/sCwb1AGZnDb+RvKoVBGmFnOt 38 | WIX56TRCZ/qOdEIk9+W/FHIvDaObhziiLGqMMlLV73fz78l7Nm/s7lQSkXjyuEZJ 39 | 6jiepTEVEBVNsKH/dF4mz0CqdqFs7sPW1WIXMuQSlkh/PQDrMZ+Sz6daa5lhXWUY 40 | 9uAdZQKCAQBrDzRuYIhn7yPPRlsy0ai4X3dsstBfRZsh/Gnx2Ax64x+yJveCY+f7 41 | /LqyvZiKDDT3PVY92ALiwW/EWX2/1JYutFCSNxhJniNtu2U6l2GTOY8HCPq6puud 42 | XCgSKWFIuOIcKax7avxuwchBc/o8cIWtgw25HkQo46ytkx2/FdU4JjQLRw/zZjl3 43 | /Eu+s8F58asnxvgcxTXM1yrYvdLNK4PqMutbI3YtqToyHEc/RqLLxFEZJPkOPm9Z 44 | pLWinXx2OV35HbCsdpJDrTvuZHD2stLkx45j26YXT8X8iP4j3JLDvtq7KZ7qGSSG 45 | b2pBWU77XPfIsL0SXkf3+VEvV+ZY7X+pAoIBABV6Mu5Yr4UxI+ZgsaKgcK8aKoyD 46 | 5GDshxkxs8R3K1i7eCF0mEjxkV25r11KX10qFvG+hPJqizKQDBOCQC2w3noqz42p 47 | QVUeBNXpDVGoImD1/4DqUvQMivTwHWS+wSAi/wYAODJ6/bWP5Kil/7iDOwCPp0WD 48 | mLd0ujjwkOw3Xksn2Gd01pXeiT4FZpkYnyh5ddWGf1TihRFATW5+vpi6t+6KX3LR 49 | hwd9zi6soSwju/n986NUfGfeewBb6/fnh6hM/vfS2a0Blvk/7yM1k2P0uN+TzLYf 50 | skhRay10UoMwtXak+q/DBzrrAbW3EwuIdV66H4dx1AV5NMq6kAAtfDXc728= 51 | -----END RSA PRIVATE KEY----- 52 | -------------------------------------------------------------------------------- /service/tunnel/.ssl/rootCA.srl: -------------------------------------------------------------------------------- 1 | B45E3577C258EF9E 2 | -------------------------------------------------------------------------------- /service/tunnel/cfg.go: -------------------------------------------------------------------------------- 1 | package tunnel 2 | 3 | import ( 4 | "bufio" 5 | "crypto/tls" 6 | "crypto/x509" 7 | "encoding/json" 8 | "errors" 9 | "io/ioutil" 10 | "net" 11 | "net/url" 12 | "os" 13 | "strings" 14 | 15 | "github.com/tongsq/go-lib/logger" 16 | "github.com/tongsq/gost" 17 | "proxy-collect/config" 18 | ) 19 | 20 | type baseConfig struct { 21 | route 22 | Routes []route 23 | Debug bool 24 | } 25 | 26 | var ( 27 | defaultCertFile = "cert.pem" 28 | defaultKeyFile = "key.pem" 29 | ) 30 | 31 | // Load the certificate from cert & key files and optional client CA file, 32 | // will use the default certificate if the provided info are invalid. 33 | func tlsConfig(certFile, keyFile, caFile string) (*tls.Config, error) { 34 | if certFile == "" || keyFile == "" { 35 | certFile, keyFile = defaultCertFile, defaultKeyFile 36 | } 37 | 38 | cert, err := tls.LoadX509KeyPair(certFile, keyFile) 39 | if err != nil { 40 | return nil, err 41 | } 42 | 43 | cfg := &tls.Config{Certificates: []tls.Certificate{cert}} 44 | 45 | if pool, _ := loadCA(caFile); pool != nil { 46 | cfg.ClientCAs = pool 47 | cfg.ClientAuth = tls.RequireAndVerifyClientCert 48 | } 49 | 50 | return cfg, nil 51 | } 52 | 53 | func loadCA(caFile string) (cp *x509.CertPool, err error) { 54 | if caFile == "" { 55 | return 56 | } 57 | cp = x509.NewCertPool() 58 | data, err := ioutil.ReadFile(caFile) 59 | if err != nil { 60 | return nil, err 61 | } 62 | if !cp.AppendCertsFromPEM(data) { 63 | return nil, errors.New("AppendCertsFromPEM failed") 64 | } 65 | return 66 | } 67 | 68 | func parseKCPConfig(configFile string) (*gost.KCPConfig, error) { 69 | if configFile == "" { 70 | return nil, nil 71 | } 72 | file, err := os.Open(configFile) 73 | if err != nil { 74 | return nil, err 75 | } 76 | defer file.Close() 77 | 78 | config := &gost.KCPConfig{} 79 | if err = json.NewDecoder(file).Decode(config); err != nil { 80 | return nil, err 81 | } 82 | return config, nil 83 | } 84 | 85 | func parseUsers(userList []config.TunnelUserConfig) (users []*url.Userinfo, err error) { 86 | if userList == nil { 87 | return 88 | } 89 | for _, user := range userList { 90 | 91 | if user.Password == "" { 92 | users = append(users, url.User(user.Username)) 93 | } else { 94 | users = append(users, url.UserPassword(user.Username, user.Password)) 95 | } 96 | } 97 | return 98 | } 99 | 100 | func parseAuthenticator(users []config.TunnelUserConfig) (gost.Authenticator, error) { 101 | kvs := map[string]string{} 102 | for _, user := range users { 103 | kvs[user.Username] = user.Password 104 | } 105 | au := gost.NewLocalAuthenticator(kvs) 106 | return au, nil 107 | } 108 | 109 | func parseLimiter(users []config.TunnelUserConfig) (gost.Limiter, error) { 110 | if users == nil || len(users) <= 0 { 111 | return nil, nil 112 | } 113 | l, _ := gost.NewLocalLimiter("", "") 114 | for _, user := range users { 115 | err := l.AddRule(user.Username, user.Limiter) 116 | if err != nil { 117 | logger.Error("parse tunnel user limiter fail", map[string]interface{}{"user": user}) 118 | } 119 | } 120 | return l, nil 121 | } 122 | 123 | func parseIP(s string, port string) (ips []string) { 124 | if s == "" { 125 | return 126 | } 127 | if port == "" { 128 | port = "8080" // default port 129 | } 130 | 131 | file, err := os.Open(s) 132 | if err != nil { 133 | ss := strings.Split(s, ",") 134 | for _, s := range ss { 135 | s = strings.TrimSpace(s) 136 | if s != "" { 137 | // TODO: support IPv6 138 | if !strings.Contains(s, ":") { 139 | s = s + ":" + port 140 | } 141 | ips = append(ips, s) 142 | } 143 | 144 | } 145 | return 146 | } 147 | 148 | scanner := bufio.NewScanner(file) 149 | for scanner.Scan() { 150 | line := strings.TrimSpace(scanner.Text()) 151 | if line == "" || strings.HasPrefix(line, "#") { 152 | continue 153 | } 154 | if !strings.Contains(line, ":") { 155 | line = line + ":" + port 156 | } 157 | ips = append(ips, line) 158 | } 159 | return 160 | } 161 | 162 | func parseBypass(s string) *gost.Bypass { 163 | if s == "" { 164 | return nil 165 | } 166 | var matchers []gost.Matcher 167 | var reversed bool 168 | if strings.HasPrefix(s, "~") { 169 | reversed = true 170 | s = strings.TrimLeft(s, "~") 171 | } 172 | 173 | f, err := os.Open(s) 174 | if err != nil { 175 | for _, s := range strings.Split(s, ",") { 176 | s = strings.TrimSpace(s) 177 | if s == "" { 178 | continue 179 | } 180 | matchers = append(matchers, gost.NewMatcher(s)) 181 | } 182 | return gost.NewBypass(reversed, matchers...) 183 | } 184 | defer f.Close() 185 | 186 | bp := gost.NewBypass(reversed) 187 | bp.Reload(f) 188 | go gost.PeriodReload(bp, s) 189 | 190 | return bp 191 | } 192 | 193 | func parseResolver(cfg string) gost.Resolver { 194 | if cfg == "" { 195 | return nil 196 | } 197 | var nss []gost.NameServer 198 | 199 | f, err := os.Open(cfg) 200 | if err != nil { 201 | for _, s := range strings.Split(cfg, ",") { 202 | s = strings.TrimSpace(s) 203 | if s == "" { 204 | continue 205 | } 206 | if strings.HasPrefix(s, "https") { 207 | p := "https" 208 | u, _ := url.Parse(s) 209 | if u == nil || u.Scheme == "" { 210 | continue 211 | } 212 | if u.Scheme == "https-chain" { 213 | p = u.Scheme 214 | } 215 | ns := gost.NameServer{ 216 | Addr: s, 217 | Protocol: p, 218 | } 219 | nss = append(nss, ns) 220 | continue 221 | } 222 | 223 | ss := strings.Split(s, "/") 224 | if len(ss) == 1 { 225 | ns := gost.NameServer{ 226 | Addr: ss[0], 227 | } 228 | nss = append(nss, ns) 229 | } 230 | if len(ss) == 2 { 231 | ns := gost.NameServer{ 232 | Addr: ss[0], 233 | Protocol: ss[1], 234 | } 235 | nss = append(nss, ns) 236 | } 237 | } 238 | return gost.NewResolver(0, nss...) 239 | } 240 | defer f.Close() 241 | 242 | resolver := gost.NewResolver(0) 243 | resolver.Reload(f) 244 | 245 | go gost.PeriodReload(resolver, cfg) 246 | 247 | return resolver 248 | } 249 | 250 | func parseHosts(s string) *gost.Hosts { 251 | f, err := os.Open(s) 252 | if err != nil { 253 | return nil 254 | } 255 | defer f.Close() 256 | 257 | hosts := gost.NewHosts() 258 | hosts.Reload(f) 259 | 260 | go gost.PeriodReload(hosts, s) 261 | 262 | return hosts 263 | } 264 | 265 | func parseIPRoutes(s string) (routes []gost.IPRoute) { 266 | if s == "" { 267 | return 268 | } 269 | 270 | file, err := os.Open(s) 271 | if err != nil { 272 | ss := strings.Split(s, ",") 273 | for _, s := range ss { 274 | if _, inet, _ := net.ParseCIDR(strings.TrimSpace(s)); inet != nil { 275 | routes = append(routes, gost.IPRoute{Dest: inet}) 276 | } 277 | } 278 | return 279 | } 280 | 281 | defer file.Close() 282 | scanner := bufio.NewScanner(file) 283 | for scanner.Scan() { 284 | line := strings.Replace(scanner.Text(), "\t", " ", -1) 285 | line = strings.TrimSpace(line) 286 | if line == "" || strings.HasPrefix(line, "#") { 287 | continue 288 | } 289 | 290 | var route gost.IPRoute 291 | var ss []string 292 | for _, s := range strings.Split(line, " ") { 293 | if s = strings.TrimSpace(s); s != "" { 294 | ss = append(ss, s) 295 | } 296 | } 297 | if len(ss) > 0 && ss[0] != "" { 298 | _, route.Dest, _ = net.ParseCIDR(strings.TrimSpace(ss[0])) 299 | if route.Dest == nil { 300 | continue 301 | } 302 | } 303 | if len(ss) > 1 && ss[1] != "" { 304 | route.Gateway = net.ParseIP(ss[1]) 305 | } 306 | routes = append(routes, route) 307 | } 308 | return routes 309 | } 310 | -------------------------------------------------------------------------------- /service/tunnel/gost.go: -------------------------------------------------------------------------------- 1 | package tunnel 2 | 3 | import ( 4 | "crypto/tls" 5 | "errors" 6 | _ "net/http/pprof" 7 | "os" 8 | "time" 9 | 10 | "github.com/go-log/log" 11 | "github.com/tongsq/go-lib/logger" 12 | "github.com/tongsq/gost" 13 | "proxy-collect/config" 14 | "proxy-collect/dao" 15 | "proxy-collect/dto" 16 | ) 17 | 18 | var ( 19 | baseCfg = &baseConfig{} 20 | NodeGroupList = []*gost.NodeGroup{} 21 | ) 22 | 23 | func StartTunnels() { 24 | gost.SetLogger(&gost.LogLogger{}) 25 | baseCfg.Debug = config.Get().Tunnel.Debug 26 | baseCfg.ServeNodes = config.Get().Tunnels 27 | baseCfg.ChainNodes = []dto.ProxyDto{} 28 | proxyList, err := getProxyList() 29 | if err != nil { 30 | logger.Error("get active ip fail", logger.Fields{"err": err}) 31 | } else { 32 | baseCfg.ChainNodes = proxyList 33 | } 34 | baseCfg.ChinLevel = config.Get().Tunnel.TunnelLevel 35 | // NOTE: as of 2.6, you can use custom cert/key files to initialize the default certificate. 36 | tlsConfig, err := tlsConfig(defaultCertFile, defaultKeyFile, "") 37 | if err != nil { 38 | // generate random self-signed certificate. 39 | cert, err := gost.GenCertificate() 40 | if err != nil { 41 | log.Log(err) 42 | os.Exit(1) 43 | } 44 | tlsConfig = &tls.Config{ 45 | Certificates: []tls.Certificate{cert}, 46 | } 47 | } else { 48 | log.Log("load TLS certificate files OK") 49 | } 50 | 51 | gost.DefaultTLSConfig = tlsConfig 52 | 53 | if err := start(); err != nil { 54 | log.Log(err) 55 | os.Exit(1) 56 | } 57 | 58 | select {} 59 | } 60 | 61 | func start() error { 62 | gost.Debug = baseCfg.Debug 63 | 64 | var routers []router 65 | rts, err := baseCfg.route.GenRouters() 66 | if err != nil { 67 | return err 68 | } 69 | routers = append(routers, rts...) 70 | 71 | for _, route := range baseCfg.Routes { 72 | rts, err := route.GenRouters() 73 | if err != nil { 74 | return err 75 | } 76 | routers = append(routers, rts...) 77 | } 78 | 79 | if len(routers) == 0 { 80 | return errors.New("invalid config") 81 | } 82 | for i := range routers { 83 | go routers[i].Serve() 84 | } 85 | go StartRefreshNodeGroupList() 86 | return nil 87 | } 88 | 89 | func getProxyList() ([]dto.ProxyDto, error) { 90 | proxies, err := dao.ProxyDao.GetActiveList() 91 | if err != nil { 92 | logger.Error("get active ip fail", logger.Fields{"err": err}) 93 | return nil, err 94 | } 95 | proxyList := []dto.ProxyDto{} 96 | for _, proxy := range proxies { 97 | var p = dto.NewProxyDto(proxy) 98 | proxyList = append(proxyList, p.ProxyDto) 99 | } 100 | return proxyList, nil 101 | } 102 | 103 | func StartRefreshNodeGroupList() { 104 | for { 105 | time.Sleep(time.Second * time.Duration(config.Get().Tunnel.Refresh)) 106 | logger.Debug("start run RefreshNodeGroupList", nil) 107 | proxyList, err := getProxyList() 108 | if err != nil { 109 | logger.Error("get active ip fail", logger.Fields{"err": err}) 110 | continue 111 | } 112 | if err = RefreshNodeGroupList(proxyList); err != nil { 113 | logger.Error("RefreshNodeGroupList fail", map[string]interface{}{"err": err}) 114 | } 115 | 116 | } 117 | } 118 | 119 | func RefreshNodeGroupList(proxyList []dto.ProxyDto) error { 120 | // parse the base nodes 121 | nodes, err := parseChainNode(proxyList) 122 | if err != nil { 123 | return err 124 | } 125 | 126 | nid := 1 // node ID 127 | for i := range nodes { 128 | nodes[i].ID = nid 129 | nid++ 130 | } 131 | for _, ngroup := range NodeGroupList { 132 | ngroup.SetNodes(nodes...) 133 | 134 | ngroup.SetSelector(nil, 135 | gost.WithFilter( 136 | &gost.FailFilter{ 137 | MaxFails: config.Get().Tunnel.MaxFails, 138 | FailTimeout: time.Duration(config.Get().Tunnel.FailTimeout) * time.Second, 139 | }, 140 | &gost.InvalidFilter{}, 141 | ), 142 | gost.WithStrategy(gost.NewStrategy(config.Get().Tunnel.Strategy)), 143 | ) 144 | } 145 | return nil 146 | } 147 | -------------------------------------------------------------------------------- /service/tunnel/route.go: -------------------------------------------------------------------------------- 1 | package tunnel 2 | 3 | import ( 4 | "crypto/sha256" 5 | "crypto/tls" 6 | "crypto/x509" 7 | "encoding/base64" 8 | "fmt" 9 | "net" 10 | "net/url" 11 | "strings" 12 | "time" 13 | 14 | "github.com/tongsq/go-lib/logger" 15 | "github.com/tongsq/gost" 16 | "proxy-collect/config" 17 | "proxy-collect/dto" 18 | "proxy-collect/service/common" 19 | ) 20 | 21 | type stringList []string 22 | 23 | func (l *stringList) String() string { 24 | return fmt.Sprintf("%s", *l) 25 | } 26 | func (l *stringList) Set(value string) error { 27 | *l = append(*l, value) 28 | return nil 29 | } 30 | 31 | type route struct { 32 | ServeNodes []config.TunnelConfig 33 | ChainNodes []dto.ProxyDto 34 | ChinLevel int 35 | Retries int 36 | Mark int 37 | } 38 | 39 | func (r *route) parseChain() (*gost.Chain, error) { 40 | chain := gost.NewChain() 41 | chain.Retries = r.Retries 42 | chain.Mark = r.Mark 43 | gid := 1 // group ID 44 | 45 | for i := 0; i < r.ChinLevel; i++ { 46 | ngroup := gost.NewNodeGroup() 47 | ngroup.ID = gid 48 | gid++ 49 | 50 | // parse the base nodes 51 | nodes, err := parseChainNode(r.ChainNodes) 52 | if err != nil || len(nodes) <= 0 { 53 | return nil, err 54 | } 55 | 56 | nid := 1 // node ID 57 | for i := range nodes { 58 | nodes[i].ID = nid 59 | nid++ 60 | } 61 | ngroup.AddNode(nodes...) 62 | 63 | ngroup.SetSelector(nil, 64 | gost.WithFilter( 65 | &gost.FailFilter{ 66 | MaxFails: config.Get().Tunnel.MaxFails, 67 | FailTimeout: time.Duration(config.Get().Tunnel.FailTimeout) * time.Second, 68 | }, 69 | &gost.InvalidFilter{}, 70 | ), 71 | gost.WithStrategy(gost.NewStrategy(config.Get().Tunnel.Strategy)), 72 | ) 73 | //refresh proxy ips 74 | NodeGroupList = append(NodeGroupList, ngroup) 75 | chain.AddNodeGroup(ngroup) 76 | } 77 | 78 | return chain, nil 79 | } 80 | 81 | func parseChainNode(nsList []dto.ProxyDto) (nodes []gost.Node, err error) { 82 | for _, nsNode := range nsList { 83 | ns := common.GetProxyUrl(&nsNode) 84 | node, err := gost.ParseNode(ns) 85 | if err != nil { 86 | logger.Error("parse chain node fail", map[string]interface{}{"ns": ns}) 87 | continue 88 | } 89 | if auth := node.Get("auth"); auth != "" && node.User == nil { 90 | c, err := base64.StdEncoding.DecodeString(auth) 91 | if err != nil { 92 | return nil, err 93 | } 94 | cs := string(c) 95 | s := strings.IndexByte(cs, ':') 96 | if s < 0 { 97 | node.User = url.User(cs) 98 | } else { 99 | node.User = url.UserPassword(cs[:s], cs[s+1:]) 100 | } 101 | } 102 | if node.User == nil && nsNode.User != "" { 103 | if nsNode.Password == "" { 104 | node.User = url.User(nsNode.User) 105 | } else { 106 | node.User = url.UserPassword(nsNode.User, nsNode.Password) 107 | } 108 | } 109 | 110 | serverName, _, _ := net.SplitHostPort(node.Addr) 111 | if serverName == "" { 112 | serverName = "localhost" // default server name 113 | } 114 | 115 | rootCAs, err := loadCA(node.Get("ca")) 116 | if err != nil { 117 | logger.Error("loadCa fail", map[string]interface{}{"err": err}) 118 | continue 119 | } 120 | tlsCfg := &tls.Config{ 121 | ServerName: serverName, 122 | InsecureSkipVerify: !node.GetBool("secure"), 123 | RootCAs: rootCAs, 124 | } 125 | 126 | // If the argument `ca` is given, but not open `secure`, we verify the 127 | // certificate manually. 128 | if rootCAs != nil && !node.GetBool("secure") { 129 | tlsCfg.VerifyConnection = func(state tls.ConnectionState) error { 130 | opts := x509.VerifyOptions{ 131 | Roots: rootCAs, 132 | CurrentTime: time.Now(), 133 | DNSName: "", 134 | Intermediates: x509.NewCertPool(), 135 | } 136 | 137 | certs := state.PeerCertificates 138 | for i, cert := range certs { 139 | if i == 0 { 140 | continue 141 | } 142 | opts.Intermediates.AddCert(cert) 143 | } 144 | 145 | _, err = certs[0].Verify(opts) 146 | return err 147 | } 148 | } 149 | 150 | if cert, err := tls.LoadX509KeyPair(node.Get("cert"), node.Get("key")); err == nil { 151 | tlsCfg.Certificates = []tls.Certificate{cert} 152 | } 153 | 154 | wsOpts := &gost.WSOptions{} 155 | wsOpts.EnableCompression = node.GetBool("compression") 156 | wsOpts.ReadBufferSize = node.GetInt("rbuf") 157 | wsOpts.WriteBufferSize = node.GetInt("wbuf") 158 | wsOpts.UserAgent = node.Get("agent") 159 | wsOpts.Path = node.Get("path") 160 | 161 | timeout := node.GetDuration("timeout") 162 | 163 | var tr gost.Transporter 164 | switch node.Transport { 165 | case "tls": 166 | tr = gost.TLSTransporter() 167 | case "mtls": 168 | tr = gost.MTLSTransporter() 169 | case "ws": 170 | tr = gost.WSTransporter(wsOpts) 171 | case "mws": 172 | tr = gost.MWSTransporter(wsOpts) 173 | case "wss": 174 | tr = gost.WSSTransporter(wsOpts) 175 | case "mwss": 176 | tr = gost.MWSSTransporter(wsOpts) 177 | case "kcp": 178 | kpcConfig, err := parseKCPConfig(node.Get("c")) 179 | if err != nil { 180 | return nil, err 181 | } 182 | if kpcConfig == nil { 183 | conf := gost.DefaultKCPConfig 184 | if node.GetBool("tcp") { 185 | conf.TCP = true 186 | } 187 | kpcConfig = &conf 188 | } 189 | tr = gost.KCPTransporter(kpcConfig) 190 | case "ssh": 191 | if node.Protocol == "direct" || node.Protocol == "remote" { 192 | tr = gost.SSHForwardTransporter() 193 | } else { 194 | tr = gost.SSHTunnelTransporter() 195 | } 196 | case "quic": 197 | quicConfig := &gost.QUICConfig{ 198 | TLSConfig: tlsCfg, 199 | KeepAlive: node.GetBool("keepalive"), 200 | Timeout: timeout, 201 | IdleTimeout: node.GetDuration("idle"), 202 | } 203 | 204 | if cipher := node.Get("cipher"); cipher != "" { 205 | sum := sha256.Sum256([]byte(cipher)) 206 | quicConfig.Key = sum[:] 207 | } 208 | 209 | tr = gost.QUICTransporter(quicConfig) 210 | case "http2": 211 | tr = gost.HTTP2Transporter(tlsCfg) 212 | case "h2": 213 | tr = gost.H2Transporter(tlsCfg, node.Get("path")) 214 | case "h2c": 215 | tr = gost.H2CTransporter(node.Get("path")) 216 | case "obfs4": 217 | tr = gost.Obfs4Transporter() 218 | case "ohttp": 219 | tr = gost.ObfsHTTPTransporter() 220 | case "otls": 221 | tr = gost.ObfsTLSTransporter() 222 | case "ftcp": 223 | tr = gost.FakeTCPTransporter() 224 | case "udp": 225 | tr = gost.UDPTransporter() 226 | default: 227 | tr = gost.TCPTransporter() 228 | } 229 | 230 | var connector gost.Connector 231 | switch node.Protocol { 232 | case "http2": 233 | connector = gost.HTTP2Connector(node.User) 234 | case "socks", "socks5": 235 | connector = gost.SOCKS5Connector(node.User) 236 | case "socks4": 237 | connector = gost.SOCKS4Connector() 238 | case "socks4a": 239 | connector = gost.SOCKS4AConnector() 240 | case "ss": 241 | connector = gost.ShadowConnector(node.User) 242 | case "ssu": 243 | connector = gost.ShadowUDPConnector(node.User) 244 | case "direct": 245 | connector = gost.SSHDirectForwardConnector() 246 | case "remote": 247 | connector = gost.SSHRemoteForwardConnector() 248 | case "forward": 249 | connector = gost.ForwardConnector() 250 | case "sni": 251 | connector = gost.SNIConnector(node.Get("host")) 252 | case "http": 253 | connector = gost.HTTPConnector(node.User) 254 | case "relay": 255 | connector = gost.RelayConnector(node.User) 256 | default: 257 | connector = gost.AutoConnector(node.User) 258 | } 259 | 260 | host := node.Get("host") 261 | if host == "" { 262 | host = node.Host 263 | } 264 | 265 | node.DialOptions = append(node.DialOptions, 266 | gost.TimeoutDialOption(timeout), 267 | gost.HostDialOption(host), 268 | ) 269 | 270 | node.ConnectOptions = []gost.ConnectOption{ 271 | gost.UserAgentConnectOption(node.Get("agent")), 272 | gost.NoTLSConnectOption(node.GetBool("notls")), 273 | gost.NoDelayConnectOption(node.GetBool("nodelay")), 274 | } 275 | 276 | sshConfig := &gost.SSHConfig{} 277 | if s := node.Get("ssh_key"); s != "" { 278 | key, err := gost.ParseSSHKeyFile(s) 279 | if err != nil { 280 | return nil, err 281 | } 282 | sshConfig.Key = key 283 | } 284 | handshakeOptions := []gost.HandshakeOption{ 285 | gost.AddrHandshakeOption(node.Addr), 286 | gost.HostHandshakeOption(host), 287 | gost.UserHandshakeOption(node.User), 288 | gost.TLSConfigHandshakeOption(tlsCfg), 289 | gost.IntervalHandshakeOption(node.GetDuration("ping")), 290 | gost.TimeoutHandshakeOption(timeout), 291 | gost.RetryHandshakeOption(node.GetInt("retry")), 292 | gost.SSHConfigHandshakeOption(sshConfig), 293 | } 294 | 295 | node.Client = &gost.Client{ 296 | Connector: connector, 297 | Transporter: tr, 298 | } 299 | 300 | node.Bypass = parseBypass(node.Get("bypass")) 301 | node.HandshakeOptions = handshakeOptions 302 | nodes = append(nodes, node) 303 | if node.Transport == "obfs4" { 304 | for i := range nodes { 305 | if err := gost.Obfs4Init(nodes[i], false); err != nil { 306 | return nil, err 307 | } 308 | } 309 | } 310 | } 311 | return 312 | } 313 | 314 | func (r *route) GenRouters() ([]router, error) { 315 | chain, err := r.parseChain() 316 | if err != nil { 317 | return nil, err 318 | } 319 | 320 | var rts []router 321 | 322 | for _, ns := range r.ServeNodes { 323 | nsUrl := common.GetTunnelUrl(&ns) 324 | node, err := gost.ParseNode(nsUrl) 325 | if err != nil { 326 | return nil, err 327 | } 328 | 329 | authenticator, err := parseAuthenticator(ns.Users) 330 | if err != nil { 331 | return nil, err 332 | } 333 | if authenticator == nil && node.User != nil { 334 | kvs := make(map[string]string) 335 | kvs[node.User.Username()], _ = node.User.Password() 336 | authenticator = gost.NewLocalAuthenticator(kvs) 337 | } 338 | if node.User == nil { 339 | if users, _ := parseUsers(ns.Users); len(users) > 0 { 340 | node.User = users[0] 341 | } 342 | } 343 | 344 | //init rate limiter 345 | limiterHandler, err := parseLimiter(ns.Users) 346 | if err != nil { 347 | return nil, err 348 | } 349 | certFile, keyFile := node.Get("cert"), node.Get("key") 350 | tlsCfg, err := tlsConfig(certFile, keyFile, node.Get("ca")) 351 | if err != nil && certFile != "" && keyFile != "" { 352 | return nil, err 353 | } 354 | 355 | wsOpts := &gost.WSOptions{} 356 | wsOpts.EnableCompression = node.GetBool("compression") 357 | wsOpts.ReadBufferSize = node.GetInt("rbuf") 358 | wsOpts.WriteBufferSize = node.GetInt("wbuf") 359 | wsOpts.Path = node.Get("path") 360 | 361 | ttl := node.GetDuration("ttl") 362 | timeout := node.GetDuration("timeout") 363 | 364 | tunRoutes := parseIPRoutes(node.Get("route")) 365 | gw := net.ParseIP(node.Get("gw")) // default gateway 366 | for i := range tunRoutes { 367 | if tunRoutes[i].Gateway == nil { 368 | tunRoutes[i].Gateway = gw 369 | } 370 | } 371 | 372 | var ln gost.Listener 373 | switch node.Transport { 374 | case "tls": 375 | ln, err = gost.TLSListener(node.Addr, tlsCfg) 376 | case "mtls": 377 | ln, err = gost.MTLSListener(node.Addr, tlsCfg) 378 | case "ws": 379 | ln, err = gost.WSListener(node.Addr, wsOpts) 380 | case "mws": 381 | ln, err = gost.MWSListener(node.Addr, wsOpts) 382 | case "wss": 383 | ln, err = gost.WSSListener(node.Addr, tlsCfg, wsOpts) 384 | case "mwss": 385 | ln, err = gost.MWSSListener(node.Addr, tlsCfg, wsOpts) 386 | case "kcp": 387 | kcpConfig, er := parseKCPConfig(node.Get("c")) 388 | if er != nil { 389 | return nil, er 390 | } 391 | if kcpConfig == nil { 392 | conf := gost.DefaultKCPConfig 393 | if node.GetBool("tcp") { 394 | conf.TCP = true 395 | } 396 | kcpConfig = &conf 397 | } 398 | ln, err = gost.KCPListener(node.Addr, kcpConfig) 399 | case "ssh": 400 | sshConfig := &gost.SSHConfig{ 401 | Authenticator: authenticator, 402 | TLSConfig: tlsCfg, 403 | } 404 | if s := node.Get("ssh_key"); s != "" { 405 | key, err := gost.ParseSSHKeyFile(s) 406 | if err != nil { 407 | return nil, err 408 | } 409 | sshConfig.Key = key 410 | } 411 | if s := node.Get("ssh_authorized_keys"); s != "" { 412 | keys, err := gost.ParseSSHAuthorizedKeysFile(s) 413 | if err != nil { 414 | return nil, err 415 | } 416 | sshConfig.AuthorizedKeys = keys 417 | } 418 | if node.Protocol == "forward" { 419 | ln, err = gost.TCPListener(node.Addr) 420 | } else { 421 | ln, err = gost.SSHTunnelListener(node.Addr, sshConfig) 422 | } 423 | case "quic": 424 | quicConfig := &gost.QUICConfig{ 425 | TLSConfig: tlsCfg, 426 | KeepAlive: node.GetBool("keepalive"), 427 | Timeout: timeout, 428 | IdleTimeout: node.GetDuration("idle"), 429 | } 430 | if cipher := node.Get("cipher"); cipher != "" { 431 | sum := sha256.Sum256([]byte(cipher)) 432 | quicConfig.Key = sum[:] 433 | } 434 | 435 | ln, err = gost.QUICListener(node.Addr, quicConfig) 436 | case "http2": 437 | ln, err = gost.HTTP2Listener(node.Addr, tlsCfg) 438 | case "h2": 439 | ln, err = gost.H2Listener(node.Addr, tlsCfg, node.Get("path")) 440 | case "h2c": 441 | ln, err = gost.H2CListener(node.Addr, node.Get("path")) 442 | case "tcp": 443 | // Directly use SSH port forwarding if the last chain node is forward+ssh 444 | if chain.LastNode().Protocol == "forward" && chain.LastNode().Transport == "ssh" { 445 | chain.Nodes()[len(chain.Nodes())-1].Client.Connector = gost.SSHDirectForwardConnector() 446 | chain.Nodes()[len(chain.Nodes())-1].Client.Transporter = gost.SSHForwardTransporter() 447 | } 448 | ln, err = gost.TCPListener(node.Addr) 449 | case "udp": 450 | ln, err = gost.UDPListener(node.Addr, &gost.UDPListenConfig{ 451 | TTL: ttl, 452 | Backlog: node.GetInt("backlog"), 453 | QueueSize: node.GetInt("queue"), 454 | }) 455 | case "rtcp": 456 | // Directly use SSH port forwarding if the last chain node is forward+ssh 457 | if chain.LastNode().Protocol == "forward" && chain.LastNode().Transport == "ssh" { 458 | chain.Nodes()[len(chain.Nodes())-1].Client.Connector = gost.SSHRemoteForwardConnector() 459 | chain.Nodes()[len(chain.Nodes())-1].Client.Transporter = gost.SSHForwardTransporter() 460 | } 461 | ln, err = gost.TCPRemoteForwardListener(node.Addr, chain) 462 | case "rudp": 463 | ln, err = gost.UDPRemoteForwardListener(node.Addr, 464 | chain, 465 | &gost.UDPListenConfig{ 466 | TTL: ttl, 467 | Backlog: node.GetInt("backlog"), 468 | QueueSize: node.GetInt("queue"), 469 | }) 470 | case "obfs4": 471 | if err = gost.Obfs4Init(node, true); err != nil { 472 | return nil, err 473 | } 474 | ln, err = gost.Obfs4Listener(node.Addr) 475 | case "ohttp": 476 | ln, err = gost.ObfsHTTPListener(node.Addr) 477 | case "otls": 478 | ln, err = gost.ObfsTLSListener(node.Addr) 479 | case "tun": 480 | cfg := gost.TunConfig{ 481 | Name: node.Get("name"), 482 | Addr: node.Get("net"), 483 | Peer: node.Get("peer"), 484 | MTU: node.GetInt("mtu"), 485 | Routes: tunRoutes, 486 | Gateway: node.Get("gw"), 487 | } 488 | ln, err = gost.TunListener(cfg) 489 | case "tap": 490 | cfg := gost.TapConfig{ 491 | Name: node.Get("name"), 492 | Addr: node.Get("net"), 493 | MTU: node.GetInt("mtu"), 494 | Routes: strings.Split(node.Get("route"), ","), 495 | Gateway: node.Get("gw"), 496 | } 497 | ln, err = gost.TapListener(cfg) 498 | case "ftcp": 499 | ln, err = gost.FakeTCPListener( 500 | node.Addr, 501 | &gost.FakeTCPListenConfig{ 502 | TTL: ttl, 503 | Backlog: node.GetInt("backlog"), 504 | QueueSize: node.GetInt("queue"), 505 | }, 506 | ) 507 | case "dns": 508 | ln, err = gost.DNSListener( 509 | node.Addr, 510 | &gost.DNSOptions{ 511 | Mode: node.Get("mode"), 512 | TLSConfig: tlsCfg, 513 | }, 514 | ) 515 | case "redu", "redirectu": 516 | ln, err = gost.UDPRedirectListener(node.Addr, &gost.UDPListenConfig{ 517 | TTL: ttl, 518 | Backlog: node.GetInt("backlog"), 519 | QueueSize: node.GetInt("queue"), 520 | }) 521 | default: 522 | ln, err = gost.TCPListener(node.Addr) 523 | } 524 | if err != nil { 525 | return nil, err 526 | } 527 | 528 | var handler gost.Handler 529 | switch node.Protocol { 530 | case "http2": 531 | handler = gost.HTTP2Handler() 532 | case "socks", "socks5": 533 | handler = gost.SOCKS5Handler() 534 | case "socks4", "socks4a": 535 | handler = gost.SOCKS4Handler() 536 | case "ss": 537 | handler = gost.ShadowHandler() 538 | case "http": 539 | handler = gost.HTTPHandler() 540 | case "tcp": 541 | handler = gost.TCPDirectForwardHandler(node.Remote) 542 | case "rtcp": 543 | handler = gost.TCPRemoteForwardHandler(node.Remote) 544 | case "udp": 545 | handler = gost.UDPDirectForwardHandler(node.Remote) 546 | case "rudp": 547 | handler = gost.UDPRemoteForwardHandler(node.Remote) 548 | case "forward": 549 | handler = gost.SSHForwardHandler() 550 | case "red", "redirect": 551 | handler = gost.TCPRedirectHandler() 552 | case "redu", "redirectu": 553 | handler = gost.UDPRedirectHandler() 554 | case "ssu": 555 | handler = gost.ShadowUDPHandler() 556 | case "sni": 557 | handler = gost.SNIHandler() 558 | case "tun": 559 | handler = gost.TunHandler() 560 | case "tap": 561 | handler = gost.TapHandler() 562 | case "dns": 563 | handler = gost.DNSHandler(node.Remote) 564 | case "relay": 565 | handler = gost.RelayHandler(node.Remote) 566 | default: 567 | // start from 2.5, if remote is not empty, then we assume that it is a forward tunnel. 568 | if node.Remote != "" { 569 | handler = gost.TCPDirectForwardHandler(node.Remote) 570 | } else { 571 | handler = gost.AutoHandler() 572 | } 573 | } 574 | 575 | var whitelist, blacklist *gost.Permissions 576 | if node.Values.Get("whitelist") != "" { 577 | if whitelist, err = gost.ParsePermissions(node.Get("whitelist")); err != nil { 578 | return nil, err 579 | } 580 | } 581 | if node.Values.Get("blacklist") != "" { 582 | if blacklist, err = gost.ParsePermissions(node.Get("blacklist")); err != nil { 583 | return nil, err 584 | } 585 | } 586 | 587 | node.Bypass = parseBypass(node.Get("bypass")) 588 | hosts := parseHosts(node.Get("hosts")) 589 | ips := parseIP(node.Get("ip"), "") 590 | 591 | resolver := parseResolver(node.Get("dns")) 592 | if resolver != nil { 593 | resolver.Init( 594 | gost.ChainResolverOption(chain), 595 | gost.TimeoutResolverOption(timeout), 596 | gost.TTLResolverOption(ttl), 597 | gost.PreferResolverOption(node.Get("prefer")), 598 | gost.SrcIPResolverOption(net.ParseIP(node.Get("ip"))), 599 | ) 600 | } 601 | 602 | handler.Init( 603 | gost.AddrHandlerOption(ln.Addr().String()), 604 | gost.ChainHandlerOption(chain), 605 | gost.UsersHandlerOption(node.User), 606 | gost.AuthenticatorHandlerOption(authenticator), 607 | gost.TLSConfigHandlerOption(tlsCfg), 608 | gost.WhitelistHandlerOption(whitelist), 609 | gost.BlacklistHandlerOption(blacklist), 610 | gost.StrategyHandlerOption(gost.NewStrategy(node.Get("strategy"))), 611 | gost.MaxFailsHandlerOption(node.GetInt("max_fails")), 612 | gost.FailTimeoutHandlerOption(node.GetDuration("fail_timeout")), 613 | gost.BypassHandlerOption(node.Bypass), 614 | gost.ResolverHandlerOption(resolver), 615 | gost.HostsHandlerOption(hosts), 616 | gost.RetryHandlerOption(node.GetInt("retry")), // override the global retry option. 617 | gost.TimeoutHandlerOption(timeout), 618 | gost.ProbeResistHandlerOption(node.Get("probe_resist")), 619 | gost.KnockingHandlerOption(node.Get("knock")), 620 | gost.NodeHandlerOption(node), 621 | gost.IPsHandlerOption(ips), 622 | gost.TCPModeHandlerOption(node.GetBool("tcp")), 623 | gost.IPRoutesHandlerOption(tunRoutes...), 624 | gost.LimiterHandlerOption(limiterHandler), 625 | ) 626 | 627 | rt := router{ 628 | node: node, 629 | server: &gost.Server{Listener: ln}, 630 | handler: handler, 631 | chain: chain, 632 | resolver: resolver, 633 | hosts: hosts, 634 | } 635 | rts = append(rts, rt) 636 | } 637 | 638 | return rts, nil 639 | } 640 | 641 | type router struct { 642 | node gost.Node 643 | server *gost.Server 644 | handler gost.Handler 645 | chain *gost.Chain 646 | resolver gost.Resolver 647 | hosts *gost.Hosts 648 | } 649 | 650 | func (r *router) Serve() error { 651 | logger.FInfo("%s on %s", r.node.String(), r.server.Addr()) 652 | return r.server.Serve(r.handler) 653 | } 654 | 655 | func (r *router) Close() error { 656 | if r == nil || r.server == nil { 657 | return nil 658 | } 659 | return r.server.Close() 660 | } 661 | -------------------------------------------------------------------------------- /tests/check_ip_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "sync" 5 | "testing" 6 | 7 | "proxy-collect/consts" 8 | "proxy-collect/service" 9 | ) 10 | 11 | //https://pzzqz.com/ 12 | func TestCheckIp(t *testing.T) { 13 | items := [][]string{ 14 | {"localhost", "9999", consts.PROTO_HTTPS}, 15 | {"localhost", "8888", consts.PROTO_HTTP}, 16 | {"localhost", "8899", consts.PROTO_SOCKS4}, 17 | {"localhost", "9988", consts.PROTO_SOCKS5, "root", "123"}, 18 | } 19 | wg := sync.WaitGroup{} 20 | 21 | wg.Add(1) 22 | go func() { 23 | for _, item := range items { 24 | proxy := service.ProxyService.ParseProxyArr(item) 25 | r, ping := service.ProxyService.CheckIpStatus(&proxy) 26 | t.Log(item, r, ping) 27 | } 28 | defer wg.Done() 29 | }() 30 | wg.Wait() 31 | } 32 | 33 | func TestCheckIpBatch(t *testing.T) { 34 | item := []string{"0.0.0.0", "8888", consts.PROTO_HTTP, "root", "123"} 35 | wg := sync.WaitGroup{} 36 | 37 | for i := 0; i < 10; i++ { 38 | wg.Add(1) 39 | go func() { 40 | proxy := service.ProxyService.ParseProxyArr(item) 41 | r, ping := service.ProxyService.CheckIpStatus(&proxy) 42 | t.Log(item, r, ping) 43 | defer wg.Done() 44 | }() 45 | } 46 | wg.Wait() 47 | } 48 | -------------------------------------------------------------------------------- /tests/conf.yaml: -------------------------------------------------------------------------------- 1 | dao: redis #set storage type -> redis|database 2 | redis: #redis config 3 | MaxIdle: 2 4 | MaxActive: 5 5 | Network: tcp 6 | Address: 127.0.0.1:6379 7 | Password: 8 | database: #database config 9 | Dialect: mysql 10 | Args: python:123456@(127.0.0.1:3306)/py?charset=utf8&loc=Local 11 | MaxIdle: 5 #max idle connections count 12 | MaxOpen: 50 #max connections count 13 | api: 14 | host: 0.0.0.0 15 | port: 8090 16 | poolSize: 500 17 | localIpDataPath: /tmp/local_ip_data.dat 18 | recheckCount: 10 #recheck count if ip is invalid 19 | maxPing: 100000 #max timeout duration, Millisecond 20 | log: 21 | logLevel: 6 #0:Panic 1: Fatal 2:error 3:warning 4:info 5:debug 6:trace 22 | errorLogFile: errors.log 23 | tunnels: 24 | - 25 | proto: http 26 | host: 0.0.0.0 27 | port: 8888 28 | users: 29 | - 30 | username: root 31 | password: 123 32 | limiter: 1,10,2 33 | getters: 34 | - 35 | method: GET 36 | regexp: (\d+\.\d+\.\d+\.\d+):(\d+) 37 | proto: http #代理类型 38 | proxy: false #是否通过代理获取 39 | agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36 40 | urls: 41 | - 42 | https://www.zdaye.com/dayProxy/1.html -------------------------------------------------------------------------------- /tests/config_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "testing" 5 | 6 | "proxy-collect/config" 7 | ) 8 | 9 | func TestConfig(t *testing.T) { 10 | config.LoadConfig() 11 | t.Log(config.Get()) 12 | } 13 | -------------------------------------------------------------------------------- /tests/get_ip_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "math/rand" 5 | "sync" 6 | "sync/atomic" 7 | "testing" 8 | "time" 9 | 10 | "github.com/tongsq/go-lib/logger" 11 | "proxy-collect/bootstrap" 12 | "proxy-collect/service" 13 | ) 14 | 15 | func TestGetIp(t *testing.T) { 16 | bootstrap.Bootstrap() 17 | s := service.CommonGetterHttp 18 | wg := sync.WaitGroup{} 19 | succ := 0 20 | for _, requestUrl := range s.GetUrlList() { 21 | t.Log(requestUrl) 22 | contentBody := s.GetContentHtml(requestUrl) 23 | if contentBody == "" { 24 | time.Sleep(time.Second * 5) 25 | continue 26 | } 27 | proxyList := s.ParseHtml(contentBody) 28 | logger.Info("get ip list:", logger.Fields{"list": proxyList}) 29 | for _, item := range proxyList { 30 | p := service.ProxyService.ParseProxyArr(item) 31 | wg.Add(1) 32 | go func() { 33 | defer wg.Done() 34 | r, ping := service.ProxyService.CheckIpStatus(&p) 35 | t.Log(p, r, ping) 36 | if r { 37 | succ++ 38 | } 39 | }() 40 | } 41 | } 42 | wg.Wait() 43 | t.Log("success count:", succ) 44 | } 45 | 46 | func TestRand(t *testing.T) { 47 | rand.Seed(time.Now().Unix()) 48 | i := rand.Intn(10) 49 | t.Log(i) 50 | } 51 | 52 | func TestAto(t *testing.T) { 53 | var num int64 = 0 54 | var num2 int64 = 0 55 | wg := sync.WaitGroup{} 56 | for i := 0; i < 10000; i++ { 57 | wg.Add(1) 58 | go func() { 59 | defer wg.Done() 60 | num2++ 61 | atomic.AddInt64(&num, 1) 62 | }() 63 | } 64 | wg.Wait() 65 | t.Log(num, num2) 66 | } 67 | -------------------------------------------------------------------------------- /tests/getter_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "encoding/json" 5 | "github.com/tongsq/go-lib/logger" 6 | "proxy-collect/bootstrap" 7 | "proxy-collect/config" 8 | "proxy-collect/global" 9 | "proxy-collect/service" 10 | "proxy-collect/service/proxy_getter" 11 | "testing" 12 | "time" 13 | ) 14 | 15 | func TestGetter(t *testing.T) { 16 | bootstrap.Bootstrap() 17 | var configGetterList []service.ProxyGetterInterface 18 | for _, conf := range config.Get().Getters { 19 | configGetterList = append(configGetterList, proxy_getter.NewGetter(&conf)) 20 | } 21 | config.RegisterConfigRefreshHandler(func(old, new *config.ConfDto) { 22 | var oldConfig, newConfig []byte 23 | oldConfig, _ = json.Marshal(old.Getters) 24 | newConfig, _ = json.Marshal(new.Getters) 25 | if string(oldConfig) != string(newConfig) { 26 | logger.Info("reload getter configs", map[string]interface{}{"old": string(oldConfig), "new": string(newConfig)}) 27 | configGetterList = []service.ProxyGetterInterface{} 28 | for _, conf := range new.Getters { 29 | configGetterList = append(configGetterList, proxy_getter.NewGetter(&conf)) 30 | } 31 | } 32 | }) 33 | for { 34 | for _, configGetter := range configGetterList { 35 | go service.ProxyService.DoGetProxy(configGetter, global.Pool) 36 | } 37 | time.Sleep(time.Second * 60) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /tests/local_ip_service_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "testing" 5 | 6 | "proxy-collect/service/ip" 7 | ) 8 | 9 | func TestGetIpInfoLocal(t *testing.T) { 10 | result := ip.GetIpInfo("139.9.64.238", "") 11 | t.Log(result) 12 | } 13 | -------------------------------------------------------------------------------- /tests/test_check_active.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/tongsq/go-lib/logger" 8 | "proxy-collect/dao" 9 | ) 10 | 11 | func TestActive(t *testing.T) { 12 | logger.FSuccess("check active ip start run") 13 | proxies, err := dao.ProxyDao.GetActiveList() 14 | if err != nil { 15 | logger.Error("get active ip fail", logger.Fields{"err": err}) 16 | } 17 | logger.FInfo("check active ip, len:%d, cap:%d", len(proxies), cap(proxies)) 18 | //pool := component.NewTaskPool(20) 19 | //defer pool.Close() 20 | for _, proxy := range proxies { 21 | //var proxyTmp model.ProxyModel = proxy 22 | proxy := proxy 23 | go func(h, p string) { 24 | fmt.Println(h, p) 25 | fmt.Println(proxy.Host, proxy.Port, "a") 26 | }(proxy.Host, proxy.Port) 27 | //component.TaskPool.RunTask(func() { service.ProxyService.CheckProxyAndSave(proxyTmp.Host, proxyTmp.Port, "") }) 28 | } 29 | } 30 | --------------------------------------------------------------------------------