├── .github └── workflows │ └── release.yml ├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── cmd └── main.go ├── config └── config.toml ├── go.mod ├── go.sum └── internal ├── api ├── handler.go └── response │ └── response.go ├── checker └── checker.go ├── config └── config.go ├── crawler ├── crawler.go └── sources │ ├── base.go │ ├── kuaidaili.go │ └── openproxylist.go ├── logger └── logger.go ├── middleware ├── auth.go ├── error.go ├── logger.go └── ratelimit.go ├── model └── proxy.go ├── storage └── redis.go └── validator └── validator.go /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release ProxyPool 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'v*' 7 | 8 | permissions: 9 | contents: write 10 | 11 | jobs: 12 | build: 13 | name: Build and Release 14 | runs-on: ubuntu-latest 15 | 16 | steps: 17 | - name: Checkout code 18 | uses: actions/checkout@v3 19 | with: 20 | fetch-depth: 0 21 | 22 | - name: Set up Go 23 | uses: actions/setup-go@v4 24 | with: 25 | go-version: '1.21' # 设置你使用的 Go 版本 26 | 27 | - name: Get version 28 | id: get_version 29 | run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV 30 | 31 | - name: Build binaries 32 | run: | 33 | mkdir -p build/release 34 | # Linux amd64 - 添加静态链接标志 35 | CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "-X main.Version=${{ env.VERSION }} -w -s" -o build/release/proxypool-linux-amd64 cmd/main.go 36 | # Linux arm64 37 | CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -ldflags "-X main.Version=${{ env.VERSION }} -w -s" -o build/release/proxypool-linux-arm64 cmd/main.go 38 | # Windows amd64 39 | CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -ldflags "-X main.Version=${{ env.VERSION }} -w -s" -o build/release/proxypool-windows-amd64.exe cmd/main.go 40 | # macOS amd64 41 | CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -ldflags "-X main.Version=${{ env.VERSION }} -w -s" -o build/release/proxypool-darwin-amd64 cmd/main.go 42 | # macOS arm64 43 | CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -ldflags "-X main.Version=${{ env.VERSION }} -w -s" -o build/release/proxypool-darwin-arm64 cmd/main.go 44 | 45 | # 为每个平台创建正确的目录结构和配置文件 46 | cd build/release 47 | 48 | # Linux AMD64 49 | mkdir -p linux-amd64/data 50 | cp ../../config/config.toml linux-amd64/data/ 51 | mv proxypool-linux-amd64 linux-amd64/ 52 | zip -r proxypool-linux-amd64.zip linux-amd64/ 53 | 54 | # Linux ARM64 55 | mkdir -p linux-arm64/data 56 | cp ../../config/config.toml linux-arm64/data/ 57 | mv proxypool-linux-arm64 linux-arm64/ 58 | zip -r proxypool-linux-arm64.zip linux-arm64/ 59 | 60 | # Windows AMD64 61 | mkdir -p windows-amd64/data 62 | cp ../../config/config.toml windows-amd64/data/ 63 | mv proxypool-windows-amd64.exe windows-amd64/ 64 | zip -r proxypool-windows-amd64.zip windows-amd64/ 65 | 66 | # macOS AMD64 67 | mkdir -p darwin-amd64/data 68 | cp ../../config/config.toml darwin-amd64/data/ 69 | mv proxypool-darwin-amd64 darwin-amd64/ 70 | zip -r proxypool-darwin-amd64.zip darwin-amd64/ 71 | 72 | # macOS ARM64 73 | mkdir -p darwin-arm64/data 74 | cp ../../config/config.toml darwin-arm64/data/ 75 | mv proxypool-darwin-arm64 darwin-arm64/ 76 | zip -r proxypool-darwin-arm64.zip darwin-arm64/ 77 | 78 | - name: Create Release 79 | id: create_release 80 | uses: softprops/action-gh-release@v1 81 | env: 82 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 83 | with: 84 | name: ProxyPool ${{ github.ref_name }} 85 | draft: false 86 | prerelease: false 87 | files: | 88 | build/release/proxypool-linux-amd64.zip 89 | build/release/proxypool-linux-arm64.zip 90 | build/release/proxypool-windows-amd64.zip 91 | build/release/proxypool-darwin-amd64.zip 92 | build/release/proxypool-darwin-arm64.zip 93 | body: | 94 | ## ProxyPool ${{ github.ref_name }} 发布说明 95 | 96 | ### 支持平台 97 | - Linux (amd64, arm64) 98 | - Windows (amd64) 99 | - macOS (amd64, arm64) 100 | 101 | ### 环境依赖 102 | - Redis 服务（必需） 103 | 104 | ### Redis 服务部署建议 105 | 106 | 1. 使用 Docker 运行 Redis（推荐）： 107 | ```bash 108 | docker pull redis:latest 109 | docker run -d --name redis -p 6379:6379 redis:latest 110 | ``` 111 | 112 | 2. 或直接安装 Redis： 113 | - Linux: `apt install redis-server` 或 `yum install redis` 114 | - macOS: `brew install redis` 115 | - Windows: 从 Redis 官网下载安装包 116 | 117 | ### 使用说明 118 | 1. 确保 Redis 服务已启动 119 | 2. 下载对应平台的压缩包 120 | 3. 解压后进入目录 121 | 4. 配置文件位于 `data/config.toml`，按需修改 Redis 连接信息 122 | 5. 运行可执行文件启动服务 123 | 124 | ### 配置说明 125 | 在 `data/config.toml` 中配置 Redis： 126 | ```toml 127 | [redis] 128 | host = "localhost" # Redis 服务器地址 129 | port = 6379 # Redis 端口 130 | password = "" # Redis 密码（如果有） 131 | db = 0 # 使用的数据库编号 132 | ``` 133 | 134 | ### 注意事项 135 | - 首次使用请先修改配置文件 136 | - 确保 data 目录与程序在同一目录下 137 | - 确保 Redis 服务正常运行且可访问 -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /build/ 2 | .idea/ 3 | .vscode/ 4 | *.log 5 | .DS_Store 6 | *.swp 7 | *.swo 8 | /vendor/ 9 | *.test 10 | *.out -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright © 2024 Jonty 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 8 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # 变量定义 2 | BINARY_NAME=proxypool 3 | BUILD_DIR=build 4 | DATA_DIR=$(BUILD_DIR)/data 5 | GO_FILES=$(shell find . -name "*.go") 6 | COMMIT_HASH=$(shell git rev-parse --short HEAD 2>/dev/null || echo "unknown") 7 | BUILD_TIME=$(shell date +%FT%T%z) 8 | 9 | # 默认目标 10 | .DEFAULT_GOAL := build 11 | 12 | # 创建必要的目录 13 | .PHONY: init 14 | init: 15 | mkdir -p $(BUILD_DIR) 16 | mkdir -p $(DATA_DIR) 17 | mkdir -p $(BUILD_DIR)/logs 18 | 19 | # 清理构建目录 20 | .PHONY: clean 21 | clean: 22 | rm -rf $(BUILD_DIR) 23 | 24 | # 编译 25 | .PHONY: build 26 | build: clean init 27 | # 编译程序 28 | go build -ldflags "-X main.CommitHash=$(COMMIT_HASH) -X main.BuildTime=$(BUILD_TIME)" -o $(BUILD_DIR)/$(BINARY_NAME) cmd/main.go 29 | # 复制配置文件 30 | cp config/config.toml $(DATA_DIR)/ 31 | 32 | # 运行 33 | .PHONY: run 34 | run: build 35 | cd $(BUILD_DIR) && ./$(BINARY_NAME) 36 | 37 | # 测试 38 | .PHONY: test 39 | test: 40 | go test -v ./... 41 | 42 | # 帮助 43 | .PHONY: help 44 | help: 45 | @echo "Make targets:" 46 | @echo " build - Build the application" 47 | @echo " clean - Remove build directory" 48 | @echo " run - Build and run the application" 49 | @echo " test - Run tests" 50 | @echo " help - Show this help" -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ProxyPool 2 | 3 | 一个基于 Go 的高性能代理池系统，支持自动抓取、验证和提供代理服务。 4 | 5 | ## 功能特点 6 | 7 | - 支持多种代理类型（HTTP/HTTPS/SOCKS4/SOCKS5） 8 | - 自动抓取免费代理 9 | - 定时验证可用性 10 | - RESTful API 接口 11 | - 代理质量评分 12 | - 安全特性 13 | - 基本认证 (Basic Auth) 14 | - API Key 认证 15 | - IP 访问频率限制 16 | - 自动封禁异常 IP 17 | - 所有安全特性可配置 18 | 19 | ## 快速开始 20 | 21 | ### 环境要求 22 | - Redis 6.0+ (必需) 23 | 24 | ### 部署步骤 25 | 26 | 1. 首先需要运行 Redis 服务（以下方式二选一）： 27 | 28 | a. 使用 Docker 运行 Redis（推荐）： 29 | ```bash 30 | docker pull redis:latest 31 | docker run -d --name redis -p 6379:6379 redis:latest 32 | ``` 33 | 34 | b. 直接安装 Redis： 35 | - Linux: `apt install redis-server` 或 `yum install redis` 36 | - macOS: `brew install redis` 37 | - Windows: 从 Redis 官网下载安装包 38 | 39 | 2. 运行 ProxyPool： 40 | ```bash 41 | # 解压下载的发布包 42 | unzip proxypool-{对应平台}.zip 43 | cd proxypool-{对应平台} 44 | 45 | # 运行服务 46 | ./proxypool-{对应平台} 47 | ``` 48 | 49 | ### Redis 配置 50 | 51 | 在 `data/config.toml` 中配置 Redis 连接信息： 52 | 53 | ```toml 54 | [redis] 55 | host = "localhost" # Redis 服务器地址 56 | port = 6379 # Redis 端口 57 | password = "" # Redis 密码（如果有） 58 | db = 0 # 使用的数据库编号 59 | ``` 60 | 61 | ### API 使用 62 | 63 | 1. 获取单个代理 64 | ```bash 65 | curl "http://localhost:8080/proxy" 66 | ``` 67 | 68 | 2. 获取指定类型代理 69 | ```bash 70 | curl "http://localhost:8080/proxy?type=http,https&count=5" 71 | ``` 72 | 73 | 3. 获取高匿代理 74 | ```bash 75 | curl "http://localhost:8080/proxy?anonymous=true" 76 | ``` 77 | 78 | ### 认证方式 79 | 80 | 1. 基本认证 (Basic Auth) 81 | ```bash 82 | # 使用用户名密码访问 83 | curl -u admin:123456 "http://localhost:8080/proxy" 84 | 85 | # 使用 base64 编码方式 86 | curl -H "Authorization: Basic YWRtaW46MTIzNDU2" "http://localhost:8080/proxy" 87 | ``` 88 | 89 | 2. API Key 认证 90 | ```bash 91 | # 在请求头中携带 API Key 92 | curl -H "X-API-Key: your-api-key" "http://localhost:8080/proxy" 93 | ``` 94 | 95 | ### 访问限制 96 | - 每个 IP 在指定时间窗口内有请求次数限制 97 | - 超过限制后 IP 会被临时封禁 98 | - 可以通过响应头查看限制情况： 99 | - X-RateLimit-Remaining: 剩余请求次数 100 | - X-RateLimit-Limit: 总请求限制 101 | - X-RateLimit-Reset: 重置时间 102 | 103 | ### 配置安全特性 104 | 105 | 在 `data/config.toml` 中配置： 106 | 107 | ```toml 108 | [security] 109 | # 基本认证 110 | auth_enabled = false # 是否启用认证 111 | username = "admin" # 认证用户名 112 | password = "123456" # 认证密码 113 | 114 | # API Key认证 115 | api_key_enabled = true # 是否启用 API Key 116 | api_keys = ["key1", "key2", "key3"] # 允许的 API Key 列表 117 | 118 | # 限流配置 119 | rate_limit_enabled = true # 是否启用限流 120 | rate_limit = 100 # 每个时间窗口最大请求数 121 | rate_window = 1 # 时间窗口（分钟） 122 | ban_duration = 24 # 封禁时长（小时） 123 | ``` 124 | 125 | ### 响应格式 126 | 127 | ```json 128 | { 129 | "code": 200, 130 | "message": "success", 131 | "data": { 132 | "ip": "1.2.3.4", 133 | "port": "8080", 134 | "type": "http", 135 | "anonymous": true, 136 | "speed_ms": 500, 137 | "score": 100 138 | } 139 | } 140 | ``` 141 | 142 | ### 配置说明 143 | 144 | 配置文件位于 `data/config.toml`，主要配置项： 145 | - Redis 连接信息（必需配置） 146 | - 代理验证参数 147 | - 爬虫更新间隔 148 | - 日志配置 149 | - 安全特性配置 150 | 151 | ## 添加代理源 152 | 153 | 1. 在 `internal/crawler/sources` 目录下创建新的源文件，例如 `myproxy.go`： 154 | 155 | ```go 156 | package sources 157 | 158 | type MyProxySource struct { 159 | BaseSource 160 | } 161 | 162 | func NewMyProxySource() *MyProxySource { 163 | return &MyProxySource{ 164 | BaseSource: BaseSource{name: "myproxy"}, 165 | } 166 | } 167 | 168 | func (s *MyProxySource) Fetch() ([]*model.Proxy, error) { 169 | // 实现代理获取逻辑 170 | proxies := make([]*model.Proxy, 0) 171 | 172 | // ... 获取代理的具体实现 ... 173 | 174 | return proxies, nil 175 | } 176 | ``` 177 | 178 | 2. 在 `internal/crawler/crawler.go` 中注册新代理源： 179 | 180 | ```go 181 | func NewManager(storage storage.Storage, validator *validator.Validator) *Manager { 182 | return &Manager{ 183 | sources: []sources.Source{ 184 | sources.NewOpenProxyListSource(), 185 | sources.NewMyProxySource(), // 添加新代理源 186 | }, 187 | storage: storage, 188 | validator: validator, 189 | } 190 | } 191 | ``` 192 | 193 | ## 常见问题 194 | 195 | 1. Redis 连接失败 196 | - 检查 Redis 服务是否正常运行 197 | - 确认配置文件中的 Redis 连接信息是否正确 198 | - 确保 Redis 端口（默认6379）未被占用 199 | 200 | 2. 配置文件找不到 201 | - 确保 `data` 目录与程序在同一目录下 202 | - 确保 `data/config.toml` 文件存在且格式正确 203 | 204 | ## License 205 | 206 | MIT License 207 | -------------------------------------------------------------------------------- /cmd/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "time" 8 | 9 | "github.com/langchou/proxyPool/internal/api" 10 | "github.com/langchou/proxyPool/internal/checker" 11 | "github.com/langchou/proxyPool/internal/config" 12 | "github.com/langchou/proxyPool/internal/crawler" 13 | "github.com/langchou/proxyPool/internal/logger" 14 | "github.com/langchou/proxyPool/internal/middleware" 15 | "github.com/langchou/proxyPool/internal/storage" 16 | "github.com/langchou/proxyPool/internal/validator" 17 | 18 | "github.com/gin-gonic/gin" 19 | "go.uber.org/zap" 20 | ) 21 | 22 | func main() { 23 | // 解析命令行参数 24 | configPath := flag.String("config", "data/config.toml", "path to config file") 25 | flag.Parse() 26 | 27 | // 加载配置文件 28 | if err := config.LoadConfig(*configPath); err != nil { 29 | panic(err) 30 | } 31 | 32 | // 初始化日志 33 | if err := logger.Init( 34 | config.GlobalConfig.Log.Level, 35 | config.GlobalConfig.Log.Output, 36 | config.GlobalConfig.Log.FilePath, 37 | ); err != nil { 38 | panic(fmt.Sprintf("Initialize logger failed: %v", err)) 39 | } 40 | defer logger.Log.Sync() 41 | 42 | logger.Log.Info("Starting proxy pool service...") 43 | 44 | // 设置gin模式 45 | gin.SetMode(config.GlobalConfig.Server.Mode) 46 | 47 | // 初始化存储 48 | store := storage.NewRedisStorage( 49 | config.GlobalConfig.GetRedisAddr(), 50 | config.GlobalConfig.Redis.Password, 51 | config.GlobalConfig.Redis.DB, 52 | ) 53 | logger.Log.Info("Redis storage initialized") 54 | 55 | // 初始化验证器 56 | validator := validator.NewValidator(config.GlobalConfig.GetValidatorTimeout()) 57 | logger.Log.Info("Proxy validator initialized") 58 | 59 | // 初始化爬虫管理器 60 | crawler := crawler.NewManager(store, validator) 61 | logger.Log.Info("Crawler manager initialized") 62 | 63 | // 初始化检查器 64 | checker := checker.NewChecker(store, validator) 65 | logger.Log.Info("Proxy checker initialized") 66 | 67 | // 启动后台爬虫任务 68 | go func() { 69 | logger.Log.Info("Starting crawler goroutine") 70 | 71 | // 创建一个用于取消的context 72 | ctx := context.Background() 73 | 74 | // 首次执行爬虫任务 75 | logger.Log.Info("Running initial proxy crawling...") 76 | if err := crawler.Run(ctx); err != nil { 77 | logger.Log.Error("Initial crawling failed", zap.Error(err)) 78 | } 79 | 80 | // 创建定时器 81 | interval := config.GlobalConfig.GetCrawlerInterval() 82 | logger.Log.Info("Setting up crawler timer", zap.Duration("interval", interval)) 83 | ticker := time.NewTicker(interval) 84 | defer ticker.Stop() 85 | 86 | for { 87 | select { 88 | case <-ticker.C: 89 | logger.Log.Info("Starting scheduled proxy crawling...") 90 | if err := crawler.Run(ctx); err != nil { 91 | logger.Log.Error("Scheduled crawling failed", zap.Error(err)) 92 | } 93 | } 94 | } 95 | }() 96 | 97 | // 启动定时检查任务 98 | go func() { 99 | logger.Log.Info("Starting checker goroutine") 100 | 101 | ctx := context.Background() 102 | 103 | // 等待30秒后开始第一次检查，给爬虫一些时间先获取代理 104 | logger.Log.Info("Waiting 30 seconds before first check...") 105 | time.Sleep(30 * time.Second) 106 | 107 | // 执行第一次检查 108 | logger.Log.Info("Running initial proxy check...") 109 | if err := checker.Run(ctx); err != nil { 110 | logger.Log.Error("Initial proxy check failed", zap.Error(err)) 111 | } 112 | 113 | // 创建定时器 114 | interval := config.GlobalConfig.GetCheckInterval() 115 | logger.Log.Info("Setting up checker timer", zap.Duration("interval", interval)) 116 | ticker := time.NewTicker(interval) 117 | defer ticker.Stop() 118 | 119 | for { 120 | select { 121 | case <-ticker.C: 122 | logger.Log.Info("Starting scheduled proxy check...") 123 | if err := checker.Run(ctx); err != nil { 124 | logger.Log.Error("Scheduled proxy check failed", zap.Error(err)) 125 | } 126 | } 127 | } 128 | }() 129 | 130 | // 启动API服务 131 | r := gin.New() 132 | r.Use(middleware.Logger()) 133 | r.Use(middleware.ErrorHandler()) 134 | 135 | // 初始化限流器（如果启用） 136 | if config.GlobalConfig.Security.RateLimitEnabled { 137 | rateLimiter := middleware.NewRateLimiter( 138 | store.GetRedisClient(), 139 | config.GlobalConfig.Security.RateLimit, 140 | time.Duration(config.GlobalConfig.Security.RateWindow)*time.Minute, 141 | time.Duration(config.GlobalConfig.Security.BanDuration)*time.Hour, 142 | ) 143 | r.Use(rateLimiter.RateLimit()) 144 | } 145 | 146 | r.Use(middleware.BasicAuth()) // 基本认证 147 | r.Use(middleware.APIKeyAuth()) // API Key 认证 148 | r.Use(gin.Recovery()) 149 | 150 | handler := api.NewHandler(store) 151 | r.GET("/proxy", handler.GetProxy) 152 | r.GET("/proxies", handler.GetAllProxies) 153 | 154 | // 添加健康检查接口 155 | r.GET("/health", func(c *gin.Context) { 156 | c.JSON(200, gin.H{ 157 | "status": "ok", 158 | "time": time.Now().Format(time.RFC3339), 159 | }) 160 | }) 161 | 162 | addr := fmt.Sprintf(":%d", config.GlobalConfig.Server.Port) 163 | logger.Log.Info("Starting HTTP server", zap.String("addr", addr)) 164 | if err := r.Run(addr); err != nil { 165 | logger.Log.Fatal("Failed to start HTTP server", zap.Error(err)) 166 | } 167 | } 168 | -------------------------------------------------------------------------------- /config/config.toml: -------------------------------------------------------------------------------- 1 | # 服务配置 2 | [server] 3 | port = 8080 4 | mode = "debug" # debug/release 5 | 6 | # Redis配置 7 | [redis] 8 | host = "localhost" 9 | port = 6379 10 | password = "" 11 | db = 0 12 | 13 | # 代理验证配置 14 | [validator] 15 | timeout = 10 # 超时时间（秒） 16 | check_interval = 10 # 定时检查间隔（分钟） 17 | test_url = "http://httpbin.org/ip" 18 | 19 | # 爬虫配置 20 | [crawler] 21 | interval = 30 # 爬取间隔（分钟） 22 | batch_size = 20 # 每批验证的代理数量 23 | fetch_delay = 2 # 每个页面爬取间隔（秒） 24 | max_retry = 3 # 最大重试次数 25 | 26 | # 日志配置 27 | [log] 28 | level = "debug" # debug/info/warn/error 29 | output = "file" # console/file 30 | file_path = "logs/proxy_pool.log" 31 | 32 | # 安全配置 33 | [security] 34 | # 基本认证 35 | auth_enabled = false # 是否启用认证 36 | username = "admin" # 基本认证用户名 37 | password = "123456" # 基本认证密码 38 | 39 | # API Key认证 40 | api_key_enabled = false # 是否启用 API Key 41 | api_keys = ["key1", "key2", "key3"] # 允许的 API Key 列表 42 | 43 | # 限流配置 44 | rate_limit_enabled = false # 是否启用限流 45 | rate_limit = 100 # 每个时间窗口最大请求数 46 | rate_window = 1 # 时间窗口（分钟） 47 | ban_duration = 24 # 封禁时长（小时） 48 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/langchou/proxyPool 2 | 3 | go 1.21 4 | 5 | require ( 6 | github.com/PuerkitoBio/goquery v1.9.1 7 | github.com/gin-gonic/gin v1.9.1 8 | github.com/redis/go-redis/v9 v9.5.1 9 | github.com/spf13/viper v1.18.2 10 | go.uber.org/zap v1.27.0 11 | golang.org/x/net v0.21.0 12 | ) 13 | 14 | require ( 15 | github.com/andybalholm/cascadia v1.3.2 // indirect 16 | github.com/bytedance/sonic v1.9.1 // indirect 17 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 18 | github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect 19 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 20 | github.com/fsnotify/fsnotify v1.7.0 // indirect 21 | github.com/gabriel-vasile/mimetype v1.4.2 // indirect 22 | github.com/gin-contrib/sse v0.1.0 // indirect 23 | github.com/go-playground/locales v0.14.1 // indirect 24 | github.com/go-playground/universal-translator v0.18.1 // indirect 25 | github.com/go-playground/validator/v10 v10.14.0 // indirect 26 | github.com/goccy/go-json v0.10.2 // indirect 27 | github.com/hashicorp/hcl v1.0.0 // indirect 28 | github.com/json-iterator/go v1.1.12 // indirect 29 | github.com/klauspost/cpuid/v2 v2.2.4 // indirect 30 | github.com/leodido/go-urn v1.2.4 // indirect 31 | github.com/magiconair/properties v1.8.7 // indirect 32 | github.com/mattn/go-isatty v0.0.19 // indirect 33 | github.com/mitchellh/mapstructure v1.5.0 // indirect 34 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 35 | github.com/modern-go/reflect2 v1.0.2 // indirect 36 | github.com/pelletier/go-toml/v2 v2.1.0 // indirect 37 | github.com/sagikazarmark/locafero v0.4.0 // indirect 38 | github.com/sagikazarmark/slog-shim v0.1.0 // indirect 39 | github.com/sourcegraph/conc v0.3.0 // indirect 40 | github.com/spf13/afero v1.11.0 // indirect 41 | github.com/spf13/cast v1.6.0 // indirect 42 | github.com/spf13/pflag v1.0.5 // indirect 43 | github.com/subosito/gotenv v1.6.0 // indirect 44 | github.com/twitchyliquid64/golang-asm v0.15.1 // indirect 45 | github.com/ugorji/go/codec v1.2.11 // indirect 46 | go.uber.org/multierr v1.10.0 // indirect 47 | golang.org/x/arch v0.3.0 // indirect 48 | golang.org/x/crypto v0.19.0 // indirect 49 | golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect 50 | golang.org/x/sys v0.17.0 // indirect 51 | golang.org/x/text v0.14.0 // indirect 52 | google.golang.org/protobuf v1.31.0 // indirect 53 | gopkg.in/ini.v1 v1.67.0 // indirect 54 | gopkg.in/yaml.v3 v3.0.1 // indirect 55 | ) 56 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/PuerkitoBio/goquery v1.9.1 h1:mTL6XjbJTZdpfL+Gwl5U2h1l9yEkJjhmlTeV9VPW7UI= 2 | github.com/PuerkitoBio/goquery v1.9.1/go.mod h1:cW1n6TmIMDoORQU5IU/P1T3tGFunOeXEpGP2WHRwkbY= 3 | github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= 4 | github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= 5 | github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= 6 | github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= 7 | github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= 8 | github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= 9 | github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= 10 | github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= 11 | github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= 12 | github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= 13 | github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 14 | github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= 15 | github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= 16 | github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= 17 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 18 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 19 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= 20 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 21 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= 22 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= 23 | github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= 24 | github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= 25 | github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= 26 | github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= 27 | github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= 28 | github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= 29 | github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= 30 | github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= 31 | github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= 32 | github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= 33 | github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= 34 | github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= 35 | github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= 36 | github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= 37 | github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= 38 | github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= 39 | github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js= 40 | github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= 41 | github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= 42 | github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= 43 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= 44 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 45 | github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= 46 | github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 47 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 48 | github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= 49 | github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= 50 | github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= 51 | github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= 52 | github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= 53 | github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= 54 | github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= 55 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 56 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 57 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 58 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 59 | github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= 60 | github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= 61 | github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= 62 | github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= 63 | github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= 64 | github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 65 | github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= 66 | github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= 67 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 68 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 69 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 70 | github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= 71 | github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= 72 | github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= 73 | github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= 74 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 75 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= 76 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 77 | github.com/redis/go-redis/v9 v9.5.1 h1:H1X4D3yHPaYrkL5X06Wh6xNVM/pX0Ft4RV0vMGvLBh8= 78 | github.com/redis/go-redis/v9 v9.5.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= 79 | github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= 80 | github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= 81 | github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= 82 | github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= 83 | github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= 84 | github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= 85 | github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= 86 | github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= 87 | github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= 88 | github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= 89 | github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= 90 | github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= 91 | github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= 92 | github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 93 | github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ= 94 | github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk= 95 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 96 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 97 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 98 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 99 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 100 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 101 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 102 | github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= 103 | github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= 104 | github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= 105 | github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= 106 | github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= 107 | github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= 108 | github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= 109 | github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= 110 | github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= 111 | github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= 112 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 113 | go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= 114 | go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= 115 | go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= 116 | go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= 117 | go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= 118 | go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= 119 | golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= 120 | golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= 121 | golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= 122 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 123 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 124 | golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= 125 | golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= 126 | golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= 127 | golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= 128 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 129 | golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= 130 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 131 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 132 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 133 | golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= 134 | golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= 135 | golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= 136 | golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= 137 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 138 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 139 | golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 140 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 141 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 142 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 143 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 144 | golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 145 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 146 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 147 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 148 | golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 149 | golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= 150 | golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 151 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 152 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 153 | golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= 154 | golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= 155 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 156 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 157 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 158 | golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 159 | golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= 160 | golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= 161 | golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= 162 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 163 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 164 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= 165 | golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= 166 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 167 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 168 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 169 | google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= 170 | google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= 171 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 172 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= 173 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 174 | gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= 175 | gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= 176 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 177 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 178 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 179 | rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= 180 | -------------------------------------------------------------------------------- /internal/api/handler.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "github.com/langchou/proxyPool/internal/api/response" 5 | "github.com/langchou/proxyPool/internal/logger" 6 | "github.com/langchou/proxyPool/internal/model" 7 | "github.com/langchou/proxyPool/internal/storage" 8 | "strconv" 9 | "strings" 10 | 11 | "github.com/gin-gonic/gin" 12 | "github.com/redis/go-redis/v9" 13 | "go.uber.org/zap" 14 | ) 15 | 16 | type Handler struct { 17 | storage storage.Storage 18 | } 19 | 20 | func NewHandler(storage storage.Storage) *Handler { 21 | return &Handler{storage: storage} 22 | } 23 | 24 | // GetProxy 获取代理 25 | // @param type: 代理类型，可选值：http,https,socks4,socks5，多个类型用逗号分隔 26 | // @param count: 返回数量，默认1 27 | // @param anonymous: 是否只返回高匿代理，可选值：true/false 28 | func (h *Handler) GetProxy(c *gin.Context) { 29 | logger.Log.Info("Received request for proxy") 30 | 31 | // 解析请求参数 32 | proxyTypes := parseProxyTypes(c.Query("type")) 33 | count := parseCount(c.Query("count"), 1) 34 | anonymous := c.Query("anonymous") == "true" 35 | 36 | // 获取所有代理 37 | proxies, err := h.storage.GetAll(c.Request.Context()) 38 | if err != nil { 39 | if err == redis.Nil { 40 | logger.Log.Warn("No proxies available") 41 | response.Success(c, []response.ProxyData{}) 42 | return 43 | } 44 | logger.Log.Error("Failed to get proxies", zap.Error(err)) 45 | response.Error(c, "Failed to get proxies") 46 | return 47 | } 48 | 49 | // 过滤代理 50 | filtered := filterProxies(proxies, proxyTypes, anonymous) 51 | if len(filtered) == 0 { 52 | response.Success(c, []response.ProxyData{}) 53 | return 54 | } 55 | 56 | // 限制返回数量 57 | if count > len(filtered) { 58 | count = len(filtered) 59 | } 60 | 61 | result := filtered[:count] 62 | logger.Log.Info("Successfully returned proxies", 63 | zap.Int("requested", count), 64 | zap.Int("returned", len(result))) 65 | 66 | response.Success(c, response.ConvertProxies(result)) 67 | } 68 | 69 | func (h *Handler) GetAllProxies(c *gin.Context) { 70 | logger.Log.Info("Received request for all proxies") 71 | 72 | // 解析请求参数 73 | proxyTypes := parseProxyTypes(c.Query("type")) 74 | anonymous := c.Query("anonymous") == "true" 75 | 76 | proxies, err := h.storage.GetAll(c.Request.Context()) 77 | if err != nil { 78 | logger.Log.Error("Failed to get all proxies", zap.Error(err)) 79 | response.Error(c, "Failed to get proxies") 80 | return 81 | } 82 | 83 | // 过滤代理 84 | filtered := filterProxies(proxies, proxyTypes, anonymous) 85 | 86 | logger.Log.Info("Successfully returned all proxies", 87 | zap.Int("total", len(filtered))) 88 | 89 | response.Success(c, response.ConvertProxies(filtered)) 90 | } 91 | 92 | // 解析代理类型 93 | func parseProxyTypes(typeStr string) []model.ProxyType { 94 | if typeStr == "" { 95 | return nil 96 | } 97 | 98 | types := strings.Split(typeStr, ",") 99 | result := make([]model.ProxyType, 0, len(types)) 100 | 101 | for _, t := range types { 102 | proxyType := model.ProxyType(strings.ToLower(strings.TrimSpace(t))) 103 | if proxyType.IsValid() { 104 | result = append(result, proxyType) 105 | } 106 | } 107 | 108 | return result 109 | } 110 | 111 | // 解析数量 112 | func parseCount(countStr string, defaultValue int) int { 113 | if countStr == "" { 114 | return defaultValue 115 | } 116 | 117 | count, err := strconv.Atoi(countStr) 118 | if err != nil || count < 1 { 119 | return defaultValue 120 | } 121 | return count 122 | } 123 | 124 | // 过滤代理 125 | func filterProxies(proxies []*model.Proxy, types []model.ProxyType, anonymous bool) []*model.Proxy { 126 | if len(proxies) == 0 { 127 | return proxies 128 | } 129 | 130 | filtered := make([]*model.Proxy, 0, len(proxies)) 131 | for _, proxy := range proxies { 132 | // 类型过滤 133 | if len(types) > 0 { 134 | typeMatched := false 135 | for _, t := range types { 136 | if proxy.Type == t { 137 | typeMatched = true 138 | break 139 | } 140 | } 141 | if !typeMatched { 142 | continue 143 | } 144 | } 145 | 146 | // 匿名性过滤 147 | if anonymous && !proxy.Anonymous { 148 | continue 149 | } 150 | 151 | filtered = append(filtered, proxy) 152 | } 153 | 154 | return filtered 155 | } 156 | -------------------------------------------------------------------------------- /internal/api/response/response.go: -------------------------------------------------------------------------------- 1 | package response 2 | 3 | import ( 4 | "net/http" 5 | "github.com/langchou/proxyPool/internal/model" 6 | 7 | "github.com/gin-gonic/gin" 8 | ) 9 | 10 | // 响应码定义 11 | const ( 12 | CodeSuccess = 200 13 | CodeNotFound = 404 14 | CodeInternalError = 500 15 | ) 16 | 17 | // Response 通用响应结构 18 | type Response struct { 19 | Code int `json:"code"` // 响应码 20 | Message string `json:"message"` // 响应信息 21 | Data interface{} `json:"data,omitempty"` // 数据，可选 22 | } 23 | 24 | // ProxyData 代理数据结构 25 | type ProxyData struct { 26 | IP string `json:"ip"` // IP地址 27 | Port string `json:"port"` // 端口 28 | Type string `json:"type"` // 代理类型 29 | Anonymous bool `json:"anonymous"` // 是否高匿 30 | Speed int64 `json:"speed_ms"` // 响应速度（毫秒） 31 | Score int `json:"score"` // 可用性评分 32 | } 33 | 34 | // Success 成功响应 35 | func Success(c *gin.Context, data interface{}) { 36 | c.JSON(http.StatusOK, Response{ 37 | Code: CodeSuccess, 38 | Message: "success", 39 | Data: data, 40 | }) 41 | } 42 | 43 | // NotFound 未找到响应 44 | func NotFound(c *gin.Context, message string) { 45 | c.JSON(http.StatusNotFound, Response{ 46 | Code: CodeNotFound, 47 | Message: message, 48 | }) 49 | } 50 | 51 | // Error 错误响应 52 | func Error(c *gin.Context, message string) { 53 | c.JSON(http.StatusInternalServerError, Response{ 54 | Code: CodeInternalError, 55 | Message: message, 56 | }) 57 | } 58 | 59 | // ConvertProxy 转换代理模型为响应数据 60 | func ConvertProxy(proxy *model.Proxy) ProxyData { 61 | return ProxyData{ 62 | IP: proxy.IP, 63 | Port: proxy.Port, 64 | Type: string(proxy.Type), 65 | Anonymous: proxy.Anonymous, 66 | Speed: proxy.Speed, 67 | Score: proxy.Score, 68 | } 69 | } 70 | 71 | // ConvertProxies 转换代理列表 72 | func ConvertProxies(proxies []*model.Proxy) []ProxyData { 73 | result := make([]ProxyData, len(proxies)) 74 | for i, proxy := range proxies { 75 | result[i] = ConvertProxy(proxy) 76 | } 77 | return result 78 | } 79 | -------------------------------------------------------------------------------- /internal/checker/checker.go: -------------------------------------------------------------------------------- 1 | package checker 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/langchou/proxyPool/internal/logger" 8 | "github.com/langchou/proxyPool/internal/storage" 9 | "github.com/langchou/proxyPool/internal/validator" 10 | "go.uber.org/zap" 11 | ) 12 | 13 | type Checker struct { 14 | storage storage.Storage 15 | validator *validator.Validator 16 | } 17 | 18 | func NewChecker(storage storage.Storage, validator *validator.Validator) *Checker { 19 | return &Checker{ 20 | storage: storage, 21 | validator: validator, 22 | } 23 | } 24 | 25 | func (c *Checker) Run(ctx context.Context) error { 26 | logger.Log.Info("Starting to check existing proxies") 27 | 28 | // 从存储中获取所有代理 29 | proxies, err := c.storage.GetAll(ctx) 30 | if err != nil { 31 | return fmt.Errorf("failed to get proxies from storage: %w", err) 32 | } 33 | 34 | logger.Log.Info("Retrieved proxies for checking", zap.Int("count", len(proxies))) 35 | 36 | for _, proxy := range proxies { 37 | select { 38 | case <-ctx.Done(): 39 | return ctx.Err() 40 | default: 41 | logger.Log.Debug("Checking proxy", 42 | zap.String("ip", proxy.IP), 43 | zap.String("port", proxy.Port)) 44 | 45 | // 验证代理 46 | valid, speed := c.validator.Validate(proxy) 47 | if valid { 48 | proxy.Speed = speed 49 | // 验证成功，更新代理信息 50 | if err := c.storage.Save(ctx, proxy); err != nil { 51 | logger.Log.Error("Failed to update proxy", 52 | zap.String("ip", proxy.IP), 53 | zap.String("port", proxy.Port), 54 | zap.Error(err)) 55 | } 56 | logger.Log.Info("Proxy check passed", 57 | zap.String("ip", proxy.IP), 58 | zap.String("port", proxy.Port), 59 | zap.Int64("speed", speed)) 60 | } else { 61 | // 验证失败，从存储中删除 62 | key := proxy.IP + ":" + proxy.Port 63 | if err := c.storage.Remove(ctx, key); err != nil { 64 | logger.Log.Error("Failed to remove invalid proxy", 65 | zap.String("ip", proxy.IP), 66 | zap.String("port", proxy.Port), 67 | zap.Error(err)) 68 | } 69 | logger.Log.Info("Removed invalid proxy", 70 | zap.String("ip", proxy.IP), 71 | zap.String("port", proxy.Port)) 72 | } 73 | } 74 | } 75 | 76 | logger.Log.Info("Finished checking all proxies") 77 | return nil 78 | } 79 | -------------------------------------------------------------------------------- /internal/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/spf13/viper" 8 | ) 9 | 10 | type Config struct { 11 | Server ServerConfig `mapstructure:"server"` 12 | Redis RedisConfig `mapstructure:"redis"` 13 | Validator ValidatorConfig `mapstructure:"validator"` 14 | Crawler CrawlerConfig `mapstructure:"crawler"` 15 | Log LogConfig `mapstructure:"log"` 16 | Security SecurityConfig `mapstructure:"security"` 17 | } 18 | 19 | type ServerConfig struct { 20 | Port int `mapstructure:"port"` 21 | Mode string `mapstructure:"mode"` 22 | } 23 | 24 | type RedisConfig struct { 25 | Host string `mapstructure:"host"` 26 | Port int `mapstructure:"port"` 27 | Password string `mapstructure:"password"` 28 | DB int `mapstructure:"db"` 29 | } 30 | 31 | type ValidatorConfig struct { 32 | Timeout int `mapstructure:"timeout"` 33 | CheckInterval int `mapstructure:"check_interval"` 34 | TestURL string `mapstructure:"test_url"` 35 | } 36 | 37 | type CrawlerConfig struct { 38 | Interval int `mapstructure:"interval"` 39 | BatchSize int `mapstructure:"batch_size"` 40 | } 41 | 42 | type LogConfig struct { 43 | Level string `mapstructure:"level"` 44 | Output string `mapstructure:"output"` 45 | FilePath string `mapstructure:"file_path"` 46 | } 47 | 48 | // SecurityConfig 安全配置 49 | type SecurityConfig struct { 50 | // 基本认证 51 | AuthEnabled bool `mapstructure:"auth_enabled"` 52 | Username string `mapstructure:"username"` 53 | Password string `mapstructure:"password"` 54 | 55 | // API Key认证 56 | APIKeyEnabled bool `mapstructure:"api_key_enabled"` 57 | APIKeys []string `mapstructure:"api_keys"` 58 | 59 | // 限流配置 60 | RateLimit int `mapstructure:"rate_limit"` // 每个时间窗口最大请求数 61 | RateWindow int `mapstructure:"rate_window"` // 时间窗口（分钟） 62 | BanDuration int `mapstructure:"ban_duration"` // 封禁时长（小时） 63 | RateLimitEnabled bool `mapstructure:"rate_limit_enabled"` // 是否启用限流 64 | } 65 | 66 | var ( 67 | GlobalConfig Config 68 | ) 69 | 70 | // LoadConfig 加载配置文件 71 | func LoadConfig(configPath string) error { 72 | viper.SetConfigFile(configPath) 73 | viper.SetConfigType("toml") 74 | 75 | if err := viper.ReadInConfig(); err != nil { 76 | return fmt.Errorf("failed to read config file: %w", err) 77 | } 78 | 79 | if err := viper.Unmarshal(&GlobalConfig); err != nil { 80 | return fmt.Errorf("failed to unmarshal config: %w", err) 81 | } 82 | 83 | return nil 84 | } 85 | 86 | // Helper functions for getting config values 87 | func (c *Config) GetRedisAddr() string { 88 | return fmt.Sprintf("%s:%d", c.Redis.Host, c.Redis.Port) 89 | } 90 | 91 | func (c *Config) GetValidatorTimeout() time.Duration { 92 | return time.Duration(c.Validator.Timeout) * time.Second 93 | } 94 | 95 | func (c *Config) GetCrawlerInterval() time.Duration { 96 | return time.Duration(c.Crawler.Interval) * time.Minute 97 | } 98 | 99 | func (c *Config) GetCheckInterval() time.Duration { 100 | return time.Duration(c.Validator.CheckInterval) * time.Minute 101 | } 102 | -------------------------------------------------------------------------------- /internal/crawler/crawler.go: -------------------------------------------------------------------------------- 1 | package crawler 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | 7 | "github.com/langchou/proxyPool/internal/crawler/sources" 8 | "github.com/langchou/proxyPool/internal/logger" 9 | "github.com/langchou/proxyPool/internal/storage" 10 | "github.com/langchou/proxyPool/internal/validator" 11 | "go.uber.org/zap" 12 | ) 13 | 14 | type Manager struct { 15 | sources []sources.Source 16 | storage storage.Storage 17 | validator *validator.Validator 18 | } 19 | 20 | func NewManager(storage storage.Storage, validator *validator.Validator) *Manager { 21 | return &Manager{ 22 | sources: []sources.Source{ 23 | sources.NewKuaidailiSource(), 24 | sources.NewOpenProxyListSource(), 25 | // 添加更多代理源 26 | }, 27 | storage: storage, 28 | validator: validator, 29 | } 30 | } 31 | 32 | func (m *Manager) Run(ctx context.Context) error { 33 | var wg sync.WaitGroup 34 | var errs []error 35 | var mu sync.Mutex 36 | 37 | for _, source := range m.sources { 38 | wg.Add(1) 39 | go func(s sources.Source) { 40 | defer wg.Done() 41 | 42 | proxies, err := s.Fetch() 43 | if err != nil { 44 | mu.Lock() 45 | errs = append(errs, err) 46 | mu.Unlock() 47 | return 48 | } 49 | 50 | // 验证和存储代理 51 | for _, proxy := range proxies { 52 | select { 53 | case <-ctx.Done(): 54 | return 55 | default: 56 | // 先验证再存储 57 | valid, speed := m.validator.Validate(proxy) 58 | if valid { 59 | proxy.Speed = speed 60 | proxy.Score = 100 // 初始分数 61 | if err := m.storage.Save(ctx, proxy); err != nil { 62 | mu.Lock() 63 | errs = append(errs, err) 64 | mu.Unlock() 65 | } 66 | logger.Log.Debug("Saved valid proxy", 67 | zap.String("ip", proxy.IP), 68 | zap.String("port", proxy.Port), 69 | zap.String("type", string(proxy.Type))) 70 | } else { 71 | // 确保验证失败的代理被删除（以防之前存在） 72 | key := proxy.IP + ":" + proxy.Port 73 | if err := m.storage.Remove(ctx, key); err != nil { 74 | logger.Log.Error("Failed to remove invalid proxy", 75 | zap.String("ip", proxy.IP), 76 | zap.String("port", proxy.Port), 77 | zap.Error(err)) 78 | } 79 | logger.Log.Debug("Removed invalid proxy", 80 | zap.String("ip", proxy.IP), 81 | zap.String("port", proxy.Port), 82 | zap.String("type", string(proxy.Type))) 83 | } 84 | } 85 | } 86 | }(source) 87 | } 88 | 89 | wg.Wait() 90 | 91 | // 如果有错误，返回第一个错误 92 | if len(errs) > 0 { 93 | return errs[0] 94 | } 95 | return nil 96 | } 97 | -------------------------------------------------------------------------------- /internal/crawler/sources/base.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import "github.com/langchou/proxyPool/internal/model" 4 | 5 | type Source interface { 6 | Name() string 7 | Fetch() ([]*model.Proxy, error) 8 | } 9 | 10 | type BaseSource struct { 11 | name string 12 | } 13 | 14 | func (s *BaseSource) Name() string { 15 | return s.name 16 | } 17 | -------------------------------------------------------------------------------- /internal/crawler/sources/kuaidaili.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "github.com/langchou/proxyPool/internal/logger" 7 | "github.com/langchou/proxyPool/internal/model" 8 | "strings" 9 | "time" 10 | 11 | "github.com/PuerkitoBio/goquery" 12 | "go.uber.org/zap" 13 | ) 14 | 15 | type KuaidailiSource struct { 16 | BaseSource 17 | } 18 | 19 | func NewKuaidailiSource() *KuaidailiSource { 20 | return &KuaidailiSource{ 21 | BaseSource: BaseSource{name: "kuaidaili"}, 22 | } 23 | } 24 | 25 | func (s *KuaidailiSource) Fetch() ([]*model.Proxy, error) { 26 | logger.Log.Info("Starting to fetch proxies from kuaidaili") 27 | proxies := make([]*model.Proxy, 0) 28 | 29 | urls := map[string]string{ 30 | "http": "https://www.kuaidaili.com/free/intr/", 31 | "https": "https://www.kuaidaili.com/free/inha/", 32 | "socks": "https://www.kuaidaili.com/ops/proxylist/", 33 | } 34 | 35 | for proxyType, baseURL := range urls { 36 | logger.Log.Debug("Fetching proxy type", zap.String("type", proxyType)) 37 | for i := 1; i <= 3; i++ { 38 | url := fmt.Sprintf("%s%d/", baseURL, i) 39 | newProxies, err := s.fetchPage(url, proxyType) 40 | if err != nil { 41 | logger.Log.Error("Failed to fetch page", 42 | zap.String("url", url), 43 | zap.Error(err)) 44 | continue 45 | } 46 | logger.Log.Debug("Fetched proxies from page", 47 | zap.String("url", url), 48 | zap.Int("count", len(newProxies))) 49 | proxies = append(proxies, newProxies...) 50 | time.Sleep(1 * time.Second) 51 | } 52 | } 53 | 54 | logger.Log.Info("Finished fetching proxies", 55 | zap.String("source", s.Name()), 56 | zap.Int("total", len(proxies))) 57 | return proxies, nil 58 | } 59 | 60 | func (s *KuaidailiSource) fetchPage(url, pageType string) ([]*model.Proxy, error) { 61 | proxies := make([]*model.Proxy, 0) 62 | 63 | req, err := http.NewRequest("GET", url, nil) 64 | if err != nil { 65 | return nil, err 66 | } 67 | 68 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36") 69 | 70 | client := &http.Client{Timeout: 10 * time.Second} 71 | resp, err := client.Do(req) 72 | if err != nil { 73 | return nil, err 74 | } 75 | defer resp.Body.Close() 76 | 77 | doc, err := goquery.NewDocumentFromReader(resp.Body) 78 | if err != nil { 79 | return nil, err 80 | } 81 | 82 | doc.Find("table tbody tr").Each(func(i int, selection *goquery.Selection) { 83 | ip := strings.TrimSpace(selection.Find("td[data-title='IP']").Text()) 84 | port := strings.TrimSpace(selection.Find("td[data-title='PORT']").Text()) 85 | typeStr := strings.ToLower(strings.TrimSpace(selection.Find("td[data-title='类型']").Text())) 86 | anonymous := strings.Contains(strings.ToLower(selection.Find("td[data-title='匿名度']").Text()), "高匿") 87 | 88 | if ip != "" && port != "" { 89 | proxyType := s.parseProxyType(typeStr) 90 | if proxyType.IsValid() { 91 | proxies = append(proxies, &model.Proxy{ 92 | IP: ip, 93 | Port: port, 94 | Type: proxyType, 95 | Anonymous: anonymous, 96 | LastCheck: time.Now(), 97 | }) 98 | } 99 | } 100 | }) 101 | 102 | return proxies, nil 103 | } 104 | 105 | func (s *KuaidailiSource) parseProxyType(typeStr string) model.ProxyType { 106 | switch strings.ToLower(typeStr) { 107 | case "http": 108 | return model.ProxyTypeHTTP 109 | case "https": 110 | return model.ProxyTypeHTTPS 111 | case "socks4": 112 | return model.ProxyTypeSOCKS4 113 | case "socks5": 114 | return model.ProxyTypeSOCKS5 115 | default: 116 | return "" 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /internal/crawler/sources/openproxylist.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "net/http" 7 | "github.com/langchou/proxyPool/internal/logger" 8 | "github.com/langchou/proxyPool/internal/model" 9 | "strings" 10 | "time" 11 | 12 | "go.uber.org/zap" 13 | ) 14 | 15 | type OpenProxyListSource struct { 16 | BaseSource 17 | } 18 | 19 | func NewOpenProxyListSource() *OpenProxyListSource { 20 | return &OpenProxyListSource{ 21 | BaseSource: BaseSource{name: "openproxylist"}, 22 | } 23 | } 24 | 25 | func (s *OpenProxyListSource) Fetch() ([]*model.Proxy, error) { 26 | logger.Log.Info("Starting to fetch proxies from openproxylist") 27 | proxies := make([]*model.Proxy, 0) 28 | 29 | // 定义要爬取的URL 30 | urls := map[string]string{ 31 | "https": "https://raw.githubusercontent.com/roosterkid/openproxylist/main/HTTPS.txt", 32 | "socks5": "https://raw.githubusercontent.com/roosterkid/openproxylist/main/SOCKS5.txt", 33 | } 34 | 35 | for proxyType, url := range urls { 36 | logger.Log.Debug("Fetching proxy type", zap.String("type", proxyType)) 37 | newProxies, err := s.fetchList(url, proxyType) 38 | if err != nil { 39 | logger.Log.Error("Failed to fetch list", 40 | zap.String("url", url), 41 | zap.Error(err)) 42 | continue 43 | } 44 | proxies = append(proxies, newProxies...) 45 | // 避免请求过快 46 | time.Sleep(2 * time.Second) 47 | } 48 | 49 | logger.Log.Info("Finished fetching proxies", 50 | zap.String("source", s.Name()), 51 | zap.Int("total", len(proxies))) 52 | return proxies, nil 53 | } 54 | 55 | func (s *OpenProxyListSource) fetchList(url, proxyType string) ([]*model.Proxy, error) { 56 | proxies := make([]*model.Proxy, 0) 57 | 58 | req, err := http.NewRequest("GET", url, nil) 59 | if err != nil { 60 | return nil, err 61 | } 62 | 63 | client := &http.Client{Timeout: 10 * time.Second} 64 | resp, err := client.Do(req) 65 | if err != nil { 66 | return nil, err 67 | } 68 | defer resp.Body.Close() 69 | 70 | scanner := bufio.NewScanner(resp.Body) 71 | for scanner.Scan() { 72 | line := scanner.Text() 73 | // 跳过注释和空行 74 | if strings.HasPrefix(line, "#") || strings.TrimSpace(line) == "" || 75 | strings.HasPrefix(line, "Support") || strings.HasPrefix(line, "BTC") || 76 | strings.HasPrefix(line, "ETH") || strings.HasPrefix(line, "LTC") || 77 | strings.HasPrefix(line, "Doge") || strings.HasPrefix(line, "Format") || 78 | strings.HasPrefix(line, "Website") || !strings.Contains(line, "]") { 79 | continue 80 | } 81 | 82 | // 解析代理信息 83 | // 格式: 🇨🇦 67.43.228.250:14395 370ms CA [GloboTech Communications] 84 | parts := strings.Fields(line) 85 | if len(parts) < 3 { 86 | continue 87 | } 88 | 89 | // 获取IP:PORT部分 90 | ipPort := strings.Split(parts[1], ":") 91 | if len(ipPort) != 2 { 92 | continue 93 | } 94 | 95 | // 解析响应时间 96 | speedStr := strings.TrimSuffix(parts[2], "ms") 97 | var speed int64 98 | if _, err := fmt.Sscanf(speedStr, "%d", &speed); err != nil { 99 | speed = 0 100 | } 101 | 102 | proxy := &model.Proxy{ 103 | IP: ipPort[0], 104 | Port: ipPort[1], 105 | Type: s.getProxyType(proxyType), 106 | Speed: speed, 107 | Anonymous: true, // 这些代理通常都是高匿的 108 | LastCheck: time.Now(), 109 | } 110 | proxies = append(proxies, proxy) 111 | } 112 | 113 | logger.Log.Debug("Fetched proxies from list", 114 | zap.String("url", url), 115 | zap.Int("count", len(proxies))) 116 | 117 | return proxies, nil 118 | } 119 | 120 | func (s *OpenProxyListSource) getProxyType(typeStr string) model.ProxyType { 121 | switch strings.ToLower(typeStr) { 122 | case "https": 123 | return model.ProxyTypeHTTPS 124 | case "socks5": 125 | return model.ProxyTypeSOCKS5 126 | default: 127 | return model.ProxyTypeHTTP 128 | } 129 | } 130 | -------------------------------------------------------------------------------- /internal/logger/logger.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "time" 8 | 9 | "go.uber.org/zap" 10 | "go.uber.org/zap/zapcore" 11 | ) 12 | 13 | var ( 14 | Log *zap.Logger 15 | ) 16 | 17 | func Init(level, output, logPath string) error { 18 | // 创建日志目录 19 | if output == "file" { 20 | dir := filepath.Dir(logPath) 21 | if err := os.MkdirAll(dir, 0755); err != nil { 22 | return fmt.Errorf("create log directory failed: %w", err) 23 | } 24 | } 25 | 26 | // 解析日志级别 27 | var zapLevel zapcore.Level 28 | switch level { 29 | case "debug": 30 | zapLevel = zapcore.DebugLevel 31 | case "info": 32 | zapLevel = zapcore.InfoLevel 33 | case "warn": 34 | zapLevel = zapcore.WarnLevel 35 | case "error": 36 | zapLevel = zapcore.ErrorLevel 37 | default: 38 | zapLevel = zapcore.InfoLevel 39 | } 40 | 41 | // 配置编码器 42 | encoderConfig := zapcore.EncoderConfig{ 43 | TimeKey: "time", 44 | LevelKey: "level", 45 | NameKey: "logger", 46 | CallerKey: "caller", 47 | FunctionKey: zapcore.OmitKey, 48 | MessageKey: "msg", 49 | StacktraceKey: "stacktrace", 50 | LineEnding: zapcore.DefaultLineEnding, 51 | EncodeLevel: zapcore.CapitalLevelEncoder, 52 | EncodeTime: zapcore.ISO8601TimeEncoder, 53 | EncodeDuration: zapcore.SecondsDurationEncoder, 54 | EncodeCaller: zapcore.ShortCallerEncoder, 55 | } 56 | 57 | // 创建Core 58 | var core zapcore.Core 59 | if output == "file" { 60 | // 创建轮转日志文件 61 | filename := fmt.Sprintf("%s.%s.log", 62 | filepath.Join(filepath.Dir(logPath), filepath.Base(logPath)), 63 | time.Now().Format("2006-01-02")) 64 | 65 | // 打开日志文件 66 | f, err := os.OpenFile(filename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) 67 | if err != nil { 68 | return fmt.Errorf("open log file failed: %w", err) 69 | } 70 | 71 | // 只输出到文件，移除控制台输出 72 | fileEncoder := zapcore.NewJSONEncoder(encoderConfig) 73 | core = zapcore.NewCore(fileEncoder, zapcore.AddSync(f), zapLevel) 74 | } else { 75 | // 如果不是文件输出，则使用控制台 76 | consoleEncoder := zapcore.NewConsoleEncoder(encoderConfig) 77 | core = zapcore.NewCore(consoleEncoder, zapcore.AddSync(os.Stdout), zapLevel) 78 | } 79 | 80 | // 创建logger 81 | Log = zap.New(core, 82 | zap.AddCaller(), 83 | zap.AddStacktrace(zapcore.ErrorLevel), 84 | zap.AddCallerSkip(1), 85 | ) 86 | 87 | return nil 88 | } 89 | 90 | // 添加一个初始化检查函数 91 | func init() { 92 | // 默认初始化为 info 级别，输出到控制台 93 | if Log == nil { 94 | if err := Init("info", "console", ""); err != nil { 95 | panic(err) 96 | } 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /internal/middleware/auth.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "github.com/langchou/proxyPool/internal/api/response" 5 | "github.com/langchou/proxyPool/internal/config" 6 | "github.com/langchou/proxyPool/internal/logger" 7 | 8 | "github.com/gin-gonic/gin" 9 | "go.uber.org/zap" 10 | ) 11 | 12 | // BasicAuth 基本认证中间件 13 | func BasicAuth() gin.HandlerFunc { 14 | return func(c *gin.Context) { 15 | if !config.GlobalConfig.Security.AuthEnabled { 16 | c.Next() 17 | return 18 | } 19 | 20 | username, password, ok := c.Request.BasicAuth() 21 | if !ok || username != config.GlobalConfig.Security.Username || 22 | password != config.GlobalConfig.Security.Password { 23 | logger.Log.Warn("Invalid basic auth attempt", 24 | zap.String("ip", c.ClientIP()), 25 | zap.String("username", username)) 26 | 27 | c.Header("WWW-Authenticate", "Basic realm=Authorization Required") 28 | response.Error(c, "Unauthorized") 29 | c.Abort() 30 | return 31 | } 32 | c.Next() 33 | } 34 | } 35 | 36 | // APIKeyAuth API Key 认证中间件 37 | func APIKeyAuth() gin.HandlerFunc { 38 | return func(c *gin.Context) { 39 | if !config.GlobalConfig.Security.APIKeyEnabled { 40 | c.Next() 41 | return 42 | } 43 | 44 | apiKey := c.GetHeader("X-API-Key") 45 | valid := false 46 | for _, key := range config.GlobalConfig.Security.APIKeys { 47 | if apiKey == key { 48 | valid = true 49 | break 50 | } 51 | } 52 | 53 | if !valid { 54 | logger.Log.Warn("Invalid API key attempt", 55 | zap.String("ip", c.ClientIP()), 56 | zap.String("api_key", apiKey)) 57 | 58 | response.Error(c, "Invalid API key") 59 | c.Abort() 60 | return 61 | } 62 | c.Next() 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /internal/middleware/error.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "net/http" 5 | "github.com/langchou/proxyPool/internal/api/response" 6 | "github.com/langchou/proxyPool/internal/logger" 7 | 8 | "github.com/gin-gonic/gin" 9 | "go.uber.org/zap" 10 | ) 11 | 12 | // ErrorHandler 处理所有未捕获的路由和错误 13 | func ErrorHandler() gin.HandlerFunc { 14 | return func(c *gin.Context) { 15 | c.Next() // 处理请求 16 | 17 | // 如果没有路由匹配，返回 404 18 | if c.Writer.Status() == http.StatusNotFound { 19 | logger.Log.Warn("Route not found", 20 | zap.String("path", c.Request.URL.Path), 21 | zap.String("method", c.Request.Method)) 22 | 23 | c.JSON(http.StatusNotFound, response.Response{ 24 | Code: response.CodeNotFound, 25 | Message: "Route not found", 26 | }) 27 | return 28 | } 29 | 30 | // 处理其他错误状态码 31 | if c.Writer.Status() >= http.StatusBadRequest { 32 | logger.Log.Error("Request error", 33 | zap.String("path", c.Request.URL.Path), 34 | zap.Int("status", c.Writer.Status())) 35 | 36 | if !c.Writer.Written() { 37 | c.JSON(c.Writer.Status(), response.Response{ 38 | Code: c.Writer.Status(), 39 | Message: http.StatusText(c.Writer.Status()), 40 | }) 41 | } 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /internal/middleware/logger.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "github.com/langchou/proxyPool/internal/logger" 5 | "time" 6 | 7 | "github.com/gin-gonic/gin" 8 | "go.uber.org/zap" 9 | ) 10 | 11 | func Logger() gin.HandlerFunc { 12 | return func(c *gin.Context) { 13 | start := time.Now() 14 | path := c.Request.URL.Path 15 | query := c.Request.URL.RawQuery 16 | 17 | logger.Log.Info("Incoming request", 18 | zap.String("path", path), 19 | zap.String("query", query), 20 | zap.String("ip", c.ClientIP()), 21 | zap.String("method", c.Request.Method)) 22 | 23 | // 处理请求 24 | c.Next() 25 | 26 | // 请求处理完成后记录 27 | latency := time.Since(start) 28 | logger.Log.Info("Request completed", 29 | zap.String("path", path), 30 | zap.Int("status", c.Writer.Status()), 31 | zap.Duration("latency", latency)) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /internal/middleware/ratelimit.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "strconv" 8 | "time" 9 | 10 | "github.com/gin-gonic/gin" 11 | "github.com/langchou/proxyPool/internal/api/response" 12 | "github.com/langchou/proxyPool/internal/logger" 13 | "github.com/redis/go-redis/v9" 14 | "go.uber.org/zap" 15 | ) 16 | 17 | type RateLimiter struct { 18 | redisClient *redis.Client 19 | maxRequests int // 最大请求次数 20 | duration time.Duration // 时间窗口 21 | banDuration time.Duration // 封禁时长 22 | } 23 | 24 | func NewRateLimiter(redisClient *redis.Client, maxRequests int, duration, banDuration time.Duration) *RateLimiter { 25 | return &RateLimiter{ 26 | redisClient: redisClient, 27 | maxRequests: maxRequests, 28 | duration: duration, 29 | banDuration: banDuration, 30 | } 31 | } 32 | 33 | func (rl *RateLimiter) RateLimit() gin.HandlerFunc { 34 | return func(c *gin.Context) { 35 | ip := c.ClientIP() 36 | 37 | // 检查是否被封禁 38 | banKey := fmt.Sprintf("ban:%s", ip) 39 | if banned, _ := rl.redisClient.Get(c.Request.Context(), banKey).Bool(); banned { 40 | logger.Log.Warn("Banned IP attempted access", 41 | zap.String("ip", ip), 42 | zap.String("path", c.Request.URL.Path)) 43 | 44 | c.JSON(http.StatusForbidden, response.Response{ 45 | Code: 403, 46 | Message: "Your IP has been banned due to excessive requests", 47 | }) 48 | c.Abort() 49 | return 50 | } 51 | 52 | // 访问计数key 53 | key := fmt.Sprintf("ratelimit:%s", ip) 54 | 55 | // 使用 Redis 的 MULTI 命令保证原子性 56 | pipe := rl.redisClient.Pipeline() 57 | incr := pipe.Incr(c.Request.Context(), key) 58 | pipe.Expire(c.Request.Context(), key, rl.duration) 59 | _, err := pipe.Exec(c.Request.Context()) 60 | 61 | if err != nil { 62 | logger.Log.Error("Redis pipeline failed", zap.Error(err)) 63 | c.Next() 64 | return 65 | } 66 | 67 | count := incr.Val() 68 | 69 | // 检查是否超过限制 70 | if count > int64(rl.maxRequests) { 71 | // 封禁IP 72 | rl.banIP(c.Request.Context(), ip) 73 | 74 | logger.Log.Warn("IP banned due to rate limit exceeded", 75 | zap.String("ip", ip), 76 | zap.Int64("request_count", count)) 77 | 78 | c.JSON(http.StatusTooManyRequests, response.Response{ 79 | Code: 429, 80 | Message: fmt.Sprintf("Rate limit exceeded. Maximum %d requests per %v", rl.maxRequests, rl.duration), 81 | }) 82 | c.Abort() 83 | return 84 | } 85 | 86 | // 设置剩余请求次数的header 87 | c.Header("X-RateLimit-Remaining", strconv.FormatInt(int64(rl.maxRequests)-count, 10)) 88 | c.Header("X-RateLimit-Limit", strconv.Itoa(rl.maxRequests)) 89 | c.Header("X-RateLimit-Reset", strconv.FormatInt(time.Now().Add(rl.duration).Unix(), 10)) 90 | 91 | c.Next() 92 | } 93 | } 94 | 95 | func (rl *RateLimiter) banIP(ctx context.Context, ip string) { 96 | banKey := fmt.Sprintf("ban:%s", ip) 97 | rl.redisClient.Set(ctx, banKey, true, rl.banDuration) 98 | } 99 | 100 | // 解封IP的方法（可用于管理API） 101 | func (rl *RateLimiter) UnbanIP(ctx context.Context, ip string) error { 102 | banKey := fmt.Sprintf("ban:%s", ip) 103 | return rl.redisClient.Del(ctx, banKey).Err() 104 | } 105 | -------------------------------------------------------------------------------- /internal/model/proxy.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import "time" 4 | 5 | // ProxyType 代理类型 6 | type ProxyType string 7 | 8 | const ( 9 | ProxyTypeHTTP ProxyType = "http" 10 | ProxyTypeHTTPS ProxyType = "https" 11 | ProxyTypeSOCKS4 ProxyType = "socks4" 12 | ProxyTypeSOCKS5 ProxyType = "socks5" 13 | ) 14 | 15 | type Proxy struct { 16 | IP string `json:"ip"` 17 | Port string `json:"port"` 18 | Type ProxyType `json:"type"` // 代理类型 19 | Anonymous bool `json:"anonymous"` // 是否高匿 20 | Speed int64 `json:"speed"` // 响应速度（毫秒） 21 | Score int `json:"score"` // 可用性评分 22 | LastCheck time.Time `json:"last_check"` 23 | } 24 | 25 | type ProxyList []*Proxy 26 | 27 | // IsValid 检查代理类型是否有效 28 | func (t ProxyType) IsValid() bool { 29 | switch t { 30 | case ProxyTypeHTTP, ProxyTypeHTTPS, ProxyTypeSOCKS4, ProxyTypeSOCKS5: 31 | return true 32 | default: 33 | return false 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /internal/storage/redis.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "math/rand" 7 | "time" 8 | 9 | "github.com/langchou/proxyPool/internal/logger" 10 | "github.com/langchou/proxyPool/internal/model" 11 | 12 | "github.com/redis/go-redis/v9" 13 | "go.uber.org/zap" 14 | ) 15 | 16 | type Storage interface { 17 | Save(context.Context, *model.Proxy) error 18 | GetAll(context.Context) ([]*model.Proxy, error) 19 | GetRandom(context.Context) (*model.Proxy, error) 20 | Remove(context.Context, string) error 21 | UpdateScore(context.Context, string, int) error 22 | } 23 | 24 | type RedisStorage struct { 25 | client *redis.Client 26 | } 27 | 28 | func NewRedisStorage(addr, password string, db int) *RedisStorage { 29 | client := redis.NewClient(&redis.Options{ 30 | Addr: addr, 31 | Password: password, 32 | DB: db, 33 | }) 34 | 35 | return &RedisStorage{client: client} 36 | } 37 | 38 | // 实现 Storage 接口的方法... 39 | 40 | func (s *RedisStorage) Save(ctx context.Context, proxy *model.Proxy) error { 41 | key := "proxy:" + proxy.IP + ":" + proxy.Port 42 | logger.Log.Debug("Saving proxy to Redis", zap.String("key", key)) 43 | 44 | // 将代理对象序列化为 JSON 45 | data, err := json.Marshal(proxy) 46 | if err != nil { 47 | logger.Log.Error("Failed to marshal proxy", zap.Error(err)) 48 | return err 49 | } 50 | 51 | // 使用 SET 命令而不是 HSET 52 | err = s.client.Set(ctx, key, data, 24*time.Hour).Err() // 设置 24 小时过期 53 | if err != nil { 54 | logger.Log.Error("Failed to save proxy", zap.String("key", key), zap.Error(err)) 55 | } 56 | return err 57 | } 58 | 59 | func (s *RedisStorage) GetAll(ctx context.Context) ([]*model.Proxy, error) { 60 | keys, err := s.client.Keys(ctx, "proxy:*").Result() 61 | if err != nil { 62 | return nil, err 63 | } 64 | 65 | proxies := make([]*model.Proxy, 0, len(keys)) 66 | for _, key := range keys { 67 | // 使用 GET 命令获取 JSON 数据 68 | data, err := s.client.Get(ctx, key).Result() 69 | if err != nil { 70 | if err != redis.Nil { 71 | logger.Log.Error("Failed to get proxy", zap.String("key", key), zap.Error(err)) 72 | } 73 | continue 74 | } 75 | 76 | var proxy model.Proxy 77 | if err := json.Unmarshal([]byte(data), &proxy); err != nil { 78 | logger.Log.Error("Failed to unmarshal proxy", zap.String("key", key), zap.Error(err)) 79 | continue 80 | } 81 | 82 | proxies = append(proxies, &proxy) 83 | } 84 | 85 | return proxies, nil 86 | } 87 | 88 | func (s *RedisStorage) GetRandom(ctx context.Context) (*model.Proxy, error) { 89 | keys, err := s.client.Keys(ctx, "proxy:*").Result() 90 | if err != nil { 91 | return nil, err 92 | } 93 | 94 | if len(keys) == 0 { 95 | return nil, redis.Nil 96 | } 97 | 98 | // 随机选择一个代理 99 | key := keys[rand.Intn(len(keys))] 100 | 101 | // 使用 GET 命令获取 JSON 数据 102 | data, err := s.client.Get(ctx, key).Result() 103 | if err != nil { 104 | logger.Log.Error("Failed to get random proxy", zap.String("key", key), zap.Error(err)) 105 | return nil, err 106 | } 107 | 108 | var proxy model.Proxy 109 | if err := json.Unmarshal([]byte(data), &proxy); err != nil { 110 | logger.Log.Error("Failed to unmarshal proxy", zap.String("key", key), zap.Error(err)) 111 | return nil, err 112 | } 113 | 114 | return &proxy, nil 115 | } 116 | 117 | func (s *RedisStorage) Remove(ctx context.Context, key string) error { 118 | return s.client.Del(ctx, "proxy:"+key).Err() 119 | } 120 | 121 | func (s *RedisStorage) UpdateScore(ctx context.Context, key string, score int) error { 122 | fullKey := "proxy:" + key 123 | 124 | // 先获取现有数据 125 | data, err := s.client.Get(ctx, fullKey).Result() 126 | if err != nil { 127 | return err 128 | } 129 | 130 | var proxy model.Proxy 131 | if err := json.Unmarshal([]byte(data), &proxy); err != nil { 132 | return err 133 | } 134 | 135 | // 更新分数 136 | proxy.Score = score 137 | 138 | // 重新保存 139 | return s.Save(ctx, &proxy) 140 | } 141 | 142 | // 在 RedisStorage 结构体中添加获取客户端的方法 143 | func (s *RedisStorage) GetRedisClient() *redis.Client { 144 | return s.client 145 | } 146 | -------------------------------------------------------------------------------- /internal/validator/validator.go: -------------------------------------------------------------------------------- 1 | package validator 2 | 3 | import ( 4 | "crypto/tls" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | "net/url" 10 | "time" 11 | 12 | "github.com/langchou/proxyPool/internal/logger" 13 | "github.com/langchou/proxyPool/internal/model" 14 | 15 | "go.uber.org/zap" 16 | "golang.org/x/net/proxy" 17 | ) 18 | 19 | type Validator struct { 20 | timeout time.Duration 21 | } 22 | 23 | // IPInfo 响应结构 24 | type IPInfo struct { 25 | IP string `json:"ip"` 26 | Hostname string `json:"hostname"` 27 | City string `json:"city"` 28 | Region string `json:"region"` 29 | Country string `json:"country"` 30 | Org string `json:"org"` 31 | } 32 | 33 | func NewValidator(timeout time.Duration) *Validator { 34 | return &Validator{timeout: timeout} 35 | } 36 | 37 | func (v *Validator) Validate(p *model.Proxy) (bool, int64) { 38 | logger.Log.Debug("Validating proxy", 39 | zap.String("ip", p.IP), 40 | zap.String("port", p.Port), 41 | zap.String("type", string(p.Type))) 42 | 43 | var client *http.Client 44 | var err error 45 | 46 | switch p.Type { 47 | case model.ProxyTypeHTTP, model.ProxyTypeHTTPS: 48 | client, err = v.createHTTPClient(p) 49 | case model.ProxyTypeSOCKS4, model.ProxyTypeSOCKS5: 50 | client, err = v.createSocksClient(p) 51 | default: 52 | logger.Log.Warn("Invalid proxy type", zap.String("type", string(p.Type))) 53 | return false, 0 54 | } 55 | 56 | if err != nil { 57 | logger.Log.Error("Failed to create HTTP client", zap.Error(err)) 58 | return false, 0 59 | } 60 | 61 | start := time.Now() 62 | 63 | resp, err := client.Get("http://ipinfo.io/json") 64 | if err != nil { 65 | logger.Log.Debug("Proxy validation failed", 66 | zap.String("ip", p.IP), 67 | zap.Error(err)) 68 | return false, 0 69 | } 70 | defer resp.Body.Close() 71 | 72 | speed := time.Since(start).Milliseconds() 73 | logger.Log.Debug("Proxy response time", 74 | zap.String("ip", p.IP), 75 | zap.Int64("speed_ms", speed)) 76 | 77 | if resp.StatusCode != http.StatusOK { 78 | logger.Log.Debug("Proxy returned non-200 status", 79 | zap.String("ip", p.IP), 80 | zap.Int("status", resp.StatusCode)) 81 | return false, speed 82 | } 83 | 84 | // 读取并解析响应 85 | body, err := io.ReadAll(resp.Body) 86 | if err != nil { 87 | return false, speed 88 | } 89 | 90 | var ipInfo IPInfo 91 | if err := json.Unmarshal(body, &ipInfo); err != nil { 92 | return false, speed 93 | } 94 | 95 | // 验证返回的IP是否与代理IP匹配 96 | // 注意：某些代理可能会返回不同的IP（级联代理），所以这里只验证是否成功获取到了IP信息 97 | return ipInfo.IP != "", speed 98 | } 99 | 100 | func (v *Validator) createHTTPClient(p *model.Proxy) (*http.Client, error) { 101 | proxyURL := fmt.Sprintf("%s://%s:%s", p.Type, p.IP, p.Port) 102 | parsedURL, err := url.Parse(proxyURL) 103 | if err != nil { 104 | return nil, err 105 | } 106 | 107 | // 创建跳过证书验证的 Transport 108 | transport := &http.Transport{ 109 | Proxy: http.ProxyURL(parsedURL), 110 | TLSClientConfig: &tls.Config{ 111 | InsecureSkipVerify: true, // 跳过证书验证 112 | }, 113 | } 114 | 115 | return &http.Client{ 116 | Transport: transport, 117 | Timeout: v.timeout, 118 | }, nil 119 | } 120 | 121 | func (v *Validator) createSocksClient(p *model.Proxy) (*http.Client, error) { 122 | dialer, err := proxy.SOCKS5("tcp", p.IP+":"+p.Port, nil, proxy.Direct) 123 | if err != nil { 124 | return nil, err 125 | } 126 | 127 | // 创建跳过证书验证的 Transport 128 | transport := &http.Transport{ 129 | Dial: dialer.Dial, 130 | TLSClientConfig: &tls.Config{ 131 | InsecureSkipVerify: true, // 跳过证书验证 132 | }, 133 | } 134 | 135 | return &http.Client{ 136 | Transport: transport, 137 | Timeout: v.timeout, 138 | }, nil 139 | } 140 | --------------------------------------------------------------------------------