├── .gitignore ├── .travis.yml ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── assets └── proxy-pool.png ├── config └── config.go ├── configs └── default.yml ├── controller ├── asset.go ├── common.go └── proxy.go ├── crawler ├── crawler.go ├── crawler_test.go ├── ip66.go ├── ip66_test.go ├── kuai.go ├── kuai_test.go ├── proxylist.go ├── proxylist_test.go ├── xici.go └── xici_test.go ├── doc ├── config.md └── crawler.md ├── go.mod ├── go.sum ├── log └── log.go ├── main.go ├── router └── router.go ├── runner.conf ├── script └── entrypoint.sh ├── service └── proxy.go └── web ├── .gitignore ├── package.json ├── public ├── favicon.ico ├── index.html ├── manifest.json └── robots.txt ├── src ├── App.js ├── App.sass ├── App.test.js ├── index.js ├── index.sass ├── request-interceptors.js ├── serviceWorker.js └── setupTests.js └── yarn.lock /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | # vendor/ 16 | 17 | tmp 18 | proxypool 19 | 20 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 21 | 22 | # dependencies 23 | node_modules 24 | .pnp 25 | .pnp.js 26 | 27 | # testing 28 | coverage 29 | 30 | # production 31 | build 32 | 33 | # misc 34 | .DS_Store 35 | .env.local 36 | .env.development.local 37 | .env.test.local 38 | .env.production.local 39 | 40 | npm-debug.log* 41 | yarn-debug.log* 42 | yarn-error.log* 43 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | sudo: required 3 | 4 | go: 5 | - "1.14" 6 | 7 | services: 8 | - docker 9 | 10 | script: 11 | - make test 12 | 13 | after_success: 14 | - docker build -t proxypool . 15 | - docker tag proxypool $DOCKER_USERNAME/proxypool:latest 16 | - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin 17 | - docker push $DOCKER_USERNAME/proxypool:latest 18 | - docker images 19 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:12-alpine as webbuilder 2 | ADD . /proxy-pool 3 | RUN cd /proxy-pool/web \ 4 | && yarn \ 5 | && yarn build \ 6 | && rm -rf node_module 7 | 8 | FROM golang:1.14-alpine as builder 9 | 10 | COPY --from=webbuilder /proxy-pool /proxy-pool 11 | 12 | RUN apk update \ 13 | && apk add git make \ 14 | && go get -u github.com/gobuffalo/packr/v2/packr2 \ 15 | && cd /proxy-pool \ 16 | && make build 17 | 18 | FROM alpine 19 | 20 | EXPOSE 4000 21 | 22 | RUN addgroup -g 1000 go \ 23 | && adduser -u 1000 -G go -s /bin/sh -D go \ 24 | && apk add --no-cache ca-certificates 25 | 26 | COPY --from=builder /proxy-pool/proxypool /usr/local/bin/proxypool 27 | COPY --from=builder /proxy-pool/script/entrypoint.sh /entrypoint.sh 28 | 29 | USER go 30 | 31 | WORKDIR /home/go 32 | 33 | HEALTHCHECK --timeout=10s CMD [ "wget", "http://127.0.0.1:4000/ping", "-q", "-O", "-"] 34 | 35 | ENTRYPOINT ["/entrypoint.sh"] 36 | CMD ["proxypool"] 37 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | export GO111MODULE = on 2 | 3 | .PHONY: default test test-cover dev 4 | 5 | # for dev 6 | dev: 7 | fresh 8 | 9 | # for test 10 | test: 11 | go test -race -cover ./... 12 | 13 | test-cover: 14 | go test -race -coverprofile=test.out ./... && go tool cover --html=test.out 15 | 16 | build: 17 | packr2 18 | go build -o proxypool 19 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # proxy-pool 2 | 3 | 虽然网上各类网站提供了一堆的免费代理地址，但是其可用性比较差，更新不及时，过多不可用的地址，以及延时较大等问题都干扰实际使用的效果。对于代理地址，期望是越多越好，但是对于代理质量有着更高的要求，宁缺勿滥，因此`proxy-pool`不再将抓取到的代理地址保存至数据库，而调整为定期从免费代理网站下抓取代理地址，使用该地址去测试其可用性（默认配置为访问baidu），测试可用则添加至可用代理地址列表中，如此循环一直抓取新的地址，一直校验。对于已校验可用的代理地址，也定期重新校验是否可用，默认校验间隔为30分钟。 4 | 5 | 注意：网页部分有增加百度统计，部署时可先删除。 6 | 7 |

8 | 9 |

10 | 11 | ## 常用配置 12 | 13 | 对于有特别需求，可以调整默认的配置，主要的配置如下： 14 | 15 | 抓取代理网站列表配置（暂时只实现了三个网站的抓取）： 16 | 17 | ```yml 18 | crawler: 19 | - xici 20 | - ip66 21 | - kuai 22 | ``` 23 | 24 | 由于各网站对访问IP频率限制的不同，可根据实际使用中调整各网站的抓取间隔，如设置`xici`的抓取延时为10分钟（如果不配置则为默认值2分钟）： 25 | 26 | ```yml 27 | xici: 28 | interval: 10m 29 | ``` 30 | 31 | 默认的检测方式是通过代理地址去访问`baidu`，可根据应用场景调整相应的配置： 32 | 33 | ```yml 34 | detect: 35 | # 检测时间（定时对现可用的代理地址重新检测） 36 | interval: 30m 37 | # 检测地址 38 | url: https://www.baidu.com/ 39 | # 检测超时 40 | timeout: 3s 41 | # 最大次数 42 | maxTimes: 3 43 | ``` 44 | 45 | ## 程序设计 46 | 47 | - [config](./doc/config.md) 48 | - [crawler](./doc/crawler.md) 49 | -------------------------------------------------------------------------------- /assets/proxy-pool.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vicanso/proxy-pool/fa848accebac33a2fc102f173bdd7a9bdbd9a395/assets/proxy-pool.png -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 tree xie 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package config 16 | 17 | import ( 18 | "bytes" 19 | "os" 20 | "time" 21 | 22 | "github.com/gobuffalo/packr/v2" 23 | "github.com/spf13/viper" 24 | ) 25 | 26 | var ( 27 | box = packr.New("config", "../configs") 28 | env = os.Getenv("GO_ENV") 29 | ) 30 | 31 | const ( 32 | // Dev development env 33 | Dev = "dev" 34 | // Test test env 35 | Test = "test" 36 | // Production production env 37 | Production = "production" 38 | ) 39 | 40 | type ( 41 | Crawler struct { 42 | Name string 43 | Interval time.Duration 44 | MaxPage int 45 | } 46 | // Detect detect config 47 | Detect struct { 48 | URL string 49 | Interval time.Duration 50 | Timeout time.Duration 51 | MaxTimes int 52 | } 53 | ) 54 | 55 | func init() { 56 | configType := "yml" 57 | configExt := "." + configType 58 | data, err := box.Find("default" + configExt) 59 | if err != nil { 60 | panic(err) 61 | } 62 | viper.SetConfigType(configType) 63 | v := viper.New() 64 | v.SetConfigType(configType) 65 | // 读取默认配置中的所有配置 66 | err = v.ReadConfig(bytes.NewReader(data)) 67 | if err != nil { 68 | panic(err) 69 | } 70 | configs := v.AllSettings() 71 | // 将default中的配置全部以默认配置写入 72 | for k, v := range configs { 73 | viper.SetDefault(k, v) 74 | } 75 | 76 | // 根据当前运行环境配置读取 77 | // 可根据不同的环境仅调整与default不一致的相关配置 78 | if env != "" { 79 | envConfigFile := env + configExt 80 | data, err = box.Find(envConfigFile) 81 | if err != nil { 82 | panic(err) 83 | } 84 | // 读取当前运行环境对应的配置 85 | err = viper.ReadConfig(bytes.NewReader(data)) 86 | if err != nil { 87 | panic(err) 88 | } 89 | } 90 | } 91 | 92 | // GetCrawlers get crawlers config 93 | func GetCrawlers() []*Crawler { 94 | crawlers := make([]*Crawler, 0) 95 | data := viper.GetStringSlice("crawler") 96 | for _, name := range data { 97 | interval := viper.GetDuration(name + ".interval") 98 | maxPage := viper.GetInt(name + ".maxPage") 99 | // 如果未配置抓取间隔时间，则设置为2分钟 100 | if interval == 0 { 101 | interval = 2 * time.Minute 102 | } 103 | crawlers = append(crawlers, &Crawler{ 104 | Name: name, 105 | Interval: interval, 106 | MaxPage: maxPage, 107 | }) 108 | } 109 | return crawlers 110 | } 111 | 112 | // GetDetect get detect config 113 | func GetDetect() *Detect { 114 | prefix := "detect." 115 | conf := &Detect{ 116 | Timeout: viper.GetDuration(prefix + "timeout"), 117 | URL: viper.GetString(prefix + "url"), 118 | Interval: viper.GetDuration(prefix + "interval"), 119 | MaxTimes: viper.GetInt(prefix + "maxTimes"), 120 | } 121 | if conf.Timeout == 0 { 122 | conf.Timeout = 3 * time.Second 123 | } 124 | if conf.Interval == 0 { 125 | conf.Interval = 30 * time.Minute 126 | } 127 | if conf.URL == "" { 128 | conf.URL = "https://www.baidu.com/" 129 | } 130 | if conf.MaxTimes <= 0 { 131 | conf.MaxTimes = 3 132 | } 133 | return conf 134 | } 135 | 136 | // GetListenAddr get listen address 137 | func GetListenAddr() string { 138 | addr := viper.GetString("listen") 139 | if addr == "" { 140 | return ":4000" 141 | } 142 | return addr 143 | } 144 | -------------------------------------------------------------------------------- /configs/default.yml: -------------------------------------------------------------------------------- 1 | # 监听地址 2 | listen: :4000 3 | # 抓取的代理网站列表 4 | crawler: 5 | - xici 6 | - ip66 7 | - kuai 8 | # 按照需要可配置不同的代理网站的抓取频率 9 | xici: 10 | interval: 10m 11 | maxPage: 100 12 | ip66: 13 | maxPage: 200 14 | kuai: 15 | maxPage: 200 16 | # 检测代理是否可用的配置 17 | detect: 18 | # 检测时间（定时对现可用的代理地址重新检测） 19 | interval: 30m 20 | # 检测地址 21 | url: https://www.baidu.com/ 22 | # 检测超时 23 | timeout: 3s 24 | # 最大次数 25 | maxTimes: 3 -------------------------------------------------------------------------------- /controller/asset.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 tree xie 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package controller 16 | 17 | import ( 18 | "bytes" 19 | "io" 20 | "os" 21 | 22 | "github.com/gobuffalo/packr/v2" 23 | "github.com/vicanso/elton" 24 | "github.com/vicanso/elton/middleware" 25 | "github.com/vicanso/proxy-pool/router" 26 | ) 27 | 28 | type ( 29 | // assetCtrl asset ctrl 30 | assetCtrl struct { 31 | } 32 | staticFile struct { 33 | box *packr.Box 34 | } 35 | ) 36 | 37 | var ( 38 | box = packr.New("asset", "../web/build") 39 | ) 40 | 41 | func (sf *staticFile) Exists(file string) bool { 42 | return sf.box.Has(file) 43 | } 44 | func (sf *staticFile) Get(file string) ([]byte, error) { 45 | return sf.box.Find(file) 46 | } 47 | func (sf *staticFile) Stat(file string) os.FileInfo { 48 | return nil 49 | } 50 | func (sf *staticFile) NewReader(file string) (io.Reader, error) { 51 | buf, err := sf.Get(file) 52 | if err != nil { 53 | return nil, err 54 | } 55 | return bytes.NewReader(buf), nil 56 | } 57 | 58 | func init() { 59 | g := router.NewGroup("") 60 | ctrl := assetCtrl{} 61 | g.GET("/", ctrl.index) 62 | g.GET("/favicon.ico", ctrl.favIcon) 63 | 64 | sf := &staticFile{ 65 | box: box, 66 | } 67 | g.GET("/static/*", middleware.NewStaticServe(sf, middleware.StaticServeConfig{ 68 | Path: "/static", 69 | // 客户端缓存一年 70 | MaxAge: 365 * 24 * 3600, 71 | // 缓存服务器缓存一个小时 72 | SMaxAge: 60 * 60, 73 | DenyQueryString: true, 74 | DisableLastModified: true, 75 | })) 76 | } 77 | 78 | func sendFile(c *elton.Context, file string) (err error) { 79 | buf, err := box.Find(file) 80 | if err != nil { 81 | return 82 | } 83 | // 根据文件后续设置类型 84 | c.SetContentTypeByExt(file) 85 | c.BodyBuffer = bytes.NewBuffer(buf) 86 | return 87 | } 88 | 89 | func (ctrl assetCtrl) index(c *elton.Context) (err error) { 90 | c.CacheMaxAge("10s") 91 | return sendFile(c, "index.html") 92 | } 93 | 94 | func (ctrl assetCtrl) favIcon(c *elton.Context) (err error) { 95 | c.SetHeader(elton.HeaderAcceptEncoding, "public, max-age=3600, s-maxage=600") 96 | return sendFile(c, "favicon.ico") 97 | } 98 | -------------------------------------------------------------------------------- /controller/common.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 tree xie 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package controller 16 | 17 | import ( 18 | "bytes" 19 | 20 | "github.com/vicanso/elton" 21 | "github.com/vicanso/proxy-pool/router" 22 | ) 23 | 24 | type ( 25 | commonCtrl struct{} 26 | ) 27 | 28 | func init() { 29 | ctrl := commonCtrl{} 30 | g := router.NewGroup("") 31 | 32 | g.GET("/ping", ctrl.ping) 33 | 34 | } 35 | 36 | func (commonCtrl) ping(c *elton.Context) (err error) { 37 | c.BodyBuffer = bytes.NewBufferString("pong") 38 | return 39 | } 40 | -------------------------------------------------------------------------------- /controller/proxy.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 tree xie 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package controller 16 | 17 | import ( 18 | "strconv" 19 | 20 | "github.com/vicanso/elton" 21 | "github.com/vicanso/proxy-pool/router" 22 | "github.com/vicanso/proxy-pool/service" 23 | ) 24 | 25 | type ( 26 | proxyCtrl struct{} 27 | ) 28 | 29 | func init() { 30 | ctrl := proxyCtrl{} 31 | g := router.NewGroup("/proxies") 32 | 33 | g.GET("", ctrl.list) 34 | g.GET("/one", ctrl.findOne) 35 | } 36 | 37 | // list get all available proxy 38 | func (proxyCtrl) list(c *elton.Context) (err error) { 39 | c.CacheMaxAge("1m") 40 | // 直接返回所有可用的proxy，暂不考虑分页等处理 41 | c.Body = map[string]interface{}{ 42 | "proxies": service.GetAvailableProxyList(), 43 | } 44 | return 45 | } 46 | 47 | // findOne get one available proxy 48 | func (proxyCtrl) findOne(c *elton.Context) (err error) { 49 | category := c.QueryParam("category") 50 | speed := -1 51 | sp := c.QueryParam("speed") 52 | if sp != "" { 53 | v, e := strconv.Atoi(sp) 54 | if e == nil { 55 | speed = v 56 | } 57 | } 58 | p := service.GetAvailableProxy(category, speed) 59 | if p == nil { 60 | c.NoContent() 61 | return 62 | } 63 | c.Body = p 64 | return 65 | } 66 | -------------------------------------------------------------------------------- /crawler/crawler.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 tree xie 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package crawler 16 | 17 | import ( 18 | "bytes" 19 | "fmt" 20 | "net" 21 | "net/http" 22 | "net/url" 23 | "sync" 24 | "sync/atomic" 25 | "time" 26 | 27 | "github.com/PuerkitoBio/goquery" 28 | "github.com/vicanso/go-axios" 29 | "github.com/vicanso/proxy-pool/config" 30 | "github.com/vicanso/proxy-pool/log" 31 | "go.uber.org/zap" 32 | ) 33 | 34 | const ( 35 | StatusRunning = iota 36 | StatusStop 37 | ) 38 | 39 | const ( 40 | detectRunning = iota + 1 41 | detectStop 42 | ) 43 | 44 | const ( 45 | defaultUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36" 46 | defaulttProxyTimeout = 10 * time.Second 47 | ) 48 | 49 | var ( 50 | speedDevides = []time.Duration{750 * time.Millisecond, 1500 * time.Millisecond} 51 | 52 | logger = log.Default() 53 | detectConfig = config.GetDetect() 54 | ) 55 | 56 | type ( 57 | // Crawler crawler 58 | Crawler struct { 59 | sync.Mutex 60 | HTTPDetectURL string 61 | HTTPSDetectURL string 62 | newProxyList ProxyList 63 | avaliableProxyList ProxyList 64 | newProxyDetectStatus int32 65 | availableProxyDetectStatus int32 66 | } 67 | // baseProxyCrawler base proxy crawler 68 | // nolint 69 | baseProxyCrawler struct { 70 | // 每次抓取代理信息间隔（需要注意不同的网站对访问频率有不同的限制，不要设置太短） 71 | interval time.Duration 72 | // axios http实例 73 | ins *axios.Instance 74 | // 获取到IP的回调函数 75 | fetchListener FetchListener 76 | // 当前页 77 | currentPage int 78 | // 最大页数 79 | maxPage int 80 | // 状态，运行中或停止 81 | status int32 82 | // 限制的最大页数 83 | limitMaxPage int 84 | } 85 | // FetchListener fetch listener 86 | FetchListener func(*Proxy) 87 | // ProxyCrawler proxy crawler 88 | ProxyCrawler interface { 89 | // OnFetch set fetch listener 90 | OnFetch(FetchListener) 91 | // Start start the crawler 92 | Start() 93 | // Stop stop the crawler 94 | Stop() 95 | } 96 | ) 97 | 98 | // OnFetch set fetch listener 99 | func (bp *baseProxyCrawler) OnFetch(fn FetchListener) { 100 | bp.fetchListener = fn 101 | } 102 | 103 | // Stop stop the crawler 104 | func (bp *baseProxyCrawler) Stop() { 105 | atomic.StoreInt32(&bp.status, StatusStop) 106 | } 107 | 108 | // fetchPage fetch html content of the current page 109 | func (bp *baseProxyCrawler) fetchPage(name, urlTemplate string) (doc *goquery.Document, err error) { 110 | ins := bp.ins 111 | // 至最后一页则重置页码 112 | if bp.maxPage != 0 && bp.currentPage == bp.maxPage { 113 | bp.currentPage = 0 114 | bp.maxPage = 0 115 | } 116 | bp.currentPage++ 117 | resp, err := ins.Get(fmt.Sprintf(urlTemplate, bp.currentPage)) 118 | // 对于抓取失败，则直接退出 119 | if err != nil || 120 | resp.Status != http.StatusOK || 121 | len(resp.Data) == 0 { 122 | logger.Error("get proxy list fail", 123 | zap.String("name", name), 124 | zap.Int("page", bp.currentPage), 125 | zap.Error(err), 126 | ) 127 | return 128 | } 129 | logger.Info("get proxy list success", 130 | zap.String("name", name), 131 | zap.Int("page", bp.currentPage), 132 | ) 133 | return goquery.NewDocumentFromReader(bytes.NewReader(resp.Data)) 134 | } 135 | 136 | // LimitMaxPage set limit max page 137 | func (bp *baseProxyCrawler) LimitMaxPage(value int) { 138 | bp.limitMaxPage = value 139 | } 140 | 141 | // NewProxyClient create a new http client with proxy 142 | func NewProxyClient(p *Proxy) *http.Client { 143 | proxyURL, _ := url.Parse(fmt.Sprintf("http://%s:%s", p.IP, p.Port)) 144 | if proxyURL == nil { 145 | return nil 146 | } 147 | return &http.Client{ 148 | Transport: &http.Transport{ 149 | Proxy: func(_ *http.Request) (*url.URL, error) { 150 | return proxyURL, nil 151 | }, 152 | DialContext: (&net.Dialer{ 153 | Timeout: 30 * time.Second, 154 | KeepAlive: 30 * time.Second, 155 | DualStack: true, 156 | }).DialContext, 157 | ForceAttemptHTTP2: true, 158 | MaxIdleConns: 100, 159 | IdleConnTimeout: 10 * time.Second, 160 | TLSHandshakeTimeout: 10 * time.Second, 161 | ExpectContinueTimeout: 1 * time.Second, 162 | }, 163 | } 164 | } 165 | 166 | // analyze check the proxy is available and speed 167 | func (c *Crawler) analyze(p *Proxy) (available bool) { 168 | httpClient := NewProxyClient(p) 169 | if httpClient == nil { 170 | return false 171 | } 172 | // 多次检测，只要一次成功则认为成功 173 | for i := 0; i < detectConfig.MaxTimes; i++ { 174 | ins := axios.NewInstance(&axios.InstanceConfig{ 175 | Timeout: detectConfig.Timeout, 176 | Client: httpClient, 177 | }) 178 | startedAt := time.Now() 179 | resp, err := ins.Get(detectConfig.URL) 180 | if err != nil { 181 | continue 182 | } 183 | if resp.Status >= http.StatusOK && resp.Status < http.StatusBadRequest { 184 | d := time.Since(startedAt) 185 | atomic.StoreInt32(&p.Speed, int32(len(speedDevides))) 186 | // 将当前proxy划分对应的分段 187 | for index, item := range speedDevides { 188 | if d < item { 189 | atomic.StoreInt32(&p.Speed, int32(index)) 190 | break 191 | } 192 | } 193 | available = true 194 | break 195 | } 196 | } 197 | return 198 | } 199 | 200 | // addNewProxy add proxy to new proxy list 201 | func (c *Crawler) addNewProxy(p *Proxy) { 202 | c.newProxyList.Add(p) 203 | } 204 | 205 | // detectProxyList detect proxy list 206 | func (c *Crawler) detectProxyList(list []*Proxy) (availableList []*Proxy, unavailableList []*Proxy) { 207 | availableList = make([]*Proxy, 0) 208 | unavailableList = make([]*Proxy, 0) 209 | w := sync.WaitGroup{} 210 | // 控制最多检测proxy的数量 211 | chans := make(chan bool, 5) 212 | for _, item := range list { 213 | w.Add(1) 214 | go func(p *Proxy) { 215 | chans <- true 216 | avaliable := c.analyze(p) 217 | atomic.StoreInt64(&p.DetectedAt, time.Now().Unix()) 218 | if avaliable { 219 | availableList = append(availableList, p) 220 | } else { 221 | unavailableList = append(unavailableList, p) 222 | } 223 | <-chans 224 | w.Done() 225 | }(item) 226 | } 227 | w.Wait() 228 | return 229 | } 230 | 231 | // detectNewProxy detect the new proxy is avaliable 232 | func (c *Crawler) detectNewProxy() { 233 | old := atomic.SwapInt32(&c.newProxyDetectStatus, detectRunning) 234 | // 如果已经在运行中，则直接退出 235 | if old == detectRunning { 236 | return 237 | } 238 | proxyList := c.newProxyList.Reset() 239 | availableList, _ := c.detectProxyList(proxyList) 240 | c.avaliableProxyList.Add(availableList...) 241 | 242 | atomic.StoreInt32(&c.newProxyDetectStatus, detectStop) 243 | // 等待1分钟后，重新运行detect new proxy 244 | time.Sleep(time.Minute) 245 | c.detectNewProxy() 246 | } 247 | 248 | // RedetectAvailableProxy redetect available proxy 249 | func (c *Crawler) RedetectAvailableProxy() { 250 | old := atomic.SwapInt32(&c.availableProxyDetectStatus, detectRunning) 251 | // 如果已经在运行中，则直接退出 252 | if old == detectRunning { 253 | return 254 | } 255 | proxyList := c.avaliableProxyList.List() 256 | availableList, unavailableList := c.detectProxyList(proxyList) 257 | 258 | // 如果成功，则重置失败次数 259 | for _, p := range availableList { 260 | atomic.StoreInt32(&p.Fails, 0) 261 | } 262 | // 如果失败，则失败次数+1 263 | failProxyList := make([]*Proxy, 0) 264 | for _, p := range unavailableList { 265 | count := atomic.AddInt32(&p.Fails, 1) 266 | if count >= 3 { 267 | failProxyList = append(failProxyList, p) 268 | } 269 | } 270 | // 对于三次检测失败的代理则删除 271 | c.avaliableProxyList.Remove(failProxyList...) 272 | 273 | atomic.StoreInt32(&c.availableProxyDetectStatus, detectStop) 274 | } 275 | 276 | // Start start fetch proxy 277 | func (c *Crawler) Start(crawlers ...ProxyCrawler) { 278 | for _, item := range crawlers { 279 | item.OnFetch(c.addNewProxy) 280 | go item.Start() 281 | } 282 | // 首次延时10秒后则执行detect new proxy 283 | go func() { 284 | time.Sleep(10 * time.Second) 285 | c.detectNewProxy() 286 | }() 287 | } 288 | 289 | // GetAvailableProxyList get available proxy list 290 | func (c *Crawler) GetAvailableProxyList() []*Proxy { 291 | return c.avaliableProxyList.List() 292 | } 293 | 294 | // GetAvailableProxy get available proxy 295 | func (c *Crawler) GetAvailableProxy(category string, speed int32) *Proxy { 296 | return c.avaliableProxyList.FindOne(category, speed) 297 | } 298 | -------------------------------------------------------------------------------- /crawler/crawler_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 tree xie 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package crawler 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/vicanso/go-axios" 21 | 22 | "github.com/stretchr/testify/assert" 23 | ) 24 | 25 | func TestBaseProxyCrawler(t *testing.T) { 26 | assert := assert.New(t) 27 | bp := new(baseProxyCrawler) 28 | 29 | assert.Nil(bp.fetchListener) 30 | bp.OnFetch(func(_ *Proxy) {}) 31 | assert.NotNil(bp.fetchListener) 32 | 33 | ins := axios.NewInstance(nil) 34 | bp.currentPage = 10 35 | bp.maxPage = 10 36 | bp.ins = ins 37 | // empty data 38 | done := ins.Mock(&axios.Response{ 39 | Status: 200, 40 | Data: []byte(""), 41 | }) 42 | doc, err := bp.fetchPage("", "%d") 43 | assert.Nil(err) 44 | assert.Nil(doc) 45 | assert.Equal(0, bp.maxPage) 46 | assert.Equal(1, bp.currentPage) 47 | done() 48 | } 49 | 50 | -------------------------------------------------------------------------------- /crawler/ip66.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 tree xie 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package crawler 16 | 17 | import ( 18 | "net/http" 19 | "strconv" 20 | "time" 21 | 22 | "github.com/PuerkitoBio/goquery" 23 | "github.com/vicanso/go-axios" 24 | ) 25 | 26 | type ( 27 | // ip66Proxy ip66 proxy 28 | ip66Proxy struct { 29 | baseProxyCrawler 30 | } 31 | ) 32 | 33 | const ( 34 | ProxyIP66 = "ip66" 35 | ) 36 | 37 | // NewIP66Proxy create a new ip66 proxy crawler 38 | func NewIP66Proxy(interval time.Duration) *ip66Proxy { 39 | header := make(http.Header) 40 | header.Set("User-Agent", defaultUserAgent) 41 | ins := axios.NewInstance(&axios.InstanceConfig{ 42 | BaseURL: "http://www.66ip.cn", 43 | Headers: header, 44 | Timeout: defaulttProxyTimeout, 45 | }) 46 | ip66 := new(ip66Proxy) 47 | ip66.interval = interval 48 | ip66.ins = ins 49 | return ip66 50 | } 51 | 52 | // Start start the crawler 53 | func (ip66 *ip66Proxy) Start() { 54 | ip66.status = StatusRunning 55 | for { 56 | if ip66.status != StatusRunning { 57 | return 58 | } 59 | _ = ip66.fetch() 60 | time.Sleep(ip66.interval) 61 | } 62 | } 63 | 64 | func (ip66 *ip66Proxy) fetch() (err error) { 65 | doc, err := ip66.fetchPage("ip66", "/%d") 66 | if err != nil || doc == nil { 67 | return 68 | } 69 | // 仅在首次获取 70 | if ip66.maxPage == 0 { 71 | pages := doc.Find("#PageList a") 72 | value := pages.Eq(pages.Length() - 2).Text() 73 | max, _ := strconv.Atoi(value) 74 | if max == 0 { 75 | max = 1 76 | } 77 | if ip66.limitMaxPage != 0 && max > ip66.limitMaxPage { 78 | max = ip66.limitMaxPage 79 | } 80 | ip66.maxPage = max 81 | } 82 | doc.Find("#main table tr").Each(func(i int, s *goquery.Selection) { 83 | // 表头忽略 84 | if i == 0 { 85 | return 86 | } 87 | tdList := s.Find("td") 88 | ip := tdList.Eq(0).Text() 89 | port := tdList.Eq(1).Text() 90 | if ip == "" || port == "" || ip66.fetchListener == nil { 91 | return 92 | } 93 | fn := ip66.fetchListener 94 | fn(&Proxy{ 95 | IP: ip, 96 | Port: port, 97 | Anonymous: true, 98 | Category: "http", 99 | }) 100 | }) 101 | return 102 | } 103 | -------------------------------------------------------------------------------- /crawler/ip66_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 tree xie 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package crawler 16 | 17 | import ( 18 | "testing" 19 | "time" 20 | 21 | "github.com/stretchr/testify/assert" 22 | "github.com/vicanso/go-axios" 23 | ) 24 | 25 | func TestIP66Proxy(t *testing.T) { 26 | assert := assert.New(t) 27 | ip66 := NewIP66Proxy(time.Minute) 28 | html := ` 29 | 30 |
31 | 10 32 | 33 |
34 |
35 | 36 | 37 | 38 |
183.166.71.939999安徽省淮南市高匿代理2019年12月14日14时 验证
39 |
40 | 41 | ` 42 | ip66.ins.Mock(&axios.Response{ 43 | Status: 200, 44 | Data: []byte(html), 45 | }) 46 | done := make(chan bool) 47 | ip66.OnFetch(func(p *Proxy) { 48 | assert.Equal("183.166.71.93", p.IP) 49 | assert.Equal("9999", p.Port) 50 | done <- true 51 | }) 52 | go ip66.Start() 53 | <-done 54 | ip66.Stop() 55 | assert.Equal(10, ip66.maxPage) 56 | } 57 | -------------------------------------------------------------------------------- /crawler/kuai.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 tree xie 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package crawler 16 | 17 | import ( 18 | "net/http" 19 | "strconv" 20 | "strings" 21 | "time" 22 | 23 | "github.com/PuerkitoBio/goquery" 24 | 25 | "github.com/vicanso/go-axios" 26 | ) 27 | 28 | type ( 29 | // kuaiProxy kuai proxy 30 | kuaiProxy struct { 31 | baseProxyCrawler 32 | } 33 | ) 34 | 35 | const ( 36 | ProxyKuai = "kuai" 37 | ) 38 | 39 | // NewKuaiProxy create a new kuai proxy crawler 40 | func NewKuaiProxy(interval time.Duration) *kuaiProxy { 41 | header := make(http.Header) 42 | header.Set("User-Agent", defaultUserAgent) 43 | ins := axios.NewInstance(&axios.InstanceConfig{ 44 | BaseURL: "https://www.kuaidaili.com/free/inha", 45 | Headers: header, 46 | Timeout: defaulttProxyTimeout, 47 | }) 48 | kuaiProxy := new(kuaiProxy) 49 | kuaiProxy.interval = interval 50 | kuaiProxy.ins = ins 51 | return kuaiProxy 52 | } 53 | 54 | // Start start the crawler 55 | func (kuai *kuaiProxy) Start() { 56 | kuai.status = StatusRunning 57 | for { 58 | if kuai.status != StatusRunning { 59 | return 60 | } 61 | // 获取proxy信息 62 | _ = kuai.fetch() 63 | time.Sleep(kuai.interval) 64 | } 65 | } 66 | 67 | // Fetch fetch proxy list from kuai dai li 68 | func (kuai *kuaiProxy) fetch() (err error) { 69 | doc, err := kuai.fetchPage("kuai daili", "/%d/") 70 | if err != nil || doc == nil { 71 | return 72 | } 73 | if kuai.maxPage == 0 { 74 | pages := doc.Find("#listnav a") 75 | value := pages.Last().Text() 76 | max, _ := strconv.Atoi(value) 77 | if max == 0 { 78 | max = 1 79 | } 80 | if kuai.limitMaxPage != 0 && max > kuai.limitMaxPage { 81 | max = kuai.limitMaxPage 82 | } 83 | kuai.maxPage = max 84 | } 85 | doc.Find("#list tbody tr").Each(func(i int, s *goquery.Selection) { 86 | tdList := s.Find("td") 87 | ip := tdList.Eq(0).Text() 88 | port := tdList.Eq(1).Text() 89 | category := strings.ToLower(tdList.Eq(3).Text()) 90 | if ip == "" || 91 | port == "" || 92 | category == "" || 93 | kuai.fetchListener == nil { 94 | return 95 | } 96 | kuai.fetchListener(&Proxy{ 97 | IP: ip, 98 | Port: port, 99 | Anonymous: true, 100 | Category: category, 101 | }) 102 | }) 103 | return 104 | } 105 | -------------------------------------------------------------------------------- /crawler/kuai_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 tree xie 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package crawler 16 | 17 | import ( 18 | "testing" 19 | "time" 20 | 21 | "github.com/vicanso/go-axios" 22 | 23 | "github.com/stretchr/testify/assert" 24 | ) 25 | 26 | func TestKuaiProxy(t *testing.T) { 27 | assert := assert.New(t) 28 | kuai := NewKuaiProxy(time.Minute) 29 | html := ` 30 | 31 |
32 | 10 33 |
34 |
35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 |
171.13.103.2139999高匿名HTTP河南省鹤壁市 电信2秒2019-12-14 15:31:01
48 |
49 | 50 | ` 51 | kuai.ins.Mock(&axios.Response{ 52 | Status: 200, 53 | Data: []byte(html), 54 | }) 55 | done := make(chan bool) 56 | kuai.OnFetch(func(p *Proxy) { 57 | assert.Equal("171.13.103.213", p.IP) 58 | assert.Equal("9999", p.Port) 59 | assert.Equal("http", p.Category) 60 | done <- true 61 | }) 62 | go kuai.Start() 63 | <-done 64 | kuai.Stop() 65 | assert.Equal(10, kuai.maxPage) 66 | } 67 | -------------------------------------------------------------------------------- /crawler/proxylist.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 tree xie 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package crawler 16 | 17 | import ( 18 | "math/rand" 19 | "sync" 20 | "time" 21 | ) 22 | 23 | type ( 24 | // Proxy proxy server 25 | Proxy struct { 26 | DetectedAt int64 `json:"detectedAt,omitempty"` 27 | IP string `json:"ip,omitempty"` 28 | Port string `json:"port,omitempty"` 29 | Category string `json:"category,omitempty"` 30 | Anonymous bool `json:"anonymous,omitempty"` 31 | Speed int32 `json:"speed,omitempty"` 32 | Fails int32 `json:"fails,omitempty"` 33 | } 34 | // ProxyList proxy list 35 | ProxyList struct { 36 | sync.RWMutex 37 | data []*Proxy 38 | } 39 | ) 40 | 41 | func (pl *ProxyList) indexOf(p *Proxy) int { 42 | index := -1 43 | for i, item := range pl.data { 44 | if item.IP == p.IP && 45 | item.Port == p.Port && 46 | item.Category == p.Category { 47 | index = i 48 | break 49 | } 50 | } 51 | return index 52 | } 53 | 54 | // Exists test whether or not the proxy exists 55 | func (pl *ProxyList) Exists(p *Proxy) bool { 56 | pl.RLock() 57 | defer pl.RUnlock() 58 | return pl.indexOf(p) != -1 59 | } 60 | 61 | // Add add proxy to list 62 | func (pl *ProxyList) Add(list ...*Proxy) { 63 | if len(list) == 0 { 64 | return 65 | } 66 | pl.Lock() 67 | defer pl.Unlock() 68 | if len(pl.data) == 0 { 69 | pl.data = make([]*Proxy, 0, 100) 70 | } 71 | for _, p := range list { 72 | if pl.indexOf(p) != -1 { 73 | continue 74 | } 75 | pl.data = append(pl.data, p) 76 | } 77 | } 78 | 79 | // Remove remove proxy from list 80 | func (pl *ProxyList) Remove(list ...*Proxy) { 81 | if len(list) == 0 { 82 | return 83 | } 84 | pl.Lock() 85 | defer pl.Unlock() 86 | if len(pl.data) == 0 { 87 | return 88 | } 89 | for _, p := range list { 90 | index := pl.indexOf(p) 91 | if index != -1 { 92 | pl.data = append(pl.data[:index], pl.data[index+1:]...) 93 | } 94 | } 95 | } 96 | 97 | // List get proxy list 98 | func (pl *ProxyList) List() []*Proxy { 99 | pl.RLock() 100 | defer pl.RUnlock() 101 | return pl.data[:] 102 | } 103 | 104 | // Reset reset proxy list 105 | func (pl *ProxyList) Reset() []*Proxy { 106 | pl.Lock() 107 | defer pl.Unlock() 108 | oldProxyList := pl.data[:] 109 | pl.data = nil 110 | return oldProxyList 111 | } 112 | 113 | // Size get the size of proxy list 114 | func (pl *ProxyList) Size() int { 115 | pl.RLock() 116 | defer pl.RUnlock() 117 | return len(pl.data) 118 | } 119 | 120 | // Replace replace the proxy list 121 | func (pl *ProxyList) Replace(list []*Proxy) { 122 | pl.Lock() 123 | defer pl.Unlock() 124 | pl.data = list 125 | } 126 | 127 | // FindOne find one proxy 128 | func (pl *ProxyList) FindOne(category string, speed int32) (p *Proxy) { 129 | pl.RLock() 130 | defer pl.RUnlock() 131 | list := pl.data 132 | // 指定了速度或者代理类型 133 | if speed >= 0 || category != "" { 134 | list = make([]*Proxy, 0, 10) 135 | for _, item := range pl.data { 136 | if speed >= 0 && item.Speed != speed { 137 | continue 138 | } 139 | if category != "" && item.Category != category { 140 | continue 141 | } 142 | list = append(list, item) 143 | } 144 | } 145 | size := len(list) 146 | if size == 0 { 147 | return 148 | } 149 | rand.Seed(time.Now().UnixNano()) 150 | return list[rand.Intn(size)] 151 | } 152 | -------------------------------------------------------------------------------- /crawler/proxylist_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 tree xie 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package crawler 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/stretchr/testify/assert" 21 | ) 22 | 23 | func TestProxyList(t *testing.T) { 24 | assert := assert.New(t) 25 | pl := new(ProxyList) 26 | 27 | p := &Proxy{ 28 | IP: "127.0.0.1", 29 | Port: "80", 30 | Category: "http", 31 | Speed: 2, 32 | } 33 | pl.Add(p) 34 | pl.Add(p) 35 | 36 | result := pl.FindOne("http", 2) 37 | assert.Equal(p, result) 38 | 39 | assert.False(pl.Exists(&Proxy{ 40 | IP: "127.0.0.1", 41 | Port: "80", 42 | Category: "https", 43 | })) 44 | assert.True(pl.Exists(&Proxy{ 45 | IP: "127.0.0.1", 46 | Port: "80", 47 | Category: "http", 48 | })) 49 | assert.Equal(1, pl.Size()) 50 | assert.Equal(p, pl.List()[0]) 51 | assert.Equal(p, pl.Reset()[0]) 52 | assert.Equal(0, pl.Size()) 53 | 54 | pl.Add(p) 55 | newList := make([]*Proxy, 0) 56 | pl.Replace(newList) 57 | assert.Equal(newList, pl.data) 58 | } 59 | -------------------------------------------------------------------------------- /crawler/xici.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 tree xie 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package crawler 16 | 17 | import ( 18 | "net/http" 19 | "strconv" 20 | "strings" 21 | "time" 22 | 23 | "github.com/PuerkitoBio/goquery" 24 | "github.com/vicanso/go-axios" 25 | ) 26 | 27 | type ( 28 | // xiciProxy xici proxy 29 | xiciProxy struct { 30 | baseProxyCrawler 31 | } 32 | ) 33 | 34 | const ( 35 | ProxyXiCi = "xici" 36 | ) 37 | 38 | // NewXiciProxy create a new xici proxy crawler 39 | func NewXiciProxy(interval time.Duration) *xiciProxy { 40 | header := make(http.Header) 41 | header.Set("User-Agent", defaultUserAgent) 42 | ins := axios.NewInstance(&axios.InstanceConfig{ 43 | BaseURL: "https://www.xicidaili.com/nn", 44 | Headers: header, 45 | Timeout: defaulttProxyTimeout, 46 | }) 47 | xiciProxy := new(xiciProxy) 48 | xiciProxy.interval = interval 49 | xiciProxy.ins = ins 50 | return xiciProxy 51 | } 52 | 53 | // Start start the crawler 54 | func (xc *xiciProxy) Start() { 55 | xc.status = StatusRunning 56 | for { 57 | if xc.status != StatusRunning { 58 | return 59 | } 60 | // 获取proxy信息 61 | _ = xc.fetch() 62 | time.Sleep(xc.interval) 63 | } 64 | } 65 | 66 | // Fetch fetch proxy list from xici 67 | func (xc *xiciProxy) fetch() (err error) { 68 | doc, err := xc.fetchPage("xici", "/%d") 69 | if err != nil || doc == nil { 70 | return 71 | } 72 | // 仅在首次获取 73 | if xc.maxPage == 0 { 74 | pages := doc.Find(".pagination a") 75 | value := pages.Eq(pages.Length() - 2).Text() 76 | max, _ := strconv.Atoi(value) 77 | if max == 0 { 78 | max = 1 79 | } 80 | if xc.limitMaxPage != 0 && max > xc.limitMaxPage { 81 | max = xc.limitMaxPage 82 | } 83 | xc.maxPage = max 84 | } 85 | // 解析表格获取代理列表 86 | doc.Find("#ip_list tr").Each(func(i int, s *goquery.Selection) { 87 | // 表头忽略 88 | if i == 0 { 89 | return 90 | } 91 | tdList := s.Find("td") 92 | ip := tdList.Eq(1).Text() 93 | port := tdList.Eq(2).Text() 94 | anonymous := tdList.Eq(4).Text() == "高匿" 95 | category := strings.ToLower(tdList.Eq(5).Text()) 96 | 97 | if ip == "" || 98 | port == "" || 99 | category == "" || 100 | xc.fetchListener == nil { 101 | return 102 | } 103 | xc.fetchListener(&Proxy{ 104 | IP: ip, 105 | Port: port, 106 | Anonymous: anonymous, 107 | Category: category, 108 | }) 109 | }) 110 | return 111 | } 112 | -------------------------------------------------------------------------------- /crawler/xici_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 tree xie 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package crawler 16 | 17 | import ( 18 | "testing" 19 | "time" 20 | 21 | "github.com/vicanso/go-axios" 22 | 23 | "github.com/stretchr/testify/assert" 24 | ) 25 | 26 | func TestXiciProxy(t *testing.T) { 27 | assert := assert.New(t) 28 | xici := NewXiciProxy(time.Minute) 29 | html := ` 30 | 31 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 45 | 46 | 47 | 54 | 61 | 62 | 63 | 64 | 65 | 66 |
Cn183.154.49.89999 43 | 浙江金华 44 | 高匿HTTP 48 |
49 |
50 | 51 |
52 |
53 | 		55 |
56 |
57 | 58 |
59 |
60 | 		1分钟19-12-14 16:21
67 | 68 | ` 69 | xici.ins.Mock(&axios.Response{ 70 | Status: 200, 71 | Data: []byte(html), 72 | }) 73 | done := make(chan bool) 74 | xici.OnFetch(func(p *Proxy) { 75 | assert.Equal("183.154.49.8", p.IP) 76 | assert.Equal("9999", p.Port) 77 | assert.Equal("http", p.Category) 78 | done <- true 79 | }) 80 | go xici.Start() 81 | <-done 82 | xici.Stop() 83 | assert.Equal(10, xici.maxPage) 84 | } 85 | -------------------------------------------------------------------------------- /doc/config.md: -------------------------------------------------------------------------------- 1 | # 应用配置 2 | 3 | 在具体抓取代理时，可根据需求设定相应的抓取列表以及检测延时等，使用[viper](https://github.com/spf13/viper)，可以方便的通过`yaml`来定义应用配置。在生成go程序，我习惯性采用单一可执行文件，将配置打包至应用程序，对于自动构建发布的流程比较适合。对于是否打包至执行文件，在不同的应用场景可按需考虑，不需要打包的在构建时不执行打包则可。 4 | 5 | ## 初始化配置 6 | 7 | ```go 8 | func init() { 9 | configType := "yml" 10 | configExt := "." + configType 11 | data, err := box.Find("default" + configExt) 12 | if err != nil { 13 | panic(err) 14 | } 15 | viper.SetConfigType(configType) 16 | v := viper.New() 17 | v.SetConfigType(configType) 18 | // 读取默认配置中的所有配置 19 | err = v.ReadConfig(bytes.NewReader(data)) 20 | if err != nil { 21 | panic(err) 22 | } 23 | configs := v.AllSettings() 24 | // 将default中的配置全部以默认配置写入 25 | for k, v := range configs { 26 | viper.SetDefault(k, v) 27 | } 28 | 29 | // 根据当前运行环境配置读取 30 | // 可根据不同的环境仅调整与default不一致的相关配置 31 | if env != "" { 32 | envConfigFile := env + configExt 33 | data, err = box.Find(envConfigFile) 34 | if err != nil { 35 | panic(err) 36 | } 37 | // 读取当前运行环境对应的配置 38 | err = viper.ReadConfig(bytes.NewReader(data)) 39 | if err != nil { 40 | panic(err) 41 | } 42 | } 43 | } 44 | ``` 45 | 46 | 初始化配置读取`default.yml`的相关配置信息作为默认配置，再根据配置的`GO_ENV`读取该环境下的配置（此配置会覆盖默认配置），适用于各运行环境（开发、测试、生产）都有个性化配置的场景。 47 | 48 | ## 获取抓取列表配置 49 | 50 | ```go 51 | type ( 52 | Crawler struct { 53 | Name string 54 | Interval time.Duration 55 | } 56 | ) 57 | 58 | // GetCrawlers get crawlers config 59 | func GetCrawlers() []*Crawler { 60 | crawlers := make([]*Crawler, 0) 61 | data := viper.GetStringSlice("crawler") 62 | for _, name := range data { 63 | interval := viper.GetDuration(name + ".interval") 64 | // 如果未配置抓取间隔时间，则设置为2分钟 65 | if interval == 0 { 66 | interval = 2 * time.Minute 67 | } 68 | crawlers = append(crawlers, &Crawler{ 69 | Name: name, 70 | Interval: interval, 71 | }) 72 | } 73 | return crawlers 74 | } 75 | ``` 76 | 77 | ## 获取检测配置 78 | 79 | ```go 80 | type ( 81 | // Detect detect config 82 | Detect struct { 83 | URL string 84 | Interval time.Duration 85 | Timeout time.Duration 86 | MaxTimes int 87 | } 88 | ) 89 | // GetDetect get detect config 90 | func GetDetect() *Detect { 91 | prefix := "detect." 92 | conf := &Detect{ 93 | Timeout: viper.GetDuration(prefix + "timeout"), 94 | URL: viper.GetString(prefix + "url"), 95 | Interval: viper.GetDuration(prefix + "interval"), 96 | MaxTimes: viper.GetInt(prefix + "maxTimes"), 97 | } 98 | if conf.Timeout == 0 { 99 | conf.Timeout = 3 * time.Second 100 | } 101 | if conf.Interval == 0 { 102 | conf.Interval = 30 * time.Minute 103 | } 104 | if conf.URL == "" { 105 | conf.URL = "https://www.baidu.com/" 106 | } 107 | if conf.MaxTimes <= 0 { 108 | conf.MaxTimes = 3 109 | } 110 | return conf 111 | } 112 | ``` -------------------------------------------------------------------------------- /doc/crawler.md: -------------------------------------------------------------------------------- 1 | # 代理IP抓取 2 | 3 | 代理IP地址主要通过抓取提供免费代理IP的网站，如`西刺`。通过获取其网页数据，筛选出代理服务器的信息，如：IP、端口、支持的代理类型以及是否匿名代理。 4 | 5 | 根据需求先定义一个Proxy的结构来保存代理信息，一个ProxyCrawler的interface，便于后面扩展从不同的网站上抓取代理信息。 6 | 7 | 8 | ```go 9 | type ( 10 | // Crawler crawler 11 | Crawler struct { 12 | sync.Mutex 13 | HTTPDetectURL string 14 | HTTPSDetectURL string 15 | newProxyList ProxyList 16 | avaliableProxyList ProxyList 17 | newProxyDetectStatus int32 18 | availableProxyDetectStatus int32 19 | } 20 | // FetchListener fetch listener 21 | FetchListener func(*Proxy) 22 | // ProxyCrawler proxy crawler 23 | ProxyCrawler interface { 24 | // OnFetch set fetch listener 25 | OnFetch(FetchListener) 26 | // Start start the crawler 27 | Start() 28 | // Stop stop the crawler 29 | Stop() 30 | } 31 | ) 32 | ``` 33 | 34 | ProxyCrawler的OnFetch函数用于设定获取proxy成功时的回调，各抓取服务在抓取到相应的代理信息之后，则触发此回调，为什么使用这样的实现形式呢？因为各网站都会对访问频率有限制，因此无法快速的获取所有代理信息，只能抓取一页数据之后，延时再去抓取的方式，使用回调则可以在获取到一个代理信息时，则立即触发回调。 35 | 36 | 下面开始编写对`西刺`的数据抓取，主要获取其高匿代理的数据，地址为：`https://www.xicidaili.com/nn/`，其数据是以html的形式返回，展示形式为表格，因此抓取的方法也比较简单，HTTP请求获取数据，解析html，从表格中获取相应的代理信息。 37 | 38 | 首先将抓取IP地址的功能提取公共模块，`baseProxyCrawler`: 39 | 40 | ```go 41 | // baseProxyCrawler base proxy crawler 42 | // nolint 43 | type baseProxyCrawler struct { 44 | // 每次抓取代理信息间隔（需要注意不同的网站对访问频率有不同的限制，不要设置太短） 45 | interval time.Duration 46 | // axios http实例 47 | ins *axios.Instance 48 | // 获取到IP的回调函数 49 | fetchListener FetchListener 50 | // 当前页 51 | currentPage int 52 | // 最大页数 53 | maxPage int 54 | // 状态，运行中或停止 55 | status int32 56 | } 57 | 58 | 59 | // OnFetch set fetch listener 60 | func (bp *baseProxyCrawler) OnFetch(fn FetchListener) { 61 | bp.fetchListener = fn 62 | } 63 | 64 | // Stop stop the crawler 65 | func (bp *baseProxyCrawler) Stop() { 66 | atomic.StoreInt32(&bp.status, StatusStop) 67 | } 68 | 69 | // fetchPage fetch html content of the current page 70 | func (bp *baseProxyCrawler) fetchPage(name, urlTemplate string) (doc *goquery.Document, err error) { 71 | ins := bp.ins 72 | // 至最后一页则重置页码 73 | if bp.maxPage != 0 && bp.currentPage == bp.maxPage { 74 | bp.currentPage = 0 75 | bp.maxPage = 0 76 | } 77 | bp.currentPage++ 78 | resp, err := ins.Get(fmt.Sprintf(urlTemplate, bp.currentPage)) 79 | // 对于抓取失败，则直接退出 80 | if err != nil || 81 | resp.Status != http.StatusOK || 82 | len(resp.Data) == 0 { 83 | logger.Error("get proxy list fail", 84 | zap.String("name", name), 85 | zap.Int("page", bp.currentPage), 86 | zap.Error(err), 87 | ) 88 | return 89 | } 90 | logger.Info("get proxy list success", 91 | zap.String("name", name), 92 | zap.Int("page", bp.currentPage), 93 | ) 94 | return goquery.NewDocumentFromReader(bytes.NewReader(resp.Data)) 95 | } 96 | ``` 97 | 98 | 下面是具体抓取`西刺`代理的免费IP地址的代码如下： 99 | 100 | ```go 101 | type ( 102 | // xiciProxy xici proxy 103 | xiciProxy struct { 104 | baseProxyCrawler 105 | } 106 | ) 107 | 108 | // NewXiciProxy create a new xici proxy crawler 109 | func NewXiciProxy(interval time.Duration) *xiciProxy { 110 | header := make(http.Header) 111 | header.Set("User-Agent", defaultUserAgent) 112 | ins := axios.NewInstance(&axios.InstanceConfig{ 113 | BaseURL: "https://www.xicidaili.com/nn", 114 | Headers: header, 115 | Timeout: defaulttProxyTimeout, 116 | }) 117 | xiciProxy := new(xiciProxy) 118 | xiciProxy.interval = interval 119 | xiciProxy.ins = ins 120 | return xiciProxy 121 | } 122 | 123 | // Start start the crawler 124 | func (xc *xiciProxy) Start() { 125 | xc.status = StatusRunning 126 | for { 127 | if xc.status != StatusRunning { 128 | return 129 | } 130 | // 获取proxy信息 131 | _ = xc.fetch() 132 | time.Sleep(xc.interval) 133 | } 134 | } 135 | 136 | // Fetch fetch proxy list from xici 137 | func (xc *xiciProxy) fetch() (err error) { 138 | doc, err := xc.fetchPage("xici", "/%d") 139 | if err != nil { 140 | return 141 | } 142 | // 仅在首次获取 143 | if xc.maxPage == 0 { 144 | pages := doc.Find(".pagination a") 145 | value := pages.Eq(pages.Length() - 2).Text() 146 | max, _ := strconv.Atoi(value) 147 | if max == 0 { 148 | max = 1 149 | } 150 | xc.maxPage = max 151 | } 152 | // 解析表格获取代理列表 153 | doc.Find("#ip_list tr").Each(func(i int, s *goquery.Selection) { 154 | // 表头忽略 155 | if i == 0 { 156 | return 157 | } 158 | tdList := s.Find("td") 159 | ip := tdList.Eq(1).Text() 160 | port := tdList.Eq(2).Text() 161 | anonymous := tdList.Eq(4).Text() == "高匿" 162 | category := strings.ToLower(tdList.Eq(5).Text()) 163 | 164 | if ip == "" || 165 | port == "" || 166 | category == "" || 167 | xc.fetchListener == nil { 168 | return 169 | } 170 | xc.fetchListener(&Proxy{ 171 | IP: ip, 172 | Port: port, 173 | Anonymous: anonymous, 174 | Category: category, 175 | }) 176 | }) 177 | return 178 | } 179 | ``` 180 | 181 | 至此抓取免费代理IP地址已经完成，后续增加了对`66ip`以及`快代理`的抓取，如果还有其它网站可供抓取，只需参考实现抓取的流程则可。 182 | 183 | # 代理IP连通性测试 184 | 185 | 在实际使用时发现大部分抓取到的IP地址都不可用，因此增加检测模块，定时对新抓取的代理IP检测（使用此IP做代理去访问baidu），访问成功的则记录至可用代理列表中。 186 | 187 | ## 代理列表 188 | 189 | 代理列表用于保存代理地址，提供判断是否存在、添加、重置等方法，主要用于保存新抓取的代理地址以及可用代理地址，部分代码如下： 190 | 191 | ```go 192 | type ( 193 | // ProxyList proxy list 194 | ProxyList struct { 195 | sync.RWMutex 196 | data []*Proxy 197 | } 198 | ) 199 | 200 | // Exists test whether or not the proxy exists 201 | func (pl *ProxyList) Exists(p *Proxy) bool { 202 | pl.RLock() 203 | defer pl.RUnlock() 204 | found := false 205 | for _, item := range pl.data { 206 | if item.IP == p.IP && 207 | item.Port == p.Port && 208 | item.Category == p.Category { 209 | found = true 210 | break 211 | } 212 | } 213 | return found 214 | } 215 | 216 | // Add add proxy to list 217 | func (pl *ProxyList) Add(p *Proxy) { 218 | if pl.Exists(p) { 219 | return 220 | } 221 | pl.Lock() 222 | defer pl.Unlock() 223 | if len(pl.data) == 0 { 224 | pl.data = make([]*Proxy, 0, 100) 225 | } 226 | pl.data = append(pl.data, p) 227 | } 228 | 229 | ``` 230 | 231 | ## 代理IP检测 232 | 233 | 对代理IP检测主要判断其可用性以及连接速度（默认为3秒如果无响应则不可用），检测逻辑比较简单，创建新的Transport，指定其使用的Proxy，如果成功返回则计算处理时间，并添加至可用代理列表。 234 | 235 | ```go 236 | // NewProxyClient create a new http client with proxy 237 | func NewProxyClient(p *Proxy) *http.Client { 238 | proxyURL, _ := url.Parse(fmt.Sprintf("http://%s:%s", p.IP, p.Port)) 239 | if proxyURL == nil { 240 | return nil 241 | } 242 | return &http.Client{ 243 | Transport: &http.Transport{ 244 | Proxy: func(_ *http.Request) (*url.URL, error) { 245 | return proxyURL, nil 246 | }, 247 | DialContext: (&net.Dialer{ 248 | Timeout: 30 * time.Second, 249 | KeepAlive: 30 * time.Second, 250 | DualStack: true, 251 | }).DialContext, 252 | ForceAttemptHTTP2: true, 253 | MaxIdleConns: 100, 254 | IdleConnTimeout: 10 * time.Second, 255 | TLSHandshakeTimeout: 10 * time.Second, 256 | ExpectContinueTimeout: 1 * time.Second, 257 | }, 258 | } 259 | } 260 | 261 | // analyze check the proxy is available and speed 262 | func (c *Crawler) analyze(p *Proxy) (available bool) { 263 | httpClient := NewProxyClient(p) 264 | if httpClient == nil { 265 | return false 266 | } 267 | // 多次检测，只要一次成功则认为成功 268 | for i := 0; i < detectConfig.MaxTimes; i++ { 269 | ins := axios.NewInstance(&axios.InstanceConfig{ 270 | Timeout: detectConfig.Timeout, 271 | Client: httpClient, 272 | }) 273 | startedAt := time.Now() 274 | resp, err := ins.Get(detectConfig.URL) 275 | if err != nil { 276 | continue 277 | } 278 | if resp.Status >= http.StatusOK && resp.Status < http.StatusBadRequest { 279 | d := time.Since(startedAt) 280 | atomic.StoreInt32(&p.Speed, int32(len(speedDevides))) 281 | // 将当前proxy划分对应的分段 282 | for index, item := range speedDevides { 283 | if d < item { 284 | atomic.StoreInt32(&p.Speed, int32(index)) 285 | break 286 | } 287 | } 288 | available = true 289 | break 290 | } 291 | } 292 | return 293 | } 294 | ``` 295 | 296 | 对于新抓取代理IP的检测使用的是定时检测的方式，在每次对当前新增代理列表检测完成之后，等待1分钟后再进去下一次检测。 297 | 298 | ```go 299 | // detectProxyList detect proxy list 300 | func (c *Crawler) detectProxyList(list []*Proxy) (availableList []*Proxy, unavailableList []*Proxy) { 301 | availableList = make([]*Proxy, 0) 302 | unavailableList = make([]*Proxy, 0) 303 | w := sync.WaitGroup{} 304 | // 控制最多检测proxy的数量 305 | chans := make(chan bool, 5) 306 | for _, item := range list { 307 | w.Add(1) 308 | go func(p *Proxy) { 309 | chans <- true 310 | avaliable := c.analyze(p) 311 | atomic.StoreInt64(&p.DetectedAt, time.Now().Unix()) 312 | if avaliable { 313 | availableList = append(availableList, p) 314 | } else { 315 | unavailableList = append(unavailableList, p) 316 | } 317 | <-chans 318 | w.Done() 319 | }(item) 320 | } 321 | w.Wait() 322 | return 323 | } 324 | 325 | // detectNewProxy detect the new proxy is avaliable 326 | func (c *Crawler) detectNewProxy() { 327 | old := atomic.SwapInt32(&c.newProxyDetectStatus, detectRunning) 328 | // 如果已经在运行中，则直接退出 329 | if old == detectRunning { 330 | return 331 | } 332 | proxyList := c.newProxyList.Reset() 333 | availableList, _ := c.detectProxyList(proxyList) 334 | c.avaliableProxyList.Add(availableList...) 335 | 336 | atomic.StoreInt32(&c.newProxyDetectStatus, detectStop) 337 | // 等待1分钟后，重新运行detect new proxy 338 | time.Sleep(time.Minute) 339 | c.detectNewProxy() 340 | } 341 | ``` 342 | 343 | 而对于可用代理IP列表，也需要定期去检测是否可还是可用，在多次检测均失败则认为此代理也不可用，从可用代理列表中删除，逻辑如下： 344 | 345 | ```go 346 | // RedetectAvailableProxy redetect available proxy 347 | func (c *Crawler) RedetectAvailableProxy() { 348 | old := atomic.SwapInt32(&c.availableProxyDetectStatus, detectRunning) 349 | // 如果已经在运行中，则直接退出 350 | if old == detectRunning { 351 | return 352 | } 353 | proxyList := c.avaliableProxyList.List() 354 | availableList, unavailableList := c.detectProxyList(proxyList) 355 | 356 | // 如果成功，则重置失败次数 357 | for _, p := range availableList { 358 | atomic.StoreInt32(&p.Fails, 0) 359 | } 360 | // 如果失败，则失败次数+1 361 | failProxyList := make([]*Proxy, 0) 362 | for _, p := range unavailableList { 363 | count := atomic.AddInt32(&p.Fails, 1) 364 | if count >= 3 { 365 | failProxyList = append(failProxyList, p) 366 | } 367 | } 368 | // 对于三次检测失败的代理则删除 369 | c.avaliableProxyList.Remove(failProxyList...) 370 | 371 | atomic.StoreInt32(&c.availableProxyDetectStatus, detectStop) 372 | } 373 | ``` 374 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/vicanso/proxy-pool 2 | 3 | go 1.14 4 | 5 | require ( 6 | github.com/PuerkitoBio/goquery v1.5.1 7 | github.com/gobuffalo/packr/v2 v2.8.0 8 | github.com/spf13/viper v1.6.3 9 | github.com/stretchr/testify v1.5.1 10 | github.com/vicanso/elton v0.5.0 11 | github.com/vicanso/go-axios v0.1.0 12 | go.uber.org/zap v1.14.1 13 | ) 14 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 3 | github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= 4 | github.com/PuerkitoBio/goquery v1.5.0 h1:uGvmFXOA73IKluu/F84Xd1tt/z07GYm8X49XKHP7EJk= 5 | github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg= 6 | github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE= 7 | github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= 8 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 9 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 10 | github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDafo4= 11 | github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= 12 | github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o= 13 | github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= 14 | github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo= 15 | github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= 16 | github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= 17 | github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= 18 | github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= 19 | github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= 20 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 21 | github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= 22 | github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= 23 | github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= 24 | github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= 25 | github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= 26 | github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= 27 | github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= 28 | github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= 29 | github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= 30 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 31 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 32 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 33 | github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= 34 | github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= 35 | github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= 36 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 37 | github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= 38 | github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 39 | github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= 40 | github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= 41 | github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 42 | github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= 43 | github.com/gobuffalo/envy v1.7.1 h1:OQl5ys5MBea7OGCdvPbBJWRgnhC/fGona6QKfvFeau8= 44 | github.com/gobuffalo/envy v1.7.1/go.mod h1:FurDp9+EDPE4aIUS3ZLyD+7/9fpx7YRt/ukY6jIHf0w= 45 | github.com/gobuffalo/logger v1.0.1 h1:ZEgyRGgAm4ZAhAO45YXMs5Fp+bzGLESFewzAVBMKuTg= 46 | github.com/gobuffalo/logger v1.0.1/go.mod h1:2zbswyIUa45I+c+FLXuWl9zSWEiVuthsk8ze5s8JvPs= 47 | github.com/gobuffalo/logger v1.0.3 h1:YaXOTHNPCvkqqA7w05A4v0k2tCdpr+sgFlgINbQ6gqc= 48 | github.com/gobuffalo/logger v1.0.3/go.mod h1:SoeejUwldiS7ZsyCBphOGURmWdwUFXs0J7TCjEhjKxM= 49 | github.com/gobuffalo/packd v0.3.0 h1:eMwymTkA1uXsqxS0Tpoop3Lc0u3kTfiMBE6nKtQU4g4= 50 | github.com/gobuffalo/packd v0.3.0/go.mod h1:zC7QkmNkYVGKPw4tHpBQ+ml7W/3tIebgeo1b36chA3Q= 51 | github.com/gobuffalo/packd v1.0.0 h1:6ERZvJHfe24rfFmA9OaoKBdC7+c9sydrytMg8SdFGBM= 52 | github.com/gobuffalo/packd v1.0.0/go.mod h1:6VTc4htmJRFB7u1m/4LeMTWjFoYrUiBkU9Fdec9hrhI= 53 | github.com/gobuffalo/packr v1.30.1 h1:hu1fuVR3fXEZR7rXNW3h8rqSML8EVAf6KNm0NKO/wKg= 54 | github.com/gobuffalo/packr/v2 v2.7.1 h1:n3CIW5T17T8v4GGK5sWXLVWJhCz7b5aNLSxW6gYim4o= 55 | github.com/gobuffalo/packr/v2 v2.7.1/go.mod h1:qYEvAazPaVxy7Y7KR0W8qYEE+RymX74kETFqjFoFlOc= 56 | github.com/gobuffalo/packr/v2 v2.8.0 h1:IULGd15bQL59ijXLxEvA5wlMxsmx/ZkQv9T282zNVIY= 57 | github.com/gobuffalo/packr/v2 v2.8.0/go.mod h1:PDk2k3vGevNE3SwVyVRgQCCXETC9SaONCNSXT1Q8M1g= 58 | github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 59 | github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= 60 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 61 | github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 62 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 63 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 64 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 65 | github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 66 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 67 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 68 | github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= 69 | github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= 70 | github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= 71 | github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= 72 | github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= 73 | github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= 74 | github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= 75 | github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= 76 | github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= 77 | github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= 78 | github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= 79 | github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= 80 | github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= 81 | github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 82 | github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= 83 | github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= 84 | github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= 85 | github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= 86 | github.com/karrick/godirwalk v1.15.3 h1:0a2pXOgtB16CqIqXTiT7+K9L73f74n/aNQUnH6Ortew= 87 | github.com/karrick/godirwalk v1.15.3/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= 88 | github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= 89 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 90 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 91 | github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 92 | github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= 93 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 94 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 95 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 96 | github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= 97 | github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= 98 | github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= 99 | github.com/markbates/errx v1.1.0 h1:QDFeR+UP95dO12JgW+tgi2UVfo0V8YBHiUIOaeBPiEI= 100 | github.com/markbates/errx v1.1.0/go.mod h1:PLa46Oex9KNbVDZhKel8v1OT7hD5JZ2eI7AHhA0wswc= 101 | github.com/markbates/oncer v1.0.0 h1:E83IaVAHygyndzPimgUYJjbshhDTALZyXxvk9FOlQRY= 102 | github.com/markbates/oncer v1.0.0/go.mod h1:Z59JA581E9GP6w96jai+TGqafHPW+cPfRxz2aSZ0mcI= 103 | github.com/markbates/safe v1.0.1 h1:yjZkbvRM6IzKj9tlu/zMJLS0n/V351OZWRnF3QfaUxI= 104 | github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= 105 | github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= 106 | github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= 107 | github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= 108 | github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= 109 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 110 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 111 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 112 | github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= 113 | github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= 114 | github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= 115 | github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= 116 | github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= 117 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 118 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 119 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 120 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 121 | github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= 122 | github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= 123 | github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= 124 | github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 125 | github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= 126 | github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= 127 | github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 128 | github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= 129 | github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= 130 | github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= 131 | github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= 132 | github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= 133 | github.com/rogpeppe/go-internal v1.3.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= 134 | github.com/rogpeppe/go-internal v1.4.0 h1:LUa41nrWTQNGhzdsZ5lTnkwbNjj6rXTdazA1cSdjkOY= 135 | github.com/rogpeppe/go-internal v1.4.0/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= 136 | github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= 137 | github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= 138 | github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 139 | github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= 140 | github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= 141 | github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= 142 | github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= 143 | github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= 144 | github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= 145 | github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= 146 | github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= 147 | github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= 148 | github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= 149 | github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= 150 | github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= 151 | github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= 152 | github.com/spf13/cobra v0.0.6/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= 153 | github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= 154 | github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= 155 | github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= 156 | github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 157 | github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= 158 | github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= 159 | github.com/spf13/viper v1.6.1 h1:VPZzIkznI1YhVMRi6vNFLHSwhnhReBfgTxIPccpfdZk= 160 | github.com/spf13/viper v1.6.1/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= 161 | github.com/spf13/viper v1.6.3 h1:pDDu1OyEDTKzpJwdq4TiuLyMsUgRa/BT5cn5O62NoHs= 162 | github.com/spf13/viper v1.6.3/go.mod h1:jUMtyi0/lB5yZH/FjyGAoH7IMNrIhlBf6pXZmbMDvzw= 163 | github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= 164 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 165 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 166 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 167 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 168 | github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= 169 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 170 | github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= 171 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= 172 | github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= 173 | github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= 174 | github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= 175 | github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= 176 | github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= 177 | github.com/vicanso/elton v0.2.2 h1:MZ5nfJFKBWDWnFPO8wRyPat8kZz3KoNBY0scemo7RFQ= 178 | github.com/vicanso/elton v0.2.2/go.mod h1:QFZ+Un4LLBANtl0mExkqLD4uqw3JLA2ZCWUHaCsHOUg= 179 | github.com/vicanso/elton v0.5.0 h1:MOx/lW6jy3Q0j+mX5jFiFp1gmGISB8nQzit/iIhpc8I= 180 | github.com/vicanso/elton v0.5.0/go.mod h1:oNCU0zM8ADN7J192eND0NMg7jfXRXVgEnLIhztkoZBc= 181 | github.com/vicanso/elton-responder v0.1.5 h1:hESrm1cpUaR4O6IU1oiKLvpLLyJq63teyO2rC9NBTnI= 182 | github.com/vicanso/elton-responder v0.1.5/go.mod h1:ySEKOvVIoxvaQvj+IlTHOpyuvUj7mZbAoUkW3BccO4s= 183 | github.com/vicanso/elton-static-serve v0.1.1 h1:45dNFHJl8GYIx322X4QS4ttBSwpTH5iWOE7r0byy1sc= 184 | github.com/vicanso/elton-static-serve v0.1.1/go.mod h1:baWS93rcTi8WjK9xB+fbmW3YNWOebrY/n6RRWRILd5s= 185 | github.com/vicanso/fresh v0.1.1 h1:L3UX/g2uZ4pNFzjc7+PyeYOlIAs2/yUfhcFnK7E+JQU= 186 | github.com/vicanso/fresh v0.1.1/go.mod h1:gr1RKSFxQ1OnQHzUMBHCigifni7KrXveJjWCTlPjICA= 187 | github.com/vicanso/go-axios v0.0.6 h1:05IrkbNDkDNynlBR/TRHehY2VfHb4cpK88jV+58wknM= 188 | github.com/vicanso/go-axios v0.0.6/go.mod h1:o3WVwN1ocD5S2SLQpiaE6qWbr6xwfuzMnJx4S1mYVXw= 189 | github.com/vicanso/go-axios v0.1.0 h1:GllDZ6wemdb0UDUf7LXg0678zg7TYFRhjKNIMB68rkY= 190 | github.com/vicanso/go-axios v0.1.0/go.mod h1:o3WVwN1ocD5S2SLQpiaE6qWbr6xwfuzMnJx4S1mYVXw= 191 | github.com/vicanso/hes v0.2.1 h1:jRFEADmiQ30koVY/sKwlkhyXM5B3QbVVizLqrjNJgPw= 192 | github.com/vicanso/hes v0.2.1/go.mod h1:QcxOFmFfBQMhASTaLgnFayXYCgevdSeBVprt+o+3eKo= 193 | github.com/vicanso/http-trace v0.0.1 h1:2aAbVm8l8qpOeoY3YwGwXZWD5YvWKP7lVyMPDCqB3Qw= 194 | github.com/vicanso/http-trace v0.0.1/go.mod h1:wuw5xL5PkzFjlhdhpzeO4R9ehpvO0nl5g9o1tBl0JJg= 195 | github.com/vicanso/intranet-ip v0.0.1 h1:cYS+mExFsKqewWSuHtFwAqw/CO66GsheB/P1BPmSTx0= 196 | github.com/vicanso/intranet-ip v0.0.1/go.mod h1:bqQ6VUhxdz0ipSb1kzd6aoZStlp+pB7CTlVmVhgLAxA= 197 | github.com/vicanso/keygrip v0.1.0 h1:/zYzoVIbREAvaxSM7bo3/oSXuuYztaP71dPBfhRoNkM= 198 | github.com/vicanso/keygrip v0.1.0/go.mod h1:cI05iOjY00NJ7oH2Z9Zdm9eJPUkpoex3XnEubK78nho= 199 | github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= 200 | github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= 201 | go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= 202 | go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= 203 | go.uber.org/atomic v1.5.0 h1:OI5t8sDa1Or+q8AeE+yKeB/SDYioSHAgcVljj9JIETY= 204 | go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= 205 | go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= 206 | go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= 207 | go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= 208 | go.uber.org/multierr v1.3.0 h1:sFPn2GLc3poCkfrpIXGhBD2X0CMIo4Q/zSULXrj/+uc= 209 | go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= 210 | go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= 211 | go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= 212 | go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= 213 | go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= 214 | go.uber.org/zap v1.13.0 h1:nR6NoDBgAf67s68NhaXbsojM+2gxp3S1hWkHDl27pVU= 215 | go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= 216 | go.uber.org/zap v1.14.1 h1:nYDKopTbvAPq/NrUVZwT15y2lpROBiLLyoRTbXOYWOo= 217 | go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= 218 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 219 | golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 220 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 221 | golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 222 | golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4 h1:ydJNl0ENAG67pFbB+9tfhiL2pYqLhfoaZFw/cjLhY4A= 223 | golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 224 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 225 | golang.org/x/crypto v0.0.0-20191122220453-ac88ee75c92c h1:/nJuwDLoL/zrqY6gf57vxC+Pi+pZ8bfhpPkicO5H7W4= 226 | golang.org/x/crypto v0.0.0-20191122220453-ac88ee75c92c/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 227 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 228 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 229 | golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 230 | golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= 231 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 232 | golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 233 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 234 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a h1:gOpx8G595UYyvj8UK4+OFyY4rx037g3fmfhe5SasG3U= 235 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 236 | golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 237 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 238 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 239 | golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= 240 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= 241 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 242 | golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 243 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8= 244 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 245 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 246 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 247 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 248 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 249 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 250 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 251 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 252 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 253 | golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 254 | golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 255 | golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 256 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 257 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 258 | golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= 259 | golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 260 | golang.org/x/sys v0.0.0-20190515120540-06a5c4944438 h1:khxRGsvPk4n2y8I/mLLjp7e5dMTJmH75wvqS6nMwUtY= 261 | golang.org/x/sys v0.0.0-20190515120540-06a5c4944438/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 262 | golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= 263 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 264 | golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 265 | golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 266 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 267 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 268 | golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 269 | golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 270 | golang.org/x/tools v0.0.0-20191004055002-72853e10c5a3/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 271 | golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 272 | golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 273 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 274 | golang.org/x/tools v0.0.0-20200308013534-11ec41452d41/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= 275 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 276 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 277 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 278 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 279 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 280 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 281 | google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= 282 | gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= 283 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 284 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 285 | gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= 286 | gopkg.in/h2non/gock.v1 v1.0.15/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE= 287 | gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= 288 | gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= 289 | gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= 290 | gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= 291 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 292 | gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= 293 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 294 | gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= 295 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 296 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 297 | honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= 298 | -------------------------------------------------------------------------------- /log/log.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 tree xie 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package log 16 | 17 | import ( 18 | "go.uber.org/zap" 19 | "go.uber.org/zap/zapcore" 20 | ) 21 | 22 | var ( 23 | defaultLogger *zap.Logger 24 | ) 25 | 26 | func init() { 27 | c := zap.NewProductionConfig() 28 | c.DisableCaller = true 29 | c.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder 30 | // 只针对panic 以上的日志增加stack trace 31 | l, err := c.Build(zap.AddStacktrace(zap.DPanicLevel)) 32 | if err != nil { 33 | panic(err) 34 | } 35 | defaultLogger = l 36 | } 37 | 38 | // Default get default logger 39 | func Default() *zap.Logger { 40 | return defaultLogger 41 | } 42 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/vicanso/elton" 5 | "github.com/vicanso/elton/middleware" 6 | "github.com/vicanso/proxy-pool/config" 7 | _ "github.com/vicanso/proxy-pool/controller" 8 | "github.com/vicanso/proxy-pool/log" 9 | "github.com/vicanso/proxy-pool/router" 10 | "go.uber.org/zap" 11 | ) 12 | 13 | func main() { 14 | logger := log.Default() 15 | e := elton.New() 16 | 17 | e.Use(func(c *elton.Context) error { 18 | c.NoCache() 19 | return c.Next() 20 | }) 21 | e.Use(middleware.NewDefaultResponder()) 22 | 23 | router.Init(e) 24 | addr := config.GetListenAddr() 25 | logger.Info("start to linstening...", 26 | zap.String("listen", addr), 27 | ) 28 | err := e.ListenAndServe(addr) 29 | if err != nil { 30 | panic(err) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /router/router.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 tree xie 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package router 16 | 17 | import ( 18 | "github.com/vicanso/elton" 19 | ) 20 | 21 | var ( 22 | // groupList 路由组列表 23 | groupList = make([]*elton.Group, 0) 24 | ) 25 | 26 | // NewGroup new router group 27 | func NewGroup(path string, handlerList ...elton.Handler) *elton.Group { 28 | // 如果配置文件中有配置路由 29 | g := elton.NewGroup(path, handlerList...) 30 | groupList = append(groupList, g) 31 | return g 32 | } 33 | 34 | // Init init router 35 | func Init(d *elton.Elton) { 36 | for _, g := range groupList { 37 | d.AddGroup(g) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /runner.conf: -------------------------------------------------------------------------------- 1 | root: . 2 | tmp_path: ./tmp 3 | build_name: proxypool 4 | build_log: proxypool-errors.log 5 | valid_ext: .go, .tpl, .tmpl, .html 6 | no_rebuild_ext: .tpl, .tmpl, .html 7 | ignored: tmp, admin, vendor, web 8 | build_delay: 600 9 | colors: 1 10 | log_color_main: cyan 11 | log_color_build: yellow 12 | log_color_runner: green 13 | log_color_watcher: magenta 14 | -------------------------------------------------------------------------------- /script/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | 4 | if [ "${1:0:1}" = '-' ]; then 5 | set -- proxypool "$@" 6 | fi 7 | 8 | exec "$@" -------------------------------------------------------------------------------- /service/proxy.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 tree xie 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package service 16 | 17 | import ( 18 | "errors" 19 | "time" 20 | 21 | "github.com/vicanso/proxy-pool/config" 22 | "github.com/vicanso/proxy-pool/crawler" 23 | ) 24 | 25 | var ( 26 | defaultCrawler = new(crawler.Crawler) 27 | ) 28 | 29 | func init() { 30 | crawlerProxyList := make([]crawler.ProxyCrawler, 0) 31 | for _, item := range config.GetCrawlers() { 32 | interval := item.Interval 33 | var c crawler.ProxyCrawler 34 | switch item.Name { 35 | case crawler.ProxyIP66: 36 | ip66 := crawler.NewIP66Proxy(interval) 37 | ip66.LimitMaxPage(item.MaxPage) 38 | c = ip66 39 | case crawler.ProxyKuai: 40 | kuai := crawler.NewKuaiProxy(interval) 41 | kuai.LimitMaxPage(item.MaxPage) 42 | c = kuai 43 | default: 44 | xici := crawler.NewXiciProxy(interval) 45 | xici.LimitMaxPage(item.MaxPage) 46 | c = xici 47 | } 48 | crawlerProxyList = append(crawlerProxyList, c) 49 | } 50 | if len(crawlerProxyList) == 0 { 51 | panic(errors.New("no proxy crawler")) 52 | } 53 | defaultCrawler.Start(crawlerProxyList...) 54 | go func() { 55 | detectConfig := config.GetDetect() 56 | for range time.NewTicker(detectConfig.Interval).C { 57 | defaultCrawler.RedetectAvailableProxy() 58 | } 59 | }() 60 | } 61 | 62 | // GetAvailableProxyList get available proxy lsit 63 | func GetAvailableProxyList() []*crawler.Proxy { 64 | return defaultCrawler.GetAvailableProxyList() 65 | } 66 | 67 | // GetAvailableProxy get available proxy 68 | func GetAvailableProxy(category string, speed int) *crawler.Proxy { 69 | return defaultCrawler.GetAvailableProxy(category, int32(speed)) 70 | } 71 | -------------------------------------------------------------------------------- /web/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vicanso/proxy-pool/fa848accebac33a2fc102f173bdd7a9bdbd9a395/web/.gitignore -------------------------------------------------------------------------------- /web/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "web", 3 | "version": "0.1.0", 4 | "private": true, 5 | "proxy": "http://localhost:4000", 6 | "dependencies": { 7 | "@testing-library/jest-dom": "^4.2.4", 8 | "@testing-library/react": "^9.3.2", 9 | "@testing-library/user-event": "^7.1.2", 10 | "antd": "^3.26.3", 11 | "axios": "^0.19.0", 12 | "react": "^16.12.0", 13 | "react-dom": "^16.12.0", 14 | "react-scripts": "3.3.0" 15 | }, 16 | "scripts": { 17 | "format": "node node_modules/.bin/prettier --write src/*.js src/**/*.js src/**/**/*.js", 18 | "start": "react-scripts start", 19 | "build": "react-scripts build", 20 | "test": "react-scripts test", 21 | "eject": "react-scripts eject" 22 | }, 23 | "eslintConfig": { 24 | "extends": "react-app" 25 | }, 26 | "browserslist": { 27 | "production": [ 28 | ">0.2%", 29 | "not dead", 30 | "not op_mini all" 31 | ], 32 | "development": [ 33 | "last 1 chrome version", 34 | "last 1 firefox version", 35 | "last 1 safari version" 36 | ] 37 | }, 38 | "devDependencies": { 39 | "node-sass": "^4.13.0", 40 | "prettier": "^1.19.1" 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /web/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vicanso/proxy-pool/fa848accebac33a2fc102f173bdd7a9bdbd9a395/web/public/favicon.ico -------------------------------------------------------------------------------- /web/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 12 | 13 | 17 | 18 | 27 | Free Proxy 28 | 37 | 38 | 39 | 40 |
41 | 51 | 52 | 53 | -------------------------------------------------------------------------------- /web/public/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "short_name": "React App", 3 | "name": "Create React App Sample", 4 | "icons": [ 5 | { 6 | "src": "favicon.ico", 7 | "sizes": "64x64 32x32 24x24 16x16", 8 | "type": "image/x-icon" 9 | }, 10 | { 11 | "src": "logo192.png", 12 | "type": "image/png", 13 | "sizes": "192x192" 14 | }, 15 | { 16 | "src": "logo512.png", 17 | "type": "image/png", 18 | "sizes": "512x512" 19 | } 20 | ], 21 | "start_url": ".", 22 | "display": "standalone", 23 | "theme_color": "#000000", 24 | "background_color": "#ffffff" 25 | } 26 | -------------------------------------------------------------------------------- /web/public/robots.txt: -------------------------------------------------------------------------------- 1 | # https://www.robotstxt.org/robotstxt.html 2 | User-agent: * 3 | -------------------------------------------------------------------------------- /web/src/App.js: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import axios from "axios"; 3 | import { 4 | Spin, 5 | message, 6 | Table, 7 | Icon, 8 | Row, 9 | Col, 10 | Select, 11 | Button, 12 | Card 13 | } from "antd"; 14 | 15 | import "./App.sass"; 16 | 17 | const { Option } = Select; 18 | const oneProxyURL = window.location.origin + "/proxies/one"; 19 | 20 | // copy 将内容复制至粘贴板 21 | export function copy(value, parent) { 22 | if (!document.execCommand) { 23 | return new Error("The browser isn't support copy."); 24 | } 25 | const input = document.createElement("input"); 26 | input.value = value; 27 | if (parent) { 28 | parent.appendChild(input); 29 | } else { 30 | document.body.appendChild(input); 31 | } 32 | input.focus(); 33 | input.select(); 34 | document.execCommand("Copy", false, null); 35 | input.remove(); 36 | } 37 | 38 | class App extends React.Component { 39 | state = { 40 | pageSize: 10, 41 | total: 0, 42 | loading: true, 43 | originalProxies: null, 44 | oneProxyURL, 45 | selectCategory: "", 46 | selectSpeed: "", 47 | proxies: null 48 | }; 49 | async componentDidMount() { 50 | this.setState({ 51 | loading: true 52 | }); 53 | try { 54 | const { data } = await axios.get("/proxies"); 55 | const proxies = data.proxies || []; 56 | proxies.forEach(item => { 57 | item.speed = item.speed || 0; 58 | }); 59 | proxies.sort((a, b) => { 60 | return b.detectedAt - a.detectedAt; 61 | }); 62 | this.setState({ 63 | proxies, 64 | originalProxies: proxies 65 | }); 66 | } catch (err) { 67 | message.error(err.message); 68 | } finally { 69 | this.setState({ 70 | loading: false 71 | }); 72 | } 73 | } 74 | handleChange(pagination, filters, sorter) { 75 | const { originalProxies } = this.state; 76 | if (!originalProxies) { 77 | return; 78 | } 79 | let arr = []; 80 | const filterKeys = Object.keys(filters); 81 | originalProxies.forEach(item => { 82 | if (filterKeys.length === 0) { 83 | arr.push(item); 84 | return; 85 | } 86 | let matched = true; 87 | filterKeys.forEach(key => { 88 | const values = filters[key]; 89 | if (!values || values.length === 0) { 90 | return; 91 | } 92 | if (!filters[key].includes(item[key])) { 93 | matched = false; 94 | } 95 | }); 96 | if (matched) { 97 | arr.push(item); 98 | } 99 | }); 100 | const { field, order } = sorter; 101 | arr.sort((a, b) => { 102 | return a[field] - b[field]; 103 | }); 104 | if (order === "descend") { 105 | arr = arr.reverse(); 106 | } 107 | 108 | this.setState({ 109 | proxies: arr 110 | }); 111 | } 112 | renderAvailablePorxySelector() { 113 | const { loading, oneProxyURL, selectCategory, selectSpeed } = this.state; 114 | if (loading) { 115 | return; 116 | } 117 | let requestUrl = oneProxyURL; 118 | const arr = []; 119 | if (selectCategory) { 120 | arr.push(`category=${selectCategory}`); 121 | } 122 | if (selectSpeed) { 123 | arr.push(`spped=${selectSpeed}`); 124 | } 125 | if (arr.length !== 0) { 126 | requestUrl += `?${arr.join("&")}`; 127 | } 128 | return ( 129 | 130 |

Select cateogry and speed to generate the request.

131 | 132 | 133 | 151 | 152 | 153 | 168 | 169 | 170 | 186 | 187 | 188 | 189 | ); 190 | } 191 | renderProxyList() { 192 | const { loading, proxies, pageSize } = this.state; 193 | if (loading) { 194 | return; 195 | } 196 | const columns = [ 197 | { 198 | title: "IP", 199 | dataIndex: "ip", 200 | key: "ip" 201 | }, 202 | { 203 | title: "Port", 204 | dataIndex: "port", 205 | key: "port" 206 | }, 207 | { 208 | title: "Speed", 209 | dataIndex: "speed", 210 | key: "speed", 211 | width: 120, 212 | sorter: true 213 | }, 214 | { 215 | title: "Type", 216 | dataIndex: "category", 217 | key: "category", 218 | width: 100, 219 | filters: [ 220 | { text: "http", value: "http" }, 221 | { text: "https", value: "https" } 222 | ] 223 | }, 224 | { 225 | title: "Anonymous", 226 | dataIndex: "anonymous", 227 | key: "anonymous", 228 | width: 130, 229 | render: v => { 230 | if (v) { 231 | return "YES"; 232 | } 233 | return "NO"; 234 | } 235 | }, 236 | { 237 | title: "DetectedAt", 238 | dataIndex: "detectedAt", 239 | key: "detectedAt", 240 | sorter: true, 241 | render: v => { 242 | const date = new Date(v * 1000); 243 | return `${date.toLocaleDateString()} ${date.toLocaleTimeString()}`; 244 | } 245 | } 246 | ]; 247 | return ( 248 | { 255 | this.setState({ 256 | pageSize: size 257 | }); 258 | } 259 | }} 260 | columns={columns} 261 | dataSource={proxies} 262 | onChange={this.handleChange.bind(this)} 263 | /> 264 | ); 265 | } 266 | render() { 267 | const { loading } = this.state; 268 | return ( 269 |
270 |
271 | 272 | Free Proxy 273 |
274 | {loading &&
279 | 280 |
} 281 |
282 | {this.renderAvailablePorxySelector()} 283 | {this.renderProxyList()} 284 |
285 |
286 | ); 287 | } 288 | } 289 | 290 | export default App; 291 | -------------------------------------------------------------------------------- /web/src/App.sass: -------------------------------------------------------------------------------- 1 | .App 2 | .header 3 | font-size: 18px 4 | padding-left: 15px 5 | line-height: 60px 6 | background-color: #fff 7 | box-shadow: 0 1px 4px rgba(0,21,41,.08) 8 | text-transform: uppercase 9 | i 10 | font-size: 24px 11 | margin-right: 5px 12 | color: #1890ff 13 | .contentWrapper 14 | margin: 10px 15 | .proxySelector 16 | margin: 10px 0 17 | table 18 | background-color: #f0f0f0 19 | border: 1px solid #e8e8e8 20 | tr:nth-child(even) 21 | background-color: #fff 22 | -------------------------------------------------------------------------------- /web/src/App.test.js: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import { render } from "@testing-library/react"; 3 | import App from "./App"; 4 | 5 | test("renders learn react link", () => { 6 | const { getByText } = render(); 7 | const linkElement = getByText(/learn react/i); 8 | expect(linkElement).toBeInTheDocument(); 9 | }); 10 | -------------------------------------------------------------------------------- /web/src/index.js: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import ReactDOM from "react-dom"; 3 | import "antd/dist/antd.css"; 4 | import "./index.sass"; 5 | import App from "./App"; 6 | import * as serviceWorker from "./serviceWorker"; 7 | import "./request-interceptors"; 8 | 9 | ReactDOM.render(, document.getElementById("root")); 10 | 11 | // If you want your app to work offline and load faster, you can change 12 | // unregister() to register() below. Note this comes with some pitfalls. 13 | // Learn more about service workers: https://bit.ly/CRA-PWA 14 | serviceWorker.unregister(); 15 | -------------------------------------------------------------------------------- /web/src/index.sass: -------------------------------------------------------------------------------- 1 | body, html 2 | height: 100% 3 | 4 | body 5 | background-color: #f0f2f5 6 | color: #333 7 | font-family: Open Sans,Helvetica Neue,Arial,Verdana,sans-serif 8 | margin: 0 9 | 10 | * 11 | box-sizing: border-box 12 | &:before, &:after 13 | box-sizing: border-box 14 | 15 | .clearfix:before, .clearfix:after 16 | display: table 17 | content: " " 18 | .clearfix:after 19 | clear: both 20 | 21 | .loadingWrapper 22 | width: 100% 23 | text-align: center 24 | padding: 50px -------------------------------------------------------------------------------- /web/src/request-interceptors.js: -------------------------------------------------------------------------------- 1 | import request from "axios"; 2 | 3 | request.interceptors.request.use(config => { 4 | if (!config.timeout) { 5 | config.timeout = 10 * 1000; 6 | } 7 | return config; 8 | }); 9 | 10 | request.interceptors.response.use(null, err => { 11 | const { response } = err; 12 | if (response) { 13 | if (response.data && response.data.message) { 14 | err.message = response.data.message; 15 | } else { 16 | err.message = `unknown error[${response.statusCode || -1}]`; 17 | } 18 | } 19 | return Promise.reject(err); 20 | }); 21 | -------------------------------------------------------------------------------- /web/src/serviceWorker.js: -------------------------------------------------------------------------------- 1 | // This optional code is used to register a service worker. 2 | // register() is not called by default. 3 | 4 | // This lets the app load faster on subsequent visits in production, and gives 5 | // it offline capabilities. However, it also means that developers (and users) 6 | // will only see deployed updates on subsequent visits to a page, after all the 7 | // existing tabs open on the page have been closed, since previously cached 8 | // resources are updated in the background. 9 | 10 | // To learn more about the benefits of this model and instructions on how to 11 | // opt-in, read https://bit.ly/CRA-PWA 12 | 13 | const isLocalhost = Boolean( 14 | window.location.hostname === "localhost" || 15 | // [::1] is the IPv6 localhost address. 16 | window.location.hostname === "[::1]" || 17 | // 127.0.0.0/8 are considered localhost for IPv4. 18 | window.location.hostname.match( 19 | /^127(?:\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}$/ 20 | ) 21 | ); 22 | 23 | export function register(config) { 24 | if (process.env.NODE_ENV === "production" && "serviceWorker" in navigator) { 25 | // The URL constructor is available in all browsers that support SW. 26 | const publicUrl = new URL(process.env.PUBLIC_URL, window.location.href); 27 | if (publicUrl.origin !== window.location.origin) { 28 | // Our service worker won't work if PUBLIC_URL is on a different origin 29 | // from what our page is served on. This might happen if a CDN is used to 30 | // serve assets; see https://github.com/facebook/create-react-app/issues/2374 31 | return; 32 | } 33 | 34 | window.addEventListener("load", () => { 35 | const swUrl = `${process.env.PUBLIC_URL}/service-worker.js`; 36 | 37 | if (isLocalhost) { 38 | // This is running on localhost. Let's check if a service worker still exists or not. 39 | checkValidServiceWorker(swUrl, config); 40 | 41 | // Add some additional logging to localhost, pointing developers to the 42 | // service worker/PWA documentation. 43 | navigator.serviceWorker.ready.then(() => { 44 | console.log( 45 | "This web app is being served cache-first by a service " + 46 | "worker. To learn more, visit https://bit.ly/CRA-PWA" 47 | ); 48 | }); 49 | } else { 50 | // Is not localhost. Just register service worker 51 | registerValidSW(swUrl, config); 52 | } 53 | }); 54 | } 55 | } 56 | 57 | function registerValidSW(swUrl, config) { 58 | navigator.serviceWorker 59 | .register(swUrl) 60 | .then(registration => { 61 | registration.onupdatefound = () => { 62 | const installingWorker = registration.installing; 63 | if (installingWorker == null) { 64 | return; 65 | } 66 | installingWorker.onstatechange = () => { 67 | if (installingWorker.state === "installed") { 68 | if (navigator.serviceWorker.controller) { 69 | // At this point, the updated precached content has been fetched, 70 | // but the previous service worker will still serve the older 71 | // content until all client tabs are closed. 72 | console.log( 73 | "New content is available and will be used when all " + 74 | "tabs for this page are closed. See https://bit.ly/CRA-PWA." 75 | ); 76 | 77 | // Execute callback 78 | if (config && config.onUpdate) { 79 | config.onUpdate(registration); 80 | } 81 | } else { 82 | // At this point, everything has been precached. 83 | // It's the perfect time to display a 84 | // "Content is cached for offline use." message. 85 | console.log("Content is cached for offline use."); 86 | 87 | // Execute callback 88 | if (config && config.onSuccess) { 89 | config.onSuccess(registration); 90 | } 91 | } 92 | } 93 | }; 94 | }; 95 | }) 96 | .catch(error => { 97 | console.error("Error during service worker registration:", error); 98 | }); 99 | } 100 | 101 | function checkValidServiceWorker(swUrl, config) { 102 | // Check if the service worker can be found. If it can't reload the page. 103 | fetch(swUrl, { 104 | headers: { "Service-Worker": "script" } 105 | }) 106 | .then(response => { 107 | // Ensure service worker exists, and that we really are getting a JS file. 108 | const contentType = response.headers.get("content-type"); 109 | if ( 110 | response.status === 404 || 111 | (contentType != null && contentType.indexOf("javascript") === -1) 112 | ) { 113 | // No service worker found. Probably a different app. Reload the page. 114 | navigator.serviceWorker.ready.then(registration => { 115 | registration.unregister().then(() => { 116 | window.location.reload(); 117 | }); 118 | }); 119 | } else { 120 | // Service worker found. Proceed as normal. 121 | registerValidSW(swUrl, config); 122 | } 123 | }) 124 | .catch(() => { 125 | console.log( 126 | "No internet connection found. App is running in offline mode." 127 | ); 128 | }); 129 | } 130 | 131 | export function unregister() { 132 | if ("serviceWorker" in navigator) { 133 | navigator.serviceWorker.ready.then(registration => { 134 | registration.unregister(); 135 | }); 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /web/src/setupTests.js: -------------------------------------------------------------------------------- 1 | // jest-dom adds custom jest matchers for asserting on DOM nodes. 2 | // allows you to do things like: 3 | // expect(element).toHaveTextContent(/react/i) 4 | // learn more: https://github.com/testing-library/jest-dom 5 | import "@testing-library/jest-dom/extend-expect"; 6 | --------------------------------------------------------------------------------