├── .gitignore
├── LICENSE
├── README.md
├── adsl_run.py
├── adsl_server
├── __init__.py
├── main.py
└── settings.py
├── example
├── __init__.py
└── spider.py
├── redis_run.py
├── redis_server
├── __init__.py
├── api.py
├── config.py
└── db.py
└── requirements.txt
/.gitignore:
--------------------------------------------------------------------------------
1 | # Byte-compiled / optimized / DLL files
2 | __pycache__/
3 | *.py[cod]
4 | *$py.class
5 |
6 | # C extensions
7 | *.so
8 |
9 | # Distribution / packaging
10 | .Python
11 | build/
12 | develop-eggs/
13 | dist/
14 | downloads/
15 | eggs/
16 | .eggs/
17 | lib/
18 | lib64/
19 | parts/
20 | sdist/
21 | var/
22 | wheels/
23 | pip-wheel-metadata/
24 | share/python-wheels/
25 | *.egg-info/
26 | .installed.cfg
27 | *.egg
28 | MANIFEST
29 |
30 | # PyInstaller
31 | # Usually these files are written by a python script from a template
32 | # before PyInstaller builds the exe, so as to inject date/other infos into it.
33 | *.manifest
34 | *.spec
35 |
36 | # Installer logs
37 | pip-log.txt
38 | pip-delete-this-directory.txt
39 |
40 | # Unit test / coverage reports
41 | htmlcov/
42 | .tox/
43 | .nox/
44 | .coverage
45 | .coverage.*
46 | .cache
47 | nosetests.xml
48 | coverage.xml
49 | *.cover
50 | *.py,cover
51 | .hypothesis/
52 | .pytest_cache/
53 |
54 | # Translations
55 | *.mo
56 | *.pot
57 |
58 | # Django stuff:
59 | *.log
60 | local_settings.py
61 | db.sqlite3
62 | db.sqlite3-journal
63 |
64 | # Flask stuff:
65 | instance/
66 | .webassets-cache
67 |
68 | # Scrapy stuff:
69 | .scrapy
70 |
71 | # Sphinx documentation
72 | docs/_build/
73 |
74 | # PyBuilder
75 | target/
76 |
77 | # Jupyter Notebook
78 | .ipynb_checkpoints
79 |
80 | # IPython
81 | profile_default/
82 | ipython_config.py
83 |
84 | # pyenv
85 | .python-version
86 |
87 | # pipenv
88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies
90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not
91 | # install all needed dependencies.
92 | #Pipfile.lock
93 |
94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow
95 | __pypackages__/
96 |
97 | # Celery stuff
98 | celerybeat-schedule
99 | celerybeat.pid
100 |
101 | # SageMath parsed files
102 | *.sage.py
103 |
104 | # Environments
105 | .env
106 | .venv
107 | env/
108 | venv/
109 | ENV/
110 | env.bak/
111 | venv.bak/
112 |
113 | # Spyder project settings
114 | .spyderproject
115 | .spyproject
116 |
117 | # Rope project settings
118 | .ropeproject
119 |
120 | # mkdocs documentation
121 | /site
122 |
123 | # mypy
124 | .mypy_cache/
125 | .dmypy.json
126 | dmypy.json
127 |
128 | # Pyre type checker
129 | .pyre/
130 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Apache License
2 | Version 2.0, January 2004
3 | http://www.apache.org/licenses/
4 |
5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6 |
7 | 1. Definitions.
8 |
9 | "License" shall mean the terms and conditions for use, reproduction,
10 | and distribution as defined by Sections 1 through 9 of this document.
11 |
12 | "Licensor" shall mean the copyright owner or entity authorized by
13 | the copyright owner that is granting the License.
14 |
15 | "Legal Entity" shall mean the union of the acting entity and all
16 | other entities that control, are controlled by, or are under common
17 | control with that entity. For the purposes of this definition,
18 | "control" means (i) the power, direct or indirect, to cause the
19 | direction or management of such entity, whether by contract or
20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
21 | outstanding shares, or (iii) beneficial ownership of such entity.
22 |
23 | "You" (or "Your") shall mean an individual or Legal Entity
24 | exercising permissions granted by this License.
25 |
26 | "Source" form shall mean the preferred form for making modifications,
27 | including but not limited to software source code, documentation
28 | source, and configuration files.
29 |
30 | "Object" form shall mean any form resulting from mechanical
31 | transformation or translation of a Source form, including but
32 | not limited to compiled object code, generated documentation,
33 | and conversions to other media types.
34 |
35 | "Work" shall mean the work of authorship, whether in Source or
36 | Object form, made available under the License, as indicated by a
37 | copyright notice that is included in or attached to the work
38 | (an example is provided in the Appendix below).
39 |
40 | "Derivative Works" shall mean any work, whether in Source or Object
41 | form, that is based on (or derived from) the Work and for which the
42 | editorial revisions, annotations, elaborations, or other modifications
43 | represent, as a whole, an original work of authorship. For the purposes
44 | of this License, Derivative Works shall not include works that remain
45 | separable from, or merely link (or bind by name) to the interfaces of,
46 | the Work and Derivative Works thereof.
47 |
48 | "Contribution" shall mean any work of authorship, including
49 | the original version of the Work and any modifications or additions
50 | to that Work or Derivative Works thereof, that is intentionally
51 | submitted to Licensor for inclusion in the Work by the copyright owner
52 | or by an individual or Legal Entity authorized to submit on behalf of
53 | the copyright owner. For the purposes of this definition, "submitted"
54 | means any form of electronic, verbal, or written communication sent
55 | to the Licensor or its representatives, including but not limited to
56 | communication on electronic mailing lists, source code control systems,
57 | and issue tracking systems that are managed by, or on behalf of, the
58 | Licensor for the purpose of discussing and improving the Work, but
59 | excluding communication that is conspicuously marked or otherwise
60 | designated in writing by the copyright owner as "Not a Contribution."
61 |
62 | "Contributor" shall mean Licensor and any individual or Legal Entity
63 | on behalf of whom a Contribution has been received by Licensor and
64 | subsequently incorporated within the Work.
65 |
66 | 2. Grant of Copyright License. Subject to the terms and conditions of
67 | this License, each Contributor hereby grants to You a perpetual,
68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69 | copyright license to reproduce, prepare Derivative Works of,
70 | publicly display, publicly perform, sublicense, and distribute the
71 | Work and such Derivative Works in Source or Object form.
72 |
73 | 3. Grant of Patent License. Subject to the terms and conditions of
74 | this License, each Contributor hereby grants to You a perpetual,
75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76 | (except as stated in this section) patent license to make, have made,
77 | use, offer to sell, sell, import, and otherwise transfer the Work,
78 | where such license applies only to those patent claims licensable
79 | by such Contributor that are necessarily infringed by their
80 | Contribution(s) alone or by combination of their Contribution(s)
81 | with the Work to which such Contribution(s) was submitted. If You
82 | institute patent litigation against any entity (including a
83 | cross-claim or counterclaim in a lawsuit) alleging that the Work
84 | or a Contribution incorporated within the Work constitutes direct
85 | or contributory patent infringement, then any patent licenses
86 | granted to You under this License for that Work shall terminate
87 | as of the date such litigation is filed.
88 |
89 | 4. Redistribution. You may reproduce and distribute copies of the
90 | Work or Derivative Works thereof in any medium, with or without
91 | modifications, and in Source or Object form, provided that You
92 | meet the following conditions:
93 |
94 | (a) You must give any other recipients of the Work or
95 | Derivative Works a copy of this License; and
96 |
97 | (b) You must cause any modified files to carry prominent notices
98 | stating that You changed the files; and
99 |
100 | (c) You must retain, in the Source form of any Derivative Works
101 | that You distribute, all copyright, patent, trademark, and
102 | attribution notices from the Source form of the Work,
103 | excluding those notices that do not pertain to any part of
104 | the Derivative Works; and
105 |
106 | (d) If the Work includes a "NOTICE" text file as part of its
107 | distribution, then any Derivative Works that You distribute must
108 | include a readable copy of the attribution notices contained
109 | within such NOTICE file, excluding those notices that do not
110 | pertain to any part of the Derivative Works, in at least one
111 | of the following places: within a NOTICE text file distributed
112 | as part of the Derivative Works; within the Source form or
113 | documentation, if provided along with the Derivative Works; or,
114 | within a display generated by the Derivative Works, if and
115 | wherever such third-party notices normally appear. The contents
116 | of the NOTICE file are for informational purposes only and
117 | do not modify the License. You may add Your own attribution
118 | notices within Derivative Works that You distribute, alongside
119 | or as an addendum to the NOTICE text from the Work, provided
120 | that such additional attribution notices cannot be construed
121 | as modifying the License.
122 |
123 | You may add Your own copyright statement to Your modifications and
124 | may provide additional or different license terms and conditions
125 | for use, reproduction, or distribution of Your modifications, or
126 | for any such Derivative Works as a whole, provided Your use,
127 | reproduction, and distribution of the Work otherwise complies with
128 | the conditions stated in this License.
129 |
130 | 5. Submission of Contributions. Unless You explicitly state otherwise,
131 | any Contribution intentionally submitted for inclusion in the Work
132 | by You to the Licensor shall be under the terms and conditions of
133 | this License, without any additional terms or conditions.
134 | Notwithstanding the above, nothing herein shall supersede or modify
135 | the terms of any separate license agreement you may have executed
136 | with Licensor regarding such Contributions.
137 |
138 | 6. Trademarks. This License does not grant permission to use the trade
139 | names, trademarks, service marks, or product names of the Licensor,
140 | except as required for reasonable and customary use in describing the
141 | origin of the Work and reproducing the content of the NOTICE file.
142 |
143 | 7. Disclaimer of Warranty. Unless required by applicable law or
144 | agreed to in writing, Licensor provides the Work (and each
145 | Contributor provides its Contributions) on an "AS IS" BASIS,
146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 | implied, including, without limitation, any warranties or conditions
148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 | PARTICULAR PURPOSE. You are solely responsible for determining the
150 | appropriateness of using or redistributing the Work and assume any
151 | risks associated with Your exercise of permissions under this License.
152 |
153 | 8. Limitation of Liability. In no event and under no legal theory,
154 | whether in tort (including negligence), contract, or otherwise,
155 | unless required by applicable law (such as deliberate and grossly
156 | negligent acts) or agreed to in writing, shall any Contributor be
157 | liable to You for damages, including any direct, indirect, special,
158 | incidental, or consequential damages of any character arising as a
159 | result of this License or out of the use or inability to use the
160 | Work (including but not limited to damages for loss of goodwill,
161 | work stoppage, computer failure or malfunction, or any and all
162 | other commercial damages or losses), even if such Contributor
163 | has been advised of the possibility of such damages.
164 |
165 | 9. Accepting Warranty or Additional Liability. While redistributing
166 | the Work or Derivative Works thereof, You may choose to offer,
167 | and charge a fee for, acceptance of support, warranty, indemnity,
168 | or other liability obligations and/or rights consistent with this
169 | License. However, in accepting such obligations, You may act only
170 | on Your own behalf and on Your sole responsibility, not on behalf
171 | of any other Contributor, and only if You agree to indemnify,
172 | defend, and hold each Contributor harmless for any liability
173 | incurred by, or claims asserted against, such Contributor by reason
174 | of your accepting any such warranty or additional liability.
175 |
176 | END OF TERMS AND CONDITIONS
177 |
178 | APPENDIX: How to apply the Apache License to your work.
179 |
180 | To apply the Apache License to your work, attach the following
181 | boilerplate notice, with the fields enclosed by brackets "[]"
182 | replaced with your own identifying information. (Don't include
183 | the brackets!) The text should be enclosed in the appropriate
184 | comment syntax for the file format. We also recommend that a
185 | file or class name and description of purpose be included on the
186 | same "printed page" as the copyright notice for easier
187 | identification within third-party archives.
188 |
189 | Copyright [yyyy] [name of copyright owner]
190 |
191 | Licensed under the Apache License, Version 2.0 (the "License");
192 | you may not use this file except in compliance with the License.
193 | You may obtain a copy of the License at
194 |
195 | http://www.apache.org/licenses/LICENSE-2.0
196 |
197 | Unless required by applicable law or agreed to in writing, software
198 | distributed under the License is distributed on an "AS IS" BASIS,
199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 | See the License for the specific language governing permissions and
201 | limitations under the License.
202 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # AdslProxy
2 |
3 | * adsl拨号代理池
4 | * 支持版本: 
5 |
6 | ### 下载安装
7 |
8 | * 下载源码:
9 |
10 | ```
11 | https://github.com/pythonyhd/AdslProxy
12 | ```
13 |
14 | * 安装依赖:
15 |
16 | ```shell
17 | pip install -r requirements.txt
18 | ```
19 |
20 | ### 使用方法
21 |
22 | 1.购买拨号服务器,确保能正常连接网络
23 |
24 | 2.安装squid,配置代理账号密码高匿https,配置教程
25 |
26 | #### 安装
27 |
28 | yum install squid -y
29 | yum install httpd-tools -y
30 |
31 | #### 生成密码文件
32 |
33 | htpasswd -cd /etc/squid/passwords martindu //账号martindu
34 | fy1812!! //提示输入密码
35 |
36 | #### 生成证书
37 |
38 | cd /etc/squid
39 | openssl req -x509 -nodes -days 3650 -newkey rsa:2048 -keyout private.key -out public.crt
40 |
41 | #### 配置squid.conf文件
42 |
43 | acl指令用来定义访问列表,http_access指令用来定义接收还是拒绝来自acl的访问
44 |
45 | vi /etc/squid/squid.conf
46 |
47 | 在最后添加,添加账号密码,添加账号密码必须注释配置文件中的http_access allow all
48 |
49 | auth_param basic program /usr/lib64/squid/basic_ncsa_auth /etc/squid/passwords
50 | auth_param basic realm welecome //设置认证时返回头里夹带的信息
51 | acl authenticated proxy_auth REQUIRED
52 | http_access allow authenticated
53 |
54 | 设置高匿名
55 |
56 | request_header_access Via deny all
57 | request_header_access X-Forwarded-For deny all
58 | request_header_access From deny all
59 |
60 | 注意:监听的端口号必须设为443.否则squid启动不了。
61 |
62 | https_port 443 cert=/etc/squid/public.crt key=/etc/squid/private.key
63 |
64 | #### 日志
65 |
66 | squid的日志位于/var/log/squid/目录下。
67 |
68 | #### 启动
69 |
70 | systemctl enable squid
71 | yum update openssl
72 | systemctl start squid
73 |
74 | squid -k reconfigure //重新加载配置文件
75 | squid -k parse //验证配置
76 | squid -s //启动
77 |
78 | #### 其他
79 |
80 | service squid start
81 | service squid stop
82 | service squid restart
83 |
84 | 3.利用Flask跟nginx搭建redis,确保拨号机跟公司服务器之间可以通信,能够把拨号机IP存储到公司服务器,具体教程不详述
85 |
86 | 4.测试使用
87 |
88 |
89 |
90 | ### 问题反馈
91 |
92 | 任何问题欢迎在[Issues](https://github.com/pythonyhd/AdslProxy/issues) 中反馈。
93 |
94 | 你的反馈会让此项目变得更加完美。
95 |
96 | ---
97 |
98 | ### TODO
99 | - [x] 兼容py2
100 |
101 | ---
102 |
103 |
104 | ### 赞助作者
105 | 甲鱼说,咖啡是灵魂的饮料,买点咖啡
106 |
107 | [谢谢这些人的☕️](./coffee.md)
108 |
109 | 直接转账打赏作者的辛苦劳动:
110 |
111 |
--------------------------------------------------------------------------------
/adsl_run.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | from adsl_server.main import SenderAdsl
3 |
4 |
5 | def main():
6 | SenderAdsl().adsl()
7 |
8 |
9 | if __name__ == '__main__':
10 | main()
--------------------------------------------------------------------------------
/adsl_server/__init__.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | # @Time : 2019/12/12 11:16
3 | # @Author : King life
4 | # @Email : 18353626676@163.com
5 | # @File : __init__.py.py
6 | # @Software: PyCharm
--------------------------------------------------------------------------------
/adsl_server/main.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | import re
3 | import platform
4 | import time
5 |
6 | import requests
7 | from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
8 | from fake_useragent import UserAgent
9 |
10 | from adsl_server.settings import ADSL_NETMASK, TEST_URL, TEST_TIMEOUT, VALID_STATUS_CODES, CLIENT_NAME, ADSL_BASH, \
11 | PROXY_PORT, USER_NAME, PASSWORD, REDIS_URI, ADSL_ERROR_CYCLE, ADSL_CYCLE, ADSL_KEEP_USE
12 |
13 | if platform.python_version().startswith('2.'):
14 | import commands as subprocess
15 | elif platform.python_version().startswith('3.'):
16 | import subprocess
17 | else:
18 | raise ValueError('python version must be 2 or 3')
19 |
20 |
21 | class SenderAdsl(object):
22 |
23 | def get_ip(self, netname=ADSL_NETMASK):
24 | """
25 | 获取拨号IP
26 | :param netname: 网卡名称
27 | :return: 拨号ip
28 | """
29 | (status, output) = subprocess.getstatusoutput('ifconfig')
30 | if status == 0:
31 | pattern = re.compile(netname + '.*?inet.*?(\d+\.\d+\.\d+\.\d+).*?netmask', re.S)
32 | result = re.search(pattern, output)
33 | if result:
34 | ip = result.group(1)
35 | return ip
36 |
37 | def test_proxy(self, proxy):
38 | """
39 | 测试代理
40 | :param proxy: 代理
41 | :return: 测试结果
42 | """
43 | proxies = {
44 | 'http': 'http://{}:{}@{}:{}'.format(USER_NAME, PASSWORD, proxy, PROXY_PORT),
45 | 'https': 'https://{}:{}@{}:{}'.format(USER_NAME, PASSWORD, proxy, PROXY_PORT),
46 | }
47 | headers = {'User-Agent': UserAgent().random}
48 | try:
49 | response = requests.get(url=TEST_URL, headers=headers, proxies=proxies, timeout=TEST_TIMEOUT)
50 | if response.status_code in VALID_STATUS_CODES:
51 | # print(response.text)
52 | return True
53 | except (ConnectTimeout, ReadTimeout, ConnectionError):
54 | return False
55 |
56 | def remove_proxy(self):
57 | """ 移除代理 """
58 | url = '{}/remove?name={}'.format(REDIS_URI, CLIENT_NAME)
59 | res = requests.get(url=url)
60 | print(res.text)
61 |
62 | def add_proxy(self, proxy):
63 | """ 添加代理 """
64 | url = '{}/put?name={}&proxy={}'.format(REDIS_URI, CLIENT_NAME, proxy)
65 | res = requests.get(url=url)
66 | print(res.text)
67 |
68 | def adsl(self):
69 | """
70 | 拨号主进程
71 | ADSL代码优化,保证拨号前把IP删掉,
72 | 确保入库的IP在拨号前都是可用的。
73 | 执行删除后再次休眠10秒,确保取出来的IP最少能用10秒
74 | """
75 | while True:
76 | try:
77 | self.remove_proxy()
78 | # 删除之后再次休眠10秒,万一在刚要拨号的时候取到那个IP,那么取出来就拨号了,IP直接不能使用
79 | # 再次休眠确保即使已经删除,取到之后还能用,但是删除后会有一段时间没有IP进来,所以需要开尽可能多的服务器
80 | # 确保池子里面有IP可用
81 | time.sleep(ADSL_KEEP_USE)
82 | except:
83 | while True:
84 | (status, output) = subprocess.getstatusoutput(ADSL_BASH)
85 | if status == 0:
86 | self.remove_proxy()
87 | break
88 | (status, output) = subprocess.getstatusoutput(ADSL_BASH)
89 | if status == 0:
90 | print('ADSL拨号成功')
91 | ip = self.get_ip()
92 | if ip:
93 | if self.test_proxy(ip):
94 | proxies = str(ip) + ":" + PROXY_PORT
95 | self.add_proxy(proxies)
96 | time.sleep(ADSL_CYCLE)
97 | else:
98 | print('代理IP不可用,测试不通过,上次已经拨号成功的IP已经不可用,删除')
99 | self.remove_proxy()
100 | else:
101 | print('没有匹配到IP')
102 | time.sleep(ADSL_ERROR_CYCLE)
103 | else:
104 | print('ADSL拨号失败')
105 | time.sleep(ADSL_ERROR_CYCLE)
106 | self.adsl()
107 |
108 |
109 | if __name__ == '__main__':
110 | conn = SenderAdsl()
111 | conn.adsl()
--------------------------------------------------------------------------------
/adsl_server/settings.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | """
3 | IP存活时间保持在10-120秒之间
4 | """
5 | """""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
6 | 需要修改的配置
7 | """""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
8 |
9 | # 拨号间隔
10 | ADSL_CYCLE = 110
11 |
12 | # 拨号出错重试间隔
13 | ADSL_ERROR_CYCLE = 3
14 |
15 | # 删除之后IP还可以用10秒
16 | ADSL_KEEP_USE = 10
17 |
18 | # 每台拨号机对应一个名字,不同机器需要更改
19 | CLIENT_NAME = 'adsl1'
20 |
21 | # ADSL命令
22 | # ADSL_BASH = 'adsl-stop;adsl-start'
23 | ADSL_BASH = 'pppoe-stop;pppoe-start'
24 |
25 | # 代理配置,根据squid的配置文件填写
26 | USER_NAME = 'martindu'
27 | PASSWORD = 'fy1812!!'
28 | PROXY_PORT = '8881'
29 |
30 | # 代理保存到自己公司redis接口
31 | REDIS_URI = 'http://3.112.239.215'
32 |
33 | """""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
34 | 基本不需要改的配置
35 | """""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
36 |
37 | # 拨号网卡
38 | ADSL_NETMASK = 'ppp0'
39 |
40 | # 拨号成功后的IP检测地址
41 | TEST_URL = 'http://httpbin.org/get'
42 |
43 | # 测试超时时间
44 | TEST_TIMEOUT = 20
45 |
46 | # 遇到以下状态码,测试通过,认定为可用IP
47 | VALID_STATUS_CODES = [200, 201, 202]
--------------------------------------------------------------------------------
/example/__init__.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | # @Time : 2019/12/13 11:16
3 | # @Author : King life
4 | # @Email : 18353626676@163.com
5 | # @File : __init__.py.py
6 | # @Software: PyCharm
--------------------------------------------------------------------------------
/example/spider.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | import random
3 |
4 | import redis
5 | import requests
6 | from fake_useragent import UserAgent
7 |
8 | pool = redis.ConnectionPool(host="localhost", port=6379, db=15, password='admin')
9 | redis_client = redis.Redis(connection_pool=pool, decode_responses=True)
10 | REDIS_KEY = "proxies"
11 | headers = {'User-Agent': UserAgent().random}
12 |
13 |
14 | def get_proxy_from_redis():
15 | proxy_list = redis_client.hvals(REDIS_KEY)
16 | proxy = random.choice(proxy_list).decode('utf-8')
17 | proxies = {
18 | 'http': 'http://martindu:fy1812!!@{}'.format(proxy),
19 | 'https': 'https://martindu:fy1812!!@{}'.format(proxy),
20 | }
21 |
22 | return proxies
23 |
24 |
25 | def crawler(url):
26 | res = requests.get(url=url, headers=headers, proxies=proxies)
27 | print(res.text)
28 |
29 |
30 | if __name__ == '__main__':
31 | proxies = get_proxy_from_redis()
32 | print(proxies)
33 | crawler('https://www.baidu.com/')
--------------------------------------------------------------------------------
/redis_run.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | from redis_server.api import app
3 | from redis_server.config import API_PORT, API_HOST
4 |
5 |
6 | def main():
7 | app.run(host=API_HOST, port=API_PORT, debug=True)
8 |
9 |
10 | if __name__ == '__main__':
11 | main()
--------------------------------------------------------------------------------
/redis_server/__init__.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | # @Time : 2019/12/11 15:49
3 | # @Author : King life
4 | # @Email : 18353626676@163.com
5 | # @File : __init__.py.py
6 | # @Software: PyCharm
--------------------------------------------------------------------------------
/redis_server/api.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | """
3 | flask提供代理的增删接口
4 | 需要nginx部署,保证redis能被访问,才能进行对应的增删操作
5 | """
6 | from flask import Flask, g
7 | from flask import request
8 |
9 | from redis_server.db import RedisClient
10 |
11 | __all__ = ['app']
12 | app = Flask(__name__)
13 |
14 |
15 | def get_conn():
16 | if not hasattr(g, 'redis'): # 用于判断对象是否包含对应的属性
17 | g.redis = RedisClient()
18 | return g.redis
19 |
20 |
21 | @app.route('/')
22 | def index():
23 | return '
欢迎来到华东的代理池系统
' 24 | 25 | 26 | @app.route('/put') 27 | def upload_proxy(): 28 | """ 将proxy上传到redis数据库中 """ 29 | conn = get_conn() 30 | proxy = request.args.get("proxy") 31 | name = request.args.get("name") 32 | # remote_ip = request.remote_addr 33 | # port = proxy.split(":")[1] 34 | proxy = "{}".format(proxy) 35 | if not proxy: 36 | return "上传代理不能为空" 37 | conn.add(name, proxy) 38 | return "已成功上传代理: {}".format(proxy) 39 | 40 | 41 | @app.route('/remove') 42 | def remove_proxy(): 43 | """ 删除redis中的代理 """ 44 | conn = get_conn() 45 | name = request.args.get("name") 46 | if not name: 47 | return "键不能为空" 48 | conn.remove(name) 49 | return "已成功删除代理:{}".format(name) 50 | 51 | 52 | @app.route('/random') 53 | def random_proxy(): 54 | """ 55 | 随机获取可用代理 56 | :return: redis里面的代理IP 57 | """ 58 | connection = get_conn() 59 | return connection.random() 60 | 61 | 62 | if __name__ == '__main__': 63 | app.run() -------------------------------------------------------------------------------- /redis_server/config.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | 4 | # redis连接配置 5 | REDIS_HOST = '127.0.0.1' 6 | REDIS_PORT = 6379 7 | REDIS_DB = 15 8 | REDIS_PASSWORD = '' 9 | 10 | # 代理池hash名 11 | PROXY_KEY = 'proxies' 12 | 13 | # flask服务 14 | API_HOST = '127.0.0.1' 15 | API_PORT = 5000 -------------------------------------------------------------------------------- /redis_server/db.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | 存储器,存储拨号成功后的IP,Hash监控每台拨号机器拨号情况 4 | """ 5 | import random 6 | 7 | import redis 8 | 9 | from redis_server.config import REDIS_HOST, REDIS_PORT, REDIS_DB, REDIS_PASSWORD, PROXY_KEY 10 | 11 | 12 | class RedisClient(object): 13 | def __init__(self): 14 | """ 15 | 初始化Redis连接 16 | :param host: Redis 地址 17 | :param port: Redis 端口 18 | :param password: Redis 密码 19 | :param db: Redis 数据库 20 | :param proxy_key: Redis 哈希表名 21 | """ 22 | if REDIS_PASSWORD: 23 | self.db = redis.StrictRedis(host=REDIS_HOST, port=REDIS_PORT, db=REDIS_DB, password=REDIS_PASSWORD, decode_responses=True) 24 | else: 25 | self.db = redis.StrictRedis(host=REDIS_HOST, port=REDIS_PORT, db=REDIS_DB, decode_responses=True) 26 | 27 | self.proxy_key = PROXY_KEY 28 | 29 | def add(self, name, proxy): 30 | """ 31 | 添加代理 32 | :param name:主机名 33 | :param proxy:代理 34 | :return:存储结果 35 | """ 36 | return self.db.hset(self.proxy_key, name, proxy) 37 | 38 | def get(self, name): 39 | """ 40 | 获取代理 41 | :param name: 主机名称 42 | :return: 代理 43 | """ 44 | return self.db.hget(self.proxy_key, name) 45 | 46 | def count(self): 47 | """ 48 | 获取代理总数 49 | :return: 代理总数 50 | """ 51 | return self.db.hlen(self.proxy_key) 52 | 53 | def remove(self, name): 54 | """ 55 | 删除代理 56 | :param name: 主机名称 57 | :return: 删除结果 58 | """ 59 | return self.db.hdel(self.proxy_key, name) 60 | 61 | def names(self): 62 | """ 63 | 获取主机名称列表 64 | :return:主机名称列表 65 | """ 66 | return self.db.hkeys(self.proxy_key) 67 | 68 | def proxies(self): 69 | """ 70 | 获取代理列表 71 | :return: 代理列表 72 | """ 73 | return self.db.hvals(self.proxy_key) 74 | 75 | def random(self): 76 | """ 随机获取代理 """ 77 | proxies = self.proxies() 78 | return random.choice(proxies) 79 | 80 | def all(self): 81 | """ 获取字典 """ 82 | return self.db.hgetall(self.proxy_key) 83 | 84 | 85 | if __name__ == '__main__': 86 | conn = RedisClient() 87 | conn.add('adsl1', '81.163.123.233:8080') 88 | # conn.remove('adsl') -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | requests==2.22.0 2 | fake-useragent==0.1.11 3 | redis==3.3.6 4 | Flask==1.1.1 --------------------------------------------------------------------------------