├── README.md
├── proxy_grabber.py
└── requirements.txt
/README.md:
--------------------------------------------------------------------------------
1 | # Proxy Grabber
2 | This script is designed to get proxies from several sites and save it to a file.
3 | If you need check the received proxies, you can use this module [Proxy Checker](https://github.com/iterweb/proxy_checker)
4 | ### Using
5 | * running in windows ```python proxy_grabber.py```
6 | * running in linux ```python3 proxy_grabber.py```
7 | * you need input some data, like: ***country*** and ***proxy type***
8 | ### Example
9 | Country ```ru```
10 | Proxy Type ```http``` (```https``` ```socks5``` ```socks4```)
11 | ### Requirements
12 | * [python 3.7+](https://www.python.org/)
13 | * windows ```pip install -r requirements.txt```
14 | * linux ```pip3 install -r requirements.txt```
15 |
--------------------------------------------------------------------------------
/proxy_grabber.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python
2 | # -*- coding: utf-8 -*-
3 | import aiohttp
4 | import asyncio
5 | import json
6 |
7 |
8 | class ProxyGrabber:
9 | def __init__(self):
10 | super().__init__()
11 | print('Country')
12 | self.country = str(input())
13 | print('Proxy Type')
14 | self.proxytype = str(input())
15 | self.proxies = []
16 | asyncio.run(self.main())
17 |
18 | async def proxyscrape(self, session, country, proxytype):
19 | url = f'https://api.proxyscrape.com/?request=displayproxies&proxytype={proxytype}&country={country}'
20 | try:
21 | async with session.get(url) as response:
22 | r = await response.text()
23 | proxies = r.split('\r\n')
24 | for proxy in proxies:
25 | self.proxies.append(proxy)
26 | except Exception as e:
27 | print(f'Site: proxyscrape.com\n{e}')
28 |
29 | async def proxyscan(self, session, country, proxytype):
30 | url = f'https://www.proxyscan.io/api/proxy?country={country}&limit=10&type={proxytype}'
31 | try:
32 | async with session.get(url) as response:
33 | r = await response.read()
34 | json_body = json.loads(r)
35 | for item in json_body:
36 | proxy = f"{item['Ip']}:{item['Port']}"
37 | self.proxies.append(proxy)
38 | except Exception as e:
39 | print(f'Site: proxyscan.io\n{e}')
40 |
41 | async def pubproxy(self, session, country, proxytype):
42 | url = f'http://pubproxy.com/api/proxy?limit=5&type={proxytype}&country={country}'
43 | try:
44 | async with session.get(url) as response:
45 | r = await response.read()
46 | json_body = json.loads(r)
47 | for item in json_body['data']:
48 | proxy = f"{item['ipPort']}"
49 | self.proxies.append(proxy)
50 | except Exception as e:
51 | print(f'Site: pubproxy.com\n{e}')
52 |
53 | async def freshproxies(self, session, country, proxytype):
54 | url = f'https://www.freshproxies.net/ProxyList?countries_1={country}&protocol={proxytype}&format=json&count=20'
55 | try:
56 | async with session.get(url) as response:
57 | r = await response.read()
58 | json_body = json.loads(r)
59 | for item in json_body['proxies']:
60 | proxy = f"{item['ip']}:{item['port']}"
61 | if not proxy.startswith('0.0.0.0'):
62 | self.proxies.append(proxy)
63 | except Exception as e:
64 | print(f'Site: freshproxies.net\n{e}')
65 |
66 | async def proxy_list(self, session, country, proxytype):
67 | url = f'https://www.proxy-list.download/api/v1/get?type={proxytype}&country={country}'
68 | try:
69 | async with session.get(url) as response:
70 | r = await response.text()
71 | proxies = r.split('\r\n')
72 | for proxy in proxies:
73 | self.proxies.append(proxy)
74 | except Exception as e:
75 | print(f'Site: www.proxy-list.download\n{e}')
76 |
77 | def create_txt(self, proxies):
78 | proxies = list(set(proxies))
79 | with open(f'{self.country}_{self.proxytype}.txt', 'a') as f:
80 | for proxy in proxies:
81 | if proxy != '':
82 | f.write(f'{proxy}\n')
83 |
84 | async def main(self):
85 |
86 | async with aiohttp.ClientSession() as session:
87 | task_1 = asyncio.create_task(self.pubproxy(session, self.country, self.proxytype))
88 | task_2 = asyncio.create_task(self.proxyscan(session, self.country, self.proxytype))
89 | task_3 = asyncio.create_task(self.proxyscrape(session, self.country, self.proxytype))
90 | task_4 = asyncio.create_task(self.freshproxies(session, self.country, self.proxytype))
91 | task_5 = asyncio.create_task(self.proxy_list(session, self.country, self.proxytype))
92 |
93 | await asyncio.gather(task_1, task_2, task_3, task_4, task_5)
94 |
95 | self.create_txt(self.proxies)
96 |
97 |
98 | if __name__ == '__main__':
99 | ProxyGrabber()
100 |
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | aiohttp==3.7.4.post0
2 | async-timeout==3.0.1
3 | attrs==21.2.0
4 | chardet==4.0.0
5 | idna==3.2
6 | multidict==5.1.0
7 | typing-extensions==3.10.0.0
8 | yarl==1.6.3
9 |
--------------------------------------------------------------------------------