├── README.md ├── aiohttp_ip_rotator ├── __init__.py └── rotator.py ├── requirements.txt ├── setup.cfg └── setup.py /README.md: -------------------------------------------------------------------------------- 1 | # aiohttp-ip-rotator 2 | An asynchronous alternative to the requests-ip-rotator (https://github.com/Ge0rg3/requests-ip-rotator) library based on aiohttp, completely copying its functionality 3 | 4 | ## Installation 5 | ```commandline 6 | pip install aiohttp-ip-rotator 7 | ``` 8 | 9 | ## Example 10 | ```python3 11 | from asyncio import get_event_loop 12 | from aiohttp_ip_rotator import RotatingClientSession 13 | 14 | 15 | async def main(): 16 | session = RotatingClientSession("https://api.ipify.org", "aws access key id", "aws access key secret") 17 | await session.start() 18 | for i in range(5): 19 | response = await session.get("https://api.ipify.org") 20 | print(f"Your ip: {await response.text()}") 21 | await session.close() 22 | 23 | 24 | if __name__ == "__main__": 25 | get_event_loop().run_until_complete(main()) 26 | ``` 27 | ## Example 2 28 | ```python3 29 | from asyncio import get_event_loop 30 | from aiohttp_ip_rotator import RotatingClientSession 31 | 32 | 33 | async def main(): 34 | async with RotatingClientSession( 35 | "https://api.ipify.org", 36 | "aws access key id", 37 | "aws access key secret" 38 | ) as session: 39 | for i in range(5): 40 | response = await session.get("https://api.ipify.org") 41 | print(f"Your ip: {await response.text()}") 42 | 43 | 44 | if __name__ == "__main__": 45 | get_event_loop().run_until_complete(main()) 46 | ``` 47 | -------------------------------------------------------------------------------- /aiohttp_ip_rotator/__init__.py: -------------------------------------------------------------------------------- 1 | from .rotator import RotatingClientSession 2 | -------------------------------------------------------------------------------- /aiohttp_ip_rotator/rotator.py: -------------------------------------------------------------------------------- 1 | from aiohttp import ClientSession 2 | from aiohttp import ClientResponse 3 | from random import choice 4 | from random import randint 5 | from struct import pack 6 | from socket import inet_ntoa 7 | from typing import Union 8 | from typing import Optional 9 | from aiobotocore.client import BaseClient 10 | from botocore.exceptions import ClientError 11 | from botocore.exceptions import EndpointConnectionError 12 | from aioboto3.session import Session 13 | from asyncio import sleep 14 | from asyncio import gather 15 | from asyncio import create_task 16 | from uuid import uuid4 17 | 18 | 19 | class RotatingClientSession(ClientSession): 20 | def __init__( 21 | self, 22 | target: str, 23 | key_id: Optional[str] = None, 24 | key_secret: Optional[str] = None, 25 | host_header: Optional[str] = None, 26 | verbose: bool = False, 27 | regions: Optional[list[str]] = [ 28 | "us-east-1", "us-east-2", "us-west-1", "us-west-2", 29 | "eu-west-1", "eu-west-2", "eu-west-3", "eu-north-1", 30 | "eu-central-1", "ca-central-1", "ap-south-1", "me-south-1", 31 | "ap-northeast-3", "ap-northeast-2", "ap-southeast-1", 32 | "ap-southeast-2", "ap-northeast-1", "sa-east-1", 33 | "ap-east-1", "af-south-1", "eu-south-1" 34 | ], 35 | *args, 36 | **kwargs 37 | ): 38 | super().__init__(*args, **kwargs) 39 | self.target = target if not target.endswith("/") else target[:-1] 40 | if not target.startswith("http://") and not target.startswith("https://"): 41 | raise ValueError("Invalid URL schema") 42 | self.key_id = key_id 43 | self.key_secret = key_secret 44 | self.host_header = host_header or self.target.split("://", 1)[1].split("/", 1)[0] 45 | self.verbose = verbose 46 | self.endpoints = [] 47 | self.name = f"IP Rotator for {self.target} ({str(uuid4())})" 48 | self.active = False 49 | self.regions = regions 50 | 51 | async def __aenter__(self): 52 | await self.start() 53 | return self 54 | 55 | async def __aexit__(self, *args, **kwargs): 56 | await self.close() 57 | 58 | async def close(self): 59 | await self._clear_apis() 60 | await super().close() 61 | self.active = False 62 | 63 | def _print_if_verbose(self, message: str): 64 | if self.verbose: print(f">> {message}") 65 | 66 | async def _get_apis(self, region: str, client: BaseClient) -> list[dict]: 67 | position = None 68 | complete = False 69 | apis = [] 70 | while not complete: 71 | try: 72 | gateways = await client.get_rest_apis(limit=500)\ 73 | if position is None \ 74 | else await client.get_rest_apis(limit=500, position=position) 75 | except (ClientError, EndpointConnectionError): 76 | self._print_if_verbose(f"Could not get list of APIs in region \"{region}\"") 77 | return [] 78 | apis.extend(gateways["items"]) 79 | position = gateways.get("position", None) 80 | if position is None: 81 | complete = True 82 | return apis 83 | 84 | async def _configure_api(self, client: BaseClient, api_id: str, api_resource_id: str, resource_id: str) -> None: 85 | await client.put_method( 86 | restApiId=api_id, 87 | resourceId=api_resource_id, 88 | httpMethod="ANY", 89 | authorizationType="NONE", 90 | requestParameters={ 91 | "method.request.path.proxy": True, 92 | "method.request.header.X-Forwarded-Header": True, 93 | "method.request.header.X-Host": True 94 | } 95 | ) 96 | await client.put_integration( 97 | restApiId=api_id, 98 | resourceId=api_resource_id, 99 | type="HTTP_PROXY", 100 | httpMethod="ANY", 101 | integrationHttpMethod="ANY", 102 | uri=self.target, 103 | connectionType="INTERNET", 104 | requestParameters={ 105 | "integration.request.path.proxy": "method.request.path.proxy", 106 | "integration.request.header.X-Forwarded-For": "method.request.header.X-Forwarded-Header", 107 | "integration.request.header.Host": "method.request.header.X-Host" 108 | } 109 | ) 110 | await client.put_method( 111 | restApiId=api_id, 112 | resourceId=resource_id, 113 | httpMethod="ANY", 114 | authorizationType="NONE", 115 | requestParameters={ 116 | "method.request.path.proxy": True, 117 | "method.request.header.X-Forwarded-Header": True, 118 | "method.request.header.X-Host": True 119 | } 120 | ) 121 | await client.put_integration( 122 | restApiId=api_id, 123 | resourceId=resource_id, 124 | type="HTTP_PROXY", 125 | httpMethod="ANY", 126 | integrationHttpMethod="ANY", 127 | uri=f"{self.target}/{{proxy}}", 128 | connectionType="INTERNET", 129 | requestParameters={ 130 | "integration.request.path.proxy": "method.request.path.proxy", 131 | "integration.request.header.X-Forwarded-For": "method.request.header.X-Forwarded-Header", 132 | "integration.request.header.Host": "method.request.header.X-Host" 133 | } 134 | ) 135 | await client.create_deployment( 136 | restApiId=api_id, 137 | stageName="proxy-stage" 138 | ) 139 | 140 | async def _create_api(self, region: str) -> Optional[str]: 141 | async with Session().client( 142 | "apigateway", 143 | region_name=region, 144 | aws_access_key_id=self.key_id, 145 | aws_secret_access_key=self.key_secret 146 | ) as client: 147 | try: 148 | api_id = (await client.create_rest_api(name=self.name, 149 | endpointConfiguration={"types": ["REGIONAL"]}))["id"] 150 | except (ClientError, EndpointConnectionError): 151 | self._print_if_verbose(f"Could not create new API in region \"{region}\"") 152 | return None 153 | api_resource_id = (await client.get_resources(restApiId=api_id))["items"][0]["id"] 154 | resource_id = (await client.create_resource(restApiId=api_id, 155 | parentId=api_resource_id, 156 | pathPart="{proxy+}"))["id"] 157 | await self._configure_api(client, api_id, api_resource_id, resource_id) 158 | self._print_if_verbose(f"Created API with id \"{api_id}\"") 159 | return f"{api_id}.execute-api.{region}.amazonaws.com" 160 | 161 | async def _clear_region_apis(self, region: str) -> None: 162 | async with Session().client( 163 | "apigateway", 164 | region_name=region, 165 | aws_access_key_id=self.key_id, 166 | aws_secret_access_key=self.key_secret 167 | ) as client: 168 | for api in await self._get_apis(region, client): 169 | if api["name"] == self.name: 170 | try: 171 | await client.delete_rest_api(restApiId=api["id"]) 172 | except ClientError as e: 173 | if e.response["Error"]["Code"] == "TooManyRequestsException": 174 | self._print_if_verbose("Too many requests when deleting rest API, sleeping for 3 seconds") 175 | await sleep(3) 176 | return await self._clear_region_apis(region) 177 | self._print_if_verbose(f"Deleted rest API with id \"{api['id']}\"") 178 | 179 | async def _clear_apis(self) -> None: 180 | await gather(*[create_task(self._clear_region_apis(region)) for region in self.regions]) 181 | self._print_if_verbose(f"All created APIs for ip rotating have been deleted") 182 | 183 | async def start(self) -> None: 184 | self._print_if_verbose(f"Starting IP Rotating APIs in {len(self.regions)} regions") 185 | endpoints = await gather(*[create_task(self._create_api(region)) for region in self.regions]) 186 | self.endpoints.extend([endpoint for endpoint in endpoints if endpoint is not None]) 187 | self._print_if_verbose(f"API launched in {len(self.endpoints)} regions out of {len(self.regions)}") 188 | self.active = True 189 | 190 | def request(self, method: str, url: str, **kwargs) -> ClientResponse: 191 | if len(self.endpoints) == 0: 192 | raise RuntimeError("To send requests using the RotatingClientSession class, " 193 | "first call [your RotatingClientSession instance].start() " 194 | "or use async with RotatingClientSession(...) as session:") 195 | if not url.startswith("http://") and not url.startswith("https://"): 196 | raise ValueError("Invalid URL schema") 197 | endpoint = choice(self.endpoints) 198 | try: path = url.split("://", 1)[1].split("/", 1)[1] 199 | except IndexError: path = "" 200 | url = f"https://{endpoint}/proxy-stage/{path}" 201 | headers = kwargs.get("headers") or dict() 202 | if not isinstance(headers, dict): 203 | raise ValueError("Headers must be a dictionary-like object") 204 | headers.pop("X-Forwarded-For", None) 205 | kwargs.pop("headers", None) 206 | headers["X-Host"] = self.host_header 207 | headers["X-Forwarded-Header"] = headers.get("X-Forwarded-For") or inet_ntoa(pack(">I", randint(1, 0xffffffff))) 208 | return super().request(method, url, headers=headers, **kwargs) 209 | 210 | async def get(self, url: str, *, allow_redirects: bool = True, **kwargs) -> ClientResponse: 211 | return await self.request("GET", url, allow_redirects=allow_redirects, **kwargs) 212 | 213 | async def options(self, url: str, *, allow_redirects: bool = True, **kwargs) -> ClientResponse: 214 | return await self.request("OPTIONS", url, allow_redirects=allow_redirects, **kwargs) 215 | 216 | async def head(self, url: str, *, allow_redirects: bool = False, **kwargs) -> ClientResponse: 217 | return await self.request("HEAD", url, allow_redirects=allow_redirects, **kwargs) 218 | 219 | async def post(self, url: str, *, data: Union[str, bytes, None] = None, **kwargs) -> ClientResponse: 220 | return await self.request("POST", url, data=data, **kwargs) 221 | 222 | async def put(self, url: str, *, data: Union[str, bytes, None] = None, **kwargs) -> ClientResponse: 223 | return await self.request("PUT", url, data=data, **kwargs) 224 | 225 | async def patch(self, url: str, *, data: Union[str, bytes, None] = None, **kwargs) -> ClientResponse: 226 | return await self.request("PATCH", url, data=data, **kwargs) 227 | 228 | async def delete(self, url: str, **kwargs) -> ClientResponse: 229 | return await self.request("DELETE", url, **kwargs) 230 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | aioboto3 2 | aiohttp -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [egg_info] 2 | tag_build = 3 | tag_date = 0 -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | from setuptools import find_packages 3 | 4 | 5 | def readme() -> str: 6 | with open("README.md", "r") as file: 7 | return file.read() 8 | 9 | 10 | def requirements() -> list[str]: 11 | return ["aiohttp", "aioboto3"] 12 | 13 | 14 | setup( 15 | name="aiohttp-ip-rotator", 16 | version="1.0", 17 | description="Change the IP address with each http request using the AWS API Gateway.", 18 | url="https://github.com/D4rkwat3r/aiohttp-ip-rotator", 19 | long_description=readme(), 20 | long_description_content_type="text/markdown", 21 | author_email="ktoya170214@gmail.com", 22 | packages=find_packages(), 23 | install_requires=requirements() 24 | ) 25 | --------------------------------------------------------------------------------