├── .gitignore ├── README.md ├── example.py ├── pyaiodl ├── __init__.py ├── __main__.py ├── errors.py ├── helper.py ├── pyaiodl.py └── utils.py ├── requirements.txt └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | venv 3 | .idea 4 | 5 | #repl stuffs 6 | poetry.lock 7 | pyproject.toml 8 | 9 | #builds 10 | build/ 11 | dist/ 12 | pyaiodl.egg-info/ -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Python Asynchronous Downloader - pyaoidl 2 | 3 | ### Don't Use it in Production or Live Projects Currently Its Unstable 4 | ___ 5 | [![Python 3.6](https://img.shields.io/badge/python-3-blue.svg)](https://www.python.org/downloads/release/python-360/) 6 | [![Maintenance](https://img.shields.io/badge/Maintained%3F-yes-green.svg)](https://github.com/aryanvikash/pyaiodl) 7 | [![PyPI license](https://img.shields.io/pypi/l/ansicolortags.svg)](https://github.com/aryanvikash/pyaiodl) 8 | [![Open Source Love png3](https://badges.frapsoft.com/os/v3/open-source.png?v=103)](https://github.com/aryanvikash/pyaiodl) 9 | ![Hits](https://hits.seeyoufarm.com/api/count/incr/badge.svg?url=https://github.com/aryanvikash/pyaiodl) 10 | ___ 11 | ## Version 12 | [![Beta badge](https://img.shields.io/badge/STATUS-BETA-red.svg)](https://github.com/aryanvikash/pyaiodl) 13 | 14 | [![PyPI version](https://badge.fury.io/py/pyaiodl.svg)](https://pypi.org/project/pyaiodl/) 15 | 16 | 17 | ## installation 18 | pypi Method (Recommended) 19 | 20 | pip3 install pyaiodl 21 | 22 | Github repo method 23 | 24 | pip3 install git+https://github.com/aryanvikash/pyaiodl.git 25 | 26 | 27 | # Available Methods 28 | - Downloader class Instance 29 | `Non-blocking , params = [fake_useragent:bool,chunk_size:int ,download_path:str] optinals` 30 | 31 | dl = Downloader() 32 | - Download [ `download(self,url)` ] 33 | 34 | uuid = await dl.download(url) 35 | - Errors [` iserror(self, uuid) `] 36 | ` : Returns - Error Or None 37 | , Even On cancel It returns an error "{uuid} Cancelled"` 38 | 39 | ``` 40 | await dl.iserror(uuid) 41 | ``` 42 | 43 | 44 | - cancel [ `cancel(self, uuid)` ] 45 | 46 | await dl.cancel(uuid) 47 | - Get Status [ `status(self, uuid)` ] 48 | 49 | response = await dl.status(uuid) 50 | 51 | 52 | 53 | returns a dict 54 | 55 | """ 56 | filename:str 57 | file_type :str 58 | total_size :int 59 | total_size_str : str 60 | downloaded :int 61 | downloaded_str :str 62 | progress:int 63 | download_speed:str 64 | complete :bool 65 | download_path:str 66 | 67 | """ 68 | 69 | - is_active returns : bool [ `is_active( self,uuid )` ]` - on cancel ,error , download complete return False` 70 | 71 | result = await dl.is_active(uuid) 72 | 73 | 74 | ## Usage 75 | Example : 76 | ___ 77 | 78 | ```py 79 | 80 | from pyaiodl import Downloader, errors 81 | import asyncio 82 | url = "https://speed.hetzner.de/100MB.bin" 83 | 84 | 85 | async def main(): 86 | dl = Downloader() 87 | # you can pass your 88 | # custom chunk size and Download Path 89 | # dl = Downloader(download_path="/your_dir/", chunk_size=10000) 90 | uuid = await dl.download(url) 91 | try: 92 | while await dl.is_active(uuid): 93 | 94 | r = await dl.status(uuid) 95 | 96 | #cancel 97 | if r['progress'] > 0: 98 | try: 99 | await dl.cancel("your_uuid") 100 | except errors.DownloadNotActive as na: 101 | print(na) 102 | 103 | 104 | print(f""" 105 | Filename: {r['filename']} 106 | Total : {r['total_size_str']} 107 | Downloaded : {r['downloaded_str']} 108 | Download Speed : {r['download_speed']} 109 | progress: {r['progress']} 110 | """) 111 | 112 | # let him breath for a second:P 113 | await asyncio.sleep(1) 114 | 115 | # If You are putting uuid manually Than its better handle This Exception 116 | except errors.InvalidId: 117 | print("not valid uuid") 118 | return 119 | 120 | # when loop Breaks There are 2 Possibility 121 | # either Its An error Or Download Complete 122 | # Cancelled Is also count as error 123 | if await dl.iserror(uuid): 124 | print(await dl.iserror(uuid)) 125 | 126 | else: 127 | # Final filename / path 128 | print("Download completed : ", r['download_path']) 129 | 130 | 131 | asyncio.get_event_loop().run_until_complete(main()) 132 | 133 | ``` 134 | 135 | ___ 136 | ### known Bugs - 137 | - None Please Report :) 138 | 139 | ___ 140 | # TODO 141 | 142 | - Multipart Download 143 | - Queue Download / Parallel Downloads Limit 144 | - [x] Better Error Handling 145 | 146 | 147 | 148 | 149 | 150 | ## Thanks ❤️ 151 | - [aiodl](https://github.com/cshuaimin/aiodl) 152 | - [Hasibul Kobir](https://github.com/HasibulKabir) 153 | - [W4RR10R](https://github.com/CW4RR10R) 154 | - [Ranaji](https://t.me/ranaji1425) 155 | 156 | ___ 157 | 158 | [![Powered badge](https://img.shields.io/badge/Powered-Aiohttp-green.svg)](https://shields.io/) -------------------------------------------------------------------------------- /example.py: -------------------------------------------------------------------------------- 1 | from pyaiodl import Downloader, errors 2 | import asyncio 3 | url = "https://speed.hetzner.de/100MB.bin" 4 | 5 | 6 | async def main(): 7 | dl = Downloader() 8 | 9 | # or 10 | # dl = Downloader(download_path="/your_dir/", chunk_size=10000) 11 | 12 | uuid = await dl.download(url) 13 | 14 | 15 | try: 16 | while await dl.is_active(uuid): 17 | 18 | r = await dl.status(uuid) 19 | 20 | #cancel 21 | if r['progress'] > 0: 22 | try: 23 | await dl.cancel("your_uuid") 24 | except errors.DownloadNotActive as na: 25 | print(na) 26 | 27 | 28 | print(f""" 29 | Filename: {r['filename']} 30 | Total : {r['total_size_str']} 31 | Downloaded : {r['downloaded_str']} 32 | Download Speed : {r['download_speed']} 33 | progress: {r['progress']} 34 | """) 35 | 36 | # let him breath for a second:P 37 | await asyncio.sleep(1) 38 | 39 | # If You are putting uuid manually Than its better handle This Exception 40 | except errors.InvalidId: 41 | print("not valid uuid") 42 | return 43 | 44 | # when loop Breaks There are 2 Possibility either Its An error Or Download Complete 45 | # Cancelled Is also count as error 46 | if await dl.iserror(uuid): 47 | print(await dl.iserror(uuid)) 48 | 49 | else: 50 | # Final filename / path 51 | print("Download completed : ", r['download_path']) 52 | 53 | 54 | asyncio.get_event_loop().run_until_complete(main()) 55 | 56 | -------------------------------------------------------------------------------- /pyaiodl/__init__.py: -------------------------------------------------------------------------------- 1 | __all__ = ['Downloader'] 2 | 3 | # import logging 4 | # logging.basicConfig(level=logging.DEBUG) 5 | # log = logging.getLogger(__name__) 6 | 7 | # Temp. removed Custom chunk size will add it later 8 | 9 | # TODO 10 | # Multi-part Downloading (i don't know if it will work on every links) 11 | # queue Download (parallel download limiter) 12 | # Better + Custom Error Handling 13 | 14 | 15 | from .pyaiodl import PrivateDl 16 | from .errors import DownloadNotActive, InvalidId 17 | 18 | 19 | class Downloader: 20 | def __init__(self, chunk_size=None, download_path=None): 21 | self._alldownloads = {} 22 | self.download_path = download_path 23 | # custom chunk size 24 | self.chunk_size = chunk_size 25 | 26 | async def download(self, url): 27 | try: 28 | _down = PrivateDl(chunk_size=self.chunk_size, 29 | download_path=self.download_path) 30 | _uuid = await _down.download(url) 31 | self._alldownloads[_uuid] = {} 32 | self._alldownloads[_uuid] = {"obj": _down} 33 | 34 | # just init things otherwise it will throw keyerror : 35 | self._alldownloads[_uuid]["iscancel"] = False 36 | 37 | except Exception as e: 38 | raise Exception(e) 39 | return _uuid 40 | 41 | # To check If Download Is active 42 | async def is_active(self, uuid): 43 | try: 44 | _tempobj = self._alldownloads[uuid]["obj"] 45 | except KeyError: 46 | raise InvalidId() 47 | 48 | return not _tempobj.task.done() 49 | 50 | # Get Status 51 | async def status(self, uuid): 52 | try: 53 | _tempobj = self._alldownloads[uuid]["obj"] 54 | except KeyError: 55 | raise InvalidId() 56 | return await _tempobj.getStatus() 57 | 58 | #Cancel your Download 59 | 60 | async def cancel(self, uuid): 61 | try: 62 | _tempobj = self._alldownloads[uuid]["obj"] 63 | 64 | # mark as cancelled 65 | self._alldownloads[uuid]["iscancel"] = True 66 | _tempobj._cancelled = True 67 | 68 | cancelstatus = await _tempobj.cancel(uuid) or False 69 | 70 | except KeyError: 71 | raise InvalidId() 72 | 73 | if _tempobj.task.done(): 74 | 75 | raise DownloadNotActive(f"{uuid} : Download Not active") 76 | 77 | return cancelstatus 78 | 79 | async def iserror(self, uuid): 80 | try: 81 | _tempobj = self._alldownloads[uuid]["obj"] 82 | _iscancel = self._alldownloads[uuid]["iscancel"] 83 | 84 | if _iscancel: 85 | return f"{uuid} Cancelled" 86 | except KeyError: 87 | raise InvalidId() 88 | 89 | return _tempobj.iserror 90 | -------------------------------------------------------------------------------- /pyaiodl/__main__.py: -------------------------------------------------------------------------------- 1 | if __name__ == "__main__": 2 | print("currently no command-line support ") -------------------------------------------------------------------------------- /pyaiodl/errors.py: -------------------------------------------------------------------------------- 1 | __all_ = ['DownloadNotActive', 'InvalidId'] 2 | 3 | 4 | class DownloadNotActive(Exception): 5 | """ Download Not active """ 6 | 7 | 8 | class InvalidId(Exception): 9 | """ on Invalid uuid """ 10 | -------------------------------------------------------------------------------- /pyaiodl/helper.py: -------------------------------------------------------------------------------- 1 | # https://github.com/cshuaimin/aiodl/blob/8c2f30eca012cea06ab244916ead569126ac0571/aiodl/utils.py#L44 2 | 3 | import asyncio 4 | import aiohttp 5 | import functools 6 | import socket 7 | 8 | 9 | class AiodlQuitError(Exception): 10 | 'Something caused aiodl to quit.' 11 | 12 | 13 | class ClosedRange: 14 | def __init__(self, begin, end): 15 | self.begin = begin 16 | self.end = end 17 | 18 | def __iter__(self): 19 | yield self.begin 20 | yield self.end 21 | 22 | def __str__(self): 23 | return '[{0.begin}, {0.end}]'.format(self) 24 | 25 | # Why not __len__() ? See https://stackoverflow.com/questions/47048561/ 26 | @property 27 | def size(self): 28 | return self.end - self.begin + 1 29 | 30 | 31 | def retry(coro_func): 32 | @functools.wraps(coro_func) 33 | async def wrapper(self, *args, **kwargs): 34 | tried = 0 35 | while True: 36 | tried += 1 37 | try: 38 | return await coro_func(self, *args, **kwargs) 39 | except (aiohttp.ClientError, socket.gaierror) as exc: 40 | try: 41 | msg = '%d %s' % (exc.code, exc.message) 42 | # For 4xx client errors, it's no use to try again :) 43 | if 400 <= exc.code < 500: 44 | print(msg) 45 | raise AiodlQuitError from exc 46 | except AttributeError: 47 | msg = str(exc) or exc.__class__.__name__ 48 | if tried <= self.max_tries: 49 | sec = tried / 2 50 | print('%s() failed: %s, retry in %.1f seconds (%d/%d)' % 51 | (coro_func.__name__, msg, 52 | sec, tried, self.max_tries) 53 | ) 54 | await asyncio.sleep(sec) 55 | else: 56 | print( 57 | '%s() failed after %d tries: %s ' % 58 | (coro_func.__name__, self.max_tries, msg) 59 | 60 | ) 61 | raise AiodlQuitError from exc 62 | except asyncio.TimeoutError: 63 | # Usually server has a fixed TCP timeout to clean dead 64 | # connections, so you can see a lot of timeouts appear 65 | # at the same time. I don't think this is an error, 66 | # So retry it without checking the max retries. 67 | print( 68 | '%s() timeout, retry in 1 second' % coro_func.__name__ 69 | ) 70 | await asyncio.sleep(1) 71 | 72 | return wrapper 73 | -------------------------------------------------------------------------------- /pyaiodl/pyaiodl.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import urllib.parse 3 | import cgi 4 | import secrets 5 | import mimetypes 6 | import os 7 | import aiohttp 8 | from .utils import human_size, gen_uuid, getspeed 9 | 10 | import aiofiles 11 | from time import time 12 | import socket 13 | from contextlib import suppress 14 | 15 | 16 | class PrivateDl: 17 | """ 18 | Downloader Class 19 | Example : 20 | dl = downloader() 21 | task = dl.download(url) 22 | task.cancel() 23 | or dl.cancel() 24 | """ 25 | 26 | def __init__(self, fake_useragent: bool = False, chunk_size = None, download_path=None): 27 | self.chunk_size = chunk_size 28 | self.total_size = 0 29 | self.downloaded = 0 30 | #get full path of file 31 | self.download_path = download_path 32 | self.download_speed = 0 33 | self.eta = "NaN" 34 | self.filename = "Unknown" 35 | self.url = None 36 | self.file_type = None 37 | self.session = None 38 | # incase if download is cancelled we can check it here 39 | 40 | # both are protected bcz we don't need mutiple value to check status 41 | self._cancelled = False 42 | self._complete = False 43 | self.uuid = None 44 | self.task = None 45 | self.fake_useragent = fake_useragent 46 | 47 | self.conn = aiohttp.TCPConnector( 48 | family=socket.AF_INET, 49 | verify_ssl=False) 50 | 51 | # TODO add retry 52 | self.max_tries = 3 53 | self.start_time = 0 54 | # basically i will hold only one uuid ;) 55 | self.__toatal_downloads = {} 56 | self.real_url = None 57 | # error goes here 58 | self.iserror = None 59 | self.downloadedstr = 0 # 10MiB 60 | if fake_useragent: 61 | from fake_useragent import UserAgent 62 | ua = UserAgent(cache=False) 63 | self.userAgent = ua.random 64 | 65 | self.progress = 0 66 | 67 | async def download(self, url: str) -> str: 68 | 69 | try: 70 | download_obj = PrivateDl() 71 | __uuid = gen_uuid() 72 | self.uuid = __uuid 73 | self.url = url 74 | __task = asyncio.ensure_future(self.__down()) 75 | 76 | self.__toatal_downloads[__uuid] = {} 77 | self.__toatal_downloads[__uuid]["obj"] = download_obj 78 | self.__toatal_downloads[self.uuid]["task"] = __task 79 | self.task = __task 80 | 81 | return self.uuid 82 | except Exception as e: 83 | await self.mark_done(e) 84 | return 85 | 86 | async def __down(self) -> None: 87 | 88 | downloaded_chunk = 0 89 | 90 | # incase need we need some fake User-agent 91 | if self.fake_useragent: 92 | headers = { 93 | "User-Agent": self.userAgent 94 | } 95 | self.session = aiohttp.ClientSession( 96 | headers=headers, raise_for_status=True, connector=self.conn) 97 | else: 98 | self.session = aiohttp.ClientSession( 99 | raise_for_status=True, connector=self.conn) 100 | try: 101 | self.filename, self.total_size, self.content_type, self.real_url = await self.__getinfo() 102 | except Exception as e: 103 | await self.mark_done(e) 104 | return 105 | 106 | try: 107 | async with self.session.get(self.url) as r: 108 | self.start_time = time() 109 | if self.download_path: 110 | if not os.path.isdir(self.download_path): 111 | try: 112 | os.makedirs(self.download_path) 113 | except Exception as e: 114 | await self.mark_done(e) 115 | return 116 | 117 | self.download_path = os.path.join( 118 | self.download_path, self.filename) 119 | else: 120 | self.download_path = self.filename 121 | 122 | async with aiofiles.open(self.download_path, mode="wb") as f: 123 | 124 | if self.chunk_size: 125 | async for chunk in r.content.iter_chunked(self.chunk_size): 126 | await f.write(chunk) 127 | downloaded_chunk += len(chunk) 128 | await self.__updateStatus(downloaded_chunk) 129 | else: 130 | async for chunk in r.content.iter_any(): 131 | await f.write(chunk) 132 | downloaded_chunk += len(chunk) 133 | await self.__updateStatus(downloaded_chunk) 134 | 135 | except Exception as e: 136 | 137 | await self.mark_done(e) 138 | return 139 | 140 | # session close 141 | self._complete = True 142 | 143 | # incase aiohtttp can't grab file size :P 144 | if self.total_size == 0: 145 | self.total_size = self.downloaded 146 | 147 | await self.session.close() 148 | 149 | async def __updateStatus(self, downloaded_chunks): 150 | self.downloaded = downloaded_chunks 151 | 152 | #update Download Speed 153 | self.download_speed = getspeed(self.start_time, self.downloaded) 154 | 155 | # Update Download progress 156 | try: 157 | self.progress = round((self.downloaded / self.total_size) * 100) 158 | except: 159 | self.progress = 0 160 | 161 | # @retry 162 | async def __getinfo(self) -> tuple: 163 | """ get Url Info like filename ,size and filetype """ 164 | 165 | async with self.session.get( 166 | self.url, 167 | allow_redirects=True 168 | 169 | ) as response: 170 | # print(response) 171 | 172 | # Use redirected URL 173 | self.url = str(response.url) 174 | try: 175 | content_disposition = cgi.parse_header( 176 | response.headers['Content-Disposition']) 177 | filename = content_disposition[1]['filename'] 178 | filename = urllib.parse.unquote_plus(filename) 179 | except KeyError: 180 | filename = response._real_url.name 181 | 182 | if not filename: 183 | guessed_extension = mimetypes.guess_extension( 184 | response.headers["Content-Type"].split(";")[0]) 185 | filename = f"{gen_uuid(size=5)}{guessed_extension}" 186 | 187 | try: 188 | size = int(response.headers['Content-Length']) 189 | except KeyError: 190 | size = 0 191 | return ( 192 | filename, 193 | size, 194 | response.headers['Content-Type'], 195 | response._real_url 196 | ) 197 | 198 | async def getStatus(self) -> dict: 199 | """ :get current status: 200 | filename:str 201 | file_type :str 202 | total_size :int 203 | total_size_str : str 204 | downloaded :int 205 | downloaded_str :str 206 | progress:int 207 | download_speed:str 208 | complete :bool 209 | download_path:str 210 | 211 | """ 212 | 213 | return { 214 | "filename": self.filename, 215 | "file_type": self.file_type, 216 | "total_size": self.total_size, 217 | "total_size_str": human_size(self.total_size), 218 | "downloaded": self.downloaded, 219 | "downloaded_str": human_size(self.downloaded), 220 | "progress": self.progress, 221 | "download_speed": self.download_speed, 222 | "complete": self._complete, 223 | "download_path": self.download_path, 224 | 225 | 226 | } 227 | 228 | async def mark_done(self, error): 229 | 230 | self.iserror = error 231 | await self.session.close() 232 | 233 | self.task.cancel() 234 | 235 | # supress CanceledError raised by asyncio cancel task 236 | with suppress(asyncio.CancelledError): 237 | await self.task 238 | 239 | 240 | async def cancel(self, uuid) -> bool: 241 | """ provide uuid returned by download method to cancel it 242 | return : bool 243 | """ 244 | await self.session.close() 245 | # check task is active or cancelled 246 | 247 | if not self.task.done(): 248 | 249 | __task = self.__toatal_downloads[uuid]["task"] 250 | __iscancel: bool = __task.cancel() 251 | 252 | return __iscancel 253 | else: 254 | return True 255 | -------------------------------------------------------------------------------- /pyaiodl/utils.py: -------------------------------------------------------------------------------- 1 | # https://stackoverflow.com/a/43690506/11110014 2 | 3 | import random 4 | from time import time 5 | import string 6 | 7 | 8 | def human_size(size, decimal_places=2): 9 | for unit in ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB']: 10 | if size < 1024.0 or unit == 'PiB': 11 | break 12 | size /= 1024.0 13 | return f"{size:.{decimal_places}f} {unit}" 14 | 15 | 16 | def gen_uuid(size=10, chars=string.ascii_uppercase + string.digits): 17 | return ''.join(random.choice(chars) for x in range(size)) 18 | 19 | 20 | # print (gen_uuid(10, "AEIOSUUGTUGFMA23")) 21 | 22 | def getspeed(start_time, downloaded_bytes): 23 | # get time in milisec 24 | total_ms = (time() - start_time) * 1000 25 | #in bytes 26 | total_bytes = downloaded_bytes * 8000 27 | return pretty_speed((total_bytes / total_ms)/8) 28 | 29 | 30 | def pretty_speed(speed): 31 | units = ['Bps', 'KBps', 'MBps', 'GBps'] 32 | unit = 0 33 | while speed >= 1024: 34 | speed /= 1024 35 | unit += 1 36 | return '%0.2f %s' % (speed, units[unit]) 37 | 38 | # Eta = size - downloaded bytes / speed 39 | 40 | 41 | def get_readable_time(seconds: int) -> str: 42 | result = '' 43 | (days, remainder) = divmod(seconds, 86400) 44 | days = int(days) 45 | if days != 0: 46 | result += f'{days}d' 47 | (hours, remainder) = divmod(remainder, 3600) 48 | hours = int(hours) 49 | if hours != 0: 50 | result += f'{hours}h' 51 | (minutes, seconds) = divmod(remainder, 60) 52 | minutes = int(minutes) 53 | if minutes != 0: 54 | result += f'{minutes}m' 55 | seconds = int(seconds) 56 | result += f'{seconds}s' 57 | return result 58 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | aiohttp 2 | aiofiles 3 | fake-useragent -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | 4 | 5 | with open("README.md", "r") as fh: 6 | long_description = fh.read() 7 | 8 | setuptools.setup( 9 | name="pyaiodl", 10 | version="0.0.5", 11 | author="Aryan Vikash", 12 | author_email="followvikash8@gmail.com", 13 | description="A Python Asynchronous Downloader - pyaiodl", 14 | long_description=long_description, 15 | long_description_content_type="text/markdown", 16 | url="https://github.com/aryanvikash/pyaiodl", 17 | packages=setuptools.find_packages(), 18 | install_requires=['aiohttp', 'fake-useragent', 'aiofiles'], 19 | classifiers=[ 20 | "Programming Language :: Python :: 3", 21 | "Programming Language :: Python :: 3.7", 22 | "Programming Language :: Python :: 3.8", 23 | "License :: OSI Approved :: MIT License", 24 | "Topic :: Internet :: WWW/HTTP", 25 | "Topic :: Software Development :: Libraries :: Python Modules", 26 | "Operating System :: OS Independent", 27 | ], 28 | 29 | python_requires='>=3.6', 30 | ) 31 | --------------------------------------------------------------------------------