├── FastTelethonhelper ├── FastTelethon.py └── __init__.py ├── LICENSE ├── MANIFENST.in ├── README.md ├── requirements.txt └── setup.py /FastTelethonhelper/FastTelethon.py: -------------------------------------------------------------------------------- 1 | """ 2 | > Based on parallel_file_transfer.py from mautrix-telegram, with permission to distribute under the MIT license 3 | > Copyright (C) 2019 Tulir Asokan - https://github.com/tulir/mautrix-telegram 4 | """ 5 | import asyncio 6 | import hashlib 7 | import inspect 8 | import logging 9 | import math 10 | import os 11 | from collections import defaultdict 12 | from typing import ( 13 | AsyncGenerator, 14 | Awaitable, 15 | BinaryIO, 16 | DefaultDict, 17 | List, 18 | Optional, 19 | Tuple, 20 | Union, 21 | ) 22 | 23 | from telethon import TelegramClient, helpers, utils 24 | from telethon.crypto import AuthKey 25 | from telethon.network import MTProtoSender 26 | from telethon.tl.alltlobjects import LAYER 27 | from telethon.tl.functions import InvokeWithLayerRequest 28 | from telethon.tl.functions.auth import ( 29 | ExportAuthorizationRequest, 30 | ImportAuthorizationRequest, 31 | ) 32 | from telethon.tl.functions.upload import ( 33 | GetFileRequest, 34 | SaveBigFilePartRequest, 35 | SaveFilePartRequest, 36 | ) 37 | from telethon.tl.types import ( 38 | Document, 39 | InputDocumentFileLocation, 40 | InputFile, 41 | InputFileBig, 42 | InputFileLocation, 43 | InputPeerPhotoFileLocation, 44 | InputPhotoFileLocation, 45 | TypeInputFile, 46 | ) 47 | 48 | filename = "" 49 | 50 | log: logging.Logger = logging.getLogger("FastTelethon") 51 | 52 | TypeLocation = Union[ 53 | Document, 54 | InputDocumentFileLocation, 55 | InputPeerPhotoFileLocation, 56 | InputFileLocation, 57 | InputPhotoFileLocation, 58 | ] 59 | 60 | 61 | class DownloadSender: 62 | client: TelegramClient 63 | sender: MTProtoSender 64 | request: GetFileRequest 65 | remaining: int 66 | stride: int 67 | 68 | def __init__( 69 | self, 70 | client: TelegramClient, 71 | sender: MTProtoSender, 72 | file: TypeLocation, 73 | offset: int, 74 | limit: int, 75 | stride: int, 76 | count: int, 77 | ) -> None: 78 | self.sender = sender 79 | self.client = client 80 | self.request = GetFileRequest(file, offset=offset, limit=limit) 81 | self.stride = stride 82 | self.remaining = count 83 | 84 | async def next(self) -> Optional[bytes]: 85 | if not self.remaining: 86 | return None 87 | result = await self.client._call(self.sender, self.request) 88 | self.remaining -= 1 89 | self.request.offset += self.stride 90 | return result.bytes 91 | 92 | def disconnect(self) -> Awaitable[None]: 93 | return self.sender.disconnect() 94 | 95 | 96 | class UploadSender: 97 | client: TelegramClient 98 | sender: MTProtoSender 99 | request: Union[SaveFilePartRequest, SaveBigFilePartRequest] 100 | part_count: int 101 | stride: int 102 | previous: Optional[asyncio.Task] 103 | loop: asyncio.AbstractEventLoop 104 | 105 | def __init__( 106 | self, 107 | client: TelegramClient, 108 | sender: MTProtoSender, 109 | file_id: int, 110 | part_count: int, 111 | big: bool, 112 | index: int, 113 | stride: int, 114 | loop: asyncio.AbstractEventLoop, 115 | ) -> None: 116 | self.client = client 117 | self.sender = sender 118 | self.part_count = part_count 119 | if big: 120 | self.request = SaveBigFilePartRequest(file_id, index, part_count, b"") 121 | else: 122 | self.request = SaveFilePartRequest(file_id, index, b"") 123 | self.stride = stride 124 | self.previous = None 125 | self.loop = loop 126 | 127 | async def next(self, data: bytes) -> None: 128 | if self.previous: 129 | await self.previous 130 | self.previous = self.loop.create_task(self._next(data)) 131 | 132 | async def _next(self, data: bytes) -> None: 133 | self.request.bytes = data 134 | await self.client._call(self.sender, self.request) 135 | self.request.file_part += self.stride 136 | 137 | async def disconnect(self) -> None: 138 | if self.previous: 139 | await self.previous 140 | return await self.sender.disconnect() 141 | 142 | 143 | class ParallelTransferrer: 144 | client: TelegramClient 145 | loop: asyncio.AbstractEventLoop 146 | dc_id: int 147 | senders: Optional[List[Union[DownloadSender, UploadSender]]] 148 | auth_key: AuthKey 149 | upload_ticker: int 150 | 151 | def __init__(self, client: TelegramClient, dc_id: Optional[int] = None) -> None: 152 | self.client = client 153 | self.loop = self.client.loop 154 | self.dc_id = dc_id or self.client.session.dc_id 155 | self.auth_key = ( 156 | None 157 | if dc_id and self.client.session.dc_id != dc_id 158 | else self.client.session.auth_key 159 | ) 160 | self.senders = None 161 | self.upload_ticker = 0 162 | 163 | async def _cleanup(self) -> None: 164 | await asyncio.gather(*[sender.disconnect() for sender in self.senders]) 165 | self.senders = None 166 | 167 | @staticmethod 168 | def _get_connection_count( 169 | file_size: int, max_count: int = 20, full_size: int = 100 * 1024 * 1024 170 | ) -> int: 171 | if file_size > full_size: 172 | return max_count 173 | return math.ceil((file_size / full_size) * max_count) 174 | 175 | async def _init_download( 176 | self, connections: int, file: TypeLocation, part_count: int, part_size: int 177 | ) -> None: 178 | minimum, remainder = divmod(part_count, connections) 179 | 180 | def get_part_count() -> int: 181 | nonlocal remainder 182 | if remainder > 0: 183 | remainder -= 1 184 | return minimum + 1 185 | return minimum 186 | 187 | # The first cross-DC sender will export+import the authorization, so we always create it 188 | # before creating any other senders. 189 | self.senders = [ 190 | await self._create_download_sender( 191 | file, 0, part_size, connections * part_size, get_part_count() 192 | ), 193 | *await asyncio.gather( 194 | *[ 195 | self._create_download_sender( 196 | file, i, part_size, connections * part_size, get_part_count() 197 | ) 198 | for i in range(1, connections) 199 | ] 200 | ), 201 | ] 202 | 203 | async def _create_download_sender( 204 | self, 205 | file: TypeLocation, 206 | index: int, 207 | part_size: int, 208 | stride: int, 209 | part_count: int, 210 | ) -> DownloadSender: 211 | return DownloadSender( 212 | self.client, 213 | await self._create_sender(), 214 | file, 215 | index * part_size, 216 | part_size, 217 | stride, 218 | part_count, 219 | ) 220 | 221 | async def _init_upload( 222 | self, connections: int, file_id: int, part_count: int, big: bool 223 | ) -> None: 224 | self.senders = [ 225 | await self._create_upload_sender(file_id, part_count, big, 0, connections), 226 | *await asyncio.gather( 227 | *[ 228 | self._create_upload_sender(file_id, part_count, big, i, connections) 229 | for i in range(1, connections) 230 | ] 231 | ), 232 | ] 233 | 234 | async def _create_upload_sender( 235 | self, file_id: int, part_count: int, big: bool, index: int, stride: int 236 | ) -> UploadSender: 237 | return UploadSender( 238 | self.client, 239 | await self._create_sender(), 240 | file_id, 241 | part_count, 242 | big, 243 | index, 244 | stride, 245 | loop=self.loop, 246 | ) 247 | 248 | async def _create_sender(self) -> MTProtoSender: 249 | dc = await self.client._get_dc(self.dc_id) 250 | sender = MTProtoSender(self.auth_key, loggers=self.client._log) 251 | await sender.connect( 252 | self.client._connection( 253 | dc.ip_address, 254 | dc.port, 255 | dc.id, 256 | loggers=self.client._log, 257 | proxy=self.client._proxy, 258 | ) 259 | ) 260 | if not self.auth_key: 261 | auth = await self.client(ExportAuthorizationRequest(self.dc_id)) 262 | self.client._init_request.query = ImportAuthorizationRequest( 263 | id=auth.id, bytes=auth.bytes 264 | ) 265 | req = InvokeWithLayerRequest(LAYER, self.client._init_request) 266 | await sender.send(req) 267 | self.auth_key = sender.auth_key 268 | return sender 269 | 270 | async def init_upload( 271 | self, 272 | file_id: int, 273 | file_size: int, 274 | part_size_kb: Optional[float] = None, 275 | connection_count: Optional[int] = None, 276 | ) -> Tuple[int, int, bool]: 277 | connection_count = connection_count or self._get_connection_count(file_size) 278 | part_size = (part_size_kb or utils.get_appropriated_part_size(file_size)) * 1024 279 | part_count = (file_size + part_size - 1) // part_size 280 | is_large = file_size > 10 * 1024 * 1024 281 | await self._init_upload(connection_count, file_id, part_count, is_large) 282 | return part_size, part_count, is_large 283 | 284 | async def upload(self, part: bytes) -> None: 285 | await self.senders[self.upload_ticker].next(part) 286 | self.upload_ticker = (self.upload_ticker + 1) % len(self.senders) 287 | 288 | async def finish_upload(self) -> None: 289 | await self._cleanup() 290 | 291 | async def download( 292 | self, 293 | file: TypeLocation, 294 | file_size: int, 295 | part_size_kb: Optional[float] = None, 296 | connection_count: Optional[int] = None, 297 | ) -> AsyncGenerator[bytes, None]: 298 | connection_count = connection_count or self._get_connection_count(file_size) 299 | part_size = (part_size_kb or utils.get_appropriated_part_size(file_size)) * 1024 300 | part_count = math.ceil(file_size / part_size) 301 | await self._init_download(connection_count, file, part_count, part_size) 302 | 303 | part = 0 304 | while part < part_count: 305 | tasks = [] 306 | for sender in self.senders: 307 | tasks.append(self.loop.create_task(sender.next())) 308 | for task in tasks: 309 | data = await task 310 | if not data: 311 | break 312 | yield data 313 | part += 1 314 | await self._cleanup() 315 | 316 | 317 | parallel_transfer_locks: DefaultDict[int, asyncio.Lock] = defaultdict( 318 | lambda: asyncio.Lock() 319 | ) 320 | 321 | 322 | def stream_file(file_to_stream: BinaryIO, chunk_size=1024): 323 | while True: 324 | data_read = file_to_stream.read(chunk_size) 325 | if not data_read: 326 | break 327 | yield data_read 328 | 329 | 330 | async def _internal_transfer_to_telegram( 331 | client: TelegramClient, response: BinaryIO, progress_callback: callable 332 | ) -> Tuple[TypeInputFile, int]: 333 | file_id = helpers.generate_random_long() 334 | file_size = os.path.getsize(response.name) 335 | 336 | hash_md5 = hashlib.md5() 337 | uploader = ParallelTransferrer(client) 338 | part_size, part_count, is_large = await uploader.init_upload(file_id, file_size) 339 | buffer = bytearray() 340 | for data in stream_file(response): 341 | if progress_callback: 342 | r = progress_callback(response.tell(), file_size) 343 | if inspect.isawaitable(r): 344 | try: 345 | await r 346 | except BaseException: 347 | pass 348 | if not is_large: 349 | hash_md5.update(data) 350 | if len(buffer) == 0 and len(data) == part_size: 351 | await uploader.upload(data) 352 | continue 353 | new_len = len(buffer) + len(data) 354 | if new_len >= part_size: 355 | cutoff = part_size - len(buffer) 356 | buffer.extend(data[:cutoff]) 357 | await uploader.upload(bytes(buffer)) 358 | buffer.clear() 359 | buffer.extend(data[cutoff:]) 360 | else: 361 | buffer.extend(data) 362 | if len(buffer) > 0: 363 | await uploader.upload(bytes(buffer)) 364 | await uploader.finish_upload() 365 | if is_large: 366 | return InputFileBig(file_id, part_count, filename), file_size 367 | else: 368 | return InputFile(file_id, part_count, filename, hash_md5.hexdigest()), file_size 369 | 370 | 371 | async def download_file( 372 | client: TelegramClient, 373 | location: TypeLocation, 374 | out: BinaryIO, 375 | progress_callback: callable = None, 376 | ) -> BinaryIO: 377 | size = location.size 378 | dc_id, location = utils.get_input_location(location) 379 | # We lock the transfers because telegram has connection count limits 380 | downloader = ParallelTransferrer(client, dc_id) 381 | downloaded = downloader.download(location, size) 382 | async for x in downloaded: 383 | out.write(x) 384 | if progress_callback: 385 | r = progress_callback(out.tell(), size) 386 | if inspect.isawaitable(r): 387 | try: 388 | await r 389 | except BaseException: 390 | pass 391 | 392 | return out 393 | 394 | 395 | async def upload_file( 396 | client: TelegramClient, 397 | file: BinaryIO, 398 | name, 399 | progress_callback: callable = None, 400 | ) -> TypeInputFile: 401 | global filename 402 | filename = name 403 | return (await _internal_transfer_to_telegram(client, file, progress_callback))[0] -------------------------------------------------------------------------------- /FastTelethonhelper/__init__.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | import pathlib 4 | import time 5 | import datetime as dt 6 | 7 | sys.path.insert(0, f"{pathlib.Path(__file__).parent.resolve()}") 8 | 9 | from FastTelethon import upload_file, download_file 10 | 11 | 12 | class Timer: 13 | def __init__(self, time_between=5): 14 | self.start_time = time.time() 15 | self.time_between = time_between 16 | 17 | def can_send(self): 18 | if time.time() > (self.start_time + self.time_between): 19 | self.start_time = time.time() 20 | return True 21 | return False 22 | 23 | def progress_bar_str(done, total): 24 | percent = round(done/total*100, 2) 25 | strin = "░░░░░░░░░░" 26 | strin = list(strin) 27 | for i in range(round(percent)//10): 28 | strin[i] = "█" 29 | strin = "".join(strin) 30 | final = f"Percent: {percent}%\n{human_readable_size(done)}/{human_readable_size(total)}\n{strin}" 31 | return final 32 | 33 | def human_readable_size(size, decimal_places=2): 34 | for unit in ['B', 'KB', 'MB', 'GB', 'TB', 'PB']: 35 | if size < 1024.0 or unit == 'PB': 36 | break 37 | size /= 1024.0 38 | return f"{size:.{decimal_places}f} {unit}" 39 | 40 | async def fast_download(client, msg, reply = None, download_folder = None, progress_bar_function = progress_bar_str): 41 | timer = Timer() 42 | 43 | async def progress_bar(downloaded_bytes, total_bytes): 44 | if timer.can_send(): 45 | data = progress_bar_function(downloaded_bytes, total_bytes) 46 | await reply.edit(f"Downloading...\n{data}") 47 | 48 | file = msg.document 49 | filename = msg.file.name 50 | dir = "downloads/" 51 | 52 | try: 53 | os.mkdir("downloads/") 54 | except: 55 | pass 56 | 57 | if not filename: 58 | filename = "video.mp4" 59 | 60 | if download_folder == None: 61 | download_location = dir + filename 62 | else: 63 | download_location = download_folder + filename 64 | 65 | with open(download_location, "wb") as f: 66 | if reply != None: 67 | await download_file( 68 | client=client, 69 | location=file, 70 | out=f, 71 | progress_callback=progress_bar 72 | ) 73 | else: 74 | await download_file( 75 | client=client, 76 | location=file, 77 | out=f, 78 | ) 79 | return download_location 80 | 81 | async def fast_upload(client, file_location, reply=None, name=None, progress_bar_function = progress_bar_str): 82 | timer = Timer() 83 | if name == None: 84 | name = file_location.split("/")[-1] 85 | async def progress_bar(downloaded_bytes, total_bytes): 86 | if timer.can_send(): 87 | data = progress_bar_function(downloaded_bytes, total_bytes) 88 | await reply.edit(f"Uploading...\n{data}") 89 | if reply != None: 90 | with open(file_location, "rb") as f: 91 | the_file = await upload_file( 92 | client=client, 93 | file=f, 94 | name=name, 95 | progress_callback=progress_bar 96 | ) 97 | else: 98 | with open(file_location, "rb") as f: 99 | the_file = await upload_file( 100 | client=client, 101 | file=f, 102 | name=name, 103 | ) 104 | 105 | return the_file 106 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 MiyukiKun 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFENST.in: -------------------------------------------------------------------------------- 1 | README.md 2 | LICENSE 3 | requirements.txt -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # FastTelethon helper 2 | 3 | - Make use of FastTelethon to upload and download files 4 | 5 | ## Usage: 6 |
7 | 8 | ## Installation: 9 | ``` 10 | pip install FastTelethonhelper 11 | ``` 12 | 13 | ### Downloads: 14 | - Usage 15 | ``` 16 | from FastTelethonhelper import fast_download 17 | ``` 18 | - When you need to download file, 19 | ``` 20 | downloaded_location = await fast_download(client, msg, reply, download_folder, progress_bar_function) 21 | ``` 22 | - `client` = Telegram Client(Required) 23 | - `msg` = The message object which has the file to be downloaded(Required) 24 | - `reply` = The message on which you want the progressbar(Optional) 25 | - `download_folder` = Location where you want file to be downloaded, defaults to ./downloads/ (Optional) 26 | - `progress_bar_function` = The function you want to use to display the string in progressbar, it needs to have 2 parameters done bytes and total bytes and must return a desired string, defaults to a function I wrote(Optional) 27 | 28 |
29 | - The function returns the download location. 30 | 31 |
32 |
33 | 34 | ### Uploads: 35 | - Usage 36 | ``` 37 | from FastTelethonhelper import fast_upload 38 | ``` 39 | - When you need to upload file, 40 | ``` 41 | await fast_upload(client, file_location, reply, name, progress_bar_function) 42 | ``` 43 | - `client` = TelegramClient(Required) 44 | - `file_location` = Where the file is located(Required) 45 | - `reply` = The message object you want the progressbar to appear(Optional) 46 | - `name` = Name of the file you want while uploading(Optional) 47 | - `progress_bar_function` = The function you want to use to display the string in progressbar, it needs to have 2 parameters done bytes and total bytes and must return a desired string, defaults to a function I wrote(Optional) 48 | - This function returns the file object which you can use in send_message in telethon example 49 | ``` 50 | await bot.send_message(file=) 51 | ``` 52 | 53 |
54 |
55 | 56 | # Credits 57 | - [MiyukiKun](https://github.com/MiyukiKun) for getting this together 58 | - [Loonami](https://github.com/LonamiWebs) for [telethon](https://github.com/LonamiWebs/Telethon) 59 | - [Tulir Asokan](https://github.com/tulir) for [mautrix-telegram](https://github.com/tulir/mautrix-telegram) 60 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | telethon 2 | telethon-tgcrypto 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | with open("README.md", "r") as fh: 4 | long_description = fh.read() 5 | 6 | 7 | setup( 8 | name="FastTelethonhelper", 9 | version="1.0.2", 10 | description="Make Telethon files upload/download faster", 11 | packages=find_packages(), 12 | install_requires=["telethon", "telethon-tgcrypto"], 13 | classifiers=[ 14 | "Programming Language :: Python :: 3.6", 15 | "Programming Language :: Python :: 3.7", 16 | "Programming Language :: Python :: 3.8", 17 | "Programming Language :: Python :: 3.9", 18 | "Operating System :: OS Independent", 19 | ], 20 | long_description=long_description, 21 | long_description_content_type="text/markdown", 22 | url="https://github.com/MiyukiKun/FastTelethonhelper", 23 | author="MiyukiKun", 24 | author_email="hardikmajethia9529@gmail.com" 25 | ) 26 | --------------------------------------------------------------------------------