├── netrc ├── bot ├── helper │ ├── __init__.py │ ├── ext_utils │ │ ├── __init__.py │ │ ├── exceptions.py │ │ ├── fs_utils.py │ │ └── bot_utils.py │ ├── mirror_utils │ │ ├── __init__.py │ │ ├── status_utils │ │ │ ├── __init__.py │ │ │ ├── listeners.py │ │ │ ├── tar_status.py │ │ │ ├── status.py │ │ │ ├── upload_status.py │ │ │ ├── telegram_download_status.py │ │ │ ├── youtube_dl_download_status.py │ │ │ └── aria_download_status.py │ │ ├── upload_utils │ │ │ ├── __init__.py │ │ │ └── gdriveTools.py │ │ └── download_utils │ │ │ ├── __init__.py │ │ │ ├── download_helper.py │ │ │ ├── aria2_download.py │ │ │ ├── telegram_downloader.py │ │ │ ├── direct_link_generator_license.md │ │ │ ├── youtube_dl_download_helper.py │ │ │ └── direct_link_generator.py │ └── telegram_helper │ │ ├── __init__.py │ │ ├── filters.py │ │ ├── bot_commands.py │ │ └── message_utils.py ├── modules │ ├── __init__.py │ ├── clone.py │ ├── list.py │ ├── mirror_status.py │ ├── watch.py │ ├── authorize.py │ ├── cancel_mirror.py │ └── mirror.py ├── __init__.py └── __main__.py ├── start.sh ├── captain-definition ├── .gitmodules ├── .gitignore ├── aria.bat ├── generate_string_session.py ├── requirements.txt ├── aria.sh ├── config_sample.env ├── Dockerfile ├── generate_drive_token.py ├── add_to_team_drive.py ├── app.json ├── README.md ├── gen_sa_accounts.py └── LICENSE /netrc: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /bot/helper/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /bot/modules/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /bot/helper/ext_utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /start.sh: -------------------------------------------------------------------------------- 1 | ./aria.sh; python3 -m bot -------------------------------------------------------------------------------- /bot/helper/mirror_utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /bot/helper/telegram_helper/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /bot/helper/mirror_utils/status_utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /bot/helper/mirror_utils/upload_utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /bot/helper/mirror_utils/download_utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /bot/helper/ext_utils/exceptions.py: -------------------------------------------------------------------------------- 1 | class DirectDownloadLinkException(Exception): 2 | pass 3 | -------------------------------------------------------------------------------- /captain-definition: -------------------------------------------------------------------------------- 1 | { 2 | "schemaVersion": 2, 3 | "dockerfilePath": "./Dockerfile" 4 | } 5 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "vendor/cmrudl.py"] 2 | path = vendor/cmrudl.py 3 | url = https://github.com/JrMasterModelBuilder/cmrudl.py.git 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | config.env 2 | *auth_token.txt 3 | *.pyc 4 | downloads/* 5 | download/* 6 | data* 7 | .vscode 8 | .idea 9 | *.json 10 | *.pickle 11 | authorized_chats.txt 12 | log.txt 13 | accounts/* -------------------------------------------------------------------------------- /aria.bat: -------------------------------------------------------------------------------- 1 | aria2c --enable-rpc --rpc-listen-all=false --rpc-listen-port 6800 --max-connection-per-server=10 --rpc-max-request-size=1024M --seed-time=0.01 --min-split-size=10M --follow-torrent=mem --split=10 --daemon=true --allow-overwrite=true 2 | -------------------------------------------------------------------------------- /generate_string_session.py: -------------------------------------------------------------------------------- 1 | from pyrogram import Client 2 | 3 | API_KEY = int(input("Enter API KEY: ")) 4 | API_HASH = input("Enter API HASH: ") 5 | with Client(':memory:', api_id=API_KEY, api_hash=API_HASH) as app: 6 | print(app.export_session_string()) -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | requests 2 | python-telegram-bot==12.6.1 3 | google-api-python-client>=1.7.11,<1.7.20 4 | google-auth-httplib2>=0.0.3,<0.1.0 5 | google-auth-oauthlib>=0.4.1,<0.10.0 6 | aria2p>=0.9.0,<0.15.0 7 | python-dotenv>=0.10 8 | tenacity>=6.0.0 9 | python-magic 10 | beautifulsoup4>=4.8.2,<4.8.10 11 | Pyrogram>=0.16.0,<0.16.10 12 | TgCrypto>=1.1.1,<1.1.10 13 | youtube-dl 14 | -------------------------------------------------------------------------------- /aria.sh: -------------------------------------------------------------------------------- 1 | export MAX_DOWNLOAD_SPEED=0 2 | export MAX_CONCURRENT_DOWNLOADS=3 3 | aria2c --enable-rpc --rpc-listen-all=false --rpc-listen-port 6800 \ 4 | --max-connection-per-server=10 --rpc-max-request-size=1024M \ 5 | --seed-time=0.01 --min-split-size=10M --follow-torrent=mem --split=10 \ 6 | --daemon=true --allow-overwrite=true --max-overall-download-limit=$MAX_DOWNLOAD_SPEED \ 7 | --max-overall-upload-limit=1K --max-concurrent-downloads=$MAX_CONCURRENT_DOWNLOADS 8 | -------------------------------------------------------------------------------- /config_sample.env: -------------------------------------------------------------------------------- 1 | #Remove this line before deploying 2 | _____REMOVE_THIS_LINE_____=True 3 | 4 | # ENTER BOT TOKEN (Get your BOT_TOKEN by talking to @botfather) 5 | BOT_TOKEN = "" 6 | GDRIVE_FOLDER_ID = "" 7 | OWNER_ID = 8 | DOWNLOAD_DIR = "/home/username/mirror-bot/downloads" 9 | DOWNLOAD_STATUS_UPDATE_INTERVAL = 5 10 | AUTO_DELETE_MESSAGE_DURATION = 20 11 | IS_TEAM_DRIVE = "" 12 | INDEX_URL = "" 13 | USER_SESSION_STRING = "" 14 | TELEGRAM_API = 15 | TELEGRAM_HASH = "" 16 | USE_SERVICE_ACCOUNTS = "" 17 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:18.04 2 | 3 | WORKDIR /usr/src/app 4 | RUN chmod 777 /usr/src/app 5 | RUN apt-get -qq update 6 | RUN apt-get -qq install -y aria2 python3 python3-pip \ 7 | locales python3-lxml \ 8 | curl pv jq ffmpeg 9 | COPY requirements.txt . 10 | RUN pip3 install --no-cache-dir -r requirements.txt 11 | RUN locale-gen en_US.UTF-8 12 | ENV LANG en_US.UTF-8 13 | ENV LANGUAGE en_US:en 14 | ENV LC_ALL en_US.UTF-8 15 | COPY . . 16 | COPY netrc /root/.netrc 17 | RUN chmod +x aria.sh 18 | 19 | CMD ["bash","start.sh"] 20 | -------------------------------------------------------------------------------- /bot/helper/telegram_helper/filters.py: -------------------------------------------------------------------------------- 1 | from telegram.ext import BaseFilter 2 | from bot import AUTHORIZED_CHATS, OWNER_ID 3 | 4 | 5 | class CustomFilters: 6 | class _OwnerFilter(BaseFilter): 7 | def filter(self, message): 8 | return bool(message.from_user.id == OWNER_ID) 9 | 10 | owner_filter = _OwnerFilter() 11 | 12 | class _AuthorizedUserFilter(BaseFilter): 13 | def filter(self, message): 14 | id = message.from_user.id 15 | return bool(id in AUTHORIZED_CHATS or id == OWNER_ID) 16 | 17 | authorized_user = _AuthorizedUserFilter() 18 | 19 | class _AuthorizedChat(BaseFilter): 20 | def filter(self, message): 21 | return bool(message.chat.id in AUTHORIZED_CHATS) 22 | 23 | authorized_chat = _AuthorizedChat() 24 | -------------------------------------------------------------------------------- /bot/helper/telegram_helper/bot_commands.py: -------------------------------------------------------------------------------- 1 | class _BotCommands: 2 | def __init__(self): 3 | self.StartCommand = 'start' 4 | self.MirrorCommand = 'mirror' 5 | self.TarMirrorCommand = 'tarmirror' 6 | self.CancelMirror = 'cancel' 7 | self.CancelAllCommand = 'cancelall' 8 | self.ListCommand = 'list' 9 | self.StatusCommand = 'status' 10 | self.AuthorizeCommand = 'authorize' 11 | self.UnAuthorizeCommand = 'unauthorize' 12 | self.PingCommand = 'ping' 13 | self.RestartCommand = 'restart' 14 | self.StatsCommand = 'stats' 15 | self.HelpCommand = 'help' 16 | self.LogCommand = 'log' 17 | self.CloneCommand = "clone" 18 | self.WatchCommand = 'watch' 19 | self.TarWatchCommand = 'tarwatch' 20 | 21 | BotCommands = _BotCommands() 22 | -------------------------------------------------------------------------------- /generate_drive_token.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | import os 3 | from google_auth_oauthlib.flow import InstalledAppFlow 4 | from google.auth.transport.requests import Request 5 | 6 | credentials = None 7 | __G_DRIVE_TOKEN_FILE = "token.pickle" 8 | __OAUTH_SCOPE = ["https://www.googleapis.com/auth/drive"] 9 | if os.path.exists(__G_DRIVE_TOKEN_FILE): 10 | with open(__G_DRIVE_TOKEN_FILE, 'rb') as f: 11 | credentials = pickle.load(f) 12 | if credentials is None or not credentials.valid: 13 | if credentials and credentials.expired and credentials.refresh_token: 14 | credentials.refresh(Request()) 15 | else: 16 | flow = InstalledAppFlow.from_client_secrets_file( 17 | 'credentials.json', __OAUTH_SCOPE) 18 | credentials = flow.run_console(port=0) 19 | 20 | # Save the credentials for the next run 21 | with open(__G_DRIVE_TOKEN_FILE, 'wb') as token: 22 | pickle.dump(credentials, token) -------------------------------------------------------------------------------- /bot/helper/mirror_utils/status_utils/listeners.py: -------------------------------------------------------------------------------- 1 | class MirrorListeners: 2 | def __init__(self, context, update): 3 | self.bot = context 4 | self.update = update 5 | self.message = update.message 6 | self.uid = self.message.message_id 7 | 8 | def onDownloadStarted(self): 9 | raise NotImplementedError 10 | 11 | def onDownloadProgress(self): 12 | raise NotImplementedError 13 | 14 | def onDownloadComplete(self): 15 | raise NotImplementedError 16 | 17 | def onDownloadError(self, error: str): 18 | raise NotImplementedError 19 | 20 | def onUploadStarted(self): 21 | raise NotImplementedError 22 | 23 | def onUploadProgress(self): 24 | raise NotImplementedError 25 | 26 | def onUploadComplete(self, link: str): 27 | raise NotImplementedError 28 | 29 | def onUploadError(self, error: str): 30 | raise NotImplementedError 31 | -------------------------------------------------------------------------------- /bot/helper/mirror_utils/status_utils/tar_status.py: -------------------------------------------------------------------------------- 1 | from .status import Status 2 | from bot.helper.ext_utils.bot_utils import get_readable_file_size, MirrorStatus 3 | 4 | 5 | class TarStatus(Status): 6 | def __init__(self, name, path, size): 7 | self.__name = name 8 | self.__path = path 9 | self.__size = size 10 | 11 | # The progress of Tar function cannot be tracked. So we just return dummy values. 12 | # If this is possible in future,we should implement it 13 | 14 | def progress(self): 15 | return '0' 16 | 17 | def speed(self): 18 | return '0' 19 | 20 | def name(self): 21 | return self.__name 22 | 23 | def path(self): 24 | return self.__path 25 | 26 | def size(self): 27 | return get_readable_file_size(self.__size) 28 | 29 | def eta(self): 30 | return '0s' 31 | 32 | def status(self): 33 | return MirrorStatus.STATUS_ARCHIVING 34 | 35 | def processed_bytes(self): 36 | return 0 37 | -------------------------------------------------------------------------------- /bot/modules/clone.py: -------------------------------------------------------------------------------- 1 | from telegram.ext import CommandHandler, run_async 2 | from bot.helper.mirror_utils.upload_utils.gdriveTools import GoogleDriveHelper 3 | from bot.helper.telegram_helper.message_utils import * 4 | from bot.helper.telegram_helper.filters import CustomFilters 5 | from bot.helper.telegram_helper.bot_commands import BotCommands 6 | from bot import dispatcher 7 | 8 | 9 | @run_async 10 | def cloneNode(update,context): 11 | args = update.message.text.split(" ",maxsplit=1) 12 | if len(args) > 1: 13 | link = args[1] 14 | msg = sendMessage(f"Cloning: {link}",context.bot,update) 15 | gd = GoogleDriveHelper() 16 | result = gd.clone(link) 17 | deleteMessage(context.bot,msg) 18 | sendMessage(result,context.bot,update) 19 | else: 20 | sendMessage("Provide G-Drive Shareable Link to Clone.",bot,update) 21 | 22 | clone_handler = CommandHandler(BotCommands.CloneCommand,cloneNode,filters=CustomFilters.authorized_chat | CustomFilters.authorized_user) 23 | dispatcher.add_handler(clone_handler) -------------------------------------------------------------------------------- /bot/modules/list.py: -------------------------------------------------------------------------------- 1 | from telegram.ext import CommandHandler, run_async 2 | from bot.helper.mirror_utils.upload_utils.gdriveTools import GoogleDriveHelper 3 | from bot import LOGGER, dispatcher 4 | from bot.helper.telegram_helper.message_utils import auto_delete_message, sendMessage 5 | from bot.helper.telegram_helper.filters import CustomFilters 6 | import threading 7 | from bot.helper.telegram_helper.bot_commands import BotCommands 8 | 9 | @run_async 10 | def list_drive(update,context): 11 | message = update.message.text 12 | search = message.split(' ',maxsplit=1)[1] 13 | LOGGER.info(f"Searching: {search}") 14 | gdrive = GoogleDriveHelper(None) 15 | msg = gdrive.drive_list(search) 16 | if msg: 17 | reply_message = sendMessage(msg, context.bot, update) 18 | else: 19 | reply_message = sendMessage('No result found', context.bot, update) 20 | 21 | threading.Thread(target=auto_delete_message, args=(context.bot, update.message, reply_message)).start() 22 | 23 | 24 | list_handler = CommandHandler(BotCommands.ListCommand, list_drive,filters=CustomFilters.authorized_chat | CustomFilters.authorized_user) 25 | dispatcher.add_handler(list_handler) 26 | -------------------------------------------------------------------------------- /bot/helper/mirror_utils/download_utils/download_helper.py: -------------------------------------------------------------------------------- 1 | # An abstract class which will be inherited by the tool specific classes like aria2_helper or mega_download_helper 2 | import threading 3 | 4 | 5 | class MethodNotImplementedError(NotImplementedError): 6 | def __init__(self): 7 | super(self, 'Not implemented method') 8 | 9 | 10 | class DownloadHelper: 11 | def __init__(self): 12 | self.name = '' # Name of the download; empty string if no download has been started 13 | self.size = 0.0 # Size of the download 14 | self.downloaded_bytes = 0.0 # Bytes downloaded 15 | self.speed = 0.0 # Download speed in bytes per second 16 | self.progress = 0.0 17 | self.progress_string = '0.00%' 18 | self.eta = 0 # Estimated time of download complete 19 | self.eta_string = '0s' # A listener class which have event callbacks 20 | self._resource_lock = threading.Lock() 21 | 22 | def add_download(self, link: str, path): 23 | raise MethodNotImplementedError 24 | 25 | def cancel_download(self): 26 | # Returns None if successfully cancelled, else error string 27 | raise MethodNotImplementedError 28 | -------------------------------------------------------------------------------- /bot/helper/mirror_utils/status_utils/status.py: -------------------------------------------------------------------------------- 1 | # Generic status class. All other status classes must inherit this class 2 | 3 | 4 | class Status: 5 | 6 | def progress(self): 7 | """ 8 | Calculates the progress of the mirror (upload or download) 9 | :return: progress in percentage 10 | """ 11 | raise NotImplementedError 12 | 13 | def speed(self): 14 | """:return: speed in bytes per second""" 15 | raise NotImplementedError 16 | 17 | def name(self): 18 | """:return name of file/directory being processed""" 19 | raise NotImplementedError 20 | 21 | def path(self): 22 | """:return path of the file/directory""" 23 | raise NotImplementedError 24 | 25 | def size(self): 26 | """:return Size of file folder""" 27 | raise NotImplementedError 28 | 29 | def eta(self): 30 | """:return ETA of the process to complete""" 31 | raise NotImplementedError 32 | 33 | def status(self): 34 | """:return String describing what is the object of this class will be tracking (upload/download/something 35 | else) """ 36 | raise NotImplementedError 37 | 38 | def processed_bytes(self): 39 | """:return The size of file that has been processed (downloaded/uploaded/archived)""" 40 | raise NotImplementedError 41 | -------------------------------------------------------------------------------- /bot/modules/mirror_status.py: -------------------------------------------------------------------------------- 1 | from telegram.ext import CommandHandler, run_async 2 | from bot import dispatcher, status_reply_dict, DOWNLOAD_STATUS_UPDATE_INTERVAL, status_reply_dict_lock 3 | from bot.helper.telegram_helper.message_utils import * 4 | from time import sleep 5 | from bot.helper.ext_utils.bot_utils import get_readable_message 6 | from telegram.error import BadRequest 7 | from bot.helper.telegram_helper.filters import CustomFilters 8 | from bot.helper.telegram_helper.bot_commands import BotCommands 9 | import threading 10 | 11 | @run_async 12 | def mirror_status(update,context): 13 | message = get_readable_message() 14 | if len(message) == 0: 15 | message = "No active downloads" 16 | reply_message = sendMessage(message, context.bot, update) 17 | threading.Thread(target=auto_delete_message, args=(bot, update.message, reply_message)).start() 18 | return 19 | index = update.effective_chat.id 20 | with status_reply_dict_lock: 21 | if index in status_reply_dict.keys(): 22 | deleteMessage(bot, status_reply_dict[index]) 23 | del status_reply_dict[index] 24 | sendStatusMessage(update,context.bot) 25 | deleteMessage(context.bot,update.message) 26 | 27 | 28 | mirror_status_handler = CommandHandler(BotCommands.StatusCommand, mirror_status, 29 | filters=CustomFilters.authorized_chat | CustomFilters.authorized_user) 30 | dispatcher.add_handler(mirror_status_handler) 31 | -------------------------------------------------------------------------------- /bot/helper/mirror_utils/status_utils/upload_status.py: -------------------------------------------------------------------------------- 1 | from .status import Status 2 | from bot.helper.ext_utils.bot_utils import MirrorStatus, get_readable_file_size, get_readable_time 3 | from bot import DOWNLOAD_DIR 4 | 5 | 6 | class UploadStatus(Status): 7 | def __init__(self, obj, size, listener): 8 | self.obj = obj 9 | self.__size = size 10 | self.uid = listener.uid 11 | self.message = listener.message 12 | 13 | def path(self): 14 | return f"{DOWNLOAD_DIR}{self.uid}" 15 | 16 | def processed_bytes(self): 17 | return self.obj.uploaded_bytes 18 | 19 | def size_raw(self): 20 | return self.__size 21 | 22 | def size(self): 23 | return get_readable_file_size(self.__size) 24 | 25 | def status(self): 26 | return MirrorStatus.STATUS_UPLOADING 27 | 28 | def name(self): 29 | return self.obj.name 30 | 31 | def progress_raw(self): 32 | try: 33 | return self.obj.uploaded_bytes / self.__size * 100 34 | except ZeroDivisionError: 35 | return 0 36 | 37 | def progress(self): 38 | return f'{round(self.progress_raw(), 2)}%' 39 | 40 | def speed_raw(self): 41 | """ 42 | :return: Upload speed in Bytes/Seconds 43 | """ 44 | return self.obj.speed() 45 | 46 | def speed(self): 47 | return f'{get_readable_file_size(self.speed_raw())}/s' 48 | 49 | def eta(self): 50 | try: 51 | seconds = (self.__size - self.obj.uploaded_bytes) / self.speed_raw() 52 | return f'{get_readable_time(seconds)}' 53 | except ZeroDivisionError: 54 | return '-' 55 | -------------------------------------------------------------------------------- /bot/helper/mirror_utils/status_utils/telegram_download_status.py: -------------------------------------------------------------------------------- 1 | from bot import DOWNLOAD_DIR 2 | from bot.helper.ext_utils.bot_utils import MirrorStatus, get_readable_file_size, get_readable_time 3 | from .status import Status 4 | 5 | 6 | class TelegramDownloadStatus(Status): 7 | def __init__(self, obj, listener): 8 | self.obj = obj 9 | self.uid = listener.uid 10 | self.message = listener.message 11 | 12 | def gid(self): 13 | return self.obj.gid 14 | 15 | def path(self): 16 | return f"{DOWNLOAD_DIR}{self.uid}" 17 | 18 | def processed_bytes(self): 19 | return self.obj.downloaded_bytes 20 | 21 | def size_raw(self): 22 | return self.obj.size 23 | 24 | def size(self): 25 | return get_readable_file_size(self.size_raw()) 26 | 27 | def status(self): 28 | return MirrorStatus.STATUS_DOWNLOADING 29 | 30 | def name(self): 31 | return self.obj.name 32 | 33 | def progress_raw(self): 34 | return self.obj.progress 35 | 36 | def progress(self): 37 | return f'{round(self.progress_raw(), 2)}%' 38 | 39 | def speed_raw(self): 40 | """ 41 | :return: Download speed in Bytes/Seconds 42 | """ 43 | return self.obj.download_speed 44 | 45 | def speed(self): 46 | return f'{get_readable_file_size(self.speed_raw())}/s' 47 | 48 | def eta(self): 49 | try: 50 | seconds = (self.size_raw() - self.processed_bytes()) / self.speed_raw() 51 | return f'{get_readable_time(seconds)}' 52 | except ZeroDivisionError: 53 | return '-' 54 | 55 | def download(self): 56 | return self.obj 57 | -------------------------------------------------------------------------------- /bot/helper/mirror_utils/status_utils/youtube_dl_download_status.py: -------------------------------------------------------------------------------- 1 | from bot import DOWNLOAD_DIR 2 | from bot.helper.ext_utils.bot_utils import MirrorStatus, get_readable_file_size, get_readable_time 3 | from .status import Status 4 | 5 | 6 | class YoutubeDLDownloadStatus(Status): 7 | def __init__(self, obj, listener): 8 | self.obj = obj 9 | self.uid = listener.uid 10 | self.message = listener.message 11 | 12 | def gid(self): 13 | return self.obj.gid 14 | 15 | def path(self): 16 | return f"{DOWNLOAD_DIR}{self.uid}" 17 | 18 | def processed_bytes(self): 19 | return self.obj.downloaded_bytes 20 | 21 | def size_raw(self): 22 | return self.obj.size 23 | 24 | def size(self): 25 | return get_readable_file_size(self.size_raw()) 26 | 27 | def status(self): 28 | return MirrorStatus.STATUS_DOWNLOADING 29 | 30 | def name(self): 31 | return self.obj.name 32 | 33 | def progress_raw(self): 34 | return self.obj.progress 35 | 36 | def progress(self): 37 | return f'{round(self.progress_raw(), 2)}%' 38 | 39 | def speed_raw(self): 40 | """ 41 | :return: Download speed in Bytes/Seconds 42 | """ 43 | return self.obj.download_speed 44 | 45 | def speed(self): 46 | return f'{get_readable_file_size(self.speed_raw())}/s' 47 | 48 | def eta(self): 49 | try: 50 | seconds = (self.size_raw() - self.processed_bytes()) / self.speed_raw() 51 | return f'{get_readable_time(seconds)}' 52 | except ZeroDivisionError: 53 | return '-' 54 | 55 | def download(self): 56 | return self.obj 57 | -------------------------------------------------------------------------------- /bot/helper/ext_utils/fs_utils.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from bot import aria2, LOGGER, DOWNLOAD_DIR 3 | import shutil 4 | import os 5 | import pathlib 6 | import magic 7 | import tarfile 8 | 9 | 10 | def clean_download(path: str): 11 | if os.path.exists(path): 12 | LOGGER.info(f"Cleaning download: {path}") 13 | shutil.rmtree(path) 14 | 15 | 16 | def start_cleanup(): 17 | try: 18 | shutil.rmtree(DOWNLOAD_DIR) 19 | except FileNotFoundError: 20 | pass 21 | 22 | 23 | def clean_all(): 24 | aria2.remove_all(True) 25 | shutil.rmtree(DOWNLOAD_DIR) 26 | 27 | 28 | def exit_clean_up(signal, frame): 29 | try: 30 | LOGGER.info("Please wait, while we clean up the downloads and stop running downloads") 31 | clean_all() 32 | sys.exit(0) 33 | except KeyboardInterrupt: 34 | LOGGER.warning("Force Exiting before the cleanup finishes!") 35 | sys.exit(1) 36 | 37 | 38 | def get_path_size(path): 39 | if os.path.isfile(path): 40 | return os.path.getsize(path) 41 | total_size = 0 42 | for root, dirs, files in os.walk(path): 43 | for f in files: 44 | abs_path = os.path.join(root, f) 45 | total_size += os.path.getsize(abs_path) 46 | return total_size 47 | 48 | 49 | def tar(org_path): 50 | tar_path = org_path + ".tar" 51 | path = pathlib.PurePath(org_path) 52 | LOGGER.info(f'Tar: orig_path: {org_path}, tar_path: {tar_path}') 53 | tar = tarfile.open(tar_path, "w") 54 | tar.add(org_path, arcname=path.name) 55 | tar.close() 56 | return tar_path 57 | 58 | 59 | def get_mime_type(file_path): 60 | mime = magic.Magic(mime=True) 61 | mime_type = mime.from_file(file_path) 62 | mime_type = mime_type if mime_type else "text/plain" 63 | return mime_type 64 | -------------------------------------------------------------------------------- /bot/modules/watch.py: -------------------------------------------------------------------------------- 1 | from telegram.ext import CommandHandler, run_async 2 | from telegram import Bot, Update 3 | from bot import Interval, DOWNLOAD_DIR, DOWNLOAD_STATUS_UPDATE_INTERVAL, dispatcher, LOGGER 4 | from bot.helper.ext_utils.bot_utils import setInterval 5 | from bot.helper.telegram_helper.message_utils import update_all_messages, sendMessage, sendStatusMessage 6 | from .mirror import MirrorListener 7 | from bot.helper.mirror_utils.download_utils.youtube_dl_download_helper import YoutubeDLHelper 8 | from bot.helper.telegram_helper.bot_commands import BotCommands 9 | from bot.helper.telegram_helper.filters import CustomFilters 10 | import threading 11 | 12 | 13 | def _watch(bot: Bot, update: Update, args: list, isTar=False): 14 | try: 15 | link = args[0] 16 | except IndexError: 17 | sendMessage(f'/{BotCommands.WatchCommand} [yt_dl supported link] to mirror with youtube_dl', bot, update) 18 | return 19 | reply_to = update.message.reply_to_message 20 | if reply_to is not None: 21 | tag = reply_to.from_user.username 22 | else: 23 | tag = None 24 | 25 | listener = MirrorListener(bot, update, isTar, tag) 26 | ydl = YoutubeDLHelper(listener) 27 | threading.Thread(target=ydl.add_download,args=(link, f'{DOWNLOAD_DIR}{listener.uid}')).start() 28 | sendStatusMessage(update, bot) 29 | if len(Interval) == 0: 30 | Interval.append(setInterval(DOWNLOAD_STATUS_UPDATE_INTERVAL, update_all_messages)) 31 | 32 | 33 | @run_async 34 | def watchTar(update, context): 35 | _watch(context.bot, update, context.args, True) 36 | 37 | 38 | def watch(update, context): 39 | _watch(context.bot, update, context.args) 40 | 41 | 42 | mirror_handler = CommandHandler(BotCommands.WatchCommand, watch, 43 | pass_args=True, 44 | filters=CustomFilters.authorized_chat | CustomFilters.authorized_user) 45 | tar_mirror_handler = CommandHandler(BotCommands.TarWatchCommand, watchTar, 46 | pass_args=True, 47 | filters=CustomFilters.authorized_chat | CustomFilters.authorized_user) 48 | dispatcher.add_handler(mirror_handler) 49 | dispatcher.add_handler(tar_mirror_handler) 50 | -------------------------------------------------------------------------------- /bot/helper/mirror_utils/status_utils/aria_download_status.py: -------------------------------------------------------------------------------- 1 | from bot import aria2, DOWNLOAD_DIR 2 | from bot.helper.ext_utils.bot_utils import MirrorStatus 3 | from .status import Status 4 | 5 | 6 | def get_download(gid): 7 | return aria2.get_download(gid) 8 | 9 | 10 | class AriaDownloadStatus(Status): 11 | 12 | def __init__(self, obj, listener): 13 | super().__init__() 14 | self.upload_name = None 15 | self.is_archiving = False 16 | self.obj = obj 17 | self.__gid = obj.gid 18 | self.__download = get_download(obj.gid) 19 | self.__uid = listener.uid 20 | self.__listener = listener 21 | self.message = listener.message 22 | self.last = None 23 | self.is_waiting = False 24 | 25 | def __update(self): 26 | self.__download = get_download(self.__gid) 27 | 28 | def progress(self): 29 | """ 30 | Calculates the progress of the mirror (upload or download) 31 | :return: returns progress in percentage 32 | """ 33 | self.__update() 34 | return self.__download.progress_string() 35 | 36 | def size_raw(self): 37 | """ 38 | Gets total size of the mirror file/folder 39 | :return: total size of mirror 40 | """ 41 | return self.aria_download().total_length 42 | 43 | def processed_bytes(self): 44 | return self.aria_download().completed_length 45 | 46 | def speed(self): 47 | return self.aria_download().download_speed_string() 48 | 49 | def name(self): 50 | return self.aria_download().name 51 | 52 | def path(self): 53 | return f"{DOWNLOAD_DIR}{self.__uid}" 54 | 55 | def size(self): 56 | return self.aria_download().total_length_string() 57 | 58 | def eta(self): 59 | return self.aria_download().eta_string() 60 | 61 | def status(self): 62 | download = self.aria_download() 63 | if download.is_waiting: 64 | status = MirrorStatus.STATUS_WAITING 65 | elif download.is_paused: 66 | status = MirrorStatus.STATUS_CANCELLED 67 | elif download.has_failed: 68 | status = MirrorStatus.STATUS_FAILED 69 | else: 70 | status = MirrorStatus.STATUS_DOWNLOADING 71 | return status 72 | 73 | def aria_download(self): 74 | self.__update() 75 | return self.__download 76 | 77 | def download(self): 78 | return self.obj 79 | 80 | def uid(self): 81 | return self.__uid 82 | 83 | def gid(self): 84 | self.__update() 85 | return self.__gid 86 | -------------------------------------------------------------------------------- /bot/modules/authorize.py: -------------------------------------------------------------------------------- 1 | from bot.helper.telegram_helper.message_utils import sendMessage 2 | from telegram.ext import run_async 3 | from bot import AUTHORIZED_CHATS, dispatcher 4 | from telegram.ext import CommandHandler 5 | from bot.helper.telegram_helper.filters import CustomFilters 6 | from telegram.ext import Filters 7 | from telegram import Update 8 | from bot.helper.telegram_helper.bot_commands import BotCommands 9 | 10 | 11 | @run_async 12 | def authorize(update,context): 13 | reply_message = update.message.reply_to_message 14 | msg = '' 15 | with open('authorized_chats.txt', 'a') as file: 16 | if reply_message is None: 17 | # Trying to authorize a chat 18 | chat_id = update.effective_chat.id 19 | if chat_id not in AUTHORIZED_CHATS: 20 | file.write(f'{chat_id}\n') 21 | AUTHORIZED_CHATS.add(chat_id) 22 | msg = 'Chat authorized' 23 | else: 24 | msg = 'Already authorized chat' 25 | else: 26 | # Trying to authorize someone in specific 27 | user_id = reply_message.from_user.id 28 | if user_id not in AUTHORIZED_CHATS: 29 | file.write(f'{user_id}\n') 30 | AUTHORIZED_CHATS.add(user_id) 31 | msg = 'Person Authorized to use the bot!' 32 | else: 33 | msg = 'Person already authorized' 34 | sendMessage(msg, context.bot, update) 35 | 36 | 37 | @run_async 38 | def unauthorize(update,context): 39 | reply_message = update.message.reply_to_message 40 | if reply_message is None: 41 | # Trying to unauthorize a chat 42 | chat_id = update.effective_chat.id 43 | if chat_id in AUTHORIZED_CHATS: 44 | AUTHORIZED_CHATS.remove(chat_id) 45 | msg = 'Chat unauthorized' 46 | else: 47 | msg = 'Already unauthorized chat' 48 | else: 49 | # Trying to authorize someone in specific 50 | user_id = reply_message.from_user.id 51 | if user_id in AUTHORIZED_CHATS: 52 | AUTHORIZED_CHATS.remove(user_id) 53 | msg = 'Person unauthorized to use the bot!' 54 | else: 55 | msg = 'Person already unauthorized!' 56 | with open('authorized_chats.txt', 'a') as file: 57 | file.truncate(0) 58 | for i in AUTHORIZED_CHATS: 59 | file.write(f'{i}\n') 60 | sendMessage(msg, context.bot, update) 61 | 62 | 63 | authorize_handler = CommandHandler(command=BotCommands.AuthorizeCommand, callback=authorize, 64 | filters=CustomFilters.owner_filter & Filters.group) 65 | unauthorize_handler = CommandHandler(command=BotCommands.UnAuthorizeCommand, callback=unauthorize, 66 | filters=CustomFilters.owner_filter & Filters.group) 67 | dispatcher.add_handler(authorize_handler) 68 | dispatcher.add_handler(unauthorize_handler) 69 | 70 | -------------------------------------------------------------------------------- /bot/modules/cancel_mirror.py: -------------------------------------------------------------------------------- 1 | from telegram.ext import CommandHandler, run_async 2 | 3 | from bot import download_dict, dispatcher, download_dict_lock, DOWNLOAD_DIR 4 | from bot.helper.ext_utils.fs_utils import clean_download 5 | from bot.helper.telegram_helper.bot_commands import BotCommands 6 | from bot.helper.telegram_helper.filters import CustomFilters 7 | from bot.helper.telegram_helper.message_utils import * 8 | 9 | from time import sleep 10 | from bot.helper.ext_utils.bot_utils import getDownloadByGid, MirrorStatus 11 | 12 | 13 | @run_async 14 | def cancel_mirror(update,context): 15 | args = update.message.text.split(" ",maxsplit=1) 16 | mirror_message = None 17 | if len(args) > 1: 18 | gid = args[1] 19 | dl = getDownloadByGid(gid) 20 | if not dl: 21 | sendMessage(f"GID: {gid} not found.",context.bot,update) 22 | return 23 | with download_dict_lock: 24 | keys = list(download_dict.keys()) 25 | mirror_message = dl.message 26 | elif update.message.reply_to_message: 27 | mirror_message = update.message.reply_to_message 28 | with download_dict_lock: 29 | keys = list(download_dict.keys()) 30 | dl = download_dict[mirror_message.message_id] 31 | if len(args) == 1: 32 | if mirror_message is None or mirror_message.message_id not in keys: 33 | if BotCommands.MirrorCommand in mirror_message.text or \ 34 | BotCommands.TarMirrorCommand in mirror_message.text: 35 | msg = "Mirror already have been cancelled" 36 | sendMessage(msg,context.bot,update) 37 | return 38 | else: 39 | msg = "Please reply to the /mirror message which was used to start the download or /cancel gid to cancel it!" 40 | sendMessage(msg,context.bot,update) 41 | return 42 | if dl.status() == "Uploading": 43 | sendMessage("Upload in Progress, Don't Cancel it.", context.bot, update) 44 | return 45 | elif dl.status() == "Archiving": 46 | sendMessage("Archival in Progress, Don't Cancel it.", context.bot, update) 47 | return 48 | else: 49 | dl.download().cancel_download() 50 | sleep(1) # Wait a Second For Aria2 To free Resources. 51 | clean_download(f'{DOWNLOAD_DIR}{mirror_message.message_id}/') 52 | 53 | 54 | @run_async 55 | def cancel_all(update, context): 56 | with download_dict_lock: 57 | count = 0 58 | for dlDetails in list(download_dict.values()): 59 | if dlDetails.status() == MirrorStatus.STATUS_DOWNLOADING\ 60 | or dlDetails.status() == MirrorStatus.STATUS_WAITING: 61 | dlDetails.download().cancel_download() 62 | count += 1 63 | delete_all_messages() 64 | sendMessage(f'Cancelled {count} downloads!', context.bot,update) 65 | 66 | 67 | cancel_mirror_handler = CommandHandler(BotCommands.CancelMirror, cancel_mirror, 68 | filters=CustomFilters.authorized_chat | CustomFilters.authorized_user) 69 | cancel_all_handler = CommandHandler(BotCommands.CancelAllCommand, cancel_all, 70 | filters=CustomFilters.owner_filter) 71 | dispatcher.add_handler(cancel_all_handler) 72 | dispatcher.add_handler(cancel_mirror_handler) 73 | -------------------------------------------------------------------------------- /add_to_team_drive.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | from google.oauth2.service_account import Credentials 3 | import googleapiclient.discovery, json, progress.bar, glob, sys, argparse, time 4 | from google_auth_oauthlib.flow import InstalledAppFlow 5 | from google.auth.transport.requests import Request 6 | import os, pickle 7 | 8 | stt = time.time() 9 | 10 | parse = argparse.ArgumentParser( 11 | description='A tool to add service accounts to a shared drive from a folder containing credential files.') 12 | parse.add_argument('--path', '-p', default='accounts', 13 | help='Specify an alternative path to the service accounts folder.') 14 | parse.add_argument('--credentials', '-c', default='./credentials.json', 15 | help='Specify the relative path for the credentials file.') 16 | parse.add_argument('--yes', '-y', default=False, action='store_true', help='Skips the sanity prompt.') 17 | parsereq = parse.add_argument_group('required arguments') 18 | parsereq.add_argument('--drive-id', '-d', help='The ID of the Shared Drive.', required=True) 19 | 20 | args = parse.parse_args() 21 | acc_dir = args.path 22 | did = args.drive_id 23 | credentials = glob.glob(args.credentials) 24 | 25 | try: 26 | open(credentials[0], 'r') 27 | print('>> Found credentials.') 28 | except IndexError: 29 | print('>> No credentials found.') 30 | sys.exit(0) 31 | 32 | if not args.yes: 33 | # input('Make sure the following client id is added to the shared drive as Manager:\n' + json.loads((open( 34 | # credentials[0],'r').read()))['installed']['client_id']) 35 | input('>> Make sure the **Google account** that has generated credentials.json\n is added into your Team Drive ' 36 | '(shared drive) as Manager\n>> (Press any key to continue)') 37 | 38 | creds = None 39 | if os.path.exists('token_sa.pickle'): 40 | with open('token_sa.pickle', 'rb') as token: 41 | creds = pickle.load(token) 42 | # If there are no (valid) credentials available, let the user log in. 43 | if not creds or not creds.valid: 44 | if creds and creds.expired and creds.refresh_token: 45 | creds.refresh(Request()) 46 | else: 47 | flow = InstalledAppFlow.from_client_secrets_file(credentials[0], scopes=[ 48 | 'https://www.googleapis.com/auth/admin.directory.group', 49 | 'https://www.googleapis.com/auth/admin.directory.group.member' 50 | ]) 51 | # creds = flow.run_local_server(port=0) 52 | creds = flow.run_console() 53 | # Save the credentials for the next run 54 | with open('token_sa.pickle', 'wb') as token: 55 | pickle.dump(creds, token) 56 | 57 | drive = googleapiclient.discovery.build("drive", "v3", credentials=creds) 58 | batch = drive.new_batch_http_request() 59 | 60 | aa = glob.glob('%s/*.json' % acc_dir) 61 | pbar = progress.bar.Bar("Readying accounts", max=len(aa)) 62 | for i in aa: 63 | ce = json.loads(open(i, 'r').read())['client_email'] 64 | batch.add(drive.permissions().create(fileId=did, supportsAllDrives=True, body={ 65 | "role": "fileOrganizer", 66 | "type": "user", 67 | "emailAddress": ce 68 | })) 69 | pbar.next() 70 | pbar.finish() 71 | print('Adding...') 72 | batch.execute() 73 | 74 | print('Complete.') 75 | hours, rem = divmod((time.time() - stt), 3600) 76 | minutes, sec = divmod(rem, 60) 77 | print("Elapsed Time:\n{:0>2}:{:0>2}:{:05.2f}".format(int(hours), int(minutes), sec)) -------------------------------------------------------------------------------- /bot/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import threading 4 | import time 5 | 6 | import aria2p 7 | import telegram.ext as tg 8 | from dotenv import load_dotenv 9 | import socket 10 | 11 | socket.setdefaulttimeout(600) 12 | 13 | botStartTime = time.time() 14 | if os.path.exists('log.txt'): 15 | with open('log.txt', 'r+') as f: 16 | f.truncate(0) 17 | 18 | logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', 19 | handlers=[logging.FileHandler('log.txt'), logging.StreamHandler()], 20 | level=logging.INFO) 21 | 22 | load_dotenv('config.env') 23 | 24 | Interval = [] 25 | 26 | 27 | def getConfig(name: str): 28 | return os.environ[name] 29 | 30 | 31 | LOGGER = logging.getLogger(__name__) 32 | 33 | try: 34 | if bool(getConfig('_____REMOVE_THIS_LINE_____')): 35 | logging.error('The README.md file there to be read! Exiting now!') 36 | exit() 37 | except KeyError: 38 | pass 39 | 40 | aria2 = aria2p.API( 41 | aria2p.Client( 42 | host="http://localhost", 43 | port=6800, 44 | secret="", 45 | ) 46 | ) 47 | 48 | DOWNLOAD_DIR = None 49 | BOT_TOKEN = None 50 | 51 | download_dict_lock = threading.Lock() 52 | status_reply_dict_lock = threading.Lock() 53 | # Key: update.effective_chat.id 54 | # Value: telegram.Message 55 | status_reply_dict = {} 56 | # Key: update.message.message_id 57 | # Value: An object of DownloadStatus 58 | download_dict = {} 59 | # Stores list of users and chats the bot is authorized to use in 60 | AUTHORIZED_CHATS = set() 61 | if os.path.exists('authorized_chats.txt'): 62 | with open('authorized_chats.txt', 'r+') as f: 63 | lines = f.readlines() 64 | for line in lines: 65 | # LOGGER.info(line.split()) 66 | AUTHORIZED_CHATS.add(int(line.split()[0])) 67 | try: 68 | BOT_TOKEN = getConfig('BOT_TOKEN') 69 | parent_id = getConfig('GDRIVE_FOLDER_ID') 70 | DOWNLOAD_DIR = getConfig('DOWNLOAD_DIR') 71 | if DOWNLOAD_DIR[-1] != '/' or DOWNLOAD_DIR[-1] != '\\': 72 | DOWNLOAD_DIR = DOWNLOAD_DIR + '/' 73 | DOWNLOAD_STATUS_UPDATE_INTERVAL = int(getConfig('DOWNLOAD_STATUS_UPDATE_INTERVAL')) 74 | OWNER_ID = int(getConfig('OWNER_ID')) 75 | AUTO_DELETE_MESSAGE_DURATION = int(getConfig('AUTO_DELETE_MESSAGE_DURATION')) 76 | USER_SESSION_STRING = getConfig('USER_SESSION_STRING') 77 | TELEGRAM_API = getConfig('TELEGRAM_API') 78 | TELEGRAM_HASH = getConfig('TELEGRAM_HASH') 79 | except KeyError as e: 80 | LOGGER.error("One or more env variables missing! Exiting now") 81 | exit(1) 82 | try: 83 | INDEX_URL = getConfig('INDEX_URL') 84 | if len(INDEX_URL) == 0: 85 | INDEX_URL = None 86 | except KeyError: 87 | INDEX_URL = None 88 | try: 89 | IS_TEAM_DRIVE = getConfig('IS_TEAM_DRIVE') 90 | if IS_TEAM_DRIVE.lower() == 'true': 91 | IS_TEAM_DRIVE = True 92 | else: 93 | IS_TEAM_DRIVE = False 94 | except KeyError: 95 | IS_TEAM_DRIVE = False 96 | 97 | try: 98 | USE_SERVICE_ACCOUNTS = getConfig('USE_SERVICE_ACCOUNTS') 99 | if USE_SERVICE_ACCOUNTS.lower() == 'true': 100 | USE_SERVICE_ACCOUNTS = True 101 | else: 102 | USE_SERVICE_ACCOUNTS = False 103 | except KeyError: 104 | USE_SERVICE_ACCOUNTS = False 105 | 106 | updater = tg.Updater(token=BOT_TOKEN,use_context=True) 107 | bot = updater.bot 108 | dispatcher = updater.dispatcher 109 | -------------------------------------------------------------------------------- /bot/helper/telegram_helper/message_utils.py: -------------------------------------------------------------------------------- 1 | from telegram.message import Message 2 | from telegram.update import Update 3 | import time 4 | from bot import AUTO_DELETE_MESSAGE_DURATION, LOGGER, bot, \ 5 | status_reply_dict, status_reply_dict_lock 6 | from bot.helper.ext_utils.bot_utils import get_readable_message 7 | from telegram.error import TimedOut, BadRequest 8 | from bot import bot 9 | 10 | 11 | def sendMessage(text: str, bot, update: Update): 12 | return bot.send_message(update.message.chat_id, 13 | reply_to_message_id=update.message.message_id, 14 | text=text, parse_mode='HTMl') 15 | 16 | 17 | def editMessage(text: str, message: Message): 18 | try: 19 | bot.edit_message_text(text=text, message_id=message.message_id, 20 | chat_id=message.chat.id, 21 | parse_mode='HTMl') 22 | except TimedOut as e: 23 | LOGGER.error(str(e)) 24 | pass 25 | 26 | 27 | def deleteMessage(bot, message: Message): 28 | try: 29 | bot.delete_message(chat_id=message.chat.id, 30 | message_id=message.message_id) 31 | except Exception as e: 32 | LOGGER.error(str(e)) 33 | 34 | 35 | def sendLogFile(bot, update: Update): 36 | with open('log.txt', 'rb') as f: 37 | bot.send_document(document=f, filename=f.name, 38 | reply_to_message_id=update.message.message_id, 39 | chat_id=update.message.chat_id) 40 | 41 | 42 | def auto_delete_message(bot, cmd_message: Message, bot_message: Message): 43 | if AUTO_DELETE_MESSAGE_DURATION != -1: 44 | time.sleep(AUTO_DELETE_MESSAGE_DURATION) 45 | try: 46 | # Skip if None is passed meaning we don't want to delete bot xor cmd message 47 | deleteMessage(bot, cmd_message) 48 | deleteMessage(bot, bot_message) 49 | except AttributeError: 50 | pass 51 | 52 | 53 | def delete_all_messages(): 54 | with status_reply_dict_lock: 55 | for message in list(status_reply_dict.values()): 56 | try: 57 | deleteMessage(bot, message) 58 | del status_reply_dict[message.chat.id] 59 | except BadRequest as e: 60 | LOGGER.info(str(e)) 61 | del status_reply_dict[message.chat.id] 62 | pass 63 | 64 | 65 | def update_all_messages(): 66 | msg = get_readable_message() 67 | with status_reply_dict_lock: 68 | for chat_id in list(status_reply_dict.keys()): 69 | if msg != status_reply_dict[chat_id].text: 70 | try: 71 | editMessage(msg, status_reply_dict[chat_id]) 72 | except BadRequest as e: 73 | LOGGER.error(str(e)) 74 | status_reply_dict[chat_id].text = msg 75 | 76 | 77 | def sendStatusMessage(msg, bot): 78 | progress = get_readable_message() 79 | with status_reply_dict_lock: 80 | if msg.message.chat.id in list(status_reply_dict.keys()): 81 | try: 82 | message = status_reply_dict[msg.message.chat.id] 83 | deleteMessage(bot, message) 84 | del status_reply_dict[msg.message.chat.id] 85 | except Exception as e: 86 | LOGGER.error(str(e)) 87 | del status_reply_dict[msg.message.chat.id] 88 | pass 89 | message = sendMessage(progress, bot, msg) 90 | status_reply_dict[msg.message.chat.id] = message 91 | -------------------------------------------------------------------------------- /app.json: -------------------------------------------------------------------------------- 1 | 2 | 3 | { 4 | 5 | "name": "Mirror to GDrive", 6 | 7 | "description": "Bot created by a Aditya\n\n Support - https://t.me/xditya_bot", 8 | 9 | "logo": "https://telegra.ph/file/f1143e4dd5a4c699e3614.jpg", 10 | 11 | "keywords": [ 12 | 13 | "telegram", 14 | 15 | "bot", 16 | 17 | "plugin", 18 | 19 | "modular", 20 | 21 | "productivity" 22 | 23 | ], 24 | 25 | "repository": "https://github.com/xditya/Mirror2GDrive", 26 | 27 | "website": "#TODO", 28 | 29 | "success_url": "#TODO", 30 | 31 | "env": { 32 | 33 | "ENV": { 34 | 35 | "description": "Setting this to ANYTHING will enable heroku.", 36 | 37 | "value": "ANYTHING" 38 | 39 | }, 40 | 41 | "BOT_TOKEN": { 42 | 43 | "description": "The telegram bot token that you get from @BotFather.", 44 | 45 | "value": "", 46 | 47 | "required": true 48 | 49 | }, 50 | 51 | "GDRIVE_FOLDER_ID": { 52 | 53 | "description": "This is the folder ID of the Google Drive Folder to which you want to upload all the mirrors.", 54 | 55 | "value": "" 56 | 57 | }, 58 | 59 | "DOWNLOAD_DIR: { 60 | 61 | "description": "The path to the local folder where the downloads should be downloaded to", 62 | 63 | "value": "" 64 | 65 | }, 66 | 67 | "DOWNLOAD_STATUS_UPDATE_INTERVAL": { 68 | 69 | "description": "A short interval of time in seconds after which the Mirror progress message is updated. (I recommend to keep it 5 seconds at least)", 70 | 71 | "value": "" 72 | 73 | }, 74 | 75 | "OWNER_ID": { 76 | 77 | "description": "The Telegram user ID (not username) of the owner of the bot.", 78 | 79 | "value": "", 80 | 81 | "required": false 82 | 83 | }, 84 | 85 | "AUTO_DELETE_MESSAGE_DURATION": { 86 | 87 | "description": "Interval of time (in seconds), after which the bot deletes it's message (and command message) which is expected to be viewed instantly. Note: Set to -1 to never automatically delete messages", 88 | 89 | "value": "", 90 | 91 | "required": false 92 | 93 | }, 94 | 95 | "IS_TEAM_DRIVE": { 96 | 97 | "description": "(Optional field) Set to "True" if GDRIVE_FOLDER_ID is from a Team Drive else False or Leave it empty..", 98 | 99 | "value": "", 100 | 101 | "required": false 102 | 103 | }, 104 | 105 | "USE_SERVICE_ACCOUNTS": { 106 | 107 | "description": "(Optional field) (Leave empty if unsure) Whether to use service accounts or not. For this to work see "Using service accounts" section below..", 108 | 109 | "value": "", 110 | 111 | "required": true 112 | 113 | }, 114 | 115 | "INDEX_URL ": { 116 | "description": "(Optional field) Refer to https://github.com/maple3142/GDIndex/ The URL should not have any trailing '/' ", 117 | "value": "", 118 | "required": false 119 | }, 120 | 121 | "TELEGRAM_API": { 122 | "description": "This is to authenticate to your telegram account for downloading Telegram files. You can get this from https://my.telegram.org DO NOT put this in quotes.", 123 | "value": "", 124 | "required": false 125 | }, 126 | 127 | "TELEGRAM_HASH": { 128 | 129 | "description": "This is to authenticate to your telegram account for downloading Telegram files. You can get this from https://my.telegram.org", 130 | 131 | "value": "", 132 | 133 | "required": false 134 | 135 | }, 136 | 137 | "USER_SESSION_STRING ": { 138 | 139 | "description": "Session string generated by running: https://telebot-sessionstring-generator.xditya.repl.run/", 140 | 141 | "value": "", 142 | 143 | "required": true 144 | } 145 | 146 | } 147 | -------------------------------------------------------------------------------- /bot/helper/mirror_utils/download_utils/aria2_download.py: -------------------------------------------------------------------------------- 1 | from bot import aria2 2 | from bot.helper.ext_utils.bot_utils import * 3 | from .download_helper import DownloadHelper 4 | from bot.helper.mirror_utils.status_utils.aria_download_status import AriaDownloadStatus 5 | from bot.helper.telegram_helper.message_utils import * 6 | import threading 7 | from aria2p import API 8 | 9 | 10 | class AriaDownloadHelper(DownloadHelper): 11 | 12 | def __init__(self, listener): 13 | super().__init__() 14 | self.gid = None 15 | self.__listener = listener 16 | self._resource_lock = threading.RLock() 17 | 18 | def __onDownloadStarted(self, api, gid): 19 | with self._resource_lock: 20 | if self.gid == gid: 21 | download = api.get_download(gid) 22 | self.name = download.name 23 | update_all_messages() 24 | 25 | def __onDownloadComplete(self, api: API, gid): 26 | with self._resource_lock: 27 | if self.gid == gid: 28 | download = api.get_download(gid) 29 | if download.followed_by_ids: 30 | self.gid = download.followed_by_ids[0] 31 | with download_dict_lock: 32 | download_dict[self.__listener.uid] = AriaDownloadStatus(self, self.__listener) 33 | if download.is_torrent: 34 | download_dict[self.__listener.uid].is_torrent = True 35 | update_all_messages() 36 | LOGGER.info(f'Changed gid from {gid} to {self.gid}') 37 | else: 38 | self.__listener.onDownloadComplete() 39 | 40 | def __onDownloadPause(self, api, gid): 41 | if self.gid == gid: 42 | LOGGER.info("Called onDownloadPause") 43 | self.__listener.onDownloadError('Download stopped by user!') 44 | 45 | def __onDownloadStopped(self, api, gid): 46 | if self.gid == gid: 47 | LOGGER.info("Called on_download_stop") 48 | self.__listener.onDownloadError('Download stopped by user!') 49 | 50 | def __onDownloadError(self, api, gid): 51 | with self._resource_lock: 52 | if self.gid == gid: 53 | download = api.get_download(gid) 54 | error = download.error_message 55 | LOGGER.info(f"Download Error: {error}") 56 | self.__listener.onDownloadError(error) 57 | 58 | def add_download(self, link: str, path): 59 | if is_magnet(link): 60 | download = aria2.add_magnet(link, {'dir': path}) 61 | else: 62 | download = aria2.add_uris([link], {'dir': path}) 63 | self.gid = download.gid 64 | with download_dict_lock: 65 | download_dict[self.__listener.uid] = AriaDownloadStatus(self, self.__listener) 66 | if download.error_message: 67 | self.__listener.onDownloadError(download.error_message) 68 | return 69 | LOGGER.info(f"Started: {self.gid} DIR:{download.dir} ") 70 | aria2.listen_to_notifications(threaded=True, on_download_start=self.__onDownloadStarted, 71 | on_download_error=self.__onDownloadError, 72 | on_download_pause=self.__onDownloadPause, 73 | on_download_stop=self.__onDownloadStopped, 74 | on_download_complete=self.__onDownloadComplete) 75 | 76 | def cancel_download(self): 77 | download = aria2.get_download(self.gid) 78 | if download.is_waiting: 79 | aria2.remove([download]) 80 | self.__listener.onDownloadError("Cancelled by user") 81 | return 82 | if len(download.followed_by_ids) != 0: 83 | downloads = aria2.get_downloads(download.followed_by_ids) 84 | aria2.pause(downloads) 85 | aria2.pause([download]) 86 | -------------------------------------------------------------------------------- /bot/helper/mirror_utils/download_utils/telegram_downloader.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import threading 3 | import time 4 | 5 | from pyrogram import Client 6 | 7 | from bot import LOGGER, download_dict, download_dict_lock, TELEGRAM_API, \ 8 | TELEGRAM_HASH, USER_SESSION_STRING 9 | from .download_helper import DownloadHelper 10 | from ..status_utils.telegram_download_status import TelegramDownloadStatus 11 | 12 | global_lock = threading.Lock() 13 | GLOBAL_GID = set() 14 | 15 | logging.getLogger("pyrogram").setLevel(logging.WARNING) 16 | 17 | 18 | class TelegramDownloadHelper(DownloadHelper): 19 | def __init__(self, listener): 20 | super().__init__() 21 | self.__listener = listener 22 | self.__resource_lock = threading.RLock() 23 | self.__name = "" 24 | self.__gid = '' 25 | self.__start_time = time.time() 26 | self.__user_bot = Client(api_id=TELEGRAM_API, 27 | api_hash=TELEGRAM_HASH, 28 | session_name=USER_SESSION_STRING) 29 | self.__user_bot.start() 30 | self.__is_cancelled = False 31 | 32 | @property 33 | def gid(self): 34 | with self.__resource_lock: 35 | return self.__gid 36 | 37 | @property 38 | def download_speed(self): 39 | with self.__resource_lock: 40 | return self.downloaded_bytes / (time.time() - self.__start_time) 41 | 42 | def __onDownloadStart(self, name, size, file_id): 43 | with download_dict_lock: 44 | download_dict[self.__listener.uid] = TelegramDownloadStatus(self, self.__listener) 45 | with global_lock: 46 | GLOBAL_GID.add(file_id) 47 | with self.__resource_lock: 48 | self.name = name 49 | self.size = size 50 | self.__gid = file_id 51 | self.__listener.onDownloadStarted() 52 | 53 | def __onDownloadProgress(self, current, total): 54 | if self.__is_cancelled: 55 | self.__onDownloadError('Cancelled by user!') 56 | self.__user_bot.stop_transmission() 57 | return 58 | with self.__resource_lock: 59 | self.downloaded_bytes = current 60 | try: 61 | self.progress = current / self.size * 100 62 | except ZeroDivisionError: 63 | self.progress = 0 64 | 65 | def __onDownloadError(self, error): 66 | with global_lock: 67 | try: 68 | GLOBAL_GID.remove(self.gid) 69 | except KeyError: 70 | pass 71 | self.__listener.onDownloadError(error) 72 | 73 | def __onDownloadComplete(self): 74 | with global_lock: 75 | GLOBAL_GID.remove(self.gid) 76 | self.__listener.onDownloadComplete() 77 | 78 | def __download(self, message, path): 79 | download = self.__user_bot.download_media(message, 80 | progress=self.__onDownloadProgress, file_name=path) 81 | if download is not None: 82 | self.__onDownloadComplete() 83 | else: 84 | if not self.__is_cancelled: 85 | self.__onDownloadError('Internal error occurred') 86 | 87 | def add_download(self, message, path): 88 | _message = self.__user_bot.get_messages(message.chat.id, message.message_id) 89 | media = None 90 | media_array = [_message.document, _message.video, _message.audio] 91 | for i in media_array: 92 | if i is not None: 93 | media = i 94 | break 95 | if media is not None: 96 | with global_lock: 97 | # For avoiding locking the thread lock for long time unnecessarily 98 | download = media.file_id not in GLOBAL_GID 99 | 100 | if download: 101 | self.__onDownloadStart(media.file_name, media.file_size, media.file_id) 102 | LOGGER.info(f'Downloading telegram file with id: {media.file_id}') 103 | threading.Thread(target=self.__download, args=(_message, path)).start() 104 | else: 105 | self.__onDownloadError('File already being downloaded!') 106 | else: 107 | self.__onDownloadError('No document in the replied message') 108 | 109 | def cancel_download(self): 110 | LOGGER.info(f'Cancelling download on user request: {self.gid}') 111 | self.__is_cancelled = True 112 | -------------------------------------------------------------------------------- /bot/helper/ext_utils/bot_utils.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import re 3 | import threading 4 | import time 5 | 6 | from bot import download_dict, download_dict_lock 7 | 8 | LOGGER = logging.getLogger(__name__) 9 | 10 | MAGNET_REGEX = r"magnet:\?xt=urn:btih:[a-zA-Z0-9]*" 11 | 12 | URL_REGEX = r"(?:(?:https?|ftp):\/\/)?[\w/\-?=%.]+\.[\w/\-?=%.]+" 13 | 14 | 15 | class MirrorStatus: 16 | STATUS_UPLOADING = "Uploading" 17 | STATUS_DOWNLOADING = "Downloading" 18 | STATUS_WAITING = "Queued" 19 | STATUS_FAILED = "Failed. Cleaning download" 20 | STATUS_CANCELLED = "Cancelled" 21 | STATUS_ARCHIVING = "Archiving" 22 | 23 | 24 | PROGRESS_MAX_SIZE = 100 // 8 25 | PROGRESS_INCOMPLETE = ['▏', '▎', '▍', '▌', '▋', '▊', '▉'] 26 | 27 | SIZE_UNITS = ['B', 'KB', 'MB', 'GB', 'TB', 'PB'] 28 | 29 | 30 | class setInterval: 31 | def __init__(self, interval, action): 32 | self.interval = interval 33 | self.action = action 34 | self.stopEvent = threading.Event() 35 | thread = threading.Thread(target=self.__setInterval) 36 | thread.start() 37 | 38 | def __setInterval(self): 39 | nextTime = time.time() + self.interval 40 | while not self.stopEvent.wait(nextTime - time.time()): 41 | nextTime += self.interval 42 | self.action() 43 | 44 | def cancel(self): 45 | self.stopEvent.set() 46 | 47 | 48 | def get_readable_file_size(size_in_bytes) -> str: 49 | if size_in_bytes is None: 50 | return '0B' 51 | index = 0 52 | while size_in_bytes >= 1024: 53 | size_in_bytes /= 1024 54 | index += 1 55 | try: 56 | return f'{round(size_in_bytes, 2)}{SIZE_UNITS[index]}' 57 | except IndexError: 58 | return 'File too large' 59 | 60 | 61 | def getDownloadByGid(gid): 62 | with download_dict_lock: 63 | for dl in download_dict.values(): 64 | if dl.status() == MirrorStatus.STATUS_DOWNLOADING or dl.status() == MirrorStatus.STATUS_WAITING: 65 | if dl.gid() == gid: 66 | return dl 67 | return None 68 | 69 | 70 | def get_progress_bar_string(status): 71 | completed = status.processed_bytes() / 8 72 | total = status.size_raw() / 8 73 | if total == 0: 74 | p = 0 75 | else: 76 | p = round(completed * 100 / total) 77 | p = min(max(p, 0), 100) 78 | cFull = p // 8 79 | cPart = p % 8 - 1 80 | p_str = '█' * cFull 81 | if cPart >= 0: 82 | p_str += PROGRESS_INCOMPLETE[cPart] 83 | p_str += ' ' * (PROGRESS_MAX_SIZE - cFull) 84 | p_str = f"[{p_str}]" 85 | return p_str 86 | 87 | 88 | def get_readable_message(): 89 | with download_dict_lock: 90 | msg = "" 91 | for download in list(download_dict.values()): 92 | msg += f"{download.name()} - " 93 | msg += download.status() 94 | if download.status() != MirrorStatus.STATUS_ARCHIVING: 95 | msg += f"\n{get_progress_bar_string(download)} {download.progress()} of " \ 96 | f"{download.size()}" \ 97 | f" at {download.speed()}, ETA: {download.eta()} " 98 | if download.status() == MirrorStatus.STATUS_DOWNLOADING: 99 | if hasattr(download, 'is_torrent'): 100 | msg += f"| P: {download.aria_download().connections} " \ 101 | f"| S: {download.aria_download().num_seeders}" 102 | msg += f"\nGID: {download.gid()}" 103 | msg += "\n\n" 104 | return msg 105 | 106 | 107 | def get_readable_time(seconds: int) -> str: 108 | result = '' 109 | (days, remainder) = divmod(seconds, 86400) 110 | days = int(days) 111 | if days != 0: 112 | result += f'{days}d' 113 | (hours, remainder) = divmod(remainder, 3600) 114 | hours = int(hours) 115 | if hours != 0: 116 | result += f'{hours}h' 117 | (minutes, seconds) = divmod(remainder, 60) 118 | minutes = int(minutes) 119 | if minutes != 0: 120 | result += f'{minutes}m' 121 | seconds = int(seconds) 122 | result += f'{seconds}s' 123 | return result 124 | 125 | 126 | def is_url(url: str): 127 | url = re.findall(URL_REGEX, url) 128 | if url: 129 | return True 130 | return False 131 | 132 | 133 | def is_magnet(url: str): 134 | magnet = re.findall(MAGNET_REGEX, url) 135 | if magnet: 136 | return True 137 | return False 138 | -------------------------------------------------------------------------------- /bot/helper/mirror_utils/download_utils/direct_link_generator_license.md: -------------------------------------------------------------------------------- 1 | RAPHIELSCAPE PUBLIC LICENSE 2 | Version 1.c, June 2019 3 | 4 | Copyright (C) 2019 Raphielscape LLC. 5 | Copyright (C) 2019 Devscapes Open Source Holding GmbH. 6 | 7 | Everyone is permitted to copy and distribute verbatim or modified 8 | copies of this license document, and changing it is allowed as long 9 | as the name is changed. 10 | 11 | RAPHIELSCAPE PUBLIC LICENSE 12 | A-1. DEFINITIONS 13 | 14 | 0. “This License” refers to version 1.c of the Raphielscape Public License. 15 | 16 | 1. “Copyright” also means copyright-like laws that apply to other kinds of works. 17 | 18 | 2. “The Work" refers to any copyrightable work licensed under this License. Each licensee is addressed as “you”. 19 | “Licensees” and “recipients” may be individuals or organizations. 20 | 21 | 3. To “modify” a work means to copy from or adapt all or part of the work in a fashion requiring copyright permission, 22 | other than the making of an exact copy. The resulting work is called a “modified version” of the earlier work 23 | or a work “based on” the earlier work. 24 | 25 | 4. Source Form. The “source form” for a work means the preferred form of the work for making modifications to it. 26 | “Object code” means any non-source form of a work. 27 | 28 | The “Corresponding Source” for a work in object code form means all the source code needed to generate, install, and 29 | (for an executable work) run the object code and to modify the work, including scripts to control those activities. 30 | 31 | The Corresponding Source need not include anything that users can regenerate automatically from other parts of the Corresponding Source. 32 | The Corresponding Source for a work in source code form is that same work. 33 | 34 | 5. "The author" refers to "author" of the code, which is the one that made the particular code which exists inside of 35 | the Corresponding Source. 36 | 37 | 6. "Owner" refers to any parties which is made the early form of the Corresponding Source. 38 | 39 | A-2. TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 40 | 41 | 0. You must give any other recipients of the Work or Derivative Works a copy of this License; and 42 | 43 | 1. You must cause any modified files to carry prominent notices stating that You changed the files; and 44 | 45 | 2. You must retain, in the Source form of any Derivative Works that You distribute, 46 | this license, all copyright, patent, trademark, authorships and attribution notices 47 | from the Source form of the Work; and 48 | 49 | 3. Respecting the author and owner of works that are distributed in any way. 50 | 51 | You may add Your own copyright statement to Your modifications and may provide 52 | additional or different license terms and conditions for use, reproduction, 53 | or distribution of Your modifications, or for any such Derivative Works as a whole, 54 | provided Your use, reproduction, and distribution of the Work otherwise complies 55 | with the conditions stated in this License. 56 | 57 | B. DISCLAIMER OF WARRANTY 58 | 59 | THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR 60 | IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND 61 | FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS 62 | BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 63 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, 64 | OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 65 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 66 | OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 67 | 68 | 69 | C. REVISED VERSION OF THIS LICENSE 70 | 71 | The Devscapes Open Source Holding GmbH. may publish revised and/or new versions of the 72 | Raphielscape Public License from time to time. Such new versions will be similar in spirit 73 | to the present version, but may differ in detail to address new problems or concerns. 74 | 75 | Each version is given a distinguishing version number. If the Program specifies that a 76 | certain numbered version of the Raphielscape Public License "or any later version" applies to it, 77 | you have the option of following the terms and conditions either of that numbered version or of 78 | any later version published by the Devscapes Open Source Holding GmbH. If the Program does not specify a 79 | version number of the Raphielscape Public License, you may choose any version ever published 80 | by the Devscapes Open Source Holding GmbH. 81 | 82 | END OF LICENSE -------------------------------------------------------------------------------- /bot/__main__.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | import signal 3 | import pickle 4 | 5 | from os import execl, path, remove 6 | from sys import executable 7 | 8 | from telegram.ext import CommandHandler, run_async 9 | from bot import dispatcher, updater, botStartTime 10 | from bot.helper.ext_utils import fs_utils 11 | from bot.helper.telegram_helper.bot_commands import BotCommands 12 | from bot.helper.telegram_helper.message_utils import * 13 | from .helper.ext_utils.bot_utils import get_readable_file_size, get_readable_time 14 | from .helper.telegram_helper.filters import CustomFilters 15 | from .modules import authorize, list, cancel_mirror, mirror_status, mirror, clone, watch 16 | 17 | 18 | @run_async 19 | def stats(update, context): 20 | currentTime = get_readable_time((time.time() - botStartTime)) 21 | total, used, free = shutil.disk_usage('.') 22 | total = get_readable_file_size(total) 23 | used = get_readable_file_size(used) 24 | free = get_readable_file_size(free) 25 | stats = f'Bot Uptime: {currentTime}\n' \ 26 | f'Total disk space: {total}\n' \ 27 | f'Used: {used}\n' \ 28 | f'Free: {free}' 29 | sendMessage(stats, context.bot, update) 30 | 31 | 32 | @run_async 33 | def start(update, context): 34 | sendMessage("This is a bot which can mirror all your links to Google drive!\n" 35 | "Type /help to get a list of available commands", context.bot, update) 36 | 37 | 38 | @run_async 39 | def restart(update, context): 40 | restart_message = sendMessage("Restarting, Please wait!", context.bot, update) 41 | # Save restart message object in order to reply to it after restarting 42 | fs_utils.clean_all() 43 | with open('restart.pickle', 'wb') as status: 44 | pickle.dump(restart_message, status) 45 | execl(executable, executable, "-m", "bot") 46 | 47 | 48 | @run_async 49 | def ping(update, context): 50 | start_time = int(round(time.time() * 1000)) 51 | reply = sendMessage("Starting Ping", context.bot, update) 52 | end_time = int(round(time.time() * 1000)) 53 | editMessage(f'{end_time - start_time} ms', reply) 54 | 55 | 56 | @run_async 57 | def log(update, context): 58 | sendLogFile(context.bot, update) 59 | 60 | 61 | @run_async 62 | def bot_help(update, context): 63 | help_string = f''' 64 | /{BotCommands.HelpCommand}: To get this message 65 | 66 | /{BotCommands.MirrorCommand} [download_url][magnet_link]: Start mirroring the link to google drive 67 | 68 | /{BotCommands.TarMirrorCommand} [download_url][magnet_link]: start mirroring and upload the archived (.tar) version of the download 69 | 70 | /{BotCommands.WatchCommand} [youtube-dl supported link]: Mirror through youtube-dl 71 | 72 | /{BotCommands.TarWatchCommand} [youtube-dl supported link]: Mirror through youtube-dl and tar before uploading 73 | 74 | /{BotCommands.CancelMirror} : Reply to the message by which the download was initiated and that download will be cancelled 75 | 76 | /{BotCommands.StatusCommand}: Shows a status of all the downloads 77 | 78 | /{BotCommands.ListCommand} [search term]: Searches the search term in the Google drive, if found replies with the link 79 | 80 | /{BotCommands.StatsCommand}: Show Stats of the machine the bot is hosted on 81 | 82 | /{BotCommands.AuthorizeCommand}: Authorize a chat or a user to use the bot (Can only be invoked by owner of the bot) 83 | 84 | /{BotCommands.LogCommand}: Get a log file of the bot. Handy for getting crash reports 85 | 86 | ''' 87 | sendMessage(help_string, context.bot, update) 88 | 89 | 90 | def main(): 91 | fs_utils.start_cleanup() 92 | # Check if the bot is restarting 93 | if path.exists('restart.pickle'): 94 | with open('restart.pickle', 'rb') as status: 95 | restart_message = pickle.load(status) 96 | restart_message.edit_text("Restarted Successfully!") 97 | remove('restart.pickle') 98 | 99 | start_handler = CommandHandler(BotCommands.StartCommand, start, 100 | filters=CustomFilters.authorized_chat | CustomFilters.authorized_user) 101 | ping_handler = CommandHandler(BotCommands.PingCommand, ping, 102 | filters=CustomFilters.authorized_chat | CustomFilters.authorized_user) 103 | restart_handler = CommandHandler(BotCommands.RestartCommand, restart, 104 | filters=CustomFilters.owner_filter) 105 | help_handler = CommandHandler(BotCommands.HelpCommand, 106 | bot_help, filters=CustomFilters.authorized_chat | CustomFilters.authorized_user) 107 | stats_handler = CommandHandler(BotCommands.StatsCommand, 108 | stats, filters=CustomFilters.authorized_chat | CustomFilters.authorized_user) 109 | log_handler = CommandHandler(BotCommands.LogCommand, log, filters=CustomFilters.owner_filter) 110 | dispatcher.add_handler(start_handler) 111 | dispatcher.add_handler(ping_handler) 112 | dispatcher.add_handler(restart_handler) 113 | dispatcher.add_handler(help_handler) 114 | dispatcher.add_handler(stats_handler) 115 | dispatcher.add_handler(log_handler) 116 | updater.start_polling() 117 | LOGGER.info("Bot Started!") 118 | signal.signal(signal.SIGINT, fs_utils.exit_clean_up) 119 | 120 | 121 | main() 122 | -------------------------------------------------------------------------------- /bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py: -------------------------------------------------------------------------------- 1 | from .download_helper import DownloadHelper 2 | import time 3 | from youtube_dl import YoutubeDL, DownloadError 4 | from bot import download_dict_lock, download_dict 5 | from ..status_utils.youtube_dl_download_status import YoutubeDLDownloadStatus 6 | import logging 7 | import re 8 | import threading 9 | 10 | LOGGER = logging.getLogger(__name__) 11 | 12 | 13 | class MyLogger: 14 | def __init__(self, obj): 15 | self.obj = obj 16 | 17 | def debug(self, msg): 18 | LOGGER.debug(msg) 19 | # Hack to fix changing changing extension 20 | match = re.search(r'.ffmpeg..Merging formats into..(.*?).$', msg) 21 | if match and not self.obj.is_playlist: 22 | self.obj.name = match.group(1) 23 | 24 | @staticmethod 25 | def warning(msg): 26 | LOGGER.warning(msg) 27 | 28 | @staticmethod 29 | def error(msg): 30 | LOGGER.error(msg) 31 | 32 | 33 | class YoutubeDLHelper(DownloadHelper): 34 | def __init__(self, listener): 35 | super().__init__() 36 | self.__name = "" 37 | self.__start_time = time.time() 38 | self.__listener = listener 39 | self.__gid = "" 40 | self.opts = { 41 | 'progress_hooks': [self.__onDownloadProgress], 42 | 'logger': MyLogger(self), 43 | 'usenetrc': True, 44 | 'format': "best/bestvideo+bestaudio" 45 | } 46 | self.__download_speed = 0 47 | self.download_speed_readable = '' 48 | self.downloaded_bytes = 0 49 | self.size = 0 50 | self.is_playlist = False 51 | self.last_downloaded = 0 52 | self.is_cancelled = False 53 | self.vid_id = '' 54 | self.__resource_lock = threading.RLock() 55 | 56 | @property 57 | def download_speed(self): 58 | with self.__resource_lock: 59 | return self.__download_speed 60 | 61 | @property 62 | def gid(self): 63 | with self.__resource_lock: 64 | return self.__gid 65 | 66 | def __onDownloadProgress(self, d): 67 | if self.is_cancelled: 68 | raise ValueError("Cancelling Download..") 69 | if d['status'] == "finished": 70 | if self.is_playlist: 71 | self.last_downloaded = 0 72 | elif d['status'] == "downloading": 73 | with self.__resource_lock: 74 | self.__download_speed = d['speed'] 75 | if self.is_playlist: 76 | progress = d['downloaded_bytes'] / d['total_bytes'] 77 | chunk_size = d['downloaded_bytes'] - self.last_downloaded 78 | self.last_downloaded = d['total_bytes'] * progress 79 | self.downloaded_bytes += chunk_size 80 | try: 81 | self.progress = (self.downloaded_bytes / self.size) * 100 82 | except ZeroDivisionError: 83 | pass 84 | else: 85 | self.download_speed_readable = d['_speed_str'] 86 | self.downloaded_bytes = d['downloaded_bytes'] 87 | 88 | def __onDownloadStart(self): 89 | with download_dict_lock: 90 | download_dict[self.__listener.uid] = YoutubeDLDownloadStatus(self, self.__listener) 91 | 92 | def __onDownloadComplete(self): 93 | self.__listener.onDownloadComplete() 94 | 95 | def onDownloadError(self, error): 96 | self.__listener.onDownloadError(error) 97 | 98 | def extractMetaData(self, link): 99 | if 'hotstar' in link: 100 | self.opts['geo_bypass_country'] = 'IN' 101 | 102 | with YoutubeDL(self.opts) as ydl: 103 | try: 104 | result = ydl.extract_info(link, download=False) 105 | name = ydl.prepare_filename(result) 106 | except DownloadError as e: 107 | self.onDownloadError(str(e)) 108 | return 109 | if result.get('direct'): 110 | return None 111 | if 'entries' in result: 112 | video = result['entries'][0] 113 | for v in result['entries']: 114 | if v.get('filesize'): 115 | self.size += float(v['filesize']) 116 | # For playlists, ydl.prepare-filename returns the following format: -.NA 117 | self.name = name.split(f"-{result['id']}")[0] 118 | self.vid_id = video.get('id') 119 | self.is_playlist = True 120 | else: 121 | video = result 122 | if video.get('filesize'): 123 | self.size = float(video.get('filesize')) 124 | self.name = name 125 | self.vid_id = video.get('id') 126 | return video 127 | 128 | def __download(self, link): 129 | try: 130 | with YoutubeDL(self.opts) as ydl: 131 | try: 132 | ydl.download([link]) 133 | except DownloadError as e: 134 | self.onDownloadError(str(e)) 135 | return 136 | self.__onDownloadComplete() 137 | except ValueError: 138 | LOGGER.info("Download Cancelled by User!") 139 | self.onDownloadError("Download Cancelled by User!") 140 | 141 | def add_download(self, link, path): 142 | self.__onDownloadStart() 143 | self.extractMetaData(link) 144 | LOGGER.info(f"Downloading with YT-DL: {link}") 145 | self.__gid = f"{self.vid_id}{self.__listener.uid}" 146 | if not self.is_playlist: 147 | self.opts['outtmpl'] = f"{path}/{self.name}" 148 | else: 149 | self.opts['outtmpl'] = f"{path}/{self.name}/%(title)s.%(ext)s" 150 | self.__download(link) 151 | 152 | def cancel_download(self): 153 | self.is_cancelled = True 154 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # What is this repo about? 2 | This is a telegram bot writen in python for mirroring files on the internet to our beloved Google Drive. 3 | 4 | # Inspiration 5 | This project is heavily inspired from @out386 's telegram bot which is written in JS. 6 | 7 | # Features supported: 8 | - Mirroring direct download links to google drive 9 | - Download progress 10 | - Upload progress 11 | - Download/upload speeds and ETAs 12 | - Docker support 13 | - Uploading To Team Drives. 14 | - Index Link support 15 | - Service account support 16 | - Mirror all youtube-dl supported links 17 | - Mirror telegram files 18 | 19 | # Upcoming features (TODOs): 20 | 21 | # How to deploy? 22 | Deploying is pretty much straight forward and is divided into several steps as follows: 23 | ## Installing requirements 24 | 25 | - Clone this repo: 26 | ``` 27 | git clone https://github.com/lzzy12/python-aria-mirror-bot mirror-bot/ 28 | cd mirror-bot 29 | ``` 30 | 31 | - Install requirements 32 | For Debian based distros 33 | ``` 34 | sudo apt install python3 35 | ``` 36 | Install Docker by following the [official docker docs](https://docs.docker.com/engine/install/debian/) 37 | 38 | 39 | - For Arch and it's derivatives: 40 | ``` 41 | sudo pacman -S docker python 42 | ``` 43 | 44 | ## Setting up config file 45 | ``` 46 | cp config_sample.env config.env 47 | ``` 48 | - Remove the first line saying: 49 | ``` 50 | _____REMOVE_THIS_LINE_____=True 51 | ``` 52 | Fill up rest of the fields. Meaning of each fields are discussed below: 53 | - **BOT_TOKEN** : The telegram bot token that you get from @BotFather 54 | - **GDRIVE_FOLDER_ID** : This is the folder ID of the Google Drive Folder to which you want to upload all the mirrors. 55 | - **DOWNLOAD_DIR** : The path to the local folder where the downloads should be downloaded to 56 | - **DOWNLOAD_STATUS_UPDATE_INTERVAL** : A short interval of time in seconds after which the Mirror progress message is updated. (I recommend to keep it 5 seconds at least) 57 | - **OWNER_ID** : The Telegram user ID (not username) of the owner of the bot 58 | - **AUTO_DELETE_MESSAGE_DURATION** : Interval of time (in seconds), after which the bot deletes it's message (and command message) which is expected to be viewed instantly. Note: Set to -1 to never automatically delete messages 59 | - **IS_TEAM_DRIVE** : (Optional field) Set to "True" if GDRIVE_FOLDER_ID is from a Team Drive else False or Leave it empty. 60 | - **USE_SERVICE_ACCOUNTS**: (Optional field) (Leave empty if unsure) Whether to use service accounts or not. For this to work see "Using service accounts" section below. 61 | - **INDEX_URL** : (Optional field) Refer to https://github.com/maple3142/GDIndex/ The URL should not have any trailing '/' 62 | - **TELEGRAM_API** : This is to authenticate to your telegram account for downloading Telegram files. You can get this from https://my.telegram.org DO NOT put this in quotes. 63 | - **TELEGRAM_HASH** : This is to authenticate to your telegram account for downloading Telegram files. You can get this from https://my.telegram.org 64 | - **USER_SESSION_STRING** : Session string generated by running: 65 | ``` 66 | python3 generate_string_session.py 67 | ``` 68 | Note: You can limit maximum concurrent downloads by changing the value of MAX_CONCURRENT_DOWNLOADS in aria.sh. By default, it's set to 2 69 | 70 | ## Getting Google OAuth API credential file 71 | 72 | - Visit the [Google Cloud Console](https://console.developers.google.com/apis/credentials) 73 | - Go to the OAuth Consent tab, fill it, and save. 74 | - Go to the Credentials tab and click Create Credentials -> OAuth Client ID 75 | - Choose Other and Create. 76 | - Use the download button to download your credentials. 77 | - Move that file to the root of mirror-bot, and rename it to credentials.json 78 | - Visit [Google API page](https://console.developers.google.com/apis/library) 79 | - Search for Drive and enable it if it is disabled 80 | - Finally, run the script to generate token file (token.pickle) for Google Drive: 81 | ``` 82 | pip install google-api-python-client google-auth-httplib2 google-auth-oauthlib 83 | python3 generate_drive_token.py 84 | ``` 85 | ## Deploying 86 | 87 | - Start docker daemon (skip if already running): 88 | ``` 89 | sudo dockerd 90 | ``` 91 | - Build Docker image: 92 | ``` 93 | sudo docker build . -t mirror-bot 94 | ``` 95 | - Run the image: 96 | ``` 97 | sudo docker run mirror-bot 98 | ``` 99 | 100 | # Using service accounts for uploading to avoid user rate limit 101 | For Service Account to work, you must set USE_SERVICE_ACCOUNTS="True" in config file or environment variables 102 | Many thanks to [AutoRClone](https://github.com/xyou365/AutoRclone) for the scripts 103 | ## Generating service accounts 104 | Step 1. Generate service accounts [What is service account](https://cloud.google.com/iam/docs/service-accounts) 105 | --------------------------------- 106 | Let us create only the service accounts that we need. 107 | **Warning:** abuse of this feature is not the aim of autorclone and we do **NOT** recommend that you make a lot of projects, just one project and 100 sa allow you plenty of use, its also possible that overabuse might get your projects banned by google. 108 | 109 | ``` 110 | Note: 1 service account can copy around 750gb a day, 1 project makes 100 service accounts so thats 75tb a day, for most users this should easily suffice. 111 | ``` 112 | 113 | `python3 gen_sa_accounts.py --quick-setup 1 --new-only` 114 | 115 | A folder named accounts will be created which will contain keys for the service accounts created 116 | 117 | NOTE: If you have created SAs in past from this script, you can also just re download the keys by running: 118 | ``` 119 | python3 gen_sa_accounts.py --download-keys project_id 120 | ``` 121 | 122 | ### Add all the service accounts to the Team Drive or folder 123 | - Run: 124 | ``` 125 | python3 add_to_team_drive.py -d SharedTeamDriveSrcID 126 | ``` 127 | 128 | # Youtube-dl authentication using .netrc file 129 | For using your premium accounts in youtube-dl, edit the netrc file (in the root directory of this repository) according to following format: 130 | ``` 131 | machine host login username password my_youtube_password 132 | ``` 133 | where host is the name of extractor (eg. youtube, twitch). Multiple accounts of different hosts can be added each separated by a new line 134 | -------------------------------------------------------------------------------- /bot/helper/mirror_utils/download_utils/direct_link_generator.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2019 The Raphielscape Company LLC. 2 | # 3 | # Licensed under the Raphielscape Public License, Version 1.c (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # 6 | """ Helper Module containing various sites direct links generators. This module is copied and modified as per need 7 | from https://github.com/AvinashReddy3108/PaperplaneExtended . I hereby take no credit of the following code other 8 | than the modifications. See https://github.com/AvinashReddy3108/PaperplaneExtended/commits/master/userbot/modules/direct_links.py 9 | for original authorship. """ 10 | 11 | import json 12 | import re 13 | import urllib.parse 14 | from os import popen 15 | from random import choice 16 | 17 | import requests 18 | from bs4 import BeautifulSoup 19 | 20 | from bot.helper.ext_utils.exceptions import DirectDownloadLinkException 21 | 22 | 23 | def direct_link_generator(link: str): 24 | """ direct links generator """ 25 | if not link: 26 | raise DirectDownloadLinkException("`No links found!`") 27 | elif 'zippyshare.com' in link: 28 | return zippy_share(link) 29 | elif 'yadi.sk' in link: 30 | return yandex_disk(link) 31 | elif 'cloud.mail.ru' in link: 32 | return cm_ru(link) 33 | elif 'mediafire.com' in link: 34 | return mediafire(link) 35 | elif 'osdn.net' in link: 36 | return osdn(link) 37 | elif 'github.com' in link: 38 | return github(link) 39 | else: 40 | raise DirectDownloadLinkException(f'No Direct link function found for {link}') 41 | 42 | 43 | def zippy_share(url: str) -> str: 44 | """ ZippyShare direct links generator 45 | Based on https://github.com/LameLemon/ziggy""" 46 | dl_url = '' 47 | try: 48 | link = re.findall(r'\bhttps?://.*zippyshare\.com\S+', url)[0] 49 | except IndexError: 50 | raise DirectDownloadLinkException("`No ZippyShare links found`\n") 51 | session = requests.Session() 52 | base_url = re.search('http.+.com', link).group() 53 | response = session.get(link) 54 | page_soup = BeautifulSoup(response.content, "lxml") 55 | scripts = page_soup.find_all("script", {"type": "text/javascript"}) 56 | for script in scripts: 57 | if "getElementById('dlbutton')" in script.text: 58 | url_raw = re.search(r'= (?P\".+\" \+ (?P\(.+\)) .+);', 59 | script.text).group('url') 60 | math = re.search(r'= (?P\".+\" \+ (?P\(.+\)) .+);', 61 | script.text).group('math') 62 | dl_url = url_raw.replace(math, '"' + str(eval(math)) + '"') 63 | break 64 | dl_url = base_url + eval(dl_url) 65 | name = urllib.parse.unquote(dl_url.split('/')[-1]) 66 | return dl_url 67 | 68 | 69 | def yandex_disk(url: str) -> str: 70 | """ Yandex.Disk direct links generator 71 | Based on https://github.com/wldhx/yadisk-direct""" 72 | try: 73 | link = re.findall(r'\bhttps?://.*yadi\.sk\S+', url)[0] 74 | except IndexError: 75 | reply = "`No Yandex.Disk links found`\n" 76 | return reply 77 | api = 'https://cloud-api.yandex.net/v1/disk/public/resources/download?public_key={}' 78 | try: 79 | dl_url = requests.get(api.format(link)).json()['href'] 80 | return dl_url 81 | except KeyError: 82 | raise DirectDownloadLinkException("`Error: File not found / Download limit reached`\n") 83 | 84 | 85 | def cm_ru(url: str) -> str: 86 | """ cloud.mail.ru direct links generator 87 | Using https://github.com/JrMasterModelBuilder/cmrudl.py""" 88 | reply = '' 89 | try: 90 | link = re.findall(r'\bhttps?://.*cloud\.mail\.ru\S+', url)[0] 91 | except IndexError: 92 | raise DirectDownloadLinkException("`No cloud.mail.ru links found`\n") 93 | command = f'vendor/cmrudl.py/cmrudl -s {link}' 94 | result = popen(command).read() 95 | result = result.splitlines()[-1] 96 | try: 97 | data = json.loads(result) 98 | except json.decoder.JSONDecodeError: 99 | raise DirectDownloadLinkException("`Error: Can't extract the link`\n") 100 | dl_url = data['download'] 101 | return dl_url 102 | 103 | 104 | def mediafire(url: str) -> str: 105 | """ MediaFire direct links generator """ 106 | try: 107 | link = re.findall(r'\bhttps?://.*mediafire\.com\S+', url)[0] 108 | except IndexError: 109 | raise DirectDownloadLinkException("`No MediaFire links found`\n") 110 | page = BeautifulSoup(requests.get(link).content, 'lxml') 111 | info = page.find('a', {'aria-label': 'Download file'}) 112 | dl_url = info.get('href') 113 | return dl_url 114 | 115 | 116 | def osdn(url: str) -> str: 117 | """ OSDN direct links generator """ 118 | osdn_link = 'https://osdn.net' 119 | try: 120 | link = re.findall(r'\bhttps?://.*osdn\.net\S+', url)[0] 121 | except IndexError: 122 | raise DirectDownloadLinkException("`No OSDN links found`\n") 123 | page = BeautifulSoup( 124 | requests.get(link, allow_redirects=True).content, 'lxml') 125 | info = page.find('a', {'class': 'mirror_link'}) 126 | link = urllib.parse.unquote(osdn_link + info['href']) 127 | mirrors = page.find('form', {'id': 'mirror-select-form'}).findAll('tr') 128 | urls = [] 129 | for data in mirrors[1:]: 130 | mirror = data.find('input')['value'] 131 | urls.append(re.sub(r'm=(.*)&f', f'm={mirror}&f', link)) 132 | return urls[0] 133 | 134 | 135 | def github(url: str) -> str: 136 | """ GitHub direct links generator """ 137 | try: 138 | re.findall(r'\bhttps?://.*github\.com.*releases\S+', url)[0] 139 | except IndexError: 140 | raise DirectDownloadLinkException("`No GitHub Releases links found`\n") 141 | download = requests.get(url, stream=True, allow_redirects=False) 142 | try: 143 | dl_url = download.headers["location"] 144 | return dl_url 145 | except KeyError: 146 | raise DirectDownloadLinkException("`Error: Can't extract the link`\n") 147 | 148 | 149 | def useragent(): 150 | """ 151 | useragent random setter 152 | """ 153 | useragents = BeautifulSoup( 154 | requests.get( 155 | 'https://developers.whatismybrowser.com/' 156 | 'useragents/explore/operating_system_name/android/').content, 157 | 'lxml').findAll('td', {'class': 'useragent'}) 158 | user_agent = choice(useragents) 159 | return user_agent.text 160 | -------------------------------------------------------------------------------- /bot/modules/mirror.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from telegram.ext import CommandHandler, run_async 3 | 4 | from bot import Interval, INDEX_URL 5 | from bot import dispatcher, DOWNLOAD_DIR, DOWNLOAD_STATUS_UPDATE_INTERVAL, download_dict, download_dict_lock 6 | from bot.helper.ext_utils import fs_utils, bot_utils 7 | from bot.helper.ext_utils.bot_utils import setInterval 8 | from bot.helper.ext_utils.exceptions import DirectDownloadLinkException 9 | from bot.helper.mirror_utils.download_utils import aria2_download 10 | from bot.helper.mirror_utils.download_utils.direct_link_generator import direct_link_generator 11 | from bot.helper.mirror_utils.download_utils.telegram_downloader import TelegramDownloadHelper 12 | from bot.helper.mirror_utils.status_utils import listeners 13 | from bot.helper.mirror_utils.status_utils.tar_status import TarStatus 14 | from bot.helper.mirror_utils.status_utils.upload_status import UploadStatus 15 | from bot.helper.mirror_utils.upload_utils import gdriveTools 16 | from bot.helper.telegram_helper.bot_commands import BotCommands 17 | from bot.helper.telegram_helper.filters import CustomFilters 18 | from bot.helper.telegram_helper.message_utils import * 19 | from bot.helper.mirror_utils.download_utils.youtube_dl_download_helper import YoutubeDLHelper 20 | import pathlib 21 | import os 22 | 23 | 24 | class MirrorListener(listeners.MirrorListeners): 25 | 26 | def __init__(self, bot, update, isTar=False, tag=None): 27 | super().__init__(bot, update) 28 | self.isTar = isTar 29 | self.tag = tag 30 | 31 | def onDownloadStarted(self): 32 | pass 33 | 34 | def onDownloadProgress(self): 35 | # We are handling this on our own! 36 | pass 37 | 38 | def clean(self): 39 | try: 40 | Interval[0].cancel() 41 | del Interval[0] 42 | delete_all_messages() 43 | except IndexError: 44 | pass 45 | 46 | def onDownloadComplete(self): 47 | with download_dict_lock: 48 | LOGGER.info(f"Download completed: {download_dict[self.uid].name()}") 49 | download = download_dict[self.uid] 50 | name = download.name() 51 | size = download.size_raw() 52 | m_path = f'{DOWNLOAD_DIR}{self.uid}/{download.name()}' 53 | if self.isTar: 54 | download.is_archiving = True 55 | try: 56 | with download_dict_lock: 57 | download_dict[self.uid] = TarStatus(name, m_path, size) 58 | path = fs_utils.tar(m_path) 59 | except FileNotFoundError: 60 | LOGGER.info('File to archive not found!') 61 | self.onUploadError('Internal error occurred!!') 62 | return 63 | else: 64 | path = f'{DOWNLOAD_DIR}{self.uid}/{download_dict[self.uid].name()}' 65 | up_name = pathlib.PurePath(path).name 66 | LOGGER.info(f"Upload Name : {up_name}") 67 | drive = gdriveTools.GoogleDriveHelper(up_name, self) 68 | if size == 0: 69 | size = fs_utils.get_path_size(m_path) 70 | upload_status = UploadStatus(drive, size, self) 71 | with download_dict_lock: 72 | download_dict[self.uid] = upload_status 73 | update_all_messages() 74 | drive.upload(up_name) 75 | 76 | def onDownloadError(self, error): 77 | error = error.replace('<', ' ') 78 | error = error.replace('>', ' ') 79 | LOGGER.info(self.update.effective_chat.id) 80 | with download_dict_lock: 81 | try: 82 | download = download_dict[self.uid] 83 | del download_dict[self.uid] 84 | LOGGER.info(f"Deleting folder: {download.path()}") 85 | fs_utils.clean_download(download.path()) 86 | LOGGER.info(f"Deleting {download.name()} from download_dict.") 87 | LOGGER.info(str(download_dict)) 88 | except Exception as e: 89 | LOGGER.error(str(e)) 90 | pass 91 | count = len(download_dict) 92 | if self.message.from_user.username: 93 | uname = f"@{self.message.from_user.username}" 94 | else: 95 | uname = f'{self.message.from_user.first_name}' 96 | msg = f"{uname} your download has been stopped due to: {error}" 97 | sendMessage(msg, self.bot, self.update) 98 | if count == 0: 99 | self.clean() 100 | else: 101 | update_all_messages() 102 | 103 | def onUploadStarted(self): 104 | pass 105 | 106 | def onUploadProgress(self): 107 | pass 108 | 109 | def onUploadComplete(self, link: str): 110 | with download_dict_lock: 111 | msg = f'{download_dict[self.uid].name()} ({download_dict[self.uid].size()})' 112 | LOGGER.info(f'Done Uploading {download_dict[self.uid].name()}') 113 | if INDEX_URL is not None: 114 | share_url = requests.utils.requote_uri(f'{INDEX_URL}/{download_dict[self.uid].name()}') 115 | if os.path.isdir(f'{DOWNLOAD_DIR}/{self.uid}/{download_dict[self.uid].name()}'): 116 | share_url += '/' 117 | msg += f'\n\n Shareable link: here' 118 | if self.tag is not None: 119 | msg += f'\ncc: @{self.tag}' 120 | try: 121 | fs_utils.clean_download(download_dict[self.uid].path()) 122 | except FileNotFoundError: 123 | pass 124 | del download_dict[self.uid] 125 | count = len(download_dict) 126 | sendMessage(msg, self.bot, self.update) 127 | if count == 0: 128 | self.clean() 129 | else: 130 | update_all_messages() 131 | 132 | def onUploadError(self, error): 133 | e_str = str(error.last_attempt.exception()).replace('<', '').replace('>', '') 134 | with download_dict_lock: 135 | try: 136 | fs_utils.clean_download(download_dict[self.uid].path()) 137 | except FileNotFoundError: 138 | pass 139 | del download_dict[self.message.message_id] 140 | count = len(download_dict) 141 | sendMessage(e_str, self.bot, self.update) 142 | if count == 0: 143 | self.clean() 144 | else: 145 | update_all_messages() 146 | 147 | 148 | def _mirror(bot, update, isTar=False): 149 | message_args = update.message.text.split(' ') 150 | try: 151 | link = message_args[1] 152 | except IndexError: 153 | link = '' 154 | LOGGER.info(link) 155 | link = link.strip() 156 | reply_to = update.message.reply_to_message 157 | if reply_to is not None: 158 | file = None 159 | tag = reply_to.from_user.username 160 | media_array = [reply_to.document, reply_to.video, reply_to.audio] 161 | for i in media_array: 162 | if i is not None: 163 | file = i 164 | break 165 | 166 | if len(link) == 0: 167 | if file is not None: 168 | if file.mime_type != "application/x-bittorrent": 169 | listener = MirrorListener(bot, update, isTar, tag) 170 | tg_downloader = TelegramDownloadHelper(listener) 171 | tg_downloader.add_download(reply_to, f'{DOWNLOAD_DIR}{listener.uid}/') 172 | sendStatusMessage(update, bot) 173 | if len(Interval) == 0: 174 | Interval.append(setInterval(DOWNLOAD_STATUS_UPDATE_INTERVAL, update_all_messages)) 175 | return 176 | else: 177 | link = file.get_file().file_path 178 | else: 179 | tag = None 180 | if not bot_utils.is_url(link) and not bot_utils.is_magnet(link): 181 | sendMessage('No download source provided', bot, update) 182 | return 183 | 184 | try: 185 | link = direct_link_generator(link) 186 | except DirectDownloadLinkException as e: 187 | LOGGER.info(f'{link}: {e}') 188 | listener = MirrorListener(bot, update, isTar, tag) 189 | aria = aria2_download.AriaDownloadHelper(listener) 190 | aria.add_download(link, f'{DOWNLOAD_DIR}/{listener.uid}/') 191 | sendStatusMessage(update, bot) 192 | if len(Interval) == 0: 193 | Interval.append(setInterval(DOWNLOAD_STATUS_UPDATE_INTERVAL, update_all_messages)) 194 | 195 | 196 | @run_async 197 | def mirror(update, context): 198 | _mirror(context.bot, update) 199 | 200 | 201 | @run_async 202 | def tar_mirror(update, context): 203 | _mirror(context.bot, update, True) 204 | 205 | 206 | mirror_handler = CommandHandler(BotCommands.MirrorCommand, mirror, 207 | filters=CustomFilters.authorized_chat | CustomFilters.authorized_user) 208 | tar_mirror_handler = CommandHandler(BotCommands.TarMirrorCommand, tar_mirror, 209 | filters=CustomFilters.authorized_chat | CustomFilters.authorized_user) 210 | dispatcher.add_handler(mirror_handler) 211 | dispatcher.add_handler(tar_mirror_handler) 212 | -------------------------------------------------------------------------------- /gen_sa_accounts.py: -------------------------------------------------------------------------------- 1 | import errno 2 | import os 3 | import pickle 4 | import sys 5 | from argparse import ArgumentParser 6 | from base64 import b64decode 7 | from glob import glob 8 | from json import loads 9 | from random import choice 10 | from time import sleep 11 | 12 | from google.auth.transport.requests import Request 13 | from google_auth_oauthlib.flow import InstalledAppFlow 14 | from googleapiclient.discovery import build 15 | from googleapiclient.errors import HttpError 16 | 17 | SCOPES = ['https://www.googleapis.com/auth/drive', 'https://www.googleapis.com/auth/cloud-platform', 18 | 'https://www.googleapis.com/auth/iam'] 19 | project_create_ops = [] 20 | current_key_dump = [] 21 | sleep_time = 30 22 | 23 | 24 | # Create count SAs in project 25 | def _create_accounts(service, project, count): 26 | batch = service.new_batch_http_request(callback=_def_batch_resp) 27 | for i in range(count): 28 | aid = _generate_id('mfc-') 29 | batch.add(service.projects().serviceAccounts().create(name='projects/' + project, body={'accountId': aid, 30 | 'serviceAccount': { 31 | 'displayName': aid}})) 32 | batch.execute() 33 | 34 | 35 | # Create accounts needed to fill project 36 | def _create_remaining_accounts(iam, project): 37 | print('Creating accounts in %s' % project) 38 | sa_count = len(_list_sas(iam, project)) 39 | while sa_count != 100: 40 | _create_accounts(iam, project, 100 - sa_count) 41 | sa_count = len(_list_sas(iam, project)) 42 | 43 | 44 | # Generate a random id 45 | def _generate_id(prefix='saf-'): 46 | chars = '-abcdefghijklmnopqrstuvwxyz1234567890' 47 | return prefix + ''.join(choice(chars) for _ in range(25)) + choice(chars[1:]) 48 | 49 | 50 | # List projects using service 51 | def _get_projects(service): 52 | return [i['projectId'] for i in service.projects().list().execute()['projects']] 53 | 54 | 55 | # Default batch callback handler 56 | def _def_batch_resp(id, resp, exception): 57 | if exception is not None: 58 | if str(exception).startswith(' 0: 219 | current_count = len(_get_projects(cloud)) 220 | if current_count + create_projects <= max_projects: 221 | print('Creating %d projects' % (create_projects)) 222 | nprjs = _create_projects(cloud, create_projects) 223 | selected_projects = nprjs 224 | else: 225 | sys.exit('No, you cannot create %d new project (s).\n' 226 | 'Please reduce value of --quick-setup.\n' 227 | 'Remember that you can totally create %d projects (%d already).\n' 228 | 'Please do not delete existing projects unless you know what you are doing' % ( 229 | create_projects, max_projects, current_count)) 230 | else: 231 | print('Will overwrite all service accounts in existing projects.\n' 232 | 'So make sure you have some projects already.') 233 | input("Press Enter to continue...") 234 | 235 | if enable_services: 236 | ste = [] 237 | ste.append(enable_services) 238 | if enable_services == '~': 239 | ste = selected_projects 240 | elif enable_services == '*': 241 | ste = _get_projects(cloud) 242 | services = [i + '.googleapis.com' for i in services] 243 | print('Enabling services') 244 | _enable_services(serviceusage, ste, services) 245 | if create_sas: 246 | stc = [] 247 | stc.append(create_sas) 248 | if create_sas == '~': 249 | stc = selected_projects 250 | elif create_sas == '*': 251 | stc = _get_projects(cloud) 252 | for i in stc: 253 | _create_remaining_accounts(iam, i) 254 | if download_keys: 255 | try: 256 | os.mkdir(path) 257 | except OSError as e: 258 | if e.errno == errno.EEXIST: 259 | pass 260 | else: 261 | raise 262 | std = [] 263 | std.append(download_keys) 264 | if download_keys == '~': 265 | std = selected_projects 266 | elif download_keys == '*': 267 | std = _get_projects(cloud) 268 | _create_sa_keys(iam, std, path) 269 | if delete_sas: 270 | std = [] 271 | std.append(delete_sas) 272 | if delete_sas == '~': 273 | std = selected_projects 274 | elif delete_sas == '*': 275 | std = _get_projects(cloud) 276 | for i in std: 277 | print('Deleting service accounts in %s' % i) 278 | _delete_sas(iam, i) 279 | 280 | 281 | if __name__ == '__main__': 282 | parse = ArgumentParser(description='A tool to create Google service accounts.') 283 | parse.add_argument('--path', '-p', default='accounts', 284 | help='Specify an alternate directory to output the credential files.') 285 | parse.add_argument('--token', default='token_sa.pickle', help='Specify the pickle token file path.') 286 | parse.add_argument('--credentials', default='credentials.json', help='Specify the credentials file path.') 287 | parse.add_argument('--list-projects', default=False, action='store_true', 288 | help='List projects viewable by the user.') 289 | parse.add_argument('--list-sas', default=False, help='List service accounts in a project.') 290 | parse.add_argument('--create-projects', type=int, default=None, help='Creates up to N projects.') 291 | parse.add_argument('--max-projects', type=int, default=12, help='Max amount of project allowed. Default: 12') 292 | parse.add_argument('--enable-services', default=None, 293 | help='Enables services on the project. Default: IAM and Drive') 294 | parse.add_argument('--services', nargs='+', default=['iam', 'drive'], 295 | help='Specify a different set of services to enable. Overrides the default.') 296 | parse.add_argument('--create-sas', default=None, help='Create service accounts in a project.') 297 | parse.add_argument('--delete-sas', default=None, help='Delete service accounts in a project.') 298 | parse.add_argument('--download-keys', default=None, help='Download keys for all the service accounts in a project.') 299 | parse.add_argument('--quick-setup', default=None, type=int, 300 | help='Create projects, enable services, create service accounts and download keys. ') 301 | parse.add_argument('--new-only', default=False, action='store_true', help='Do not use exisiting projects.') 302 | args = parse.parse_args() 303 | # If credentials file is invalid, search for one. 304 | if not os.path.exists(args.credentials): 305 | options = glob('*.json') 306 | print('No credentials found at %s. Please enable the Drive API in:\n' 307 | 'https://developers.google.com/drive/api/v3/quickstart/python\n' 308 | 'and save the json file as credentials.json' % args.credentials) 309 | if len(options) < 1: 310 | exit(-1) 311 | else: 312 | i = 0 313 | print('Select a credentials file below.') 314 | inp_options = [str(i) for i in list(range(1, len(options) + 1))] + options 315 | while i < len(options): 316 | print(' %d) %s' % (i + 1, options[i])) 317 | i += 1 318 | inp = None 319 | while True: 320 | inp = input('> ') 321 | if inp in inp_options: 322 | break 323 | if inp in options: 324 | args.credentials = inp 325 | else: 326 | args.credentials = options[int(inp) - 1] 327 | print('Use --credentials %s next time to use this credentials file.' % args.credentials) 328 | if args.quick_setup: 329 | opt = '*' 330 | if args.new_only: 331 | opt = '~' 332 | args.services = ['iam', 'drive'] 333 | args.create_projects = args.quick_setup 334 | args.enable_services = opt 335 | args.create_sas = opt 336 | args.download_keys = opt 337 | resp = serviceaccountfactory( 338 | path=args.path, 339 | token=args.token, 340 | credentials=args.credentials, 341 | list_projects=args.list_projects, 342 | list_sas=args.list_sas, 343 | create_projects=args.create_projects, 344 | max_projects=args.max_projects, 345 | create_sas=args.create_sas, 346 | delete_sas=args.delete_sas, 347 | enable_services=args.enable_services, 348 | services=args.services, 349 | download_keys=args.download_keys 350 | ) 351 | if resp is not None: 352 | if args.list_projects: 353 | if resp: 354 | print('Projects (%d):' % len(resp)) 355 | for i in resp: 356 | print(' ' + i) 357 | else: 358 | print('No projects.') 359 | elif args.list_sas: 360 | if resp: 361 | print('Service accounts in %s (%d):' % (args.list_sas, len(resp))) 362 | for i in resp: 363 | print(' %s (%s)' % (i['email'], i['uniqueId'])) 364 | else: 365 | print('No service accounts.') 366 | -------------------------------------------------------------------------------- /bot/helper/mirror_utils/upload_utils/gdriveTools.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pickle 3 | import urllib.parse as urlparse 4 | from urllib.parse import parse_qs 5 | 6 | import re 7 | import json 8 | import requests 9 | 10 | from google.auth.transport.requests import Request 11 | from google.oauth2 import service_account 12 | from google_auth_oauthlib.flow import InstalledAppFlow 13 | from googleapiclient.discovery import build 14 | from googleapiclient.errors import HttpError 15 | from googleapiclient.http import MediaFileUpload 16 | from tenacity import * 17 | 18 | from bot import parent_id, DOWNLOAD_DIR, IS_TEAM_DRIVE, INDEX_URL, \ 19 | USE_SERVICE_ACCOUNTS 20 | from bot.helper.ext_utils.bot_utils import * 21 | from bot.helper.ext_utils.fs_utils import get_mime_type 22 | 23 | LOGGER = logging.getLogger(__name__) 24 | logging.getLogger('googleapiclient.discovery').setLevel(logging.ERROR) 25 | SERVICE_ACCOUNT_INDEX = 0 26 | 27 | 28 | class GoogleDriveHelper: 29 | def __init__(self, name=None, listener=None): 30 | self.__G_DRIVE_TOKEN_FILE = "token.pickle" 31 | # Check https://developers.google.com/drive/scopes for all available scopes 32 | self.__OAUTH_SCOPE = ['https://www.googleapis.com/auth/drive'] 33 | # Redirect URI for installed apps, can be left as is 34 | self.__REDIRECT_URI = "urn:ietf:wg:oauth:2.0:oob" 35 | self.__G_DRIVE_DIR_MIME_TYPE = "application/vnd.google-apps.folder" 36 | self.__G_DRIVE_BASE_DOWNLOAD_URL = "https://drive.google.com/uc?id={}&export=download" 37 | self.__G_DRIVE_DIR_BASE_DOWNLOAD_URL = "https://drive.google.com/drive/folders/{}" 38 | self.__listener = listener 39 | self.__service = self.authorize() 40 | self.__listener = listener 41 | self._file_uploaded_bytes = 0 42 | self.uploaded_bytes = 0 43 | self.UPDATE_INTERVAL = 5 44 | self.start_time = 0 45 | self.total_time = 0 46 | self._should_update = True 47 | self.is_uploading = True 48 | self.is_cancelled = False 49 | self.status = None 50 | self.updater = None 51 | self.name = name 52 | self.update_interval = 3 53 | 54 | def cancel(self): 55 | self.is_cancelled = True 56 | self.is_uploading = False 57 | 58 | def speed(self): 59 | """ 60 | It calculates the average upload speed and returns it in bytes/seconds unit 61 | :return: Upload speed in bytes/second 62 | """ 63 | try: 64 | return self.uploaded_bytes / self.total_time 65 | except ZeroDivisionError: 66 | return 0 67 | 68 | @staticmethod 69 | def getIdFromUrl(link: str): 70 | if "folders" in link or "file" in link: 71 | regex = r"https://drive\.google\.com/(drive)?/?u?/?\d?/?(mobile)?/?(file)?(folders)?/?d?/([-\w]+)[?+]?/?(w+)?" 72 | res = re.search(regex,link) 73 | if res is None: 74 | raise IndexError("GDrive ID not found.") 75 | return res.group(5) 76 | parsed = urlparse.urlparse(link) 77 | return parse_qs(parsed.query)['id'][0] 78 | 79 | @retry(wait=wait_exponential(multiplier=2, min=3, max=6), stop=stop_after_attempt(5), 80 | retry=retry_if_exception_type(HttpError), before=before_log(LOGGER, logging.DEBUG)) 81 | def _on_upload_progress(self): 82 | if self.status is not None: 83 | chunk_size = self.status.total_size * self.status.progress() - self._file_uploaded_bytes 84 | self._file_uploaded_bytes = self.status.total_size * self.status.progress() 85 | LOGGER.debug(f'Uploading {self.name}, chunk size: {get_readable_file_size(chunk_size)}') 86 | self.uploaded_bytes += chunk_size 87 | self.total_time += self.update_interval 88 | 89 | def __upload_empty_file(self, path, file_name, mime_type, parent_id=None): 90 | media_body = MediaFileUpload(path, 91 | mimetype=mime_type, 92 | resumable=False) 93 | file_metadata = { 94 | 'name': file_name, 95 | 'description': 'mirror', 96 | 'mimeType': mime_type, 97 | } 98 | if parent_id is not None: 99 | file_metadata['parents'] = [parent_id] 100 | return self.__service.files().create(supportsTeamDrives=True, 101 | body=file_metadata, media_body=media_body).execute() 102 | 103 | def switchServiceAccount(self): 104 | global SERVICE_ACCOUNT_INDEX 105 | service_account_count = len(os.listdir("accounts")) 106 | if SERVICE_ACCOUNT_INDEX == service_account_count - 1: 107 | SERVICE_ACCOUNT_INDEX = 0 108 | SERVICE_ACCOUNT_INDEX += 1 109 | LOGGER.info(f"Switching to {SERVICE_ACCOUNT_INDEX}.json service account") 110 | self.__service = self.authorize() 111 | 112 | @retry(wait=wait_exponential(multiplier=2, min=3, max=6), stop=stop_after_attempt(5), 113 | retry=retry_if_exception_type(HttpError), before=before_log(LOGGER, logging.DEBUG)) 114 | def __set_permission(self, drive_id): 115 | permissions = { 116 | 'role': 'reader', 117 | 'type': 'anyone', 118 | 'value': None, 119 | 'withLink': True 120 | } 121 | return self.__service.permissions().create(supportsTeamDrives=True, fileId=drive_id, 122 | body=permissions).execute() 123 | 124 | @retry(wait=wait_exponential(multiplier=2, min=3, max=6), stop=stop_after_attempt(5), 125 | retry=retry_if_exception_type(HttpError), before=before_log(LOGGER, logging.DEBUG)) 126 | def upload_file(self, file_path, file_name, mime_type, parent_id): 127 | # File body description 128 | file_metadata = { 129 | 'name': file_name, 130 | 'description': 'mirror', 131 | 'mimeType': mime_type, 132 | } 133 | if parent_id is not None: 134 | file_metadata['parents'] = [parent_id] 135 | 136 | if os.path.getsize(file_path) == 0: 137 | media_body = MediaFileUpload(file_path, 138 | mimetype=mime_type, 139 | resumable=False) 140 | response = self.__service.files().create(supportsTeamDrives=True, 141 | body=file_metadata, media_body=media_body).execute() 142 | if not IS_TEAM_DRIVE: 143 | self.__set_permission(response['id']) 144 | 145 | drive_file = self.__service.files().get(supportsTeamDrives=True, 146 | fileId=response['id']).execute() 147 | download_url = self.__G_DRIVE_BASE_DOWNLOAD_URL.format(drive_file.get('id')) 148 | return download_url 149 | media_body = MediaFileUpload(file_path, 150 | mimetype=mime_type, 151 | resumable=True, 152 | chunksize=50 * 1024 * 1024) 153 | 154 | # Insert a file 155 | drive_file = self.__service.files().create(supportsTeamDrives=True, 156 | body=file_metadata, media_body=media_body) 157 | response = None 158 | while response is None: 159 | if self.is_cancelled: 160 | return None 161 | try: 162 | self.status, response = drive_file.next_chunk() 163 | except HttpError as err: 164 | if err.resp.get('content-type', '').startswith('application/json'): 165 | reason = json.loads(err.content).get('error').get('errors')[0].get('reason') 166 | if reason == 'userRateLimitExceeded' or reason == 'dailyLimitExceeded': 167 | if USE_SERVICE_ACCOUNTS: 168 | self.switchServiceAccount() 169 | LOGGER.info(f"Got: {reason}, Trying Again.") 170 | self.upload_file(file_path, file_name, mime_type, parent_id) 171 | else: 172 | raise err 173 | self._file_uploaded_bytes = 0 174 | # Insert new permissions 175 | if not IS_TEAM_DRIVE: 176 | self.__set_permission(response['id']) 177 | # Define file instance and get url for download 178 | drive_file = self.__service.files().get(supportsTeamDrives=True, fileId=response['id']).execute() 179 | download_url = self.__G_DRIVE_BASE_DOWNLOAD_URL.format(drive_file.get('id')) 180 | return download_url 181 | 182 | def upload(self, file_name: str): 183 | if USE_SERVICE_ACCOUNTS: 184 | self.service_account_count = len(os.listdir("accounts")) 185 | self.__listener.onUploadStarted() 186 | file_dir = f"{DOWNLOAD_DIR}{self.__listener.message.message_id}" 187 | file_path = f"{file_dir}/{file_name}" 188 | LOGGER.info("Uploading File: " + file_path) 189 | self.start_time = time.time() 190 | self.updater = setInterval(self.update_interval, self._on_upload_progress) 191 | if os.path.isfile(file_path): 192 | try: 193 | mime_type = get_mime_type(file_path) 194 | link = self.upload_file(file_path, file_name, mime_type, parent_id) 195 | if link is None: 196 | raise Exception('Upload has been manually cancelled') 197 | LOGGER.info("Uploaded To G-Drive: " + file_path) 198 | except Exception as e: 199 | if isinstance(e, RetryError): 200 | LOGGER.info(f"Total Attempts: {e.last_attempt.attempt_number}") 201 | err = e.last_attempt.exception() 202 | else: 203 | err = e 204 | LOGGER.error(err) 205 | self.__listener.onUploadError(str(err)) 206 | return 207 | finally: 208 | self.updater.cancel() 209 | else: 210 | try: 211 | dir_id = self.create_directory(os.path.basename(os.path.abspath(file_name)), parent_id) 212 | result = self.upload_dir(file_path, dir_id) 213 | if result is None: 214 | raise Exception('Upload has been manually cancelled!') 215 | LOGGER.info("Uploaded To G-Drive: " + file_name) 216 | link = f"https://drive.google.com/folderview?id={dir_id}" 217 | except Exception as e: 218 | if isinstance(e, RetryError): 219 | LOGGER.info(f"Total Attempts: {e.last_attempt.attempt_number}") 220 | err = e.last_attempt.exception() 221 | else: 222 | err = e 223 | LOGGER.error(err) 224 | self.__listener.onUploadError(str(err)) 225 | return 226 | finally: 227 | self.updater.cancel() 228 | LOGGER.info(download_dict) 229 | self.__listener.onUploadComplete(link) 230 | LOGGER.info("Deleting downloaded file/folder..") 231 | return link 232 | 233 | @retry(wait=wait_exponential(multiplier=2, min=3, max=6), stop=stop_after_attempt(5), 234 | retry=retry_if_exception_type(HttpError), before=before_log(LOGGER, logging.DEBUG)) 235 | def copyFile(self, file_id, dest_id): 236 | body = { 237 | 'parents': [dest_id] 238 | } 239 | 240 | try: 241 | res = self.__service.files().copy(supportsAllDrives=True,fileId=file_id,body=body).execute() 242 | return res 243 | except HttpError as err: 244 | if err.resp.get('content-type', '').startswith('application/json'): 245 | reason = json.loads(err.content).get('error').get('errors')[0].get('reason') 246 | if reason == 'userRateLimitExceeded' or reason == 'dailyLimitExceeded': 247 | if USE_SERVICE_ACCOUNTS: 248 | self.switchServiceAccount() 249 | LOGGER.info(f"Got: {reason}, Trying Again.") 250 | self.copyFile(file_id,dest_id) 251 | else: 252 | raise err 253 | 254 | def clone(self, link): 255 | self.transferred_size = 0 256 | try: 257 | file_id = self.getIdFromUrl(link) 258 | except (KeyError,IndexError): 259 | msg = "Google drive ID could not be found in the provided link" 260 | return msg 261 | msg = "" 262 | LOGGER.info(f"File ID: {file_id}") 263 | try: 264 | meta = self.__service.files().get(supportsAllDrives=True, fileId=file_id, 265 | fields="name,id,mimeType,size").execute() 266 | except Exception as e: 267 | return f"{str(e).replace('>', '').replace('<', '')}" 268 | if meta.get("mimeType") == self.__G_DRIVE_DIR_MIME_TYPE: 269 | dir_id = self.create_directory(meta.get('name'), parent_id) 270 | try: 271 | result = self.cloneFolder(meta.get('name'), meta.get('name'), meta.get('id'), dir_id) 272 | except Exception as e: 273 | if isinstance(e, RetryError): 274 | LOGGER.info(f"Total Attempts: {e.last_attempt.attempt_number}") 275 | err = e.last_attempt.exception() 276 | else: 277 | err = str(e).replace('>', '').replace('<', '') 278 | LOGGER.error(err) 279 | return err 280 | msg += f'{meta.get("name")}' \ 281 | f' ({get_readable_file_size(self.transferred_size)})' 282 | if INDEX_URL is not None: 283 | url = requests.utils.requote_uri(f'{INDEX_URL}/{meta.get("name")}/') 284 | msg += f' | Index URL' 285 | else: 286 | try: 287 | file = self.copyFile(meta.get('id'), parent_id) 288 | except Exception as e: 289 | if isinstance(e, RetryError): 290 | LOGGER.info(f"Total Attempts: {e.last_attempt.attempt_number}") 291 | err = e.last_attempt.exception() 292 | else: 293 | err = str(e).replace('>', '').replace('<', '') 294 | LOGGER.error(err) 295 | return err 296 | msg += f'{file.get("name")}' 297 | try: 298 | msg += f' ({get_readable_file_size(int(meta.get("size")))}) ' 299 | if INDEX_URL is not None: 300 | url = requests.utils.requote_uri(f'{INDEX_URL}/{file.get("name")}') 301 | msg += f' | Index URL' 302 | except TypeError: 303 | pass 304 | return msg 305 | 306 | def cloneFolder(self, name, local_path, folder_id, parent_id): 307 | page_token = None 308 | q = f"'{folder_id}' in parents" 309 | files = [] 310 | LOGGER.info(f"Syncing: {local_path}") 311 | new_id = None 312 | while True: 313 | response = self.__service.files().list(supportsTeamDrives=True, 314 | includeTeamDriveItems=True, 315 | q=q, 316 | spaces='drive', 317 | fields='nextPageToken, files(id, name, mimeType,size)', 318 | pageToken=page_token).execute() 319 | for file in response.get('files', []): 320 | files.append(file) 321 | page_token = response.get('nextPageToken', None) 322 | if page_token is None: 323 | break 324 | if len(files) == 0: 325 | return parent_id 326 | for file in files: 327 | if file.get('mimeType') == self.__G_DRIVE_DIR_MIME_TYPE: 328 | file_path = os.path.join(local_path, file.get('name')) 329 | current_dir_id = self.create_directory(file.get('name'), parent_id) 330 | new_id = self.cloneFolder(file.get('name'), file_path, file.get('id'), current_dir_id) 331 | else: 332 | try: 333 | self.transferred_size += int(file.get('size')) 334 | except TypeError: 335 | pass 336 | try: 337 | self.copyFile(file.get('id'), parent_id) 338 | new_id = parent_id 339 | except Exception as e: 340 | if isinstance(e, RetryError): 341 | LOGGER.info(f"Total Attempts: {e.last_attempt.attempt_number}") 342 | err = e.last_attempt.exception() 343 | else: 344 | err = e 345 | LOGGER.error(err) 346 | return new_id 347 | 348 | @retry(wait=wait_exponential(multiplier=2, min=3, max=6), stop=stop_after_attempt(5), 349 | retry=retry_if_exception_type(HttpError), before=before_log(LOGGER, logging.DEBUG)) 350 | def create_directory(self, directory_name, parent_id): 351 | file_metadata = { 352 | "name": directory_name, 353 | "mimeType": self.__G_DRIVE_DIR_MIME_TYPE 354 | } 355 | if parent_id is not None: 356 | file_metadata["parents"] = [parent_id] 357 | file = self.__service.files().create(supportsTeamDrives=True, body=file_metadata).execute() 358 | file_id = file.get("id") 359 | if not IS_TEAM_DRIVE: 360 | self.__set_permission(file_id) 361 | LOGGER.info("Created Google-Drive Folder:\nName: {}\nID: {} ".format(file.get("name"), file_id)) 362 | return file_id 363 | 364 | def upload_dir(self, input_directory, parent_id): 365 | list_dirs = os.listdir(input_directory) 366 | if len(list_dirs) == 0: 367 | return parent_id 368 | new_id = None 369 | for item in list_dirs: 370 | current_file_name = os.path.join(input_directory, item) 371 | if self.is_cancelled: 372 | return None 373 | if os.path.isdir(current_file_name): 374 | current_dir_id = self.create_directory(item, parent_id) 375 | new_id = self.upload_dir(current_file_name, current_dir_id) 376 | else: 377 | mime_type = get_mime_type(current_file_name) 378 | file_name = current_file_name.split("/")[-1] 379 | # current_file_name will have the full path 380 | self.upload_file(current_file_name, file_name, mime_type, parent_id) 381 | new_id = parent_id 382 | return new_id 383 | 384 | def authorize(self): 385 | # Get credentials 386 | credentials = None 387 | if not USE_SERVICE_ACCOUNTS: 388 | if os.path.exists(self.__G_DRIVE_TOKEN_FILE): 389 | with open(self.__G_DRIVE_TOKEN_FILE, 'rb') as f: 390 | credentials = pickle.load(f) 391 | if credentials is None or not credentials.valid: 392 | if credentials and credentials.expired and credentials.refresh_token: 393 | credentials.refresh(Request()) 394 | else: 395 | flow = InstalledAppFlow.from_client_secrets_file( 396 | 'credentials.json', self.__OAUTH_SCOPE) 397 | LOGGER.info(flow) 398 | credentials = flow.run_console(port=0) 399 | 400 | # Save the credentials for the next run 401 | with open(self.__G_DRIVE_TOKEN_FILE, 'wb') as token: 402 | pickle.dump(credentials, token) 403 | else: 404 | LOGGER.info(f"Authorizing with {SERVICE_ACCOUNT_INDEX}.json service account") 405 | credentials = service_account.Credentials.from_service_account_file( 406 | f'accounts/{SERVICE_ACCOUNT_INDEX}.json', 407 | scopes=self.__OAUTH_SCOPE) 408 | return build('drive', 'v3', credentials=credentials, cache_discovery=False) 409 | 410 | def drive_list(self, fileName): 411 | msg = "" 412 | # Create Search Query for API request. 413 | query = f"'{parent_id}' in parents and (name contains '{fileName}')" 414 | response = self.__service.files().list(supportsTeamDrives=True, 415 | includeTeamDriveItems=True, 416 | q=query, 417 | spaces='drive', 418 | pageSize=20, 419 | fields='files(id, name, mimeType, size)', 420 | orderBy='modifiedTime desc').execute() 421 | for file in response.get('files', []): 422 | if file.get( 423 | 'mimeType') == "application/vnd.google-apps.folder": # Detect Whether Current Entity is a Folder or File. 424 | msg += f"⁍ {file.get('name')}" \ 425 | f" (folder)" 426 | if INDEX_URL is not None: 427 | url = requests.utils.requote_uri(f'{INDEX_URL}/{file.get("name")}/') 428 | msg += f' | Index URL' 429 | else: 430 | msg += f"⁍ {file.get('name')} ({get_readable_file_size(int(file.get('size')))})" 432 | if INDEX_URL is not None: 433 | url = requests.utils.requote_uri(f'{INDEX_URL}/{file.get("name")}') 434 | msg += f' | Index URL' 435 | msg += '\n' 436 | return msg 437 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU General Public License is a free, copyleft license for 11 | software and other kinds of works. 12 | 13 | The licenses for most software and other practical works are designed 14 | to take away your freedom to share and change the works. By contrast, 15 | the GNU General Public License is intended to guarantee your freedom to 16 | share and change all versions of a program--to make sure it remains free 17 | software for all its users. We, the Free Software Foundation, use the 18 | GNU General Public License for most of our software; it applies also to 19 | any other work released this way by its authors. You can apply it to 20 | your programs, too. 21 | 22 | When we speak of free software, we are referring to freedom, not 23 | price. Our General Public Licenses are designed to make sure that you 24 | have the freedom to distribute copies of free software (and charge for 25 | them if you wish), that you receive source code or can get it if you 26 | want it, that you can change the software or use pieces of it in new 27 | free programs, and that you know you can do these things. 28 | 29 | To protect your rights, we need to prevent others from denying you 30 | these rights or asking you to surrender the rights. Therefore, you have 31 | certain responsibilities if you distribute copies of the software, or if 32 | you modify it: responsibilities to respect the freedom of others. 33 | 34 | For example, if you distribute copies of such a program, whether 35 | gratis or for a fee, you must pass on to the recipients the same 36 | freedoms that you received. You must make sure that they, too, receive 37 | or can get the source code. And you must show them these terms so they 38 | know their rights. 39 | 40 | Developers that use the GNU GPL protect your rights with two steps: 41 | (1) assert copyright on the software, and (2) offer you this License 42 | giving you legal permission to copy, distribute and/or modify it. 43 | 44 | For the developers' and authors' protection, the GPL clearly explains 45 | that there is no warranty for this free software. For both users' and 46 | authors' sake, the GPL requires that modified versions be marked as 47 | changed, so that their problems will not be attributed erroneously to 48 | authors of previous versions. 49 | 50 | Some devices are designed to deny users access to install or run 51 | modified versions of the software inside them, although the manufacturer 52 | can do so. This is fundamentally incompatible with the aim of 53 | protecting users' freedom to change the software. The systematic 54 | pattern of such abuse occurs in the area of products for individuals to 55 | use, which is precisely where it is most unacceptable. Therefore, we 56 | have designed this version of the GPL to prohibit the practice for those 57 | products. If such problems arise substantially in other domains, we 58 | stand ready to extend this provision to those domains in future versions 59 | of the GPL, as needed to protect the freedom of users. 60 | 61 | Finally, every program is threatened constantly by software patents. 62 | States should not allow patents to restrict development and use of 63 | software on general-purpose computers, but in those that do, we wish to 64 | avoid the special danger that patents applied to a free program could 65 | make it effectively proprietary. To prevent this, the GPL assures that 66 | patents cannot be used to render the program non-free. 67 | 68 | The precise terms and conditions for copying, distribution and 69 | modification follow. 70 | 71 | TERMS AND CONDITIONS 72 | 73 | 0. Definitions. 74 | 75 | "This License" refers to version 3 of the GNU General Public License. 76 | 77 | "Copyright" also means copyright-like laws that apply to other kinds of 78 | works, such as semiconductor masks. 79 | 80 | "The Program" refers to any copyrightable work licensed under this 81 | License. Each licensee is addressed as "you". "Licensees" and 82 | "recipients" may be individuals or organizations. 83 | 84 | To "modify" a work means to copy from or adapt all or part of the work 85 | in a fashion requiring copyright permission, other than the making of an 86 | exact copy. The resulting work is called a "modified version" of the 87 | earlier work or a work "based on" the earlier work. 88 | 89 | A "covered work" means either the unmodified Program or a work based 90 | on the Program. 91 | 92 | To "propagate" a work means to do anything with it that, without 93 | permission, would make you directly or secondarily liable for 94 | infringement under applicable copyright law, except executing it on a 95 | computer or modifying a private copy. Propagation includes copying, 96 | distribution (with or without modification), making available to the 97 | public, and in some countries other activities as well. 98 | 99 | To "convey" a work means any kind of propagation that enables other 100 | parties to make or receive copies. Mere interaction with a user through 101 | a computer network, with no transfer of a copy, is not conveying. 102 | 103 | An interactive user interface displays "Appropriate Legal Notices" 104 | to the extent that it includes a convenient and prominently visible 105 | feature that (1) displays an appropriate copyright notice, and (2) 106 | tells the user that there is no warranty for the work (except to the 107 | extent that warranties are provided), that licensees may convey the 108 | work under this License, and how to view a copy of this License. If 109 | the interface presents a list of user commands or options, such as a 110 | menu, a prominent item in the list meets this criterion. 111 | 112 | 1. Source Code. 113 | 114 | The "source code" for a work means the preferred form of the work 115 | for making modifications to it. "Object code" means any non-source 116 | form of a work. 117 | 118 | A "Standard Interface" means an interface that either is an official 119 | standard defined by a recognized standards body, or, in the case of 120 | interfaces specified for a particular programming language, one that 121 | is widely used among developers working in that language. 122 | 123 | The "System Libraries" of an executable work include anything, other 124 | than the work as a whole, that (a) is included in the normal form of 125 | packaging a Major Component, but which is not part of that Major 126 | Component, and (b) serves only to enable use of the work with that 127 | Major Component, or to implement a Standard Interface for which an 128 | implementation is available to the public in source code form. A 129 | "Major Component", in this context, means a major essential component 130 | (kernel, window system, and so on) of the specific operating system 131 | (if any) on which the executable work runs, or a compiler used to 132 | produce the work, or an object code interpreter used to run it. 133 | 134 | The "Corresponding Source" for a work in object code form means all 135 | the source code needed to generate, install, and (for an executable 136 | work) run the object code and to modify the work, including scripts to 137 | control those activities. However, it does not include the work's 138 | System Libraries, or general-purpose tools or generally available free 139 | programs which are used unmodified in performing those activities but 140 | which are not part of the work. For example, Corresponding Source 141 | includes interface definition files associated with source files for 142 | the work, and the source code for shared libraries and dynamically 143 | linked subprograms that the work is specifically designed to require, 144 | such as by intimate data communication or control flow between those 145 | subprograms and other parts of the work. 146 | 147 | The Corresponding Source need not include anything that users 148 | can regenerate automatically from other parts of the Corresponding 149 | Source. 150 | 151 | The Corresponding Source for a work in source code form is that 152 | same work. 153 | 154 | 2. Basic Permissions. 155 | 156 | All rights granted under this License are granted for the term of 157 | copyright on the Program, and are irrevocable provided the stated 158 | conditions are met. This License explicitly affirms your unlimited 159 | permission to run the unmodified Program. The output from running a 160 | covered work is covered by this License only if the output, given its 161 | content, constitutes a covered work. This License acknowledges your 162 | rights of fair use or other equivalent, as provided by copyright law. 163 | 164 | You may make, run and propagate covered works that you do not 165 | convey, without conditions so long as your license otherwise remains 166 | in force. You may convey covered works to others for the sole purpose 167 | of having them make modifications exclusively for you, or provide you 168 | with facilities for running those works, provided that you comply with 169 | the terms of this License in conveying all material for which you do 170 | not control copyright. Those thus making or running the covered works 171 | for you must do so exclusively on your behalf, under your direction 172 | and control, on terms that prohibit them from making any copies of 173 | your copyrighted material outside their relationship with you. 174 | 175 | Conveying under any other circumstances is permitted solely under 176 | the conditions stated below. Sublicensing is not allowed; section 10 177 | makes it unnecessary. 178 | 179 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 180 | 181 | No covered work shall be deemed part of an effective technological 182 | measure under any applicable law fulfilling obligations under article 183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 184 | similar laws prohibiting or restricting circumvention of such 185 | measures. 186 | 187 | When you convey a covered work, you waive any legal power to forbid 188 | circumvention of technological measures to the extent such circumvention 189 | is effected by exercising rights under this License with respect to 190 | the covered work, and you disclaim any intention to limit operation or 191 | modification of the work as a means of enforcing, against the work's 192 | users, your or third parties' legal rights to forbid circumvention of 193 | technological measures. 194 | 195 | 4. Conveying Verbatim Copies. 196 | 197 | You may convey verbatim copies of the Program's source code as you 198 | receive it, in any medium, provided that you conspicuously and 199 | appropriately publish on each copy an appropriate copyright notice; 200 | keep intact all notices stating that this License and any 201 | non-permissive terms added in accord with section 7 apply to the code; 202 | keep intact all notices of the absence of any warranty; and give all 203 | recipients a copy of this License along with the Program. 204 | 205 | You may charge any price or no price for each copy that you convey, 206 | and you may offer support or warranty protection for a fee. 207 | 208 | 5. Conveying Modified Source Versions. 209 | 210 | You may convey a work based on the Program, or the modifications to 211 | produce it from the Program, in the form of source code under the 212 | terms of section 4, provided that you also meet all of these conditions: 213 | 214 | a) The work must carry prominent notices stating that you modified 215 | it, and giving a relevant date. 216 | 217 | b) The work must carry prominent notices stating that it is 218 | released under this License and any conditions added under section 219 | 7. This requirement modifies the requirement in section 4 to 220 | "keep intact all notices". 221 | 222 | c) You must license the entire work, as a whole, under this 223 | License to anyone who comes into possession of a copy. This 224 | License will therefore apply, along with any applicable section 7 225 | additional terms, to the whole of the work, and all its parts, 226 | regardless of how they are packaged. This License gives no 227 | permission to license the work in any other way, but it does not 228 | invalidate such permission if you have separately received it. 229 | 230 | d) If the work has interactive user interfaces, each must display 231 | Appropriate Legal Notices; however, if the Program has interactive 232 | interfaces that do not display Appropriate Legal Notices, your 233 | work need not make them do so. 234 | 235 | A compilation of a covered work with other separate and independent 236 | works, which are not by their nature extensions of the covered work, 237 | and which are not combined with it such as to form a larger program, 238 | in or on a volume of a storage or distribution medium, is called an 239 | "aggregate" if the compilation and its resulting copyright are not 240 | used to limit the access or legal rights of the compilation's users 241 | beyond what the individual works permit. Inclusion of a covered work 242 | in an aggregate does not cause this License to apply to the other 243 | parts of the aggregate. 244 | 245 | 6. Conveying Non-Source Forms. 246 | 247 | You may convey a covered work in object code form under the terms 248 | of sections 4 and 5, provided that you also convey the 249 | machine-readable Corresponding Source under the terms of this License, 250 | in one of these ways: 251 | 252 | a) Convey the object code in, or embodied in, a physical product 253 | (including a physical distribution medium), accompanied by the 254 | Corresponding Source fixed on a durable physical medium 255 | customarily used for software interchange. 256 | 257 | b) Convey the object code in, or embodied in, a physical product 258 | (including a physical distribution medium), accompanied by a 259 | written offer, valid for at least three years and valid for as 260 | long as you offer spare parts or customer support for that product 261 | model, to give anyone who possesses the object code either (1) a 262 | copy of the Corresponding Source for all the software in the 263 | product that is covered by this License, on a durable physical 264 | medium customarily used for software interchange, for a price no 265 | more than your reasonable cost of physically performing this 266 | conveying of source, or (2) access to copy the 267 | Corresponding Source from a network server at no charge. 268 | 269 | c) Convey individual copies of the object code with a copy of the 270 | written offer to provide the Corresponding Source. This 271 | alternative is allowed only occasionally and noncommercially, and 272 | only if you received the object code with such an offer, in accord 273 | with subsection 6b. 274 | 275 | d) Convey the object code by offering access from a designated 276 | place (gratis or for a charge), and offer equivalent access to the 277 | Corresponding Source in the same way through the same place at no 278 | further charge. You need not require recipients to copy the 279 | Corresponding Source along with the object code. If the place to 280 | copy the object code is a network server, the Corresponding Source 281 | may be on a different server (operated by you or a third party) 282 | that supports equivalent copying facilities, provided you maintain 283 | clear directions next to the object code saying where to find the 284 | Corresponding Source. Regardless of what server hosts the 285 | Corresponding Source, you remain obligated to ensure that it is 286 | available for as long as needed to satisfy these requirements. 287 | 288 | e) Convey the object code using peer-to-peer transmission, provided 289 | you inform other peers where the object code and Corresponding 290 | Source of the work are being offered to the general public at no 291 | charge under subsection 6d. 292 | 293 | A separable portion of the object code, whose source code is excluded 294 | from the Corresponding Source as a System Library, need not be 295 | included in conveying the object code work. 296 | 297 | A "User Product" is either (1) a "consumer product", which means any 298 | tangible personal property which is normally used for personal, family, 299 | or household purposes, or (2) anything designed or sold for incorporation 300 | into a dwelling. In determining whether a product is a consumer product, 301 | doubtful cases shall be resolved in favor of coverage. For a particular 302 | product received by a particular user, "normally used" refers to a 303 | typical or common use of that class of product, regardless of the status 304 | of the particular user or of the way in which the particular user 305 | actually uses, or expects or is expected to use, the product. A product 306 | is a consumer product regardless of whether the product has substantial 307 | commercial, industrial or non-consumer uses, unless such uses represent 308 | the only significant mode of use of the product. 309 | 310 | "Installation Information" for a User Product means any methods, 311 | procedures, authorization keys, or other information required to install 312 | and execute modified versions of a covered work in that User Product from 313 | a modified version of its Corresponding Source. The information must 314 | suffice to ensure that the continued functioning of the modified object 315 | code is in no case prevented or interfered with solely because 316 | modification has been made. 317 | 318 | If you convey an object code work under this section in, or with, or 319 | specifically for use in, a User Product, and the conveying occurs as 320 | part of a transaction in which the right of possession and use of the 321 | User Product is transferred to the recipient in perpetuity or for a 322 | fixed term (regardless of how the transaction is characterized), the 323 | Corresponding Source conveyed under this section must be accompanied 324 | by the Installation Information. But this requirement does not apply 325 | if neither you nor any third party retains the ability to install 326 | modified object code on the User Product (for example, the work has 327 | been installed in ROM). 328 | 329 | The requirement to provide Installation Information does not include a 330 | requirement to continue to provide support service, warranty, or updates 331 | for a work that has been modified or installed by the recipient, or for 332 | the User Product in which it has been modified or installed. Access to a 333 | network may be denied when the modification itself materially and 334 | adversely affects the operation of the network or violates the rules and 335 | protocols for communication across the network. 336 | 337 | Corresponding Source conveyed, and Installation Information provided, 338 | in accord with this section must be in a format that is publicly 339 | documented (and with an implementation available to the public in 340 | source code form), and must require no special password or key for 341 | unpacking, reading or copying. 342 | 343 | 7. Additional Terms. 344 | 345 | "Additional permissions" are terms that supplement the terms of this 346 | License by making exceptions from one or more of its conditions. 347 | Additional permissions that are applicable to the entire Program shall 348 | be treated as though they were included in this License, to the extent 349 | that they are valid under applicable law. If additional permissions 350 | apply only to part of the Program, that part may be used separately 351 | under those permissions, but the entire Program remains governed by 352 | this License without regard to the additional permissions. 353 | 354 | When you convey a copy of a covered work, you may at your option 355 | remove any additional permissions from that copy, or from any part of 356 | it. (Additional permissions may be written to require their own 357 | removal in certain cases when you modify the work.) You may place 358 | additional permissions on material, added by you to a covered work, 359 | for which you have or can give appropriate copyright permission. 360 | 361 | Notwithstanding any other provision of this License, for material you 362 | add to a covered work, you may (if authorized by the copyright holders of 363 | that material) supplement the terms of this License with terms: 364 | 365 | a) Disclaiming warranty or limiting liability differently from the 366 | terms of sections 15 and 16 of this License; or 367 | 368 | b) Requiring preservation of specified reasonable legal notices or 369 | author attributions in that material or in the Appropriate Legal 370 | Notices displayed by works containing it; or 371 | 372 | c) Prohibiting misrepresentation of the origin of that material, or 373 | requiring that modified versions of such material be marked in 374 | reasonable ways as different from the original version; or 375 | 376 | d) Limiting the use for publicity purposes of names of licensors or 377 | authors of the material; or 378 | 379 | e) Declining to grant rights under trademark law for use of some 380 | trade names, trademarks, or service marks; or 381 | 382 | f) Requiring indemnification of licensors and authors of that 383 | material by anyone who conveys the material (or modified versions of 384 | it) with contractual assumptions of liability to the recipient, for 385 | any liability that these contractual assumptions directly impose on 386 | those licensors and authors. 387 | 388 | All other non-permissive additional terms are considered "further 389 | restrictions" within the meaning of section 10. If the Program as you 390 | received it, or any part of it, contains a notice stating that it is 391 | governed by this License along with a term that is a further 392 | restriction, you may remove that term. If a license document contains 393 | a further restriction but permits relicensing or conveying under this 394 | License, you may add to a covered work material governed by the terms 395 | of that license document, provided that the further restriction does 396 | not survive such relicensing or conveying. 397 | 398 | If you add terms to a covered work in accord with this section, you 399 | must place, in the relevant source files, a statement of the 400 | additional terms that apply to those files, or a notice indicating 401 | where to find the applicable terms. 402 | 403 | Additional terms, permissive or non-permissive, may be stated in the 404 | form of a separately written license, or stated as exceptions; 405 | the above requirements apply either way. 406 | 407 | 8. Termination. 408 | 409 | You may not propagate or modify a covered work except as expressly 410 | provided under this License. Any attempt otherwise to propagate or 411 | modify it is void, and will automatically terminate your rights under 412 | this License (including any patent licenses granted under the third 413 | paragraph of section 11). 414 | 415 | However, if you cease all violation of this License, then your 416 | license from a particular copyright holder is reinstated (a) 417 | provisionally, unless and until the copyright holder explicitly and 418 | finally terminates your license, and (b) permanently, if the copyright 419 | holder fails to notify you of the violation by some reasonable means 420 | prior to 60 days after the cessation. 421 | 422 | Moreover, your license from a particular copyright holder is 423 | reinstated permanently if the copyright holder notifies you of the 424 | violation by some reasonable means, this is the first time you have 425 | received notice of violation of this License (for any work) from that 426 | copyright holder, and you cure the violation prior to 30 days after 427 | your receipt of the notice. 428 | 429 | Termination of your rights under this section does not terminate the 430 | licenses of parties who have received copies or rights from you under 431 | this License. If your rights have been terminated and not permanently 432 | reinstated, you do not qualify to receive new licenses for the same 433 | material under section 10. 434 | 435 | 9. Acceptance Not Required for Having Copies. 436 | 437 | You are not required to accept this License in order to receive or 438 | run a copy of the Program. Ancillary propagation of a covered work 439 | occurring solely as a consequence of using peer-to-peer transmission 440 | to receive a copy likewise does not require acceptance. However, 441 | nothing other than this License grants you permission to propagate or 442 | modify any covered work. These actions infringe copyright if you do 443 | not accept this License. Therefore, by modifying or propagating a 444 | covered work, you indicate your acceptance of this License to do so. 445 | 446 | 10. Automatic Licensing of Downstream Recipients. 447 | 448 | Each time you convey a covered work, the recipient automatically 449 | receives a license from the original licensors, to run, modify and 450 | propagate that work, subject to this License. You are not responsible 451 | for enforcing compliance by third parties with this License. 452 | 453 | An "entity transaction" is a transaction transferring control of an 454 | organization, or substantially all assets of one, or subdividing an 455 | organization, or merging organizations. If propagation of a covered 456 | work results from an entity transaction, each party to that 457 | transaction who receives a copy of the work also receives whatever 458 | licenses to the work the party's predecessor in interest had or could 459 | give under the previous paragraph, plus a right to possession of the 460 | Corresponding Source of the work from the predecessor in interest, if 461 | the predecessor has it or can get it with reasonable efforts. 462 | 463 | You may not impose any further restrictions on the exercise of the 464 | rights granted or affirmed under this License. For example, you may 465 | not impose a license fee, royalty, or other charge for exercise of 466 | rights granted under this License, and you may not initiate litigation 467 | (including a cross-claim or counterclaim in a lawsuit) alleging that 468 | any patent claim is infringed by making, using, selling, offering for 469 | sale, or importing the Program or any portion of it. 470 | 471 | 11. Patents. 472 | 473 | A "contributor" is a copyright holder who authorizes use under this 474 | License of the Program or a work on which the Program is based. The 475 | work thus licensed is called the contributor's "contributor version". 476 | 477 | A contributor's "essential patent claims" are all patent claims 478 | owned or controlled by the contributor, whether already acquired or 479 | hereafter acquired, that would be infringed by some manner, permitted 480 | by this License, of making, using, or selling its contributor version, 481 | but do not include claims that would be infringed only as a 482 | consequence of further modification of the contributor version. For 483 | purposes of this definition, "control" includes the right to grant 484 | patent sublicenses in a manner consistent with the requirements of 485 | this License. 486 | 487 | Each contributor grants you a non-exclusive, worldwide, royalty-free 488 | patent license under the contributor's essential patent claims, to 489 | make, use, sell, offer for sale, import and otherwise run, modify and 490 | propagate the contents of its contributor version. 491 | 492 | In the following three paragraphs, a "patent license" is any express 493 | agreement or commitment, however denominated, not to enforce a patent 494 | (such as an express permission to practice a patent or covenant not to 495 | sue for patent infringement). To "grant" such a patent license to a 496 | party means to make such an agreement or commitment not to enforce a 497 | patent against the party. 498 | 499 | If you convey a covered work, knowingly relying on a patent license, 500 | and the Corresponding Source of the work is not available for anyone 501 | to copy, free of charge and under the terms of this License, through a 502 | publicly available network server or other readily accessible means, 503 | then you must either (1) cause the Corresponding Source to be so 504 | available, or (2) arrange to deprive yourself of the benefit of the 505 | patent license for this particular work, or (3) arrange, in a manner 506 | consistent with the requirements of this License, to extend the patent 507 | license to downstream recipients. "Knowingly relying" means you have 508 | actual knowledge that, but for the patent license, your conveying the 509 | covered work in a country, or your recipient's use of the covered work 510 | in a country, would infringe one or more identifiable patents in that 511 | country that you have reason to believe are valid. 512 | 513 | If, pursuant to or in connection with a single transaction or 514 | arrangement, you convey, or propagate by procuring conveyance of, a 515 | covered work, and grant a patent license to some of the parties 516 | receiving the covered work authorizing them to use, propagate, modify 517 | or convey a specific copy of the covered work, then the patent license 518 | you grant is automatically extended to all recipients of the covered 519 | work and works based on it. 520 | 521 | A patent license is "discriminatory" if it does not include within 522 | the scope of its coverage, prohibits the exercise of, or is 523 | conditioned on the non-exercise of one or more of the rights that are 524 | specifically granted under this License. You may not convey a covered 525 | work if you are a party to an arrangement with a third party that is 526 | in the business of distributing software, under which you make payment 527 | to the third party based on the extent of your activity of conveying 528 | the work, and under which the third party grants, to any of the 529 | parties who would receive the covered work from you, a discriminatory 530 | patent license (a) in connection with copies of the covered work 531 | conveyed by you (or copies made from those copies), or (b) primarily 532 | for and in connection with specific products or compilations that 533 | contain the covered work, unless you entered into that arrangement, 534 | or that patent license was granted, prior to 28 March 2007. 535 | 536 | Nothing in this License shall be construed as excluding or limiting 537 | any implied license or other defenses to infringement that may 538 | otherwise be available to you under applicable patent law. 539 | 540 | 12. No Surrender of Others' Freedom. 541 | 542 | If conditions are imposed on you (whether by court order, agreement or 543 | otherwise) that contradict the conditions of this License, they do not 544 | excuse you from the conditions of this License. If you cannot convey a 545 | covered work so as to satisfy simultaneously your obligations under this 546 | License and any other pertinent obligations, then as a consequence you may 547 | not convey it at all. For example, if you agree to terms that obligate you 548 | to collect a royalty for further conveying from those to whom you convey 549 | the Program, the only way you could satisfy both those terms and this 550 | License would be to refrain entirely from conveying the Program. 551 | 552 | 13. Use with the GNU Affero General Public License. 553 | 554 | Notwithstanding any other provision of this License, you have 555 | permission to link or combine any covered work with a work licensed 556 | under version 3 of the GNU Affero General Public License into a single 557 | combined work, and to convey the resulting work. The terms of this 558 | License will continue to apply to the part which is the covered work, 559 | but the special requirements of the GNU Affero General Public License, 560 | section 13, concerning interaction through a network will apply to the 561 | combination as such. 562 | 563 | 14. Revised Versions of this License. 564 | 565 | The Free Software Foundation may publish revised and/or new versions of 566 | the GNU General Public License from time to time. Such new versions will 567 | be similar in spirit to the present version, but may differ in detail to 568 | address new problems or concerns. 569 | 570 | Each version is given a distinguishing version number. If the 571 | Program specifies that a certain numbered version of the GNU General 572 | Public License "or any later version" applies to it, you have the 573 | option of following the terms and conditions either of that numbered 574 | version or of any later version published by the Free Software 575 | Foundation. If the Program does not specify a version number of the 576 | GNU General Public License, you may choose any version ever published 577 | by the Free Software Foundation. 578 | 579 | If the Program specifies that a proxy can decide which future 580 | versions of the GNU General Public License can be used, that proxy's 581 | public statement of acceptance of a version permanently authorizes you 582 | to choose that version for the Program. 583 | 584 | Later license versions may give you additional or different 585 | permissions. However, no additional obligations are imposed on any 586 | author or copyright holder as a result of your choosing to follow a 587 | later version. 588 | 589 | 15. Disclaimer of Warranty. 590 | 591 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 592 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 596 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 597 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 599 | 600 | 16. Limitation of Liability. 601 | 602 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 610 | SUCH DAMAGES. 611 | 612 | 17. Interpretation of Sections 15 and 16. 613 | 614 | If the disclaimer of warranty and limitation of liability provided 615 | above cannot be given local legal effect according to their terms, 616 | reviewing courts shall apply local law that most closely approximates 617 | an absolute waiver of all civil liability in connection with the 618 | Program, unless a warranty or assumption of liability accompanies a 619 | copy of the Program in return for a fee. 620 | 621 | END OF TERMS AND CONDITIONS 622 | 623 | How to Apply These Terms to Your New Programs 624 | 625 | If you develop a new program, and you want it to be of the greatest 626 | possible use to the public, the best way to achieve this is to make it 627 | free software which everyone can redistribute and change under these terms. 628 | 629 | To do so, attach the following notices to the program. It is safest 630 | to attach them to the start of each source file to most effectively 631 | state the exclusion of warranty; and each file should have at least 632 | the "copyright" line and a pointer to where the full notice is found. 633 | 634 | 635 | Copyright (C) 636 | 637 | This program is free software: you can redistribute it and/or modify 638 | it under the terms of the GNU General Public License as published by 639 | the Free Software Foundation, either version 3 of the License, or 640 | (at your option) any later version. 641 | 642 | This program is distributed in the hope that it will be useful, 643 | but WITHOUT ANY WARRANTY; without even the implied warranty of 644 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 645 | GNU General Public License for more details. 646 | 647 | You should have received a copy of the GNU General Public License 648 | along with this program. If not, see . 649 | 650 | Also add information on how to contact you by electronic and paper mail. 651 | 652 | If the program does terminal interaction, make it output a short 653 | notice like this when it starts in an interactive mode: 654 | 655 | Copyright (C) 656 | This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 657 | This is free software, and you are welcome to redistribute it 658 | under certain conditions; type `show c' for details. 659 | 660 | The hypothetical commands `show w' and `show c' should show the appropriate 661 | parts of the General Public License. Of course, your program's commands 662 | might be different; for a GUI interface, you would use an "about box". 663 | 664 | You should also get your employer (if you work as a programmer) or school, 665 | if any, to sign a "copyright disclaimer" for the program, if necessary. 666 | For more information on this, and how to apply and follow the GNU GPL, see 667 | . 668 | 669 | The GNU General Public License does not permit incorporating your program 670 | into proprietary programs. If your program is a subroutine library, you 671 | may consider it more useful to permit linking proprietary applications with 672 | the library. If this is what you want to do, use the GNU Lesser General 673 | Public License instead of this License. But first, please read 674 | . 675 | --------------------------------------------------------------------------------