├── start.sh ├── Procfile ├── requirements.txt ├── bot ├── __init__.py ├── fs_utils.py ├── msg_utils.py ├── decorators.py ├── config.py ├── clone_status.py ├── __main__.py └── gDrive.py ├── print_emails.py ├── generate_drive_token.py ├── LICENSE ├── app.json ├── .gitignore ├── add_to_team_drive.py ├── README.md └── gen_sa_accounts.py /start.sh: -------------------------------------------------------------------------------- 1 | python3 -m bot -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | worker: python3 -m bot -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | requests 2 | python-telegram-bot==12.6.1 3 | google-api-python-client>=1.7.11,<1.7.20 4 | google-auth-httplib2>=0.0.3,<0.1.0 5 | google-auth-oauthlib>=0.4.1,<0.10.0 6 | tenacity>=6.0.0 7 | python-magic -------------------------------------------------------------------------------- /bot/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import time 3 | import os 4 | import telegram.ext as tg 5 | from bot.config import BOT_TOKEN 6 | 7 | if os.path.exists('log.txt'): 8 | with open('log.txt', 'r+') as f: 9 | f.truncate(0) 10 | 11 | logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', 12 | handlers=[logging.FileHandler('log.txt'), logging.StreamHandler()], 13 | level=logging.INFO) 14 | 15 | LOGGER = logging.getLogger(__name__) 16 | updater = tg.Updater(token=BOT_TOKEN, use_context=True, workers=16) 17 | bot = updater.bot 18 | dispatcher = updater.dispatcher -------------------------------------------------------------------------------- /bot/fs_utils.py: -------------------------------------------------------------------------------- 1 | import magic 2 | 3 | SIZE_UNITS = ['B', 'KB', 'MB', 'GB', 'TB', 'PB'] 4 | 5 | def get_mime_type(file_path): 6 | mime = magic.Magic(mime=True) 7 | mime_type = mime.from_file(file_path) 8 | mime_type = mime_type if mime_type else "text/plain" 9 | return mime_type 10 | 11 | 12 | def get_readable_file_size(size_in_bytes) -> str: 13 | if size_in_bytes is None: 14 | return '0B' 15 | index = 0 16 | while size_in_bytes >= 1024: 17 | size_in_bytes /= 1024 18 | index += 1 19 | try: 20 | return f'{round(size_in_bytes, 2)}{SIZE_UNITS[index]}' 21 | except IndexError: 22 | return 'File too large' 23 | -------------------------------------------------------------------------------- /print_emails.py: -------------------------------------------------------------------------------- 1 | import json, os, glob 2 | from bot.config import USE_SERVICE_ACCOUNTS 3 | path = 'accounts' 4 | strr= '' 5 | 6 | if USE_SERVICE_ACCOUNTS and os.path.exists(os.path.join(os.getcwd(), path)): 7 | for count, file in enumerate(glob.glob(os.path.join(os.getcwd(), path, '*.json'))): 8 | x = json.load(open(file, 'rb')) 9 | strr += x['client_email'] + ', ' 10 | 11 | if (count + 1)% 10 == 0: 12 | strr = strr[:-2] 13 | strr += '\n\n' 14 | strr += '-------------------------------------\n\n' 15 | strr = strr[:-3] 16 | print(strr) 17 | else: 18 | print('Please Set `USE_SERVICE_ACCOUNTS` to True in config.py file.') 19 | -------------------------------------------------------------------------------- /bot/msg_utils.py: -------------------------------------------------------------------------------- 1 | from bot import LOGGER 2 | from telegram.message import Message 3 | from telegram.update import Update 4 | 5 | def deleteMessage(bot, message: Message): 6 | try: 7 | bot.delete_message(chat_id=message.chat.id, 8 | message_id=message.message_id) 9 | except Exception as e: 10 | LOGGER.error(str(e)) 11 | 12 | 13 | def sendMessage(text: str, bot, update: Update, parse_mode='HTMl'): 14 | return bot.send_message(update.message.chat_id, 15 | reply_to_message_id=update.message.message_id, 16 | text=text, parse_mode=parse_mode) 17 | 18 | 19 | #To-do: One clone message for all clones; clone cancel command -------------------------------------------------------------------------------- /bot/decorators.py: -------------------------------------------------------------------------------- 1 | from functools import wraps 2 | from bot.config import OWNER_ID, AUTHORISED_USERS 3 | 4 | def is_authorised(func): 5 | @wraps(func) 6 | def wrapper(*args, **kwargs): 7 | if args[0].effective_message.from_user.id in AUTHORISED_USERS or args[0].message.chat_id in AUTHORISED_USERS: 8 | return func(*args, **kwargs) 9 | elif args[0].effective_message.from_user.id == OWNER_ID: 10 | return func(*args, **kwargs) 11 | else: 12 | return False 13 | return wrapper 14 | 15 | def is_owner(func): 16 | @wraps(func) 17 | def wrapper(*args, **kwargs): 18 | if args[0].effective_message.from_user.id == OWNER_ID: 19 | return func(*args, **kwargs) 20 | else: 21 | return False 22 | return wrapper 23 | -------------------------------------------------------------------------------- /generate_drive_token.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | import os 3 | from google_auth_oauthlib.flow import InstalledAppFlow 4 | from google.auth.transport.requests import Request 5 | 6 | credentials = None 7 | __G_DRIVE_TOKEN_FILE = "token.pickle" 8 | __OAUTH_SCOPE = ["https://www.googleapis.com/auth/drive"] 9 | if os.path.exists(__G_DRIVE_TOKEN_FILE): 10 | with open(__G_DRIVE_TOKEN_FILE, 'rb') as f: 11 | credentials = pickle.load(f) 12 | if credentials is None or not credentials.valid: 13 | if credentials and credentials.expired and credentials.refresh_token: 14 | credentials.refresh(Request()) 15 | else: 16 | flow = InstalledAppFlow.from_client_secrets_file( 17 | 'credentials.json', __OAUTH_SCOPE) 18 | credentials = flow.run_console(port=0) 19 | 20 | # Save the credentials for the next run 21 | with open(__G_DRIVE_TOKEN_FILE, 'wb') as token: 22 | pickle.dump(credentials, token) -------------------------------------------------------------------------------- /bot/config.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | from distutils.util import strtobool as stb 4 | 5 | # -------------------------------------- 6 | BOT_TOKEN = "" 7 | GDRIVE_FOLDER_ID = "" 8 | # Default folder id. 9 | OWNER_ID = 123455673 10 | # Example: OWNER_ID = 619418070 11 | AUTHORISED_USERS = [] 12 | # Example: AUTHORISED_USERS = [63055333, 100483029, -1003943959] 13 | INDEX_URL = "" 14 | IS_TEAM_DRIVE = True 15 | USE_SERVICE_ACCOUNTS = True 16 | # -------------------------------------- 17 | 18 | # dont edit below this > 19 | 20 | 21 | 22 | BOT_TOKEN = os.environ.get('BOT_TOKEN', BOT_TOKEN) 23 | GDRIVE_FOLDER_ID = os.environ.get('GDRIVE_FOLDER_ID', GDRIVE_FOLDER_ID) 24 | OWNER_ID = int(os.environ.get('OWNER_ID', OWNER_ID)) 25 | AUTHORISED_USERS = json.loads(os.environ.get('AUTHORISED_USERS', json.dumps(AUTHORISED_USERS))) 26 | INDEX_URL = os.environ.get('INDEX_URL', INDEX_URL) 27 | IS_TEAM_DRIVE = stb(os.environ.get('IS_TEAM_DRIVE', str(IS_TEAM_DRIVE))) 28 | USE_SERVICE_ACCOUNTS = stb(os.environ.get('USE_SERVICE_ACCOUNTS', str(USE_SERVICE_ACCOUNTS))) 29 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Jagrit Thapar 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /bot/clone_status.py: -------------------------------------------------------------------------------- 1 | from bot.fs_utils import get_readable_file_size 2 | 3 | class CloneStatus: 4 | def __init__(self, size=0): 5 | self.size = size 6 | self.name = '' 7 | self.status = False 8 | self.checking = False 9 | self.MainFolderName = '' 10 | self.MainFolderLink = '' 11 | self.DestinationFolderName = '' 12 | self.DestinationFolderLink = '' 13 | 14 | 15 | def get_size(self): 16 | return get_readable_file_size(int(self.size)) 17 | 18 | def add_size(self, value): 19 | self.size += int(value) 20 | 21 | def set_name(self, name=''): 22 | self.name = name 23 | 24 | def get_name(self): 25 | return self.name 26 | 27 | def set_status(self, stat): 28 | self.status = stat 29 | 30 | def done(self): 31 | return self.status 32 | 33 | def checkFileExist(self, checking=False): 34 | self.checking = checking 35 | 36 | def checkFileStatus(self): 37 | return self.checking 38 | 39 | def SetMainFolder(self, folder_name, link): 40 | self.MainFolderName = folder_name 41 | self.MainFolderLink = link 42 | 43 | def SetDestinationFolder(self, folder_name, link): 44 | self.DestinationFolderName = folder_name 45 | self.DestinationFolderLink = link -------------------------------------------------------------------------------- /app.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "TelegramCloneBot", 3 | "description": "Telegram CloneBot by @jagrit007", 4 | "logo": "https://i.imgur.com/ZLi4nDP.jpg", 5 | "keywords": [ 6 | "telegram", 7 | "clone", 8 | "google drive", 9 | "clone bot" 10 | ], 11 | "repository": "https://github.com/jagrit007/Telegram-CloneBot", 12 | "website": "https://github.com/jagrit007/Telegram-CloneBot", 13 | "success_url": "https://t.me/scippletech", 14 | "env": { 15 | "BOT_TOKEN": { 16 | "description": "Get this value from @BotFather on Telegram", 17 | "value": "", 18 | "required": true 19 | }, 20 | "GDRIVE_FOLDER_ID": { 21 | "description": "Google Drive Folder ID to Clone to. (Dont paste a link)", 22 | "value": "", 23 | "required": true 24 | }, 25 | "OWNER_ID": { 26 | "description": "Get this value by sending /id to @kelverbot on Telegram.", 27 | "value": "", 28 | "required": true 29 | }, 30 | "AUTHORISED_USERS" : { 31 | "description": "List of user ids to allow bot access to. Note: Write within [] and seperate with ,", 32 | "value": "[]", 33 | "required": false 34 | }, 35 | "INDEX_URL": { 36 | "description": "(Optional) CloudFlare Workers Index Link from GDINDEX (dont put / at end)", 37 | "required": false 38 | }, 39 | "IS_TEAM_DRIVE": { 40 | "description": "Does you 'GDRIVE_FOLDER_ID' lead to a Team Drive or Normal Google Drive Folder", 41 | "required": false, 42 | "value": "False" 43 | }, 44 | "USE_SERVICE_ACCOUNTS": { 45 | "description": "If you are directly deploying from GitHub, set this to False. But if you later add Service Accounts, go to 'Config Vars' in app settings and set this to True.", 46 | "value": "False", 47 | "required": false 48 | } 49 | }, 50 | "buildpacks": [{ 51 | "url": "heroku/python" 52 | }] 53 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | -------------------------------------------------------------------------------- /add_to_team_drive.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | from google.oauth2.service_account import Credentials 3 | import googleapiclient.discovery, json, progress.bar, glob, sys, argparse, time 4 | from google_auth_oauthlib.flow import InstalledAppFlow 5 | from google.auth.transport.requests import Request 6 | import os, pickle 7 | 8 | stt = time.time() 9 | 10 | parse = argparse.ArgumentParser( 11 | description='A tool to add service accounts to a shared drive from a folder containing credential files.') 12 | parse.add_argument('--path', '-p', default='accounts', 13 | help='Specify an alternative path to the service accounts folder.') 14 | parse.add_argument('--credentials', '-c', default='./credentials.json', 15 | help='Specify the relative path for the credentials file.') 16 | parse.add_argument('--yes', '-y', default=False, action='store_true', help='Skips the sanity prompt.') 17 | parsereq = parse.add_argument_group('required arguments') 18 | parsereq.add_argument('--drive-id', '-d', help='The ID of the Shared Drive.', required=True) 19 | 20 | args = parse.parse_args() 21 | acc_dir = args.path 22 | did = args.drive_id 23 | credentials = glob.glob(args.credentials) 24 | 25 | try: 26 | open(credentials[0], 'r') 27 | print('>> Found credentials.') 28 | except IndexError: 29 | print('>> No credentials found.') 30 | sys.exit(0) 31 | 32 | if not args.yes: 33 | # input('Make sure the following client id is added to the shared drive as Manager:\n' + json.loads((open( 34 | # credentials[0],'r').read()))['installed']['client_id']) 35 | input('>> Make sure the **Google account** that has generated credentials.json\n is added into your Team Drive ' 36 | '(shared drive) as Manager\n>> (Press any key to continue)') 37 | 38 | creds = None 39 | if os.path.exists('token_sa.pickle'): 40 | with open('token_sa.pickle', 'rb') as token: 41 | creds = pickle.load(token) 42 | # If there are no (valid) credentials available, let the user log in. 43 | if not creds or not creds.valid: 44 | if creds and creds.expired and creds.refresh_token: 45 | creds.refresh(Request()) 46 | else: 47 | flow = InstalledAppFlow.from_client_secrets_file(credentials[0], scopes=[ 48 | 'https://www.googleapis.com/auth/admin.directory.group', 49 | 'https://www.googleapis.com/auth/admin.directory.group.member' 50 | ]) 51 | # creds = flow.run_local_server(port=0) 52 | creds = flow.run_console() 53 | # Save the credentials for the next run 54 | with open('token_sa.pickle', 'wb') as token: 55 | pickle.dump(creds, token) 56 | 57 | drive = googleapiclient.discovery.build("drive", "v3", credentials=creds) 58 | batch = drive.new_batch_http_request() 59 | 60 | aa = glob.glob('%s/*.json' % acc_dir) 61 | pbar = progress.bar.Bar("Readying accounts", max=len(aa)) 62 | for i in aa: 63 | ce = json.loads(open(i, 'r').read())['client_email'] 64 | batch.add(drive.permissions().create(fileId=did, supportsAllDrives=True, body={ 65 | "role": "fileOrganizer", 66 | "type": "user", 67 | "emailAddress": ce 68 | })) 69 | pbar.next() 70 | pbar.finish() 71 | print('Adding...') 72 | batch.execute() 73 | 74 | print('Complete.') 75 | hours, rem = divmod((time.time() - stt), 3600) 76 | minutes, sec = divmod(rem, 60) 77 | print("Elapsed Time:\n{:0>2}:{:0>2}:{:05.2f}".format(int(hours), int(minutes), sec)) -------------------------------------------------------------------------------- /bot/__main__.py: -------------------------------------------------------------------------------- 1 | from telegram.ext import CommandHandler, run_async 2 | from bot.gDrive import GoogleDriveHelper 3 | from bot.fs_utils import get_readable_file_size 4 | from bot import LOGGER, dispatcher, updater, bot 5 | from bot.config import BOT_TOKEN, OWNER_ID, GDRIVE_FOLDER_ID 6 | from bot.decorators import is_authorised, is_owner 7 | from telegram.error import TimedOut, BadRequest 8 | from bot.clone_status import CloneStatus 9 | from bot.msg_utils import deleteMessage, sendMessage 10 | import time 11 | 12 | REPO_LINK = "https://github.com/jagrit007/Telegram-CloneBot" 13 | # Soon to be used for direct updates from within the bot. 14 | 15 | @run_async 16 | def start(update, context): 17 | sendMessage("Hello! Please send me a Google Drive Shareable Link to Clone to your Drive!" \ 18 | "\nSend /help for checking all available commands.", 19 | context.bot, update, 'Markdown') 20 | # ;-; 21 | 22 | @run_async 23 | def helper(update, context): 24 | sendMessage("Here are the available commands of the bot\n\n" \ 25 | "*Usage:* `/clone [DESTINATION_ID]`\n*Example:* \n1. `/clone https://drive.google.com/drive/u/1/folders/0AO-ISIXXXXXXXXXXXX`\n2. `/clone 0AO-ISIXXXXXXXXXXXX`" \ 26 | "\n*DESTIONATION_ID* is optional. It can be either link or ID to where you wish to store a particular clone." \ 27 | "\n\nYou can also *ignore folders* from clone process by doing the following:\n" \ 28 | "`/clone [DESTINATION] [id1,id2,id3]`\n In this example: id1, id2 and id3 would get ignored from cloning\nDo not use <> or [] in actual message." \ 29 | "*Make sure to not put any space between commas (,).*\n" \ 30 | f"Source of this bot: [GitHub]({REPO_LINK})", context.bot, update, 'Markdown') 31 | 32 | # TODO Cancel Clones with /cancel command. 33 | @run_async 34 | @is_authorised 35 | def cloneNode(update, context): 36 | args = update.message.text.split(" ") 37 | if len(args) > 1: 38 | link = args[1] 39 | try: 40 | ignoreList = args[-1].split(',') 41 | except IndexError: 42 | ignoreList = [] 43 | 44 | DESTINATION_ID = GDRIVE_FOLDER_ID 45 | try: 46 | DESTINATION_ID = args[2] 47 | print(DESTINATION_ID) 48 | except IndexError: 49 | pass 50 | # Usage: /clone , 51 | 52 | msg = sendMessage(f"Cloning: {link}", context.bot, update) 53 | status_class = CloneStatus() 54 | gd = GoogleDriveHelper(GFolder_ID=DESTINATION_ID) 55 | sendCloneStatus(update, context, status_class, msg, link) 56 | result = gd.clone(link, status_class, ignoreList=ignoreList) 57 | deleteMessage(context.bot, msg) 58 | status_class.set_status(True) 59 | sendMessage(result, context.bot, update) 60 | else: 61 | sendMessage("Please Provide a Google Drive Shared Link to Clone.", bot, update) 62 | 63 | 64 | @run_async 65 | def sendCloneStatus(update, context, status, msg, link): 66 | old_text = '' 67 | while not status.done(): 68 | sleeper(3) 69 | try: 70 | text=f'🔗 *Cloning:* [{status.MainFolderName}]({status.MainFolderLink})\n━━━━━━━━━━━━━━\n🗃️ *Current File:* `{status.get_name()}`\n⬆️ *Transferred*: `{status.get_size()}`\n📁 *Destination:* [{status.DestinationFolderName}]({status.DestinationFolderLink})' 71 | if status.checkFileStatus(): 72 | text += f"\n🕒 *Checking Existing Files:* `{str(status.checkFileStatus())}`" 73 | if not text == old_text: 74 | msg.edit_text(text=text, parse_mode="Markdown", timeout=200) 75 | old_text = text 76 | except Exception as e: 77 | LOGGER.error(e) 78 | if str(e) == "Message to edit not found": 79 | break 80 | sleeper(2) 81 | continue 82 | return 83 | 84 | def sleeper(value, enabled=True): 85 | time.sleep(int(value)) 86 | return 87 | 88 | @run_async 89 | @is_owner 90 | def sendLogs(update, context): 91 | with open('log.txt', 'rb') as f: 92 | bot.send_document(document=f, filename=f.name, 93 | reply_to_message_id=update.message.message_id, 94 | chat_id=update.message.chat_id) 95 | 96 | def main(): 97 | LOGGER.info("Bot Started!") 98 | clone_handler = CommandHandler('clone', cloneNode) 99 | start_handler = CommandHandler('start', start) 100 | help_handler = CommandHandler('help', helper) 101 | log_handler = CommandHandler('logs', sendLogs) 102 | dispatcher.add_handler(log_handler) 103 | dispatcher.add_handler(start_handler) 104 | dispatcher.add_handler(clone_handler) 105 | dispatcher.add_handler(help_handler) 106 | updater.start_polling() 107 | 108 | main() -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Why? 2 | For all my friends using my TDs who now need to store everything in it instead of their Drive. [Need help?](https://t.me/tgclonebot) 3 | 4 |

5 | drawing 6 |

7 | 8 | ## Guide: 9 | - YouTube Guide: [Google Drive Clone Bot Set-Up Tutorial | Telegram Bot Setup Guide](https://www.youtube.com/watch?v=2r3_jR7SvUo&feature=youtu.be) 10 | - Follow the above guide for Heroku. 11 | - If you wish to run on a VPS, Do all the stuff I did on the VPS Terminal ;) 12 | - Wish to run anywhere else? Follow the guide till the part where I download ZIP Archive from Repl.it. Use that zip on any device you'd like to run the bot on. 13 | - Don't forget to install requirements.txt 14 | ``` 15 | pip3 install -r requirements.txt 16 | ``` 17 | - [Adding Service Accounts to Google Group/TeamDrive](https://youtu.be/pBfsmJhYr78) 18 | 19 | ## Setting up config file (present in bot/config.py) 20 | - **BOT_TOKEN** : The telegram bot token that you get from @BotFather 21 | - **GDRIVE_FOLDER_ID** : This is the folder ID of the Google Drive Folder to which you want to clone. 22 | - **OWNER_ID** : The Telegram user ID (not username) of the owner of the bot (if you do not have that, send /id to @kelverbot ) 23 | - **AUTHORISED_USERS** : The Telegram user IDs (not username) of people you wish to allow for bot access.It can also be group chat id. Write like: [123456, 4030394, -1003823820] 24 | - **IS_TEAM_DRIVE** : (Optional field) Set to True if GDRIVE_FOLDER_ID is from a Team Drive else False or Leave it empty. 25 | - **USE_SERVICE_ACCOUNTS**: (Optional field) (Leave empty if unsure) Whether to use service accounts or not. For this to work see "Using service accounts" section below. 26 | - **INDEX_URL** : (Optional field) Refer to https://github.com/maple3142/GDIndex/ The URL should not have any trailing '/' 27 | 28 | ## Getting Google OAuth API credential file 29 | 30 | - Visit the [Google Cloud Console](https://console.developers.google.com/apis/credentials) 31 | - Go to the OAuth Consent tab, fill it, and save. 32 | - Go to the Credentials tab and click Create Credentials -> OAuth Client ID 33 | - Choose Other and Create. 34 | - Use the download button to download your credentials. 35 | - Move that file to the root of clone-bot, and rename it to credentials.json 36 | - Visit [Google API page](https://console.developers.google.com/apis/library) 37 | - Search for Drive and enable it if it is disabled 38 | - Finally, run the script to generate token file (token.pickle) for Google Drive: 39 | ``` 40 | pip install google-api-python-client google-auth-httplib2 google-auth-oauthlib 41 | python3 generate_drive_token.py 42 | ``` 43 | # Running 44 | - To run this bot (locally) (suggested) 45 | ``` 46 | python3 -m bot 47 | ``` 48 | - Deploying to Heroku (Optional) (Not Suitable for very big Clones!) 49 | 50 | [![Deploy](https://www.herokucdn.com/deploy/button.svg)](https://dashboard.heroku.com/new?template=https://github.com/jagrit007/Telegram-CloneBot) 51 | 52 | - Please know that after using this button, your work isn't done. You gotta [clone heroku app](https://devcenter.heroku.com/articles/git-clone-heroku-app) and add credentials.json and token.pickle (By now you would know how to make it.) and this is the perfect time to generate service accounts if you wish to use them. After it's all done, [Push changes to Heroku (Step1-2 only).](https://docs.railsbridge.org/intro-to-rails/deploying_to_heroku_again) 53 | 54 | **Tip: Instead of using Termux or local machine, use [repl.it](https://repl.it/), atleast it won't throw any errors in installing Python requirements. From [repl.it](https://repl.it/) you could push to a private GitHub repo and attach that to Heroku.** 55 | 56 | 57 | # Using service accounts for uploading to avoid user rate limit 58 | For Service Account to work, you must set USE_SERVICE_ACCOUNTS=True in config file or environment variables 59 | Many thanks to [AutoRClone](https://github.com/xyou365/AutoRclone) for the scripts 60 | ## Generating service accounts 61 | Step 1. Generate service accounts [What is service account](https://cloud.google.com/iam/docs/service-accounts) 62 | --------------------------------- 63 | Let us create only the service accounts that we need. 64 | **Warning:** abuse of this feature is not the aim of autorclone and we do **NOT** recommend that you make a lot of projects, just one project and 100 sa allow you plenty of use, its also possible that overabuse might get your projects banned by google. 65 | 66 | ``` 67 | Note: 1 service account can copy around 750gb a day, 1 project makes 100 service accounts so thats 75tb a day, for most users this should easily suffice. 68 | ``` 69 | 70 | `python3 gen_sa_accounts.py --quick-setup 1 --new-only` 71 | 72 | A folder named accounts will be created which will contain keys for the service accounts created 73 | 74 | NOTE: If you have created SAs in past from this script, you can also just re download the keys by running: 75 | ``` 76 | python3 gen_sa_accounts.py --download-keys project_id 77 | ``` 78 | 79 | ### Add all the service accounts to the Team Drive or folder 80 | - Run: 81 | ``` 82 | python3 add_to_team_drive.py -d SharedTeamDriveSrcID 83 | ``` 84 | 85 | ### Credits 86 | - https://github.com/jagrit007 87 | - https://github.com/lzzy12/python-aria-mirror-bot 88 | - https://github.com/xyou365/AutoRclone 89 | -------------------------------------------------------------------------------- /gen_sa_accounts.py: -------------------------------------------------------------------------------- 1 | import errno 2 | import os 3 | import pickle 4 | import sys 5 | from argparse import ArgumentParser 6 | from base64 import b64decode 7 | from glob import glob 8 | from json import loads 9 | from random import choice 10 | from time import sleep 11 | 12 | from google.auth.transport.requests import Request 13 | from google_auth_oauthlib.flow import InstalledAppFlow 14 | from googleapiclient.discovery import build 15 | from googleapiclient.errors import HttpError 16 | 17 | SCOPES = ['https://www.googleapis.com/auth/drive', 'https://www.googleapis.com/auth/cloud-platform', 18 | 'https://www.googleapis.com/auth/iam'] 19 | project_create_ops = [] 20 | current_key_dump = [] 21 | sleep_time = 30 22 | 23 | 24 | # Create count SAs in project 25 | def _create_accounts(service, project, count): 26 | batch = service.new_batch_http_request(callback=_def_batch_resp) 27 | for i in range(count): 28 | aid = _generate_id('mfc-') 29 | batch.add(service.projects().serviceAccounts().create(name='projects/' + project, body={'accountId': aid, 30 | 'serviceAccount': { 31 | 'displayName': aid}})) 32 | batch.execute() 33 | 34 | 35 | # Create accounts needed to fill project 36 | def _create_remaining_accounts(iam, project): 37 | print('Creating accounts in %s' % project) 38 | sa_count = len(_list_sas(iam, project)) 39 | while sa_count != 100: 40 | _create_accounts(iam, project, 100 - sa_count) 41 | sa_count = len(_list_sas(iam, project)) 42 | 43 | 44 | # Generate a random id 45 | def _generate_id(prefix='saf-'): 46 | chars = '-abcdefghijklmnopqrstuvwxyz1234567890' 47 | return prefix + ''.join(choice(chars) for _ in range(25)) + choice(chars[1:]) 48 | 49 | 50 | # List projects using service 51 | def _get_projects(service): 52 | return [i['projectId'] for i in service.projects().list().execute()['projects']] 53 | 54 | 55 | # Default batch callback handler 56 | def _def_batch_resp(id, resp, exception): 57 | if exception is not None: 58 | if str(exception).startswith(' 0: 219 | current_count = len(_get_projects(cloud)) 220 | if current_count + create_projects <= max_projects: 221 | print('Creating %d projects' % (create_projects)) 222 | nprjs = _create_projects(cloud, create_projects) 223 | selected_projects = nprjs 224 | else: 225 | sys.exit('No, you cannot create %d new project (s).\n' 226 | 'Please reduce value of --quick-setup.\n' 227 | 'Remember that you can totally create %d projects (%d already).\n' 228 | 'Please do not delete existing projects unless you know what you are doing' % ( 229 | create_projects, max_projects, current_count)) 230 | else: 231 | print('Will overwrite all service accounts in existing projects.\n' 232 | 'So make sure you have some projects already.') 233 | input("Press Enter to continue...") 234 | 235 | if enable_services: 236 | ste = [] 237 | ste.append(enable_services) 238 | if enable_services == '~': 239 | ste = selected_projects 240 | elif enable_services == '*': 241 | ste = _get_projects(cloud) 242 | services = [i + '.googleapis.com' for i in services] 243 | print('Enabling services') 244 | _enable_services(serviceusage, ste, services) 245 | if create_sas: 246 | stc = [] 247 | stc.append(create_sas) 248 | if create_sas == '~': 249 | stc = selected_projects 250 | elif create_sas == '*': 251 | stc = _get_projects(cloud) 252 | for i in stc: 253 | _create_remaining_accounts(iam, i) 254 | if download_keys: 255 | try: 256 | os.mkdir(path) 257 | except OSError as e: 258 | if e.errno == errno.EEXIST: 259 | pass 260 | else: 261 | raise 262 | std = [] 263 | std.append(download_keys) 264 | if download_keys == '~': 265 | std = selected_projects 266 | elif download_keys == '*': 267 | std = _get_projects(cloud) 268 | _create_sa_keys(iam, std, path) 269 | if delete_sas: 270 | std = [] 271 | std.append(delete_sas) 272 | if delete_sas == '~': 273 | std = selected_projects 274 | elif delete_sas == '*': 275 | std = _get_projects(cloud) 276 | for i in std: 277 | print('Deleting service accounts in %s' % i) 278 | _delete_sas(iam, i) 279 | 280 | 281 | if __name__ == '__main__': 282 | parse = ArgumentParser(description='A tool to create Google service accounts.') 283 | parse.add_argument('--path', '-p', default='accounts', 284 | help='Specify an alternate directory to output the credential files.') 285 | parse.add_argument('--token', default='token_sa.pickle', help='Specify the pickle token file path.') 286 | parse.add_argument('--credentials', default='credentials.json', help='Specify the credentials file path.') 287 | parse.add_argument('--list-projects', default=False, action='store_true', 288 | help='List projects viewable by the user.') 289 | parse.add_argument('--list-sas', default=False, help='List service accounts in a project.') 290 | parse.add_argument('--create-projects', type=int, default=None, help='Creates up to N projects.') 291 | parse.add_argument('--max-projects', type=int, default=12, help='Max amount of project allowed. Default: 12') 292 | parse.add_argument('--enable-services', default=None, 293 | help='Enables services on the project. Default: IAM and Drive') 294 | parse.add_argument('--services', nargs='+', default=['iam', 'drive'], 295 | help='Specify a different set of services to enable. Overrides the default.') 296 | parse.add_argument('--create-sas', default=None, help='Create service accounts in a project.') 297 | parse.add_argument('--delete-sas', default=None, help='Delete service accounts in a project.') 298 | parse.add_argument('--download-keys', default=None, help='Download keys for all the service accounts in a project.') 299 | parse.add_argument('--quick-setup', default=None, type=int, 300 | help='Create projects, enable services, create service accounts and download keys. ') 301 | parse.add_argument('--new-only', default=False, action='store_true', help='Do not use exisiting projects.') 302 | args = parse.parse_args() 303 | # If credentials file is invalid, search for one. 304 | if not os.path.exists(args.credentials): 305 | options = glob('*.json') 306 | print('No credentials found at %s. Please enable the Drive API in:\n' 307 | 'https://developers.google.com/drive/api/v3/quickstart/python\n' 308 | 'and save the json file as credentials.json' % args.credentials) 309 | if len(options) < 1: 310 | exit(-1) 311 | else: 312 | i = 0 313 | print('Select a credentials file below.') 314 | inp_options = [str(i) for i in list(range(1, len(options) + 1))] + options 315 | while i < len(options): 316 | print(' %d) %s' % (i + 1, options[i])) 317 | i += 1 318 | inp = None 319 | while True: 320 | inp = input('> ') 321 | if inp in inp_options: 322 | break 323 | if inp in options: 324 | args.credentials = inp 325 | else: 326 | args.credentials = options[int(inp) - 1] 327 | print('Use --credentials %s next time to use this credentials file.' % args.credentials) 328 | if args.quick_setup: 329 | opt = '*' 330 | if args.new_only: 331 | opt = '~' 332 | args.services = ['iam', 'drive'] 333 | args.create_projects = args.quick_setup 334 | args.enable_services = opt 335 | args.create_sas = opt 336 | args.download_keys = opt 337 | resp = serviceaccountfactory( 338 | path=args.path, 339 | token=args.token, 340 | credentials=args.credentials, 341 | list_projects=args.list_projects, 342 | list_sas=args.list_sas, 343 | create_projects=args.create_projects, 344 | max_projects=args.max_projects, 345 | create_sas=args.create_sas, 346 | delete_sas=args.delete_sas, 347 | enable_services=args.enable_services, 348 | services=args.services, 349 | download_keys=args.download_keys 350 | ) 351 | if resp is not None: 352 | if args.list_projects: 353 | if resp: 354 | print('Projects (%d):' % len(resp)) 355 | for i in resp: 356 | print(' ' + i) 357 | else: 358 | print('No projects.') 359 | elif args.list_sas: 360 | if resp: 361 | print('Service accounts in %s (%d):' % (args.list_sas, len(resp))) 362 | for i in resp: 363 | print(' %s (%s)' % (i['email'], i['uniqueId'])) 364 | else: 365 | print('No service accounts.') -------------------------------------------------------------------------------- /bot/gDrive.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pickle 3 | import urllib.parse as urlparse 4 | from urllib.parse import parse_qs 5 | from bot import LOGGER 6 | 7 | import json 8 | import logging 9 | import re 10 | import requests 11 | import socket 12 | 13 | from google.auth.transport.requests import Request 14 | from google.oauth2 import service_account 15 | from google_auth_oauthlib.flow import InstalledAppFlow 16 | from googleapiclient.discovery import build 17 | from googleapiclient.errors import HttpError 18 | from googleapiclient.http import MediaFileUpload 19 | from tenacity import * 20 | 21 | from bot.config import IS_TEAM_DRIVE, \ 22 | USE_SERVICE_ACCOUNTS, GDRIVE_FOLDER_ID, INDEX_URL 23 | from bot.fs_utils import get_mime_type 24 | 25 | logging.getLogger('googleapiclient.discovery').setLevel(logging.ERROR) 26 | socket.setdefaulttimeout(650) # https://github.com/googleapis/google-api-python-client/issues/632#issuecomment-541973021 27 | SERVICE_ACCOUNT_INDEX = 0 28 | 29 | def clean_name(name): 30 | name = name.replace("'", "\\'") 31 | return name 32 | 33 | class GoogleDriveHelper: 34 | def __init__(self, name=None, listener=None, GFolder_ID=GDRIVE_FOLDER_ID): 35 | self.__G_DRIVE_TOKEN_FILE = "token.pickle" 36 | # Check https://developers.google.com/drive/scopes for all available scopes 37 | self.__OAUTH_SCOPE = ['https://www.googleapis.com/auth/drive'] 38 | # Redirect URI for installed apps, can be left as is 39 | self.__REDIRECT_URI = "urn:ietf:wg:oauth:2.0:oob" 40 | self.__G_DRIVE_DIR_MIME_TYPE = "application/vnd.google-apps.folder" 41 | self.__G_DRIVE_BASE_DOWNLOAD_URL = "https://drive.google.com/uc?id={}&export=download" 42 | self.__G_DRIVE_DIR_BASE_DOWNLOAD_URL = "https://drive.google.com/drive/folders/{}" 43 | self.__listener = listener 44 | self.__service = self.authorize() 45 | self.__listener = listener 46 | self._file_uploaded_bytes = 0 47 | self.uploaded_bytes = 0 48 | self.UPDATE_INTERVAL = 5 49 | self.start_time = 0 50 | self.total_time = 0 51 | self._should_update = True 52 | self.is_uploading = True 53 | self.is_cancelled = False 54 | self.status = None 55 | self.updater = None 56 | self.name = name 57 | self.update_interval = 3 58 | if not len(GFolder_ID) == 33 or not len(GFolder_ID) == 19: 59 | self.gparentid = self.getIdFromUrl(GFolder_ID) 60 | else: 61 | self.gparentid = GFolder_ID 62 | 63 | def cancel(self): 64 | self.is_cancelled = True 65 | self.is_uploading = False 66 | 67 | def speed(self): 68 | """ 69 | It calculates the average upload speed and returns it in bytes/seconds unit 70 | :return: Upload speed in bytes/second 71 | """ 72 | try: 73 | return self.uploaded_bytes / self.total_time 74 | except ZeroDivisionError: 75 | return 0 76 | 77 | @staticmethod 78 | def getIdFromUrl(link: str): 79 | if len(link) in [33, 19]: 80 | return link 81 | if "folders" in link or "file" in link: 82 | regex = r"https://drive\.google\.com/(drive)?/?u?/?\d?/?(mobile)?/?(file)?(folders)?/?d?/(?P[-\w]+)[?+]?/?(w+)?" 83 | res = re.search(regex,link) 84 | if res is None: 85 | raise IndexError("GDrive ID not found.") 86 | return res.group('id') 87 | parsed = urlparse.urlparse(link) 88 | return parse_qs(parsed.query)['id'][0] 89 | 90 | def switchServiceAccount(self): 91 | global SERVICE_ACCOUNT_INDEX 92 | service_account_count = len(os.listdir("accounts")) 93 | if SERVICE_ACCOUNT_INDEX == service_account_count - 1: 94 | SERVICE_ACCOUNT_INDEX = 0 95 | SERVICE_ACCOUNT_INDEX += 1 96 | LOGGER.info(f"Switching to {SERVICE_ACCOUNT_INDEX}.json service account") 97 | self.__service = self.authorize() 98 | 99 | @retry(wait=wait_exponential(multiplier=2, min=3, max=6), stop=stop_after_attempt(15), 100 | retry=retry_if_exception_type(HttpError), before=before_log(LOGGER, logging.DEBUG)) 101 | def __set_permission(self, drive_id): 102 | permissions = { 103 | 'role': 'reader', 104 | 'type': 'anyone', 105 | 'value': None, 106 | 'withLink': True 107 | } 108 | return self.__service.permissions().create(supportsTeamDrives=True, fileId=drive_id, 109 | body=permissions).execute() 110 | 111 | 112 | @retry(wait=wait_exponential(multiplier=2, min=3, max=6), stop=stop_after_attempt(15), 113 | retry=retry_if_exception_type(HttpError), before=before_log(LOGGER, logging.DEBUG)) 114 | def copyFile(self, file_id, dest_id, status): 115 | body = { 116 | 'parents': [dest_id] 117 | } 118 | 119 | try: 120 | res = self.__service.files().copy(supportsAllDrives=True,fileId=file_id,body=body).execute() 121 | return res 122 | except HttpError as err: 123 | if err.resp.get('content-type', '').startswith('application/json'): 124 | reason = json.loads(err.content).get('error').get('errors')[0].get('reason') 125 | if reason == 'userRateLimitExceeded' or reason == 'dailyLimitExceeded': 126 | if USE_SERVICE_ACCOUNTS: 127 | self.switchServiceAccount() 128 | LOGGER.info(f"Got: {reason}, Trying Again.") 129 | self.copyFile(file_id, dest_id, status) 130 | else: 131 | raise err 132 | 133 | def clone(self, link, status, ignoreList=[]): 134 | self.transferred_size = 0 135 | try: 136 | file_id = self.getIdFromUrl(link) 137 | except (KeyError,IndexError): 138 | msg = "Google drive ID could not be found in the provided link" 139 | return msg 140 | msg = "" 141 | LOGGER.info(f"File ID: {file_id}") 142 | try: 143 | meta = self.__service.files().get(supportsAllDrives=True, fileId=file_id, 144 | fields="name,id,mimeType,size").execute() 145 | dest_meta = self.__service.files().get(supportsAllDrives=True, fileId=self.gparentid, 146 | fields="name,id,size").execute() 147 | status.SetMainFolder(meta.get('name'), self.__G_DRIVE_DIR_BASE_DOWNLOAD_URL.format(meta.get('id'))) 148 | status.SetDestinationFolder(dest_meta.get('name'), self.__G_DRIVE_DIR_BASE_DOWNLOAD_URL.format(dest_meta.get('id'))) 149 | except Exception as e: 150 | return f"{str(e).replace('>', '').replace('<', '')}" 151 | if meta.get("mimeType") == self.__G_DRIVE_DIR_MIME_TYPE: 152 | dir_id = self.check_folder_exists(meta.get('name'), self.gparentid) 153 | if not dir_id: 154 | dir_id = self.create_directory(meta.get('name'), self.gparentid) 155 | try: 156 | self.cloneFolder(meta.get('name'), meta.get('name'), meta.get('id'), dir_id, status, ignoreList) 157 | except Exception as e: 158 | if isinstance(e, RetryError): 159 | LOGGER.info(f"Total Attempts: {e.last_attempt.attempt_number}") 160 | err = e.last_attempt.exception() 161 | else: 162 | err = str(e).replace('>', '').replace('<', '') 163 | LOGGER.error(err) 164 | return err 165 | status.set_status(True) 166 | msg += f'{meta.get("name")}' \ 167 | f' ({get_readable_file_size(self.transferred_size)})' 168 | if INDEX_URL: 169 | url = requests.utils.requote_uri(f'{INDEX_URL}/{meta.get("name")}/') 170 | msg += f' | Index URL' 171 | else: 172 | try: 173 | file = self.check_file_exists(meta.get('id'), self.gparentid) 174 | if file: 175 | status.checkFileExist(True) 176 | if not file: 177 | status.checkFileExist(False) 178 | file = self.copyFile(meta.get('id'), self.gparentid, status) 179 | except Exception as e: 180 | if isinstance(e, RetryError): 181 | LOGGER.info(f"Total Attempts: {e.last_attempt.attempt_number}") 182 | err = e.last_attempt.exception() 183 | else: 184 | err = str(e).replace('>', '').replace('<', '') 185 | LOGGER.error(err) 186 | return err 187 | msg += f'{file.get("name")}' 188 | try: 189 | msg += f' ({get_readable_file_size(int(meta.get("size")))}) ' 190 | if INDEX_URL is not None: 191 | url = requests.utils.requote_uri(f'{INDEX_URL}/{file.get("name")}') 192 | msg += f' | Index URL' 193 | except TypeError: 194 | pass 195 | return msg 196 | 197 | def cloneFolder(self, name, local_path, folder_id, parent_id, status, ignoreList=[]): 198 | page_token = None 199 | q = f"'{folder_id}' in parents" 200 | files = [] 201 | LOGGER.info(f"Syncing: {local_path}") 202 | while True: 203 | response = self.__service.files().list(supportsTeamDrives=True, 204 | includeTeamDriveItems=True, 205 | q=q, 206 | spaces='drive', 207 | fields='nextPageToken, files(id, name, mimeType,size)', 208 | pageToken=page_token).execute() 209 | for file in response.get('files', []): 210 | files.append(file) 211 | page_token = response.get('nextPageToken', None) 212 | if page_token is None: 213 | break 214 | if len(files) == 0: 215 | return parent_id 216 | for file in files: 217 | if file.get('mimeType') == self.__G_DRIVE_DIR_MIME_TYPE: 218 | file_path = os.path.join(local_path, file.get('name')) 219 | current_dir_id = self.check_folder_exists(file.get('name'), parent_id) 220 | if not current_dir_id: 221 | current_dir_id = self.create_directory(file.get('name'), parent_id) 222 | if not str(file.get('id')) in ignoreList: 223 | self.cloneFolder(file.get('name'), file_path, file.get('id'), current_dir_id, status, ignoreList) 224 | else: 225 | LOGGER.info("Ignoring FolderID from clone: " + str(file.get('id'))) 226 | else: 227 | try: 228 | if not self.check_file_exists(file.get('name'), parent_id): 229 | status.checkFileExist(False) 230 | self.copyFile(file.get('id'), parent_id, status) 231 | self.transferred_size += int(file.get('size')) 232 | status.set_name(file.get('name')) 233 | status.add_size(int(file.get('size'))) 234 | else: 235 | status.checkFileExist(True) 236 | except TypeError: 237 | pass 238 | except Exception as e: 239 | if isinstance(e, RetryError): 240 | LOGGER.info(f"Total Attempts: {e.last_attempt.attempt_number}") 241 | err = e.last_attempt.exception() 242 | else: 243 | err = e 244 | LOGGER.error(err) 245 | 246 | @retry(wait=wait_exponential(multiplier=2, min=3, max=6), stop=stop_after_attempt(15), 247 | retry=retry_if_exception_type(HttpError), before=before_log(LOGGER, logging.DEBUG)) 248 | def create_directory(self, directory_name, parent_id): 249 | file_metadata = { 250 | "name": directory_name, 251 | "mimeType": self.__G_DRIVE_DIR_MIME_TYPE 252 | } 253 | if parent_id is not None: 254 | file_metadata["parents"] = [parent_id] 255 | file = self.__service.files().create(supportsTeamDrives=True, body=file_metadata).execute() 256 | file_id = file.get("id") 257 | if not IS_TEAM_DRIVE: 258 | self.__set_permission(file_id) 259 | LOGGER.info("Created Google-Drive Folder:\nName: {}\nID: {} ".format(file.get("name"), file_id)) 260 | return file_id 261 | 262 | 263 | def authorize(self): 264 | # Get credentials 265 | credentials = None 266 | if not USE_SERVICE_ACCOUNTS: 267 | if os.path.exists(self.__G_DRIVE_TOKEN_FILE): 268 | with open(self.__G_DRIVE_TOKEN_FILE, 'rb') as f: 269 | credentials = pickle.load(f) 270 | if credentials is None or not credentials.valid: 271 | if credentials and credentials.expired and credentials.refresh_token: 272 | credentials.refresh(Request()) 273 | else: 274 | flow = InstalledAppFlow.from_client_secrets_file( 275 | 'credentials.json', self.__OAUTH_SCOPE) 276 | LOGGER.info(flow) 277 | credentials = flow.run_console(port=0) 278 | 279 | # Save the credentials for the next run 280 | with open(self.__G_DRIVE_TOKEN_FILE, 'wb') as token: 281 | pickle.dump(credentials, token) 282 | else: 283 | LOGGER.info(f"Authorizing with {SERVICE_ACCOUNT_INDEX}.json service account") 284 | credentials = service_account.Credentials.from_service_account_file( 285 | f'accounts/{SERVICE_ACCOUNT_INDEX}.json', 286 | scopes=self.__OAUTH_SCOPE) 287 | return build('drive', 'v3', credentials=credentials, cache_discovery=False) 288 | 289 | 290 | @retry(wait=wait_exponential(multiplier=2, min=3, max=6), stop=stop_after_attempt(15), 291 | retry=retry_if_exception_type(HttpError), before=before_log(LOGGER, logging.DEBUG)) 292 | def check_folder_exists(self, fileName, u_parent_id): 293 | fileName = clean_name(fileName) 294 | # Create Search Query for API request. 295 | query = f"'{u_parent_id}' in parents and (name contains '{fileName}' and trashed=false)" 296 | response = self.__service.files().list(supportsTeamDrives=True, 297 | includeTeamDriveItems=True, 298 | q=query, 299 | spaces='drive', 300 | pageSize=5, 301 | fields='files(id, name, mimeType, size)', 302 | orderBy='modifiedTime desc').execute() 303 | for file in response.get('files', []): 304 | if file.get('mimeType') == "application/vnd.google-apps.folder": # Detect Whether Current Entity is a Folder or File. 305 | driveid = file.get('id') 306 | return driveid 307 | 308 | @retry(wait=wait_exponential(multiplier=2, min=3, max=6), stop=stop_after_attempt(15), 309 | retry=retry_if_exception_type(HttpError), before=before_log(LOGGER, logging.DEBUG)) 310 | def check_file_exists(self, fileName, u_parent_id): 311 | fileName = clean_name(fileName) 312 | # Create Search Query for API request. 313 | query = f"'{u_parent_id}' in parents and (name contains '{fileName}' and trashed=false)" 314 | response = self.__service.files().list(supportsTeamDrives=True, 315 | includeTeamDriveItems=True, 316 | q=query, 317 | spaces='drive', 318 | pageSize=5, 319 | fields='files(id, name, mimeType, size)', 320 | orderBy='modifiedTime desc').execute() 321 | for file in response.get('files', []): 322 | if file.get('mimeType') != "application/vnd.google-apps.folder": 323 | # driveid = file.get('id') 324 | return file 325 | 326 | 327 | def get_readable_file_size(size_in_bytes) -> str: 328 | SIZE_UNITS = ['B', 'KB', 'MB', 'GB', 'TB', 'PB'] 329 | if size_in_bytes is None: 330 | return '0B' 331 | index = 0 332 | while size_in_bytes >= 1024: 333 | size_in_bytes /= 1024 334 | index += 1 335 | try: 336 | return f'{round(size_in_bytes, 2)}{SIZE_UNITS[index]}' 337 | except IndexError: 338 | return 'File too large' 339 | 340 | --------------------------------------------------------------------------------