├── copier.py ├── generate_drive_token.py ├── helper.py ├── readme.md └── requirements.txt /copier.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from helper import GoogleDriveHelper 3 | 4 | def cloneNode(link): 5 | if 'https://drive.google.com/' in link: 6 | print(f"Cloning: {link}") 7 | gd = GoogleDriveHelper() 8 | result = gd.clone(link) 9 | print(result) 10 | else: 11 | print("Provide G-Drive Shareable Link to Clone.") 12 | 13 | cloneNode(sys.argv[1]) -------------------------------------------------------------------------------- /generate_drive_token.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | import os 3 | from google_auth_oauthlib.flow import InstalledAppFlow 4 | from google.auth.transport.requests import Request 5 | 6 | credentials = None 7 | __G_DRIVE_TOKEN_FILE = "token.pickle" 8 | __OAUTH_SCOPE = ["https://www.googleapis.com/auth/drive"] 9 | if os.path.exists(__G_DRIVE_TOKEN_FILE): 10 | with open(__G_DRIVE_TOKEN_FILE, 'rb') as f: 11 | credentials = pickle.load(f) 12 | if credentials is None or not credentials.valid: 13 | if credentials and credentials.expired and credentials.refresh_token: 14 | credentials.refresh(Request()) 15 | else: 16 | flow = InstalledAppFlow.from_client_secrets_file( 17 | 'credentials.json', __OAUTH_SCOPE) 18 | credentials = flow.run_console(port=0) 19 | 20 | # Save the credentials for the next run 21 | with open(__G_DRIVE_TOKEN_FILE, 'wb') as token: 22 | pickle.dump(credentials, token) -------------------------------------------------------------------------------- /helper.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pickle 3 | import urllib.parse as urlparse 4 | from urllib.parse import parse_qs 5 | 6 | import re 7 | import json 8 | import requests 9 | import logging 10 | import sys 11 | 12 | from google.auth.transport.requests import Request 13 | from google.oauth2 import service_account 14 | from google_auth_oauthlib.flow import InstalledAppFlow 15 | from googleapiclient.discovery import build 16 | from googleapiclient.errors import HttpError 17 | from googleapiclient.http import MediaFileUpload 18 | from tenacity import * 19 | 20 | 21 | file_handler = logging.FileHandler(filename='tmp.log') 22 | stdout_handler = logging.StreamHandler(sys.stdout) 23 | handlers = [file_handler, stdout_handler] 24 | 25 | logging.basicConfig( 26 | level=logging.INFO, 27 | format='[%(asctime)s] {%(filename)s:%(lineno)d} %(levelname)s - %(message)s', 28 | handlers=handlers 29 | ) 30 | LOGGER = logging.getLogger(__name__) 31 | 32 | parent_id = "" #folder id of the destination , where the file is to be copied 33 | IS_TEAM_DRIVE = #True or False 34 | SIZE_UNITS = ['B', 'KB', 'MB', 'GB', 'TB', 'PB'] 35 | def get_readable_file_size(size_in_bytes) -> str: 36 | if size_in_bytes is None: 37 | return '0B' 38 | index = 0 39 | while size_in_bytes >= 1024: 40 | size_in_bytes /= 1024 41 | index += 1 42 | try: 43 | return f'{round(size_in_bytes, 2)}{SIZE_UNITS[index]}' 44 | except IndexError: 45 | return 'File too large' 46 | 47 | class GoogleDriveHelper: 48 | def __init__(self, name=None, listener=None): 49 | self.__G_DRIVE_TOKEN_FILE = os.path.join(os.getcwd(),'token.pickle') 50 | # Check https://developers.google.com/drive/scopes for all available scopes 51 | self.__OAUTH_SCOPE = ['https://www.googleapis.com/auth/drive'] 52 | self.__G_DRIVE_DIR_MIME_TYPE = "application/vnd.google-apps.folder" 53 | self.__G_DRIVE_BASE_DOWNLOAD_URL = "https://drive.google.com/uc?id={}&export=download" 54 | self.__G_DRIVE_DIR_BASE_DOWNLOAD_URL = "https://drive.google.com/drive/folders/{}" 55 | self.__service = self.authorize() 56 | 57 | @staticmethod 58 | def getIdFromUrl(link: str): 59 | if "folders" in link or "file" in link: 60 | regex = r"https://drive\.google\.com/(drive)?/?u?/?\d?/?(mobile)?/?(file)?(folders)?/?d?/([-\w]+)[?+]?/?(w+)?" 61 | res = re.search(regex,link) 62 | if res is None: 63 | raise IndexError("GDrive ID not found.") 64 | return res.group(5) 65 | parsed = urlparse.urlparse(link) 66 | return parse_qs(parsed.query)['id'][0] 67 | 68 | 69 | @retry(wait=wait_exponential(multiplier=2, min=3, max=6), stop=stop_after_attempt(5), 70 | retry=retry_if_exception_type(HttpError), before=before_log(LOGGER, logging.DEBUG)) 71 | def __set_permission(self, drive_id): 72 | permissions = { 73 | 'role': 'reader', 74 | 'type': 'anyone', 75 | 'value': None, 76 | 'withLink': True 77 | } 78 | return self.__service.permissions().create(supportsTeamDrives=True, fileId=drive_id, 79 | body=permissions).execute() 80 | 81 | @retry(wait=wait_exponential(multiplier=2, min=3, max=6), stop=stop_after_attempt(5), 82 | retry=retry_if_exception_type(HttpError), before=before_log(LOGGER, logging.DEBUG)) 83 | def getFileMetadata(self,file_id): 84 | return self.__service.files().get(supportsAllDrives=True, fileId=file_id, 85 | fields="name,id,mimeType,size").execute() 86 | 87 | @retry(wait=wait_exponential(multiplier=2, min=3, max=6), stop=stop_after_attempt(5), 88 | retry=retry_if_exception_type(HttpError), before=before_log(LOGGER, logging.DEBUG)) 89 | def getFilesByFolderId(self,folder_id): 90 | page_token = None 91 | q = f"'{folder_id}' in parents" 92 | files = [] 93 | while True: 94 | response = self.__service.files().list(supportsTeamDrives=True, 95 | includeTeamDriveItems=True, 96 | q=q, 97 | spaces='drive', 98 | pageSize=200, 99 | fields='nextPageToken, files(id, name, mimeType,size)', 100 | pageToken=page_token).execute() 101 | for file in response.get('files', []): 102 | files.append(file) 103 | page_token = response.get('nextPageToken', None) 104 | if page_token is None: 105 | break 106 | return files 107 | 108 | def copyFile(self, file_id, dest_id): 109 | body = { 110 | 'parents': [dest_id] 111 | } 112 | 113 | try: 114 | res = self.__service.files().copy(supportsAllDrives=True,fileId=file_id,body=body).execute() 115 | return res 116 | except HttpError as err: 117 | if err.resp.get('content-type', '').startswith('application/json'): 118 | reason = json.loads(err.content).get('error').get('errors')[0].get('reason') 119 | if reason == 'userRateLimitExceeded' or reason == 'dailyLimitExceeded': 120 | print('Limit Exceeded') 121 | else: 122 | raise err 123 | 124 | def clone(self, link): 125 | self.transferred_size = 0 126 | try: 127 | file_id = self.getIdFromUrl(link) 128 | except (KeyError,IndexError): 129 | msg = "Google drive ID could not be found in the provided link" 130 | return msg 131 | msg = "" 132 | LOGGER.info(f"File ID: {file_id}") 133 | try: 134 | meta = self.getFileMetadata(file_id) 135 | if meta.get("mimeType") == self.__G_DRIVE_DIR_MIME_TYPE: 136 | dir_id = self.create_directory(meta.get('name'), parent_id) 137 | result = self.cloneFolder(meta.get('name'), meta.get('name'), meta.get('id'), dir_id) 138 | msg += f'{self.__G_DRIVE_DIR_BASE_DOWNLOAD_URL.format(dir_id)} {meta.get("name")}' \ 139 | f' ({get_readable_file_size(self.transferred_size)})' 140 | else: 141 | file = self.copyFile(meta.get('id'), parent_id) 142 | msg += f'{self.__G_DRIVE_BASE_DOWNLOAD_URL.format(file.get("id"))} {file.get("name")}' 143 | try: 144 | msg += f' ({get_readable_file_size(int(meta.get("size")))}) ' 145 | except TypeError: 146 | pass 147 | except Exception as err: 148 | if isinstance(err, RetryError): 149 | LOGGER.info(f"Total Attempts: {err.last_attempt.attempt_number}") 150 | err = err.last_attempt.exception() 151 | err = str(err).replace('>', '').replace('<', '') 152 | LOGGER.error(err) 153 | return err 154 | return msg 155 | 156 | def cloneFolder(self, name, local_path, folder_id, parent_id): 157 | LOGGER.info(f"Syncing: {local_path}") 158 | files = self.getFilesByFolderId(folder_id) 159 | new_id = None 160 | if len(files) == 0: 161 | return parent_id 162 | for file in files: 163 | if file.get('mimeType') == self.__G_DRIVE_DIR_MIME_TYPE: 164 | file_path = os.path.join(local_path, file.get('name')) 165 | current_dir_id = self.create_directory(file.get('name'), parent_id) 166 | new_id = self.cloneFolder(file.get('name'), file_path, file.get('id'), current_dir_id) 167 | else: 168 | try: 169 | self.transferred_size += int(file.get('size')) 170 | except TypeError: 171 | pass 172 | try: 173 | self.copyFile(file.get('id'), parent_id) 174 | new_id = parent_id 175 | except Exception as e: 176 | if isinstance(e, RetryError): 177 | LOGGER.info(f"Total Attempts: {e.last_attempt.attempt_number}") 178 | err = e.last_attempt.exception() 179 | else: 180 | err = e 181 | LOGGER.error(err) 182 | return new_id 183 | 184 | @retry(wait=wait_exponential(multiplier=2, min=3, max=6), stop=stop_after_attempt(5), 185 | retry=retry_if_exception_type(HttpError), before=before_log(LOGGER, logging.DEBUG)) 186 | def create_directory(self, directory_name, parent_id): 187 | file_metadata = { 188 | "name": directory_name, 189 | "mimeType": self.__G_DRIVE_DIR_MIME_TYPE 190 | } 191 | if parent_id is not None: 192 | file_metadata["parents"] = [parent_id] 193 | file = self.__service.files().create(supportsTeamDrives=True, body=file_metadata).execute() 194 | file_id = file.get("id") 195 | if not IS_TEAM_DRIVE: 196 | self.__set_permission(file_id) 197 | LOGGER.info("Created Google-Drive Folder:\nName: {}\nID: {} ".format(file.get("name"), file_id)) 198 | return file_id 199 | 200 | def authorize(self): 201 | # Get credentials 202 | credentials = None 203 | if os.path.exists(self.__G_DRIVE_TOKEN_FILE): 204 | with open(self.__G_DRIVE_TOKEN_FILE, 'rb') as f: 205 | credentials = pickle.load(f) 206 | if credentials is None or not credentials.valid: 207 | if credentials and credentials.expired and credentials.refresh_token: 208 | credentials.refresh(Request()) 209 | else: 210 | flow = InstalledAppFlow.from_client_secrets_file( 211 | 'credentials.json', self.__OAUTH_SCOPE) 212 | LOGGER.info(flow) 213 | credentials = flow.run_console(port=0) 214 | 215 | # Save the credentials for the next run 216 | with open(self.__G_DRIVE_TOKEN_FILE, 'wb') as token: 217 | pickle.dump(credentials, token) 218 | 219 | return build('drive', 'v3', credentials=credentials, cache_discovery=False) 220 | 221 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # GOOGLE DRIVE COPIER 2 | 3 | This is a simple python app that can copy files and folder from your drive or publiv link to your personal or team drive. 4 | This code is extracted from [python-aria-mirror-bot](https://github.com/lzzy12/python-aria-mirror-bot). 5 | 6 | ## Installation 7 | 8 | ### Requirements 9 | - Python3 installed 10 | 11 | ### Clone the repo and install required modules 12 | ``` 13 | git clone https://github.com/adarshadhungel/drive-copier.git 14 | cd drive-copier 15 | pip install -r requirements.txt 16 | ``` 17 | ### Getting Credentials.json for authenicating to drive 18 | - Visit the [Google Cloud Console](https://console.developers.google.com/apis/credentials) 19 | - Go to the OAuth Consent tab, fill it, and save. 20 | - Go to the Credentials tab and click Create Credentials -> OAuth Client ID 21 | - Choose Other and Create. 22 | - Use the download button to download your credentials. 23 | - Move that file to the root of drive-copier, and rename it to credentials.json 24 | - Visit [Google API page](https://console.developers.google.com/apis/library) 25 | - Search for Drive and enable it if it is disabled 26 | - Finally generate token file 27 | ``` 28 | python3 generate_drive_token.py 29 | ``` 30 | 31 | ### Configuring the destination path 32 | - Open file helper.py 33 | - On line 23 set up parent_id with folder id of destination folder like 34 | ``` 35 | parent_id = "1ndsjbfdbjh3332" 36 | ``` 37 | - Folder id is the string after https://drive.google.com/drive/folders/ in the drive link. 38 | ![Imgur Image](https://i.imgur.com/rs2yDyX.jpg) 39 | - This is the folder where the copied files are stored. 40 | - Set the value for line 24 to **True** or **False** for your case. Do not put quotation mark around these values 41 | 42 | ### Running the code 43 | Just type 44 | ``` 45 | python3 copier.py 46 | ``` 47 | It will copy the content of the link to parent_id provided above and gives the link of the copy in output 48 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | requests 2 | tenacity 3 | google-api-python-client 4 | google-auth-httplib2 5 | google-auth-oauthlib --------------------------------------------------------------------------------