├── LICENSE ├── README.md ├── requirements.txt └── ripper.py /LICENSE: -------------------------------------------------------------------------------- 1 | This is free and unencumbered software released into the public domain. 2 | 3 | Anyone is free to copy, modify, publish, use, compile, sell, or 4 | distribute this software, either in source code form or as a compiled 5 | binary, for any purpose, commercial or non-commercial, and by any 6 | means. 7 | 8 | In jurisdictions that recognize copyright laws, the author or authors 9 | of this software dedicate any and all copyright interest in the 10 | software to the public domain. We make this dedication for the benefit 11 | of the public at large and to the detriment of our heirs and 12 | successors. We intend this dedication to be an overt act of 13 | relinquishment in perpetuity of all present and future rights to this 14 | software under copyright law. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 19 | IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 | OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | OTHER DEALINGS IN THE SOFTWARE. 23 | 24 | For more information, please refer to 25 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Manual temporary workaround: 2 | https://www.reddit.com/r/DataHoarder/comments/13hjddd/riptok_tiktok_download_manual_edition/ 3 | 4 | # RipTok - currentlly not working due to changes in TikTok and unoffical API 5 | Script provided as is. Absolutely no guarantee. 6 | 7 | A TikTok ripper based on TikTokApi and YouTube-dl. Some assembly may be required. 8 | 9 | ``` 10 | positional arguments: 11 | user Target username to rip 12 | 13 | optional arguments: 14 | -h, --help show this help message and exit 15 | --download_dir DOWNLOAD_DIR 16 | Path to the directory where videos should be downloaded 17 | --skip_existing SKIP_EXISTING 18 | Skip videos which are already in the download directory 19 | --timezone TIMEZONE Override UTC with another timezone code 20 | --delay_min DELAY_MIN 21 | The minimum sleep delay between downloads (in Seconds) 22 | --delay_max DELAY_MAX 23 | The maximum sleep delay between downloads (in Seconds) 24 | 25 | Have fun! 26 | ``` 27 | 28 | Ripper by default is creating a new folder in running diectory ("_rips") and puts video files in folders by username. 29 | Follow instructions in callback when installing TikTok_Api and instruction on project sites and documenetation. 30 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | TikTokApi==3.9.5 2 | youtube-dl==2021.4.1 3 | pytz==2020.5 4 | numpy==1.19.4 5 | -------------------------------------------------------------------------------- /ripper.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import logging 3 | import os 4 | import re 5 | from datetime import datetime 6 | from time import sleep 7 | 8 | import pytz 9 | import youtube_dl 10 | from TikTokApi import TikTokApi 11 | from numpy import random 12 | 13 | logger = logging.getLogger(__name__) 14 | logger.setLevel(logging.DEBUG) 15 | logger.propagate = False 16 | 17 | 18 | def _format_timestamp_iso(tz, timestamp): 19 | return datetime.fromtimestamp(int(timestamp), tz).isoformat()[:-6].replace(":", "_") 20 | 21 | 22 | def _format_bytes(num): 23 | for unit in ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"]: 24 | if abs(num) < 1024.0: 25 | return "%3.1f%sB" % (num, unit) 26 | num /= 1024.0 27 | return "%.1f%s" % (num, "YiB",) 28 | 29 | 30 | class Ripper: 31 | def __init__(self, username, download_path, skip_downloaded, timezone_code, sleep_min, sleep_max): 32 | """Initialize a Ripper 33 | :param username: TikTok user to rip 34 | :param download_path: Path in which to create user folder and download videos 35 | :param skip_downloaded: If True, skip downloading videos whose IDs are already in the download folder on disk 36 | :param timezone_code: Formatted like UTC 37 | :param sleep_min: Minimum delay between downloads, in Seconds 38 | :param sleep_max: Maximum delay between downloads, in Seconds 39 | """ 40 | self.api = TikTokApi().get_instance() 41 | self.username = username 42 | self.download_path = download_path 43 | self.skip_downloaded = skip_downloaded 44 | self.tz = pytz.timezone(timezone_code) 45 | self.sleep_min = sleep_min 46 | self.sleep_max = sleep_max 47 | logger.info("Fetching video list of user @" + username + " with TikTokApi") 48 | self.video_count = self.api.get_user(self.username)["userInfo"]["stats"]["videoCount"] 49 | # Supposedly the by_username method is limited to around 2000, according to TikTokApi documentation 50 | if self.video_count > 1900: 51 | logger.warning("TikTokApi may encounter issues if a user has posted ~2000 videos. Video count: " + 52 | str(self.video_count)) 53 | self.videos = self.api.by_username(self.username, count=self.video_count) 54 | self.fallback_counter = 0 55 | self.ytdl_downloaderror_counter = 0 56 | self.other_error_counter = 0 57 | logger.debug("Ripper init complete") 58 | 59 | def __repr__(self): 60 | """Override str() when used on a Ripper object 61 | :return: String representation of this Ripper instance 62 | """ 63 | return "Username: " + str(self.username) + \ 64 | ", Video count in metadata: " + str(self.video_count) + \ 65 | ", Videos IDs found: " + str(len(self.videos)) 66 | 67 | @staticmethod 68 | def _download_with_ytdl(file_path, video_url): 69 | ydl_opts = {"outtmpl": file_path} 70 | with youtube_dl.YoutubeDL(ydl_opts) as ydl: 71 | ydl.download([video_url]) 72 | 73 | def _download_with_api(self, file_path, video_url): 74 | logger.debug("Downloading video with TikTokApi: " + video_url) 75 | video_bytes = self.api.get_Video_By_Url(video_url) 76 | size = len(video_bytes) 77 | if size < 1024: 78 | raise AssertionError("Only " + _format_bytes(size) + " received") 79 | logger.debug("Writing " + _format_bytes(size) + " video to file: " + file_path) 80 | with open(file_path, "wb") as f: 81 | f.write(video_bytes) 82 | 83 | @staticmethod 84 | def _format_video_url(tiktok_object): 85 | return "https://www.tiktok.com/@{}/video/{}?lang=en".format(tiktok_object["author"]["uniqueId"], 86 | tiktok_object["id"]) 87 | 88 | def _format_file_name(self, timestamp, video_id): 89 | return "{}_{}.mp4".format(_format_timestamp_iso(self.tz, timestamp), video_id) 90 | 91 | @staticmethod 92 | def _parse_file_name(file_name: str): 93 | match = re.search(r"(.+)_(.+?)\.mp4", file_name, re.IGNORECASE) 94 | if match: 95 | return {"timestamp": datetime.strptime(match.group(1), '%Y-%m-%dT%H_%M_%S'), "id": match.group(2)} 96 | else: 97 | return None 98 | 99 | def download_video(self, file_path, video_url, video_creation_time): 100 | """Download a single one of the user's videos 101 | :param file_path: path to file download location 102 | :param video_url: TikTok video URL 103 | :param video_creation_time: timestamp of video creation 104 | :return: True if success, False if failure 105 | """ 106 | logger.debug("Downloading video created at " + _format_timestamp_iso(self.tz, video_creation_time) + " from " 107 | + video_url + " to " + file_path) 108 | failed = False 109 | try: 110 | self._download_with_api(file_path, video_url) 111 | except Exception as e: 112 | logger.debug("Video download failed using TikTokApi: " + str(e)) 113 | failed = True 114 | if not os.path.isfile(file_path): 115 | failed = True 116 | logger.debug("No file was created by TikTokApi at " + file_path) 117 | elif os.stat(file_path).st_size < 1024: 118 | failed = True 119 | try: 120 | os.remove(file_path) 121 | logger.debug("Deleted malformed TikTokApi download at " + file_path) 122 | except Exception as ee: 123 | logger.error("Unable to delete malformed TikTokApi download at " + str(ee)) 124 | if failed: 125 | sleep_time = random.uniform(self.sleep_min, self.sleep_max) 126 | logger.info("Sleeping for: " + str(sleep_time) + " seconds") 127 | sleep(sleep_time) 128 | try: 129 | logger.debug("Falling back to YouTube-dl") 130 | self.fallback_counter += 1 131 | self._download_with_ytdl(file_path, video_url) 132 | if not os.path.isfile(file_path): 133 | raise AssertionError("No file was created by YouTube-dl at " + file_path) 134 | elif os.stat(file_path).st_size < 1024: 135 | try: 136 | os.remove(file_path) 137 | logger.debug("Deleted malformed YouTube-dl download at " + file_path) 138 | except Exception as ee: 139 | raise AssertionError("Malformed file was created at " + file_path + 140 | " and could not be removed: " + str(ee)) 141 | raise AssertionError("Malformed file was created at " + file_path + " and was removed") 142 | failed = False 143 | except youtube_dl.utils.DownloadError as ee: 144 | logger.error("YouTube-dl DownloadError: " + str(ee)) 145 | self.ytdl_downloaderror_counter += 1 146 | failed = True 147 | except Exception as ee: 148 | logger.error("Video download failed with YouTube-dl: " + str(ee)) 149 | self.other_error_counter += 1 150 | failed = True 151 | if not failed: 152 | try: 153 | os.utime(file_path, (video_creation_time, video_creation_time)) 154 | except Exception as e: 155 | logger.debug("Unable to set utime of " + str(video_creation_time) + " on file " + file_path + 156 | ", Error: " + str(e)) 157 | return True 158 | return False 159 | 160 | def download_all(self): 161 | """Download all of the user's videos 162 | :return: a Dict with a list of successful IDs, failed IDs, and skipped (already downloaded) IDs 163 | """ 164 | download_path = os.path.join(self.download_path, self.username) 165 | already_downloaded = [] 166 | successful_downloads = [] 167 | failed_downloads = [] 168 | if not os.path.exists(download_path): 169 | os.makedirs(download_path) 170 | elif not os.path.isdir(download_path): 171 | raise NotADirectoryError("Download path is not a directory: " + download_path) 172 | elif self.skip_downloaded: 173 | for item in os.listdir(download_path): 174 | file_path = str(os.path.join(download_path, item)) 175 | if os.path.isfile(file_path): 176 | parsed_file = self._parse_file_name(os.path.basename(file_path)) 177 | if parsed_file is not None: 178 | already_downloaded.append(parsed_file["id"]) 179 | for index, item in enumerate(self.videos): 180 | # Don't download it if the user has set that option, and the tiktok already exists on the disk 181 | if item["id"] in already_downloaded: 182 | logger.info("Already downloaded video with id: " + item["id"]) 183 | continue 184 | file_name = self._format_file_name(item["createTime"], item["id"]) 185 | file_path = os.path.join(download_path, file_name) 186 | logger.info("Downloading video: " + file_name + " (" + str(index + 1) + "/" + str(len(self.videos)) + ")") 187 | video_url = self._format_video_url(item) 188 | success = self.download_video(file_path, video_url, item["createTime"]) 189 | if success: 190 | successful_downloads.append(video_url) 191 | else: 192 | failed_downloads.append(video_url) 193 | sleep_time = random.uniform(self.sleep_min, self.sleep_max) 194 | logger.info("Sleeping for: " + str(sleep_time) + " seconds") 195 | sleep(sleep_time) 196 | logger.info("Processed all {} videos".format(self.video_count)) 197 | logger.debug("Fallback counter: " + str(self.fallback_counter)) 198 | logger.debug("YouTube-dl DownloadError counter: " + str(self.fallback_counter)) 199 | logger.debug("Other error counter: " + str(self.other_error_counter)) 200 | return {"successful_downloads": successful_downloads, 201 | "failed_downloads": failed_downloads, 202 | "skipped_downloads": already_downloaded} 203 | 204 | 205 | if __name__ == '__main__': 206 | # Define launch arguments 207 | parser = argparse.ArgumentParser( 208 | description='''A TikTok ripper based on TikTokApi and YouTube-dl''', 209 | epilog="""Have fun!""") 210 | user_arg = "user" 211 | download_dir_arg = "download_dir" 212 | skip_existing_arg = "skip_existing" 213 | timezone_arg = "timezone" 214 | delay_min_arg = "delay_min" 215 | delay_max_arg = "delay_max" 216 | parser.add_argument(user_arg, type=str, 217 | help="Target username to rip") 218 | parser.add_argument("--" + download_dir_arg, type=str, required=False, default="_rips", 219 | help="Path to the directory where videos should be downloaded") 220 | parser.add_argument("--" + skip_existing_arg, type=bool, required=False, default=True, 221 | help="Skip videos which are already in the download directory") 222 | parser.add_argument("--" + timezone_arg, type=str, required=False, default="UTC", 223 | help="Override UTC with another timezone code") 224 | parser.add_argument("--" + delay_min_arg, type=int, required=False, default=1, 225 | help="The minimum sleep delay between downloads (in Seconds)") 226 | parser.add_argument("--" + delay_max_arg, type=int, required=False, default=3, 227 | help="The maximum sleep delay between downloads (in Seconds)") 228 | logger.debug("Parsing launch arguments") 229 | args = parser.parse_args() 230 | user: str = args.__dict__[user_arg] 231 | download_dir = os.path.join(args.__dict__[download_dir_arg]) 232 | skip_existing: bool = args.__dict__[skip_existing_arg] 233 | timezone: str = args.__dict__[timezone_arg] 234 | delay_min: str = args.__dict__[delay_min_arg] 235 | delay_max: str = args.__dict__[delay_max_arg] 236 | # Configure logging to the console and to a file 237 | console_handler = logging.StreamHandler() 238 | console_formatter = logging.Formatter("%(asctime)s %(name)-12s %(levelname)-8s %(message)s") 239 | console_handler.setFormatter(console_formatter) 240 | console_handler.setLevel(logging.INFO) 241 | logger.addHandler(console_handler) 242 | if not os.path.exists("logs"): 243 | os.makedirs("logs") 244 | file_handler = logging.FileHandler( 245 | "logs/RipTok_log_" + _format_timestamp_iso(pytz.timezone(timezone), datetime.now().timestamp()) + ".log") 246 | file_formatter = logging.Formatter("%(asctime)s %(name)-12s %(levelname)-8s %(message)s") 247 | file_handler.setFormatter(file_formatter) 248 | file_handler.setLevel(logging.DEBUG) # Increased logging detail in the file 249 | logger.addHandler(file_handler) 250 | # Log all the launch arguments to aid with debugging 251 | logger.debug(user_arg + ": " + user) 252 | logger.debug(download_dir_arg + ": " + download_dir) 253 | logger.debug(skip_existing_arg + ": " + str(skip_existing)) 254 | logger.debug(timezone_arg + ": " + timezone) 255 | logger.debug(delay_min_arg + ": " + str(delay_min)) 256 | logger.debug(delay_max_arg + ": " + str(delay_max)) 257 | if user.startswith("@"): # handle @username format 258 | user = user.replace("@", "", 1) 259 | logger.debug("Stripped @ from " + user) 260 | logger.info("Starting rip of TikTok user @" + user + " to " + download_dir) 261 | rip = Ripper(user, download_dir, skip_existing, timezone, delay_min, delay_max) 262 | logger.info(str(rip)) 263 | result = rip.download_all() 264 | logger.info("Downloaded " + str(len(result["successful_downloads"])) + "/" + str(rip.video_count) + 265 | " videos. " + str(len(result["failed_downloads"])) + " failed, and " + 266 | str(len(result["skipped_downloads"])) + " were already downloaded.") 267 | --------------------------------------------------------------------------------