23 |
24 |
25 |
27 |
28 |
29 |
30 |
40 |
41 |
42 |
43 |
59 |
60 |
61 |
62 |
75 |
76 |
77 |
78 |
79 |
80 |
93 |
94 |
95 |
96 |
97 |
98 |
104 |
105 |
106 |
107 |
108 | TubeTube
26 |
31 |
33 |
38 |
39 |
44 |
58 |
45 |
57 |
46 |
51 |
47 |
48 |
49 |
50 |
52 |
55 |
56 |
63 |
74 |
64 |
67 |
68 |
69 |
72 |
73 | | 82 | 83 | | 84 |# | 85 |Title | 86 |Status | 87 |Progress | 88 |
|---|
109 |
123 |
124 |
125 |
143 |
144 |
145 |
146 |
147 |
148 |
149 |
--------------------------------------------------------------------------------
/tubetube/tubetube.py:
--------------------------------------------------------------------------------
1 | import logging
2 | import threading
3 | from flask import Flask, render_template
4 | from flask_socketio import SocketIO
5 | from settings import Settings, Config
6 | from yt_downloader import DownloadManager
7 |
8 |
9 | logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
10 |
11 |
12 | class WebApp(Settings, DownloadManager):
13 | def __init__(self):
14 | Settings.__init__(self)
15 | DownloadManager.__init__(self)
16 | self.app = Flask(__name__)
17 | self.app.secret_key = Config.SECRET_KEY
18 | self.socketio = SocketIO(self.app, cors_allowed_origins=Config.SOCKETIO_CORS_ALLOWED_ORIGINS)
19 |
20 | @self.app.route("/")
21 | def handle_index():
22 | return render_template("index.html")
23 |
24 | @self.socketio.on("connect")
25 | def handle_connect():
26 | threading.Thread(target=self.client_connect, daemon=True).start()
27 |
28 | @self.socketio.on("download")
29 | def handle_download(item_info):
30 | threading.Thread(target=self.download_stuff, daemon=True, args=(item_info,)).start()
31 |
32 | @self.socketio.on("remove_items")
33 | def handle_remove_items(item_ids):
34 | threading.Thread(target=self.remove_items, args=(item_ids,), daemon=True).start()
35 |
36 | @self.socketio.on("cancel_items")
37 | def handle_cancel_items(item_ids):
38 | threading.Thread(target=self.cancel_items, args=(item_ids,), daemon=True).start()
39 |
40 | def client_connect(self):
41 | self.socketio.emit(
42 | "update_folder_locations",
43 | {"audio": self.audio_locations, "video": self.video_locations},
44 | )
45 | self.socketio.emit("update_download_list", self.all_items)
46 |
47 | def download_stuff(self, item_info):
48 | folder_name = item_info.get("folder_name")
49 | if folder_name not in self.folder_locations:
50 | logging.warning(f"Invalid folder selected")
51 | return
52 |
53 | download_settings = self.folder_locations.get(folder_name, {})
54 | item_info["download_settings"] = download_settings
55 | self.add_to_queue(item_info)
56 |
57 | def run_app(self):
58 | self.socketio.run(self.app, host="0.0.0.0", port=5000)
59 |
60 | def get_app(self):
61 | return self.app
62 |
63 |
64 | web_app = WebApp()
65 | if __name__ == "__main__":
66 | web_app.run_app()
67 | else:
68 | app = web_app.get_app()
69 |
--------------------------------------------------------------------------------
/tubetube/yt_downloader.py:
--------------------------------------------------------------------------------
1 | import re
2 | import os
3 | import queue
4 | import logging
5 | import platform
6 | import threading
7 | import random
8 | import yt_dlp
9 | from settings import DownloadCancelledException
10 | import helpers
11 |
12 |
13 | class DownloadManager:
14 | def __init__(self):
15 | self.download_queue = queue.Queue()
16 | self.all_items = {}
17 | self.lock = threading.Lock()
18 | self.stop_signals = {}
19 |
20 | os_system = platform.system()
21 | logging.info(f"OS: {os_system}")
22 |
23 | self.ffmpeg_location = "D:\\" if os_system == "Windows" else "/usr/bin/ffmpeg"
24 | logging.info(f"FFmpeg location set to: {self.ffmpeg_location}")
25 |
26 | self.verbose_ytdlp = bool(os.getenv("VERBOSE_LOGS", "False").lower() == "true")
27 | logging.info(f"Verbose logging for yt-dlp set to: {self.verbose_ytdlp}")
28 |
29 | self.trim_metadata = bool(os.getenv("TRIM_METADATA", "False").lower() == "true")
30 | logging.info(f"Trim Metadata set to: {self.trim_metadata}")
31 |
32 | self.preferred_language = os.getenv("PREFERRED_LANGUAGE", "en")
33 | logging.info(f"Preferred Audio Language: {self.preferred_language}")
34 |
35 | self.preferred_audio_codec = os.getenv("PREFERRED_AUDIO_CODEC", "aac")
36 | logging.info(f"Preferred Audio Codec: {self.preferred_audio_codec}")
37 |
38 | self.preferred_video_codec = os.getenv("PREFERRED_VIDEO_CODEC", "vp9")
39 | logging.info(f"Preferred Video Codec: {self.preferred_video_codec}")
40 |
41 | self.preferred_video_ext = os.getenv("PREFERRED_VIDEO_EXT", "mp4")
42 | logging.info(f"Preferred Video Ext: {self.preferred_video_ext}")
43 |
44 | self.embed_subs = bool(os.getenv("EMBED_SUBS", "False").lower() == "true")
45 | logging.info(f"Embed Subtitles: {self.embed_subs}")
46 |
47 | self.write_subs = bool(os.getenv("WRITE_SUBS", "False").lower() == "true")
48 | logging.info(f"Write Subtitles: {self.write_subs}")
49 |
50 | self.allow_auto_subs = bool(os.getenv("ALLOW_AUTO_SUBS", "True").lower() == "true")
51 | logging.info(f"Automatic Subtitles Enabled: {self.allow_auto_subs}")
52 |
53 | self.subtitle_format = os.getenv("SUBTITLE_FORMAT", "vtt")
54 | logging.info(f"Subtitle Format: {self.subtitle_format}")
55 |
56 | subtitle_langs_env = os.getenv("SUBTITLE_LANGUAGES", "en")
57 | self.subtitle_languages = [lang.strip() for lang in subtitle_langs_env.split(",") if lang.strip()]
58 | logging.info(f"Subtitle Languages: {self.subtitle_languages}")
59 |
60 | self.subtitle_config = {
61 | "subtitlesformat": "best",
62 | "subtitleslangs": self.subtitle_languages,
63 | "writeautomaticsub": self.allow_auto_subs,
64 | "writesubtitles": self.write_subs,
65 | }
66 | self.subtitle_pps = []
67 | if self.write_subs:
68 | self.subtitle_pps.append({"key": "FFmpegSubtitlesConvertor", "format": self.subtitle_format, "when": "before_dl"})
69 | if self.embed_subs:
70 | self.subtitle_pps.append({"key": "FFmpegEmbedSubtitle", "already_have_subtitle": self.write_subs})
71 |
72 | self.thread_count = int(os.getenv("THREAD_COUNT", "4"))
73 | logging.info(f"Thread Count: {self.thread_count}")
74 |
75 | for i in range(self.thread_count):
76 | worker = threading.Thread(target=self._process_queue, daemon=True, name=f"Worker-{i}")
77 | worker.start()
78 | logging.info(f"Started thread: {worker.name}")
79 |
80 | parsing_opts = {
81 | "quiet": True,
82 | "no_color": True,
83 | "extract_flat": True,
84 | "ignore_no_formats_error": True,
85 | "force_generic_extractor": False,
86 | "cachedir": "/temp/cache",
87 | "noprogress": True,
88 | "no_warnings": True,
89 | }
90 | self.ydl_for_parsing = yt_dlp.YoutubeDL(parsing_opts)
91 |
92 | self.temp_folder = "/temp"
93 | self.cleanup_temp_folder()
94 |
95 | def cleanup_temp_folder(self):
96 | try:
97 | removable_extensions = (".tmp", ".part", ".webp", ".ytdl", ".png", f".{self.subtitle_format}")
98 | for file_name in os.listdir(self.temp_folder):
99 | file_path = os.path.join(self.temp_folder, file_name)
100 | if os.path.isfile(file_path) and file_name.endswith(removable_extensions):
101 | os.remove(file_path)
102 | logging.info(f"Deleted file: {file_path}")
103 |
104 | except Exception as e:
105 | logging.error(f"Error cleaning up temporary folder: {e}")
106 |
107 | def add_to_queue(self, item_info):
108 | url = item_info.get("url", "")
109 | logging.info(f"Processing URL: {url}")
110 |
111 | if "&list=" in url:
112 | url = re.sub(r"&list=.*", "", url)
113 |
114 | with self.lock:
115 | parsed_identifier = helpers.parse_video_id(url)
116 | if any(item["url"] == url or item["video_identifier"] == parsed_identifier for item in self.all_items.values()):
117 | logging.info(f"URL {url} is already in the queue or being downloaded.")
118 | self.socketio.emit("toast", {"title": "Duplicate URL", "body": f"The video '{url}' is already in the queue or being processed."})
119 | return
120 |
121 | try:
122 | yt_info_dict = self.ydl_for_parsing.extract_info(url, download=False)
123 | logging.info(f"Extracted info for {yt_info_dict.get('title', 'unknown')}")
124 |
125 | except Exception as e:
126 | logging.error(f"Error extracting info: {e}")
127 | logging.error(f"Nothing Added to Queue")
128 | self.socketio.emit("toast", {"title": "Failed to add item to the queue.", "body": f"Please check the URL.\n\n {str(e)}"})
129 | return
130 |
131 | if "entries" in yt_info_dict:
132 | playlist_name = re.sub(r'[<>:"/\\|?*]', "-", yt_info_dict.get("title"))
133 | item_info["folder_name"] = f'{item_info.get("folder_name")}/{playlist_name}'
134 | logging.info(f"Adding playlist: {playlist_name} to queue")
135 | for entry in yt_info_dict["entries"]:
136 | self._enqueue_item(entry, item_info)
137 | else:
138 | self._enqueue_item(yt_info_dict, item_info)
139 |
140 | def _enqueue_item(self, yt_info_dict, item_info):
141 | try:
142 | download_id = max(self.all_items.keys(), default=-1) + 1
143 | url = yt_info_dict.get("webpage_url", yt_info_dict.get("url"))
144 | item = {
145 | "video_identifier": yt_info_dict.get("id"),
146 | "id": download_id,
147 | "title": yt_info_dict.get("title"),
148 | "url": url,
149 | "status": "Pending",
150 | "progress": "0%",
151 | "folder_name": item_info.get("folder_name"),
152 | "download_settings": item_info.get("download_settings"),
153 | "audio_only": item_info.get("audio_only"),
154 | "skipped": False,
155 | }
156 | with self.lock:
157 | self.all_items[download_id] = item
158 | self.stop_signals[download_id] = threading.Event()
159 | self.download_queue.put(download_id)
160 | logging.info(f'Queued item: {item["title"]} with ID: {download_id}')
161 | self.socketio.emit("update_download_list", self.all_items)
162 |
163 | except Exception as e:
164 | logging.error(f"Error enqueuing item: {e}")
165 | logging.warning(f'Failed to add: {yt_info_dict.get("title")} to the queue.')
166 |
167 | else:
168 | logging.info(f'Added: {yt_info_dict.get("title")} to queue.')
169 |
170 | def _process_queue(self):
171 | while True:
172 | try:
173 | download_id = self.download_queue.get()
174 | logging.info(f"Processing download ID: {download_id} in thread {threading.current_thread().name}")
175 |
176 | if self.all_items[download_id]["skipped"]:
177 | self.all_items[download_id]["status"] = "Cancelled"
178 | logging.info(f"Item {download_id} marked as skipped.")
179 | self.socketio.emit("update_download_item", {"item": self.all_items[download_id]})
180 |
181 | else:
182 | self._download_item(download_id)
183 |
184 | except Exception as e:
185 | logging.error(f"Processing error for ID {download_id}: {e}")
186 |
187 | finally:
188 | self.download_queue.task_done()
189 | if self.download_queue.empty():
190 | logging.info(f"Queue is empty.")
191 |
192 | def _download_item(self, download_id):
193 | item = self.all_items[download_id]
194 | item["status"] = "In Progress"
195 | self.socketio.emit("update_download_item", {"item": item})
196 |
197 | download_settings = item.get("download_settings")
198 | folder_name = item.get("folder_name")
199 |
200 | video_format_id = download_settings.get("video_format_id", {})
201 | audio_format_id = download_settings.get("audio_format_id", {})
202 |
203 | if item.get("audio_only"):
204 | download_format = f"{audio_format_id}/bestaudio/best"
205 | else:
206 | download_format = f"{video_format_id}+{audio_format_id}/bestvideo+bestaudio/best"
207 |
208 | item_title = re.sub(r'[<>:"/\\|?*]', "-", item.get("title"))
209 | final_path = f"/data/{folder_name}"
210 |
211 | ydl_opts = {
212 | "ignore_no_formats_error": True,
213 | "noplaylist": True,
214 | "outtmpl": f"{item_title}.%(ext)s",
215 | "progress_hooks": [lambda d: self._progress_hook(d, download_id)],
216 | "ffmpeg_location": self.ffmpeg_location,
217 | "writethumbnail": True,
218 | "quiet": not self.verbose_ytdlp,
219 | "extract_flat": True,
220 | "format": download_format,
221 | "updatetime": False,
222 | "live_from_start": True,
223 | "extractor_args": {"youtubetab": {"skip": ["authcheck"]}},
224 | "paths": {"home": final_path, "temp": self.temp_folder},
225 | "no_overwrites": True,
226 | "verbose": self.verbose_ytdlp,
227 | "no_mtime": True,
228 | "format_sort": [f"lang:{self.preferred_language}", f"acodec:{self.preferred_audio_codec}", "quality", "size", f"vcodec:{self.preferred_video_codec}", f"vext:{self.preferred_video_ext}"],
229 | }
230 |
231 | post_processors = [
232 | {"key": "SponsorBlock", "categories": ["sponsor"]},
233 | {"key": "ModifyChapters", "remove_sponsor_segments": ["sponsor"]},
234 | ]
235 |
236 | if item.get("audio_only"):
237 | audio_ext = download_settings.get("audio_ext", "m4a")
238 | post_processors.extend([{"key": "FFmpegExtractAudio", "preferredcodec": audio_ext, "preferredquality": "0"}])
239 |
240 | post_processors.append({"key": "FFmpegThumbnailsConvertor", "format": "png", "when": "before_dl"})
241 | post_processors.append({"key": "EmbedThumbnail"})
242 | post_processors.append({"key": "FFmpegMetadata"})
243 |
244 | if not item.get("audio_only"):
245 | ydl_opts["merge_output_format"] = "mp4"
246 |
247 | if self.cookies_file:
248 | ydl_opts["cookiefile"] = self.cookies_file
249 |
250 | if self.write_subs or self.embed_subs:
251 | ydl_opts.update(self.subtitle_config)
252 | post_processors.extend(self.subtitle_pps)
253 |
254 | ydl_opts["postprocessors"] = post_processors
255 |
256 | try:
257 | logging.info(f'Starting {threading.current_thread().name} Download: {item.get("title")}')
258 | ydl = yt_dlp.YoutubeDL(ydl_opts)
259 | if self.trim_metadata:
260 | ydl.add_post_processor(helpers.TrimDescriptionPP(), when="before_dl")
261 | result = ydl.download([item["url"]])
262 | item["progress"] = "Done" if result == 0 else "Incomplete"
263 | item["status"] = "Complete"
264 | logging.info(f'Finished {threading.current_thread().name} Download: {item.get("title")}')
265 |
266 | except DownloadCancelledException:
267 | item["status"] = "Cancelled"
268 | logging.info(f'Download cancelled: {item.get("title")}')
269 |
270 | except Exception as e:
271 | item["status"] = f"Failed: {type(e).__name__}"
272 | item["progress"] = "Error"
273 | logging.error(f'Error downloading: {item.get("title")} - {str(e)}')
274 |
275 | finally:
276 | self.socketio.emit("update_download_item", {"item": item})
277 | ydl.close()
278 |
279 | def _progress_hook(self, d, download_id):
280 | if self.stop_signals[download_id].is_set():
281 | raise DownloadCancelledException("Cancelled")
282 |
283 | if d["status"] == "downloading":
284 | if random.randint(1, 10) != 1:
285 | return
286 |
287 | with self.lock:
288 | item = self.all_items[download_id]
289 | live = d.get("info_dict", {}).get("is_live", False)
290 | if live:
291 | fragment_index_str = d.get("fragment_index", 1)
292 | elapsed_str = re.sub(r"\x1b\[[0-9;]*m", "", d.get("_elapsed_str", "")).strip()
293 | progress_message = f"Frag: {fragment_index_str} ({elapsed_str})"
294 | else:
295 | percent_str = re.sub(r"\x1b\[[0-9;]*m", "", d.get("_percent_str", "")).strip()
296 | speed_str = re.sub(r"\x1b\[[0-9;]*m", "", d.get("_speed_str", "")).strip()
297 | progress_message = f"{percent_str} at {speed_str}"
298 |
299 | item["progress"] = progress_message
300 | item["status"] = "Downloading"
301 | self.socketio.emit("update_download_item", {"item": item})
302 |
303 | elif d["status"] == "finished":
304 | with self.lock:
305 | item = self.all_items[download_id]
306 | item["progress"] = "Downloaded"
307 | item["status"] = "Processing"
308 | logging.info(f'Download finished: {item.get("title")} - processing now')
309 | self.socketio.emit("update_download_item", {"item": item})
310 |
311 | def cancel_items(self, item_ids):
312 | with self.lock:
313 | for item_id in item_ids:
314 | if item_id in self.all_items:
315 | self.all_items[item_id]["skipped"] = True
316 | self.all_items[item_id]["status"] = "Cancelling"
317 | logging.info(f"Item {item_id} marked for cancellation.")
318 | if item_id in self.stop_signals:
319 | self.stop_signals[item_id].set()
320 | self.socketio.emit("update_download_item", {"item": self.all_items[item_id]})
321 |
322 | def remove_items(self, item_ids):
323 | with self.lock:
324 | for item_id in item_ids:
325 | if item_id in self.all_items:
326 | logging.info(f"Removing item {item_id}")
327 | if item_id in self.stop_signals:
328 | self.stop_signals[item_id].set()
329 | del self.all_items[item_id]
330 | if item_id in self.stop_signals:
331 | del self.stop_signals[item_id]
332 | self.socketio.emit("remove_download_item", {"id": item_id})
333 |
--------------------------------------------------------------------------------
110 |
111 |
122 |
112 |
121 |
113 | Toast
114 | Now
115 |
116 |
117 |
118 | Message.
119 |
120 |