14 | 15 |
17 | 18 | ## Requirements 19 | 1. [Pafy](https://pythonhosted.org/Pafy/) 20 | 2. [python-telegram-bot](https://python-telegram-bot.org/) 21 | 3. urlextract 22 | 4. requests 23 | 5. hashlib 24 | 6. [psycopg2](https://www.psycopg.org/) 25 | 26 | ## Deploy the bot 27 | Create a bot on telegram using [@botfather](t.me/botfather) 28 | Create an app on [heroku](https://dashboard.heroku.com/) 29 | Create a public channel on telegram 30 | Set environment variables in heroku app 31 | - BOT_TOKEN = your bot API token 32 | - HEROKU_APP_NAME = name of your heroku app 33 | - POLLING : if you're running this on your local machine, please set POLLING = True else leave this if you're deploying on heroku 34 | - AUDIO_DB : PostgreSQL DB link for audio management 35 | - USER_DB : PostgreSQL DB link for user management 36 | - OPEN_CHANNEL_USERNAME : Universal upload channel username, bot will upload files here and forward it to the user 37 | 38 | Deploy the bot [read here](https://devcenter.heroku.com/articles/getting-started-with-python) 39 | 40 | ## Contributions 41 | 42 | 1. [Icon by Alpha Designer](https://instagram.com/alpha_designer18) 43 | 44 | ## Future plans 45 | 46 | 1. Making a desktop application that can be used by everyone on their machine by providing their bot API token 47 | -------------------------------------------------------------------------------- /bot.py: -------------------------------------------------------------------------------- 1 | from telegram import Update 2 | from telegram.ext import (CallbackContext, CommandHandler, Filters, 3 | MessageHandler, Updater) 4 | 5 | from const import (DBHANDLER, HEROKU_APP_NAME, LOGGER, OPEN_CHANNEL_USERNAME, 6 | POLLING, PORT, TOKEN) 7 | from databasehandler import add_to_db, check_in_db 8 | from helper import * 9 | from logger import log 10 | from ytadllib import (YTADL, FileDownloadError, FileSizeExceeded, 11 | UnableToDownload) 12 | 13 | OWNER_CHAT_ID = 0 14 | 15 | 16 | def start(update: Update, context: CallbackContext) -> None: 17 | """Command handler to send start message""" 18 | log(update, LOGGER) 19 | update.message.reply_text( 20 | f"Hello {update.effective_user.first_name}, I'm youtube audio downloader bot! " 21 | f"send me any youtube url and I'll download it for you!" 22 | ) 23 | 24 | 25 | def help_bot(update: Update, context: CallbackContext) -> None: 26 | """Command handler for sending help message""" 27 | log(update, LOGGER) 28 | update.message.reply_text( 29 | f"Use telegram inline bot @vid to search youtube video and send me the link\n" 30 | f"Type '@vid Video name' in msg and click on appropriate youtube video\n" 31 | ) 32 | 33 | 34 | def donate(update: Update, context: CallbackContext) -> None: 35 | """Command handler for sending help message""" 36 | log(update, LOGGER) 37 | update.message.reply_text(f"Visit @donateatamaka to donate me") 38 | 39 | 40 | def download_url(update: Update, context: CallbackContext, url: str) -> None: 41 | """Function to download and send single youtube url downloaded audio file. Sends appropriate error message to 42 | user""" 43 | log(update, LOGGER) 44 | audio = None 45 | title = "" 46 | artist = "" 47 | try: 48 | audio = YTADL(url, url_only=False) 49 | if "-" in audio.pafy_obj.title: 50 | title = audio.pafy_obj.title.split("-")[1].strip() 51 | artist = audio.pafy_obj.title.split("-")[0].strip() 52 | 53 | except ValueError: 54 | update.message.reply_text("Invalid URL") 55 | 56 | if audio: 57 | if OPEN_CHANNEL_USERNAME: 58 | db_status = check_in_db(audio.url, DBHANDLER) 59 | if db_status: 60 | try: 61 | context.bot.forward_message( 62 | chat_id=update.effective_chat.id, 63 | from_chat_id=OPEN_CHANNEL_USERNAME, 64 | message_id=db_status, 65 | ) 66 | return 67 | except Exception as e: 68 | print("Msg not found on channel") 69 | try: 70 | audio.processor_url() 71 | except FileSizeExceeded: 72 | update.message.reply_text("File size limit exceeded") 73 | return 74 | 75 | try: 76 | audio.download() 77 | except FileDownloadError: 78 | update.message.reply_text("Unable to download file") 79 | return 80 | except UnableToDownload: 81 | update.message.reply_text("Unable to download file") 82 | return 83 | 84 | if OPEN_CHANNEL_USERNAME: 85 | try: 86 | msg = context.bot.send_audio( 87 | chat_id=OPEN_CHANNEL_USERNAME, 88 | audio=audio.audio_file, 89 | title=title, 90 | thumb=audio.thumbnail, 91 | performer=artist, 92 | duration=get_sec(audio.pafy_obj.duration), 93 | timeout=60, 94 | ) 95 | context.bot.forward_message( 96 | chat_id=update.effective_chat.id, 97 | from_chat_id=OPEN_CHANNEL_USERNAME, 98 | message_id=msg.message_id, 99 | ) 100 | try: 101 | add_to_db(audio.url, msg.message_id, DBHANDLER) 102 | except Exception as e: 103 | print(e) 104 | print("DB not working") 105 | except Exception as e: 106 | print(e) 107 | print("Upload to open channel failed") 108 | context.bot.send_audio( 109 | chat_id=update.message.chat_id, 110 | audio=audio.audio_file, 111 | title=title, 112 | thumb=audio.thumbnail, 113 | performer=artist, 114 | duration=get_sec(audio.pafy_obj.duration), 115 | timeout=60, 116 | ) 117 | else: 118 | context.bot.send_audio( 119 | chat_id=update.message.chat_id, 120 | audio=audio.audio_file, 121 | title=title, 122 | thumb=audio.thumbnail, 123 | performer=artist, 124 | duration=get_sec(audio.pafy_obj.duration), 125 | timeout=60, 126 | ) 127 | 128 | 129 | def extract_url_download(update: Update, context: CallbackContext) -> None: 130 | """Extract YouTube urls from the random text send to the bot and starts downloading and sending from url""" 131 | received_text = update.message.text 132 | yt_urls = get_links_from_text(received_text) 133 | yt_urls_msg = update.message.reply_text( 134 | pretty_url_string(yt_urls), disable_web_page_preview=True 135 | ) 136 | if len(yt_urls) > 0: 137 | for url in yt_urls: 138 | if "list=" in url: 139 | download_playlist_url(update, context, url) 140 | else: 141 | try: 142 | download_url(update, context, url) 143 | except FileSizeExceeded: 144 | update.message.reply_text("File size exceeded") 145 | context.bot.delete_message( 146 | message_id=yt_urls_msg.message_id, chat_id=yt_urls_msg.chat_id 147 | ) 148 | 149 | 150 | def download_playlist_url( 151 | update: Update, context: CallbackContext, pl_link: str 152 | ) -> None: 153 | """Extract YouTube urls from the playlist url send to the bot and starts downloading and sending each file""" 154 | pl_link = get_pl_link_from_url(pl_link) 155 | yt_urls = get_yt_links_from_pl(pl_link) 156 | yt_urls_msg = update.message.reply_text( 157 | pretty_url_string(yt_urls), disable_web_page_preview=True 158 | ) 159 | if len(yt_urls) > 0: 160 | for url in yt_urls: 161 | download_url(update, context, url) 162 | context.bot.delete_message( 163 | message_id=yt_urls_msg.message_id, chat_id=yt_urls_msg.chat_id 164 | ) 165 | 166 | 167 | def main() -> None: 168 | updater = None 169 | try: 170 | updater = Updater(TOKEN) 171 | except Exception: 172 | print("Token is incorrect") 173 | exit(1) 174 | # Command handlers 175 | updater.dispatcher.add_handler( 176 | CommandHandler("start", start, run_async=True)) 177 | updater.dispatcher.add_handler( 178 | CommandHandler("help", help_bot, run_async=True)) 179 | updater.dispatcher.add_handler( 180 | CommandHandler("donate", donate, run_async=True)) 181 | # Message handler 182 | updater.dispatcher.add_handler( 183 | MessageHandler( 184 | Filters.text & ~Filters.command, extract_url_download, run_async=True 185 | ) 186 | ) 187 | # Polling method to test on local machine 188 | if POLLING: 189 | updater.start_polling() 190 | else: 191 | # webhook method for heroku 192 | updater.start_webhook( 193 | listen="0.0.0.0", 194 | port=PORT, 195 | url_path=TOKEN, 196 | webhook_url="https://{}.herokuapp.com/{}".format( 197 | HEROKU_APP_NAME, TOKEN), 198 | ) 199 | updater.idle() 200 | LOGGER.close() 201 | DBHANDLER.close() 202 | 203 | 204 | if __name__ == "__main__": 205 | main() 206 | -------------------------------------------------------------------------------- /const.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from databasehandler import start_dbhandler 4 | from logger import start_logger 5 | 6 | TOKEN = os.environ.get("BOT_TOKEN", None) 7 | PORT = int(os.environ.get("PORT", "8443")) 8 | HEROKU_APP_NAME = os.environ.get("HEROKU_APP_NAME", None) 9 | PASS_HASH = os.environ.get("PASS_HASH", None) 10 | # only enable pooling for testing on local machine, Heroku will only work on webhook method 11 | POLLING = os.environ.get("POOLING", False) 12 | OPEN_CHANNEL_USERNAME = os.environ.get("OPEN_CHANNEL_USERNAME", None) 13 | # Database connection urls 14 | USER_DB = os.environ.get("USER_DB", None) 15 | AUDIO_DB = os.environ.get("AUDIO_DB", None) 16 | 17 | # start the logger 18 | LOGGER = start_logger(user_db=USER_DB) 19 | DBHANDLER = start_dbhandler(audio_db=AUDIO_DB) 20 | -------------------------------------------------------------------------------- /databasehandler.py: -------------------------------------------------------------------------------- 1 | import urllib.parse as urlparse 2 | 3 | import psycopg2 4 | 5 | STATUS = False 6 | 7 | 8 | def start_dbhandler(audio_db: str): 9 | global STATUS 10 | try: 11 | result = urlparse.urlparse(audio_db) 12 | username = result.username 13 | password = result.password 14 | database = result.path[1:] 15 | hostname = result.hostname 16 | port = result.port 17 | except Exception as e: 18 | print(e) 19 | print("DB url is not valid") 20 | STATUS = False 21 | return None 22 | try: 23 | conn = psycopg2.connect( 24 | database=database, 25 | user=username, 26 | password=password, 27 | host=hostname, 28 | port=port, 29 | ) 30 | STATUS = True 31 | except Exception as e: 32 | print(e) 33 | print("Connection establishment failed") 34 | STATUS = False 35 | return None 36 | if conn: 37 | cur = conn.cursor() 38 | try: 39 | cur.execute( 40 | "CREATE TABLE IF NOT EXISTS Audios(" 41 | "yt_link TEXT PRIMARY KEY," 42 | "msg_id INTEGER)" 43 | ) 44 | except Exception as e: 45 | print(e) 46 | print("Table creation failed") 47 | STATUS = False 48 | return None 49 | conn.commit() 50 | return conn 51 | return False 52 | 53 | 54 | def check_in_db(url, conn) -> int or None: 55 | if STATUS: 56 | cur = conn.cursor() 57 | row = [] 58 | try: 59 | cur.execute( 60 | "SELECT msg_id from Audios where yt_link = {}".format( 61 | "'" + url + "'") 62 | ) 63 | row = cur.fetchall() 64 | except Exception as e: 65 | print(e) 66 | print("Can't run find command!") 67 | 68 | if len(row) > 0: 69 | return int(row[0][0]) 70 | else: 71 | return None 72 | return None 73 | 74 | 75 | def add_to_db(url: str, msg_id: int, conn): 76 | if STATUS: 77 | cur = conn.cursor() 78 | cur.execute( 79 | "INSERT INTO Audios(yt_link,msg_id)" 80 | "VALUES ({},{}) ON CONFLICT UPDATE msg_id = {}".format( 81 | "'" + url + "'", msg_id, msg_id 82 | ) 83 | ) 84 | conn.commit() 85 | else: 86 | print("DB is not active cannot add the entry!") 87 | -------------------------------------------------------------------------------- /helper.py: -------------------------------------------------------------------------------- 1 | import re 2 | from urllib import error, request 3 | from urllib.parse import parse_qs, urlparse 4 | 5 | import requests 6 | from urlextract import URLExtract 7 | 8 | 9 | def get_vid_id(url: str) -> str: 10 | query = urlparse(url) 11 | if query.hostname == "youtu.be": 12 | return query.path[1:] 13 | if query.hostname in ("www.youtube.com", "youtube.com"): 14 | if query.path == "/watch": 15 | p = parse_qs(query.query) 16 | return p["v"][0] 17 | if query.path[:7] == "/embed/": 18 | return query.path.split("/")[2] 19 | if query.path[:3] == "/v/": 20 | return query.path.split("/")[2] 21 | return "" 22 | 23 | 24 | def is_yt_url(video_id: str) -> bool: 25 | checker_url = "https://www.youtube.com/oembed?url=" 26 | video_url = checker_url + video_id 27 | req = requests.get(video_url) 28 | return req.status_code == 200 29 | 30 | 31 | def is_pl_url(url: str): 32 | if "list=" in url and "youtu" in url.lower(): 33 | return True 34 | return False 35 | 36 | 37 | def get_links_from_text(text: str) -> list: 38 | urls = URLExtract().find_urls(text) 39 | yt_urls = [] 40 | for url in urls: 41 | if is_yt_url(url) or is_pl_url(url): 42 | yt_urls.append(url) 43 | return yt_urls 44 | 45 | 46 | def pretty_url_string(urls: list) -> str: 47 | pretty_url = "" 48 | if len(urls) == 0: 49 | return "No valid url found" 50 | elif len(urls) > 1: 51 | pretty_url += f"Received {len(urls)} links\n" 52 | i = 1 53 | for url in urls: 54 | pretty_url += f"{i}. {url}\n" 55 | i += 1 56 | pretty_url += "Downloading ..." 57 | 58 | return pretty_url[:4096] 59 | 60 | 61 | def is_yt_playlist(url: str) -> bool: 62 | return url.startswith("https://www.youtube.com/playlist?list=") 63 | 64 | 65 | def get_pl_link_from_url(url: str) -> str: 66 | pl_id = "" 67 | if "watch?v=" in url and "&" in url: 68 | idx = url.find("list=") 69 | pl_id = url[idx + 5:] 70 | elif "list=" in url: 71 | eq_idx = url.index("=") + 1 72 | pl_id = url[eq_idx:] 73 | if "&" in url: 74 | amp = url.index("&") 75 | pl_id = url[eq_idx:amp] 76 | else: 77 | return "" 78 | if "&" in pl_id: 79 | idx = pl_id.find("&") 80 | pl_id = pl_id[:idx] 81 | return "https://www.youtube.com/playlist?list=" + pl_id 82 | 83 | 84 | def get_yt_links_from_pl(url: str) -> list: 85 | page = None 86 | urls = [] 87 | try: 88 | page = request.urlopen(url).read() 89 | page = str(page) 90 | except error.URLError as e: 91 | print(e.reason) 92 | idx = url.find("list=") 93 | playlist_id = url[idx + 5:] 94 | if page: 95 | vid_url_pat = re.compile(r"watch\?v=\S+?list=" + playlist_id) 96 | vid_url_matches = list(set(re.findall(vid_url_pat, page))) 97 | if vid_url_matches: 98 | for vid_url in vid_url_matches: 99 | urls.append("https://www.youtube.com/" + vid_url[:19]) 100 | return urls 101 | 102 | 103 | def get_sec(time_str) -> int: 104 | """Get Seconds from time.""" 105 | h, m, s = time_str.split(":") 106 | return int(h) * 3600 + int(m) * 60 + int(s) 107 | -------------------------------------------------------------------------------- /image/demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/atamakahere-git/ytadlbot/2005001a6738f6be094dc9105e15e2afc755f9ba/image/demo.gif -------------------------------------------------------------------------------- /image/icon.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/atamakahere-git/ytadlbot/2005001a6738f6be094dc9105e15e2afc755f9ba/image/icon.jpg -------------------------------------------------------------------------------- /image/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/atamakahere-git/ytadlbot/2005001a6738f6be094dc9105e15e2afc755f9ba/image/icon.png -------------------------------------------------------------------------------- /logger.py: -------------------------------------------------------------------------------- 1 | import urllib.parse as urlparse 2 | 3 | import psycopg2 4 | from telegram import Update 5 | 6 | STATUS = False 7 | SQL_BASE_CMD = """ INSERT INTO Users(chat_id,first_name,last_name,username,usage) 8 | VALUES({},{},{},{},{}) """ 9 | SQL_UPDATE_CMD = """UPDATE Users 10 | SET usage = usage + 1 11 | WHERE chat_id = {};""" 12 | 13 | 14 | def start_logger(user_db: str): 15 | global STATUS 16 | result = urlparse.urlparse(user_db) 17 | username = result.username 18 | password = result.password 19 | database = result.path[1:] 20 | hostname = result.hostname 21 | port = result.port 22 | try: 23 | conn = psycopg2.connect( 24 | database=database, 25 | user=username, 26 | password=password, 27 | host=hostname, 28 | port=port, 29 | ) 30 | except Exception as e: 31 | print(e) 32 | print("Connection establishment failed") 33 | STATUS = False 34 | return None 35 | if conn: 36 | STATUS = True 37 | cur = conn.cursor() 38 | try: 39 | cur.execute( 40 | "CREATE TABLE IF NOT EXISTS Users(" 41 | "chat_id INTEGER PRIMARY KEY," 42 | "first_name TEXT," 43 | "last_name TEXT," 44 | "username TEXT," 45 | "usage INTEGER DEFAULT 0)" 46 | ) 47 | except Exception as e: 48 | print(e) 49 | print("Table creation failed") 50 | STATUS = False 51 | return None 52 | conn.commit() 53 | return conn 54 | return None 55 | 56 | 57 | def log(update: Update, conn): 58 | global STATUS 59 | if not STATUS: 60 | print("DB is not connected, cannot log!") 61 | return 62 | global SQL_BASE_CMD, SQL_UPDATE_CMD 63 | chat_id = update.effective_chat.id 64 | first_name = update.effective_chat.first_name 65 | last_name = update.effective_chat.last_name 66 | username = update.effective_chat.username 67 | cur = conn.cursor() 68 | try: 69 | cur.execute( 70 | SQL_BASE_CMD.format( 71 | chat_id, 72 | "'" + first_name + "'", 73 | "'" + last_name + "'", 74 | "'" + username + "'", 75 | 0, 76 | ) 77 | ) 78 | except Exception: 79 | try: 80 | cur.execute(SQL_UPDATE_CMD.format(chat_id)) 81 | except Exception as e: 82 | print(e) 83 | print(f"Update failed for {chat_id}") 84 | 85 | conn.commit() 86 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | import bot 2 | from const import HEROKU_APP_NAME, POLLING, PORT, TOKEN 3 | 4 | if __name__ == "__main__": 5 | if TOKEN is None: 6 | print("Token not found") 7 | elif PORT is None and not POLLING: 8 | print("Port not found") 9 | elif HEROKU_APP_NAME is None and not POLLING: 10 | print("App name not found") 11 | else: 12 | bot.main() 13 | -------------------------------------------------------------------------------- /pafy/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = "0.5.5" 2 | __author__ = "np1" 3 | __license__ = "LGPLv3" 4 | 5 | 6 | # External api 7 | from .pafy import new 8 | from .pafy import set_api_key 9 | from .pafy import load_cache, dump_cache 10 | from .pafy import get_categoryname 11 | from .pafy import backend 12 | from .util import GdataError, call_gdata 13 | from .playlist import get_playlist, get_playlist2 14 | from .channel import get_channel 15 | -------------------------------------------------------------------------------- /pafy/backend_internal.py: -------------------------------------------------------------------------------- 1 | import os 2 | import hashlib 3 | import tempfile 4 | import json 5 | import re 6 | import sys 7 | import time 8 | import logging 9 | from xml.etree import ElementTree 10 | 11 | if sys.version_info[:2] >= (3, 0): 12 | # pylint: disable=E0611,F0401,I0011 13 | from urllib.parse import parse_qs, unquote_plus 14 | uni, pyver = str, 3 15 | 16 | else: 17 | from urllib import unquote_plus 18 | from urlparse import parse_qs 19 | uni, pyver = unicode, 2 20 | 21 | early_py_version = sys.version_info[:2] < (2, 7) 22 | 23 | from . import g 24 | from .pafy import fetch_decode, dbg, get_categoryname 25 | from .backend_shared import BasePafy, BaseStream 26 | from .jsinterp import JSInterpreter 27 | 28 | 29 | funcmap = {} 30 | 31 | 32 | class InternPafy(BasePafy): 33 | def __init__(self, *args, **kwargs): 34 | self.sm = [] 35 | self.asm = [] 36 | self.dash = [] 37 | self.js_url = None # if js_url is set then has new stream map 38 | self._dashurl = None 39 | self.age_ver = False 40 | self._formats = None 41 | self.ciphertag = None # used by Stream class in url property def 42 | super(InternPafy, self).__init__(*args, **kwargs) 43 | 44 | 45 | def _fetch_basic(self): 46 | """ Fetch basic data and streams. """ 47 | if self._have_basic: 48 | return 49 | 50 | allinfo = get_video_info(self.videoid, self.callback) 51 | 52 | if self.callback: 53 | self.callback("Fetched video info") 54 | 55 | def _get_lst(key, default="unknown", dic=allinfo): 56 | """ Dict get function, returns first index. """ 57 | retval = dic.get(key, default) 58 | return retval[0] if retval != default else default 59 | 60 | self._title = _get_lst('title') 61 | self._dashurl = _get_lst('dashmpd') 62 | self._author = _get_lst('author') 63 | self._rating = float(_get_lst('avg_rating', 0.0)) 64 | self._length = int(_get_lst('length_seconds', 0)) 65 | self._viewcount = int(_get_lst('view_count', 0)) 66 | self._thumb = unquote_plus(_get_lst('thumbnail_url', "")) 67 | self._formats = [x.split("/") for x in _get_lst('fmt_list').split(",")] 68 | self._keywords = _get_lst('keywords', "").split(',') 69 | self._bigthumb = _get_lst('iurlsd', "") 70 | self._bigthumbhd = _get_lst('iurlsdmaxres', "") 71 | self.ciphertag = _get_lst("use_cipher_signature") == "True" 72 | self.sm = _extract_smap(g.UEFSM, allinfo, True) 73 | self.asm = _extract_smap(g.AF, allinfo, True) 74 | dbg("extracted stream maps") 75 | 76 | sm_ciphertag = "s" in self.sm[0] 77 | 78 | if self.ciphertag != sm_ciphertag: 79 | dbg("ciphertag mismatch") 80 | self.ciphertag = not self.ciphertag 81 | 82 | watch_url = g.urls['watchv'] % self.videoid 83 | if self.callback: 84 | self.callback("Fetching watch page") 85 | watchinfo = fetch_decode(watch_url) # unicode 86 | dbg("Fetched watch page") 87 | if self.callback: 88 | self.callback("Fetched watch page") 89 | self.age_ver = re.search(r'player-age-gate-content">', watchinfo) is not None 90 | 91 | if self.ciphertag: 92 | dbg("Encrypted signature detected.") 93 | 94 | if not self.age_ver: 95 | smaps, js_url, mainfunc = get_js_sm(watchinfo, self.callback) 96 | funcmap[js_url] = mainfunc 97 | self.sm, self.asm = smaps 98 | self.js_url = js_url 99 | dashsig = re.search(r"/s/([\w\.]+)", self._dashurl).group(1) 100 | dbg("decrypting dash sig") 101 | goodsig = _decodesig(dashsig, js_url, self.callback) 102 | self._dashurl = re.sub(r"/s/[\w\.]+", 103 | "/signature/%s" % goodsig, self._dashurl) 104 | 105 | else: 106 | s = re.search(r"/s/([\w\.]+)", self._dashurl).group(1) 107 | s = s[2:63] + s[82] + s[64:82] + s[63] 108 | self._dashurl = re.sub(r"/s/[\w\.]+", 109 | "/signature/%s" % s, self._dashurl) 110 | 111 | if self._dashurl != 'unknown': 112 | self.dash = _extract_dash(self._dashurl) 113 | self._have_basic = 1 114 | self._process_streams() 115 | self.expiry = time.time() + g.lifespan 116 | 117 | 118 | def _fetch_gdata(self): 119 | """ Extract gdata values, fetch gdata if necessary. """ 120 | if self._have_gdata: 121 | return 122 | 123 | item = self._get_video_gdata(self.videoid)['items'][0] 124 | snippet = item['snippet'] 125 | self._published = uni(snippet['publishedAt']) 126 | self._description = uni(snippet["description"]) 127 | self._category = get_categoryname(snippet['categoryId']) 128 | # TODO: Make sure actual usename is not available through the api 129 | self._username = uni(snippet['channelTitle']) 130 | statistics = item["statistics"] 131 | self._likes = int(statistics["likeCount"]) 132 | self._dislikes = int(statistics["dislikeCount"]) 133 | self._have_gdata = 1 134 | 135 | 136 | def _process_streams(self): 137 | """ Create Stream object lists from internal stream maps. """ 138 | if not self._have_basic: 139 | self._fetch_basic() 140 | 141 | streams = [InternStream(z, self) for z in self.sm] 142 | streams = [x for x in streams if x.itag in g.itags] 143 | adpt_streams = [InternStream(z, self) for z in self.asm] 144 | adpt_streams = [x for x in adpt_streams if x.itag in g.itags] 145 | dash_streams = [InternStream(z, self) for z in self.dash] 146 | dash_streams = [x for x in dash_streams if x.itag in g.itags] 147 | audiostreams = [x for x in adpt_streams if x.bitrate] 148 | videostreams = [x for x in adpt_streams if not x.bitrate] 149 | dash_itags = [x.itag for x in dash_streams] 150 | audiostreams = [x for x in audiostreams if x.itag not in dash_itags] 151 | videostreams = [x for x in videostreams if x.itag not in dash_itags] 152 | audiostreams += [x for x in dash_streams if x.mediatype == "audio"] 153 | videostreams += [x for x in dash_streams if x.mediatype != "audio"] 154 | audiostreams = sorted(audiostreams, key=lambda x: x.rawbitrate, 155 | reverse=True) 156 | videostreams = sorted(videostreams, key=lambda x: x.dimensions, 157 | reverse=True) 158 | m4astreams = [x for x in audiostreams if x.extension == "m4a"] 159 | oggstreams = [x for x in audiostreams if x.extension == "ogg"] 160 | self._streams = streams 161 | self._audiostreams = audiostreams 162 | self._videostreams = videostreams 163 | self._m4astreams, self._oggstreams = m4astreams, oggstreams 164 | self._allstreams = streams + videostreams + audiostreams 165 | 166 | 167 | class InternStream(BaseStream): 168 | def __init__(self, sm, parent): 169 | super(InternStream, self).__init__(parent) 170 | 171 | self._itag = sm['itag'] 172 | # is_dash = "width" in sm and "height" in sm 173 | is_dash = "dash" in sm 174 | 175 | if self._itag not in g.itags: 176 | logging.warning("Unknown itag: %s", self._itag) 177 | return 178 | 179 | self._mediatype = g.itags[self.itag][2] 180 | self._threed = 'stereo3d' in sm and sm['stereo3d'] == '1' 181 | 182 | if is_dash: 183 | if sm['width'] != "None": # dash video 184 | self._resolution = "%sx%s" % (sm['width'], sm['height']) 185 | self._quality = self._resolution 186 | self._dimensions = (int(sm['width']), int(sm['height'])) 187 | 188 | else: # dash audio 189 | self._resolution = "0x0" 190 | self._dimensions = (0, 0) 191 | self._rawbitrate = int(sm['bitrate']) 192 | # self._bitrate = uni(int(sm['bitrate']) // 1024) + "k" 193 | self._bitrate = g.itags[self.itag][0] 194 | self._quality = self._bitrate 195 | 196 | self._fsize = int(sm['size'] or 0) 197 | # self._bitrate = sm['bitrate'] 198 | # self._rawbitrate = uni(int(self._bitrate) // 1024) + "k" 199 | 200 | else: # not dash 201 | self._resolution = g.itags[self.itag][0] 202 | self._dimensions = tuple(self.resolution.split("-")[0].split("x")) 203 | self._dimensions = tuple([int(x) if x.isdigit() else x for x in 204 | self._dimensions]) 205 | self._quality = self.resolution 206 | 207 | self._extension = g.itags[self.itag][1] 208 | self._title = parent.title 209 | self.encrypted = 's' in sm 210 | self._parent = parent 211 | self._filename = self.generate_filename() 212 | self._notes = g.itags[self.itag][3] 213 | self._rawurl = sm['url'] 214 | self._sig = sm['s'] if self.encrypted else sm.get("sig") 215 | self._active = False 216 | 217 | if self.mediatype == "audio" and not is_dash: 218 | self._dimensions = (0, 0) 219 | self._bitrate = self.resolution 220 | self._quality = self.bitrate 221 | self._resolution = "0x0" 222 | self._rawbitrate = int(sm["bitrate"]) 223 | 224 | @property 225 | def url(self): 226 | """ Return the url, decrypt if required. """ 227 | if not self._url: 228 | 229 | if self._parent.age_ver: 230 | 231 | if self._sig: 232 | s = self._sig 233 | self._sig = s[2:63] + s[82] + s[64:82] + s[63] 234 | 235 | elif self.encrypted: 236 | self._sig = _decodesig(self._sig, self._parent.js_url, 237 | self._parent.callback) 238 | 239 | self._url = _make_url(self._rawurl, self._sig) 240 | 241 | return self._url 242 | 243 | 244 | def parseqs(data): 245 | """ parse_qs, return unicode. """ 246 | if type(data) == uni: 247 | return parse_qs(data) 248 | 249 | elif pyver == 3: 250 | data = data.decode("utf8") 251 | data = parse_qs(data) 252 | 253 | else: 254 | data = parse_qs(data) 255 | out = {} 256 | 257 | for k, v in data.items(): 258 | k = k.decode("utf8") 259 | out[k] = [x.decode("utf8") for x in v] 260 | data = out 261 | 262 | return data 263 | 264 | 265 | def get_video_info(video_id, callback, newurl=None): 266 | """ Return info for video_id. Returns dict. """ 267 | # TODO: see if there is a way to avoid retrieving the embed page 268 | # just for this, or to use it for more. This was coppied from 269 | # youtube-dl. 270 | embed_webpage = fetch_decode(g.urls['embed']) 271 | sts = re.search(r'sts"\s*:\s*(\d+)', embed_webpage).group(1) 272 | 273 | url = g.urls['vidinfo'] % (video_id, video_id, sts) 274 | url = newurl if newurl else url 275 | info = fetch_decode(url) # bytes 276 | info = parseqs(info) # unicode dict 277 | dbg("Fetched video info%s", " (age ver)" if newurl else "") 278 | 279 | if info['status'][0] == "fail": 280 | reason = info['reason'][0] or "Bad video argument" 281 | raise IOError("Youtube says: %s [%s]" % (reason, video_id)) 282 | 283 | return info 284 | 285 | 286 | def _extract_smap(map_name, dic, zero_idx=True): 287 | """ Extract stream map, returns list of dicts. """ 288 | if map_name in dic: 289 | smap = dic.get(map_name) 290 | smap = smap[0] if zero_idx else smap 291 | smap = smap.split(",") 292 | smap = [parseqs(x) for x in smap] 293 | return [dict((k, v[0]) for k, v in x.items()) for x in smap] 294 | 295 | return [] 296 | 297 | 298 | def _extract_dash(dashurl): 299 | """ Download dash url and extract some data. """ 300 | # pylint: disable = R0914 301 | dbg("Fetching dash page") 302 | dashdata = fetch_decode(dashurl) 303 | dbg("DASH list fetched") 304 | ns = "{urn:mpeg:DASH:schema:MPD:2011}" 305 | ytns = "{http://youtube.com/yt/2012/10/10}" 306 | tree = ElementTree.fromstring(dashdata) 307 | tlist = tree.findall(".//%sRepresentation" % ns) 308 | dashmap = [] 309 | 310 | for x in tlist: 311 | baseurl = x.find("%sBaseURL" % ns) 312 | url = baseurl.text 313 | size = baseurl.get("%scontentLength" % ytns) 314 | bitrate = x.get("bandwidth") 315 | itag = uni(x.get("id")) 316 | width = uni(x.get("width")) 317 | height = uni(x.get("height")) 318 | dashmap.append(dict(bitrate=bitrate, 319 | dash=True, 320 | itag=itag, 321 | width=width, 322 | height=height, 323 | url=url, 324 | size=size)) 325 | return dashmap 326 | 327 | 328 | def _get_mainfunc_from_js(js): 329 | """ Return main signature decryption function from javascript as dict. """ 330 | dbg("Scanning js for main function.") 331 | m = re.search(r'\.sig\|\|([a-zA-Z0-9$]+)\(', js) 332 | funcname = m.group(1) 333 | dbg("Found main function: %s", funcname) 334 | jsi = JSInterpreter(js) 335 | return jsi.extract_function(funcname) 336 | 337 | 338 | def _decodesig(sig, js_url, callback): 339 | """ Return decrypted sig given an encrypted sig and js_url key. """ 340 | # lookup main function in funcmap dict 341 | mainfunction = funcmap[js_url] 342 | 343 | # fill in function argument with signature 344 | if callback: 345 | callback("Decrypting signature") 346 | solved = mainfunction([sig]) 347 | dbg("Decrypted sig = %s...", solved[:30]) 348 | if callback: 349 | callback("Decrypted signature") 350 | return solved 351 | 352 | 353 | def fetch_cached(url, callback, encoding=None, dbg_ref="", file_prefix=""): 354 | """ Fetch url - from tmpdir if already retrieved. """ 355 | tmpdir = os.path.join(tempfile.gettempdir(), "pafy") 356 | 357 | if not os.path.exists(tmpdir): 358 | os.makedirs(tmpdir) 359 | 360 | url_md5 = hashlib.md5(url.encode("utf8")).hexdigest() 361 | cached_filename = os.path.join(tmpdir, file_prefix + url_md5) 362 | 363 | if os.path.exists(cached_filename): 364 | dbg("fetched %s from cache", dbg_ref) 365 | 366 | with open(cached_filename) as f: 367 | retval = f.read() 368 | 369 | return retval 370 | 371 | else: 372 | data = fetch_decode(url, "utf8") # unicode 373 | dbg("Fetched %s", dbg_ref) 374 | if callback: 375 | callback("Fetched %s" % dbg_ref) 376 | 377 | with open(cached_filename, "w") as f: 378 | f.write(data) 379 | 380 | # prune files after write 381 | prune_files(tmpdir, file_prefix) 382 | return data 383 | 384 | 385 | def prune_files(path, prefix="", age_max=3600 * 24 * 14, count_max=4): 386 | """ Remove oldest files from path that start with prefix. 387 | 388 | remove files older than age_max, leave maximum of count_max files. 389 | """ 390 | tempfiles = [] 391 | 392 | if not os.path.isdir(path): 393 | return 394 | 395 | for f in os.listdir(path): 396 | filepath = os.path.join(path, f) 397 | 398 | if os.path.isfile(filepath) and f.startswith(prefix): 399 | age = time.time() - os.path.getmtime(filepath) 400 | 401 | if age > age_max: 402 | os.unlink(filepath) 403 | 404 | else: 405 | tempfiles.append((filepath, age)) 406 | 407 | tempfiles = sorted(tempfiles, key=lambda x: x[1], reverse=True) 408 | 409 | for f in tempfiles[:-count_max]: 410 | os.unlink(f[0]) 411 | 412 | 413 | def get_js_sm(watchinfo, callback): 414 | """ Fetch watchinfo page and extract stream map and js funcs if not known. 415 | 416 | This function is needed by videos with encrypted signatures. 417 | If the js url referred to in the watchv page is not a key in funcmap, 418 | the javascript is fetched and functions extracted. 419 | 420 | Returns streammap (list of dicts), js url (str) and funcs (dict) 421 | 422 | """ 423 | m = re.search(g.jsplayer, watchinfo) 424 | myjson = json.loads(m.group(1)) 425 | stream_info = myjson['args'] 426 | sm = _extract_smap(g.UEFSM, stream_info, False) 427 | asm = _extract_smap(g.AF, stream_info, False) 428 | js_url = myjson['assets']['js'] 429 | js_url = "https:" + js_url if js_url.startswith("//") else js_url 430 | mainfunc = funcmap.get(js_url) 431 | 432 | if not mainfunc: 433 | dbg("Fetching javascript") 434 | if callback: 435 | callback("Fetching javascript") 436 | javascript = fetch_cached(js_url, callback, encoding="utf8", 437 | dbg_ref="javascript", file_prefix="js-") 438 | mainfunc = _get_mainfunc_from_js(javascript) 439 | 440 | elif mainfunc: 441 | dbg("Using functions in memory extracted from %s", js_url) 442 | dbg("Mem contains %s js func sets", len(funcmap)) 443 | 444 | return (sm, asm), js_url, mainfunc 445 | 446 | 447 | def _make_url(raw, sig, quick=True): 448 | """ Return usable url. Set quick=False to disable ratebypass override. """ 449 | if quick and "ratebypass=" not in raw: 450 | raw += "&ratebypass=yes" 451 | 452 | if "signature=" not in raw: 453 | 454 | if sig is None: 455 | raise IOError("Error retrieving url") 456 | 457 | raw += "&signature=" + sig 458 | 459 | return raw 460 | -------------------------------------------------------------------------------- /pafy/backend_shared.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | import sys 4 | import time 5 | import logging 6 | import subprocess 7 | 8 | if sys.version_info[:2] >= (3, 0): 9 | # pylint: disable=E0611,F0401,I0011 10 | from urllib.request import urlopen, build_opener 11 | from urllib.error import HTTPError, URLError 12 | from urllib.parse import parse_qs, urlparse 13 | uni, pyver = str, 3 14 | 15 | else: 16 | from urllib2 import urlopen, build_opener, HTTPError, URLError 17 | from urlparse import parse_qs, urlparse 18 | uni, pyver = unicode, 2 19 | 20 | early_py_version = sys.version_info[:2] < (2, 7) 21 | 22 | from . import __version__, g 23 | from .pafy import call_gdata 24 | from .playlist import get_playlist2 25 | from .util import xenc 26 | 27 | dbg = logging.debug 28 | 29 | 30 | def extract_video_id(url): 31 | """ Extract the video id from a url, return video id as str. """ 32 | idregx = re.compile(r'[\w-]{11}$') 33 | url = str(url).strip() 34 | 35 | if idregx.match(url): 36 | return url # ID of video 37 | 38 | if '://' not in url: 39 | url = '//' + url 40 | parsedurl = urlparse(url) 41 | if parsedurl.netloc in ('youtube.com', 'www.youtube.com', 'm.youtube.com', 'gaming.youtube.com'): 42 | query = parse_qs(parsedurl.query) 43 | if 'v' in query and idregx.match(query['v'][0]): 44 | return query['v'][0] 45 | elif parsedurl.netloc in ('youtu.be', 'www.youtu.be'): 46 | vidid = parsedurl.path.split('/')[-1] if parsedurl.path else '' 47 | if idregx.match(vidid): 48 | return vidid 49 | 50 | err = "Need 11 character video id or the URL of the video. Got %s" 51 | raise ValueError(err % url) 52 | 53 | 54 | class BasePafy(object): 55 | 56 | """ Class to represent a YouTube video. """ 57 | 58 | def __init__(self, video_url, basic=True, gdata=False, 59 | size=False, callback=None, ydl_opts=None): 60 | """ Set initial values. """ 61 | self.version = __version__ 62 | self.videoid = extract_video_id(video_url) 63 | self.watchv_url = g.urls['watchv'] % self.videoid 64 | 65 | self.callback = callback 66 | self._have_basic = False 67 | self._have_gdata = False 68 | 69 | self._description = None 70 | self._likes = None 71 | self._dislikes = None 72 | self._category = None 73 | self._published = None 74 | self._username = None 75 | 76 | self._streams = [] 77 | self._oggstreams = [] 78 | self._m4astreams = [] 79 | self._allstreams = [] 80 | self._videostreams = [] 81 | self._audiostreams = [] 82 | 83 | self._title = None 84 | self._rating = None 85 | self._length = None 86 | self._author = None 87 | self._duration = None 88 | self._keywords = None 89 | self._bigthumb = None 90 | self._viewcount = None 91 | self._bigthumbhd = None 92 | self._bestthumb = None 93 | self._mix_pl = None 94 | self.expiry = None 95 | 96 | if basic: 97 | self._fetch_basic() 98 | 99 | if gdata: 100 | self._fetch_gdata() 101 | 102 | if size: 103 | for s in self.allstreams: 104 | # pylint: disable=W0104 105 | s.get_filesize() 106 | 107 | 108 | def _fetch_basic(self): 109 | """ Fetch basic data and streams. """ 110 | raise NotImplementedError 111 | 112 | 113 | def _fetch_gdata(self): 114 | """ Extract gdata values, fetch gdata if necessary. """ 115 | raise NotImplementedError 116 | 117 | 118 | def _get_video_gdata(self, video_id): 119 | """ Return json string containing video metadata from gdata api. """ 120 | if self.callback: 121 | self.callback("Fetching video gdata") 122 | query = {'part': 'id,snippet,statistics', 123 | 'maxResults': 1, 124 | 'id': video_id} 125 | gdata = call_gdata('videos', query) 126 | dbg("Fetched video gdata") 127 | if self.callback: 128 | self.callback("Fetched video gdata") 129 | return gdata 130 | 131 | 132 | def _process_streams(self): 133 | """ Create Stream object lists from internal stream maps. """ 134 | raise NotImplementedError 135 | 136 | 137 | def __repr__(self): 138 | """ Print video metadata. Return utf8 string. """ 139 | if self._have_basic: 140 | info = [("Title", self.title), 141 | ("Author", self.author), 142 | ("ID", self.videoid), 143 | ("Duration", self.duration), 144 | ("Rating", self.rating), 145 | ("Views", self.viewcount), 146 | ("Thumbnail", self.thumb)] 147 | 148 | nfo = "\n".join(["%s: %s" % i for i in info]) 149 | 150 | else: 151 | nfo = "Pafy object: %s [%s]" % (self.videoid, 152 | self.title[:45] + "..") 153 | 154 | return nfo.encode("utf8", "replace") if pyver == 2 else nfo 155 | 156 | @property 157 | def streams(self): 158 | """ The streams for a video. Returns list.""" 159 | if not self._streams: 160 | self._process_streams() 161 | 162 | return self._streams 163 | 164 | @property 165 | def allstreams(self): 166 | """ All stream types for a video. Returns list. """ 167 | if not self._allstreams: 168 | self._process_streams() 169 | 170 | return self._allstreams 171 | 172 | @property 173 | def audiostreams(self): 174 | """ Return a list of audio Stream objects. """ 175 | if not self._audiostreams: 176 | self._process_streams() 177 | 178 | return self._audiostreams 179 | 180 | @property 181 | def videostreams(self): 182 | """ The video streams for a video. Returns list. """ 183 | if not self._videostreams: 184 | self._process_streams() 185 | 186 | return self._videostreams 187 | 188 | @property 189 | def oggstreams(self): 190 | """ Return a list of ogg encoded Stream objects. """ 191 | if not self._oggstreams: 192 | self._process_streams() 193 | 194 | return self._oggstreams 195 | 196 | @property 197 | def m4astreams(self): 198 | """ Return a list of m4a encoded Stream objects. """ 199 | if not self._m4astreams: 200 | self._process_streams() 201 | 202 | return self._m4astreams 203 | 204 | @property 205 | def title(self): 206 | """ Return YouTube video title as a string. """ 207 | if not self._title: 208 | self._fetch_basic() 209 | 210 | return self._title 211 | 212 | @property 213 | def author(self): 214 | """ The uploader of the video. Returns str. """ 215 | if not self._author: 216 | self._fetch_basic() 217 | 218 | return self._author 219 | 220 | @property 221 | def rating(self): 222 | """ Rating for a video. Returns float. """ 223 | if not self._rating: 224 | self._fetch_basic() 225 | 226 | return self._rating 227 | 228 | @property 229 | def length(self): 230 | """ Length of a video in seconds. Returns int. """ 231 | if not self._length: 232 | self._fetch_basic() 233 | 234 | return self._length 235 | 236 | @property 237 | def viewcount(self): 238 | """ Number of views for a video. Returns int. """ 239 | if not self._viewcount: 240 | self._fetch_basic() 241 | 242 | return self._viewcount 243 | 244 | @property 245 | def bigthumb(self): 246 | """ Large thumbnail image url. Returns str. """ 247 | self._fetch_basic() 248 | return self._bigthumb 249 | 250 | @property 251 | def bigthumbhd(self): 252 | """ Extra large thumbnail image url. Returns str. """ 253 | self._fetch_basic() 254 | return self._bigthumbhd 255 | 256 | @property 257 | def thumb(self): 258 | """ Thumbnail image url. Returns str. """ 259 | return g.urls['thumb'] % self.videoid 260 | 261 | @property 262 | def duration(self): 263 | """ Duration of a video (HH:MM:SS). Returns str. """ 264 | if not self._length: 265 | self._fetch_basic() 266 | 267 | self._duration = time.strftime('%H:%M:%S', time.gmtime(self._length)) 268 | self._duration = uni(self._duration) 269 | 270 | return self._duration 271 | 272 | @property 273 | def keywords(self): 274 | """ Return keywords as list of str. """ 275 | if not self._keywords: 276 | self._fetch_gdata() 277 | 278 | return self._keywords 279 | 280 | @property 281 | def category(self): 282 | """ YouTube category of the video. Returns string. """ 283 | if not self._category: 284 | self._fetch_gdata() 285 | 286 | return self._category 287 | 288 | @property 289 | def description(self): 290 | """ Description of the video. Returns string. """ 291 | if not self._description: 292 | self._fetch_gdata() 293 | 294 | return self._description 295 | 296 | @property 297 | def username(self): 298 | """ Return the username of the uploader. """ 299 | if not self._username: 300 | self._fetch_basic() 301 | 302 | return self._username 303 | 304 | @property 305 | def published(self): 306 | """ The upload date and time of the video. Returns string. """ 307 | if not self._published: 308 | self._fetch_gdata() 309 | 310 | return self._published.replace(".000Z", "").replace("T", " ") 311 | 312 | @property 313 | def likes(self): 314 | """ The number of likes for the video. Returns int. """ 315 | if not self._likes: 316 | self._fetch_basic() 317 | 318 | return self._likes 319 | 320 | @property 321 | def dislikes(self): 322 | """ The number of dislikes for the video. Returns int. """ 323 | if not self._dislikes: 324 | self._fetch_basic() 325 | 326 | return self._dislikes 327 | 328 | @property 329 | def mix(self): 330 | """ The playlist for the related YouTube mix. Returns a Playlist object. """ 331 | if self._mix_pl is None: 332 | try: 333 | self._mix_pl = get_playlist2("RD" + self.videoid) 334 | except IOError: 335 | return None 336 | return self._mix_pl 337 | 338 | def _getbest(self, preftype="any", ftypestrict=True, vidonly=False): 339 | """ 340 | Return the highest resolution video available. 341 | 342 | Select from video-only streams if vidonly is True 343 | """ 344 | streams = self.videostreams if vidonly else self.streams 345 | 346 | if not streams: 347 | return None 348 | 349 | def _sortkey(x, key3d=0, keyres=0, keyftype=0): 350 | """ sort function for max(). """ 351 | key3d = "3D" not in x.resolution 352 | keyres = int(x.resolution.split("x")[0]) 353 | keyftype = preftype == x.extension 354 | strict = (key3d, keyftype, keyres) 355 | nonstrict = (key3d, keyres, keyftype) 356 | return strict if ftypestrict else nonstrict 357 | 358 | r = max(streams, key=_sortkey) 359 | 360 | if ftypestrict and preftype != "any" and r.extension != preftype: 361 | return None 362 | 363 | else: 364 | return r 365 | 366 | def getbestvideo(self, preftype="any", ftypestrict=True): 367 | """ 368 | Return the best resolution video-only stream. 369 | 370 | set ftypestrict to False to return a non-preferred format if that 371 | has a higher resolution 372 | """ 373 | return self._getbest(preftype, ftypestrict, vidonly=True) 374 | 375 | def getbest(self, preftype="any", ftypestrict=True): 376 | """ 377 | Return the highest resolution video+audio stream. 378 | 379 | set ftypestrict to False to return a non-preferred format if that 380 | has a higher resolution 381 | """ 382 | return self._getbest(preftype, ftypestrict, vidonly=False) 383 | 384 | def getbestaudio(self, preftype="any", ftypestrict=True): 385 | """ Return the highest bitrate audio Stream object.""" 386 | if not self.audiostreams: 387 | return None 388 | 389 | def _sortkey(x, keybitrate=0, keyftype=0): 390 | """ Sort function for max(). """ 391 | keybitrate = int(x.rawbitrate) 392 | keyftype = preftype == x.extension 393 | strict, nonstrict = (keyftype, keybitrate), (keybitrate, keyftype) 394 | return strict if ftypestrict else nonstrict 395 | 396 | r = max(self.audiostreams, key=_sortkey) 397 | 398 | if ftypestrict and preftype != "any" and r.extension != preftype: 399 | return None 400 | 401 | else: 402 | return r 403 | 404 | @classmethod 405 | def _content_available(cls, url): 406 | try: 407 | response = urlopen(url) 408 | except HTTPError: 409 | return False 410 | else: 411 | return response.getcode() < 300 412 | 413 | def getbestthumb(self): 414 | """ Return the best available thumbnail.""" 415 | if not self._bestthumb: 416 | part_url = "http://i.ytimg.com/vi/%s/" % self.videoid 417 | # Thumbnail resolution sorted in descending order 418 | thumbs = ("maxresdefault.jpg", 419 | "sddefault.jpg", 420 | "hqdefault.jpg", 421 | "mqdefault.jpg", 422 | "default.jpg") 423 | for thumb in thumbs: 424 | url = part_url + thumb 425 | if self._content_available(url): 426 | return url 427 | 428 | return self._bestthumb 429 | 430 | def populate_from_playlist(self, pl_data): 431 | """ Populate Pafy object with items fetched from playlist data. """ 432 | self._title = pl_data.get("title") 433 | self._author = pl_data.get("author") 434 | self._length = int(pl_data.get("length_seconds", 0)) 435 | self._rating = pl_data.get("rating", 0.0) 436 | self._viewcount = "".join(re.findall(r"\d", "{0}".format(pl_data.get("views", "0")))) 437 | self._viewcount = int(self._viewcount) 438 | self._description = pl_data.get("description") 439 | 440 | 441 | class BaseStream(object): 442 | 443 | """ YouTube video stream class. """ 444 | 445 | def __init__(self, parent): 446 | """ Set initial values. """ 447 | self._itag = None 448 | self._mediatype = None 449 | self._threed = None 450 | self._rawbitrate = None 451 | self._resolution = None 452 | self._quality = None 453 | self._dimensions = None 454 | self._bitrate = None 455 | self._extension = None 456 | self.encrypted = None 457 | self._notes = None 458 | self._url = None 459 | self._rawurl = None 460 | 461 | self._parent = parent 462 | self._filename = None 463 | self._fsize = None 464 | self._active = False 465 | 466 | def generate_filename(self, meta=False, max_length=None): 467 | """ Generate filename. """ 468 | ok = re.compile(r'[^/]') 469 | 470 | if os.name == "nt": 471 | ok = re.compile(r'[^\\/:*?"<>|]') 472 | 473 | filename = "".join(x if ok.match(x) else "_" for x in self.title) 474 | 475 | if meta: 476 | filename += " - %s - %s" % (self._parent.videoid, self.itag) 477 | 478 | if max_length: 479 | max_length = max_length + 1 + len(self.extension) 480 | if len(filename) > max_length: 481 | filename = filename[:max_length-3] + '...' 482 | 483 | filename += "." + self.extension 484 | return xenc(filename) 485 | 486 | @property 487 | def rawbitrate(self): 488 | """ Return raw bitrate value. """ 489 | return self._rawbitrate 490 | 491 | @property 492 | def threed(self): 493 | """ Return bool, True if stream is 3D. """ 494 | return self._threed 495 | 496 | @property 497 | def itag(self): 498 | """ Return itag value of stream. """ 499 | return self._itag 500 | 501 | @property 502 | def resolution(self): 503 | """ Return resolution of stream as str. 0x0 if audio. """ 504 | return self._resolution 505 | 506 | @property 507 | def dimensions(self): 508 | """ Return dimensions of stream as tuple. (0, 0) if audio. """ 509 | return self._dimensions 510 | 511 | @property 512 | def quality(self): 513 | """ Return quality of stream (bitrate or resolution). 514 | 515 | eg, 128k or 640x480 (str) 516 | """ 517 | return self._quality 518 | 519 | @property 520 | def title(self): 521 | """ Return YouTube video title as a string. """ 522 | return self._parent.title 523 | 524 | @property 525 | def extension(self): 526 | """ Return appropriate file extension for stream (str). 527 | 528 | Possible values are: 3gp, m4a, m4v, mp4, webm, ogg 529 | """ 530 | return self._extension 531 | 532 | @property 533 | def bitrate(self): 534 | """ Return bitrate of an audio stream. """ 535 | return self._bitrate 536 | 537 | @property 538 | def mediatype(self): 539 | """ Return mediatype string (normal, audio or video). 540 | 541 | (normal means a stream containing both video and audio.) 542 | """ 543 | return self._mediatype 544 | 545 | @property 546 | def notes(self): 547 | """ Return additional notes regarding the stream format. """ 548 | return self._notes 549 | 550 | @property 551 | def filename(self): 552 | """ Return filename of stream; derived from title and extension. """ 553 | if not self._filename: 554 | self._filename = self.generate_filename() 555 | return self._filename 556 | 557 | @property 558 | def url(self): 559 | """ Return the url, decrypt if required. """ 560 | return self._url 561 | 562 | @property 563 | def url_https(self): 564 | """ Return https url. """ 565 | return self.url.replace("http://", "https://") 566 | 567 | def __repr__(self): 568 | """ Return string representation. """ 569 | out = "%s:%s@%s" % (self.mediatype, self.extension, self.quality) 570 | return out 571 | 572 | def get_filesize(self): 573 | """ Return filesize of the stream in bytes. Set member variable. """ 574 | if not self._fsize: 575 | 576 | try: 577 | dbg("Getting stream size") 578 | cl = "content-length" 579 | self._fsize = int(g.opener.open(self.url).headers[cl]) 580 | dbg("Got stream size") 581 | 582 | except (AttributeError, HTTPError, URLError): 583 | self._fsize = 0 584 | 585 | return self._fsize 586 | 587 | def cancel(self): 588 | """ Cancel an active download. """ 589 | if self._active: 590 | self._active = False 591 | return True 592 | 593 | def download(self, filepath="", quiet=False, progress="Bytes", 594 | callback=None, meta=False, remux_audio=False): 595 | """ Download. Use quiet=True to supress output. Return filename. 596 | 597 | Use meta=True to append video id and itag to generated filename 598 | Use remax_audio=True to remux audio file downloads 599 | 600 | """ 601 | # pylint: disable=R0912,R0914 602 | # Too many branches, too many local vars 603 | savedir = filename = "" 604 | 605 | if filepath and os.path.isdir(filepath): 606 | savedir, filename = filepath, self.generate_filename(max_length=256-len('.temp')) 607 | 608 | elif filepath: 609 | savedir, filename = os.path.split(filepath) 610 | 611 | else: 612 | filename = self.generate_filename(meta=meta, max_length=256-len('.temp')) 613 | 614 | filepath = os.path.join(savedir, filename) 615 | temp_filepath = filepath + ".temp" 616 | 617 | progress_available = ["KB", "MB", "GB"] 618 | if progress not in progress_available: 619 | progress = "Bytes" 620 | 621 | status_string = get_status_string(progress) 622 | 623 | response = g.opener.open(self.url) 624 | total = int(response.info()['Content-Length'].strip()) 625 | chunksize, bytesdone, t0 = 16384, 0, time.time() 626 | 627 | fmode, offset = "wb", 0 628 | 629 | if os.path.exists(temp_filepath): 630 | if os.stat(temp_filepath).st_size < total: 631 | 632 | offset = os.stat(temp_filepath).st_size 633 | fmode = "ab" 634 | 635 | outfh = open(temp_filepath, fmode) 636 | 637 | if offset: 638 | # partial file exists, resume download 639 | resuming_opener = build_opener() 640 | resuming_opener.addheaders = [('User-Agent', g.user_agent), 641 | ("Range", "bytes=%s-" % offset)] 642 | response = resuming_opener.open(self.url) 643 | bytesdone = offset 644 | 645 | self._active = True 646 | 647 | while self._active: 648 | chunk = response.read(chunksize) 649 | outfh.write(chunk) 650 | elapsed = time.time() - t0 651 | bytesdone += len(chunk) 652 | if elapsed: 653 | rate = ((float(bytesdone) - float(offset)) / 1024.0) / elapsed 654 | eta = (total - bytesdone) / (rate * 1024) 655 | else: # Avoid ZeroDivisionError 656 | rate = 0 657 | eta = 0 658 | 659 | progress_stats = (get_size_done(bytesdone, progress), 660 | bytesdone * 1.0 / total, rate, eta) 661 | 662 | if not chunk: 663 | outfh.close() 664 | break 665 | 666 | if not quiet: 667 | status = status_string.format(*progress_stats) 668 | sys.stdout.write("\r" + status + ' ' * 4 + "\r") 669 | sys.stdout.flush() 670 | 671 | if callback: 672 | callback(total, *progress_stats) 673 | 674 | if self._active: 675 | 676 | if remux_audio and self.mediatype == "audio": 677 | remux(temp_filepath, filepath, quiet=quiet, muxer=remux_audio) 678 | 679 | else: 680 | os.rename(temp_filepath, filepath) 681 | 682 | return filepath 683 | 684 | else: # download incomplete, return temp filepath 685 | outfh.close() 686 | return temp_filepath 687 | 688 | 689 | def remux(infile, outfile, quiet=False, muxer="ffmpeg"): 690 | """ Remux audio. """ 691 | muxer = muxer if isinstance(muxer, str) else "ffmpeg" 692 | 693 | for tool in set([muxer, "ffmpeg", "avconv"]): 694 | cmd = [tool, "-y", "-i", infile, "-acodec", "copy", "-vn", outfile] 695 | 696 | try: 697 | with open(os.devnull, "w") as devnull: 698 | subprocess.call(cmd, stdout=devnull, stderr=subprocess.STDOUT) 699 | 700 | except OSError: 701 | dbg("Failed to remux audio using %s", tool) 702 | 703 | else: 704 | os.unlink(infile) 705 | dbg("remuxed audio file using %s" % tool) 706 | 707 | if not quiet: 708 | sys.stdout.write("\nAudio remuxed.\n") 709 | 710 | break 711 | 712 | else: 713 | logging.warning("audio remux failed") 714 | os.rename(infile, outfile) 715 | 716 | 717 | def get_size_done(bytesdone, progress): 718 | _progress_dict = {'KB': 1024.0, 'MB': 1048576.0, 'GB': 1073741824.0} 719 | return round(bytesdone/_progress_dict.get(progress, 1.0), 2) 720 | 721 | 722 | def get_status_string(progress): 723 | status_string = (' {:,} ' + progress + ' [{:.2%}] received. Rate: [{:4.0f} ' 724 | 'KB/s]. ETA: [{:.0f} secs]') 725 | 726 | if early_py_version: 727 | status_string = (' {0:} ' + progress + ' [{1:.2%}] received. Rate:' 728 | ' [{2:4.0f} KB/s]. ETA: [{3:.0f} secs]') 729 | 730 | return status_string 731 | -------------------------------------------------------------------------------- /pafy/backend_youtube_dl.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import time 3 | import logging 4 | import os 5 | import subprocess 6 | 7 | if sys.version_info[:2] >= (3, 0): 8 | # pylint: disable=E0611,F0401,I0011 9 | uni = str 10 | else: 11 | uni = unicode 12 | 13 | import youtube_dl 14 | 15 | from . import g 16 | from .backend_shared import BasePafy, BaseStream, remux, get_status_string, get_size_done 17 | 18 | dbg = logging.debug 19 | 20 | 21 | early_py_version = sys.version_info[:2] < (2, 7) 22 | 23 | 24 | class YtdlPafy(BasePafy): 25 | def __init__(self, *args, **kwargs): 26 | self._ydl_info = None 27 | self._ydl_opts = g.def_ydl_opts 28 | ydl_opts = kwargs.get("ydl_opts") 29 | if ydl_opts: 30 | self._ydl_opts.update(ydl_opts) 31 | super(YtdlPafy, self).__init__(*args, **kwargs) 32 | 33 | def _fetch_basic(self): 34 | """ Fetch basic data and streams. """ 35 | if self._have_basic: 36 | return 37 | 38 | with youtube_dl.YoutubeDL(self._ydl_opts) as ydl: 39 | try: 40 | self._ydl_info = ydl.extract_info(self.videoid, download=False) 41 | # Turn into an IOError since that is what pafy previously raised 42 | except youtube_dl.utils.DownloadError as e: 43 | raise IOError(str(e).replace('YouTube said', 'Youtube says')) 44 | 45 | if self.callback: 46 | self.callback("Fetched video info") 47 | 48 | self._title = self._ydl_info['title'] 49 | self._author = self._ydl_info['uploader'] 50 | self._rating = self._ydl_info['average_rating'] 51 | self._length = self._ydl_info['duration'] 52 | self._viewcount = self._ydl_info['view_count'] 53 | self._likes = self._ydl_info['like_count'] 54 | self._dislikes = self._ydl_info.get('dislike_count',0) 55 | self._username = self._ydl_info['uploader_id'] 56 | self._category = self._ydl_info['categories'][0] if self._ydl_info['categories'] else '' 57 | self._bestthumb = self._ydl_info['thumbnails'][0]['url'] 58 | self._bigthumb = g.urls['bigthumb'] % self.videoid 59 | self._bigthumbhd = g.urls['bigthumbhd'] % self.videoid 60 | self.expiry = time.time() + g.lifespan 61 | 62 | self._have_basic = True 63 | 64 | def _fetch_gdata(self): 65 | """ Extract gdata values, fetch gdata if necessary. """ 66 | if self._have_gdata: 67 | return 68 | 69 | item = self._get_video_gdata(self.videoid)['items'][0] 70 | snippet = item['snippet'] 71 | self._published = uni(snippet['publishedAt']) 72 | self._description = uni(snippet["description"]) 73 | # Note: using snippet.get since some videos have no tags object 74 | self._keywords = [uni(i) for i in snippet.get('tags', ())] 75 | self._have_gdata = True 76 | 77 | def _process_streams(self): 78 | """ Create Stream object lists from internal stream maps. """ 79 | 80 | if not self._have_basic: 81 | self._fetch_basic() 82 | 83 | allstreams = [YtdlStream(z, self) for z in self._ydl_info['formats']] 84 | self._streams = [i for i in allstreams if i.mediatype == 'normal'] 85 | self._audiostreams = [i for i in allstreams if i.mediatype == 'audio'] 86 | self._videostreams = [i for i in allstreams if i.mediatype == 'video'] 87 | self._m4astreams = [i for i in allstreams if i.extension == 'm4a'] 88 | self._oggstreams = [i for i in allstreams if i.extension == 'ogg'] 89 | self._allstreams = allstreams 90 | 91 | 92 | class YtdlStream(BaseStream): 93 | def __init__(self, info, parent): 94 | super(YtdlStream, self).__init__(parent) 95 | self._itag = info['format_id'] 96 | 97 | if (info.get('acodec') != 'none' and 98 | info.get('vcodec') == 'none'): 99 | self._mediatype = 'audio' 100 | elif (info.get('acodec') == 'none' and 101 | info.get('vcodec') != 'none'): 102 | self._mediatype = 'video' 103 | else: 104 | self._mediatype = 'normal' 105 | 106 | self._threed = info.get('format_note') == '3D' 107 | self._rawbitrate = info.get('abr', 0) * 1024 108 | 109 | height = info.get('height') or 0 110 | width = info.get('width') or 0 111 | self._resolution = str(width) + 'x' + str(height) 112 | self._dimensions = width, height 113 | self._bitrate = str(info.get('abr', 0)) + 'k' 114 | self._quality = self._bitrate if self._mediatype == 'audio' else self._resolution 115 | 116 | self._extension = info['ext'] 117 | self._notes = info.get('format_note') or '' 118 | self._url = info.get('url') 119 | 120 | self._info = info 121 | 122 | def get_filesize(self): 123 | """ Return filesize of the stream in bytes. Set member variable. """ 124 | 125 | # Faster method 126 | if 'filesize' in self._info and self._info['filesize'] is not None: 127 | return self._info['filesize'] 128 | 129 | # Fallback 130 | return super(YtdlStream, self).get_filesize() 131 | 132 | def download(self, filepath="", quiet=False, progress="Bytes", 133 | callback=None, meta=False, remux_audio=False): 134 | 135 | downloader = youtube_dl.downloader.http.HttpFD(ydl(), 136 | {'http_chunk_size': 10485760}) 137 | 138 | progress_available = ["KB", "MB", "GB"] 139 | if progress not in progress_available: 140 | progress = "Bytes" 141 | 142 | status_string = get_status_string(progress) 143 | 144 | def progress_hook(s): 145 | if s['status'] == 'downloading': 146 | bytesdone = s['downloaded_bytes'] 147 | total = s['total_bytes'] 148 | if s.get('speed') is not None: 149 | rate = s['speed'] / 1024 150 | else: 151 | rate = 0 152 | if s.get('eta') is None: 153 | eta = 0 154 | else: 155 | eta = s['eta'] 156 | 157 | progress_stats = (get_size_done(bytesdone, progress), 158 | bytesdone*1.0/total, rate, eta) 159 | if not quiet: 160 | status = status_string.format(*progress_stats) 161 | sys.stdout.write("\r" + status + ' ' * 4 + "\r") 162 | sys.stdout.flush() 163 | 164 | if callback: 165 | callback(total, *progress_stats) 166 | 167 | downloader._progress_hooks = [progress_hook] 168 | 169 | if filepath and os.path.isdir(filepath): 170 | filename = self.generate_filename(max_length=256 - len('.temp')) 171 | filepath = os.path.join(filepath, filename) 172 | 173 | elif filepath: 174 | pass 175 | 176 | else: 177 | filepath = self.generate_filename(meta=meta, max_length=256 - len('.temp')) 178 | 179 | infodict = {'url': self.url} 180 | 181 | downloader.download(filepath, infodict) 182 | print() 183 | 184 | if remux_audio and self.mediatype == "audio": 185 | subprocess.run(['mv', filepath, filepath + '.temp']) 186 | remux(filepath + '.temp', filepath, quiet=quiet, muxer=remux_audio) 187 | 188 | 189 | class ydl: 190 | def urlopen(self, url): 191 | return g.opener.open(url) 192 | 193 | def to_screen(self, *args, **kwargs): 194 | pass 195 | 196 | def to_console_title(self, *args, **kwargs): 197 | pass 198 | 199 | def trouble(self, *args, **kwargs): 200 | pass 201 | 202 | def report_warning(self, *args, **kwargs): 203 | pass 204 | 205 | def report_error(self, *args, **kwargs): 206 | pass 207 | -------------------------------------------------------------------------------- /pafy/channel.py: -------------------------------------------------------------------------------- 1 | import re 2 | from .pafy import call_gdata 3 | from .playlist import Playlist 4 | from .backend_shared import pyver 5 | 6 | 7 | def get_channel(channel_url, basic=False, gdata=False, 8 | size=False, callback=None): 9 | """Return a Channel object 10 | 11 | The returned Pafy and Playlist objects are initialised using the arguments 12 | to get_channel() in the manner documented for pafy.new() 13 | 14 | """ 15 | 16 | return Channel.from_url(channel_url, basic, gdata, size, callback) 17 | 18 | 19 | class Channel(object): 20 | def __init__(self, channel_url, basic, gdata, size, callback): 21 | 22 | self._channel_url = channel_url 23 | self._channel_id = None 24 | self._title = None 25 | self._description = None 26 | self._logo = None 27 | self._subscriberCount = None 28 | self._uploads = None 29 | self._basic = basic 30 | self._gdata = gdata 31 | self._size = size 32 | self._callback = callback 33 | self._playlists = None 34 | self._subscriptions = None 35 | self._have_basic = False 36 | 37 | @classmethod 38 | def from_dict(cls, ch, basic, gdata, size, callback): 39 | t = cls(ch['id'], basic, gdata, size, callback) 40 | t._channel_id = ch['id'] 41 | t._title = ch['title'] 42 | t._description = ch['description'] 43 | t._logo = ch['logo'] 44 | t._subscriberCount = ch['subscriberCount'] 45 | t._uploads = ch['uploads'] 46 | t._have_basic = True 47 | 48 | return t 49 | 50 | @classmethod 51 | def from_url(cls, url, basic, gdata, size, callback): 52 | t = cls(url, basic, gdata, size, callback) 53 | t._fetch_basic() 54 | return t 55 | 56 | @property 57 | def channel_id(self): 58 | if not self._have_basic: 59 | self._fetch_basic() 60 | return self._channel_id 61 | 62 | @property 63 | def title(self): 64 | if not self._have_basic: 65 | self._fetch_basic() 66 | return self._title 67 | 68 | @property 69 | def description(self): 70 | if not self._have_basic: 71 | self._fetch_basic() 72 | return self._description 73 | 74 | @property 75 | def logo(self): 76 | if not self._have_basic: 77 | self._fetch_basic() 78 | return self._logo 79 | 80 | @property 81 | def subscriberCount(self): 82 | if not self._have_basic: 83 | self._fetch_basic() 84 | return self._subscriberCount 85 | 86 | @property 87 | def uploads(self): 88 | if not self._uploads: 89 | self._fetch_basic() 90 | if type(self._uploads) != Playlist: 91 | self._uploads = Playlist.from_url(self._uploads, self._basic, 92 | self._gdata, self._size, 93 | self._callback) 94 | 95 | return self._uploads 96 | 97 | @property 98 | def playlists(self): 99 | if self._playlists is not None: 100 | for playlist in self._playlists: 101 | yield playlist 102 | return 103 | 104 | playlists = [] 105 | 106 | query = {'part': 'snippet,contentDetails', 107 | 'maxResults': 50, 108 | 'channelId': self.channel_id} 109 | 110 | while True: 111 | playlistList = call_gdata('playlists', query) 112 | 113 | for pl in playlistList['items']: 114 | try: 115 | thumbnail = pl['snippet']['thumbnails']['standard']['url'] 116 | except KeyError: 117 | thumbnail = None 118 | pl_data = dict( 119 | id=pl['id'], 120 | title=pl['snippet']['title'], 121 | author=pl['snippet']['channelTitle'], 122 | description=pl['snippet']['description'], 123 | thumbnail=thumbnail, 124 | len=pl['contentDetails']['itemCount'] 125 | ) 126 | 127 | pl_obj = Playlist.from_dict(pl_data, self._basic, self._gdata, 128 | self._size, self._callback) 129 | playlists.append(pl_obj) 130 | if self._callback: 131 | self._callback("Added playlist: %s" % pl_data['title']) 132 | yield pl_obj 133 | 134 | if not playlistList.get('nextPageToken'): 135 | break 136 | query['pageToken'] = playlistList['nextPageToken'] 137 | 138 | self._playlists = playlists 139 | 140 | @property 141 | def subscriptions(self): 142 | if self._subscriptions is not None: 143 | for sub in self._subscriptions: 144 | yield sub 145 | return 146 | 147 | subscriptions = [] 148 | query = {'part': 'snippet', 149 | 'maxResults': 50, 150 | 'channelId': self.channel_id} 151 | 152 | while True: 153 | subs_data = call_gdata('subscriptions', query) 154 | sub_ids = [] 155 | 156 | for sub in subs_data['items']: 157 | sub_ids.append(sub['snippet']['resourceId']['channelId']) 158 | 159 | query2 = {'part': 'snippet, contentDetails, statistics', 160 | 'id': ','.join(sub_ids), 161 | 'maxResults': 50} 162 | 163 | data = call_gdata('channels', query2) 164 | 165 | for ch in data['items']: 166 | channel_data = dict( 167 | id=ch['id'], 168 | title=ch['snippet']['title'], 169 | description=ch['snippet']['description'], 170 | logo=ch['snippet']['thumbnails']['default']['url'], 171 | subscriberCount=ch['statistics']['subscriberCount'], 172 | uploads=ch['contentDetails']['relatedPlaylists']['uploads'] 173 | ) 174 | sub_obj = Channel.from_dict(channel_data, self._basic, 175 | self._gdata, self._size, 176 | self._callback) 177 | subscriptions.append(sub_obj) 178 | yield sub_obj 179 | 180 | if not subs_data.get('nextPageToken'): 181 | break 182 | query['pageToken'] = subs_data['nextPageToken'] 183 | 184 | self._subscriptions = subscriptions 185 | 186 | def __repr__(self): 187 | if not self._have_basic: 188 | self._fetch_basic() 189 | 190 | info = [("Type", "Channel"), 191 | ("Title", self.title), 192 | ("Description", self.description), 193 | ("SubscriberCount", self.subscriberCount)] 194 | 195 | nfo = "\n".join(["%s: %s" % i for i in info]) 196 | 197 | return nfo.encode("utf8", "replace") if pyver == 2 else nfo 198 | 199 | def _fetch_basic(self): 200 | query = None 201 | chanR = re.compile('.+channel\/([^\/]+)$') 202 | userR = re.compile('.+user\/([^\/]+)$') 203 | channel_id = None 204 | channel_url = self._channel_url 205 | if chanR.match(channel_url): 206 | channel_id = chanR.search(channel_url).group(1) 207 | elif userR.match(channel_url): 208 | username = userR.search(channel_url).group(1) 209 | query = {'part': 'snippet, contentDetails, statistics', 210 | 'forUsername': username} 211 | elif len(channel_url) == 24 and channel_url[:2] == 'UC': 212 | channel_id = channel_url 213 | else: 214 | username = channel_url 215 | query = {'part': 'snippet, contentDetails, statistics', 216 | 'forUsername': username} 217 | 218 | if query is None: 219 | query = {'part': 'snippet, contentDetails, statistics', 220 | 'id': channel_id} 221 | allinfo = call_gdata('channels', query) 222 | 223 | try: 224 | ch = allinfo['items'][0] 225 | except IndexError: 226 | err = "Unrecognized channel id, url or name : %s" 227 | raise ValueError(err % channel_url) 228 | 229 | self._channel_id = ch['id'] 230 | self._title = ch['snippet']['title'] 231 | self._description = ch['snippet']['description'] 232 | self._logo = ch['snippet']['thumbnails']['default']['url'] 233 | self._subscriberCount = ch['statistics']['subscriberCount'] 234 | self._uploads = ch['contentDetails']['relatedPlaylists']['uploads'] 235 | self._have_basic = True 236 | -------------------------------------------------------------------------------- /pafy/g.py: -------------------------------------------------------------------------------- 1 | import sys 2 | if sys.version_info[:2] >= (3, 0): 3 | # pylint: disable=E0611,F0401,I0011 4 | from urllib.request import build_opener 5 | else: 6 | from urllib2 import build_opener 7 | 8 | from . import __version__ 9 | 10 | urls = { 11 | 'gdata': "https://www.googleapis.com/youtube/v3/", 12 | 'watchv': "http://www.youtube.com/watch?v=%s", 13 | 'playlist': ('http://www.youtube.com/list_ajax?' 14 | 'style=json&action_get_list=1&list=%s'), 15 | 'thumb': "http://i.ytimg.com/vi/%s/default.jpg", 16 | 'bigthumb': "http://i.ytimg.com/vi/%s/mqdefault.jpg", 17 | 'bigthumbhd': "http://i.ytimg.com/vi/%s/hqdefault.jpg", 18 | 19 | # For internal backend 20 | 'vidinfo': ('https://www.youtube.com/get_video_info?video_id=%s&' 21 | 'eurl=https://youtube.googleapis.com/v/%s&sts=%s'), 22 | 'embed': "https://youtube.com/embed/%s" 23 | } 24 | api_key = "AIzaSyCIM4EzNqi1in22f4Z3Ru3iYvLaY8tc3bo" 25 | user_agent = "pafy " + __version__ 26 | lifespan = 60 * 60 * 5 # 5 hours 27 | opener = build_opener() 28 | opener.addheaders = [('User-Agent', user_agent)] 29 | cache = {} 30 | def_ydl_opts = {'quiet': True, 'prefer_insecure': False, 'no_warnings': True} 31 | 32 | # The following are specific to the internal backend 33 | UEFSM = 'url_encoded_fmt_stream_map' 34 | AF = 'adaptive_fmts' 35 | jsplayer = r';ytplayer\.config\s*=\s*({.*?});' 36 | itags = { 37 | '5': ('320x240', 'flv', "normal", ''), 38 | '17': ('176x144', '3gp', "normal", ''), 39 | '18': ('640x360', 'mp4', "normal", ''), 40 | '22': ('1280x720', 'mp4', "normal", ''), 41 | '34': ('640x360', 'flv', "normal", ''), 42 | '35': ('854x480', 'flv', "normal", ''), 43 | '36': ('320x240', '3gp', "normal", ''), 44 | '37': ('1920x1080', 'mp4', "normal", ''), 45 | '38': ('4096x3072', 'mp4', "normal", '4:3 hi-res'), 46 | '43': ('640x360', 'webm', "normal", ''), 47 | '44': ('854x480', 'webm', "normal", ''), 48 | '45': ('1280x720', 'webm', "normal", ''), 49 | '46': ('1920x1080', 'webm', "normal", ''), 50 | '82': ('640x360-3D', 'mp4', "normal", ''), 51 | '83': ('640x480-3D', 'mp4', 'normal', ''), 52 | '84': ('1280x720-3D', 'mp4', "normal", ''), 53 | '100': ('640x360-3D', 'webm', "normal", ''), 54 | '102': ('1280x720-3D', 'webm', "normal", ''), 55 | '133': ('426x240', 'm4v', 'video', ''), 56 | '134': ('640x360', 'm4v', 'video', ''), 57 | '135': ('854x480', 'm4v', 'video', ''), 58 | '136': ('1280x720', 'm4v', 'video', ''), 59 | '137': ('1920x1080', 'm4v', 'video', ''), 60 | '138': ('4096x3072', 'm4v', 'video', ''), 61 | '139': ('48k', 'm4a', 'audio', ''), 62 | '140': ('128k', 'm4a', 'audio', ''), 63 | '141': ('256k', 'm4a', 'audio', ''), 64 | '160': ('256x144', 'm4v', 'video', ''), 65 | '167': ('640x480', 'webm', 'video', ''), 66 | '168': ('854x480', 'webm', 'video', ''), 67 | '169': ('1280x720', 'webm', 'video', ''), 68 | '170': ('1920x1080', 'webm', 'video', ''), 69 | '171': ('128k', 'ogg', 'audio', ''), 70 | '172': ('192k', 'ogg', 'audio', ''), 71 | '218': ('854x480', 'webm', 'video', 'VP8'), 72 | '219': ('854x480', 'webm', 'video', 'VP8'), 73 | '242': ('360x240', 'webm', 'video', 'VP9'), 74 | '243': ('480x360', 'webm', 'video', 'VP9'), 75 | '244': ('640x480', 'webm', 'video', 'VP9 low'), 76 | '245': ('640x480', 'webm', 'video', 'VP9 med'), 77 | '246': ('640x480', 'webm', 'video', 'VP9 high'), 78 | '247': ('720x480', 'webm', 'video', 'VP9'), 79 | '248': ('1920x1080', 'webm', 'video', 'VP9'), 80 | '249': ('48k', 'opus', 'audio', 'Opus'), 81 | '250': ('56k', 'opus', 'audio', 'Opus'), 82 | '251': ('128k', 'opus', 'audio', 'Opus'), 83 | '256': ('192k', 'm4a', 'audio', '6-channel'), 84 | '258': ('320k', 'm4a', 'audio', '6-channel'), 85 | '264': ('2560x1440', 'm4v', 'video', ''), 86 | '266': ('3840x2160', 'm4v', 'video', 'AVC'), 87 | '271': ('1920x1280', 'webm', 'video', 'VP9'), 88 | '272': ('3414x1080', 'webm', 'video', 'VP9'), 89 | '278': ('256x144', 'webm', 'video', 'VP9'), 90 | '298': ('1280x720', 'm4v', 'video', '60fps'), 91 | '299': ('1920x1080', 'm4v', 'video', '60fps'), 92 | '302': ('1280x720', 'webm', 'video', 'VP9'), 93 | '303': ('1920x1080', 'webm', 'video', 'VP9'), 94 | } 95 | -------------------------------------------------------------------------------- /pafy/jsinterp.py: -------------------------------------------------------------------------------- 1 | # Copied from youtube_dl 2 | 3 | from __future__ import unicode_literals 4 | 5 | import json 6 | import operator 7 | import re 8 | import sys 9 | import traceback 10 | 11 | 12 | class ExtractorError(Exception): 13 | """Error during info extraction.""" 14 | 15 | def __init__(self, msg, tb=None, expected=False, cause=None, video_id=None): 16 | """ tb, if given, is the original traceback (so that it can be printed out). 17 | If expected is set, this is a normal error message and most likely not a bug in youtube-dl. 18 | """ 19 | 20 | if video_id is not None: 21 | msg = video_id + ': ' + msg 22 | if cause: 23 | msg += ' (caused by %r)' % cause 24 | super(ExtractorError, self).__init__(msg) 25 | 26 | self.traceback = tb 27 | self.exc_info = sys.exc_info() # preserve original exception 28 | self.cause = cause 29 | self.video_id = video_id 30 | 31 | def format_traceback(self): 32 | if self.traceback is None: 33 | return None 34 | return ''.join(traceback.format_tb(self.traceback)) 35 | 36 | 37 | _OPERATORS = [ 38 | ('|', operator.or_), 39 | ('^', operator.xor), 40 | ('&', operator.and_), 41 | ('>>', operator.rshift), 42 | ('<<', operator.lshift), 43 | ('-', operator.sub), 44 | ('+', operator.add), 45 | ('%', operator.mod), 46 | ('/', operator.truediv), 47 | ('*', operator.mul), 48 | ] 49 | _ASSIGN_OPERATORS = [(op + '=', opfunc) for op, opfunc in _OPERATORS] 50 | _ASSIGN_OPERATORS.append(('=', lambda cur, right: right)) 51 | 52 | _NAME_RE = r'[a-zA-Z_$][a-zA-Z_$0-9]*' 53 | 54 | 55 | class JSInterpreter(object): 56 | def __init__(self, code, objects=None): 57 | if objects is None: 58 | objects = {} 59 | self.code = code 60 | self._functions = {} 61 | self._objects = objects 62 | 63 | def interpret_statement(self, stmt, local_vars, allow_recursion=100): 64 | if allow_recursion < 0: 65 | raise ExtractorError('Recursion limit reached') 66 | 67 | should_abort = False 68 | stmt = stmt.lstrip() 69 | stmt_m = re.match(r'var\s', stmt) 70 | if stmt_m: 71 | expr = stmt[len(stmt_m.group(0)):] 72 | else: 73 | return_m = re.match(r'return(?:\s+|$)', stmt) 74 | if return_m: 75 | expr = stmt[len(return_m.group(0)):] 76 | should_abort = True 77 | else: 78 | # Try interpreting it as an expression 79 | expr = stmt 80 | 81 | v = self.interpret_expression(expr, local_vars, allow_recursion) 82 | return v, should_abort 83 | 84 | def interpret_expression(self, expr, local_vars, allow_recursion): 85 | expr = expr.strip() 86 | 87 | if expr == '': # Empty expression 88 | return None 89 | 90 | if expr.startswith('('): 91 | parens_count = 0 92 | for m in re.finditer(r'[()]', expr): 93 | if m.group(0) == '(': 94 | parens_count += 1 95 | else: 96 | parens_count -= 1 97 | if parens_count == 0: 98 | sub_expr = expr[1:m.start()] 99 | sub_result = self.interpret_expression( 100 | sub_expr, local_vars, allow_recursion) 101 | remaining_expr = expr[m.end():].strip() 102 | if not remaining_expr: 103 | return sub_result 104 | else: 105 | expr = json.dumps(sub_result) + remaining_expr 106 | break 107 | else: 108 | raise ExtractorError('Premature end of parens in %r' % expr) 109 | 110 | for op, opfunc in _ASSIGN_OPERATORS: 111 | m = re.match(r'''(?x) 112 | (?P
[^}]+)}',
251 | fields)
252 | for f in fields_m:
253 | argnames = f.group('args').split(',')
254 | obj[f.group('key')] = self.build_function(argnames, f.group('code'))
255 |
256 | return obj
257 |
258 | def extract_function(self, funcname):
259 | func_m = re.search(
260 | r'''(?x)
261 | (?:function\s+%s|[{;,]\s*%s\s*=\s*function|var\s+%s\s*=\s*function)\s*
262 | \((?P[^)]*)\)\s*
263 | \{(?P[^}]+)\}''' % (
264 | re.escape(funcname), re.escape(funcname), re.escape(funcname)),
265 | self.code)
266 | if func_m is None:
267 | raise ExtractorError('Could not find JS function %r' % funcname)
268 | argnames = func_m.group('args').split(',')
269 |
270 | return self.build_function(argnames, func_m.group('code'))
271 |
272 | def call_function(self, funcname, *args):
273 | f = self.extract_function(funcname)
274 | return f(args)
275 |
276 | def build_function(self, argnames, code):
277 | def resf(args):
278 | local_vars = dict(zip(argnames, args))
279 | for stmt in code.split(';'):
280 | res, abort = self.interpret_statement(stmt, local_vars)
281 | if abort:
282 | break
283 | return res
284 | return resf
285 |
--------------------------------------------------------------------------------
/pafy/pafy.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | """
4 | pafy.py.
5 |
6 | Python library to download YouTube content and retrieve metadata
7 |
8 | https://github.com/np1/pafy
9 |
10 | Copyright (C) 2013-2014 np1
11 |
12 | This program is free software: you can redistribute it and/or modify it under
13 | the terms of the GNU Lesser General Public License as published by the Free
14 | Software Foundation, either version 3 of the License, or (at your option) any
15 | later version.
16 |
17 | This program is distributed in the hope that it will be useful, but WITHOUT ANY
18 | WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
19 | PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
20 |
21 | You should have received a copy of the GNU Lesser General Public License along
22 | with this program. If not, see