├── README.md ├── channel.py ├── client_init.py ├── config.py ├── constant.py ├── database.py ├── downloader.py ├── flower_tasks.py ├── install.sh ├── limit.py ├── requirements.txt ├── split-video.sh ├── tasks.py ├── termux ├── install.sh └── ytdl_termux.py ├── utils.py └── ytdl_bot.py /README.md: -------------------------------------------------------------------------------- 1 | # ytdlbot 2 | 3 | install YouTube downloader on server and termux 4 | install 5 | ``` 6 | bash <(curl -fsSL https://raw.githubusercontent.com/Ptechgithub/ytdl/main/install.sh) 7 | ``` 8 | ![6](https://raw.githubusercontent.com/Ptechgithub/configs/main/media/6.jpg) 9 | # Features 10 | 11 | 1. fast download and upload. 12 | 2. ads free 13 | 3. support progress bar 14 | 4. audio conversion 15 | 5. playlist support 16 | 6. payment support 17 | 7. support different video resolutions 18 | 8. support sending as file or streaming as video 19 | 9. supports celery worker distribution - faster than before. 20 | 10. subscriptions to YouTube Channels 21 | 11. cache mechanism - download once for the same video. 22 | 12. support instagram posts 23 | 24 | 25 | 26 | # How to deploy? 27 | 28 | This bot can be deployed on any platform that supports Python. 29 | 30 | ## Run natively on your machine 31 | 32 | To deploy this bot, follow these steps: 33 | 34 | 1. Clone the code from the repository. 35 | 2. Install FFmpeg. 36 | 3. Install Python 3.6 or a later version. 37 | 4. Install Aria2 and add it to the PATH. 38 | 5. Install the required packages by running `pip3 install -r requirements.txt`. 39 | 6. Set the environment variables `TOKEN`, `APP_ID`, `APP_HASH`, and any others that you may need. 40 | 7. Run `python3 ytdl_bot.py`. 41 | 42 | ## 2. create data directory 43 | 44 | ```shell 45 | mkdir data 46 | mkdir env 47 | ``` 48 | 49 | ## 3. configuration 50 | 51 | ### 3.1. set environment variables 52 | 53 | ```shell 54 | vim env/ytdl.env 55 | ``` 56 | 57 | You can configure all the following environment variables: 58 | 59 | * WORKERS: workers count for celery 60 | * PYRO_WORKERS: number of workers for pyrogram, default is 100 61 | * APP_ID: **REQUIRED**, get it from https://core.telegram.org/ 62 | * APP_HASH: **REQUIRED** 63 | * TOKEN: **REQUIRED** 64 | * REDIS: **REQUIRED if you need VIP mode and cache** ⚠️ Don't publish your redis server on the internet. ⚠️ 65 | * EXPIRE: token expire time, default: 1 day 66 | * ENABLE_VIP: enable VIP mode 67 | * OWNER: owner username 68 | * AUTHORIZED_USER: only authorized users can use the bot 69 | * REQUIRED_MEMBERSHIP: group or channel username, user must join this group to use the bot 70 | * ENABLE_CELERY: celery mode, default: disable 71 | * ENABLE_QUEUE: celery queue 72 | * BROKER: celery broker, should be redis://redis:6379/0 73 | * MYSQL_HOST:MySQL host 74 | * MYSQL_USER: MySQL username 75 | * MYSQL_PASS: MySQL password 76 | * AUDIO_FORMAT: default audio format 77 | * ARCHIVE_ID: forward all downloads to this group/channel 78 | * IPv6 = os.getenv("IPv6", False) 79 | * ENABLE_FFMPEG = os.getenv("ENABLE_FFMPEG", False) 80 | * PROVIDER_TOKEN: stripe token on Telegram payment 81 | * PLAYLIST_SUPPORT: download playlist support 82 | * ENABLE_ARIA2: enable aria2c download 83 | * FREE_DOWNLOAD: free download count per day 84 | * TOKEN_PRICE: token price per 1 USD 85 | * GOOGLE_API_KEY: YouTube API key, required for YouTube video subscription. 86 | * RCLONE_PATH: rclone path to upload files to cloud storage 87 | ## 3.2 Set up init data 88 | 89 | # Command 90 | 91 | ``` 92 | start - Let's start 93 | about - What's this bot? 94 | ping - Bot running status 95 | help - Help 96 | ytdl - Download video in group 97 | settings - Set your preference 98 | buy - Buy token 99 | direct - Download file directly 100 | sub - Subscribe to YouTube Channel 101 | unsub - Unsubscribe from YouTube Channel 102 | sub_count - Check subscription status, owner only. 103 | uncache - Delete cache for this link, owner only. 104 | purge - Delete all tasks, owner only. 105 | ``` 106 | 107 | 108 | Forked: [tgbot-collection](https://github.com/tgbot-collection/ytdlbot) 109 | -------------------------------------------------------------------------------- /channel.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # coding: utf-8 3 | import http 4 | import logging 5 | import os 6 | import re 7 | 8 | import requests 9 | from bs4 import BeautifulSoup 10 | 11 | from config import ENABLE_VIP 12 | from limit import Payment 13 | 14 | 15 | class Channel(Payment): 16 | def subscribe_channel(self, user_id: int, share_link: str) -> str: 17 | if not re.findall(r"youtube\.com|youtu\.be", share_link): 18 | raise ValueError("Is this a valid YouTube Channel link?") 19 | if ENABLE_VIP: 20 | self.cur.execute("select count(user_id) from subscribe where user_id=%s", (user_id,)) 21 | usage = int(self.cur.fetchone()[0]) 22 | if usage >= 10: 23 | logging.warning("User %s has subscribed %s channels", user_id, usage) 24 | return "You have subscribed too many channels. Maximum 5 channels." 25 | 26 | data = self.get_channel_info(share_link) 27 | channel_id = data["channel_id"] 28 | 29 | self.cur.execute("select user_id from subscribe where user_id=%s and channel_id=%s", (user_id, channel_id)) 30 | if self.cur.fetchall(): 31 | raise ValueError("You have already subscribed this channel.") 32 | 33 | self.cur.execute( 34 | "INSERT IGNORE INTO channel values" 35 | "(%(link)s,%(title)s,%(description)s,%(channel_id)s,%(playlist)s,%(last_video)s)", 36 | data, 37 | ) 38 | self.cur.execute("INSERT INTO subscribe values(%s,%s, NULL)", (user_id, channel_id)) 39 | self.con.commit() 40 | logging.info("User %s subscribed channel %s", user_id, data["title"]) 41 | return "Subscribed to {}".format(data["title"]) 42 | 43 | def unsubscribe_channel(self, user_id: int, channel_id: str) -> int: 44 | affected_rows = self.cur.execute( 45 | "DELETE FROM subscribe WHERE user_id=%s AND channel_id=%s", (user_id, channel_id) 46 | ) 47 | self.con.commit() 48 | logging.info("User %s tried to unsubscribe channel %s", user_id, channel_id) 49 | return affected_rows 50 | 51 | @staticmethod 52 | def extract_canonical_link(url: str) -> str: 53 | # canonic link works for many websites. It will strip out unnecessary stuff 54 | props = ["canonical", "alternate", "shortlinkUrl"] 55 | headers = { 56 | "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36" 57 | } 58 | cookie = {"CONSENT": "PENDING+197"} 59 | # send head request first 60 | r = requests.head(url, headers=headers, allow_redirects=True, cookies=cookie) 61 | if r.status_code != http.HTTPStatus.METHOD_NOT_ALLOWED and "text/html" not in r.headers.get("content-type", ""): 62 | # get content-type, if it's not text/html, there's no need to issue a GET request 63 | logging.warning("%s Content-type is not text/html, no need to GET for extract_canonical_link", url) 64 | return url 65 | 66 | html_doc = requests.get(url, headers=headers, cookies=cookie, timeout=5).text 67 | soup = BeautifulSoup(html_doc, "html.parser") 68 | for prop in props: 69 | element = soup.find("link", rel=prop) 70 | try: 71 | href = element["href"] 72 | if href not in ["null", "", None]: 73 | return href 74 | except Exception: 75 | logging.warning("Canonical exception %s", url) 76 | 77 | return url 78 | 79 | def get_channel_info(self, url: str) -> dict: 80 | api_key = os.getenv("GOOGLE_API_KEY") 81 | canonical_link = self.extract_canonical_link(url) 82 | try: 83 | channel_id = canonical_link.split("youtube.com/channel/")[1] 84 | except IndexError: 85 | channel_id = canonical_link.split("/")[-1] 86 | channel_api = ( 87 | f"https://www.googleapis.com/youtube/v3/channels?part=snippet,contentDetails&id={channel_id}&key={api_key}" 88 | ) 89 | 90 | data = requests.get(channel_api).json() 91 | snippet = data["items"][0]["snippet"] 92 | title = snippet["title"] 93 | description = snippet["description"] 94 | playlist = data["items"][0]["contentDetails"]["relatedPlaylists"]["uploads"] 95 | 96 | return { 97 | "link": url, 98 | "title": title, 99 | "description": description, 100 | "channel_id": channel_id, 101 | "playlist": playlist, 102 | "last_video": self.get_latest_video(playlist), 103 | } 104 | 105 | @staticmethod 106 | def get_latest_video(playlist_id: str) -> str: 107 | api_key = os.getenv("GOOGLE_API_KEY") 108 | video_api = ( 109 | f"https://www.googleapis.com/youtube/v3/playlistItems?part=snippet&maxResults=1&" 110 | f"playlistId={playlist_id}&key={api_key}" 111 | ) 112 | data = requests.get(video_api).json() 113 | video_id = data["items"][0]["snippet"]["resourceId"]["videoId"] 114 | logging.info(f"Latest video %s from %s", video_id, data["items"][0]["snippet"]["channelTitle"]) 115 | return f"https://www.youtube.com/watch?v={video_id}" 116 | 117 | def has_newer_update(self, channel_id: str) -> str: 118 | self.cur.execute("SELECT playlist,latest_video FROM channel WHERE channel_id=%s", (channel_id,)) 119 | data = self.cur.fetchone() 120 | playlist_id = data[0] 121 | old_video = data[1] 122 | newest_video = self.get_latest_video(playlist_id) 123 | if old_video != newest_video: 124 | logging.info("Newer update found for %s %s", channel_id, newest_video) 125 | self.cur.execute("UPDATE channel SET latest_video=%s WHERE channel_id=%s", (newest_video, channel_id)) 126 | self.con.commit() 127 | return newest_video 128 | 129 | def get_user_subscription(self, user_id: int) -> str: 130 | self.cur.execute( 131 | """ 132 | select title, link, channel.channel_id from channel, subscribe 133 | where subscribe.user_id = %s and channel.channel_id = subscribe.channel_id 134 | """, 135 | (user_id,), 136 | ) 137 | data = self.cur.fetchall() 138 | text = "" 139 | for item in data: 140 | text += "[{}]({}) `{}\n`".format(*item) 141 | return text 142 | 143 | def group_subscriber(self) -> dict: 144 | # {"channel_id": [user_id, user_id, ...]} 145 | self.cur.execute("select * from subscribe where is_valid=1") 146 | data = self.cur.fetchall() 147 | group = {} 148 | for item in data: 149 | group.setdefault(item[1], []).append(item[0]) 150 | logging.info("Checking periodic subscriber...") 151 | return group 152 | 153 | def deactivate_user_subscription(self, user_id: int): 154 | self.cur.execute("UPDATE subscribe set is_valid=0 WHERE user_id=%s", (user_id,)) 155 | self.con.commit() 156 | 157 | def sub_count(self) -> str: 158 | sql = """ 159 | select user_id, channel.title, channel.link 160 | from subscribe, channel where subscribe.channel_id = channel.channel_id 161 | """ 162 | self.cur.execute(sql) 163 | data = self.cur.fetchall() 164 | text = f"Total {len(data)} subscriptions found.\n\n" 165 | for item in data: 166 | text += "{} ==> [{}]({})\n".format(*item) 167 | return text 168 | 169 | def del_cache(self, user_link: str) -> int: 170 | unique = self.extract_canonical_link(user_link) 171 | caches = self.r.hgetall("cache") 172 | count = 0 173 | for key in caches: 174 | if key.startswith(unique): 175 | count += self.del_send_cache(key) 176 | return count 177 | 178 | 179 | if __name__ == "__main__": 180 | Channel.extract_canonical_link("https://www.youtube.com/watch?v=zfsPk9moAQk") 181 | -------------------------------------------------------------------------------- /client_init.py: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python3 2 | # coding: utf-8 3 | 4 | # ytdlbot - client_init.py 5 | # 12/29/21 16:20 6 | # 7 | 8 | __author__ = "Peyman" 9 | 10 | from pyrogram import Client 11 | 12 | from config import APP_HASH, APP_ID, PYRO_WORKERS, TOKEN, IPv6 13 | 14 | 15 | def create_app(session: str, workers: int = PYRO_WORKERS) -> Client: 16 | _app = Client( 17 | session, 18 | APP_ID, 19 | APP_HASH, 20 | bot_token=TOKEN, 21 | workers=workers, 22 | ipv6=IPv6, 23 | ) 24 | 25 | return _app 26 | -------------------------------------------------------------------------------- /config.py: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python3 2 | # coding: utf-8 3 | 4 | # ytdlbot - config.py 5 | # 8/28/21 15:01 6 | # 7 | 8 | __author__ = "peyman>" 9 | 10 | import os 11 | 12 | # general settings 13 | WORKERS: int = int(os.getenv("WORKERS", 100)) 14 | PYRO_WORKERS: int = int(os.getenv("PYRO_WORKERS", min(64, (os.cpu_count() + 4) * 10))) 15 | APP_ID: int = int(os.getenv("APP_ID", A)) 16 | APP_HASH = os.getenv("APP_HASH", "B") 17 | TOKEN = os.getenv("TOKEN", "C") 18 | 19 | REDIS = os.getenv("REDIS", "redis") 20 | 21 | TG_MAX_SIZE = 2000 * 1024 * 1024 22 | # TG_MAX_SIZE = 10 * 1024 * 1024 23 | 24 | EXPIRE = 24 * 3600 25 | 26 | ENABLE_VIP = os.getenv("VIP", True) 27 | AFD_LINK = os.getenv("AFD_LINK", "https://afdian.net/@ppppp") 28 | COFFEE_LINK = os.getenv("COFFEE_LINK", "https://www.buymeacoffee.com/pppppp") 29 | COFFEE_TOKEN = os.getenv("COFFEE_TOKEN") 30 | AFD_TOKEN = os.getenv("AFD_TOKEN") 31 | AFD_USER_ID = os.getenv("AFD_USER_ID") 32 | OWNER = os.getenv("OWNER", "Peyman") 33 | 34 | # limitation settings 35 | AUTHORIZED_USER: str = os.getenv("AUTHORIZED_USER", "") 36 | # membership requires: the format could be username(without @ sign)/chat_id of channel or group. 37 | # You need to add the bot to this group/channel as admin 38 | REQUIRED_MEMBERSHIP: str = os.getenv("REQUIRED_MEMBERSHIP", "") 39 | 40 | # celery related 41 | ENABLE_CELERY = os.getenv("ENABLE_CELERY", False) 42 | ENABLE_QUEUE = os.getenv("ENABLE_QUEUE", True) 43 | BROKER = os.getenv("BROKER", f"redis://{REDIS}:6379/4") 44 | 45 | MYSQL_HOST = os.getenv("MYSQL_HOST", "mysql") 46 | MYSQL_USER = os.getenv("MYSQL_USER", "root") 47 | MYSQL_PASS = os.getenv("MYSQL_PASS", "root") 48 | 49 | AUDIO_FORMAT = os.getenv("AUDIO_FORMAT") 50 | ARCHIVE_ID = os.getenv("ARCHIVE_ID") 51 | 52 | IPv6 = os.getenv("IPv6", False) 53 | ENABLE_FFMPEG = os.getenv("ENABLE_FFMPEG", True) 54 | 55 | # Stripe setting 56 | PROVIDER_TOKEN = os.getenv("PROVIDER_TOKEN") or "1234" 57 | 58 | PLAYLIST_SUPPORT = os.getenv("PLAYLIST_SUPPORT", True) 59 | ENABLE_ARIA2 = os.getenv("ENABLE_ARIA2", True) 60 | 61 | FREE_DOWNLOAD = os.getenv("FREE_DOWNLOAD", 20) 62 | TOKEN_PRICE = os.getenv("BUY_UNIT", 20) # one USD=20 credits 63 | 64 | RATE_LIMIT = os.getenv("RATE_LIMIT", 20) 65 | 66 | SS_YOUTUBE = os.getenv("SS_YOUTUBE", "https://ytdlbot.dmesg.app?token=123456") 67 | RCLONE_PATH = os.getenv("RCLONE") 68 | -------------------------------------------------------------------------------- /constant.py: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python3 2 | # coding: utf-8 3 | 4 | # ytdlbot - constant.py 5 | # 8/16/21 16:59 6 | # 7 | 8 | __author__ = "Peyman" 9 | 10 | import os 11 | 12 | from config import ( 13 | AFD_LINK, 14 | COFFEE_LINK, 15 | ENABLE_CELERY, 16 | FREE_DOWNLOAD, 17 | REQUIRED_MEMBERSHIP, 18 | TOKEN_PRICE, 19 | ) 20 | from database import InfluxDB 21 | from utils import get_func_queue 22 | 23 | 24 | class BotText: 25 | start ="🖐به ربات دانلودر خوش آمدید. برای راهنمایی /help را ارسال کنید." 26 | help = f""" 27 | 1. این ربات به درستی در حال اجرا است . اگر کار نمی‌کند، لطفاً چند دقیقه صبر کنید و دوباره لینک را ارسال کنید. 28 | 29 | 4. سورس ربات: https://github.com/Ptechgithub/ytdl 30 | 31 | 💢 دستورات 32 | /start 33 | /help 34 | /settings 35 | /about 36 | """ 37 | 38 | about = "✅️ ربات دانلودر یوتیوب\n\nآدرس گیتهاب:\n https://github.com/Ptechgithub/ytdl" 39 | 40 | buy = f""" 41 | **Terms:** 42 | 1. You can use this service free of charge for up to {FREE_DOWNLOAD} downloads within a 24-hour period, regardless of whether the download is successful or not. 43 | 44 | 2. You can purchase additional download tokens, which will be valid indefinitely. 45 | 46 | 3. I will not gather any personal information, so I won't know how many or which videos you have downloaded. 47 | 48 | 4. Refunds are possible, but you will be responsible for the processing fee charged by the payment provider (Stripe, Buy Me a Coffee, etc.). 49 | 50 | 5. I will record your unique ID after a successful payment, which is usually your payment ID or email address. 51 | 52 | 6. Paid user can change default download mode to Local mode in settings, which is faster. If your used up all your tokens, you will be reset to default mode. 53 | 54 | **Download token price:** 55 | 1. Everyone: {FREE_DOWNLOAD} tokens per 24 hours, free of charge. 56 | 2. 1 USD == {TOKEN_PRICE} tokens, valid indefinitely. 57 | 58 | **Payment option:** 59 | 1. AFDIAN(AliPay, WeChat Pay and PayPal): {AFD_LINK} 60 | 2. Buy me a coffee: {COFFEE_LINK} 61 | 3. Telegram Payment(Stripe), see following invoice. 62 | 63 | **After payment:** 64 | 65 | 1. Afdian: Provide your order number with the /redeem command (e.g., `/redeem 123456`). 66 | 2. Buy Me a Coffee: Provide your email with the /redeem command (e.g., `/redeem some@one.com`). **Use different email each time.** 67 | 3. Telegram Payment: Your payment will be automatically activated. 68 | 69 | Want to buy more token at once? Let's say 100? Here you go! `/buy 123` 70 | """ 71 | private = "This bot is for private use" 72 | membership_require = f"You need to join this group or channel to use this bot\n\nhttps://t.me/{REQUIRED_MEMBERSHIP}" 73 | 74 | settings = """ 75 | لطفاً فرمت و کیفیت مورد نظر برای ویدیوی خود را انتخاب کنید. توجه داشته باشید که این تنظیمات فقط برای ویدیوهای یوتیوب اعمال می‌شوند. 76 | 77 | کیفیت بالا توصیه می‌شود. کیفیت متوسط معادل 720P است، در حالی که کیفیت پایین معادل 480P می‌باشد. 78 | 79 | لطفاً به یاد داشته باشید که اگر انتخاب کنید ویدیو را به عنوان یک سند ارسال کنید، امکان استریم آن وجود ندارد. 80 | 81 | تنظیمات فعلی شما: 82 | کیفیت ویدیو: {0} 83 | فرمت ارسال: {1} 84 | """ 85 | custom_text = os.getenv("CUSTOM_TEXT", "") 86 | 87 | @staticmethod 88 | def get_receive_link_text() -> str: 89 | reserved = get_func_queue("reserved") 90 | if ENABLE_CELERY and reserved: 91 | text = f"درخواست بیش از حد مجاز. درخواست شما در لیست انتظار قرار گرفت. {reserved}." 92 | else: 93 | text = "درخواست شما به لیست انتظار اضافه شد.\n در حال پردازش لطفا صبور باشید🌹...\n\n" 94 | 95 | return text 96 | 97 | @staticmethod 98 | def ping_worker() -> str: 99 | from tasks import app as celery_app 100 | 101 | workers = InfluxDB().extract_dashboard_data() 102 | # [{'celery@BennyのMBP': 'abc'}, {'celery@BennyのMBP': 'abc'}] 103 | response = celery_app.control.broadcast("ping_revision", reply=True) 104 | revision = {} 105 | for item in response: 106 | revision.update(item) 107 | 108 | text = "" 109 | for worker in workers: 110 | fields = worker["fields"] 111 | hostname = worker["tags"]["hostname"] 112 | status = {True: "✅"}.get(fields["status"], "❌") 113 | active = fields["active"] 114 | load = "{},{},{}".format(fields["load1"], fields["load5"], fields["load15"]) 115 | rev = revision.get(hostname, "") 116 | text += f"{status}{hostname} **{active}** {load} {rev}\n" 117 | 118 | return text 119 | -------------------------------------------------------------------------------- /database.py: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python3 2 | # coding: utf-8 3 | 4 | # ytdlbot - database.py 5 | # 12/7/21 16:57 6 | # 7 | 8 | __author__ = "Peyman" 9 | 10 | import base64 11 | import contextlib 12 | import datetime 13 | import logging 14 | import os 15 | import re 16 | import sqlite3 17 | import subprocess 18 | import time 19 | from io import BytesIO 20 | 21 | import fakeredis 22 | import pymysql 23 | import redis 24 | import requests 25 | from beautifultable import BeautifulTable 26 | from influxdb import InfluxDBClient 27 | 28 | from config import MYSQL_HOST, MYSQL_PASS, MYSQL_USER, REDIS 29 | 30 | init_con = sqlite3.connect(":memory:", check_same_thread=False) 31 | 32 | 33 | class FakeMySQL: 34 | @staticmethod 35 | def cursor() -> "Cursor": 36 | return Cursor() 37 | 38 | def commit(self): 39 | pass 40 | 41 | def close(self): 42 | pass 43 | 44 | def ping(self, reconnect): 45 | pass 46 | 47 | 48 | class Cursor: 49 | def __init__(self): 50 | self.con = init_con 51 | self.cur = self.con.cursor() 52 | 53 | def execute(self, *args, **kwargs): 54 | sql = self.sub(args[0]) 55 | new_args = (sql,) + args[1:] 56 | with contextlib.suppress(sqlite3.OperationalError): 57 | return self.cur.execute(*new_args, **kwargs) 58 | 59 | def fetchall(self): 60 | return self.cur.fetchall() 61 | 62 | def fetchone(self): 63 | return self.cur.fetchone() 64 | 65 | @staticmethod 66 | def sub(sql): 67 | sql = re.sub(r"CHARSET.*|charset.*", "", sql, re.IGNORECASE) 68 | sql = sql.replace("%s", "?") 69 | return sql 70 | 71 | 72 | class Redis: 73 | def __init__(self): 74 | try: 75 | self.r = redis.StrictRedis(host=REDIS, db=0, decode_responses=True) 76 | self.r.ping() 77 | except redis.RedisError: 78 | self.r = fakeredis.FakeStrictRedis(host=REDIS, db=0, decode_responses=True) 79 | 80 | db_banner = "=" * 20 + "DB data" + "=" * 20 81 | quota_banner = "=" * 20 + "Celery" + "=" * 20 82 | metrics_banner = "=" * 20 + "Metrics" + "=" * 20 83 | usage_banner = "=" * 20 + "Usage" + "=" * 20 84 | vnstat_banner = "=" * 20 + "vnstat" + "=" * 20 85 | self.final_text = f""" 86 | {db_banner} 87 | %s 88 | 89 | 90 | {vnstat_banner} 91 | %s 92 | 93 | 94 | {quota_banner} 95 | %s 96 | 97 | 98 | {metrics_banner} 99 | %s 100 | 101 | 102 | {usage_banner} 103 | %s 104 | """ 105 | super().__init__() 106 | 107 | def __del__(self): 108 | self.r.close() 109 | 110 | def update_metrics(self, metrics: str): 111 | logging.info(f"Setting metrics: {metrics}") 112 | all_ = f"all_{metrics}" 113 | today = f"today_{metrics}" 114 | self.r.hincrby("metrics", all_) 115 | self.r.hincrby("metrics", today) 116 | 117 | @staticmethod 118 | def generate_table(header, all_data: list): 119 | table = BeautifulTable() 120 | for data in all_data: 121 | table.rows.append(data) 122 | table.columns.header = header 123 | table.rows.header = [str(i) for i in range(1, len(all_data) + 1)] 124 | return table 125 | 126 | def show_usage(self): 127 | db = MySQL() 128 | db.cur.execute("select user_id,payment_amount,old_user,token from payment") 129 | data = db.cur.fetchall() 130 | fd = [] 131 | for item in data: 132 | fd.append([item[0], item[1], item[2], item[3]]) 133 | db_text = self.generate_table(["ID", "pay amount", "old user", "token"], fd) 134 | 135 | fd = [] 136 | hash_keys = self.r.hgetall("metrics") 137 | for key, value in hash_keys.items(): 138 | if re.findall(r"^today|all", key): 139 | fd.append([key, value]) 140 | fd.sort(key=lambda x: x[0]) 141 | metrics_text = self.generate_table(["name", "count"], fd) 142 | 143 | fd = [] 144 | for key, value in hash_keys.items(): 145 | if re.findall(r"\d+", key): 146 | fd.append([key, value]) 147 | fd.sort(key=lambda x: int(x[-1]), reverse=True) 148 | usage_text = self.generate_table(["UserID", "count"], fd) 149 | 150 | worker_data = InfluxDB.get_worker_data() 151 | fd = [] 152 | for item in worker_data["data"]: 153 | fd.append( 154 | [ 155 | item.get("hostname", 0), 156 | item.get("status", 0), 157 | item.get("active", 0), 158 | item.get("processed", 0), 159 | item.get("task-failed", 0), 160 | item.get("task-succeeded", 0), 161 | ",".join(str(i) for i in item.get("loadavg", [])), 162 | ] 163 | ) 164 | 165 | worker_text = self.generate_table( 166 | ["worker name", "status", "active", "processed", "failed", "succeeded", "Load Average"], fd 167 | ) 168 | 169 | # vnstat 170 | if os.uname().sysname == "Darwin": 171 | cmd = "/opt/homebrew/bin/vnstat -i en0".split() 172 | else: 173 | cmd = "/usr/bin/vnstat -i eth0".split() 174 | vnstat_text = subprocess.check_output(cmd).decode("u8") 175 | return self.final_text % (db_text, vnstat_text, worker_text, metrics_text, usage_text) 176 | 177 | def reset_today(self): 178 | pairs = self.r.hgetall("metrics") 179 | for k in pairs: 180 | if k.startswith("today"): 181 | self.r.hdel("metrics", k) 182 | 183 | def user_count(self, user_id): 184 | self.r.hincrby("metrics", user_id) 185 | 186 | def generate_file(self): 187 | text = self.show_usage() 188 | file = BytesIO() 189 | file.write(text.encode("u8")) 190 | date = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time())) 191 | file.name = f"{date}.txt" 192 | return file 193 | 194 | def add_send_cache(self, unique: str, file_id: str): 195 | self.r.hset("cache", unique, file_id) 196 | 197 | def get_send_cache(self, unique) -> str: 198 | return self.r.hget("cache", unique) 199 | 200 | def del_send_cache(self, unique): 201 | return self.r.hdel("cache", unique) 202 | 203 | 204 | class MySQL: 205 | vip_sql = """ 206 | CREATE TABLE if not exists `payment` 207 | ( 208 | `user_id` bigint NOT NULL, 209 | `payment_amount` float DEFAULT NULL, 210 | `payment_id` varchar(256) DEFAULT NULL, 211 | `old_user` tinyint(1) DEFAULT NULL, 212 | `token` int DEFAULT NULL, 213 | UNIQUE KEY `payment_id` (`payment_id`) 214 | ) CHARSET = utf8mb4 215 | """ 216 | 217 | settings_sql = """ 218 | create table if not exists settings 219 | ( 220 | user_id bigint not null, 221 | resolution varchar(128) null, 222 | method varchar(64) null, 223 | mode varchar(32) default 'Celery' null, 224 | constraint settings_pk 225 | primary key (user_id) 226 | ); 227 | """ 228 | 229 | channel_sql = """ 230 | create table if not exists channel 231 | ( 232 | link varchar(256) null, 233 | title varchar(256) null, 234 | description text null, 235 | channel_id varchar(256), 236 | playlist varchar(256) null, 237 | latest_video varchar(256) null, 238 | constraint channel_pk 239 | primary key (channel_id) 240 | ) CHARSET=utf8mb4; 241 | """ 242 | 243 | subscribe_sql = """ 244 | create table if not exists subscribe 245 | ( 246 | user_id bigint null, 247 | channel_id varchar(256) null, 248 | is_valid boolean default 1 null 249 | ) CHARSET=utf8mb4; 250 | """ 251 | 252 | def __init__(self): 253 | try: 254 | self.con = pymysql.connect( 255 | host=MYSQL_HOST, user=MYSQL_USER, passwd=MYSQL_PASS, db="ytdl", charset="utf8mb4" 256 | ) 257 | except pymysql.err.OperationalError: 258 | self.con = FakeMySQL() 259 | 260 | self.con.ping(reconnect=True) 261 | self.cur = self.con.cursor() 262 | self.init_db() 263 | super().__init__() 264 | 265 | def init_db(self): 266 | self.cur.execute(self.vip_sql) 267 | self.cur.execute(self.settings_sql) 268 | self.cur.execute(self.channel_sql) 269 | self.cur.execute(self.subscribe_sql) 270 | self.con.commit() 271 | 272 | def __del__(self): 273 | self.con.close() 274 | 275 | def get_user_settings(self, user_id: int) -> tuple: 276 | cur = self.con.cursor() 277 | cur.execute("SELECT * FROM settings WHERE user_id = %s", (user_id,)) 278 | data = cur.fetchone() 279 | if data is None: 280 | return 100, "high", "video", "Celery" 281 | return data 282 | 283 | def set_user_settings(self, user_id: int, field: str, value: str): 284 | cur = self.con.cursor() 285 | cur.execute("SELECT * FROM settings WHERE user_id = %s", (user_id,)) 286 | data = cur.fetchone() 287 | if data is None: 288 | resolution = method = "" 289 | if field == "resolution": 290 | method = "video" 291 | resolution = value 292 | if field == "method": 293 | method = value 294 | resolution = "high" 295 | cur.execute("INSERT INTO settings VALUES (%s,%s,%s,%s)", (user_id, resolution, method, "Celery")) 296 | else: 297 | cur.execute(f"UPDATE settings SET {field} =%s WHERE user_id = %s", (value, user_id)) 298 | self.con.commit() 299 | 300 | 301 | class InfluxDB: 302 | def __init__(self): 303 | self.client = InfluxDBClient(host=os.getenv("INFLUX_HOST", "192.168.7.233"), database="celery") 304 | self.data = None 305 | 306 | def __del__(self): 307 | self.client.close() 308 | 309 | @staticmethod 310 | def get_worker_data() -> dict: 311 | username = os.getenv("FLOWER_USERNAME", "benny") 312 | password = os.getenv("FLOWER_PASSWORD", "123456abc") 313 | token = base64.b64encode(f"{username}:{password}".encode()).decode() 314 | headers = {"Authorization": f"Basic {token}"} 315 | r = requests.get("https://celery.dmesg.app/dashboard?json=1", headers=headers) 316 | if r.status_code != 200: 317 | return dict(data=[]) 318 | return r.json() 319 | 320 | def extract_dashboard_data(self): 321 | self.data = self.get_worker_data() 322 | json_body = [] 323 | for worker in self.data["data"]: 324 | load1, load5, load15 = worker["loadavg"] 325 | t = { 326 | "measurement": "tasks", 327 | "tags": { 328 | "hostname": worker["hostname"], 329 | }, 330 | "time": datetime.datetime.utcnow(), 331 | "fields": { 332 | "task-received": worker.get("task-received", 0), 333 | "task-started": worker.get("task-started", 0), 334 | "task-succeeded": worker.get("task-succeeded", 0), 335 | "task-failed": worker.get("task-failed", 0), 336 | "active": worker.get("active", 0), 337 | "status": worker.get("status", False), 338 | "load1": load1, 339 | "load5": load5, 340 | "load15": load15, 341 | }, 342 | } 343 | json_body.append(t) 344 | return json_body 345 | 346 | def __fill_worker_data(self): 347 | json_body = self.extract_dashboard_data() 348 | self.client.write_points(json_body) 349 | 350 | def __fill_overall_data(self): 351 | active = sum([i["active"] for i in self.data["data"]]) 352 | json_body = [{"measurement": "active", "time": datetime.datetime.utcnow(), "fields": {"active": active}}] 353 | self.client.write_points(json_body) 354 | 355 | def __fill_redis_metrics(self): 356 | json_body = [{"measurement": "metrics", "time": datetime.datetime.utcnow(), "fields": {}}] 357 | r = Redis().r 358 | hash_keys = r.hgetall("metrics") 359 | for key, value in hash_keys.items(): 360 | if re.findall(r"^today", key): 361 | json_body[0]["fields"][key] = int(value) 362 | 363 | self.client.write_points(json_body) 364 | 365 | def collect_data(self): 366 | if os.getenv("INFLUX_HOST") is None: 367 | return 368 | 369 | with contextlib.suppress(Exception): 370 | self.data = self.get_worker_data() 371 | self.__fill_worker_data() 372 | self.__fill_overall_data() 373 | self.__fill_redis_metrics() 374 | logging.debug("InfluxDB data was collected.") 375 | -------------------------------------------------------------------------------- /downloader.py: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python3 2 | # coding: utf-8 3 | 4 | # ytdlbot - downloader.py 5 | # 8/14/21 16:53 6 | # 7 | 8 | __author__ = "Peyman" 9 | 10 | import logging 11 | import os 12 | import pathlib 13 | import random 14 | import re 15 | import subprocess 16 | import time 17 | import traceback 18 | from io import StringIO 19 | from unittest.mock import MagicMock 20 | 21 | import fakeredis 22 | import ffmpeg 23 | import ffpb 24 | import filetype 25 | import requests 26 | import yt_dlp as ytdl 27 | from tqdm import tqdm 28 | 29 | from config import AUDIO_FORMAT, ENABLE_ARIA2, ENABLE_FFMPEG, TG_MAX_SIZE, IPv6, SS_YOUTUBE 30 | from limit import Payment 31 | from utils import adjust_formats, apply_log_formatter, current_time, sizeof_fmt 32 | 33 | r = fakeredis.FakeStrictRedis() 34 | apply_log_formatter() 35 | 36 | 37 | def edit_text(bot_msg, text: str): 38 | key = f"{bot_msg.chat.id}-{bot_msg.message_id}" 39 | # if the key exists, we shouldn't send edit message 40 | if not r.exists(key): 41 | time.sleep(random.random()) 42 | r.set(key, "ok", ex=3) 43 | bot_msg.edit_text(text) 44 | 45 | 46 | def tqdm_progress(desc, total, finished, speed="", eta=""): 47 | def more(title, initial): 48 | if initial: 49 | return f"{title} {initial}" 50 | else: 51 | return "" 52 | 53 | f = StringIO() 54 | tqdm( 55 | total=total, 56 | initial=finished, 57 | file=f, 58 | ascii=False, 59 | unit_scale=True, 60 | ncols=30, 61 | bar_format="{l_bar}{bar} |{n_fmt}/{total_fmt} ", 62 | ) 63 | raw_output = f.getvalue() 64 | tqdm_output = raw_output.split("|") 65 | progress = f"`[{tqdm_output[1]}]`" 66 | detail = tqdm_output[2].replace("[A", "") 67 | text = f""" 68 | {desc} 69 | 70 | {progress} 71 | {detail} 72 | {more("Speed:", speed)} 73 | {more("ETA:", eta)} 74 | """ 75 | f.close() 76 | return text 77 | 78 | 79 | def remove_bash_color(text): 80 | return re.sub(r"\u001b|\[0;94m|\u001b\[0m|\[0;32m|\[0m|\[0;33m", "", text) 81 | 82 | 83 | def download_hook(d: dict, bot_msg): 84 | # since we're using celery, server location may be located in different continent. 85 | # Therefore, we can't trigger the hook very often. 86 | # the key is user_id + download_link 87 | original_url = d["info_dict"]["original_url"] 88 | key = f"{bot_msg.chat.id}-{original_url}" 89 | 90 | if d["status"] == "downloading": 91 | downloaded = d.get("downloaded_bytes", 0) 92 | total = d.get("total_bytes") or d.get("total_bytes_estimate", 0) 93 | if total > TG_MAX_SIZE: 94 | raise Exception(f"Your download file size {sizeof_fmt(total)} is too large for Telegram.") 95 | 96 | # percent = remove_bash_color(d.get("_percent_str", "N/A")) 97 | speed = remove_bash_color(d.get("_speed_str", "N/A")) 98 | eta = remove_bash_color(d.get("_eta_str", d.get("eta"))) 99 | text = tqdm_progress("Downloading...", total, downloaded, speed, eta) 100 | edit_text(bot_msg, text) 101 | r.set(key, "ok", ex=5) 102 | 103 | 104 | def upload_hook(current, total, bot_msg): 105 | text = tqdm_progress("Uploading...", total, current) 106 | edit_text(bot_msg, text) 107 | 108 | 109 | def convert_to_mp4(video_paths: list, bot_msg): 110 | default_type = ["video/x-flv", "video/webm"] 111 | # all_converted = [] 112 | for path in video_paths: 113 | # if we can't guess file type, we assume it's video/mp4 114 | mime = getattr(filetype.guess(path), "mime", "video/mp4") 115 | if mime in default_type: 116 | if not can_convert_mp4(path, bot_msg.chat.id): 117 | logging.warning("Conversion abort for %s", bot_msg.chat.id) 118 | bot_msg._client.send_message(bot_msg.chat.id, "Can't convert your video. ffmpeg has been disabled.") 119 | break 120 | edit_text(bot_msg, f"{current_time()}: Converting {path.name} to mp4. Please wait.") 121 | new_file_path = path.with_suffix(".mp4") 122 | logging.info("Detected %s, converting to mp4...", mime) 123 | run_ffmpeg_progressbar(["ffmpeg", "-y", "-i", path, new_file_path], bot_msg) 124 | index = video_paths.index(path) 125 | video_paths[index] = new_file_path 126 | 127 | 128 | class ProgressBar(tqdm): 129 | b = None 130 | 131 | def __init__(self, *args, **kwargs): 132 | super().__init__(*args, **kwargs) 133 | self.bot_msg = self.b 134 | 135 | def update(self, n=1): 136 | super().update(n) 137 | t = tqdm_progress("Converting...", self.total, self.n) 138 | edit_text(self.bot_msg, t) 139 | 140 | 141 | def run_ffmpeg_progressbar(cmd_list: list, bm): 142 | cmd_list = cmd_list.copy()[1:] 143 | ProgressBar.b = bm 144 | ffpb.main(cmd_list, tqdm=ProgressBar) 145 | 146 | 147 | def can_convert_mp4(video_path, uid): 148 | if not ENABLE_FFMPEG: 149 | return False 150 | return True 151 | 152 | 153 | def ytdl_download(url: str, tempdir: str, bm, **kwargs) -> list: 154 | payment = Payment() 155 | chat_id = bm.chat.id 156 | hijack = kwargs.get("hijack") 157 | output = pathlib.Path(tempdir, "%(title).70s.%(ext)s").as_posix() 158 | ydl_opts = { 159 | "progress_hooks": [lambda d: download_hook(d, bm)], 160 | "outtmpl": output, 161 | "restrictfilenames": False, 162 | "quiet": True, 163 | } 164 | if ENABLE_ARIA2: 165 | ydl_opts["external_downloader"] = "aria2c" 166 | ydl_opts["external_downloader_args"] = [ 167 | "--min-split-size=1M", 168 | "--max-connection-per-server=16", 169 | "--max-concurrent-downloads=16", 170 | "--split=16", 171 | ] 172 | formats = [ 173 | # webm , vp9 and av01 are not streamable on telegram, so we'll extract mp4 and not av01 codec 174 | "bestvideo[ext=mp4][vcodec!*=av01][vcodec!*=vp09]+bestaudio[ext=m4a]/bestvideo+bestaudio", 175 | "bestvideo[vcodec^=avc]+bestaudio[acodec^=mp4a]/best[vcodec^=avc]/best", 176 | None, 177 | ] 178 | adjust_formats(chat_id, url, formats, hijack) 179 | if download_instagram(url, tempdir): 180 | return list(pathlib.Path(tempdir).glob("*")) 181 | 182 | address = ["::", "0.0.0.0"] if IPv6 else [None] 183 | error = None 184 | video_paths = None 185 | for format_ in formats: 186 | ydl_opts["format"] = format_ 187 | for addr in address: 188 | # IPv6 goes first in each format 189 | ydl_opts["source_address"] = addr 190 | try: 191 | logging.info("Downloading for %s with format %s", url, format_) 192 | with ytdl.YoutubeDL(ydl_opts) as ydl: 193 | ydl.download([url]) 194 | video_paths = list(pathlib.Path(tempdir).glob("*")) 195 | break 196 | except Exception: 197 | error = traceback.format_exc() 198 | logging.error("Download failed for %s - %s, try another way", format_, url) 199 | if error is None: 200 | break 201 | 202 | if not video_paths: 203 | raise Exception(error) 204 | 205 | # convert format if necessary 206 | settings = payment.get_user_settings(chat_id) 207 | if settings[2] == "video" or isinstance(settings[2], MagicMock): 208 | # only convert if send type is video 209 | convert_to_mp4(video_paths, bm) 210 | if settings[2] == "audio" or hijack == "bestaudio[ext=m4a]": 211 | convert_audio_format(video_paths, bm) 212 | # split_large_video(video_paths) 213 | return video_paths 214 | 215 | 216 | def convert_audio_format(video_paths: list, bm): 217 | # 1. file is audio, default format 218 | # 2. file is video, default format 219 | # 3. non default format 220 | 221 | for path in video_paths: 222 | streams = ffmpeg.probe(path)["streams"] 223 | if AUDIO_FORMAT is None and len(streams) == 1 and streams[0]["codec_type"] == "audio": 224 | logging.info("%s is audio, default format, no need to convert", path) 225 | elif AUDIO_FORMAT is None and len(streams) >= 2: 226 | logging.info("%s is video, default format, need to extract audio", path) 227 | audio_stream = {"codec_name": "m4a"} 228 | for stream in streams: 229 | if stream["codec_type"] == "audio": 230 | audio_stream = stream 231 | break 232 | ext = audio_stream["codec_name"] 233 | new_path = path.with_suffix(f".{ext}") 234 | run_ffmpeg_progressbar(["ffmpeg", "-y", "-i", path, "-vn", "-acodec", "copy", new_path], bm) 235 | path.unlink() 236 | index = video_paths.index(path) 237 | video_paths[index] = new_path 238 | else: 239 | logging.info("Not default format, converting %s to %s", path, AUDIO_FORMAT) 240 | new_path = path.with_suffix(f".{AUDIO_FORMAT}") 241 | run_ffmpeg_progressbar(["ffmpeg", "-y", "-i", path, new_path], bm) 242 | path.unlink() 243 | index = video_paths.index(path) 244 | video_paths[index] = new_path 245 | 246 | 247 | def download_instagram(url: str, tempdir: str): 248 | if url.startswith("https://www.instagram.com"): 249 | logging.info("Requesting instagram download link for %s", url) 250 | api = SS_YOUTUBE + f"&url={url}" 251 | res = requests.get(api).json() 252 | if isinstance(res, dict): 253 | downloadable = {i["url"]: i["ext"] for i in res["url"]} 254 | else: 255 | downloadable = {i["url"]: i["ext"] for item in res for i in item["url"]} 256 | 257 | for link, ext in downloadable.items(): 258 | save_path = pathlib.Path(tempdir, f"{id(link)}.{ext}") 259 | with open(save_path, "wb") as f: 260 | f.write(requests.get(link, stream=True).content) 261 | # telegram send webp as sticker, so we'll convert it to png 262 | for path in pathlib.Path(tempdir).glob("*.webp"): 263 | logging.info("Converting %s to png", path) 264 | new_path = path.with_suffix(".jpg") 265 | ffmpeg.input(path).output(new_path.as_posix()).run() 266 | path.unlink() 267 | return True 268 | 269 | 270 | def split_large_video(video_paths: list): 271 | original_video = None 272 | split = False 273 | for original_video in video_paths: 274 | size = os.stat(original_video).st_size 275 | if size > TG_MAX_SIZE: 276 | split = True 277 | logging.warning("file is too large %s, splitting...", size) 278 | subprocess.check_output(f"sh split-video.sh {original_video} {TG_MAX_SIZE * 0.95} ".split()) 279 | os.remove(original_video) 280 | 281 | if split and original_video: 282 | return [i for i in pathlib.Path(original_video).parent.glob("*")] 283 | 284 | 285 | if __name__ == "__main__": 286 | a = download_instagram("https://www.instagram.com/p/CrEAz-AI99Y/", "tmp") 287 | print(a) 288 | -------------------------------------------------------------------------------- /flower_tasks.py: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python3 2 | # coding: utf-8 3 | 4 | # ytdlbot - flower_tasks.py 5 | # 1/2/22 10:17 6 | # 7 | 8 | __author__ = "Peyman" 9 | 10 | from celery import Celery 11 | 12 | from config import BROKER 13 | 14 | app = Celery('tasks', broker=BROKER, timezone="Asia/Tehran") 15 | -------------------------------------------------------------------------------- /install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #colors 4 | red='\033[0;31m' 5 | green='\033[0;32m' 6 | yellow='\033[0;33m' 7 | blue='\033[0;34m' 8 | purple='\033[0;35m' 9 | cyan='\033[0;36m' 10 | white='\033[0;37m' 11 | rest='\033[0m' 12 | 13 | detect_distribution() { 14 | # Detect the Linux distribution 15 | local supported_distributions=("ubuntu" "debian" "centos" "fedora") 16 | 17 | if [ -f /etc/os-release ]; then 18 | source /etc/os-release 19 | if [[ "${ID}" = "ubuntu" || "${ID}" = "debian" || "${ID}" = "centos" || "${ID}" = "fedora" ]]; then 20 | PM="apt-get" 21 | [ "${ID}" = "centos" ] && PM="yum" 22 | [ "${ID}" = "fedora" ] && PM="dnf" 23 | else 24 | echo "Unsupported distribution!" 25 | exit 1 26 | fi 27 | else 28 | echo "Unsupported distribution!" 29 | exit 1 30 | fi 31 | } 32 | 33 | check_dependencies() { 34 | detect_distribution 35 | 36 | local dependencies=("git" "python3" "ffmpeg" "aria2" "python3-pip") 37 | 38 | for dep in "${dependencies[@]}"; do 39 | if ! command -v "${dep}" &> /dev/null; then 40 | echo "${dep} is not installed. Installing..." 41 | "${PM}" update && "${PM}" upgrade -y 42 | "${PM}" install "${dep}" -y 43 | 44 | fi 45 | done 46 | } 47 | 48 | inputs() { 49 | clear 50 | 51 | read -p "Please enter Telegram APP_ID: " APP_ID 52 | sed -i "s/APP_ID: int = int(os.getenv(\"APP_ID\", A))/APP_ID: int = int(os.getenv(\"APP_ID\", $APP_ID))/" config.py 53 | 54 | read -p "Please enter Telegtam APP_HASH: " APP_HASH 55 | sed -i "s/APP_HASH = os.getenv(\"APP_HASH\", \"B\")/APP_HASH = os.getenv(\"APP_HASH\", \"$APP_HASH\")/" config.py 56 | 57 | read -p "Please enter Telegram Bot TOKEN: " TOKEN 58 | sed -i "s/TOKEN = os.getenv(\"TOKEN\", \"C\")/TOKEN = os.getenv(\"TOKEN\", \"$TOKEN\")/" config.py 59 | 60 | read -p "Please enter the number of free downloads [default : 20] : " FREE_DOWNLOAD 61 | sed -i "s/FREE_DOWNLOAD = os.getenv(\"FREE_DOWNLOAD\", 20)/FREE_DOWNLOAD = os.getenv(\"FREE_DOWNLOAD\", $FREE_DOWNLOAD)/" config.py 62 | } 63 | 64 | #install 65 | install() { 66 | if ! systemctl is-active --quiet ytdl.service; then 67 | check_dependencies 68 | git clone https://github.com/Ptechgithub/ytdl.git 69 | cd ytdl 70 | pip3 install -r requirements.txt 71 | inputs 72 | service 73 | else 74 | echo "The ytdl service is already installed" 75 | fi 76 | } 77 | 78 | service() { 79 | cat < /etc/systemd/system/ytdl.service 80 | [Unit] 81 | Description=YouTube Downloader Service 82 | After=network.target 83 | 84 | [Service] 85 | WorkingDirectory=/root/ytdl 86 | ExecStart=python3 /root/ytdl/ytdl_bot.py 87 | Restart=always 88 | 89 | [Install] 90 | WantedBy=multi-user.target 91 | EOL 92 | 93 | systemctl daemon-reload 94 | systemctl enable ytdl.service 95 | systemctl start ytdl.service 96 | } 97 | 98 | uninstall() { 99 | if systemctl is-active --quiet ytdl.service; then 100 | systemctl stop ytdl.service 101 | systemctl disable ytdl.service 102 | rm /etc/systemd/system/ytdl.service 103 | systemctl daemon-reload 104 | rm -rf /root/ytdl 105 | else 106 | echo "ytdl is not installed." 107 | fi 108 | } 109 | 110 | check_ytdl_status() { 111 | # Check the status of the ytdl service 112 | if sudo systemctl is-active --quiet ytdl.service; then 113 | echo -e "${yellow}YouTube Downloader is: ${green} [running ✔]${rest}" 114 | else 115 | echo -e "${yellow}ytdl is:${red} [Not running ✗ ]${rest}" 116 | fi 117 | } 118 | 119 | #Termux 120 | install_termux() { 121 | bash <(curl -fsSL https://raw.githubusercontent.com/Ptechgithub/ytdl/main/termux/install.sh) 122 | } 123 | 124 | # Main menu 125 | clear 126 | echo -e "${cyan}By --> Peyman * Github.com/Ptechgithub * ${rest}" 127 | check_ytdl_status 128 | echo -e "${yellow} ----YouTube downloader Telegram bot---- ${rest}" 129 | echo -e "${green}1) Install on server${rest}" 130 | echo -e "${red}2) Uninstall${rest}" 131 | echo -e "${yellow} ----------------------------------------- ${rest}" 132 | echo -e "${green}3) Install on Termux${rest}" 133 | echo -e "${yellow} ----------------------------------------- ${rest}" 134 | echo -e "${yellow}0) Exit${rest}" 135 | read -p "Please choose: " choice 136 | 137 | case $choice in 138 | 1) 139 | install 140 | ;; 141 | 2) 142 | uninstall 143 | ;; 144 | 3) 145 | install_termux 146 | ;; 147 | 0) 148 | exit 149 | ;; 150 | *) 151 | echo "Invalid choice. Please try again." 152 | ;; 153 | esac 154 | -------------------------------------------------------------------------------- /limit.py: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python3 2 | # coding: utf-8 3 | 4 | # ytdlbot - limit.py 5 | # 8/15/21 18:23 6 | # 7 | 8 | __author__ = "Peyman" 9 | 10 | import hashlib 11 | import logging 12 | import time 13 | 14 | import requests 15 | 16 | from config import ( 17 | AFD_TOKEN, 18 | AFD_USER_ID, 19 | COFFEE_TOKEN, 20 | EXPIRE, 21 | FREE_DOWNLOAD, 22 | OWNER, 23 | TOKEN_PRICE, 24 | ) 25 | from database import MySQL, Redis 26 | from utils import apply_log_formatter, current_time 27 | 28 | apply_log_formatter() 29 | 30 | 31 | class BuyMeACoffee: 32 | def __init__(self): 33 | self._token = COFFEE_TOKEN 34 | self._url = "https://developers.buymeacoffee.com/api/v1/supporters" 35 | self._data = [] 36 | 37 | def _get_data(self, url): 38 | d = requests.get(url, headers={"Authorization": f"Bearer {self._token}"}).json() 39 | self._data.extend(d["data"]) 40 | next_page = d["next_page_url"] 41 | if next_page: 42 | self._get_data(next_page) 43 | 44 | def _get_bmac_status(self, email: str) -> dict: 45 | self._get_data(self._url) 46 | for user in self._data: 47 | if user["payer_email"] == email or user["support_email"] == email: 48 | return user 49 | return {} 50 | 51 | def get_user_payment(self, email: str) -> (int, "float", str): 52 | order = self._get_bmac_status(email) 53 | price = float(order.get("support_coffee_price", 0)) 54 | cups = float(order.get("support_coffees", 1)) 55 | amount = price * cups 56 | return amount, email 57 | 58 | 59 | class Afdian: 60 | def __init__(self): 61 | self._token = AFD_TOKEN 62 | self._user_id = AFD_USER_ID 63 | self._url = "https://afdian.net/api/open/query-order" 64 | 65 | def _generate_signature(self): 66 | data = { 67 | "user_id": self._user_id, 68 | "params": '{"x":0}', 69 | "ts": int(time.time()), 70 | } 71 | sign_text = "{token}params{params}ts{ts}user_id{user_id}".format( 72 | token=self._token, params=data["params"], ts=data["ts"], user_id=data["user_id"] 73 | ) 74 | 75 | md5 = hashlib.md5(sign_text.encode("u8")) 76 | md5 = md5.hexdigest() 77 | data["sign"] = md5 78 | 79 | return data 80 | 81 | def _get_afdian_status(self, trade_no: str) -> dict: 82 | req_data = self._generate_signature() 83 | data = requests.post(self._url, json=req_data).json() 84 | # latest 50 85 | for order in data["data"]["list"]: 86 | if order["out_trade_no"] == trade_no: 87 | return order 88 | 89 | return {} 90 | 91 | def get_user_payment(self, trade_no: str) -> (int, float, str): 92 | order = self._get_afdian_status(trade_no) 93 | amount = float(order.get("show_amount", 0)) 94 | # convert to USD 95 | return amount / 7, trade_no 96 | 97 | 98 | class Payment(Redis, MySQL): 99 | def check_old_user(self, user_id: int) -> tuple: 100 | self.cur.execute("SELECT * FROM payment WHERE user_id=%s AND old_user=1", (user_id,)) 101 | data = self.cur.fetchone() 102 | return data 103 | 104 | def get_pay_token(self, user_id: int) -> int: 105 | self.cur.execute("SELECT token, old_user FROM payment WHERE user_id=%s", (user_id,)) 106 | data = self.cur.fetchall() or [(0, False)] 107 | number = sum([i[0] for i in data if i[0]]) 108 | if number == 0 and data[0][1] != 1: 109 | # not old user, no token 110 | logging.warning("User %s has no token, set download mode to Celery", user_id) 111 | # change download mode to Celery 112 | self.set_user_settings(user_id, "mode", "Celery") 113 | return number 114 | 115 | def get_free_token(self, user_id: int) -> int: 116 | if self.r.exists(user_id): 117 | return int(self.r.get(user_id)) 118 | else: 119 | # set and return 120 | self.r.set(user_id, FREE_DOWNLOAD, ex=EXPIRE) 121 | return FREE_DOWNLOAD 122 | 123 | def get_token(self, user_id: int): 124 | ttl = self.r.ttl(user_id) 125 | return self.get_free_token(user_id), self.get_pay_token(user_id), current_time(time.time() + ttl) 126 | 127 | def use_free_token(self, user_id: int): 128 | if self.r.exists(user_id): 129 | self.r.decr(user_id, 1) 130 | else: 131 | # first time download 132 | self.r.set(user_id, 5 - 1, ex=EXPIRE) 133 | 134 | def use_pay_token(self, user_id: int): 135 | # a user may pay multiple times, so we'll need to filter the first payment with valid token 136 | self.cur.execute("SELECT payment_id FROM payment WHERE user_id=%s AND token>0", (user_id,)) 137 | data = self.cur.fetchone() 138 | payment_id = data[0] 139 | logging.info("User %s use pay token with payment_id %s", user_id, payment_id) 140 | self.cur.execute("UPDATE payment SET token=token-1 WHERE payment_id=%s", (payment_id,)) 141 | self.con.commit() 142 | 143 | def use_token(self, user_id: int): 144 | free = self.get_free_token(user_id) 145 | if free > 0: 146 | self.use_free_token(user_id) 147 | else: 148 | self.use_pay_token(user_id) 149 | 150 | def add_pay_user(self, pay_data: list): 151 | self.cur.execute("INSERT INTO payment VALUES (%s,%s,%s,%s,%s)", pay_data) 152 | self.con.commit() 153 | 154 | def verify_payment(self, user_id: int, unique: str) -> str: 155 | pay = BuyMeACoffee() if "@" in unique else Afdian() 156 | self.cur.execute("SELECT * FROM payment WHERE payment_id=%s ", (unique,)) 157 | data = self.cur.fetchone() 158 | if data: 159 | # TODO what if a user pay twice with the same email address? 160 | return ( 161 | f"Failed. Payment has been verified by other users. Please contact @{OWNER} if you have any questions." 162 | ) 163 | 164 | amount, pay_id = pay.get_user_payment(unique) 165 | logging.info("User %s paid %s, identifier is %s", user_id, amount, unique) 166 | # amount is already in USD 167 | if amount == 0: 168 | return "Payment not found. Please check your payment ID or email address" 169 | self.add_pay_user([user_id, amount, pay_id, 0, amount * TOKEN_PRICE]) 170 | return "Thanks! Your payment has been verified. /start to get your token details" 171 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pyrogram==1.4.16 2 | tgcrypto==1.2.5 3 | yt-dlp==2023.7.6 4 | APScheduler==3.10.4 5 | beautifultable==1.1.0 6 | ffmpeg-python==0.2.0 7 | PyMySQL==1.1.0 8 | celery==5.3.1 9 | filetype==1.2.0 10 | flower==2.0.1 11 | psutil==5.9.5 12 | influxdb==5.3.1 13 | beautifulsoup4==4.12.2 14 | fakeredis==2.18.0 15 | supervisor==4.2.5 16 | tgbot-ping==1.0.7 17 | redis==5.0.0 18 | requests==2.31.0 19 | tqdm==4.66.1 20 | requests-toolbelt==1.0.0 21 | ffpb==0.4.1 22 | youtube-search-python==1.6.6 23 | token-bucket==0.3.0 24 | coloredlogs==15.0.1 25 | -------------------------------------------------------------------------------- /split-video.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Short script to split videos by filesize using ffmpeg by LukeLR 3 | 4 | if [ $# -ne 2 ]; then 5 | echo 'Illegal number of parameters. Needs 2 parameters:' 6 | echo 'Usage:' 7 | echo './split-video.sh FILE SIZELIMIT "FFMPEG_ARGS' 8 | echo 9 | echo 'Parameters:' 10 | echo ' - FILE: Name of the video file to split' 11 | echo ' - SIZELIMIT: Maximum file size of each part (in bytes)' 12 | echo ' - FFMPEG_ARGS: Additional arguments to pass to each ffmpeg-call' 13 | echo ' (video format and quality options etc.)' 14 | exit 1 15 | fi 16 | 17 | FILE="$1" 18 | SIZELIMIT="$2" 19 | FFMPEG_ARGS="$3" 20 | 21 | # Duration of the source video 22 | DURATION=$(ffprobe -i "$FILE" -show_entries format=duration -v quiet -of default=noprint_wrappers=1:nokey=1|cut -d. -f1) 23 | 24 | # Duration that has been encoded so far 25 | CUR_DURATION=0 26 | 27 | # Filename of the source video (without extension) 28 | BASENAME="${FILE%.*}" 29 | 30 | # Extension for the video parts 31 | #EXTENSION="${FILE##*.}" 32 | EXTENSION="mp4" 33 | 34 | # Number of the current video part 35 | i=1 36 | 37 | # Filename of the next video part 38 | NEXTFILENAME="$BASENAME-$i.$EXTENSION" 39 | 40 | echo "Duration of source video: $DURATION" 41 | 42 | # Until the duration of all partial videos has reached the duration of the source video 43 | while [[ $CUR_DURATION -lt $DURATION ]]; do 44 | # Encode next part 45 | echo ffmpeg -i "$FILE" -ss "$CUR_DURATION" -fs "$SIZELIMIT" $FFMPEG_ARGS "$NEXTFILENAME" 46 | ffmpeg -ss "$CUR_DURATION" -i "$FILE" -fs "$SIZELIMIT" $FFMPEG_ARGS "$NEXTFILENAME" 47 | 48 | # Duration of the new part 49 | NEW_DURATION=$(ffprobe -i "$NEXTFILENAME" -show_entries format=duration -v quiet -of default=noprint_wrappers=1:nokey=1|cut -d. -f1) 50 | 51 | # Total duration encoded so far 52 | CUR_DURATION=$((CUR_DURATION + NEW_DURATION)) 53 | 54 | i=$((i + 1)) 55 | 56 | echo "Duration of $NEXTFILENAME: $NEW_DURATION" 57 | echo "Part No. $i starts at $CUR_DURATION" 58 | 59 | NEXTFILENAME="$BASENAME-$i.$EXTENSION" 60 | done -------------------------------------------------------------------------------- /tasks.py: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python3 2 | # coding: utf-8 3 | 4 | # ytdlbot - tasks.py 5 | # 12/29/21 14:57 6 | # 7 | 8 | __author__ = "Peyman" 9 | 10 | import logging 11 | import math 12 | import os 13 | import pathlib 14 | import random 15 | import re 16 | import shutil 17 | import subprocess 18 | import tempfile 19 | import threading 20 | import time 21 | import traceback 22 | import typing 23 | from hashlib import md5 24 | from urllib.parse import quote_plus 25 | 26 | import filetype 27 | import psutil 28 | import pyrogram.errors 29 | import requests 30 | from apscheduler.schedulers.background import BackgroundScheduler 31 | from celery import Celery 32 | from celery.worker.control import Panel 33 | from pyrogram import Client, idle, types 34 | from pyrogram.types import InlineKeyboardButton, InlineKeyboardMarkup, Message 35 | from requests_toolbelt import MultipartEncoder, MultipartEncoderMonitor 36 | 37 | from channel import Channel 38 | from client_init import create_app 39 | from config import ( 40 | ARCHIVE_ID, 41 | BROKER, 42 | ENABLE_CELERY, 43 | ENABLE_QUEUE, 44 | ENABLE_VIP, 45 | OWNER, 46 | RCLONE_PATH, 47 | RATE_LIMIT, 48 | WORKERS, 49 | ) 50 | from constant import BotText 51 | from database import Redis 52 | from downloader import edit_text, tqdm_progress, upload_hook, ytdl_download 53 | from limit import Payment 54 | from utils import ( 55 | apply_log_formatter, 56 | auto_restart, 57 | customize_logger, 58 | get_metadata, 59 | get_revision, 60 | sizeof_fmt, 61 | ) 62 | 63 | customize_logger(["pyrogram.client", "pyrogram.session.session", "pyrogram.connection.connection"]) 64 | apply_log_formatter() 65 | bot_text = BotText() 66 | logging.getLogger("apscheduler.executors.default").propagate = False 67 | 68 | # celery -A tasks worker --loglevel=info --pool=solo 69 | # app = Celery('celery', broker=BROKER, accept_content=['pickle'], task_serializer='pickle') 70 | app = Celery("tasks", broker=BROKER) 71 | redis = Redis() 72 | channel = Channel() 73 | 74 | session = "ytdl-celery" 75 | celery_client = create_app(session) 76 | 77 | 78 | def get_messages(chat_id, message_id): 79 | try: 80 | return celery_client.get_messages(chat_id, message_id) 81 | except ConnectionError as e: 82 | logging.critical("WTH!!! %s", e) 83 | celery_client.start() 84 | return celery_client.get_messages(chat_id, message_id) 85 | 86 | 87 | @app.task(rate_limit=f"{RATE_LIMIT}/m") 88 | def ytdl_download_task(chat_id, message_id, url: str): 89 | logging.info("YouTube celery tasks started for %s", url) 90 | bot_msg = get_messages(chat_id, message_id) 91 | ytdl_normal_download(celery_client, bot_msg, url) 92 | logging.info("YouTube celery tasks ended.") 93 | 94 | 95 | @app.task() 96 | def audio_task(chat_id, message_id): 97 | logging.info("Audio celery tasks started for %s-%s", chat_id, message_id) 98 | bot_msg = get_messages(chat_id, message_id) 99 | normal_audio(celery_client, bot_msg) 100 | logging.info("Audio celery tasks ended.") 101 | 102 | 103 | def get_unique_clink(original_url: str, user_id: int): 104 | payment = Payment() 105 | settings = payment.get_user_settings(user_id) 106 | clink = channel.extract_canonical_link(original_url) 107 | try: 108 | # different user may have different resolution settings 109 | unique = "{}?p={}{}".format(clink, *settings[1:]) 110 | except IndexError: 111 | unique = clink 112 | return unique 113 | 114 | 115 | @app.task() 116 | def direct_download_task(chat_id, message_id, url): 117 | logging.info("Direct download celery tasks started for %s", url) 118 | bot_msg = get_messages(chat_id, message_id) 119 | direct_normal_download(celery_client, bot_msg, url) 120 | logging.info("Direct download celery tasks ended.") 121 | 122 | 123 | def forward_video(client, bot_msg, url: str): 124 | chat_id = bot_msg.chat.id 125 | unique = get_unique_clink(url, chat_id) 126 | cached_fid = redis.get_send_cache(unique) 127 | if not cached_fid: 128 | redis.update_metrics("cache_miss") 129 | return False 130 | 131 | res_msg: "Message" = upload_processor(client, bot_msg, url, cached_fid) 132 | obj = res_msg.document or res_msg.video or res_msg.audio or res_msg.animation or res_msg.photo 133 | 134 | caption, _ = gen_cap(bot_msg, url, obj) 135 | res_msg.edit_text(caption, reply_markup=gen_video_markup()) 136 | bot_msg.edit_text(f"دانلود با موفقیت انجام شد!✅✅✅") 137 | redis.update_metrics("cache_hit") 138 | return True 139 | 140 | 141 | def ytdl_download_entrance(client: Client, bot_msg: types.Message, url: str, mode=None): 142 | payment = Payment() 143 | chat_id = bot_msg.chat.id 144 | try: 145 | if forward_video(client, bot_msg, url): 146 | return 147 | mode = mode or payment.get_user_settings(chat_id)[-1] 148 | if ENABLE_CELERY and mode in [None, "Celery"]: 149 | async_task(ytdl_download_task, chat_id, bot_msg.message_id, url) 150 | # ytdl_download_task.delay(chat_id, bot_msg.message_id, url) 151 | else: 152 | ytdl_normal_download(client, bot_msg, url) 153 | except Exception as e: 154 | logging.error("دانلود ناموفق بود %s, error: %s", url, e) 155 | bot_msg.edit_text(f"دانلود ناموفق!❌\n\n`{traceback.format_exc()[0:4000]}`", disable_web_page_preview=True) 156 | 157 | 158 | def direct_download_entrance(client: Client, bot_msg: typing.Union[types.Message, typing.Coroutine], url: str): 159 | if ENABLE_CELERY: 160 | direct_normal_download(client, bot_msg, url) 161 | # direct_download_task.delay(bot_msg.chat.id, bot_msg.message_id, url) 162 | else: 163 | direct_normal_download(client, bot_msg, url) 164 | 165 | 166 | def audio_entrance(client, bot_msg): 167 | if ENABLE_CELERY: 168 | async_task(audio_task, bot_msg.chat.id, bot_msg.message_id) 169 | # audio_task.delay(bot_msg.chat.id, bot_msg.message_id) 170 | else: 171 | normal_audio(client, bot_msg) 172 | 173 | 174 | def direct_normal_download(client: Client, bot_msg: typing.Union[types.Message, typing.Coroutine], url: str): 175 | chat_id = bot_msg.chat.id 176 | headers = { 177 | "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36" 178 | } 179 | length = 0 180 | 181 | req = None 182 | try: 183 | req = requests.get(url, headers=headers, stream=True) 184 | length = int(req.headers.get("content-length")) 185 | filename = re.findall("filename=(.+)", req.headers.get("content-disposition"))[0] 186 | except TypeError: 187 | filename = getattr(req, "url", "").rsplit("/")[-1] 188 | except Exception as e: 189 | bot_msg.edit_text(f"دانلود ناموف!❌\n\n```{e}```", disable_web_page_preview=True) 190 | return 191 | 192 | if not filename: 193 | filename = quote_plus(url) 194 | 195 | with tempfile.TemporaryDirectory(prefix="ytdl-") as f: 196 | filepath = f"{f}/{filename}" 197 | # consume the req.content 198 | downloaded = 0 199 | for chunk in req.iter_content(1024 * 1024): 200 | text = tqdm_progress("Downloading...", length, downloaded) 201 | edit_text(bot_msg, text) 202 | with open(filepath, "ab") as fp: 203 | fp.write(chunk) 204 | downloaded += len(chunk) 205 | logging.info("Downloaded file %s", filename) 206 | st_size = os.stat(filepath).st_size 207 | 208 | client.send_chat_action(chat_id, "upload_document") 209 | client.send_document( 210 | bot_msg.chat.id, 211 | filepath, 212 | caption=f"filesize: {sizeof_fmt(st_size)}", 213 | progress=upload_hook, 214 | progress_args=(bot_msg,), 215 | ) 216 | bot_msg.edit_text("داتلود با موفقیت انجام شد!✅") 217 | 218 | 219 | def normal_audio(client: Client, bot_msg: typing.Union[types.Message, typing.Coroutine]): 220 | chat_id = bot_msg.chat.id 221 | # fn = getattr(bot_msg.video, "file_name", None) or getattr(bot_msg.document, "file_name", None) 222 | status_msg: typing.Union[types.Message, typing.Coroutine] = bot_msg.reply_text( 223 | "در حال تبدیل به صدا...لطفا صبور باشید. صبوری چیز خوبیه🌹", quote=True 224 | ) 225 | orig_url: str = re.findall(r"https?://.*", bot_msg.caption)[0] 226 | with tempfile.TemporaryDirectory(prefix="ytdl-") as tmp: 227 | client.send_chat_action(chat_id, "record_audio") 228 | # just try to download the audio using yt-dlp 229 | filepath = ytdl_download(orig_url, tmp, status_msg, hijack="bestaudio[ext=m4a]") 230 | status_msg.edit_text("در حال ارسال صدا...") 231 | client.send_chat_action(chat_id, "upload_audio") 232 | for f in filepath: 233 | client.send_audio(chat_id, f) 234 | status_msg.edit_text("✅ تبدیل کامل شد.") 235 | Redis().update_metrics("audio_success") 236 | 237 | 238 | def get_dl_source(): 239 | worker_name = os.getenv("WORKER_NAME") 240 | if worker_name: 241 | return f"Downloaded by {worker_name}" 242 | return "" 243 | 244 | 245 | def upload_transfer_sh(bm, paths: list) -> str: 246 | d = {p.name: (md5(p.name.encode("utf8")).hexdigest() + p.suffix, p.open("rb")) for p in paths} 247 | monitor = MultipartEncoderMonitor(MultipartEncoder(fields=d), lambda x: upload_hook(x.bytes_read, x.len, bm)) 248 | headers = {"Content-Type": monitor.content_type} 249 | try: 250 | req = requests.post("https://transfer.sh", data=monitor, headers=headers) 251 | bm.edit_text(f"دانلود بادموفقیت انجام شد!✅") 252 | return re.sub(r"https://", "\nhttps://", req.text) 253 | except requests.exceptions.RequestException as e: 254 | return f"دانلود ناموفق!❌\n\n```{e}```" 255 | 256 | 257 | def flood_owner_message(client, ex): 258 | client.send_message(OWNER, f"CRITICAL INFO: {ex}") 259 | 260 | 261 | def ytdl_normal_download(client: Client, bot_msg: typing.Union[types.Message, typing.Coroutine], url: str): 262 | chat_id = bot_msg.chat.id 263 | temp_dir = tempfile.TemporaryDirectory(prefix="ytdl-") 264 | 265 | video_paths = ytdl_download(url, temp_dir.name, bot_msg) 266 | logging.info("دانلود تکمیل شد.") 267 | client.send_chat_action(chat_id, "upload_document") 268 | bot_msg.edit_text("دانلود انجام شد. در حال ارسال...") 269 | try: 270 | upload_processor(client, bot_msg, url, video_paths) 271 | except pyrogram.errors.Flood as e: 272 | logging.critical("FloodWait from Telegram: %s", e) 273 | client.send_message( 274 | chat_id, 275 | f"من توسط تلگرام محدود شده ام. ویدیوی شما پس از {e.x} ثانیه می آید. لطفا صبور باشید.", 276 | ) 277 | flood_owner_message(client, e) 278 | time.sleep(e.x) 279 | upload_processor(client, bot_msg, url, video_paths) 280 | 281 | bot_msg.edit_text("دانلود با موفقیت انجام شد!✅") 282 | 283 | # setup rclone environment var to back up the downloaded file 284 | if RCLONE_PATH: 285 | for item in os.listdir(temp_dir.name): 286 | logging.info("Copying %s to %s", item, RCLONE_PATH) 287 | shutil.copy(os.path.join(temp_dir.name, item), RCLONE_PATH) 288 | temp_dir.cleanup() 289 | 290 | 291 | def generate_input_media(file_paths: list, cap: str) -> list: 292 | input_media = [] 293 | for path in file_paths: 294 | mime = filetype.guess_mime(path) 295 | if "video" in mime: 296 | input_media.append(pyrogram.types.InputMediaVideo(media=path)) 297 | elif "image" in mime: 298 | input_media.append(pyrogram.types.InputMediaPhoto(media=path)) 299 | elif "audio" in mime: 300 | input_media.append(pyrogram.types.InputMediaAudio(media=path)) 301 | else: 302 | input_media.append(pyrogram.types.InputMediaDocument(media=path)) 303 | 304 | input_media[0].caption = cap 305 | return input_media 306 | 307 | 308 | def upload_processor(client, bot_msg, url, vp_or_fid: typing.Union[str, list]): 309 | # raise pyrogram.errors.exceptions.FloodWait(13) 310 | # if is str, it's a file id; else it's a list of paths 311 | payment = Payment() 312 | chat_id = bot_msg.chat.id 313 | markup = gen_video_markup() 314 | if isinstance(vp_or_fid, list) and len(vp_or_fid) > 1: 315 | # just generate the first for simplicity, send as media group(2-20) 316 | cap, meta = gen_cap(bot_msg, url, vp_or_fid[0]) 317 | res_msg = client.send_media_group(chat_id, generate_input_media(vp_or_fid, cap)) 318 | # TODO no cache for now 319 | return res_msg[0] 320 | elif isinstance(vp_or_fid, list) and len(vp_or_fid) == 1: 321 | # normal download, just contains one file in video_paths 322 | vp_or_fid = vp_or_fid[0] 323 | cap, meta = gen_cap(bot_msg, url, vp_or_fid) 324 | else: 325 | # just a file id as string 326 | cap, meta = gen_cap(bot_msg, url, vp_or_fid) 327 | 328 | settings = payment.get_user_settings(chat_id) 329 | if ARCHIVE_ID and isinstance(vp_or_fid, pathlib.Path): 330 | chat_id = ARCHIVE_ID 331 | 332 | if settings[2] == "document": 333 | logging.info("Sending as document") 334 | try: 335 | # send as document could be sent as video even if it's a document 336 | res_msg = client.send_document( 337 | chat_id, 338 | vp_or_fid, 339 | caption=cap, 340 | progress=upload_hook, 341 | progress_args=(bot_msg,), 342 | reply_markup=markup, 343 | thumb=meta["thumb"], 344 | force_document=True, 345 | ) 346 | except ValueError: 347 | logging.error("Retry to send as video") 348 | res_msg = client.send_video( 349 | chat_id, 350 | vp_or_fid, 351 | supports_streaming=True, 352 | caption=cap, 353 | progress=upload_hook, 354 | progress_args=(bot_msg,), 355 | reply_markup=markup, 356 | **meta, 357 | ) 358 | elif settings[2] == "audio": 359 | logging.info("Sending as audio") 360 | res_msg = client.send_audio( 361 | chat_id, 362 | vp_or_fid, 363 | caption=cap, 364 | progress=upload_hook, 365 | progress_args=(bot_msg,), 366 | ) 367 | else: 368 | # settings==video 369 | logging.info("Sending as video") 370 | try: 371 | res_msg = client.send_video( 372 | chat_id, 373 | vp_or_fid, 374 | supports_streaming=True, 375 | caption=cap, 376 | progress=upload_hook, 377 | progress_args=(bot_msg,), 378 | reply_markup=markup, 379 | **meta, 380 | ) 381 | except Exception: 382 | # try to send as annimation, photo 383 | try: 384 | logging.warning("Retry to send as animation") 385 | res_msg = client.send_animation( 386 | chat_id, 387 | vp_or_fid, 388 | caption=cap, 389 | progress=upload_hook, 390 | progress_args=(bot_msg,), 391 | reply_markup=markup, 392 | **meta, 393 | ) 394 | except Exception: 395 | # this is likely a photo 396 | logging.warning("Retry to send as photo") 397 | res_msg = client.send_photo( 398 | chat_id, 399 | vp_or_fid, 400 | caption=cap, 401 | progress=upload_hook, 402 | progress_args=(bot_msg,), 403 | ) 404 | 405 | unique = get_unique_clink(url, bot_msg.chat.id) 406 | obj = res_msg.document or res_msg.video or res_msg.audio or res_msg.animation or res_msg.photo 407 | redis.add_send_cache(unique, getattr(obj, "file_id", None)) 408 | redis.update_metrics("video_success") 409 | if ARCHIVE_ID and isinstance(vp_or_fid, pathlib.Path): 410 | client.forward_messages(bot_msg.chat.id, ARCHIVE_ID, res_msg.message_id) 411 | return res_msg 412 | 413 | 414 | def gen_cap(bm, url, video_path): 415 | payment = Payment() 416 | chat_id = bm.chat.id 417 | user = bm.chat 418 | try: 419 | user_info = "@{}({})-{}".format(user.username or "N/A", user.first_name or "" + user.last_name or "", user.id) 420 | except Exception: 421 | user_info = "" 422 | 423 | if isinstance(video_path, pathlib.Path): 424 | meta = get_metadata(video_path) 425 | file_name = video_path.name 426 | file_size = sizeof_fmt(os.stat(video_path).st_size) 427 | else: 428 | file_name = getattr(video_path, "file_name", "") 429 | file_size = sizeof_fmt(getattr(video_path, "file_size", (2 << 2) + ((2 << 2) + 1) + (2 << 5))) 430 | meta = dict( 431 | width=getattr(video_path, "width", 0), 432 | height=getattr(video_path, "height", 0), 433 | duration=getattr(video_path, "duration", 0), 434 | thumb=getattr(video_path, "thumb", None), 435 | ) 436 | free = payment.get_free_token(chat_id) 437 | pay = payment.get_pay_token(chat_id) 438 | if ENABLE_VIP: 439 | remain = f"تعداد توکن دانلود: رایگان {free}, پرداختی {pay}" 440 | else: 441 | remain = "" 442 | worker = get_dl_source() 443 | cap = ( 444 | f"{user_info}\n{file_name}\n\n{url}\n\nاطلاعات: {meta['width']}x{meta['height']} {file_size}\t" 445 | f"{meta['duration']}s\n{remain}\n{worker}\n{bot_text.custom_text}" 446 | ) 447 | return cap, meta 448 | 449 | 450 | def gen_video_markup(): 451 | markup = InlineKeyboardMarkup( 452 | [ 453 | [ # First row 454 | InlineKeyboardButton( # Generates a callback query when pressed 455 | "تبدیل به صدا", callback_data="convert" 456 | ) 457 | ] 458 | ] 459 | ) 460 | return markup 461 | 462 | 463 | @Panel.register 464 | def ping_revision(*args): 465 | return get_revision() 466 | 467 | 468 | @Panel.register 469 | def hot_patch(*args): 470 | app_path = pathlib.Path().cwd().parent 471 | logging.info("Hot patching on path %s...", app_path) 472 | 473 | apk_install = "xargs apk add < apk.txt" 474 | pip_install = "pip install -r requirements.txt" 475 | unset = "git config --unset http.https://github.com/.extraheader" 476 | pull_unshallow = "git pull origin --unshallow" 477 | pull = "git pull" 478 | 479 | subprocess.call(unset, shell=True, cwd=app_path) 480 | if subprocess.call(pull_unshallow, shell=True, cwd=app_path) != 0: 481 | logging.info("Already unshallow, pulling now...") 482 | subprocess.call(pull, shell=True, cwd=app_path) 483 | 484 | logging.info("Code is updated, applying hot patch now...") 485 | subprocess.call(apk_install, shell=True, cwd=app_path) 486 | subprocess.call(pip_install, shell=True, cwd=app_path) 487 | psutil.Process().kill() 488 | 489 | 490 | def async_task(task_name, *args): 491 | if not ENABLE_QUEUE: 492 | task_name.delay(*args) 493 | return 494 | 495 | t0 = time.time() 496 | inspect = app.control.inspect() 497 | worker_stats = inspect.stats() 498 | route_queues = [] 499 | padding = math.ceil(sum([i["pool"]["max-concurrency"] for i in worker_stats.values()]) / len(worker_stats)) 500 | for worker_name, stats in worker_stats.items(): 501 | route = worker_name.split("@")[1] 502 | concurrency = stats["pool"]["max-concurrency"] 503 | route_queues.extend([route] * (concurrency + padding)) 504 | destination = random.choice(route_queues) 505 | logging.info("Selecting worker %s from %s in %.2fs", destination, route_queues, time.time() - t0) 506 | task_name.apply_async(args=args, queue=destination) 507 | 508 | 509 | def run_celery(): 510 | worker_name = os.getenv("WORKER_NAME", "") 511 | argv = ["-A", "tasks", "worker", "--loglevel=info", "--pool=threads", f"--concurrency={WORKERS}", "-n", worker_name] 512 | if ENABLE_QUEUE: 513 | argv.extend(["-Q", worker_name]) 514 | app.worker_main(argv) 515 | 516 | 517 | def purge_tasks(): 518 | count = app.control.purge() 519 | return f"purged {count} tasks." 520 | 521 | 522 | if __name__ == "__main__": 523 | # celery_client.start() 524 | print("Bootstrapping Celery worker now.....") 525 | time.sleep(5) 526 | threading.Thread(target=run_celery, daemon=True).start() 527 | 528 | scheduler = BackgroundScheduler(timezone="Asia/Tehran") 529 | scheduler.add_job(auto_restart, "interval", seconds=900) 530 | scheduler.start() 531 | 532 | idle() 533 | celery_client.stop() 534 | -------------------------------------------------------------------------------- /termux/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | apt-get update && apt-get upgrade -y 4 | termux-setup-storage 5 | # Check if the operating system is Windows 6 | if [ "$OSTYPE" == "msys" ]; then 7 | # Windows OS detected 8 | python_cmd="python" 9 | else 10 | # Assume non-Windows (Linux/Mac) OS 11 | python_cmd="python3" 12 | fi 13 | 14 | # Check if Python is installed 15 | if ! command -v $python_cmd &>/dev/null; then 16 | echo "$python_cmd is not installed. Installing $python_cmd..." 17 | # Install Python based on the detected command (python or python3) 18 | if [ "$OSTYPE" == "msys" ]; then 19 | # Windows OS 20 | choco install python -y 21 | else 22 | # Linux/Mac OS 23 | apt-get install $python_cmd -y 24 | fi 25 | else 26 | echo "$python_cmd is already installed." 27 | fi 28 | 29 | # Install pytube Python package 30 | echo "Installing pytube..." 31 | $python_cmd -m pip install pytube 32 | 33 | # Clear the terminal based on the OS 34 | if [ "$OSTYPE" == "msys" ]; then 35 | # Windows OS 36 | echo "Clearing the terminal (Windows)..." 37 | cls 38 | else 39 | # Linux/Mac OS 40 | echo "Clearing the terminal (Linux/Mac)..." 41 | clear 42 | fi 43 | 44 | # Install bot.py from the external link 45 | echo "Downloading bot.py..." 46 | curl -O https://raw.githubusercontent.com/Ptechgithub/ytdl/main/termux/ytdl_termux.py 47 | 48 | # Run the bot.py script 49 | echo "Running bot.py with $python_cmd..." 50 | $python_cmd ytdl_termux.py 51 | -------------------------------------------------------------------------------- /termux/ytdl_termux.py: -------------------------------------------------------------------------------- 1 | import os 2 | from pytube import YouTube 3 | import platform 4 | 5 | def on_progress(stream, chunk, bytes_remaining): 6 | total_size = stream.filesize 7 | bytes_downloaded = total_size - bytes_remaining 8 | percentage = (bytes_downloaded / total_size) * 100 9 | print(f"Downloading : {percentage:.2f}%", end='\r') 10 | 11 | # Clear the terminal based on the OS 12 | if platform.system() == 'Windows': 13 | os.system('cls') 14 | else: 15 | os.system('clear') 16 | 17 | try: 18 | # Get the current working directory 19 | current_dir = os.getcwd() 20 | 21 | # Create the directory if it doesn't exist 22 | download_dir = os.path.join(current_dir, 'YouTube_dl') 23 | os.makedirs(download_dir, exist_ok=True) 24 | 25 | # Get the video link from the user 26 | video_url = input("Enter the video link: ") 27 | 28 | # Create a YouTube object using the link 29 | yt = YouTube(video_url, on_progress_callback=on_progress) 30 | 31 | # Video information 32 | print("----------------------------------------------------------") 33 | print(f"Video Title: {yt.title}") 34 | print(f"Channel Name: {yt.author}") 35 | print(f"Video Duration: {yt.length // 60} minutes {yt.length % 60} seconds") 36 | print(f"Views: {yt.views}") 37 | print(f"Published Date: {yt.publish_date}") 38 | print("----------------------------------------------------------") 39 | print(" ") 40 | 41 | # Get all available download formats 42 | all_formats = yt.streams 43 | 44 | # Filter and sort available formats 45 | audio_formats = [stream for stream in all_formats if stream.includes_audio_track and stream.resolution is not None] 46 | sorted_formats = sorted(audio_formats, key=lambda x: (int(x.resolution[:-1]), -x.filesize), reverse=True) 47 | 48 | # Display sorted quality, format, and file size with numbers 49 | for i, stream in enumerate(sorted_formats): 50 | print(f"{i + 1}. Quality: {stream.resolution}, F/Rate: {stream.fps} fps, Bitrate: {stream.bitrate / 1000} Kbps, Format: {stream.mime_type}, Filesize: {stream.filesize / 1024 / 1024:.2f} MB") 51 | print(" -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- ") 52 | 53 | # Let the user choose a desired format by entering a number 54 | choice = int(input("Enter the number of quality: ")) - 1 55 | 56 | # Download the video with the selected quality 57 | selected_format = sorted_formats[choice] 58 | print(f"Downloading with Quality: {selected_format.resolution} ") 59 | selected_format.download(output_path=download_dir) 60 | print(f"Download was successful & Saved to {download_dir}") 61 | 62 | except Exception as e: 63 | print("An error occurred:", str(e)) 64 | -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python3 2 | # coding: utf-8 3 | 4 | # ytdlbot - utils.py 5 | # 9/1/21 22:50 6 | # 7 | 8 | __author__ = "Peyman" 9 | 10 | import contextlib 11 | import inspect as pyinspect 12 | import logging 13 | import os 14 | import pathlib 15 | import shutil 16 | import subprocess 17 | import tempfile 18 | import time 19 | import uuid 20 | 21 | import coloredlogs 22 | import ffmpeg 23 | import psutil 24 | 25 | from flower_tasks import app 26 | 27 | inspect = app.control.inspect() 28 | 29 | 30 | def apply_log_formatter(): 31 | coloredlogs.install( 32 | level=logging.INFO, 33 | fmt="[%(asctime)s %(filename)s:%(lineno)d %(levelname).1s] %(message)s", 34 | datefmt="%Y-%m-%d %H:%M:%S", 35 | ) 36 | 37 | 38 | def customize_logger(logger: list): 39 | apply_log_formatter() 40 | for log in logger: 41 | logging.getLogger(log).setLevel(level=logging.INFO) 42 | 43 | 44 | def sizeof_fmt(num: int, suffix="B"): 45 | for unit in ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"]: 46 | if abs(num) < 1024.0: 47 | return "%3.1f%s%s" % (num, unit, suffix) 48 | num /= 1024.0 49 | return "%.1f%s%s" % (num, "Yi", suffix) 50 | 51 | 52 | def is_youtube(url: str): 53 | if url.startswith("https://www.youtube.com/") or url.startswith("https://youtu.be/"): 54 | return True 55 | 56 | 57 | def adjust_formats(user_id: int, url: str, formats: list, hijack=None): 58 | from database import MySQL 59 | 60 | # high: best quality 1080P, 2K, 4K, 8K 61 | # medium: 720P 62 | # low: 480P 63 | if hijack: 64 | formats.insert(0, hijack) 65 | return 66 | 67 | mapping = {"high": [], "medium": [720], "low": [480]} 68 | settings = MySQL().get_user_settings(user_id) 69 | if settings and is_youtube(url): 70 | for m in mapping.get(settings[1], []): 71 | formats.insert(0, f"bestvideo[ext=mp4][height={m}]+bestaudio[ext=m4a]") 72 | formats.insert(1, f"bestvideo[vcodec^=avc][height={m}]+bestaudio[acodec^=mp4a]/best[vcodec^=avc]/best") 73 | 74 | if settings[2] == "audio": 75 | formats.insert(0, "bestaudio[ext=m4a]") 76 | 77 | 78 | def get_metadata(video_path): 79 | width, height, duration = 1280, 720, 0 80 | try: 81 | video_streams = ffmpeg.probe(video_path, select_streams="v") 82 | for item in video_streams.get("streams", []): 83 | height = item["height"] 84 | width = item["width"] 85 | duration = int(float(video_streams["format"]["duration"])) 86 | except Exception as e: 87 | logging.error(e) 88 | try: 89 | thumb = pathlib.Path(video_path).parent.joinpath(f"{uuid.uuid4().hex}-thunmnail.png").as_posix() 90 | ffmpeg.input(video_path, ss=duration / 2).filter("scale", width, -1).output(thumb, vframes=1).run() 91 | except ffmpeg._run.Error: 92 | thumb = None 93 | 94 | return dict(height=height, width=width, duration=duration, thumb=thumb) 95 | 96 | 97 | def current_time(ts=None): 98 | return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(ts)) 99 | 100 | 101 | def get_revision(): 102 | with contextlib.suppress(subprocess.SubprocessError): 103 | return subprocess.check_output("git -C ../ rev-parse --short HEAD".split()).decode("u8").replace("\n", "") 104 | return "unknown" 105 | 106 | 107 | def get_func_queue(func) -> int: 108 | try: 109 | count = 0 110 | data = getattr(inspect, func)() or {} 111 | for _, task in data.items(): 112 | count += len(task) 113 | return count 114 | except Exception: 115 | return 0 116 | 117 | 118 | def tail_log(f, lines=1, _buffer=4098): 119 | """Tail a file and get X lines from the end""" 120 | # placeholder for the lines found 121 | lines_found = [] 122 | 123 | # block counter will be multiplied by buffer 124 | # to get the block size from the end 125 | block_counter = -1 126 | 127 | # loop until we find X lines 128 | while len(lines_found) < lines: 129 | try: 130 | f.seek(block_counter * _buffer, os.SEEK_END) 131 | except IOError: # either file is too small, or too many lines requested 132 | f.seek(0) 133 | lines_found = f.readlines() 134 | break 135 | 136 | lines_found = f.readlines() 137 | 138 | # we found enough lines, get out 139 | # Removed this line because it was redundant the while will catch 140 | # it, I left it for history 141 | # if len(lines_found) > lines: 142 | # break 143 | 144 | # decrement the block counter to get the 145 | # next X bytes 146 | block_counter -= 1 147 | 148 | return lines_found[-lines:] 149 | 150 | 151 | class Detector: 152 | def __init__(self, logs: str): 153 | self.logs = logs 154 | 155 | @staticmethod 156 | def func_name(): 157 | with contextlib.suppress(Exception): 158 | return pyinspect.stack()[1][3] 159 | return "N/A" 160 | 161 | def updates_too_long_detector(self): 162 | # If you're seeing this, that means you have logged more than 10 device 163 | # and the earliest account was kicked out. Restart the program could get you back in. 164 | indicators = [ 165 | "types.UpdatesTooLong", 166 | "Got shutdown from remote", 167 | "Code is updated", 168 | 'Retrying "messages.GetMessages"', 169 | "OSError: Connection lost", 170 | "[Errno -3] Try again", 171 | "MISCONF", 172 | ] 173 | for indicator in indicators: 174 | if indicator in self.logs: 175 | logging.warning("Potential crash detected by %s, it's time to commit suicide...", self.func_name()) 176 | return True 177 | logging.debug("No crash detected.") 178 | 179 | def next_salt_detector(self): 180 | text = "Next salt in" 181 | if self.logs.count(text) >= 4: 182 | logging.warning("Potential crash detected by %s, it's time to commit suicide...", self.func_name()) 183 | return True 184 | 185 | # def idle_detector(self): 186 | # mtime = os.stat("/var/log/ytdl.log").st_mtime 187 | # cur_ts = time.time() 188 | # if cur_ts - mtime > 7200: 189 | # logging.warning("Potential crash detected by %s, it's time to commit suicide...", self.func_name()) 190 | # return True 191 | 192 | 193 | def auto_restart(): 194 | log_path = "/var/log/ytdl.log" 195 | if not os.path.exists(log_path): 196 | return 197 | with open(log_path) as f: 198 | logs = "".join(tail_log(f, lines=10)) 199 | 200 | det = Detector(logs) 201 | method_list = [getattr(det, func) for func in dir(det) if func.endswith("_detector")] 202 | for method in method_list: 203 | if method(): 204 | logging.critical("Bye bye world!☠️") 205 | for item in pathlib.Path(tempfile.gettempdir()).glob("ytdl-*"): 206 | shutil.rmtree(item, ignore_errors=True) 207 | 208 | psutil.Process().kill() 209 | 210 | 211 | def clean_tempfile(): 212 | for item in pathlib.Path(tempfile.gettempdir()).glob("ytdl-*"): 213 | if time.time() - item.stat().st_ctime > 3600: 214 | shutil.rmtree(item, ignore_errors=True) 215 | 216 | 217 | if __name__ == "__main__": 218 | auto_restart() 219 | -------------------------------------------------------------------------------- /ytdl_bot.py: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python3 2 | # coding: utf-8 3 | 4 | # ytdlbot - new.py 5 | # 8/14/21 14:37 6 | # 7 | 8 | __author__ = "Peyman" 9 | 10 | import contextlib 11 | import logging 12 | import os 13 | import random 14 | import re 15 | import tempfile 16 | import time 17 | import traceback 18 | import typing 19 | from io import BytesIO 20 | 21 | import pyrogram.errors 22 | import requests 23 | import yt_dlp 24 | from apscheduler.schedulers.background import BackgroundScheduler 25 | from pyrogram import Client, filters, types 26 | from pyrogram.errors.exceptions.bad_request_400 import UserNotParticipant 27 | from pyrogram.raw import functions 28 | from pyrogram.raw import types as raw_types 29 | from pyrogram.types import InlineKeyboardButton, InlineKeyboardMarkup 30 | from tgbot_ping import get_runtime 31 | 32 | from channel import Channel 33 | from client_init import create_app 34 | from config import ( 35 | AUTHORIZED_USER, 36 | ENABLE_CELERY, 37 | ENABLE_FFMPEG, 38 | ENABLE_VIP, 39 | OWNER, 40 | PLAYLIST_SUPPORT, 41 | PROVIDER_TOKEN, 42 | REQUIRED_MEMBERSHIP, 43 | TOKEN_PRICE, 44 | ) 45 | from constant import BotText 46 | from database import InfluxDB, MySQL, Redis 47 | from limit import Payment 48 | from tasks import app as celery_app 49 | from tasks import ( 50 | audio_entrance, 51 | direct_download_entrance, 52 | hot_patch, 53 | purge_tasks, 54 | ytdl_download_entrance, 55 | ) 56 | from utils import auto_restart, clean_tempfile, customize_logger, get_revision 57 | 58 | customize_logger(["pyrogram.client", "pyrogram.session.session", "pyrogram.connection.connection"]) 59 | logging.getLogger("apscheduler.executors.default").propagate = False 60 | 61 | session = "ytdl-main" 62 | app = create_app(session) 63 | 64 | logging.info("Authorized users are %s", AUTHORIZED_USER) 65 | redis = Redis() 66 | channel = Channel() 67 | 68 | 69 | def private_use(func): 70 | def wrapper(client: Client, message: types.Message): 71 | chat_id = getattr(message.from_user, "id", None) 72 | 73 | # message type check 74 | if message.chat.type != "private" and not message.text.lower().startswith("/ytdl"): 75 | logging.debug("%s, it's annoying me...🙄️ ", message.text) 76 | return 77 | 78 | # authorized users check 79 | if AUTHORIZED_USER: 80 | users = [int(i) for i in AUTHORIZED_USER.split(",")] 81 | else: 82 | users = [] 83 | 84 | if users and chat_id and chat_id not in users: 85 | message.reply_text(BotText.private, quote=True) 86 | return 87 | 88 | if REQUIRED_MEMBERSHIP: 89 | try: 90 | member: typing.Union[types.ChatMember, typing.Coroutine] = app.get_chat_member( 91 | REQUIRED_MEMBERSHIP, chat_id 92 | ) 93 | if member.status not in [ 94 | "creator", 95 | "administrator", 96 | "member", 97 | "owner", 98 | ]: 99 | raise UserNotParticipant() 100 | else: 101 | logging.info("user %s check passed for group/channel %s.", chat_id, REQUIRED_MEMBERSHIP) 102 | except UserNotParticipant: 103 | logging.warning("user %s is not a member of group/channel %s", chat_id, REQUIRED_MEMBERSHIP) 104 | message.reply_text(BotText.membership_require, quote=True) 105 | return 106 | 107 | return func(client, message) 108 | 109 | return wrapper 110 | 111 | 112 | @app.on_message(filters.command(["start"])) 113 | def start_handler(client: Client, message: types.Message): 114 | payment = Payment() 115 | from_id = message.from_user.id 116 | logging.info("Welcome to youtube-dl bot!") 117 | client.send_chat_action(from_id, "typing") 118 | is_old_user = payment.check_old_user(from_id) 119 | if is_old_user: 120 | info = "" 121 | elif ENABLE_VIP: 122 | free_token, pay_token, reset = payment.get_token(from_id) 123 | info = f"Free token: {free_token}, Pay token: {pay_token}, Reset: {reset}" 124 | else: 125 | info = "" 126 | text = f"{BotText.start}\n\n{info}\n{BotText.custom_text}" 127 | client.send_message(message.chat.id, text) 128 | 129 | 130 | @app.on_message(filters.command(["help"])) 131 | def help_handler(client: Client, message: types.Message): 132 | chat_id = message.chat.id 133 | client.send_chat_action(chat_id, "typing") 134 | client.send_message(chat_id, BotText.help, disable_web_page_preview=True) 135 | 136 | 137 | @app.on_message(filters.command(["about"])) 138 | def about_handler(client: Client, message: types.Message): 139 | chat_id = message.chat.id 140 | client.send_chat_action(chat_id, "typing") 141 | client.send_message(chat_id, BotText.about) 142 | 143 | 144 | @app.on_message(filters.command(["sub"])) 145 | def subscribe_handler(client: Client, message: types.Message): 146 | chat_id = message.chat.id 147 | client.send_chat_action(chat_id, "typing") 148 | if message.text == "/sub": 149 | result = channel.get_user_subscription(chat_id) 150 | else: 151 | link = message.text.split()[1] 152 | try: 153 | result = channel.subscribe_channel(chat_id, link) 154 | except (IndexError, ValueError): 155 | result = f"Error: \n{traceback.format_exc()}" 156 | client.send_message(chat_id, result or "You have no subscription.", disable_web_page_preview=True) 157 | 158 | 159 | @app.on_message(filters.command(["unsub"])) 160 | def unsubscribe_handler(client: Client, message: types.Message): 161 | chat_id = message.chat.id 162 | client.send_chat_action(chat_id, "typing") 163 | text = message.text.split(" ") 164 | if len(text) == 1: 165 | client.send_message(chat_id, "/unsub channel_id", disable_web_page_preview=True) 166 | return 167 | 168 | rows = channel.unsubscribe_channel(chat_id, text[1]) 169 | if rows: 170 | text = f"Unsubscribed from {text[1]}" 171 | else: 172 | text = "Unable to find the channel." 173 | client.send_message(chat_id, text, disable_web_page_preview=True) 174 | 175 | 176 | @app.on_message(filters.command(["patch"])) 177 | def patch_handler(client: Client, message: types.Message): 178 | username = message.from_user.username 179 | chat_id = message.chat.id 180 | if username == OWNER: 181 | celery_app.control.broadcast("hot_patch") 182 | client.send_chat_action(chat_id, "typing") 183 | client.send_message(chat_id, "Oorah!") 184 | hot_patch() 185 | 186 | 187 | @app.on_message(filters.command(["uncache"])) 188 | def uncache_handler(client: Client, message: types.Message): 189 | username = message.from_user.username 190 | link = message.text.split()[1] 191 | if username == OWNER: 192 | count = channel.del_cache(link) 193 | message.reply_text(f"{count} cache(s) deleted.", quote=True) 194 | 195 | 196 | @app.on_message(filters.command(["purge"])) 197 | def purge_handler(client: Client, message: types.Message): 198 | username = message.from_user.username 199 | if username == OWNER: 200 | message.reply_text(purge_tasks(), quote=True) 201 | 202 | 203 | @app.on_message(filters.command(["ping"])) 204 | def ping_handler(client: Client, message: types.Message): 205 | chat_id = message.chat.id 206 | client.send_chat_action(chat_id, "typing") 207 | if os.uname().sysname == "Darwin" or ".heroku" in os.getenv("PYTHONHOME", ""): 208 | bot_info = "ping unavailable." 209 | else: 210 | bot_info = get_runtime("ytdlbot_ytdl_1", "YouTube-dl") 211 | if message.chat.username == OWNER: 212 | stats = BotText.ping_worker()[:1000] 213 | client.send_document(chat_id, redis.generate_file(), caption=f"{bot_info}\n\n{stats}") 214 | else: 215 | client.send_message(chat_id, f"{bot_info.split('CPU')[0]}") 216 | 217 | 218 | @app.on_message(filters.command(["sub_count"])) 219 | def sub_count_handler(client: Client, message: types.Message): 220 | username = message.from_user.username 221 | chat_id = message.chat.id 222 | if username == OWNER: 223 | with BytesIO() as f: 224 | f.write(channel.sub_count().encode("u8")) 225 | f.name = "subscription count.txt" 226 | client.send_document(chat_id, f) 227 | 228 | 229 | @app.on_message(filters.command(["direct"])) 230 | def direct_handler(client: Client, message: types.Message): 231 | chat_id = message.from_user.id 232 | client.send_chat_action(chat_id, "typing") 233 | url = re.sub(r"/direct\s*", "", message.text) 234 | logging.info("direct start %s", url) 235 | if not re.findall(r"^https?://", url.lower()): 236 | redis.update_metrics("bad_request") 237 | message.reply_text("Send me a DIRECT LINK.", quote=True) 238 | return 239 | 240 | bot_msg = message.reply_text("Request received.", quote=True) 241 | redis.update_metrics("direct_request") 242 | direct_download_entrance(client, bot_msg, url) 243 | 244 | 245 | @app.on_message(filters.command(["settings"])) 246 | def settings_handler(client: Client, message: types.Message): 247 | chat_id = message.chat.id 248 | payment = Payment() 249 | client.send_chat_action(chat_id, "typing") 250 | data = MySQL().get_user_settings(chat_id) 251 | set_mode = data[-1] 252 | text = {"Local": "Celery", "Celery": "Local"}.get(set_mode, "Local") 253 | mode_text = f"Download mode: **{set_mode}**" 254 | if message.chat.username == OWNER or payment.get_pay_token(chat_id): 255 | extra = [InlineKeyboardButton(f"Change download mode to {text}", callback_data=text)] 256 | else: 257 | extra = [] 258 | 259 | markup = InlineKeyboardMarkup( 260 | [ 261 | [ # First row 262 | InlineKeyboardButton("send as document", callback_data="document"), 263 | InlineKeyboardButton("send as video", callback_data="video"), 264 | InlineKeyboardButton("send as audio", callback_data="audio"), 265 | ], 266 | [ # second row 267 | InlineKeyboardButton("High Quality", callback_data="high"), 268 | InlineKeyboardButton("Medium Quality", callback_data="medium"), 269 | InlineKeyboardButton("Low Quality", callback_data="low"), 270 | ], 271 | extra, 272 | ] 273 | ) 274 | 275 | try: 276 | client.send_message(chat_id, BotText.settings.format(data[1], data[2]) + mode_text, reply_markup=markup) 277 | except: 278 | client.send_message( 279 | chat_id, BotText.settings.format(data[1] + ".", data[2] + ".") + mode_text, reply_markup=markup 280 | ) 281 | 282 | 283 | @app.on_message(filters.command(["buy"])) 284 | def buy_handler(client: Client, message: types.Message): 285 | # process as chat.id, not from_user.id 286 | chat_id = message.chat.id 287 | text = message.text.strip() 288 | client.send_chat_action(chat_id, "typing") 289 | client.send_message(chat_id, BotText.buy, disable_web_page_preview=True) 290 | # generate telegram invoice here 291 | payload = f"{message.chat.id}-buy" 292 | token_count = message.text.replace("/buy", "").strip() 293 | # currency USD 294 | if token_count.isdigit(): 295 | price = int(int(token_count) / TOKEN_PRICE * 100) 296 | else: 297 | price = 100 298 | invoice = generate_invoice( 299 | price, f"Buy {TOKEN_PRICE} download tokens", "You can pay by Telegram payment or using link above", payload 300 | ) 301 | 302 | app.send( 303 | functions.messages.SendMedia( 304 | peer=(raw_types.InputPeerUser(user_id=chat_id, access_hash=0)), 305 | media=invoice, 306 | random_id=app.rnd_id(), 307 | message="Buy more download token", 308 | ) 309 | ) 310 | client.send_message(chat_id, "In /settings, change your download mode to Local will make the download faster!") 311 | 312 | 313 | @app.on_message(filters.command(["redeem"])) 314 | def redeem_handler(client: Client, message: types.Message): 315 | payment = Payment() 316 | chat_id = message.chat.id 317 | text = message.text.strip() 318 | unique = text.replace("/redeem", "").strip() 319 | msg = payment.verify_payment(chat_id, unique) 320 | message.reply_text(msg, quote=True) 321 | 322 | 323 | def generate_invoice(amount: int, title: str, description: str, payload: str): 324 | invoice = raw_types.input_media_invoice.InputMediaInvoice( 325 | invoice=raw_types.invoice.Invoice( 326 | currency="USD", prices=[raw_types.LabeledPrice(label="price", amount=amount)] 327 | ), 328 | title=title, 329 | description=description, 330 | provider=PROVIDER_TOKEN, 331 | provider_data=raw_types.DataJSON(data="{}"), 332 | payload=payload.encode(), 333 | start_param=payload, 334 | ) 335 | return invoice 336 | 337 | 338 | def search(kw: str): 339 | api = f"https://dmesg.app/ytdlbot/search.php?search={kw}" 340 | # title, url, time, image 341 | text, index = "", 1 342 | for item in requests.get(api).json()["results"][:10]: 343 | item["index"] = index 344 | text += "{index}. {title}\n{url}\n{time}\n\n".format(**item) 345 | index += 1 346 | return text 347 | 348 | 349 | def link_checker(url: str) -> str: 350 | if url.startswith("https://www.instagram.com"): 351 | return "" 352 | ytdl = yt_dlp.YoutubeDL() 353 | 354 | if not PLAYLIST_SUPPORT and ( 355 | re.findall(r"^https://www\.youtube\.com/channel/", Channel.extract_canonical_link(url)) or "list" in url 356 | ): 357 | return "Playlist or channel links are disabled." 358 | 359 | if re.findall(r"m3u8|\.m3u8|\.m3u$", url.lower()): 360 | return "m3u8 links are disabled." 361 | 362 | with contextlib.suppress(yt_dlp.utils.DownloadError): 363 | if ytdl.extract_info(url, download=False).get("live_status") == "is_live": 364 | return "Live stream links are disabled. Please download it after the stream ends." 365 | 366 | 367 | @app.on_message(filters.incoming & (filters.text | filters.document)) 368 | @private_use 369 | def download_handler(client: Client, message: types.Message): 370 | payment = Payment() 371 | chat_id = message.from_user.id 372 | client.send_chat_action(chat_id, "typing") 373 | redis.user_count(chat_id) 374 | if message.document: 375 | with tempfile.NamedTemporaryFile(mode="r+") as tf: 376 | logging.info("Downloading file to %s", tf.name) 377 | message.download(tf.name) 378 | contents = open(tf.name, "r").read() # don't know why 379 | urls = contents.split() 380 | else: 381 | urls = [re.sub(r"/ytdl\s*", "", message.text)] 382 | logging.info("start %s", urls) 383 | 384 | for url in urls: 385 | # check url 386 | if not re.findall(r"^https?://", url.lower()): 387 | redis.update_metrics("bad_request") 388 | text = search(url) 389 | message.reply_text(text, quote=True) 390 | return 391 | 392 | if text := link_checker(url): 393 | message.reply_text(text, quote=True) 394 | redis.update_metrics("reject_link_checker") 395 | return 396 | 397 | # old user is not limited by token 398 | if ENABLE_VIP and not payment.check_old_user(chat_id): 399 | free, pay, reset = payment.get_token(chat_id) 400 | if free + pay <= 0: 401 | message.reply_text( 402 | f"شما توکن کافی ندارید. لطفاً تا {reset} صبر کنید یا برای خرید توکن /buy را ارسال کنید.", quote=True 403 | ) 404 | redis.update_metrics("reject_token") 405 | return 406 | else: 407 | payment.use_token(chat_id) 408 | 409 | redis.update_metrics("video_request") 410 | 411 | text = BotText.get_receive_link_text() 412 | try: 413 | # raise pyrogram.errors.exceptions.FloodWait(10) 414 | bot_msg: typing.Union[types.Message, typing.Coroutine] = message.reply_text(text, quote=True) 415 | except pyrogram.errors.Flood as e: 416 | f = BytesIO() 417 | f.write(str(e).encode()) 418 | f.write(b"Your job will be done soon. Just wait! Don't rush.") 419 | f.name = "Please don't flood me.txt" 420 | bot_msg = message.reply_document( 421 | f, caption=f"Flood wait! Please wait {e.x} seconds...." f"Your job will start automatically", quote=True 422 | ) 423 | f.close() 424 | client.send_message(OWNER, f"Flood wait! 🙁 {e.x} seconds....") 425 | time.sleep(e.x) 426 | 427 | client.send_chat_action(chat_id, "upload_video") 428 | bot_msg.chat = message.chat 429 | ytdl_download_entrance(client, bot_msg, url) 430 | 431 | 432 | @app.on_callback_query(filters.regex(r"document|video|audio")) 433 | def send_method_callback(client: Client, callback_query: types.CallbackQuery): 434 | chat_id = callback_query.message.chat.id 435 | data = callback_query.data 436 | logging.info("Setting %s file type to %s", chat_id, data) 437 | MySQL().set_user_settings(chat_id, "method", data) 438 | callback_query.answer(f"Your send type was set to {callback_query.data}") 439 | 440 | 441 | @app.on_callback_query(filters.regex(r"high|medium|low")) 442 | def download_resolution_callback(client: Client, callback_query: types.CallbackQuery): 443 | chat_id = callback_query.message.chat.id 444 | data = callback_query.data 445 | logging.info("Setting %s file type to %s", chat_id, data) 446 | MySQL().set_user_settings(chat_id, "resolution", data) 447 | callback_query.answer(f"Your default download quality was set to {callback_query.data}") 448 | 449 | 450 | @app.on_callback_query(filters.regex(r"convert")) 451 | def audio_callback(client: Client, callback_query: types.CallbackQuery): 452 | if not ENABLE_FFMPEG: 453 | callback_query.answer("Audio conversion is disabled now.") 454 | callback_query.message.reply_text("Audio conversion is disabled now.") 455 | return 456 | 457 | callback_query.answer(f"Converting to audio...please wait patiently") 458 | redis.update_metrics("audio_request") 459 | vmsg = callback_query.message 460 | audio_entrance(client, vmsg) 461 | 462 | 463 | @app.on_callback_query(filters.regex(r"Local|Celery")) 464 | def owner_local_callback(client: Client, callback_query: types.CallbackQuery): 465 | chat_id = callback_query.message.chat.id 466 | MySQL().set_user_settings(chat_id, "mode", callback_query.data) 467 | callback_query.answer(f"Download mode was changed to {callback_query.data}") 468 | 469 | 470 | def periodic_sub_check(): 471 | exceptions = pyrogram.errors.exceptions 472 | for cid, uids in channel.group_subscriber().items(): 473 | video_url = channel.has_newer_update(cid) 474 | if video_url: 475 | logging.info(f"periodic update:{video_url} - {uids}") 476 | for uid in uids: 477 | try: 478 | bot_msg: typing.Union[types.Message, typing.Coroutine] = app.send_message( 479 | uid, f"{video_url} is out. Watch it on YouTube" 480 | ) 481 | # ytdl_download_entrance(app, bot_msg, video_url, mode="direct") 482 | except (exceptions.bad_request_400.PeerIdInvalid, exceptions.bad_request_400.UserIsBlocked) as e: 483 | logging.warning("User is blocked or deleted. %s", e) 484 | channel.deactivate_user_subscription(uid) 485 | except Exception as e: 486 | logging.error("Unknown error when sending message to user. %s", traceback.format_exc()) 487 | finally: 488 | time.sleep(random.random() * 3) 489 | 490 | 491 | @app.on_raw_update() 492 | def raw_update(client: Client, update, users, chats): 493 | payment = Payment() 494 | action = getattr(getattr(update, "message", None), "action", None) 495 | if update.QUALNAME == "types.UpdateBotPrecheckoutQuery": 496 | client.send( 497 | functions.messages.SetBotPrecheckoutResults( 498 | query_id=update.query_id, 499 | success=True, 500 | ) 501 | ) 502 | elif action and action.QUALNAME == "types.MessageActionPaymentSentMe": 503 | logging.info("Payment received. %s", action) 504 | uid = update.message.peer_id.user_id 505 | amount = action.total_amount / 100 506 | payment.add_pay_user([uid, amount, action.charge.provider_charge_id, 0, amount * TOKEN_PRICE]) 507 | client.send_message(uid, f"Thank you {uid}. Payment received: {amount} {action.currency}") 508 | 509 | 510 | if __name__ == "__main__": 511 | MySQL() 512 | scheduler = BackgroundScheduler(timezone="Asia/Tehran", job_defaults={"max_instances": 5}) 513 | scheduler.add_job(redis.reset_today, "cron", hour=0, minute=0) 514 | scheduler.add_job(auto_restart, "interval", seconds=600) 515 | scheduler.add_job(clean_tempfile, "interval", seconds=60) 516 | scheduler.add_job(InfluxDB().collect_data, "interval", seconds=60) 517 | # default quota allocation of 10,000 units per day 518 | scheduler.add_job(periodic_sub_check, "interval", seconds=3600) 519 | scheduler.start() 520 | banner = f""" 521 | ▌ ▌ ▀▛▘ ▌ ▛▀▖ ▜ ▌ 522 | ▝▞ ▞▀▖ ▌ ▌ ▌ ▌ ▌ ▛▀▖ ▞▀▖ ▌ ▌ ▞▀▖ ▌ ▌ ▛▀▖ ▐ ▞▀▖ ▝▀▖ ▞▀▌ 523 | ▌ ▌ ▌ ▌ ▌ ▌ ▌ ▌ ▌ ▌ ▛▀ ▌ ▌ ▌ ▌ ▐▐▐ ▌ ▌ ▐ ▌ ▌ ▞▀▌ ▌ ▌ 524 | ▘ ▝▀ ▝▀▘ ▘ ▝▀▘ ▀▀ ▝▀▘ ▀▀ ▝▀ ▘▘ ▘ ▘ ▘ ▝▀ ▝▀▘ ▝▀▘ 525 | 526 | By @Peyman, VIP mode: {ENABLE_VIP}, Celery Mode: {ENABLE_CELERY} 527 | Version: {get_revision()} 528 | """ 529 | print(banner) 530 | app.run() 531 | --------------------------------------------------------------------------------