├── src
├── __init__.py
├── api_handler
│ ├── __init__.py
│ ├── side_processes
│ │ ├── __init__.py
│ │ ├── base.py
│ │ └── purge_old_files.py
│ ├── app.py
│ └── routes.py
├── downloader
│ ├── __init__.py
│ ├── utils
│ │ ├── __init__.py
│ │ ├── check_if_file_exists.py
│ │ ├── is_playlist.py
│ │ ├── video_info.py
│ │ ├── create_download_link.py
│ │ ├── is_valid_url.py
│ │ ├── filename_collector.py
│ │ ├── base.py
│ │ ├── extract_metadata.py
│ │ ├── extract_playlist_metadata.py
│ │ ├── find_appropriate_res.py
│ │ ├── create_response.py
│ │ ├── format_title.py
│ │ └── ydl_opts_builder.py
│ └── runner.py
└── db_handler
│ ├── __init__.py
│ └── tables.py
├── .gitignore
├── mountpoint
└── database
│ └── ytflex_database.db
├── requirements.txt
├── Dockerfile
├── docker-compose.yml
├── .github
└── workflows
│ ├── deploy_to_server.yml
│ ├── build_and_push_to_docker_hub.yml
│ └── pylint.yml
├── config.py
├── main.py
├── config.yaml
├── readme.md
└── index.html
/src/__init__.py:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/src/api_handler/__init__.py:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/src/downloader/__init__.py:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/src/downloader/utils/__init__.py:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/src/api_handler/side_processes/__init__.py:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/src/downloader/utils/check_if_file_exists.py:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | __pycache__
2 | .vscode
3 | mountpoint/*/*
4 | test*
5 | other
--------------------------------------------------------------------------------
/mountpoint/database/ytflex_database.db:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/arda-y/YTFlex/HEAD/mountpoint/database/ytflex_database.db
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | uvicorn # asgi server
2 | fastapi # asgi framework
3 | yt_dlp # youtube-dl fork
4 | pyyaml # yaml parser
5 | sqlalchemy # database orm
--------------------------------------------------------------------------------
/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM python:3.11
2 |
3 | WORKDIR /app
4 |
5 | COPY . ./
6 |
7 | RUN apt-get update && \
8 | apt-get install -y ffmpeg && \
9 | pip install --no-cache-dir --upgrade -r requirements.txt
10 |
11 | CMD ["python", "./main.py"]
12 |
--------------------------------------------------------------------------------
/src/api_handler/side_processes/base.py:
--------------------------------------------------------------------------------
1 | """
2 | Easy import for all side processes.
3 | """
4 |
5 | from src.api_handler.side_processes.purge_old_files import purge_old_files
6 |
7 | # makes it easier to import everything from this file
8 |
9 | __all__ = ["purge_old_files"]
10 |
--------------------------------------------------------------------------------
/docker-compose.yml:
--------------------------------------------------------------------------------
1 | # all values can be changed to your liking, provided you know what you are doing
2 | version: '3'
3 |
4 | services:
5 | ytflex:
6 | image: ytflex:latest
7 | container_name: ytflex
8 | restart: always
9 | ports:
10 | - "2002:2002"
11 | volumes:
12 | - /app/ytflex/:/app/mountpoint/
--------------------------------------------------------------------------------
/src/downloader/utils/is_playlist.py:
--------------------------------------------------------------------------------
1 | """
2 | Utility function for checking if a URL is a playlist.
3 | """
4 |
5 |
6 | def is_playlist(link: str) -> bool:
7 | """Checks if a URL is a playlist or not"""
8 |
9 | if len(link) == 34 and "playlist?list=" not in link and "watch?v=" not in link:
10 | # probably a playlist id since it's 34 characters long
11 | return True
12 |
13 | return "playlist?list=" in link
14 |
--------------------------------------------------------------------------------
/src/downloader/utils/video_info.py:
--------------------------------------------------------------------------------
1 | """
2 | Utility class to store video information.
3 | """
4 |
5 | from dataclasses import dataclass
6 |
7 |
8 | @dataclass
9 | class VideoInfo:
10 | """Data class to store video information."""
11 |
12 | def __init__(self, title: str, duration: int, url: str, thumbnail: str):
13 | self.url = url
14 | self.title = title
15 | self.duration = duration
16 | self.thumbnail = thumbnail
17 |
--------------------------------------------------------------------------------
/.github/workflows/deploy_to_server.yml:
--------------------------------------------------------------------------------
1 | name: Deploy to Server
2 |
3 | on: [push]
4 |
5 | jobs:
6 | replace-live-image:
7 | runs-on: penguin-devops
8 | steps:
9 | - uses: actions/checkout@v4
10 |
11 | - name: Stop Old Container
12 | run: docker stop ytflex || true
13 |
14 | - name: Remove Old Container
15 | run: docker rm ytflex || true
16 |
17 | - name: Create New Image
18 | run: docker build -t ytflex .
19 |
20 | - name: Run New Image
21 | run: docker compose up -d
--------------------------------------------------------------------------------
/src/downloader/utils/create_download_link.py:
--------------------------------------------------------------------------------
1 | """
2 | Utility function to create a download link for a file.
3 | """
4 |
5 | from urllib.parse import quote
6 | import os
7 | import config
8 |
9 |
10 | def create_download_link(filename: str):
11 | """
12 | Creates a download link for the file
13 | Args:
14 | filename: FULL filename of the file, including the extension
15 | """
16 |
17 | ip_or_domain = os.getenv("ip_or_domain", config.get("IP_OR_DOMAIN"))
18 |
19 | return f"{ip_or_domain}/ytflex-cdn/{quote(filename)}"
20 |
--------------------------------------------------------------------------------
/.github/workflows/build_and_push_to_docker_hub.yml:
--------------------------------------------------------------------------------
1 | name: Build and Push to Docker Hub
2 |
3 | on: [push]
4 |
5 | jobs:
6 | build-and-push-image:
7 | runs-on: ubuntu-latest
8 | steps:
9 | - uses: actions/checkout@v4
10 |
11 | - name: Login to Docker Hub
12 | uses: docker/login-action@v3
13 | with:
14 | username: kirellkekw
15 | password: ${{ secrets.DOCKERHUB_TOKEN }}
16 |
17 | - name: Build and push
18 | uses: docker/build-push-action@v5
19 | with:
20 | push: true
21 | tags: |
22 | kirellkekw/ytflex:${{ github.sha }}
23 | kirellkekw/ytflex:latest
--------------------------------------------------------------------------------
/src/downloader/utils/is_valid_url.py:
--------------------------------------------------------------------------------
1 | """
2 | Utility function to check if a URL is valid.
3 | """
4 |
5 | import requests
6 |
7 |
8 | def is_valid_url(link: str) -> bool:
9 | """
10 | Checks if a URL is valid or not.
11 |
12 | Args:
13 | link: The URL to check.
14 | Returns:
15 | True if the URL is valid, False otherwise.
16 | """
17 | if len(link) == 11:
18 | # probably a video id, check if the link is alive
19 | return (
20 | requests.head(
21 | f"https://www.youtube.com/watch?v={link}", timeout=5
22 | ).status_code
23 | == 200
24 | )
25 |
26 | return requests.head(link, timeout=5).status_code == 200
27 |
--------------------------------------------------------------------------------
/.github/workflows/pylint.yml:
--------------------------------------------------------------------------------
1 | name: Pylint
2 |
3 | on: [push]
4 |
5 | jobs:
6 | build:
7 | runs-on: ubuntu-latest
8 | strategy:
9 | matrix:
10 | python-version: ["3.10", "3.11"]
11 | steps:
12 | - uses: actions/checkout@v3
13 | - name: Set up Python ${{ matrix.python-version }}
14 | uses: actions/setup-python@v3
15 | with:
16 | python-version: ${{ matrix.python-version }}
17 | - name: Install dependencies
18 | run: |
19 | python -m pip install --upgrade pip
20 | pip install -r requirements.txt
21 | pip install pylint
22 | - name: Analyzing the code with pylint
23 | run: |
24 | pylint $(git ls-files '*.py')
25 |
--------------------------------------------------------------------------------
/src/downloader/utils/filename_collector.py:
--------------------------------------------------------------------------------
1 | """
2 | A post processor that collects the filename of the downloaded video.
3 | """
4 |
5 | from yt_dlp.postprocessor.common import PostProcessor
6 |
7 |
8 | class FilenameCollectorPP(PostProcessor):
9 | """
10 | A post processor that collects the filename of the downloaded video.
11 | Access the last downloaded filename by using:
12 |
13 | obj.filenames[-1]
14 |
15 | You might need to use os.path.basename() to get the actual
16 | filename if you've set a custom directory.
17 | """
18 |
19 | def __init__(self):
20 | super().__init__(None)
21 | self.filenames = []
22 |
23 | def run(self, information):
24 | self.filenames.append(information["filepath"])
25 | return [], information
26 |
--------------------------------------------------------------------------------
/src/api_handler/app.py:
--------------------------------------------------------------------------------
1 | """
2 | This file contains the FastAPI app.
3 | It also runs the sub-processes in the background.
4 | """
5 |
6 | import asyncio
7 | from fastapi import FastAPI
8 | from fastapi.middleware.cors import CORSMiddleware
9 | from src.api_handler.side_processes.base import purge_old_files
10 | import config
11 |
12 |
13 | app = FastAPI(docs_url="/docs", redoc_url=None)
14 | origins = config.get("ALLOWED_DOMAINS")
15 |
16 | # Add CORS middleware
17 | app.add_middleware(
18 | CORSMiddleware,
19 | allow_origins=origins,
20 | allow_credentials=True,
21 | allow_methods=["*"],
22 | allow_headers=["*"],
23 | )
24 |
25 |
26 | @app.on_event("startup") # run this function when the server starts
27 | async def startup_event():
28 | """Creates sub-processes to run in the background when the server starts."""
29 | asyncio.create_task(purge_old_files())
30 |
--------------------------------------------------------------------------------
/src/api_handler/routes.py:
--------------------------------------------------------------------------------
1 | """
2 | All API routes for the YTFlex project.
3 | """
4 |
5 | from src.api_handler.app import app
6 | from src.downloader.runner import download_files
7 |
8 | __all__ = ["root", "audio_download", "video_download"]
9 |
10 |
11 | @app.get("/root")
12 | async def root():
13 | """To check if the server is running without much hassle."""
14 |
15 | return {"message": "Hello World"}
16 |
17 |
18 | @app.get("/download/audio")
19 | async def audio_download(link: str):
20 | """API route for downloading audio files."""
21 |
22 | # bundle the download info
23 | raw_dl_info = download_files(passed_urls=link, is_video_request=False)
24 |
25 | return raw_dl_info
26 |
27 |
28 | @app.get("/download/video")
29 | async def video_download(link: str, res: int, mp4: bool = False):
30 | """API route for downloading video files."""
31 |
32 | raw_dl_info = download_files(
33 | passed_urls=link, is_video_request=True, preferred_res=res, convert_to_mp4=mp4
34 | )
35 |
36 | return raw_dl_info
37 |
--------------------------------------------------------------------------------
/src/db_handler/__init__.py:
--------------------------------------------------------------------------------
1 | """
2 | Database handler package for the ytflex project.
3 | Contains the database engine and the base object for all tables.
4 | """
5 |
6 | from pathlib import Path
7 | from sqlalchemy import create_engine
8 | from sqlalchemy.orm import declarative_base
9 | import config
10 |
11 | __all__ = ["_engine", "_Base"] # private namespace since it won't leave the package
12 |
13 | _db_string = config.get(
14 | "database_url",
15 | f"sqlite:///{Path(__file__).resolve().parent.parent.parent.as_posix()}"
16 | + "/mountpoint/database/ytflex_database.db",
17 | )
18 | # Create the database engine
19 | _engine = create_engine(_db_string)
20 |
21 | # Base object for all tables
22 | _Base = declarative_base()
23 | # modules are in private namespace since they are only used by the adapters
24 | # and not meant to be used outside of the db_handler package
25 | # if you're creating a new table, import _Base from here, despite the fact that it's private
26 | # it's not a very elegant solution, but it's the best I could come up with
27 | # bear with me here, thanks
28 |
--------------------------------------------------------------------------------
/config.py:
--------------------------------------------------------------------------------
1 | """
2 | Module for reading the config.yaml file.
3 |
4 | Use the get function to get the value of a setting.
5 |
6 | config.yaml file is read seperately every time the get function is called to allow
7 | hot swapping values without restarting the server.
8 | """
9 |
10 | import os
11 | import yaml
12 |
13 |
14 | def get(setting_name: str, default_value=None) -> any:
15 | """
16 | Basically an `os.getenv()` function but for the config.yaml file.
17 |
18 | Returns the value of a setting from the config.yaml file with the given `setting_name`.
19 |
20 | If the `setting_name` is not found, the `default_value` is returned if provided,
21 | otherwise `None` is returned.
22 | """
23 |
24 | this_dir = os.path.dirname(os.path.abspath(__file__))
25 | config_path = os.path.join(this_dir, "config.yaml")
26 |
27 | with open(config_path, "r", encoding="UTF_8") as file:
28 | # safe_load is used to prevent code execution from the file
29 | cfg = yaml.safe_load(file)
30 | file.close() # closing the file allows the file to be modified
31 | try:
32 | return cfg[setting_name]
33 | except KeyError:
34 | return default_value
35 |
--------------------------------------------------------------------------------
/src/api_handler/side_processes/purge_old_files.py:
--------------------------------------------------------------------------------
1 | """
2 | Utility side process to purge old files from the download directory after a certain time.
3 | """
4 |
5 | import asyncio
6 | import os
7 | import time
8 | import config
9 |
10 |
11 | async def purge_old_files():
12 | """Purge files older than max_file_age defined in config.py from download_directory"""
13 |
14 | print("File purge subprocess started.") # debug
15 |
16 | download_path = "./mountpoint/downloads"
17 | # create mountpoint if it doesn't exist
18 | if not os.path.exists(download_path):
19 | os.makedirs(download_path)
20 |
21 | while True:
22 | max_file_age = config.get("MAX_FILE_AGE")
23 |
24 | await asyncio.sleep(60) # check every minute
25 | for file in os.listdir(download_path):
26 | # get creation time of file
27 | last_change = os.path.getctime(os.path.join(download_path, file))
28 | now = time.time()
29 | file_age = int(now - last_change)
30 | # print(file, file_age)
31 | if file_age > max_file_age:
32 | print(f"Deleting {file}, {file_age} seconds old")
33 | os.remove(os.path.join(download_path, file))
34 |
--------------------------------------------------------------------------------
/src/downloader/utils/base.py:
--------------------------------------------------------------------------------
1 | """
2 | This module is used as an easy export point for all the utility functions.
3 | """
4 |
5 | from src.downloader.utils.create_download_link import create_download_link
6 | from src.downloader.utils.create_response import (
7 | create_response,
8 | create_error_response,
9 | )
10 | from src.downloader.utils.extract_metadata import extract_info
11 | from src.downloader.utils.filename_collector import FilenameCollectorPP
12 | from src.downloader.utils.ydl_opts_builder import ydl_opts_builder
13 | from src.downloader.utils.format_title import format_title
14 | from src.downloader.utils.is_playlist import is_playlist
15 | from src.downloader.utils.is_valid_url import is_valid_url
16 |
17 | # from src.downloader.utils.extract_playlist_metadata import parse_playlist
18 |
19 | # from engine.downloader.utils.check_if_file_exists import check_if_file_exists
20 |
21 | __all__ = [
22 | "create_download_link",
23 | # "check_if_file_exists",
24 | "create_response",
25 | "create_error_response",
26 | "extract_info",
27 | "FilenameCollectorPP",
28 | "ydl_opts_builder",
29 | "format_title",
30 | "is_playlist",
31 | "is_valid_url",
32 | # "parse_playlist",
33 | ]
34 |
--------------------------------------------------------------------------------
/src/downloader/utils/extract_metadata.py:
--------------------------------------------------------------------------------
1 | """
2 | Utility file to extract metadata from a video.
3 | """
4 |
5 | from yt_dlp import YoutubeDL
6 | from yt_dlp.utils import DownloadError
7 | from src.downloader.utils.video_info import VideoInfo
8 | from src.downloader.utils.format_title import format_title
9 | import config
10 |
11 |
12 | def extract_info(url: str):
13 | """Try to extract info, return None if the video is unavailable"""
14 |
15 | show_yt_dlp_output = config.get("SHOW_YT_DLP_OUTPUT")
16 |
17 | try:
18 | info = YoutubeDL({"quiet": not show_yt_dlp_output}).extract_info(
19 | url=url, download=False
20 | )
21 | except DownloadError:
22 | # if the video is unavailable, we'll just skip it
23 | return None
24 |
25 | # get the info we need
26 | try:
27 | title = info["title"]
28 | except KeyError:
29 | title = ""
30 | try:
31 | thumbnail = info["thumbnail"]
32 | except KeyError:
33 | thumbnail = ""
34 | try:
35 | duration = info["duration"]
36 | except KeyError:
37 | duration = -1
38 |
39 | title = format_title(title)
40 | return VideoInfo(title=title, duration=duration, url=url, thumbnail=thumbnail)
41 |
--------------------------------------------------------------------------------
/src/downloader/utils/extract_playlist_metadata.py:
--------------------------------------------------------------------------------
1 | """
2 | Utility function to parse a playlist and return a list of VideoInfo objects(see below).
3 | """
4 |
5 | from yt_dlp import YoutubeDL
6 | from yt_dlp.utils import DownloadError
7 | import config
8 | from src.downloader.utils.video_info import VideoInfo
9 |
10 |
11 | def parse_playlist(link: str):
12 | """Parses the playlist and returns a list of VideoInfo objects"""
13 |
14 | show_yt_dlp_output = config.get("SHOW_YT_DLP_OUTPUT")
15 |
16 | parsed_data: list[VideoInfo] = []
17 | try:
18 | data = YoutubeDL({"quiet": not show_yt_dlp_output}).extract_info(
19 | link, download=False
20 | )
21 | for index in range(len(data["entries"])):
22 | try:
23 | title = data["entries"][index]["title"]
24 | duration = data["entries"][index]["duration"]
25 | vid_link = data["entries"][index]["webpage_url"]
26 | thumbnail = data["entries"][index]["thumbnail"]
27 | parsed_data.append(VideoInfo(title, duration, vid_link, thumbnail))
28 | except KeyError:
29 | # if the video is unavailable, we'll just skip it
30 | parsed_data.append(None)
31 | except DownloadError:
32 | return []
33 | return parsed_data
34 |
--------------------------------------------------------------------------------
/main.py:
--------------------------------------------------------------------------------
1 | """
2 | Entry point for the project.
3 | """
4 |
5 | # pylint: disable=wildcard-import, unused-wildcard-import
6 | #
7 | # this is a false positive from pylint's side
8 | # you can just import the routes and not use them as long as they are registered in the FastAPI app
9 |
10 | import uvicorn
11 | import config
12 |
13 | # rebasing the imports for future changes, bare with me for a while
14 | from src.api_handler.routes import *
15 |
16 |
17 | if __name__ == "__main__":
18 | port = config.get("PORT")
19 | root_path = config.get("ROOT_PATH", "/ytflex")
20 |
21 | uvicorn.run(
22 | "src.api_handler.app:app", # FastAPI app instance, routes are registered to this app
23 | host="0.0.0.0", # listen on all interfaces, won't matter since it's running in a container
24 | port=port, # check config.yaml for the port
25 | reload=False, # reload when code changes, useful for development, set to False otherwise
26 | log_level="debug", # log level, "info" or "debug" recommended, "error" for production
27 | loop="asyncio", # use asyncio event loop for sub-processes
28 | lifespan="on", # run startup and shutdown events, not really sure what this does
29 | root_path=root_path, # root path for the API, useful for reverse proxies
30 | )
31 |
--------------------------------------------------------------------------------
/src/downloader/utils/find_appropriate_res.py:
--------------------------------------------------------------------------------
1 | """
2 | Utility to find the appropriate resolution to download.
3 | """
4 |
5 | import config
6 |
7 |
8 | def find_appropriate_res(preferred_res: int | str):
9 | """
10 | Returns the nearest resolution that is lower than the preferred resolution defined in config.py.
11 |
12 | Also attempts to convert the passed resolution to an integer if it is a string.
13 |
14 | Args:
15 | preferred_res: The preferred resolution to download.
16 | """
17 |
18 | res_list: list[int] = config.get("RES_LIST")
19 |
20 | if preferred_res == str:
21 | # someone might have accidentally passed with p at the end
22 | # being the good dev we are, we'll just remove it
23 | preferred_res = preferred_res.replace("p", "")
24 | try:
25 | preferred_res = int(preferred_res) # convert to int
26 | except ValueError:
27 | # i sincerely hope you know how resolutions and integers work if you're reading this
28 | print("Resolution must be an integer")
29 | return 720 # default to 720p
30 |
31 | if preferred_res not in res_list:
32 | # find a resolution that is lower than the preferred resolution
33 | for res in res_list:
34 | if preferred_res > res:
35 | preferred_res = res
36 | break
37 | # i really hope you're not desperate enough to download 144p
38 | # i'm not stopping you though
39 | preferred_res = max(preferred_res, res_list[-1])
40 |
41 | return preferred_res
42 |
--------------------------------------------------------------------------------
/config.yaml:
--------------------------------------------------------------------------------
1 | # Configuration options for the API
2 |
3 | # Reverse proxied path to the API.
4 | # Required for Swagger UI to work properly.
5 | # Sync with: nginx reverse proxied path
6 | ROOT_PATH: "/ytflex" # type: str
7 |
8 | # The port to run the API server on.
9 | # Sync with: docker-compose port argument, nginx reverse proxied port
10 | PORT: 2002 # type: int
11 |
12 | # The resolutions the API can attempt to download.
13 | # Comment out any resolution you don't want to be available.
14 | RES_LIST: # type: list[int]
15 | # - 2160
16 | # - 1440
17 | - 1080
18 | - 720
19 | - 480
20 | - 360
21 | - 240
22 | - 144
23 |
24 | # Maximum file size in megabytes. If a resolution is not available due to this limitation,
25 | # it will download the next best resolution.
26 | # This only limits the filesize of the video, not the audio.
27 | # If a video is 197mb and audio is 5mb, the video will be 202mb and will be downloaded anyway.
28 | # Max audio file size is the same when downloading audio only.
29 | MAX_FILE_SIZE: 200 # type: int
30 |
31 | # The IP or domain to use when generating the download link.
32 | # Has no effect on nginx or docker command, but is used in the API message returned.
33 | # Passing environment variable from docker run command will override this.
34 | IP_OR_DOMAIN: "https://ihateall.tech" # type: str
35 |
36 | # Max file age in seconds.
37 | # 3600 = 1 hour, 86400 = 1 day, 604800 = 1 week
38 | MAX_FILE_AGE: 3600 # type: int
39 |
40 | # Sets if download jobs should be silent or not.
41 | # If set to false, the output from yt-dlp will not be shown.
42 | SHOW_YT_DLP_OUTPUT: true # type: bool
43 |
44 | # Sets the allowed domains for CORS requests.
45 | # If the request is not from one of these domains, it will be rejected.
46 | # If you want to allow all domains, remove the already present domains and add "*".
47 | ALLOWED_DOMAINS: # type: list[str]
48 | - "https://ihateall.tech" # subdomains are allowed by default
49 | - "https://ytflex.vercel.app"
50 |
--------------------------------------------------------------------------------
/src/downloader/utils/create_response.py:
--------------------------------------------------------------------------------
1 | """
2 | Utility function to create a response object for the client.
3 | """
4 |
5 | import os
6 |
7 |
8 | def create_response(
9 | cdn_link: str,
10 | thumbnail: str,
11 | filename: str,
12 | duration: int,
13 | already_downloaded: bool,
14 | ):
15 | """
16 | Creates and returns a response object for the client.
17 |
18 | Args:
19 | resolution: the resolution of the file, if it's audio, it'll be None
20 | link: the download link of the file
21 | title: the title of the file
22 | thumbnail: the thumbnail of the file
23 | duration: the duration of the file
24 | filename: the filename of the file
25 | already_downloaded: whether the file was already downloaded or not
26 | """
27 |
28 | if already_downloaded:
29 | message = "File is already downloaded"
30 | else:
31 | message = "File has been downloaded successfully"
32 |
33 | response = [
34 | {
35 | "link": cdn_link,
36 | "message": message,
37 | "metadata": {
38 | "title": filename.split(".")[0],
39 | "duration": duration,
40 | "thumbnail": thumbnail,
41 | "already_downloaded": already_downloaded,
42 | "filename": filename,
43 | "extension": os.path.splitext(filename)[1],
44 | },
45 | }
46 | ]
47 |
48 | return response
49 |
50 |
51 | def create_error_response(message: str) -> dict:
52 | """
53 | Creates a response with error message.
54 |
55 | Args:
56 | error: The error message.
57 |
58 | Returns:
59 | A dictionary containing the error message.
60 | """
61 |
62 | # responses = {
63 | # "invalidurl": "Invalid URL/ID provided",
64 | # "unavailable": "Video is not accessible in our server location",
65 | # "file_too_big": "Video is too big to download",
66 | # "resolution_unavailable": "Requested resolution is not available for the video",
67 | # "invalid_resolution": "Invalid resolution provided",
68 | # }
69 | # this will be put to use in the following commit(s), pinky promise
70 |
71 | return {"message": message}
72 |
--------------------------------------------------------------------------------
/src/downloader/utils/format_title.py:
--------------------------------------------------------------------------------
1 | """
2 | Utility function to format the title of a video to be suitable for filenames.
3 | """
4 |
5 |
6 | def format_title(title: str, remove_spaces: bool = False) -> str:
7 | """
8 | Formats the title to be suitable for filenames
9 |
10 | Args:
11 | title: the string to format
12 | remove_spaces: replace spaces with underscores if set to True
13 | """
14 | # not adding these three lines are a great way learn from mistakes, trust me
15 | title = title.replace(":", "").replace("\\", "").replace("/", "").replace(".", "")
16 |
17 | # luckily spaces are allowed in filenames, but i'll leave the choice to you
18 | # just set remove_spaces to True if you want to remove spaces.
19 | # this won't be added to the config file, because it's not that important
20 | # but if you're reading this, you can add it yourself if you want to
21 | if remove_spaces:
22 | title = title.replace(" ", "_")
23 |
24 | # cyrillic characters work fine, but for the sake of consistency,
25 | # we'll replace them with latin characters
26 | def cyrillic_to_latin(text: str):
27 | """
28 | Replaces cyrillic letters in the passed string with latin letters
29 | """
30 | cyrillic_to_latin_map = {
31 | "а": "a",
32 | "б": "b",
33 | "в": "v",
34 | "г": "g",
35 | "д": "d",
36 | "е": "e",
37 | "ё": "e",
38 | "ж": "zh",
39 | "з": "z",
40 | "и": "i",
41 | "й": "i",
42 | "к": "k",
43 | "л": "l",
44 | "м": "m",
45 | "н": "n",
46 | "о": "o",
47 | "п": "p",
48 | "р": "r",
49 | "с": "s",
50 | "т": "t",
51 | "у": "u",
52 | "ф": "f",
53 | "х": "h",
54 | "ц": "ts",
55 | "ч": "ch",
56 | "ш": "sh",
57 | "щ": "shch",
58 | "ы": "y",
59 | "э": "e",
60 | "ю": "iu",
61 | "я": "ia",
62 | }
63 | return "".join(cyrillic_to_latin_map.get(char.lower(), char) for char in text)
64 |
65 | title = cyrillic_to_latin(title)
66 |
67 | return title
68 |
--------------------------------------------------------------------------------
/src/downloader/utils/ydl_opts_builder.py:
--------------------------------------------------------------------------------
1 | """
2 | Utility function to build the options for yt-dlp.
3 | """
4 |
5 | import os
6 | import config
7 | from src.downloader.utils.find_appropriate_res import find_appropriate_res
8 |
9 |
10 | def ydl_opts_builder(
11 | title: str,
12 | is_video_request: bool,
13 | preferred_res: int = 720,
14 | convert_to_mp4: bool = False,
15 | ):
16 | """
17 | Utility function for building the options for yt-dlp.
18 |
19 | Args:
20 | title: The title of the file to download.
21 |
22 | is_video_request: Whether the request is for a video or audio file.
23 |
24 | preferred_res: The preferred resolution to download. Defaults to 720p.
25 | Not used if downloading audio only.
26 |
27 | convert_to_mp4: Whether to convert the downloaded file to mp4 or not. Defaults to False.
28 | Will have no effect if downloading audio only.
29 | """
30 |
31 | download_path = "./mountpoint/downloads"
32 | max_file_size = config.get("MAX_FILE_SIZE")
33 | show_yt_dlp_output = config.get("SHOW_YT_DLP_OUTPUT")
34 |
35 | if is_video_request:
36 | # format string for yt-dlp
37 | preferred_res = find_appropriate_res(preferred_res)
38 |
39 | ydl_opts = {
40 | "format": f"bestvideo[height<={preferred_res}][filesize<{max_file_size}M]+"
41 | + f"bestaudio/best[height<={preferred_res}][filesize<{int(max_file_size/4)}M]",
42 | "outtmpl": os.path.join(download_path, f"{title}-%(height)sp.%(ext)s"),
43 | "windowsfilenames": True,
44 | "quiet": not show_yt_dlp_output,
45 | }
46 |
47 | if convert_to_mp4:
48 | ydl_opts["postprocessors"] = [
49 | {"key": "FFmpegVideoConvertor", "preferedformat": "mp4"}
50 | ]
51 |
52 | else:
53 | ydl_opts = {
54 | "format": f"bestaudio/best[filesize<{int(max_file_size)}M]",
55 | "outtmpl": os.path.join(download_path, f"{title}"),
56 | "windowsfilenames": True,
57 | "quiet": not show_yt_dlp_output,
58 | "postprocessors": [
59 | {
60 | "key": "FFmpegExtractAudio",
61 | "preferredcodec": "mp3",
62 | "preferredquality": "192",
63 | }
64 | ],
65 | }
66 |
67 | return ydl_opts
68 |
--------------------------------------------------------------------------------
/src/db_handler/tables.py:
--------------------------------------------------------------------------------
1 | # pylint: disable=invalid-name
2 | #
3 | # API_Key is technically not PascalCase
4 |
5 | """This module contains the SQLAlchemy tables for the database."""
6 |
7 | import datetime
8 | from dataclasses import dataclass
9 | from sqlalchemy import (
10 | Column,
11 | Integer,
12 | String,
13 | DateTime,
14 | Boolean,
15 | SmallInteger,
16 | )
17 | from sqlalchemy.orm import Session
18 | from src.db_handler import _Base, _engine
19 |
20 | __all__ = [
21 | "DownloadLog",
22 | "Api_Key",
23 | ] # classes the objects of which are to be used in the code
24 |
25 |
26 | def get_session():
27 | """Returns a new session to the database."""
28 | return Session(_engine)
29 |
30 |
31 | @dataclass
32 | class DownloadLog(_Base):
33 | """Table for storing download information."""
34 |
35 | __tablename__ = "ytflex_downloads"
36 | id = Column(Integer, primary_key=True, autoincrement=True)
37 | request_string = Column(
38 | String(200), nullable=False
39 | ) # parsed ID of the video, can track parsing errors with this
40 | time_of_request = Column(
41 | DateTime, nullable=False, default=datetime.datetime.now
42 | ) # pass the time of the request no matter what
43 | type_of_request = Column(
44 | String(10), nullable=False
45 | ) # type of request, either "video", "audio", "playlist_video" or "playlist_audio"
46 | status_code = Column(SmallInteger) # inside codes for the status of the download
47 | requested_resolution = Column(SmallInteger)
48 | downloaded_resolution = Column(SmallInteger)
49 | title = Column(String(100))
50 | duration = Column(SmallInteger)
51 | api_key = Column(String(40))
52 | file_size = Column(SmallInteger)
53 | time_of_response = Column(DateTime)
54 | playlist_url_count = Column(SmallInteger)
55 | custom_file_life = Column(SmallInteger)
56 |
57 |
58 | @dataclass
59 | class Api_Key(_Base):
60 | """Table for storing API keys."""
61 |
62 | @staticmethod
63 | def _default_expiration():
64 | """Returns datetime object of 30 days from now."""
65 | return datetime.datetime.now() + datetime.timedelta(days=30)
66 |
67 | __tablename__ = "ytflex_api_keys"
68 | id = Column(Integer, primary_key=True, autoincrement=True)
69 | key = Column(String(40), nullable=False)
70 | owner = Column(String(40), nullable=False)
71 | created_at = Column(DateTime, nullable=False, default=datetime.datetime.now)
72 | tier = Column(SmallInteger, default=1)
73 | is_active = Column(Boolean, default=True)
74 | additional_file_life_requested = Column(SmallInteger, default=0) # in hours
75 | credits_used = Column(
76 | Integer, default=0
77 | ) # hours * video size, no penalty for < 1 hour
78 | last_used = Column(DateTime)
79 | last_ip = Column(String(20))
80 | usage_count = Column(Integer, default=0, autoincrement=True)
81 | expires_at = Column(DateTime, default=_default_expiration)
82 |
83 |
84 | _Base.metadata.create_all(_engine)
85 |
--------------------------------------------------------------------------------
/src/downloader/runner.py:
--------------------------------------------------------------------------------
1 | """
2 | The head file of all utility functions. This file will be imported by the API handler.
3 | Will contain all logic involved in downloading files by either importing from other files or
4 | by having the logic here.
5 | """
6 |
7 | # pylint: disable=too-many-branches
8 | #
9 | # this file is the main file for the downloader module
10 | # and it requires too many branches to be reduced into a sane number
11 | # so we disable the warning for this file only
12 |
13 | # pylint: disable=wildcard-import
14 | #
15 | # the wildcard import is used to import all utility functions which are
16 | # going to be used in this file.
17 |
18 | # pylint: disable=fixme
19 | #
20 | # this is for pylint to not fail the build because of the fixme comments
21 |
22 |
23 | import os
24 | from yt_dlp import YoutubeDL
25 | from src.downloader.utils.base import *
26 |
27 |
28 | def download_files(
29 | passed_urls: list[str] | str,
30 | is_video_request: bool,
31 | preferred_res: int = 720,
32 | convert_to_mp4: bool = False,
33 | ):
34 | """
35 | Downloads files from youtube using yt-dlp.
36 | If a preferred resolution is given, it will attempt to download that resolution.
37 | If the preferred resolution is not available, it will download the next best resolution.
38 | If no preferred resolution is given, it will download audio only instead.
39 |
40 | Args:
41 | passed_urls: List of urls to download, or a single url as a string.
42 | Incompatible with multiple playlists.
43 |
44 | is_video_request: Whether the request is for a video or audio file.
45 |
46 | preferred_res: The preferred resolution to download. Defaults to 720p.
47 | If not available, audio will be downloaded instead.
48 |
49 | convert_to_mp4: Whether to convert the downloaded file to mp4 or not. Defaults to False.
50 | Will have no effect if downloading audio only.
51 | """
52 |
53 | # url's will be collected here
54 | parsed_links_list = []
55 |
56 | if isinstance(passed_urls, str):
57 | # check if url is valid
58 | if not is_valid_url(passed_urls):
59 | return create_error_response("Invalid URL")
60 |
61 | # check if url is playlist
62 | if is_playlist(passed_urls):
63 | return create_error_response(
64 | "Can't download playlists. Please try again with a single video."
65 | )
66 | # this will be re-enabled once i figure out the rest of the code.
67 | # parsed_links_list = parse_playlist(passed_urls)
68 |
69 | # if not a playlist then it's a single video
70 | parsed_links_list.append(extract_info(passed_urls))
71 |
72 | elif isinstance(passed_urls, list):
73 | for video in passed_urls:
74 | # can't have multiple playlists
75 | if is_playlist(video):
76 | return create_error_response(
77 | "Can't download multiple playlists. "
78 | + "Please try again with a single playlist."
79 | )
80 | # check url's are valid
81 | if not is_valid_url(video):
82 | # remove invalid url's from list
83 | passed_urls.remove(video)
84 |
85 | # now we can parse the list
86 | for video in passed_urls:
87 | parsed_links_list += extract_info(video)
88 |
89 | # if no urls are valid
90 | if parsed_links_list == []:
91 | return create_error_response(
92 | "Invalid URL(s) passed. " + "Please check your URL(s) and try again."
93 | )
94 |
95 | # create a list of download info per url
96 | download_info = []
97 | for video in parsed_links_list:
98 | if video is None:
99 | download_info.append([])
100 | continue
101 |
102 | # format title
103 | video.title = format_title(video.title)
104 |
105 | # this function will create the required options for yt-dlp
106 | # regardless of whether the request is for a video or audio file
107 | # by checking if the request is for a video or audio file internally
108 | ydl_opts = ydl_opts_builder(
109 | video.title, is_video_request, preferred_res, convert_to_mp4
110 | )
111 |
112 | # create a download object
113 | filename_collector = FilenameCollectorPP()
114 | ydl = YoutubeDL(ydl_opts)
115 | ydl.add_post_processor(filename_collector)
116 | ydl.download([video.url])
117 | last_downloaded_dir: str = filename_collector.filenames[-1]
118 | filename: str = os.path.basename(last_downloaded_dir)
119 |
120 | cdn_link: str = create_download_link(filename)
121 |
122 | download_info.append(
123 | create_response(cdn_link, video.thumbnail, filename, video.duration, False)
124 | )
125 |
126 | return download_info
127 |
--------------------------------------------------------------------------------
/readme.md:
--------------------------------------------------------------------------------
1 | # YTFlex
2 |
3 | [](https://hub.docker.com/r/kirellkekw/ytflex)
4 |
5 | [](https://github.com/pylint-dev/pylint)
6 | [](https://github.com/kirellkekw/YTFlex/actions/workflows/deploy_to_server.yml)
7 | [](https://github.com/psf/black)
8 |
9 | Deployment ready, easy to use and fast YouTube downloader API written in Python with CDN and reverse proxy setup guide.
10 |
11 | []()
12 | []()
13 | []()
14 | []()
15 | []()
16 | []()
17 |
18 | # 1- Installing the requirements
19 |
20 | This guide assumes you have a Linux machine with root access, a public IP address, or a domain name, and the required storage space for the downloaded files.
21 |
22 | ## 1.1- Download the dependencies
23 |
24 | * Docker
25 | * Nginx
26 |
27 | For detailed instructions on how to install those, please refer to the following links:
28 |
29 | * [Docker Install Guide](https://docs.docker.com/engine/install/)
30 | * [Nginx Install Guide](https://www.nginx.com/resources/wiki/start/topics/tutorials/install/)
31 |
32 | # 2- Setting up the API server with Docker
33 |
34 | * This guide has been tested on NixOS with distro specific commands on both Docker and Podman OCI runtimes, but it has been written with the assumption that you are using Docker on a Debian based distro. If you are using a different distro, you might need to change the commands accordingly.
35 |
36 | ## 2.1- Clone the repository(and change directory to it)
37 |
38 | ```bash
39 | git clone https://github.com/kirellkekw/YTFlex.git
40 | cd YTFlex
41 | ```
42 |
43 | ## 2.2- Edit the configuration file
44 |
45 | * Open the `config.yaml` file and change the settings according to your needs with your favorite text editor.
46 |
47 | * Here's a quick explanation of what each setting does:
48 |
49 | | Setting | Description | Type |
50 | | --- | --- | --- |
51 | | `res_list` | A list of resolutions API can attempt to download. | `list[int]` |
52 | | `root_path` | Reverse proxied path to the API. | `str` |
53 | | `max_file_size` | The maximum file size in megabytes the API can download. | `int` |
54 | | `port` | The port the API server will listen on. | `int` |
55 | | `ip_or_domain` | The address API will use to create a CDN link. If you pass this value in docker run command, config.yaml value will be ignored. | `str` |
56 | | `max_file_age` | Maximum age of a file in seconds before it gets deleted. | `int` |
57 | | `show_yt_dlp_output` | Decides if yt_dlp output is printed to the console or not. | `bool` |
58 | | `allowed_domains` | A list of allowed domains for CORS requests. | `list[str]` |
59 |
60 | ## 2.3- Edit the docker-compose file
61 |
62 | * Open `docker-compose.yml` and change the values of `volumes` to the path you want to attach to the container. You can also change the port the container will listen on, but you should also change the Nginx configuration file accordingly.
63 |
64 | ## 2.4- Build the Docker image
65 |
66 | ```bash
67 | sudo docker build -t ytflex .
68 | ```
69 |
70 | ## 2.5- Run the image with docker-compose
71 |
72 | ```bash
73 | sudo docker-compose up -d
74 | ```
75 |
76 | * You can later stop and remove the container with the following commands:
77 |
78 | ```bash
79 | sudo docker stop ytflex
80 | sudo docker remove ytflex
81 | ```
82 |
83 | * Or you can change directory to the cloned repository and run the following command to stop the container:
84 |
85 | ```bash
86 | sudo docker-compose down
87 | ```
88 |
89 | # 3- Running the CDN server with Nginx
90 |
91 | ## 3.1- Create new location blocks in your Nginx configuration file, usually located at `/etc/nginx/nginx.conf`
92 |
93 | ```nginx
94 | # use your favorite text editor add the following server blocks to your nginx.conf as you see fit
95 | server {
96 | # other server blocks and listen directives
97 |
98 | location /ytflex/ {
99 | proxy_pass http://127.0.0.1:2002/; # remember to change the port if you have changed it in the docker-compose file
100 | proxy_set_header Host $host;
101 | proxy_set_header X-Real-IP $remote_addr;
102 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
103 | proxy_set_header X-Forwarded-Proto $scheme;
104 | }
105 | location /ytflex-cdn/ {
106 | alias /app/ytflex/downloads/; # change this to your download path attached to the container, or leave as is if you're using the default path
107 | add_header Content-Disposition 'attachment'; # forces browser to download the file instead of playing it
108 | }
109 | }
110 |
111 | ```
112 |
113 | ## 3.2- Restart Nginx
114 |
115 | * The way you restart Nginx depends on your distro, but in a Debian based distro, you can restart Nginx with the following command:
116 |
117 | ```bash
118 | sudo systemctl restart nginx
119 | ```
120 |
121 | ## 3.3- Test the server
122 |
123 | Try to access the following URL in your browser:
124 |
125 | `http://localhost/ytflex/root`
126 |
127 | If you get a JSON response with the following content, then you are good to go!
128 |
129 | ```json
130 | {"message": "Hello World"}
131 | ```
132 |
133 | # 4- TODO
134 |
135 | ***(in no particular order)***
136 |
137 | * [x] Add support for multiple file resolutions
138 | * [x] Add option to limit the file size of the downloaded files
139 | * [x] Add support for Docker for easier deployment
140 | * [x] Add support for Nginx for reverse proxying and CDN
141 | * [x] Add CDN support
142 | * [x] Write a proper readme(ironic, isn't it?)
143 | * [x] Add option to purge files after a certain amount of time
144 | * [x] Add option to download mp3 files(this is actually easier than video files ~~,but i haven't gotten around to it yet~~)
145 | * [x] Handle multiple file links more gracefully
146 | * [x] Add graceful error handling for invalid links
147 | * [x] Open source the project
148 | * [x] Restructure the backend to be more modular
149 | * [x] Add option to download playlists
150 | * [x] Allow passing video or playlist id's as a parameter instead of a link
151 | * [ ] Add a control block before downloads to prevent redownloading same files
152 | * [ ] Anonymize the file access links
153 | * [ ] Add a frontend
154 | * [ ] Create a special message if:
155 | * [x] The file is not available for download, or if the link is invalid
156 | * [ ] If the file is already downloaded and not expired
157 | * [ ] The file is too large
158 | * [ ] The file is too long
159 | * [ ] The file is not available in the requested resolution
160 | * [ ] Add support to sqlite database for:
161 | * [ ] Logging downloads
162 | * [ ] Logging errors
163 | * [ ] Logging file purges
164 | * [ ] Adding API key support for uncapped file size and higher resolution
165 | * [ ] Adding option to choose when a file will be purged to user
166 | * [ ] Adding a mechanism to limit the number of concurrent downloads per IP to prevent abuse
167 |
--------------------------------------------------------------------------------
/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 | Video Downloader
8 |
122 |
123 |
124 |
125 |
148 |
149 |
235 |
236 |
237 |
--------------------------------------------------------------------------------