├── tests ├── __init__.py ├── cli │ ├── __init__.py │ └── test_detect_compat_mode.py ├── core │ ├── __init__.py │ └── test_get_extractors.py ├── ytdl │ ├── __init__.py │ └── test_extractors_registry.py ├── config │ ├── __init__.py │ ├── conftest.py │ ├── test_get_config_path.py │ ├── test_get_config_home.py │ ├── test_config_load.py │ └── test_get_data_home.py ├── extractor │ ├── __init__.py │ ├── test_peqn.py │ └── test_plugin.py ├── testlib.py └── conftest.py ├── .github └── FUNDING.yml ├── src └── dl_plus │ ├── __main__.py │ ├── cli │ ├── __init__.py │ ├── commands │ │ ├── config │ │ │ ├── __init__.py │ │ │ └── show │ │ │ │ ├── __init__.py │ │ │ │ ├── main.py │ │ │ │ └── backends.py │ │ ├── __init__.py │ │ ├── extractor │ │ │ ├── __init__.py │ │ │ ├── uninstall.py │ │ │ ├── update.py │ │ │ ├── install.py │ │ │ ├── base.py │ │ │ └── list.py │ │ ├── backend │ │ │ ├── __init__.py │ │ │ ├── uninstall.py │ │ │ ├── info.py │ │ │ ├── list.py │ │ │ ├── update.py │ │ │ ├── install.py │ │ │ └── base.py │ │ └── base.py │ ├── args.py │ ├── command.py │ └── cli.py │ ├── const.py │ ├── utils.py │ ├── extractor │ ├── __init__.py │ ├── extractor.py │ ├── machinery.py │ ├── peqn.py │ └── plugin.py │ ├── exceptions.py │ ├── deprecated.py │ ├── core.py │ ├── ytdl.py │ ├── pypi.py │ ├── backend.py │ └── config.py ├── requirements.dev.txt ├── .gitignore ├── .flake8 ├── .editorconfig ├── scripts ├── build_pyz.py └── build_exe.py ├── tox.ini ├── justfile ├── LICENSE ├── dl-plus.spec ├── docs ├── available-extractor-plugins.md └── extractor-plugin-authoring-guide.md ├── pyproject.toml ├── README.md └── CHANGELOG.md /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/cli/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/core/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/ytdl/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/config/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/extractor/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | ko_fi: un1def 2 | -------------------------------------------------------------------------------- /src/dl_plus/__main__.py: -------------------------------------------------------------------------------- 1 | from dl_plus.cli import main 2 | 3 | 4 | main() 5 | -------------------------------------------------------------------------------- /requirements.dev.txt: -------------------------------------------------------------------------------- 1 | flake8 2 | isort 3 | mypy 4 | pytest 5 | tox 6 | build 7 | -------------------------------------------------------------------------------- /src/dl_plus/cli/__init__.py: -------------------------------------------------------------------------------- 1 | from .cli import main 2 | 3 | 4 | __all__ = ['main'] 5 | -------------------------------------------------------------------------------- /src/dl_plus/const.py: -------------------------------------------------------------------------------- 1 | DL_PLUS_VERSION = '0.10.1' 2 | PLUGINS_PACKAGE = 'dl_plus.extractors' 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .* 2 | !.gitignore 3 | !.editorconfig 4 | !.flake8 5 | !.github/ 6 | 7 | *.py[co] 8 | 9 | build/ 10 | dist/ 11 | *.egg-info/ 12 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | filename = 3 | ./setup.py 4 | ./src/**.py 5 | ./tests/**.py 6 | ./scripts/**.py 7 | ./dl-plus.spec 8 | show_source = true 9 | statistics = true 10 | ignore = F824, W503 11 | -------------------------------------------------------------------------------- /src/dl_plus/cli/commands/config/__init__.py: -------------------------------------------------------------------------------- 1 | from dl_plus.cli.commands.base import CommandGroup 2 | 3 | from .show import ConfigShowCommandGroup 4 | 5 | 6 | class ConfigCommandGroup(CommandGroup): 7 | short_description = 'Config commands' 8 | 9 | commands = ( 10 | ConfigShowCommandGroup, 11 | ) 12 | -------------------------------------------------------------------------------- /src/dl_plus/utils.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | 3 | 4 | class NotSet(Enum): 5 | # Am I the only one who finds this hideous? 6 | # https://github.com/python/typing/issues/236 7 | # https://www.python.org/dev/peps/pep-0484/#support-for-singleton-types\ 8 | # -in-unions 9 | NOTSET = 0 10 | 11 | 12 | NOTSET = NotSet.NOTSET 13 | -------------------------------------------------------------------------------- /tests/config/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | @pytest.fixture 5 | def config_home(): 6 | # set dl_plus.config._config_home to None to disable caching 7 | return None 8 | 9 | 10 | @pytest.fixture(autouse=True) 11 | def _autopatch_config_home(monkeypatch, config_home): 12 | monkeypatch.setattr('dl_plus.config._config_home', config_home) 13 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | charset = utf-8 5 | end_of_line = lf 6 | insert_final_newline = true 7 | trim_trailing_whitespace = true 8 | 9 | [*.py] 10 | indent_style = space 11 | indent_size = 4 12 | 13 | [*.md] 14 | indent_style = space 15 | indent_size = 2 16 | 17 | [{*.toml,*.ini,.flake8}] 18 | indent_style = space 19 | indent_size = 4 20 | 21 | [Makefile] 22 | indent_style = tab 23 | indent_size = 8 24 | -------------------------------------------------------------------------------- /scripts/build_pyz.py: -------------------------------------------------------------------------------- 1 | import os 2 | import zipapp 3 | from pathlib import Path 4 | 5 | 6 | repo_dir = Path(__file__).parent.parent 7 | os.chdir(repo_dir) 8 | 9 | dest_dir = Path('dist') 10 | os.makedirs(dest_dir, exist_ok=True) 11 | zipapp.create_archive( 12 | source='src', 13 | target=dest_dir / 'dl-plus', 14 | interpreter='/usr/bin/env python3', 15 | main='dl_plus.cli:main', 16 | compressed=True, 17 | ) 18 | -------------------------------------------------------------------------------- /tests/testlib.py: -------------------------------------------------------------------------------- 1 | class ExtractorMock(type): 2 | 3 | def __new__(cls, name): 4 | return super().__new__(cls, name, (), {'IE_NAME': name}) 5 | 6 | def __repr__(self): 7 | return f'' 8 | 9 | def __eq__(self, other): 10 | if not isinstance(other, ExtractorMock): 11 | return NotImplemented 12 | return self.IE_NAME == other.IE_NAME 13 | 14 | def __hash__(self): 15 | return hash(self.IE_NAME) 16 | -------------------------------------------------------------------------------- /src/dl_plus/extractor/__init__.py: -------------------------------------------------------------------------------- 1 | from .plugin import ExtractorPlugin 2 | 3 | 4 | _LAZY_LOAD_NAMES = [ 5 | 'Extractor', 6 | 'ExtractorError', 7 | ] 8 | 9 | __all__ = _LAZY_LOAD_NAMES + ['ExtractorPlugin'] 10 | 11 | 12 | def __getattr__(name): 13 | if name not in _LAZY_LOAD_NAMES: 14 | raise AttributeError(name) 15 | from . import extractor 16 | _globals = globals() 17 | for _name in _LAZY_LOAD_NAMES: 18 | _globals[_name] = getattr(extractor, _name) 19 | return _globals[name] 20 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = 3 | py39 4 | py310 5 | py311 6 | py312 7 | py313 8 | flake8 9 | isort 10 | 11 | [testenv] 12 | deps = 13 | yt-dlp 14 | pytest 15 | commands = pytest {posargs} 16 | 17 | [testenv:flake8] 18 | skip_install = true 19 | deps = flake8 20 | commands = flake8 {posargs} 21 | 22 | [testenv:isort] 23 | skip_install = true 24 | deps = isort 25 | commands = isort {posargs:. -c} 26 | 27 | [testenv:mypy] 28 | skip_install = true 29 | deps = mypy 30 | commands = mypy {posargs} 31 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pathlib 3 | 4 | 5 | def pytest_addoption(parser): 6 | parser.addoption('--backend') 7 | 8 | 9 | def pytest_configure(config): 10 | dlp_vars = [key for key in os.environ if key.startswith('DL_PLUS_')] 11 | for dlp_var in dlp_vars: 12 | del os.environ[dlp_var] 13 | 14 | from dl_plus import config as conf 15 | from dl_plus.backend import init_backend 16 | 17 | conf._config_home = conf._data_home = pathlib.Path('fake-dl-plus-home') 18 | init_backend(config.getoption('backend')) 19 | -------------------------------------------------------------------------------- /src/dl_plus/cli/commands/__init__.py: -------------------------------------------------------------------------------- 1 | from dl_plus.cli import args as cli_args 2 | 3 | from .backend import BackendCommandGroup 4 | from .base import CommandGroup 5 | from .config import ConfigCommandGroup 6 | from .extractor import ExtractorCommandGroup 7 | 8 | 9 | class RootCommandGroup(CommandGroup): 10 | short_description = 'commands' 11 | 12 | arguments = ( 13 | cli_args.dlp_config, 14 | ) 15 | 16 | commands = ( 17 | BackendCommandGroup, 18 | ExtractorCommandGroup, 19 | ConfigCommandGroup, 20 | ) 21 | -------------------------------------------------------------------------------- /src/dl_plus/cli/commands/extractor/__init__.py: -------------------------------------------------------------------------------- 1 | from dl_plus.cli.commands.base import CommandGroup 2 | 3 | from .install import ExtractorInstallCommand 4 | from .list import ExtractorListCommand 5 | from .uninstall import ExtractorUninstallCommand 6 | from .update import ExtractorUpdateCommand 7 | 8 | 9 | class ExtractorCommandGroup(CommandGroup): 10 | 11 | short_description = 'Extractors management commands' 12 | 13 | commands = ( 14 | ExtractorListCommand, 15 | ExtractorInstallCommand, 16 | ExtractorUninstallCommand, 17 | ExtractorUpdateCommand, 18 | ) 19 | -------------------------------------------------------------------------------- /src/dl_plus/cli/commands/extractor/uninstall.py: -------------------------------------------------------------------------------- 1 | from dl_plus.cli.args import Arg, assume_yes_arg 2 | from dl_plus.cli.commands.base import BaseUninstallCommand 3 | 4 | from .base import ExtractorInstallUninstallUpdateCommandMixin 5 | 6 | 7 | class ExtractorUninstallCommand( 8 | ExtractorInstallUninstallUpdateCommandMixin, BaseUninstallCommand, 9 | ): 10 | 11 | short_description = 'Uninstall extractor plugin' 12 | 13 | arguments = ( 14 | Arg( 15 | 'name', metavar='NAME', 16 | help='Extractor plugin name.' 17 | ), 18 | assume_yes_arg, 19 | ) 20 | -------------------------------------------------------------------------------- /tests/cli/test_detect_compat_mode.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from dl_plus.cli.cli import _detect_compat_mode 4 | 5 | 6 | @pytest.mark.parametrize('name', [ 7 | 'yt-dlp', 'yt-dlp.exe', 8 | '/path/to/bin/yt-dlp', '/path/to/bin/yt-dlp.exe', 9 | 'youtube-dl', 'youtube-dl.exe', 10 | ]) 11 | def test_compat_mode(name): 12 | assert _detect_compat_mode(name) is True 13 | 14 | 15 | @pytest.mark.parametrize('name', [ 16 | 'dl-plus', 'dl-plus.exe', 17 | 'youtube-dl-enhanced', 18 | 'yt-dlp.enhanced', 19 | '/path/to/yt_dlp/__main__.py', 20 | ]) 21 | def test_full_mode(name): 22 | assert _detect_compat_mode(name) is False 23 | -------------------------------------------------------------------------------- /src/dl_plus/cli/commands/backend/__init__.py: -------------------------------------------------------------------------------- 1 | from dl_plus.cli.commands.base import CommandGroup 2 | 3 | from .info import BackendInfoCommand 4 | from .install import BackendInstallCommand 5 | from .list import BackendListCommand 6 | from .uninstall import BackendUninstallCommand 7 | from .update import BackendUpdateCommand 8 | 9 | 10 | class BackendCommandGroup(CommandGroup): 11 | 12 | short_description = 'Backend management commands' 13 | 14 | commands = ( 15 | BackendListCommand, 16 | BackendInfoCommand, 17 | BackendInstallCommand, 18 | BackendUninstallCommand, 19 | BackendUpdateCommand, 20 | ) 21 | -------------------------------------------------------------------------------- /src/dl_plus/exceptions.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, Union 2 | 3 | 4 | class DLPlusException(Exception): 5 | 6 | def __init__(self, message: Union[str, Exception, None] = None) -> None: 7 | if message is not None: 8 | message = str(message) 9 | self._message = message 10 | 11 | def __str__(self): 12 | if self._message is None: 13 | if self.__cause__: 14 | self._message = str(self.__cause__) 15 | else: 16 | self._message = '' 17 | return self._message 18 | 19 | @property 20 | def error(self) -> Optional[BaseException]: 21 | return self.__cause__ 22 | -------------------------------------------------------------------------------- /src/dl_plus/cli/commands/extractor/update.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from dl_plus.cli.args import Arg 4 | from dl_plus.cli.commands.base import BaseUpdateCommand 5 | 6 | from .base import ExtractorInstallUninstallUpdateCommandMixin 7 | 8 | 9 | class ExtractorUpdateCommand( 10 | ExtractorInstallUninstallUpdateCommandMixin, BaseUpdateCommand, 11 | ): 12 | 13 | short_description = 'Update extractor plugin' 14 | 15 | arguments = ( 16 | Arg( 17 | 'name', metavar='NAME', 18 | help='Extractor plugin name.' 19 | ), 20 | ) 21 | 22 | def get_project_name(self) -> str: 23 | return self.project_name 24 | -------------------------------------------------------------------------------- /src/dl_plus/cli/commands/config/show/__init__.py: -------------------------------------------------------------------------------- 1 | from dl_plus.cli.args import Arg, ExclusiveArgGroup 2 | from dl_plus.cli.commands.base import CommandGroup 3 | 4 | from .backends import ConfigShowBackendsCommand 5 | from .main import ConfigShowMainCommand 6 | 7 | 8 | class ConfigShowCommandGroup(CommandGroup): 9 | short_description = 'Config show commands' 10 | 11 | arguments = ( 12 | ExclusiveArgGroup( 13 | Arg('--default', action='store_true', help='Show default config.'), 14 | Arg('--merged', action='store_true', help='Show merged configs.'), 15 | title='show options', 16 | ), 17 | ) 18 | 19 | commands = ( 20 | ConfigShowMainCommand, 21 | ConfigShowBackendsCommand, 22 | ) 23 | -------------------------------------------------------------------------------- /scripts/build_exe.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess 3 | import tempfile 4 | import venv 5 | from pathlib import Path 6 | 7 | 8 | repo_dir = Path(__file__).parent.parent 9 | os.chdir(repo_dir) 10 | 11 | with tempfile.TemporaryDirectory() as tmp: 12 | work_dir = Path(tmp) 13 | env_dir = work_dir / 'env' 14 | env = venv.EnvBuilder(with_pip=True) 15 | # EnvBuilder.create() doesn't return context 16 | ctx = env.ensure_directories(env_dir) 17 | env.create(env_dir) 18 | pyexe = ctx.env_exe 19 | subprocess.check_call([ 20 | pyexe, '-m', 'pip', 'install', 'pyinstaller', 'stdlib-list']) 21 | subprocess.check_call([ 22 | pyexe, '-m', 'PyInstaller', 23 | '--clean', 24 | '--distpath', 'dist', 25 | '--workpath', str(work_dir), 26 | 'dl-plus.spec', 27 | ]) 28 | -------------------------------------------------------------------------------- /src/dl_plus/deprecated.py: -------------------------------------------------------------------------------- 1 | import warnings 2 | from typing import Optional 3 | 4 | 5 | __all__ = ['DLPlusDeprecationWarning', 'warn'] 6 | 7 | 8 | _filters_configured = False 9 | 10 | 11 | def _maybe_configure_filters(): 12 | global _filters_configured 13 | if not _filters_configured: 14 | warnings.filterwarnings('default', category=DLPlusDeprecationWarning) 15 | _filters_configured = True 16 | 17 | 18 | def _calculate_stacklevel(stacklevel: Optional[int]) -> int: 19 | if stacklevel is None: 20 | stacklevel = 2 21 | return stacklevel + 1 22 | 23 | 24 | class DLPlusDeprecationWarning(DeprecationWarning): 25 | pass 26 | 27 | 28 | def warn(message: str, stacklevel: Optional[int] = None) -> None: 29 | _maybe_configure_filters() 30 | warnings.warn( 31 | message, category=DLPlusDeprecationWarning, 32 | stacklevel=_calculate_stacklevel(stacklevel), 33 | ) 34 | -------------------------------------------------------------------------------- /src/dl_plus/cli/commands/backend/uninstall.py: -------------------------------------------------------------------------------- 1 | from dl_plus.cli.args import Arg, assume_yes_arg 2 | from dl_plus.cli.commands.base import BaseUninstallCommand 3 | 4 | from .base import BackendInstallUninstallUpdateCommandMixin 5 | 6 | 7 | class BackendUninstallCommand( 8 | BackendInstallUninstallUpdateCommandMixin, BaseUninstallCommand, 9 | ): 10 | 11 | short_description = 'Uninstall backend' 12 | 13 | arguments = ( 14 | Arg( 15 | 'name', metavar='NAME', 16 | help='Backend plugin name.' 17 | ), 18 | assume_yes_arg, 19 | ) 20 | 21 | fallback_to_config = False 22 | allow_autodetect = False 23 | init_backend = True 24 | 25 | def init(self) -> None: 26 | super().init() 27 | package_dir = self.get_package_dir() 28 | assert self.backend_info is not None 29 | if not package_dir.exists() and not self.backend_info.is_managed: 30 | name = self.get_short_name() 31 | self.die(f'{name} is not managed by dl-plus, unable to uninstall') 32 | -------------------------------------------------------------------------------- /justfile: -------------------------------------------------------------------------------- 1 | src_dir := justfile_dir() / 'src' 2 | scripts_dir := justfile_dir() / 'scripts' 3 | python := 'python3' 4 | _pythonpath := env('PYTHONPATH', '') 5 | export PYTHONPATH := if _pythonpath == '' { src_dir } else { src_dir + ':' + _pythonpath } 6 | 7 | 8 | @_list: 9 | just --list --unsorted 10 | 11 | @clean: 12 | find {{src_dir}} -depth -name '__pycache__' -type d -exec rm -rf '{}' \; 13 | find {{src_dir}} -name '*.py[co]' -type f -delete 14 | rm -rf build dist {{src_dir}}/*.egg-info 15 | 16 | dist: clean 17 | {{python}} -m build 18 | 19 | exe: clean 20 | {{python}} {{scripts_dir}}/build_exe.py 21 | 22 | pyz: clean 23 | {{python}} {{scripts_dir}}/build_pyz.py 24 | 25 | fix: 26 | isort . 27 | 28 | lint: 29 | isort . -c 30 | flake8 31 | 32 | [positional-arguments] 33 | @test *args: 34 | pytest "${@}" 35 | 36 | [positional-arguments] 37 | @tox *args: 38 | tox run "${@}" 39 | 40 | [positional-arguments] 41 | @run *args: 42 | {{python}} -m dl_plus "${@}" 43 | 44 | [positional-arguments] 45 | @cmd *args: 46 | just run --cmd "${@}" 47 | -------------------------------------------------------------------------------- /src/dl_plus/cli/commands/backend/info.py: -------------------------------------------------------------------------------- 1 | from dl_plus.cli.args import Arg 2 | from dl_plus.cli.commands.base import Command 3 | 4 | from .base import BackendCommandMixin 5 | 6 | 7 | class BackendInfoCommand(BackendCommandMixin, Command): 8 | 9 | short_description = 'Show backend information' 10 | 11 | arguments = ( 12 | Arg( 13 | 'name', nargs='?', metavar='NAME', 14 | help='Backend name.' 15 | ), 16 | ) 17 | 18 | fallback_to_config = True 19 | allow_autodetect = True 20 | init_backend = True 21 | 22 | def run(self): 23 | backend_info = self.backend_info 24 | assert backend_info is not None 25 | metadata = backend_info.metadata 26 | if metadata: 27 | self.print('project name:', metadata.name) 28 | self.print('project version:', metadata.version) 29 | self.print('import name:', backend_info.import_name) 30 | self.print('version:', backend_info.version) 31 | self.print('path:', str(backend_info.path)) 32 | self.print('managed:', 'yes' if backend_info.is_managed else 'no') 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020–2025 Dmitry Meyer 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/dl_plus/cli/commands/extractor/install.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import Optional, Tuple 4 | 5 | from dl_plus.cli.args import Arg 6 | from dl_plus.cli.commands.base import BaseInstallCommand 7 | 8 | from .base import ExtractorInstallUninstallUpdateCommandMixin 9 | 10 | 11 | class ExtractorInstallCommand( 12 | ExtractorInstallUninstallUpdateCommandMixin, BaseInstallCommand, 13 | ): 14 | 15 | short_description = 'Install extractor plugin' 16 | 17 | arguments = ( 18 | Arg( 19 | 'name', metavar='NAME', 20 | help='Extractor plugin name.' 21 | ), 22 | Arg( 23 | 'version', nargs='?', metavar='VERSION', 24 | help='Extractor plugin version. Default is latest.', 25 | ), 26 | Arg( 27 | '-f', '--force', action='store_true', 28 | help='Force installation if the same version is already installed.' 29 | ), 30 | ) 31 | 32 | def get_project_name_version_tuple(self) -> Tuple[str, Optional[str]]: 33 | return (self.project_name, self.args.version) 34 | 35 | def get_force_flag(self) -> bool: 36 | return self.args.force 37 | -------------------------------------------------------------------------------- /dl-plus.spec: -------------------------------------------------------------------------------- 1 | from stdlib_list import stdlib_list 2 | 3 | 4 | hiddenimports = [ 5 | m for m in stdlib_list() 6 | if not m.startswith('ensurepip') 7 | and not m.startswith('idlelib') 8 | and not m.startswith('lib2to3') 9 | and not m.startswith('test') 10 | and not m.startswith('tkinter') 11 | and not m.startswith('turtle') 12 | and not m.startswith('venv') 13 | ] 14 | 15 | 16 | a = Analysis( # noqa: F821 17 | ['src\\dl_plus\\__main__.py'], 18 | pathex=['src'], 19 | binaries=[], 20 | datas=[], 21 | hiddenimports=hiddenimports, 22 | hookspath=[], 23 | runtime_hooks=[], 24 | excludes=[], 25 | win_no_prefer_redirects=False, 26 | win_private_assemblies=False, 27 | cipher=None, 28 | noarchive=True, 29 | ) 30 | pyz = PYZ( # noqa: F821 31 | a.pure, 32 | a.zipped_data, 33 | cipher=None, 34 | ) 35 | exe = EXE( # noqa: F821 36 | pyz, 37 | a.scripts, 38 | a.binaries, 39 | a.zipfiles, 40 | a.datas, 41 | [], 42 | name='dl-plus', 43 | debug=False, 44 | bootloader_ignore_signals=False, 45 | strip=False, 46 | upx=True, 47 | upx_exclude=[], 48 | runtime_tmpdir=None, 49 | console=True, 50 | ) 51 | -------------------------------------------------------------------------------- /src/dl_plus/cli/commands/extractor/base.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import re 4 | from typing import TYPE_CHECKING 5 | 6 | from dl_plus.core import get_extractor_plugin_dir 7 | 8 | 9 | if TYPE_CHECKING: 10 | from pathlib import Path 11 | 12 | from dl_plus.cli.commands.base import BaseInstallUpdateCommand as _base 13 | else: 14 | _base = object 15 | 16 | 17 | PLUGIN_NAME_REGEX = re.compile( 18 | r'^(?:dl-plus-extractor-)?(?P[a-z0-9]+)[/-](?P[a-z0-9]+)$') 19 | 20 | 21 | class ExtractorInstallUninstallUpdateCommandMixin(_base): 22 | ns: str 23 | plugin: str 24 | project_name: str 25 | 26 | def init(self): 27 | super().init() 28 | plugin_name = self.args.name 29 | match = PLUGIN_NAME_REGEX.fullmatch(plugin_name) 30 | if not match: 31 | self.die(f'invalid extractor plugin name: {plugin_name}') 32 | self.ns, self.plugin = match.groups() 33 | self.project_name = f'dl-plus-extractor-{self.ns}-{self.plugin}' 34 | 35 | def get_package_dir(self) -> Path: 36 | return get_extractor_plugin_dir(self.ns, self.plugin) 37 | 38 | def get_short_name(self) -> str: 39 | return f'{self.ns}/{self.plugin}' 40 | -------------------------------------------------------------------------------- /tests/config/test_get_config_path.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from dl_plus.config import ConfigError, get_config_path 4 | 5 | 6 | @pytest.fixture 7 | def config_home(tmp_path): 8 | _config_home = tmp_path / 'dl-plus-config-home' 9 | _config_home.mkdir() 10 | return _config_home 11 | 12 | 13 | def test_default_path(config_home, monkeypatch): 14 | monkeypatch.delenv('DL_PLUS_CONFIG', raising=False) 15 | config_path = config_home / 'config.ini' 16 | assert get_config_path() is None 17 | config_path.touch() 18 | assert get_config_path() == config_path 19 | 20 | 21 | def test_path_from_envvar(config_home, monkeypatch): 22 | config_path = config_home / 'another-config.ini' 23 | monkeypatch.setenv('DL_PLUS_CONFIG', str(config_path)) 24 | with pytest.raises(ConfigError, match='is not a file'): 25 | get_config_path() 26 | config_path.touch() 27 | assert get_config_path() == config_path 28 | 29 | 30 | def test_path_from_argument(config_home): 31 | config_path = config_home / 'another-config.ini' 32 | with pytest.raises(ConfigError, match='is not a file'): 33 | get_config_path(config_path) 34 | config_path.touch() 35 | assert get_config_path(config_path) == config_path 36 | -------------------------------------------------------------------------------- /src/dl_plus/cli/commands/backend/list.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from dl_plus.backend import get_backends_dir 4 | from dl_plus.cli.args import Arg 5 | from dl_plus.cli.commands.base import Command 6 | from dl_plus.pypi import load_metadata 7 | 8 | 9 | class BackendListCommand(Command): 10 | 11 | short_description = 'list installed backends' 12 | 13 | arguments = ( 14 | Arg( 15 | '-s', '--short', action='store_true', 16 | help='Print only backend names.' 17 | ), 18 | ) 19 | 20 | def run(self): 21 | backends_dir = get_backends_dir() 22 | if not backends_dir.exists(): 23 | return 24 | name: str 25 | version: str | None 26 | short = self.args.short 27 | for backend_dir in sorted(backends_dir.iterdir()): 28 | if metadata := load_metadata(backend_dir): 29 | name = metadata.name 30 | version = metadata.version 31 | else: 32 | name = backend_dir.name.replace('_', '-') 33 | version = None 34 | if short or version is None: 35 | self.print(name) 36 | else: 37 | self.print(f'{name} {version}') 38 | -------------------------------------------------------------------------------- /src/dl_plus/cli/commands/backend/update.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from dl_plus.cli.args import Arg 4 | from dl_plus.cli.commands.base import BaseUpdateCommand 5 | 6 | from .base import BackendInstallUninstallUpdateCommandMixin 7 | 8 | 9 | class BackendUpdateCommand( 10 | BackendInstallUninstallUpdateCommandMixin, BaseUpdateCommand, 11 | ): 12 | 13 | short_description = 'Update backend' 14 | 15 | arguments = ( 16 | Arg( 17 | 'name', nargs='?', metavar='NAME', 18 | help='Backend plugin name.' 19 | ), 20 | ) 21 | 22 | fallback_to_config = True 23 | allow_autodetect = True 24 | init_backend = True 25 | 26 | def init(self) -> None: 27 | super().init() 28 | assert self.backend_info is not None 29 | if not self.backend_info.is_managed: 30 | name = self.get_short_name() 31 | self.die( 32 | f'{name} is not managed by dl-plus, ' 33 | f'install it first with `backend install {name}`' 34 | ) 35 | 36 | def get_project_name(self) -> str: 37 | return self.project_name 38 | 39 | def get_extras(self) -> list[str] | None: 40 | if self.backend is not None: 41 | return self.backend.extras 42 | return None 43 | -------------------------------------------------------------------------------- /src/dl_plus/cli/commands/backend/install.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from dl_plus.cli.args import Arg 4 | from dl_plus.cli.commands.base import BaseInstallCommand 5 | 6 | from .base import BackendInstallUninstallUpdateCommandMixin 7 | 8 | 9 | class BackendInstallCommand( 10 | BackendInstallUninstallUpdateCommandMixin, BaseInstallCommand, 11 | ): 12 | 13 | short_description = 'Install backend' 14 | 15 | arguments = ( 16 | Arg( 17 | 'name', metavar='NAME', 18 | help='Backend name.' 19 | ), 20 | Arg( 21 | 'version', nargs='?', metavar='VERSION', 22 | help='Backend version. Default is latest.', 23 | ), 24 | Arg( 25 | '-f', '--force', action='store_true', 26 | help='Force installation if the same version is already installed.' 27 | ), 28 | ) 29 | 30 | fallback_to_config = False 31 | allow_autodetect = False 32 | init_backend = False 33 | 34 | def get_project_name_version_tuple(self) -> tuple[str, str | None]: 35 | return (self.project_name, self.args.version) 36 | 37 | def get_extras(self) -> list[str] | None: 38 | if self.backend is not None: 39 | return self.backend.extras 40 | return None 41 | 42 | def get_force_flag(self) -> bool: 43 | return self.args.force 44 | -------------------------------------------------------------------------------- /docs/available-extractor-plugins.md: -------------------------------------------------------------------------------- 1 | # Available Extractor Plugins 2 | 3 | This is a list of the extractor plugins known to/recommended by the `dl-plus` developers. 4 | 5 | Try searching for `dl-plus-extractor` on [PyPI](https://pypi.org/search/?q=dl-plus-extractor)/[GitHub](https://github.com/search?q=dl-plus-extractor)/[GitLab](https://gitlab.com/explore/projects?name=dl-plus-extractor)/etc. to get more plugins, but remember — this is at your own risk. 6 | 7 | | Service | Plugin Name | Plugin Package | Notes | 8 | | --- | --- | --- | --- | 9 | | [GoodGame](https://goodgame.ru/) | [un1def/goodgame](https://github.com/un-def/dl-plus-extractor-un1def-goodgame) | [dl-plus-extractor-un1def-goodgame](https://pypi.org/project/dl-plus-extractor-un1def-goodgame/) | | 10 | | [Rinse FM](https://rinse.fm/) | [un1def/rinsefm](https://github.com/un-def/dl-plus-extractor-un1def-rinsefm) | [dl-plus-extractor-un1def-rinsefm](https://pypi.org/project/dl-plus-extractor-un1def-rinsefm/) | | 11 | | [NTS Radio](https://www.nts.live/) | [un1def/ntsradio](https://github.com/un-def/dl-plus-extractor-un1def-ntsradio) | [dl-plus-extractor-un1def-ntsradio](https://pypi.org/project/dl-plus-extractor-un1def-ntsradio/) | | 12 | | ~~[WASD.TV](https://wasd.tv/)~~ | [un1def/wasdtv](https://github.com/un-def/dl-plus-extractor-un1def-wasdtv) | [dl-plus-extractor-un1def-wasdtv](https://pypi.org/project/dl-plus-extractor-un1def-wasdtv/) | The service was closed on 2023-04-12 | 13 | -------------------------------------------------------------------------------- /src/dl_plus/cli/commands/config/show/main.py: -------------------------------------------------------------------------------- 1 | import io 2 | 3 | from dl_plus.cli.commands.base import Command 4 | from dl_plus.config import DEFAULT_CONFIG 5 | 6 | 7 | class ConfigShowMainCommand(Command): 8 | short_description = 'Show main config (config.ini)' 9 | long_description = f""" 10 | {short_description}. 11 | 12 | By default, the user config is shown, that is, the content of 13 | the user config file. 14 | 15 | If the --default flag is passed, the default config is shown, that is, 16 | the configuration as if there is no user config. 17 | 18 | If the --merged flag is passed, the result of merging the user config 19 | with the default config is shown, that is, the resulting configuration 20 | as it used by dl-plus. 21 | """ 22 | 23 | def run(self): 24 | if self.args.default: 25 | self._show_default_config() 26 | elif self.args.merged: 27 | self._show_merged_config() 28 | else: 29 | self._show_user_config() 30 | 31 | def _show_default_config(self): 32 | self.print(DEFAULT_CONFIG.strip()) 33 | 34 | def _show_merged_config(self): 35 | with io.StringIO() as buf: 36 | self.config.write(buf) 37 | self.print(buf.getvalue().strip()) 38 | 39 | def _show_user_config(self): 40 | if self.config_path: 41 | print(f'Config file: {self.config_path}\n') 42 | with open(self.config_path) as fobj: 43 | self.print(fobj.read().strip()) 44 | else: 45 | print('No config file found') 46 | -------------------------------------------------------------------------------- /tests/core/test_get_extractors.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from dl_plus.core import get_extractors 4 | 5 | from tests.testlib import ExtractorMock as EM 6 | 7 | 8 | def mock_ytdl_get_all_extractors(include_generic): 9 | extractors = [EM('foo'), EM('bar'), EM('baz')] 10 | if include_generic: 11 | extractors.append(EM('generic')) 12 | return extractors 13 | 14 | 15 | def mock_ytdl_get_extractors_by_name(name): 16 | return [EM(name)] 17 | 18 | 19 | def mock_machinery_load_all_extractors(): 20 | return [EM('ns1/plugin:foo'), EM('ns1/plugin:bar'), EM('ns2/plugin')] 21 | 22 | 23 | def mock_machinery_load_extractors_by_peqn(name): 24 | name_str = str(name) 25 | if name_str == 'ns1/plugin': 26 | return [EM('ns1/plugin:foo'), EM('ns1/plugin:bar')] 27 | return [EM(name_str)] 28 | 29 | 30 | @pytest.fixture 31 | def mock_loaders(monkeypatch): 32 | monkeypatch.setattr( 33 | 'dl_plus.ytdl.get_all_extractors', mock_ytdl_get_all_extractors) 34 | monkeypatch.setattr( 35 | 'dl_plus.ytdl.get_extractors_by_name', 36 | mock_ytdl_get_extractors_by_name, 37 | ) 38 | monkeypatch.setattr( 39 | 'dl_plus.extractor.machinery.load_all_extractors', 40 | mock_machinery_load_all_extractors, 41 | ) 42 | monkeypatch.setattr( 43 | 'dl_plus.extractor.machinery.load_extractors_by_peqn', 44 | mock_machinery_load_extractors_by_peqn, 45 | ) 46 | 47 | 48 | @pytest.mark.parametrize('names,expected', [ 49 | (['bar', 'bar', 'bar'], [EM('bar')]), 50 | (['baz', ':builtins:'], [EM('baz'), EM('foo'), EM('bar')]), 51 | (['ns2/plugin', ':plugins:', 'foo'], [ 52 | EM('ns2/plugin'), EM('ns1/plugin:foo'), EM('ns1/plugin:bar'), 53 | EM('foo'), 54 | ]), 55 | (['ns1/plugin', 'ns1/plugin:foo'], [ 56 | EM('ns1/plugin:foo'), EM('ns1/plugin:bar'), 57 | ]), 58 | ]) 59 | def test(mock_loaders, names, expected): 60 | assert get_extractors(names) == expected 61 | -------------------------------------------------------------------------------- /src/dl_plus/cli/commands/extractor/list.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import sys 4 | 5 | from dl_plus.backend import init_backend 6 | from dl_plus.cli.args import Arg 7 | from dl_plus.cli.commands.base import Command 8 | from dl_plus.core import get_extractor_plugins_dir 9 | from dl_plus.extractor.machinery import load_extractors_by_peqn 10 | from dl_plus.pypi import load_metadata 11 | 12 | 13 | class ExtractorListCommand(Command): 14 | 15 | short_description = 'list installed extractor plugins' 16 | 17 | arguments = ( 18 | Arg( 19 | '-s', '--short', action='store_true', 20 | help='Print only plugin names.' 21 | ), 22 | ) 23 | 24 | def run(self): 25 | extractor_plugins_dir = get_extractor_plugins_dir() 26 | if not extractor_plugins_dir.exists(): 27 | return 28 | plugin: str 29 | version: str | None = None 30 | extractors: list[str] | None = None 31 | short = self.args.short 32 | if not short: 33 | init_backend(self.config.backend) 34 | for extractor_plugin_dir in sorted(extractor_plugins_dir.iterdir()): 35 | plugin = extractor_plugin_dir.name.replace('-', '/') 36 | if not short: 37 | if metadata := load_metadata(extractor_plugin_dir): 38 | version = metadata.version 39 | else: 40 | version = None 41 | sys.path.insert(0, str(extractor_plugin_dir)) 42 | peqns = [ie.IE_NAME for ie in load_extractors_by_peqn(plugin)] 43 | if len(peqns) == 1 and ':' not in peqns[0]: 44 | extractors = None 45 | else: 46 | extractors = [peqn.partition(':')[2] for peqn in peqns] 47 | if version is None: 48 | self.print(plugin) 49 | else: 50 | self.print(f'{plugin} {version}') 51 | if extractors: 52 | for extractor in extractors: 53 | self.print(' ', extractor) 54 | -------------------------------------------------------------------------------- /src/dl_plus/extractor/extractor.py: -------------------------------------------------------------------------------- 1 | import re 2 | from typing import Match, Optional 3 | 4 | from dl_plus import deprecated, ytdl 5 | 6 | 7 | InfoExtractor = ytdl.import_from('extractor.common', 'InfoExtractor') 8 | ExtractorError = ytdl.import_from('utils', 'ExtractorError') 9 | 10 | 11 | class Extractor(InfoExtractor): 12 | """ 13 | A base class for pluggable extractors 14 | """ 15 | 16 | # Set by `ExtractorPlugin.register`, do not override. 17 | IE_NAME: Optional[str] = None 18 | 19 | @classmethod 20 | def ie_key(cls): 21 | return cls.IE_NAME 22 | 23 | if not hasattr(InfoExtractor, '_match_valid_url'): 24 | 25 | @classmethod 26 | def _match_valid_url(cls, url: str) -> Optional[Match[str]]: 27 | """Emulates yt-dlp's method with the same name.""" 28 | valid_url = cls._VALID_URL 29 | if valid_url is False: 30 | return None 31 | if not isinstance(valid_url, str): 32 | # multiple _VALID_URL is not (yet?) supported 33 | raise ExtractorError( 34 | f'_VALID_URL: string expected, got: {valid_url!r}') 35 | # a copy/paste from youtube-dl 36 | if '_VALID_URL_RE' not in cls.__dict__: 37 | cls._VALID_URL_RE = re.compile(valid_url) 38 | return cls._VALID_URL_RE.match(url) 39 | 40 | if ( 41 | ytdl.get_ytdl_module_name() == 'yt_dlp' 42 | and ytdl.get_ytdl_module_version() >= '2023.01.02' 43 | ): 44 | 45 | # suppress some warnings 46 | def _sort_formats(self, formats, field_preference=None): 47 | if not formats or not field_preference: 48 | return 49 | super()._sort_formats(formats, field_preference) 50 | 51 | # dl-plus extra attributes/methods 52 | 53 | DLP_BASE_URL: Optional[str] = None 54 | DLP_REL_URL: Optional[str] = None 55 | 56 | @classmethod 57 | def dlp_match(cls, url: str) -> Optional[Match[str]]: 58 | deprecated.warn( 59 | 'dlp_match() is deprecated, use _match_valid_url() instead') 60 | return cls._match_valid_url(url) 61 | -------------------------------------------------------------------------------- /src/dl_plus/core.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from pathlib import Path 3 | from typing import TYPE_CHECKING, Dict, Iterable, List, Set, Type 4 | 5 | from dl_plus import ytdl 6 | from dl_plus.config import ConfigValue, get_data_home 7 | from dl_plus.extractor import machinery 8 | from dl_plus.extractor.peqn import PEQN 9 | 10 | 11 | if TYPE_CHECKING: 12 | from .extractor.extractor import Extractor 13 | 14 | 15 | def get_extractor_plugins_dir() -> Path: 16 | return get_data_home() / 'extractors' 17 | 18 | 19 | def get_extractor_plugin_dir(ns: str, plugin: str) -> Path: 20 | return get_extractor_plugins_dir() / f'{ns}-{plugin}' 21 | 22 | 23 | def get_extractors(names: Iterable[str]) -> List[Type['Extractor']]: 24 | extractors_dict: Dict[Type['Extractor'], bool] = {} 25 | added_search_paths: Set[str] = set() 26 | 27 | def maybe_add_search_path(path: Path) -> None: 28 | if not path.is_dir(): 29 | return 30 | path_str = str(path) 31 | if path_str not in added_search_paths: 32 | sys.path.insert(0, path_str) 33 | added_search_paths.add(path_str) 34 | 35 | for name in names: 36 | if name == ConfigValue.Extractor.BUILTINS: 37 | extractors = ytdl.get_all_extractors(include_generic=False) 38 | elif name == ConfigValue.Extractor.PLUGINS: 39 | extractor_plugins_dir = get_extractor_plugins_dir() 40 | if extractor_plugins_dir.is_dir(): 41 | for path in extractor_plugins_dir.iterdir(): 42 | maybe_add_search_path(path) 43 | extractors = machinery.load_all_extractors() 44 | elif '/' in name: 45 | peqn = PEQN.from_string(name) 46 | maybe_add_search_path(get_extractor_plugin_dir( 47 | peqn.ns, peqn.plugin)) 48 | extractors = machinery.load_extractors_by_peqn(peqn) 49 | else: 50 | extractors = ytdl.get_extractors_by_name(name) 51 | for extractor in extractors: 52 | if extractor not in extractors_dict: 53 | extractors_dict[extractor] = True 54 | return list(extractors_dict.keys()) 55 | 56 | 57 | def enable_extractors(names) -> None: 58 | extractors = get_extractors(names) 59 | ytdl.patch_extractors(extractors) 60 | -------------------------------------------------------------------------------- /src/dl_plus/cli/commands/config/show/backends.py: -------------------------------------------------------------------------------- 1 | import io 2 | 3 | from dl_plus.backend import ( 4 | DEFAULT_BACKENDS_CONFIG, get_backends_config_path, get_known_backends, 5 | ) 6 | from dl_plus.cli.commands.base import Command 7 | from dl_plus.config import _Config 8 | 9 | 10 | class ConfigShowBackendsCommand(Command): 11 | short_description = 'Show backends config (backends.ini)' 12 | long_description = f""" 13 | {short_description}. 14 | 15 | By default, the user config is shown, that is, the content of 16 | the user config file. 17 | 18 | If the --default flag is passed, the default config is shown, that is, 19 | the configuration as if there is no user config. 20 | 21 | If the --merged flag is passed, the result of merging the user config 22 | with the default config is shown, that is, the resulting configuration 23 | as it used by dl-plus. 24 | """ 25 | 26 | def run(self): 27 | if self.args.default: 28 | self._show_default_config() 29 | elif self.args.merged: 30 | self._show_merged_config() 31 | else: 32 | self._show_user_config() 33 | 34 | def _show_default_config(self): 35 | self.print(DEFAULT_BACKENDS_CONFIG.strip()) 36 | 37 | def _show_merged_config(self): 38 | config = _Config() 39 | for alias, backend in get_known_backends().items(): 40 | # revert [backend_alias] normalization 41 | alias = alias.replace('_', '-') 42 | config.add_section(alias) 43 | for field, value in backend._asdict().items(): 44 | if field == 'extras': 45 | if not value: 46 | continue 47 | value = ' '.join(value) 48 | config.set(alias, field.replace('_', '-'), value) 49 | with io.StringIO() as buf: 50 | config.write(buf) 51 | self.print(buf.getvalue().strip()) 52 | 53 | def _show_user_config(self): 54 | if config_path := get_backends_config_path(): 55 | print(f'Config file: {config_path}\n') 56 | with open(config_path) as fobj: 57 | self.print(fobj.read().strip()) 58 | else: 59 | print('No config file found') 60 | -------------------------------------------------------------------------------- /src/dl_plus/cli/args.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import TYPE_CHECKING 4 | 5 | 6 | if TYPE_CHECKING: 7 | from argparse import _ActionsContainer as _ArgumentParserLike 8 | 9 | 10 | class Arg: 11 | 12 | __slots__ = ('args', 'kwargs') 13 | 14 | def __init__(self, *args, **kwargs): 15 | self.args = args 16 | self.kwargs = kwargs 17 | 18 | def add_to_parser(self, parser: _ArgumentParserLike): 19 | return parser.add_argument(*self.args, **self.kwargs) 20 | 21 | 22 | class ArgGroup: 23 | 24 | __slots__ = ('arguments', 'title', 'description') 25 | 26 | def __init__( 27 | self, *arguments: Arg, title: str, description: str | None = None, 28 | ): 29 | self.arguments = arguments 30 | self.title = title 31 | self.description = description 32 | 33 | def add_to_parser(self, parser: _ArgumentParserLike): 34 | group = parser.add_argument_group( 35 | title=self.title, description=self.description) 36 | for arg in self.arguments: 37 | arg.add_to_parser(group) 38 | 39 | 40 | class ExclusiveArgGroup: 41 | 42 | __slots__ = ('arguments', 'title', 'description', 'required') 43 | 44 | def __init__( 45 | self, *arguments: Arg, required: bool = False, 46 | title: str | None = None, description: str | None = None, 47 | ): 48 | self.arguments = arguments 49 | self.required = required 50 | self.title = title 51 | self.description = description 52 | 53 | def add_to_parser(self, parser: _ArgumentParserLike): 54 | if self.title: 55 | parent = parser.add_argument_group( 56 | title=self.title, description=self.description) 57 | else: 58 | parent = parser 59 | group = parent.add_mutually_exclusive_group(required=self.required) 60 | for arg in self.arguments: 61 | arg.add_to_parser(group) 62 | 63 | 64 | dlp_config = ExclusiveArgGroup( 65 | Arg('--dlp-config', metavar='PATH', help='dl-plus config path.'), 66 | Arg( 67 | '--no-dlp-config', action='store_true', 68 | help='Do not read dl-plus config.', 69 | ), 70 | title='config options', 71 | ) 72 | 73 | 74 | assume_yes_arg = Arg( 75 | '-y', '--assume-yes', action='store_true', 76 | help='Automatic yes to prompts.' 77 | ) 78 | -------------------------------------------------------------------------------- /src/dl_plus/extractor/machinery.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | import itertools 3 | import os 4 | import os.path 5 | import pkgutil 6 | 7 | from dl_plus.const import PLUGINS_PACKAGE 8 | from dl_plus.exceptions import DLPlusException 9 | 10 | from .peqn import PEQN 11 | 12 | 13 | class ExtractorLoadError(DLPlusException): 14 | 15 | pass 16 | 17 | 18 | def discover_extractor_plugins_gen(): 19 | try: 20 | extractors_package = importlib.import_module(PLUGINS_PACKAGE) 21 | except ImportError: 22 | return 23 | for ns_path in extractors_package.__path__: 24 | if not os.path.isdir(ns_path): 25 | continue 26 | for entry in os.scandir(ns_path): 27 | if not entry.is_dir(): 28 | continue 29 | ns = entry.name 30 | for _, name, _ in pkgutil.iter_modules([entry.path]): 31 | yield f'{PLUGINS_PACKAGE}.{ns}.{name}' 32 | 33 | 34 | def load_extractors_by_plugin_import_path(plugin_import_path, names=None): 35 | try: 36 | plugin_module = importlib.import_module(plugin_import_path) 37 | except ImportError as exc: 38 | raise ExtractorLoadError(f'cannot import {plugin_import_path}: {exc}') 39 | try: 40 | plugin = plugin_module.plugin 41 | except AttributeError: 42 | raise ExtractorLoadError( 43 | f'{plugin_import_path} does not marked as an extractor plugin') 44 | if names: 45 | try: 46 | extractors = list(map(plugin.get_extractor, names)) 47 | except KeyError as exc: 48 | raise ExtractorLoadError( 49 | f'{plugin_import_path} does not contain {exc.args[0]}') 50 | else: 51 | extractors = plugin.get_all_extractors() 52 | return extractors 53 | 54 | 55 | def load_extractors_by_peqn(peqn): 56 | if not isinstance(peqn, PEQN): 57 | try: 58 | peqn = PEQN.from_string(peqn) 59 | except ValueError: 60 | raise ExtractorLoadError(f'failed to parse PEQN: {peqn}') 61 | if peqn.name: 62 | names = [peqn.name] 63 | else: 64 | names = None 65 | return load_extractors_by_plugin_import_path( 66 | peqn.plugin_import_path, names) 67 | 68 | 69 | def load_all_extractors(): 70 | return list(itertools.chain.from_iterable( 71 | map( 72 | load_extractors_by_plugin_import_path, 73 | discover_extractor_plugins_gen(), 74 | ) 75 | )) 76 | -------------------------------------------------------------------------------- /tests/config/test_get_config_home.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from pathlib import Path 3 | 4 | import pytest 5 | 6 | from dl_plus.config import get_config_home 7 | 8 | 9 | @pytest.fixture(autouse=True) 10 | def _unset_config_home_envvar(monkeypatch): 11 | monkeypatch.delenv('DL_PLUS_CONFIG_HOME', raising=False) 12 | monkeypatch.delenv('DL_PLUS_HOME', raising=False) 13 | 14 | 15 | @pytest.mark.skipif( 16 | not sys.platform.startswith('linux'), reason='requires linux') 17 | class TestLinux: 18 | 19 | def test_location_from_config_home_envvar(self, monkeypatch): 20 | monkeypatch.setenv('DL_PLUS_CONFIG_HOME', '/dl/plus/config/home') 21 | # should be ignored 22 | monkeypatch.setenv('DL_PLUS_HOME', '/dl/plus/home') 23 | assert get_config_home() == Path('/dl/plus/config/home') 24 | 25 | def test_location_from_home_envvar(self, monkeypatch): 26 | monkeypatch.setenv('DL_PLUS_HOME', '/dl/plus/home') 27 | assert get_config_home() == Path('/dl/plus/home') 28 | 29 | def test_default_location_no_xdg_config_home_envvar(self, monkeypatch): 30 | monkeypatch.delenv('XDG_CONFIG_HOME', raising=False) 31 | monkeypatch.setattr(Path, 'home', lambda: Path('/fakehome')) 32 | assert get_config_home() == Path('/fakehome/.config/dl-plus') 33 | 34 | def test_default_location_with_xdg_config_home_envvar(self, monkeypatch): 35 | monkeypatch.setenv('XDG_CONFIG_HOME', '/xdgconfig') 36 | assert get_config_home() == Path('/xdgconfig/dl-plus') 37 | 38 | 39 | @pytest.mark.skipif( 40 | not sys.platform.startswith('win'), reason='requires windows') 41 | class TestWindows: 42 | 43 | def test_location_from_config_home_envvar(self, monkeypatch): 44 | monkeypatch.setenv('DL_PLUS_CONFIG_HOME', 'X:/dl/plus/config/home') 45 | # should be ignored 46 | monkeypatch.setenv('DL_PLUS_HOME', r'X:\dl\plus\home') 47 | assert get_config_home() == Path('X:/dl/plus/config/home') 48 | 49 | def test_location_from_home_envvar(self, monkeypatch): 50 | monkeypatch.setenv('DL_PLUS_HOME', r'X:\dl\plus\home') 51 | assert get_config_home() == Path('X:/dl/plus/home') 52 | 53 | def test_default_location_no_appdata_envvar(self, monkeypatch): 54 | monkeypatch.delenv('AppData', raising=False) 55 | monkeypatch.setattr(Path, 'home', lambda: Path('X:/fakehome')) 56 | assert get_config_home() == Path('X:/fakehome/AppData/Roaming/dl-plus') 57 | 58 | def test_default_location_with_appdata_envvar(self, monkeypatch): 59 | monkeypatch.setenv('AppData', 'X:/appdata') 60 | assert get_config_home() == Path('X:/appdata/dl-plus') 61 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ['setuptools'] 3 | build-backend = 'setuptools.build_meta' 4 | 5 | [project] 6 | name = 'dl-plus' 7 | description = 'A youtube-dl extension with pluggable extractors' 8 | readme = 'README.md' 9 | license = {text = 'MIT'} 10 | authors = [ 11 | {name = 'Dmitry Meyer', email = 'me@undef.im'}, 12 | ] 13 | classifiers = [ 14 | 'Development Status :: 4 - Beta', 15 | 'License :: OSI Approved :: MIT License', 16 | 'Programming Language :: Python :: 3', 17 | 'Programming Language :: Python :: 3.9', 18 | 'Programming Language :: Python :: 3.10', 19 | 'Programming Language :: Python :: 3.11', 20 | 'Programming Language :: Python :: 3.12', 21 | 'Programming Language :: Python :: 3.13', 22 | ] 23 | requires-python = '>= 3.9' 24 | dynamic = ['version'] 25 | 26 | [project.urls] 27 | Homepage = 'https://github.com/un-def/dl-plus' 28 | Repository = 'https://github.com/un-def/dl-plus.git' 29 | Changelog = 'https://github.com/un-def/dl-plus/blob/master/CHANGELOG.md' 30 | Issues = 'https://github.com/un-def/dl-plus/issues' 31 | 32 | [project.scripts] 33 | dl-plus = 'dl_plus.cli:main' 34 | 35 | [tool.setuptools] 36 | zip-safe = false 37 | include-package-data = false 38 | 39 | [tool.setuptools.dynamic] 40 | version = {attr = 'dl_plus.const.DL_PLUS_VERSION'} 41 | 42 | [tool.isort] 43 | lines_after_imports = 2 44 | multi_line_output = 5 45 | include_trailing_comma = true 46 | use_parentheses = true 47 | sections = ['FUTURE', 'STDLIB', 'THIRDPARTY', 'FIRSTPARTY', 'TESTS', 'LOCALFOLDER'] 48 | known_tests = ['tests'] 49 | skip_glob = ['.*'] 50 | 51 | [tool.mypy] 52 | mypy_path = '${MYPY_CONFIG_FILE_DIR}/src' 53 | packages = 'dl_plus' 54 | python_version = '3.9' # the lowest Python version we support 55 | namespace_packages = true 56 | disallow_any_unimported = true 57 | disallow_any_expr = true 58 | disallow_any_decorated = true 59 | disallow_any_explicit = true 60 | disallow_any_generics = true 61 | disallow_subclassing_any = true 62 | disallow_untyped_calls = true 63 | disallow_untyped_defs = true 64 | disallow_incomplete_defs = true 65 | check_untyped_defs = true 66 | disallow_untyped_decorators = true 67 | no_implicit_optional = true 68 | strict_optional = true 69 | warn_redundant_casts = true 70 | warn_unused_ignores = true 71 | warn_no_return = true 72 | warn_return_any = true 73 | warn_unreachable = true 74 | implicit_reexport = false 75 | strict_equality = true 76 | show_error_context = true 77 | show_column_numbers = true 78 | show_error_codes = true 79 | warn_unused_configs = true 80 | 81 | [tool.pytest.ini_options] 82 | minversion = '6.0' 83 | addopts = '--strict-markers' 84 | testpaths = ['tests'] 85 | markers = [ 86 | 'extractors', 87 | ] 88 | -------------------------------------------------------------------------------- /src/dl_plus/cli/commands/backend/base.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import TYPE_CHECKING, ClassVar 4 | 5 | from dl_plus import backend 6 | from dl_plus.backend import ( 7 | Backend, BackendInfo, get_known_backend, init_backend, 8 | is_project_name_valid, 9 | ) 10 | from dl_plus.config import ConfigValue 11 | 12 | 13 | if TYPE_CHECKING: 14 | from pathlib import Path 15 | 16 | from dl_plus.cli.commands.base import BaseInstallUpdateCommand as _base 17 | else: 18 | _base = object 19 | 20 | 21 | class BackendCommandMixin(_base): 22 | fallback_to_config: ClassVar[bool] 23 | allow_autodetect: ClassVar[bool] 24 | init_backend: ClassVar[bool] 25 | 26 | project_name: str 27 | backend_alias: str | None = None # [section_name] is backends.ini 28 | backend: Backend | None = None 29 | backend_info: BackendInfo | None = None 30 | 31 | def init(self): 32 | super().init() 33 | project_name_or_backend_alias: str | None = self.args.name 34 | if project_name_or_backend_alias is None and self.fallback_to_config: 35 | project_name_or_backend_alias = self.config.backend 36 | if project_name_or_backend_alias == ConfigValue.Backend.AUTODETECT: 37 | if not self.allow_autodetect: 38 | self.die( 39 | f'{ConfigValue.Backend.AUTODETECT} is not allowed ' 40 | f'in {self.name} command' 41 | ) 42 | self.backend_info = init_backend() 43 | project_name_or_backend_alias = self.backend_info.alias 44 | if project_name_or_backend_alias is None: 45 | self.die('Backend argument is required') 46 | backend = get_known_backend(project_name_or_backend_alias) 47 | if backend is not None: 48 | self.project_name = backend.project_name 49 | self.backend_alias = project_name_or_backend_alias 50 | self.backend = backend 51 | else: 52 | self.project_name = project_name_or_backend_alias 53 | if not is_project_name_valid(self.project_name): 54 | self.die(f'invalid backend name: {self.project_name}') 55 | if self.backend_info is None and self.init_backend: 56 | self.backend_info = init_backend(project_name_or_backend_alias) 57 | 58 | 59 | class BackendInstallUninstallUpdateCommandMixin(BackendCommandMixin): 60 | 61 | def get_package_dir(self) -> Path: 62 | if self.backend_alias is not None: 63 | _backend = self.backend_alias 64 | else: 65 | _backend = self.project_name 66 | return backend.get_backend_dir(_backend) 67 | 68 | def get_short_name(self) -> str: 69 | short_name = self.backend_alias 70 | if short_name is None: 71 | short_name = self.project_name 72 | return short_name 73 | -------------------------------------------------------------------------------- /tests/extractor/test_peqn.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from dl_plus.extractor.peqn import PEQN 4 | 5 | 6 | @pytest.mark.parametrize('path,expected_ns,expected_plugin', [ 7 | ('dl_plus.extractors.foo.bar', 'foo', 'bar'), 8 | ('dl_plus.extractors._1foo._2bar', '1foo', '2bar'), 9 | ]) 10 | def test_from_plugin_import_path(path, expected_ns, expected_plugin): 11 | peqn = PEQN.from_plugin_import_path(path) 12 | assert peqn.ns == expected_ns 13 | assert peqn.plugin == expected_plugin 14 | assert peqn.name is None 15 | 16 | 17 | @pytest.mark.parametrize('path,expected_error', [ 18 | ('somepackage.extractors.foo.bar', 'not in plugins package'), 19 | ('dl_plus.extractors.foo', 'not enough parts'), 20 | ('dl_plus.extractors.foo.bar.baz', 'too many parts'), 21 | ('dl_plus.extractors.foo.', 'empty plugin part'), 22 | ('dl_plus.extractors._.bar', 'empty ns part'), 23 | ('dl_plus.extractors.Foo.bar', 'bad ns part'), 24 | ('dl_plus.extractors.foo.bar_baz', 'bad plugin part'), 25 | ]) 26 | def test_from_plugin_import_path_error(path, expected_error): 27 | with pytest.raises(ValueError, match=expected_error): 28 | PEQN.from_plugin_import_path(path) 29 | 30 | 31 | def test_from_string_without_name(): 32 | peqn = PEQN.from_string('foo/1bar') 33 | assert peqn.ns == 'foo' 34 | assert peqn.plugin == '1bar' 35 | assert peqn.name is None 36 | 37 | 38 | def test_from_string_with_name(): 39 | peqn = PEQN.from_string('1foo/bar:baz') 40 | assert peqn.ns == '1foo' 41 | assert peqn.plugin == 'bar' 42 | assert peqn.name == 'baz' 43 | 44 | 45 | @pytest.mark.parametrize('string', [ 46 | 'Foo/bar', '/bar', 'foo/bar_baz', '', 'foo', 47 | ]) 48 | def test_from_string_error(string): 49 | with pytest.raises(ValueError, match='bad PEQN'): 50 | PEQN.from_string(string) 51 | 52 | 53 | @pytest.mark.parametrize( 54 | 'orig,copy_args,expected_ns,expected_plugin,expected_name', [ 55 | (PEQN('foo', 'bar'), dict(), 'foo', 'bar', None), 56 | (PEQN('foo', 'bar'), dict(plugin='bar1'), 'foo', 'bar1', None), 57 | (PEQN('foo', 'bar', 'baz'), dict(ns='foo1'), 'foo1', 'bar', 'baz'), 58 | (PEQN('foo', 'bar', 'baz'), dict(name=None), 'foo', 'bar', None), 59 | ] 60 | ) 61 | def test_copy(orig, copy_args, expected_ns, expected_plugin, expected_name): 62 | copy = orig.copy(**copy_args) 63 | assert copy.ns == expected_ns 64 | assert copy.plugin == expected_plugin 65 | assert copy.name == expected_name 66 | 67 | 68 | @pytest.mark.parametrize('peqn,expected', [ 69 | (PEQN('foo', 'bar'), 'foo/bar'), 70 | (PEQN('foo', 'bar', 'baz'), 'foo/bar:baz'), 71 | ]) 72 | def test_to_str(peqn, expected): 73 | assert str(peqn) == expected 74 | 75 | 76 | @pytest.mark.parametrize('peqn,expected', [ 77 | (PEQN('foo', 'bar'), 'dl_plus.extractors.foo.bar'), 78 | (PEQN('foo', 'bar', 'baz'), 'dl_plus.extractors.foo.bar'), 79 | (PEQN('1foo', '22bar'), 'dl_plus.extractors._1foo._22bar'), 80 | ]) 81 | def test_plugin_import_path_property(peqn, expected): 82 | assert peqn.plugin_import_path == expected 83 | -------------------------------------------------------------------------------- /src/dl_plus/cli/command.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import argparse 4 | from typing import TYPE_CHECKING, List, Type 5 | 6 | from .commands import RootCommandGroup 7 | from .commands.base import CommandGroup 8 | 9 | 10 | if TYPE_CHECKING: 11 | from .commands.base import Command 12 | 13 | 14 | __all__ = ['run_command'] 15 | 16 | 17 | _COMMAND_DEST = 'command' 18 | 19 | 20 | class CommandNamespace(argparse.Namespace): 21 | 22 | def __init__(self, **kwargs): 23 | super().__init__(**kwargs) 24 | self.__dict__[_COMMAND_DEST] = [] 25 | 26 | def _get_command(self): 27 | return self.__dict__[_COMMAND_DEST] 28 | 29 | def _set_command(self, value): 30 | if isinstance(value, list): 31 | self.__dict__[_COMMAND_DEST].extend(value) 32 | else: 33 | assert isinstance(value, str), repr(str) 34 | self.__dict__[_COMMAND_DEST].append(value) 35 | 36 | command = property(_get_command, _set_command) 37 | 38 | 39 | class CommandArgParser(argparse.ArgumentParser): 40 | 41 | def __init__(self, *args, **kwargs): 42 | kwargs.setdefault( 43 | 'formatter_class', argparse.RawDescriptionHelpFormatter) 44 | super().__init__(*args, **kwargs) 45 | 46 | def parse_known_args(self, args=None, namespace=None): 47 | if namespace is None: 48 | namespace = CommandNamespace() 49 | return super().parse_known_args(args, namespace) 50 | 51 | def add_command_arguments(self, command: Type[Command]): 52 | for parent in command.get_parents(): 53 | for arg in parent.arguments: 54 | arg.add_to_parser(self) 55 | for arg in command.arguments: 56 | arg.add_to_parser(self) 57 | 58 | def add_command_group(self, command_group: Type[CommandGroup]): 59 | command_group_subparsers = self.add_command_group_subparsers( 60 | title=command_group.short_description) 61 | for command_or_group in command_group.commands: 62 | description = ( 63 | command_or_group.long_description 64 | or command_or_group.short_description 65 | ) 66 | command_parser = command_group_subparsers.add_parser( 67 | command_or_group.name, 68 | help=command_or_group.short_description, 69 | description=description, 70 | ) 71 | if issubclass(command_or_group, CommandGroup): 72 | command_parser.add_command_group(command_or_group) 73 | else: 74 | command_parser.add_command_arguments(command_or_group) 75 | 76 | def add_command_group_subparsers(self, *args, **kwargs): 77 | kwargs.setdefault('dest', _COMMAND_DEST) 78 | kwargs.setdefault('metavar', 'COMMAND') 79 | kwargs.setdefault('required', True) 80 | subparsers = self.add_subparsers(*args, **kwargs) 81 | return subparsers 82 | 83 | 84 | def run_command(prog: str, cmd_arg: str, args: List[str]) -> None: 85 | parser = CommandArgParser(prog=f'{prog} {cmd_arg}') 86 | parser.add_argument( 87 | cmd_arg, action='store_true', required=True, help=argparse.SUPPRESS) 88 | parser.add_command_group(RootCommandGroup) 89 | parsed_args = parser.parse_args(args) 90 | command_cls = RootCommandGroup.get_command(parsed_args.command) 91 | command_cls(parsed_args).run() 92 | -------------------------------------------------------------------------------- /tests/config/test_config_load.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from dl_plus.config import Config 4 | 5 | 6 | @pytest.fixture 7 | def config_home(tmp_path): 8 | _config_home = tmp_path / 'dl-plus-config-home' 9 | _config_home.mkdir() 10 | return _config_home 11 | 12 | 13 | @pytest.fixture 14 | def config(): 15 | _config = Config() 16 | return _config 17 | 18 | 19 | @pytest.fixture 20 | def write_config(config_home): 21 | 22 | def _write_config(name='config.ini', backend=None): 23 | path = config_home / name 24 | main_section = [] 25 | if backend: 26 | main_section.append(f'backend = {backend}\n') 27 | lines = [] 28 | if main_section: 29 | lines.append('[main]\n') 30 | lines.extend(main_section) 31 | with open(path, 'wt') as fobj: 32 | fobj.writelines(lines) 33 | return path 34 | 35 | return _write_config 36 | 37 | 38 | class TestConfigLoad: 39 | DEFAULT_BACKEND = ':autodetect:' 40 | 41 | @pytest.fixture(autouse=True) 42 | def setup(self, monkeypatch, config, write_config): 43 | monkeypatch.delenv('DL_PLUS_BACKEND', raising=False) 44 | self.config = config 45 | self.write_config = write_config 46 | 47 | @pytest.mark.parametrize('args', [(), (None,), (True,)]) 48 | def test_default_config_location(self, args): 49 | self.write_config(backend='config-ini') 50 | self.config.load(*args) 51 | assert self.config.backend == 'config-ini' 52 | 53 | def test_another_config_location(self): 54 | self.write_config(backend='config-ini') 55 | another_path = self.write_config( 56 | 'another-config.ini', backend='another-config-ini') 57 | self.config.load(another_path) 58 | assert self.config.backend == 'another-config-ini' 59 | 60 | def test_default_config_location_with_envvar(self, monkeypatch): 61 | self.write_config(backend='config-ini') 62 | monkeypatch.setenv('DL_PLUS_BACKEND', 'environ') 63 | self.config.load() 64 | assert self.config.backend == 'environ' 65 | 66 | def test_another_config_location_with_envvar(self, monkeypatch): 67 | self.write_config(backend='config-ini') 68 | another_path = self.write_config( 69 | 'another-config.ini', backend='another-config-ini') 70 | monkeypatch.setenv('DL_PLUS_BACKEND', 'environ') 71 | self.config.load(another_path) 72 | assert self.config.backend == 'environ' 73 | 74 | def test_disable_config_file(self): 75 | self.write_config(backend='config-ini') 76 | self.config.load(False) 77 | assert self.config.backend == self.DEFAULT_BACKEND 78 | 79 | def test_disable_config_file_with_envvar(self, monkeypatch): 80 | self.write_config(backend='config-ini') 81 | monkeypatch.setenv('DL_PLUS_BACKEND', 'environ') 82 | self.config.load(False) 83 | assert self.config.backend == 'environ' 84 | 85 | def test_disable_environ(self, monkeypatch): 86 | self.write_config(backend='config-ini') 87 | monkeypatch.setenv('DL_PLUS_BACKEND', 'environ') 88 | self.config.load(environ=False) 89 | assert self.config.backend == 'config-ini' 90 | 91 | def test_disable_config_file_disable_environ(self, monkeypatch): 92 | self.write_config(backend='config-ini') 93 | monkeypatch.setenv('DL_PLUS_BACKEND', 'environ') 94 | self.config.load(False, False) 95 | assert self.config.backend == self.DEFAULT_BACKEND 96 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # dl-plus 2 | 3 | A [youtube-dl][youtube-dl-website] extension with pluggable extractors 4 | 5 | ## Description 6 | 7 | `dl-plus` is an extension and a drop-in replacement of `youtube-dl` (or any compatible fork, e.g., `yt-dlp`). The main goal of the project is to add an easy-to-use extractor plugin system to `youtube-dl` while maintaining full backward compatibility. 8 | 9 | `dl-plus` is not a fork of `youtube-dl` and does not contain code from `youtube-dl`, it is a pure dynamic wrapper (thanks to Python dynamic nature) hacking some `youtube-dl` internals. 10 | 11 | ## Installation 12 | 13 | 1. Install `dl-plus`: 14 | 15 | * using [pipx][pipx-website]: 16 | 17 | ``` 18 | pipx install dl-plus 19 | ``` 20 | 21 | * using pip: 22 | 23 | ``` 24 | pip install dl-plus 25 | ``` 26 | 27 | (**\*nix**) Alternatively, you can download a single file binary (zipapp) and put it somewhere in your `PATH`: 28 | 29 | ``` 30 | curl -L https://github.com/un-def/dl-plus/releases/latest/download/dl-plus -o dl-plus 31 | chmod a+x dl-plus 32 | ``` 33 | 34 | 2. Install a backend — `youtube-dl` or any compatible package (fork), e.g., `yt-dlp`: 35 | 36 | * using `dl-plus` itself: 37 | 38 | ``` 39 | dl-plus --cmd backend install yt-dlp 40 | ``` 41 | 42 | * using [pipx][pipx-website]: 43 | 44 | ``` 45 | pipx inject dl-plus yt-dlp 46 | ``` 47 | 48 | * using pip: 49 | 50 | ``` 51 | pip install yt-dlp 52 | ``` 53 | 54 | 3. (optional) Install some extractor plugins: 55 | 56 | * using `dl-plus` itself: 57 | 58 | ``` 59 | dl-plus --cmd extractor install un1def/goodgame 60 | ``` 61 | 62 | PyPI package names are supported too: 63 | 64 | ``` 65 | dl-plus --cmd extractor install dl-plus-extractor-un1def-goodgame 66 | ``` 67 | 68 | * using [pipx][pipx-website]: 69 | 70 | ``` 71 | pipx inject dl-plus dl-plus-extractor-un1def-goodgame 72 | ``` 73 | 74 | * using pip: 75 | 76 | ``` 77 | pip install dl-plus-extractor-un1def-goodgame 78 | ``` 79 | 80 | 4. (optional) Create `dl-plus` → `youtube-dl` symlink (for apps relying on `youtube-dl` executable in `PATH`, e.g., [mpv][mpv-website]): 81 | 82 | - **\*nix**: 83 | 84 | ```shell 85 | dlp=$(command -v dl-plus 2>&1) && ln -s "$dlp" "$(dirname "$dlp")/youtube-dl" 86 | ``` 87 | 88 | Use `ln -sf` instead of `ln -s` to overwrite an existing `youtube-dl` executable. 89 | 90 | - **Windows** (PowerShell, requires administrative privileges): 91 | 92 | ```powershell 93 | $dlp = (Get-Command -ErrorAction:Stop dl-plus).Path; New-Item -ItemType SymbolicLink -Path ((Get-Item $dlp).Directory.FullName + "\youtube-dl.exe") -Target $dlp 94 | ``` 95 | 96 | ## Extractor Plugin Authoring Guide 97 | 98 | See [docs/extractor-plugin-authoring-guide.md](https://github.com/un-def/dl-plus/blob/master/docs/extractor-plugin-authoring-guide.md). 99 | 100 | ## Available Extractor Plugins 101 | 102 | See [docs/available-extractor-plugins.md](https://github.com/un-def/dl-plus/blob/master/docs/available-extractor-plugins.md). 103 | 104 | ## License 105 | 106 | The [MIT License][license]. 107 | 108 | 109 | [youtube-dl-website]: https://youtube-dl.org/ 110 | [pipx-website]: https://pipxproject.github.io/pipx/ 111 | [mpv-website]: https://mpv.io/ 112 | [license]: https://github.com/un-def/dl-plus/blob/master/LICENSE 113 | -------------------------------------------------------------------------------- /src/dl_plus/extractor/peqn.py: -------------------------------------------------------------------------------- 1 | import re 2 | from typing import Optional, Union 3 | 4 | from dl_plus.const import PLUGINS_PACKAGE 5 | from dl_plus.utils import NOTSET, NotSet 6 | 7 | 8 | PLUGIN_IMPORT_PATH_PREFIX = f'{PLUGINS_PACKAGE}.' 9 | PART_REGEX = re.compile(r'^[a-z0-9]+$') 10 | PEQN_REGEX = re.compile( 11 | r'^(?P[a-z0-9]+)/(?P[a-z0-9]+)(?::(?P[a-z0-9]+))?$') 12 | 13 | 14 | def _check_part(part: str, part_name: str) -> None: 15 | if not part: 16 | raise ValueError(f'empty {part_name} part') 17 | if not PART_REGEX.fullmatch(part): 18 | raise ValueError(f'bad {part_name} part: {part}') 19 | 20 | 21 | def _escape_import_part(part: str) -> str: 22 | try: 23 | if part[0].isdigit(): 24 | return '_' + part 25 | except IndexError: 26 | pass 27 | return part 28 | 29 | 30 | def _unescape_import_part(part: str) -> str: 31 | try: 32 | if part[0] == '_': 33 | return part[1:] 34 | except IndexError: 35 | pass 36 | return part 37 | 38 | 39 | class PEQN: 40 | """ 41 | Pluggable Extractor Qualified Name 42 | """ 43 | 44 | def __init__( 45 | self, ns: str, plugin: str, name: Optional[str] = None, 46 | ) -> None: 47 | _check_part(ns, 'ns') 48 | _check_part(plugin, 'plugin') 49 | if name is not None: 50 | _check_part(name, 'name') 51 | self._ns = ns 52 | self._plugin = plugin 53 | self._name = name 54 | 55 | @classmethod 56 | def from_plugin_import_path(cls, path: str) -> 'PEQN': 57 | try: 58 | if not path.startswith(PLUGIN_IMPORT_PATH_PREFIX): 59 | raise ValueError('not in plugins package') 60 | parts = path[len(PLUGIN_IMPORT_PATH_PREFIX):].split('.') 61 | if len(parts) < 2: 62 | raise ValueError('not enough parts') 63 | elif len(parts) > 2: 64 | raise ValueError('too many parts') 65 | ns = _unescape_import_part(parts[0]) 66 | plugin = _unescape_import_part(parts[1]) 67 | return cls(ns, plugin) 68 | except ValueError as exc: 69 | raise ValueError(f'bad plugin import path: {path}: {exc}') 70 | 71 | @classmethod 72 | def from_string(cls, peqn: str) -> 'PEQN': 73 | match = PEQN_REGEX.fullmatch(peqn) 74 | if not match: 75 | raise ValueError(f'bad PEQN: {peqn}') 76 | return cls(**match.groupdict()) 77 | 78 | def copy( 79 | self, 80 | ns: Union[str, NotSet] = NOTSET, 81 | plugin: Union[str, NotSet] = NOTSET, 82 | name: Union[str, None, NotSet] = NOTSET, 83 | ) -> 'PEQN': 84 | if ns is NOTSET: 85 | ns = self._ns 86 | if plugin is NOTSET: 87 | plugin = self._plugin 88 | if name is NOTSET: 89 | name = self._name 90 | return self.__class__(ns, plugin, name) 91 | 92 | def __str__(self) -> str: 93 | if self._name: 94 | return f'{self._ns}/{self._plugin}:{self._name}' 95 | return f'{self._ns}/{self._plugin}' 96 | 97 | @property 98 | def ns(self) -> str: 99 | return self._ns 100 | 101 | @property 102 | def plugin(self) -> str: 103 | return self._plugin 104 | 105 | @property 106 | def name(self) -> Optional[str]: 107 | return self._name 108 | 109 | @property 110 | def plugin_import_path(self) -> str: 111 | ns = _escape_import_part(self._ns) 112 | plugin = _escape_import_part(self._plugin) 113 | return f'{PLUGINS_PACKAGE}.{ns}.{plugin}' 114 | -------------------------------------------------------------------------------- /tests/ytdl/test_extractors_registry.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from dl_plus import ytdl 4 | 5 | from tests.testlib import ExtractorMock as EM 6 | 7 | 8 | @pytest.fixture(autouse=True) 9 | def reset_cache(): 10 | ytdl._extractors = ytdl._NOT_SET 11 | ytdl._extractors_registry = ytdl._NOT_SET 12 | 13 | 14 | @pytest.fixture(autouse=True) 15 | def monkeypatch_get_all_extractors(request, monkeypatch, reset_cache): 16 | extractors = None 17 | try: 18 | extractors = request.getfixturevalue('extractors') 19 | except pytest.FixtureLookupError: 20 | marker = request.node.get_closest_marker('extractors') 21 | if marker: 22 | extractors = marker.args 23 | if extractors is not None: 24 | monkeypatch.setattr( 25 | 'dl_plus.ytdl.get_all_extractors', lambda **kw: extractors) 26 | 27 | 28 | @pytest.mark.parametrize('extractors,expected', [ 29 | # 0 30 | ([EM('foo'), EM('bar')], { 31 | 'foo': EM('foo'), 32 | 'bar': EM('bar'), 33 | }), 34 | # 1 35 | ([EM('foo:sub1'), EM('foo:sub2'), EM('bar')], { 36 | 'foo': { 37 | 'sub1': EM('foo:sub1'), 38 | 'sub2': EM('foo:sub2'), 39 | }, 40 | 'bar': EM('bar'), 41 | }), 42 | # 2 43 | ([EM('foo'), EM('foo:sub'), EM('bar')], { 44 | 'foo': { 45 | '_': EM('foo'), 46 | 'sub': EM('foo:sub'), 47 | }, 48 | 'bar': EM('bar'), 49 | }), 50 | # 3 51 | ([EM('foo:sub'), EM('foo'), EM('bar')], { 52 | 'foo': { 53 | '_': EM('foo'), 54 | 'sub': EM('foo:sub'), 55 | }, 56 | 'bar': EM('bar'), 57 | }), 58 | # 4 59 | ([EM('foo:sub1'), EM('foo'), EM('foo:sub2')], { 60 | 'foo': { 61 | '_': EM('foo'), 62 | 'sub1': EM('foo:sub1'), 63 | 'sub2': EM('foo:sub2'), 64 | }, 65 | }), 66 | # 5 67 | ([EM('foo:sub'), EM('foo'), EM('foo:sub:subsub')], { 68 | 'foo': { 69 | '_': EM('foo'), 70 | 'sub': { 71 | '_': EM('foo:sub'), 72 | 'subsub': EM('foo:sub:subsub'), 73 | }, 74 | }, 75 | }), 76 | ]) 77 | def test_build_extractors_registry(extractors, expected): 78 | assert ytdl._build_extractors_registry() == expected 79 | 80 | 81 | @pytest.mark.parametrize('extractors,expected', [ 82 | ([EM('foo'), EM('bar'), EM('foo')], EM('foo')), 83 | ([EM('foo:sub'), EM('foo'), EM('foo:sub')], EM('foo:sub')), 84 | ([EM('foo:sub'), EM('foo:sub:subsub'), EM('foo:sub')], EM('foo:sub')), 85 | ]) 86 | def test_build_extractors_registry_error_duplicate_name(extractors, expected): 87 | with pytest.raises( 88 | ytdl.YoutubeDLError, match=f'duplicate name: {expected!r}'): 89 | ytdl._build_extractors_registry() 90 | 91 | 92 | extractors_marker = pytest.mark.extractors( 93 | EM('foo'), EM('foo:sub'), EM('foo:sub:subsub1'), EM('foo:sub:subsub2'), 94 | EM('bar:sub1'), EM('bar:sub2'), 95 | EM('baz'), 96 | ) 97 | 98 | 99 | @pytest.mark.parametrize('name,expected', [ 100 | ('foo', [ 101 | EM('foo'), EM('foo:sub'), EM('foo:sub:subsub1'), EM('foo:sub:subsub2'), 102 | ]), 103 | ('foo:_', [EM('foo')]), 104 | ('foo:sub', [EM('foo:sub'), EM('foo:sub:subsub1'), EM('foo:sub:subsub2')]), 105 | ('foo:sub:_', [EM('foo:sub')]), 106 | ('foo:sub:subsub1', [EM('foo:sub:subsub1')]), 107 | ('bar', [EM('bar:sub1'), EM('bar:sub2')]), 108 | ('baz', [EM('baz')]), 109 | ]) 110 | @extractors_marker 111 | def test_get_extractors_by_name(name, expected): 112 | assert ytdl.get_extractors_by_name(name) == expected 113 | 114 | 115 | @pytest.mark.parametrize('name', [ 116 | 'foo:sub:subsub3', 'bar:sub1:subsub', 'baz:_']) 117 | @extractors_marker 118 | def test_get_extractors_by_name_error_unknown(name): 119 | with pytest.raises(ytdl.UnknownBuiltinExtractor, match=name): 120 | ytdl.get_extractors_by_name(name) 121 | -------------------------------------------------------------------------------- /tests/config/test_get_data_home.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from pathlib import Path 3 | 4 | import pytest 5 | 6 | from dl_plus.config import get_data_home 7 | from dl_plus.deprecated import DLPlusDeprecationWarning 8 | 9 | 10 | @pytest.fixture(autouse=True) 11 | def _autopatch_data_home(monkeypatch): 12 | # set dl_plus.config._data_home to None to disable caching 13 | monkeypatch.setattr('dl_plus.config._data_home', None) 14 | 15 | 16 | @pytest.fixture(autouse=True) 17 | def _unset_data_home_envvar(monkeypatch): 18 | monkeypatch.delenv('DL_PLUS_DATA_HOME', raising=False) 19 | monkeypatch.delenv('DL_PLUS_HOME', raising=False) 20 | 21 | 22 | @pytest.mark.skipif( 23 | not sys.platform.startswith('linux'), reason='requires linux') 24 | class TestLinux: 25 | 26 | def test_location_from_data_home_envvar(self, monkeypatch): 27 | monkeypatch.setenv('DL_PLUS_DATA_HOME', '/dl/plus/data/home') 28 | # should be ignored 29 | monkeypatch.setenv('DL_PLUS_HOME', '/dl/plus/home') 30 | assert get_data_home() == Path('/dl/plus/data/home') 31 | 32 | def test_location_from_home_envvar(self, monkeypatch): 33 | monkeypatch.setenv('DL_PLUS_HOME', '/dl/plus/home') 34 | assert get_data_home() == Path('/dl/plus/home') 35 | 36 | def test_default_location_no_xdg_data_home_envvar(self, monkeypatch): 37 | monkeypatch.setattr(Path, 'home', lambda: Path('/fakehome')) 38 | monkeypatch.delenv('XDG_DATA_HOME', raising=False) 39 | assert get_data_home() == Path('/fakehome/.local/share/dl-plus') 40 | 41 | def test_default_location_with_xdg_data_home_envvar(self, monkeypatch): 42 | monkeypatch.setenv('XDG_DATA_HOME', '/xdgdata') 43 | assert get_data_home() == Path('/xdgdata/dl-plus') 44 | 45 | # config_home fallback tests; remove all class code below when 46 | # fallback is removed 47 | 48 | @pytest.fixture 49 | def config_home(self, tmp_path): 50 | _config_home = tmp_path / 'dl-plus-config-home' 51 | _config_home.mkdir() 52 | return _config_home 53 | 54 | @pytest.mark.parametrize('data_dir', ['backends', 'extractors']) 55 | def test_fallback_with_one_dir( 56 | self, monkeypatch, tmp_path, config_home, data_dir, 57 | ): 58 | monkeypatch.setenv('XDG_DATA_HOME', str(tmp_path / 'xdgdata')) 59 | (config_home / data_dir).mkdir() 60 | with pytest.warns(DLPlusDeprecationWarning) as warn_record: 61 | assert get_data_home() == config_home 62 | assert len(warn_record) == 1 63 | assert f"move '{data_dir}' directory" in warn_record[0].message.args[0] 64 | 65 | def test_fallback_with_both_dirs(self, monkeypatch, tmp_path, config_home): 66 | monkeypatch.setenv('XDG_DATA_HOME', str(tmp_path / 'xdgdata')) 67 | (config_home / 'backends').mkdir() 68 | (config_home / 'extractors').mkdir() 69 | with pytest.warns(DLPlusDeprecationWarning) as warn_record: 70 | assert get_data_home() == config_home 71 | assert len(warn_record) == 2 72 | assert "move 'backends' directory" in warn_record[0].message.args[0] 73 | assert "move 'extractors' directory" in warn_record[1].message.args[0] 74 | 75 | def test_no_fallback(self, monkeypatch, tmp_path): 76 | monkeypatch.setenv('XDG_DATA_HOME', str(tmp_path / 'xdgdata')) 77 | assert get_data_home() == tmp_path / 'xdgdata' / 'dl-plus' 78 | 79 | 80 | @pytest.mark.skipif( 81 | not sys.platform.startswith('win'), reason='requires windows') 82 | class TestWindows: 83 | 84 | def test_location_from_data_home_envvar(self, monkeypatch): 85 | monkeypatch.setenv('DL_PLUS_DATA_HOME', 'X:/dl/plus/data/home') 86 | # should be ignored 87 | monkeypatch.setenv('DL_PLUS_HOME', r'X:\dl\plus\home') 88 | assert get_data_home() == Path('X:/dl/plus/data/home') 89 | 90 | def test_location_from_home_envvar(self, monkeypatch): 91 | monkeypatch.setenv('DL_PLUS_HOME', r'X:\dl\plus\home') 92 | assert get_data_home() == Path('X:/dl/plus/home') 93 | 94 | def test_default_location_no_appdata_envvar(self, monkeypatch): 95 | monkeypatch.delenv('AppData', raising=False) 96 | monkeypatch.setattr(Path, 'home', lambda: Path('X:/fakehome')) 97 | assert get_data_home() == Path('X:/fakehome/AppData/Roaming/dl-plus') 98 | 99 | def test_default_location_with_appdata_envvar(self, monkeypatch): 100 | monkeypatch.setenv('AppData', 'X:/appdata') 101 | assert get_data_home() == Path('X:/appdata/dl-plus') 102 | -------------------------------------------------------------------------------- /src/dl_plus/cli/cli.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import pathlib 3 | import sys 4 | from textwrap import dedent 5 | from typing import Union 6 | 7 | from dl_plus import core, ytdl 8 | from dl_plus.backend import get_known_backends, init_backend 9 | from dl_plus.config import Config 10 | from dl_plus.const import DL_PLUS_VERSION 11 | from dl_plus.exceptions import DLPlusException 12 | 13 | from . import args as cli_args 14 | 15 | 16 | __all__ = ['main'] 17 | 18 | _PROG = 'dl-plus' 19 | _CMD = '--cmd' 20 | 21 | 22 | def _dedent(text): 23 | return dedent(text).strip() 24 | 25 | 26 | def _detect_compat_mode(program_name: str) -> bool: 27 | path = pathlib.Path(program_name) 28 | if path.suffix == '.exe': 29 | name = path.stem 30 | else: 31 | name = path.name 32 | return name in ( 33 | backend.executable_name for backend 34 | in get_known_backends().values() 35 | ) 36 | 37 | 38 | class _MainArgParser(argparse.ArgumentParser): 39 | 40 | def format_help(self): 41 | return super().format_help() + ytdl.get_help() 42 | 43 | 44 | def _get_main_parser() -> argparse.ArgumentParser: 45 | parser = _MainArgParser( 46 | prog=_PROG, 47 | usage=( 48 | '%(prog)s ' 49 | '[--dlp-config PATH | --no-dlp-config] ' 50 | '[--backend BACKEND] [--extractor EXTRACTOR] ' 51 | '[YOUTUBE-DL OPTIONS] URL [URL...]' 52 | ), 53 | description=_dedent(""" 54 | %(prog)s is a youtube-dl extension with pluggable extractors. 55 | 56 | The following are %(prog)s options: 57 | """), 58 | epilog='The following are youtube-dl options:', 59 | add_help=False, 60 | formatter_class=argparse.RawTextHelpFormatter, 61 | ) 62 | cli_args.dlp_config.add_to_parser(parser) 63 | parser.add_argument( 64 | '--backend', 65 | metavar='BACKEND', 66 | help='youtube-dl backend.', 67 | ) 68 | parser.add_argument( 69 | '--dlp-version', 70 | action='version', 71 | version=DL_PLUS_VERSION, 72 | help='Print dl-plus version and exit.', 73 | ) 74 | extractor_group = parser.add_mutually_exclusive_group() 75 | extractor_group.add_argument( 76 | '-E', '--extractor', 77 | action='append', 78 | help=_dedent(""" 79 | Extractor name. Can be specified multiple times: -E foo -E bar. 80 | """), 81 | ) 82 | extractor_group.add_argument( 83 | '--force-generic-extractor', 84 | action='store_true', 85 | help=argparse.SUPPRESS, 86 | ) 87 | parser.add_argument( 88 | '-h', '--help', 89 | action='store_true', 90 | help=argparse.SUPPRESS, 91 | ) 92 | return parser 93 | 94 | 95 | def _main(argv): 96 | args = argv[1:] 97 | if '-U' in args or '--update' in args: 98 | raise DLPlusException('update is not yet supported') 99 | compat_mode = _detect_compat_mode(argv[0]) 100 | config = Config() 101 | backend = None 102 | if not compat_mode: 103 | if _CMD in args: 104 | from .command import run_command 105 | run_command(prog=_PROG, cmd_arg=_CMD, args=args) 106 | return 107 | parser = _get_main_parser() 108 | parsed_args, ytdl_args = parser.parse_known_args(args) 109 | backend = parsed_args.backend 110 | config_file: Union[str, bool, None] 111 | if parsed_args.no_dlp_config: 112 | config_file = False 113 | else: 114 | config_file = parsed_args.dlp_config 115 | config.load(config_file) 116 | else: 117 | ytdl_args = args 118 | config.load() 119 | if not backend: 120 | backend = config.backend 121 | init_backend(backend) 122 | force_generic_extractor = False 123 | extractors = None 124 | if not compat_mode: 125 | if parsed_args.help: 126 | parser.print_help() 127 | return 128 | force_generic_extractor = parsed_args.force_generic_extractor 129 | extractors = parsed_args.extractor 130 | if force_generic_extractor: 131 | ytdl_args.append('--force-generic-extractor') 132 | else: 133 | if not extractors: 134 | extractors = config.extractors 135 | core.enable_extractors(extractors) 136 | backend_options = config.backend_options 137 | if backend_options is not None: 138 | ytdl_args = ['--ignore-config'] + backend_options + ytdl_args 139 | ytdl.run(ytdl_args) 140 | 141 | 142 | def main(argv=None): 143 | if argv is None: 144 | argv = sys.argv 145 | try: 146 | _main(argv) 147 | except DLPlusException as exc: 148 | sys.exit(exc) 149 | -------------------------------------------------------------------------------- /docs/extractor-plugin-authoring-guide.md: -------------------------------------------------------------------------------- 1 | # Extractor Plugin Authoring Guide 2 | 3 | **NOTE**: you can use the [un1def/goodgame][un1def-goodgame-extractor-repo] plugin repository as an example. 4 | 5 | 1. Choose a namespace. Namespaces are used to avoid name conflicts of different plugins created by different authors. It's recommended to use your name, username, or organization name as a namespace. Make sure that the namespace is not already taken (at least search for `dl-plus-extractor-` on [PyPI][pypi-website]). 6 | 7 | The namespace must consist only of lowercase latin letters and digits. It should be reasonable short to save typing. 8 | 9 | 2. Choose a plugin name. The plugin name should reflect the name of the service the plugin is intended for. 10 | 11 | The same restrictions and recommendations regarding allowed characters and length are apply to the plugin name as to the namespace. 12 | 13 | 3. Create the following directory structure: 14 | 15 | ``` 16 | dl_plus/ 17 | extractors/ 18 | / 19 | .py 20 | ``` 21 | 22 | Please note that there are no `__init__.py` files in any directory. It's crucial for the `dl-plus` plugin system, a single `__init__.py` can break all other plugins or even `dl-plus` itself. 23 | 24 | However, it's OK to use `__init__.py` inside the plugin *package*, that is, the following structure is allowed: 25 | 26 | ``` 27 | dl_plus/ 28 | extractors/ 29 | / 30 | / 31 | __init__.py 32 | extractor.py 33 | utils.py 34 | ... 35 | ``` 36 | 37 | The latter form can be used for plugins with a lot of code. 38 | 39 | If your namespace or plugin name starts with a digit, prepend a single underscore (`_`) to it. 40 | 41 | For example, if some user called `ZX_2000` wants to create a plugin for some service named `2chan TV`, they should create the following structure: 42 | 43 | ``` 44 | dl_plus/ 45 | extractors/ 46 | zx2000/ 47 | _2chantv.py 48 | ``` 49 | 50 | 4. Add plugin initialization code. Put the following lines in `.py`: 51 | 52 | ```python 53 | from dl_plus.extractor import Extractor, ExtractorPlugin 54 | 55 | plugin = ExtractorPlugin(__name__) 56 | ``` 57 | 58 | Please note that the plugin object must be available in the module's globals under the `plugin` name as in the code above. 59 | 60 | 5. Write your extractor class using the template from the `youtube-dl`'s [developer instructions][youtube-dl-extractor-guide] except for the following points: 61 | 62 | * Python 2 backward compatibility is redundant. Skip this part: 63 | ```python 64 | # coding: utf-8 65 | from __future__ import unicode_literals 66 | ``` 67 | * Use `dl_plus.extractor.Extractor` instead of `youtube_dl.extractor.common.InfoExtractor` as a base class. 68 | * Use `dl_plus.extractor.ExtractorError` instead of `youtube_dl.utils.ExtractorError`. 69 | * Do not import from `youtube_dl` directly, use `import_module`/`import_from` helpers from `dl_plus.ytdl`: 70 | ```python 71 | # from youtube_dl import utils 72 | utils = dl_plus.ytdl.import_module('utils') 73 | # from youtube_dl.utils import try_get 74 | try_get = dl_plus.ytdl.import_from('utils', 'try_get') 75 | # from yotube_dl.utils import int_or_none, float_or_none 76 | int_or_none, float_or_none = dl_plus.ytdl.import_from('utils', ['int_or_none', 'float_or_none']) 77 | ``` 78 | * `_TEST`/`_TESTS` are not supported at the moment (and will probably not be supported in the future). 79 | * Do not define the `IE_NAME` attribute, it will be generated by the plugin system automatically. 80 | 81 | ```python 82 | class MyPluginExtractor(Extractor): 83 | _VALID_URL = r'https?://...' 84 | 85 | def _real_extract(self, url): 86 | ... 87 | ``` 88 | 89 | 6. Register your extractor. There are two options: 90 | 91 | * If there is only one extractor in the plugin, you can either leave the extractor unnamed or give it a name. 92 | * If there are two or more extractors in the plugin, all of them must be named. 93 | 94 | The unnamed extractor is identified by the plugin name: `/`. The named extractor is identified by a combination of the plugin name and its own name: `/:`. 95 | 96 | The same restrictions and recommendations regarding allowed characters and length are apply to the extractor name as to the namespace and the plugin name. 97 | 98 | * Register the unnamed extractor: 99 | 100 | ```python 101 | @plugin.register 102 | class MyUnnamedExtractor(Extractor): 103 | ... 104 | ``` 105 | 106 | * Register the named extractor under the `playlist` name: 107 | 108 | ```python 109 | @plugin.register('playlist') 110 | class MyPlaylistExtractor(Extractor): 111 | ... 112 | ``` 113 | 114 | 7. Make your plugin importable by `dl-plus`. That is, ensure that the plugin's directory (the top-level directory, `dl_plus`) is in `sys.path`. The most common way is to install the plugin into the same virtual environment as `dl-plus` (or globally if `dl-plus` is installed globally), but you can also use the `PYTHONPATH` variable, `.pth` files – you name it. 115 | 116 | 8. If you want to upload your plugin to the [PyPI][pypi-website], use the following name format for the package: 117 | 118 | ``` 119 | dl-plus-extractor-- 120 | ``` 121 | 122 | The same name should be used for the project's public repository. 123 | 124 | 125 | [un1def-goodgame-extractor-repo]: https://github.com/un-def/dl-plus-extractor-un1def-goodgame 126 | [pypi-website]: https://pypi.org/ 127 | [youtube-dl-extractor-guide]: https://github.com/ytdl-org/youtube-dl#adding-support-for-a-new-site 128 | -------------------------------------------------------------------------------- /src/dl_plus/extractor/plugin.py: -------------------------------------------------------------------------------- 1 | from functools import partial 2 | from typing import ( 3 | TYPE_CHECKING, Callable, Dict, List, Optional, Type, Union, cast, overload, 4 | ) 5 | 6 | from dl_plus.exceptions import DLPlusException 7 | 8 | from .peqn import PEQN 9 | 10 | 11 | if TYPE_CHECKING: 12 | from .extractor import Extractor 13 | 14 | 15 | EXTRACTOR_PEQN_ATTR = '_dl_plus_peqn' 16 | 17 | 18 | ExtractorType = Type['Extractor'] 19 | RegisterDecoratorType = Callable[[ExtractorType], ExtractorType] 20 | 21 | 22 | class ExtractorPluginError(DLPlusException): 23 | 24 | pass 25 | 26 | 27 | class ExtractorPlugin: 28 | """ 29 | An extractor plugin object 30 | 31 | :param str import_path: An import path of the plugin module. 32 | The :code:`__name__` module attribute should be used as a value: 33 | 34 | .. code-block:: 35 | 36 | # dl_plus/extractors/somenamespace/someplugin.py 37 | plugin = ExtractorPlugin(__name__) 38 | 39 | :raises ExtractorPluginError: if the passed import path does not conform to 40 | plugin structure rules. 41 | """ 42 | 43 | def __init__(self, import_path: str) -> None: 44 | try: 45 | self._base_peqn = PEQN.from_plugin_import_path(import_path) 46 | except ValueError as exc: 47 | raise ExtractorPluginError(exc) 48 | self._extractors: Dict[Optional[str], ExtractorType] = {} 49 | 50 | @overload 51 | def register(self, __extractor_cls: ExtractorType) -> ExtractorType: 52 | ... 53 | 54 | @overload 55 | def register( 56 | self, __extractor_cls: ExtractorType, *, name: str, 57 | ) -> ExtractorType: 58 | ... 59 | 60 | @overload 61 | def register(self, __name: str) -> RegisterDecoratorType: 62 | ... 63 | 64 | def register( 65 | self, __extractor_cls_or_name: Union[ExtractorType, str, None] = None, 66 | *, name: Optional[str] = None, 67 | ) -> Union[ExtractorType, RegisterDecoratorType]: 68 | """ 69 | Register the given :class:`dl_plus.extractor.Extractor` class 70 | in the plugin's registry 71 | 72 | This method can be used as: 73 | 74 | * a regular method: 75 | 76 | .. code-block:: 77 | 78 | plugin.register(UnnamedExtractor) 79 | 80 | .. code-block:: 81 | 82 | plugin.register(NamedExtractor, name='clip') 83 | 84 | * a decorator: 85 | 86 | .. code-block:: 87 | 88 | @plugin.register 89 | class UnnamedExtractor(Extractor): 90 | ... 91 | 92 | .. code-block:: 93 | 94 | @plugin.register('clip') 95 | class NamedExtractor(Extractor): 96 | ... 97 | 98 | :raises ExtractorPluginError: 99 | """ 100 | # A special case for `@plugin.extractor()` syntax 101 | if __extractor_cls_or_name is None: 102 | return partial(self._register, name=name) 103 | if isinstance(__extractor_cls_or_name, str): 104 | return partial(self._register, name=__extractor_cls_or_name) 105 | return self._register(__extractor_cls_or_name, name) 106 | 107 | def _register( 108 | self, extractor_cls: ExtractorType, name: Optional[str], 109 | ) -> ExtractorType: 110 | from .extractor import Extractor 111 | if not issubclass(extractor_cls, Extractor): 112 | raise ExtractorPluginError( 113 | f'Extractor subclass expected, got: {extractor_cls!r}') 114 | if EXTRACTOR_PEQN_ATTR in extractor_cls.__dict__: # type: ignore 115 | peqn = cast( 116 | PEQN, 117 | extractor_cls.__dict__[EXTRACTOR_PEQN_ATTR], # type: ignore 118 | ) 119 | raise ExtractorPluginError( 120 | f'the extractor {extractor_cls!r} is already registered ' 121 | f'as "{peqn}"' 122 | ) 123 | if extractor_cls.IE_NAME is not None: 124 | raise ExtractorPluginError( 125 | f'the extractor {extractor_cls!r} has non-None IE_NAME value') 126 | if name and name in self._extractors: 127 | raise ExtractorPluginError( 128 | f'the plugin already contains an extractor called "{name}"') 129 | if name and None in self._extractors or not name and self._extractors: 130 | raise ExtractorPluginError( 131 | f'{extractor_cls!r}: the unnamed extractor must be ' 132 | f'the only extractor in the plugin' 133 | ) 134 | if 'DLP_REL_URL' in extractor_cls.__dict__: # type: ignore 135 | rel_url = extractor_cls.DLP_REL_URL 136 | if not rel_url: 137 | raise ExtractorPluginError( 138 | f'{extractor_cls!r}: DLP_REL_URL is not set') 139 | base_url = extractor_cls.DLP_BASE_URL 140 | if not base_url: 141 | raise ExtractorPluginError( 142 | f'{extractor_cls!r}: DLP_REL_URL without DLP_BASE_URL') 143 | if '_VALID_URL' in extractor_cls.__dict__: # type: ignore 144 | raise ExtractorPluginError( 145 | f'{extractor_cls!r}: DLP_REL_URL and _VALID_URL ' 146 | f'are mutually exclusive' 147 | ) 148 | extractor_cls._VALID_URL = base_url + rel_url 149 | peqn = self._base_peqn.copy(name=name) 150 | setattr(extractor_cls, EXTRACTOR_PEQN_ATTR, peqn) 151 | extractor_cls.IE_NAME = str(peqn) 152 | self._extractors[name or None] = extractor_cls 153 | return extractor_cls 154 | 155 | def get_extractor(self, name: Optional[str] = None) -> ExtractorType: 156 | """ 157 | Get the extractor class from the plugin's registry 158 | 159 | :param str name: (optional) The name of the extractor. The same value 160 | that was used to register the extractor (see :meth:`register`). 161 | Omit this parameter to get the unnamed extractor. 162 | :rtype: dl_plus.extractor.Extractor 163 | :raises KeyError: if no extractor found 164 | """ 165 | return self._extractors[name] 166 | 167 | def get_all_extractors(self) -> List[ExtractorType]: 168 | """ 169 | Get all extractors from the plugin's registry 170 | 171 | :rtype: list[dl_plus.extractor.Extractor] 172 | """ 173 | return list(self._extractors.values()) 174 | -------------------------------------------------------------------------------- /src/dl_plus/ytdl.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | import sys 3 | from io import StringIO 4 | 5 | from .exceptions import DLPlusException 6 | 7 | 8 | class YoutubeDLError(DLPlusException): 9 | 10 | pass 11 | 12 | 13 | class UnknownBuiltinExtractor(YoutubeDLError): 14 | 15 | def __init__(self, name): 16 | self.name = name 17 | 18 | def __str__(self): 19 | return f'unknown built-in extractor: {self.name}' 20 | 21 | 22 | _NAME_PART_SURROGATE = '_' 23 | 24 | 25 | _NOT_SET = object() 26 | 27 | 28 | _ytdl_module = _NOT_SET 29 | _ytdl_module_name = _NOT_SET 30 | 31 | _extractors = _NOT_SET 32 | _extractors_registry = _NOT_SET 33 | 34 | _lazy_load_extractor_base = _NOT_SET 35 | 36 | 37 | def _check_initialized(): 38 | global _ytdl_module 39 | if _ytdl_module is _NOT_SET: 40 | raise YoutubeDLError('not initialized') 41 | 42 | 43 | def init(ytdl_module_name: str) -> None: 44 | global _ytdl_module 45 | if _ytdl_module is not _NOT_SET: 46 | raise YoutubeDLError('already initialized') 47 | try: 48 | _ytdl_module = importlib.import_module(ytdl_module_name) 49 | except ImportError as exc: 50 | raise YoutubeDLError(f'failed to initialize: {exc}') from exc 51 | global _ytdl_module_name 52 | _ytdl_module_name = ytdl_module_name 53 | 54 | 55 | def run(args): 56 | _check_initialized() 57 | global _ytdl_module 58 | global _ytdl_module_name 59 | orig_sys_argv = sys.argv 60 | try: 61 | sys.argv = [_ytdl_module_name.replace('_', '-'), *args] 62 | _ytdl_module.main() 63 | finally: 64 | sys.argv = orig_sys_argv 65 | 66 | 67 | def get_ytdl_module(): 68 | _check_initialized() 69 | global _ytdl_module 70 | return _ytdl_module 71 | 72 | 73 | def get_ytdl_module_name(): 74 | _check_initialized() 75 | global _ytdl_module_name 76 | return _ytdl_module_name 77 | 78 | 79 | def get_ytdl_module_version(): 80 | return import_from('version', '__version__') 81 | 82 | 83 | def get_help(): 84 | _check_initialized() 85 | global _ytdl_module 86 | with StringIO() as buffer: 87 | stdout, stderr = sys.stdout, sys.stderr 88 | sys.stdout = sys.stderr = buffer 89 | try: 90 | _ytdl_module.main(['--help']) 91 | except SystemExit: 92 | pass 93 | finally: 94 | sys.stdout, sys.stderr = stdout, stderr 95 | return buffer.getvalue().partition('Options:')[2] 96 | 97 | 98 | def import_module(module_name): 99 | _check_initialized() 100 | global _ytdl_module_name 101 | return importlib.import_module(f'{_ytdl_module_name}.{module_name}') 102 | 103 | 104 | def _import_from(module, name): 105 | try: 106 | return getattr(module, name) 107 | except Exception as exc: 108 | raise ImportError(f'failed to import {name}: {exc}') from exc 109 | 110 | 111 | def import_from(module_name, names): 112 | module = import_module(module_name) 113 | if isinstance(names, str): 114 | return _import_from(module, names) 115 | return tuple(_import_from(module, name) for name in names) 116 | 117 | 118 | def get_all_extractors(*, include_generic: bool): 119 | _check_initialized() 120 | global _extractors 121 | if _extractors is _NOT_SET: 122 | _extractors = tuple( 123 | import_from('extractor', 'gen_extractor_classes')()) 124 | if include_generic: 125 | return _extractors 126 | return _extractors[:-1] 127 | 128 | 129 | def _get_real_extractor(extractor): 130 | global _lazy_load_extractor_base 131 | if _lazy_load_extractor_base is None: 132 | return extractor 133 | if _lazy_load_extractor_base is _NOT_SET: 134 | try: 135 | _lazy_load_extractor_base = import_from( 136 | 'extractor.lazy_extractors', 'LazyLoadExtractor') 137 | except ImportError: 138 | _lazy_load_extractor_base = None 139 | return extractor 140 | if not issubclass(extractor, _lazy_load_extractor_base): 141 | return extractor 142 | if 'real_class' in _lazy_load_extractor_base.__dict__: 143 | return extractor.real_class 144 | if '_get_real_class' in _lazy_load_extractor_base.__dict__: 145 | return extractor._get_real_class() 146 | return extractor 147 | 148 | 149 | def _get_extractor_name(extractor): 150 | ie_name = extractor.IE_NAME 151 | if isinstance(ie_name, property): 152 | return extractor().IE_NAME 153 | return ie_name 154 | 155 | 156 | def _build_extractors_registry(): 157 | registry = {} 158 | for extractor in get_all_extractors(include_generic=True): 159 | extractor = _get_real_extractor(extractor) 160 | name_parts = _get_extractor_name(extractor).split(':') 161 | name_parts.reverse() 162 | _store_extractor_in_registry(extractor, name_parts, registry) 163 | return registry 164 | 165 | 166 | def _store_extractor_in_registry(extractor, name_parts, registry): 167 | name_part = name_parts.pop() 168 | if name_part in registry: 169 | stored = registry[name_part] 170 | if not isinstance(stored, dict): 171 | subregistry = registry[name_part] = {_NAME_PART_SURROGATE: stored} 172 | else: 173 | subregistry = stored 174 | if not name_parts: 175 | if _NAME_PART_SURROGATE in subregistry: 176 | raise YoutubeDLError(f'duplicate name: {extractor!r}') 177 | subregistry[_NAME_PART_SURROGATE] = extractor 178 | else: 179 | _store_extractor_in_registry(extractor, name_parts, subregistry) 180 | elif not name_parts: 181 | registry[name_part] = extractor 182 | else: 183 | subregistry = registry[name_part] = {} 184 | _store_extractor_in_registry(extractor, name_parts, subregistry) 185 | 186 | 187 | def _flatten_registry_gen(registry): 188 | for value in registry.values(): 189 | if not isinstance(value, dict): 190 | yield value 191 | else: 192 | yield from _flatten_registry_gen(value) 193 | 194 | 195 | def _get_extractors_from_registry(name_parts, registry): 196 | name_part = name_parts.pop() 197 | stored = registry[name_part] 198 | if not name_parts: 199 | if not isinstance(stored, dict): 200 | return [stored] 201 | return list(_flatten_registry_gen(stored)) 202 | if not isinstance(stored, dict): 203 | raise KeyError(name_part) 204 | return _get_extractors_from_registry(name_parts, stored) 205 | 206 | 207 | def get_extractors_by_name(name): 208 | _check_initialized() 209 | global _extractors_registry 210 | if _extractors_registry is _NOT_SET: 211 | _extractors_registry = _build_extractors_registry() 212 | name_parts = name.split(':') 213 | name_parts.reverse() 214 | try: 215 | return _get_extractors_from_registry(name_parts, _extractors_registry) 216 | except KeyError: 217 | raise UnknownBuiltinExtractor(name) 218 | 219 | 220 | def patch_extractors(extractors): 221 | extractor_module = import_module('extractor') 222 | extractor_module.gen_extractor_classes = lambda: extractors 223 | ie_keys_extractors_map = { 224 | extractor.ie_key(): extractor for extractor in extractors} 225 | extractor_module.get_info_extractor = ( 226 | lambda ie_key: ie_keys_extractors_map[ie_key]) 227 | importlib.reload(import_module('YoutubeDL')) 228 | -------------------------------------------------------------------------------- /src/dl_plus/pypi.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import hashlib 4 | import json 5 | import os 6 | import shutil 7 | import subprocess 8 | import sys 9 | import tempfile 10 | import zipfile 11 | from collections.abc import Iterable 12 | from io import BytesIO 13 | from pathlib import Path 14 | from typing import ClassVar, Dict, NamedTuple, Optional 15 | from urllib.error import HTTPError 16 | from urllib.request import urlopen 17 | 18 | from dl_plus.exceptions import DLPlusException 19 | 20 | 21 | _HTTP_TIMEOUT = 30 22 | 23 | 24 | class Wheel(NamedTuple): 25 | name: str 26 | version: str 27 | metadata: Metadata 28 | filename: str 29 | url: str 30 | sha256: str 31 | 32 | 33 | class PyPIClientError(DLPlusException): 34 | 35 | pass 36 | 37 | 38 | class RequestError(PyPIClientError): 39 | 40 | pass 41 | 42 | 43 | class ParseError(PyPIClientError): 44 | 45 | pass 46 | 47 | 48 | class DownloadError(RequestError): 49 | 50 | def __init__( 51 | self, error: str, 52 | project_name: Optional[str] = None, version: Optional[str] = None, 53 | ) -> None: 54 | if project_name: 55 | if version: 56 | message = f'{project_name} {version}: {error}' 57 | else: 58 | message = f'{project_name}: {error}' 59 | else: 60 | message = error 61 | super().__init__(message) 62 | 63 | 64 | class Metadata(dict): 65 | 66 | @property 67 | def name(self) -> str: 68 | return self['info']['name'] 69 | 70 | @property 71 | def version(self) -> str: 72 | return self['info']['version'] 73 | 74 | @property 75 | def urls(self) -> Dict[str, Dict]: 76 | return self['urls'] 77 | 78 | @property 79 | def extras(self) -> Optional[list[str]]: 80 | return self['info']['provides_extra'] 81 | 82 | 83 | def save_metadata(backend_dir: Path, metadata: Metadata) -> None: 84 | with open(backend_dir / 'metadata.json', 'w') as fobj: 85 | json.dump(metadata, fobj) 86 | 87 | 88 | def load_metadata(backend_dir: Path) -> Optional[Metadata]: 89 | try: 90 | with open(backend_dir / 'metadata.json') as fobj: 91 | return Metadata(json.load(fobj)) 92 | except OSError: 93 | return None 94 | 95 | 96 | class PyPIClient: 97 | 98 | JSON_BASE_URL = 'https://pypi.org/pypi' 99 | 100 | def build_json_url( 101 | self, project_name: str, version: Optional[str] = None, 102 | ) -> str: 103 | parts = [self.JSON_BASE_URL, project_name] 104 | if version: 105 | parts.append(version) 106 | parts.append('json') 107 | return '/'.join(parts) 108 | 109 | def fetch_metadata( 110 | self, project_name: str, version: Optional[str] = None, 111 | ) -> Metadata: 112 | url = self.build_json_url(project_name, version) 113 | try: 114 | with urlopen(url, timeout=_HTTP_TIMEOUT) as response: 115 | return Metadata(json.load(response)) 116 | except (OSError, ValueError) as exc: 117 | raise RequestError from exc 118 | 119 | def _is_wheel_release(self, release: Dict) -> bool: 120 | return ( 121 | release['packagetype'] == 'bdist_wheel' and not release['yanked']) 122 | 123 | def fetch_wheel_info( 124 | self, project_name: str, version: Optional[str] = None, 125 | ) -> Wheel: 126 | try: 127 | metadata = self.fetch_metadata(project_name, version) 128 | except RequestError as exc: 129 | if isinstance(exc.error, HTTPError) and exc.error.code == 404: 130 | error = 'not found' 131 | else: 132 | error = 'unexpected error' 133 | raise DownloadError(error, project_name, version) from exc 134 | try: 135 | release = next(filter(self._is_wheel_release, metadata.urls)) 136 | except StopIteration: 137 | raise DownloadError('no wheel distribution', project_name, version) 138 | return Wheel( 139 | name=metadata.name, 140 | version=metadata.version, 141 | metadata=metadata, 142 | filename=release['filename'], 143 | url=release['url'], 144 | sha256=release['digests']['sha256'], 145 | ) 146 | 147 | 148 | def _is_pip_available(): 149 | exit_status = subprocess.call( 150 | [sys.executable, '-m', 'pip', '--version'], 151 | stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, 152 | ) 153 | return exit_status == 0 154 | 155 | 156 | class WheelInstaller: 157 | identifier: ClassVar[str] 158 | 159 | def __new__(cls) -> 'WheelInstaller': 160 | if cls is WheelInstaller: 161 | if _is_pip_available(): 162 | return PipWheelInstaller() 163 | else: 164 | return BuiltinWheelInstaller() 165 | return super().__new__(cls) 166 | 167 | def install( 168 | self, wheel: Wheel, output_dir: Path, 169 | extras: Optional[Iterable[str]] = None, 170 | ) -> None: 171 | _extras: tuple[str, ...] 172 | if extras is None: 173 | _extras = () 174 | else: 175 | _extras = tuple(extras) 176 | with tempfile.TemporaryDirectory() as _tmp_dir: 177 | tmp_dir = Path(_tmp_dir) / output_dir.name 178 | self._install(wheel, tmp_dir, _extras) 179 | if output_dir.exists(): 180 | shutil.rmtree(output_dir) 181 | else: 182 | os.makedirs(output_dir.parent, exist_ok=True) 183 | shutil.move(tmp_dir, output_dir.parent) 184 | save_metadata(output_dir, wheel.metadata) 185 | 186 | def _install(self, wheel: Wheel, tmp_dir: Path) -> None: 187 | raise NotImplementedError 188 | 189 | 190 | class BuiltinWheelInstaller(WheelInstaller): 191 | # builtin installer cannot install wheel dependencies 192 | identifier = 'builtin' 193 | 194 | def _install( 195 | self, wheel: Wheel, tmp_dir: Path, extras: tuple[str, ...], 196 | ) -> None: 197 | with self._download(wheel.url, wheel.sha256) as fobj: 198 | with zipfile.ZipFile(fobj) as zfobj: 199 | zfobj.extractall(tmp_dir) 200 | 201 | def _download(self, url: str, sha256: str) -> BytesIO: 202 | try: 203 | with urlopen(url, timeout=_HTTP_TIMEOUT) as response: 204 | buffer = BytesIO(response.read()) 205 | except OSError as exc: 206 | raise DownloadError(f'{url}: {exc}') from exc 207 | digest = hashlib.sha256(buffer.getvalue()) 208 | hexdigest = digest.hexdigest() 209 | if hexdigest != sha256: 210 | raise DownloadError( 211 | f'{url}: sha256 mismatch: expected {sha256}, got {hexdigest}') 212 | return buffer 213 | 214 | 215 | class PipWheelInstaller(WheelInstaller): 216 | # pip installer installs wheel dependencies 217 | identifier = 'pip' 218 | 219 | def _install( 220 | self, wheel: Wheel, tmp_dir: Path, extras: tuple[str, ...], 221 | ) -> None: 222 | if extras: 223 | _extras = f'[{",".join(extras)}]' 224 | else: 225 | _extras = '' 226 | subprocess.check_call([ 227 | sys.executable, '-m', 'pip', 'install', 228 | '--quiet', '--disable-pip-version-check', 229 | '--target', str(tmp_dir), 230 | '--only-binary', ':all:', 231 | f'{wheel.name}{_extras} @ {wheel.url}#sha256={wheel.sha256}', 232 | ]) 233 | -------------------------------------------------------------------------------- /tests/extractor/test_plugin.py: -------------------------------------------------------------------------------- 1 | import random 2 | import string 3 | 4 | import pytest 5 | 6 | from dl_plus import ytdl 7 | from dl_plus.extractor import Extractor 8 | from dl_plus.extractor.plugin import ExtractorPlugin, ExtractorPluginError 9 | 10 | 11 | def create_extractor(__base=None, **attrs): 12 | return type('TestExtractor', (__base or Extractor,), attrs) 13 | 14 | 15 | def create_plugin(ns=None, plugin=None): 16 | ns = ns or ''.join(random.choices(string.ascii_lowercase, k=6)) 17 | plugin = plugin or ''.join(random.choices(string.ascii_lowercase, k=6)) 18 | return ExtractorPlugin(f'dl_plus.extractors.{ns}.{plugin}') 19 | 20 | 21 | @pytest.fixture 22 | def plugin(): 23 | return create_plugin(ns='foo', plugin='bar') 24 | 25 | 26 | @pytest.mark.parametrize('path', [ 27 | 'somepackage.extractors.foo.bar', 28 | 'dl_plus.extractors.foo', 29 | 'dl_plus.extractors.foo.bar.baz', 30 | ]) 31 | def test_init_error_bad_import_path(path): 32 | with pytest.raises(ExtractorPluginError, match='bad plugin import path'): 33 | ExtractorPlugin(path) 34 | 35 | 36 | class TestRegister: 37 | 38 | NS = 'foo' 39 | PLUGIN = 'bar' 40 | 41 | @pytest.fixture(autouse=True) 42 | def setup(self): 43 | self.plugin = create_plugin(self.NS, self.PLUGIN) 44 | 45 | @property 46 | def registry(self): 47 | return self.plugin._extractors 48 | 49 | def test_decorator_unnamed_extractor(self): 50 | @self.plugin.register 51 | class TestExtractor(Extractor): 52 | pass 53 | assert self.registry == {None: TestExtractor} 54 | 55 | def test_decorator_unnamed_extractor_with_parentheses(self): 56 | @self.plugin.register() 57 | class TestExtractor(Extractor): 58 | pass 59 | assert self.registry == {None: TestExtractor} 60 | 61 | def test_decorator_named_extractor(self): 62 | @self.plugin.register('test') 63 | class TestExtractor(Extractor): 64 | pass 65 | assert self.registry == {'test': TestExtractor} 66 | 67 | def test_decorator_named_extractor_name_kw(self): 68 | @self.plugin.register(name='test') 69 | class TestExtractor(Extractor): 70 | pass 71 | assert self.registry == {'test': TestExtractor} 72 | 73 | def test_method_call_two_named_extractors(self): 74 | extractor_1 = self.plugin.register(create_extractor(), name='quux') 75 | extractor_2 = self.plugin.register(create_extractor(), name='quuz') 76 | assert self.registry == { 77 | 'quux': extractor_1, 78 | 'quuz': extractor_2, 79 | } 80 | 81 | def test_ie_name_unnamed_extractor(self): 82 | extractor = create_extractor() 83 | assert extractor.IE_NAME is None 84 | self.plugin.register(extractor) 85 | assert extractor.IE_NAME == f'{self.NS}/{self.PLUGIN}' 86 | 87 | def test_ie_name_named_extractor(self): 88 | extractor = create_extractor() 89 | assert extractor.IE_NAME is None 90 | self.plugin.register(extractor, name='quux') 91 | assert extractor.IE_NAME == f'{self.NS}/{self.PLUGIN}:quux' 92 | 93 | @pytest.mark.parametrize('base,rel,expected', [ 94 | ('foo://bar/', 'baz', 'foo://bar/baz'), 95 | ('foo://bar/', '/baz', 'foo://bar//baz'), 96 | ('foo://bar', 'baz', 'foo://barbaz'), 97 | ]) 98 | def test_dlp_url(self, base, rel, expected): 99 | extractor_base = create_extractor(DLP_BASE_URL=base) 100 | extractor = create_extractor(extractor_base, DLP_REL_URL=rel) 101 | self.plugin.register(extractor) 102 | assert extractor._VALID_URL == expected 103 | 104 | def test_error_dlp_rel_url_without_base_url(self): 105 | extractor_base = create_extractor() 106 | extractor = create_extractor(extractor_base, DLP_REL_URL='baz') 107 | with pytest.raises(ExtractorPluginError, match='without DLP_BASE_URL'): 108 | self.plugin.register(extractor) 109 | 110 | def test_error_dlp_rel_url_and_valid_url_conflict(self): 111 | extractor_base = create_extractor(DLP_BASE_URL='foo://bar/') 112 | extractor = create_extractor( 113 | extractor_base, DLP_REL_URL='baz', _VALID_URL='qux://quux') 114 | with pytest.raises(ExtractorPluginError, match='mutually exclusive'): 115 | self.plugin.register(extractor) 116 | 117 | def test_error_bad_superclass(self): 118 | InfoExtractor = ytdl.import_from('extractor.common', 'InfoExtractor') 119 | extractor = create_extractor(InfoExtractor) 120 | with pytest.raises(ExtractorPluginError, match='subclass expected'): 121 | self.plugin.register(extractor) 122 | 123 | def test_error_ie_name_is_set(self): 124 | extractor = create_extractor(IE_NAME='clip') 125 | with pytest.raises(ExtractorPluginError, match='non-None IE_NAME'): 126 | self.plugin.register(extractor) 127 | 128 | def test_error_already_registered_same_plugin(self): 129 | extractor = create_extractor() 130 | self.plugin.register(extractor, name='quux') 131 | with pytest.raises(ExtractorPluginError, match='already registered'): 132 | self.plugin.register(extractor, name='quuz') 133 | 134 | def test_error_already_registered_another_plugin(self): 135 | another_plugin = create_plugin() 136 | extractor = create_extractor() 137 | another_plugin.register(extractor, name='quux') 138 | with pytest.raises(ExtractorPluginError, match='already registered'): 139 | self.plugin.register(extractor, name='quuz') 140 | 141 | @pytest.mark.parametrize('name', [None, 'quux']) 142 | def test_error_unnamed_extractor_when_registry_is_not_empty(self, name): 143 | self.plugin.register(create_extractor(), name=name) 144 | with pytest.raises(ExtractorPluginError, match='the only extractor'): 145 | self.plugin.register(create_extractor()) 146 | 147 | def test_error_named_extractor_name_collision(self): 148 | self.plugin.register(create_extractor(), name='quux') 149 | with pytest.raises(ExtractorPluginError, match='already contains'): 150 | self.plugin.register(create_extractor(), name='quux') 151 | 152 | def test_error_named_extractor_when_unnamed_extractor_in_registry(self): 153 | self.plugin.register(create_extractor()) 154 | with pytest.raises(ExtractorPluginError, match='the only extractor'): 155 | self.plugin.register(create_extractor(), name='quux') 156 | 157 | 158 | def test_get_extractor_unnamed(plugin): 159 | extractor = plugin.register(create_extractor()) 160 | assert plugin.get_extractor() is extractor 161 | 162 | 163 | def test_get_extractor_named(plugin): 164 | extractor = plugin.register(create_extractor(), name='quux') 165 | assert plugin.get_extractor('quux') is extractor 166 | 167 | 168 | def test_get_extractor_unnamed_error(plugin): 169 | plugin.register(create_extractor(), name='quux') 170 | with pytest.raises(KeyError): 171 | plugin.get_extractor() 172 | 173 | 174 | def test_get_extractor_named_error(plugin): 175 | plugin.register(create_extractor(), name='quux') 176 | with pytest.raises(KeyError): 177 | plugin.get_extractor('quuz') 178 | 179 | 180 | def get_all_extractors_empty_registry(plugin): 181 | assert plugin.get_all_extractors() == [] 182 | 183 | 184 | def get_all_extractors_unnamed_extractor(plugin): 185 | extractor = plugin.register(create_extractor()) 186 | assert plugin.get_all_extractors() == [extractor] 187 | 188 | 189 | def get_all_extractors_two_named_extractors(plugin): 190 | extractor_1 = plugin.register(create_extractor(), name='quux') 191 | extractor_2 = plugin.register(create_extractor(), name='quuz') 192 | assert sorted(plugin.get_all_extractors()) == sorted( 193 | [extractor_1, extractor_2]) 194 | -------------------------------------------------------------------------------- /src/dl_plus/backend.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import re 4 | import sys 5 | from pathlib import Path 6 | from typing import TYPE_CHECKING, Iterable, NamedTuple, Optional 7 | 8 | from dl_plus import ytdl 9 | from dl_plus.config import ( 10 | ConfigError, ConfigValue, _Config, get_config_home, get_data_home, 11 | ) 12 | from dl_plus.exceptions import DLPlusException 13 | from dl_plus.pypi import load_metadata 14 | 15 | 16 | if TYPE_CHECKING: 17 | from .pypi import Metadata 18 | 19 | 20 | PROJECT_NAME_REGEX = re.compile(r'^[A-Za-z][A-Za-z0-9_-]*$') 21 | 22 | 23 | def _normalize(string: str) -> str: 24 | return string.replace('-', '_') 25 | 26 | 27 | class Backend(NamedTuple): 28 | # a name of the project ("distribution" in PyPA terms) 29 | # *.dist-info/METADATA->Name 30 | project_name: str 31 | # a name of the root package directory 32 | # *.dist-info/top_level.txt 33 | import_name: str 34 | # a name of the the executable script 35 | # *.dist-info/entry_points.txt->console_scripts). 36 | executable_name: str 37 | # a list of extras to install 38 | # *.dist-info/METADATA->Provides-Extra 39 | extras: list[str] 40 | 41 | 42 | DEFAULT_BACKENDS_CONFIG = """ 43 | [yt-dlp] 44 | project-name = yt-dlp 45 | import-name = yt_dlp 46 | executable-name = yt-dlp 47 | extras = default curl-cffi secretstorage 48 | 49 | [youtube-dl-nightly] 50 | project-name = youtube-dl-nightly 51 | import-name = youtube_dl 52 | executable-name = youtube-dl 53 | 54 | [youtube-dl] 55 | project-name = youtube_dl 56 | import-name = youtube_dl 57 | executable-name = youtube-dl 58 | 59 | [youtube-dlc] 60 | project-name = youtube-dlc 61 | import-name = youtube_dlc 62 | executable-name = youtube-dlc 63 | """ 64 | 65 | 66 | def is_project_name_valid(name: str) -> bool: 67 | return PROJECT_NAME_REGEX.fullmatch(name) is not None 68 | 69 | 70 | def parse_backends_config(content: str) -> dict[str, Backend]: 71 | config = _Config() 72 | config.read_string(content) 73 | backends = {} 74 | for alias in config.sections(): 75 | section = config[alias] 76 | project_name = section['project-name'] 77 | if not is_project_name_valid(project_name): 78 | raise ConfigError(f'invalid project-name: {project_name}') 79 | backends[_normalize(alias)] = Backend( 80 | project_name=project_name, 81 | import_name=section['import-name'], 82 | executable_name=section['executable-name'], 83 | extras=section.get('extras', '').split(), 84 | ) 85 | return backends 86 | 87 | 88 | _known_backends: dict[str, Backend] | None = None 89 | 90 | 91 | def get_backends_config_path() -> Optional[Path]: 92 | path = get_config_home() / 'backends.ini' 93 | if not path.is_file(): 94 | return None 95 | return path 96 | 97 | 98 | def get_known_backends() -> dict[str, Backend]: 99 | global _known_backends 100 | if _known_backends is not None: 101 | return _known_backends 102 | _known_backends = parse_backends_config(DEFAULT_BACKENDS_CONFIG) 103 | config_path = get_backends_config_path() 104 | if config_path: 105 | with open(config_path) as fobj: 106 | _known_backends.update(parse_backends_config(fobj.read())) 107 | return _known_backends 108 | 109 | 110 | def get_known_backend(alias: str) -> Optional[Backend]: 111 | return get_known_backends().get(_normalize(alias)) 112 | 113 | 114 | class BackendInfo(NamedTuple): 115 | alias: str | None 116 | import_name: str 117 | version: str 118 | path: Path 119 | is_managed: bool 120 | metadata: Metadata | None 121 | 122 | 123 | class BackendError(DLPlusException): 124 | 125 | pass 126 | 127 | 128 | class AutodetectFailed(BackendError): 129 | 130 | def __init__(self, candidates: Iterable[str]) -> None: 131 | self._candidates = tuple(candidates) 132 | 133 | def __str__(self) -> str: 134 | return 'failed to autodetect backend (candidates tested: {})'.format( 135 | ', '.join(self._candidates)) 136 | 137 | 138 | def _is_managed(location: Path) -> bool: 139 | try: 140 | location.relative_to(get_backends_dir()) 141 | return True 142 | except ValueError: 143 | return False 144 | 145 | 146 | def get_backends_dir() -> Path: 147 | return get_data_home() / 'backends' 148 | 149 | 150 | def get_backend_dir(backend: str) -> Path: 151 | return get_backends_dir() / _normalize(backend) 152 | 153 | 154 | def parse_backend_string(backend_string: str) -> tuple[bool, Path | None, str]: 155 | # backend_string is one of: 156 | # * alias ([section-name] in the backends.ini, e.g., 'yt-dlp'); 157 | # * 'import_name', e.g., 'youtube_dl'; 158 | # * 'project-name/import_name', e.g., 'youtube-dl-nightly/youtube_dl'. 159 | # is_alias is only True if [alias] != project-name, e.g., 160 | # * [yt-dlp] with project-name = yt-dlp -> False 161 | # * [yt-dlp-noextras] with project-name = yt-dlp -> True 162 | is_alias: bool 163 | if '/' in backend_string: 164 | is_alias = False 165 | project_name, _, import_name = backend_string.partition('/') 166 | backend_dir = get_backend_dir(project_name) 167 | if not backend_dir.is_dir(): 168 | raise BackendError( 169 | f'{backend_dir} does not exist or is not a directory') 170 | elif backend := get_known_backend(backend_string): 171 | is_alias = ( 172 | _normalize(backend_string) != _normalize(backend.project_name)) 173 | import_name = backend.import_name 174 | backend_dir = get_backend_dir(backend_string) 175 | if not backend_dir.is_dir(): 176 | # in case of backends not managed by dl-plus 177 | backend_dir = None 178 | else: 179 | is_alias = False 180 | import_name = backend_string 181 | backend_dir = get_backend_dir(backend_string) 182 | if not backend_dir.is_dir(): 183 | backend_dir = None 184 | return is_alias, backend_dir, _normalize(import_name) 185 | 186 | 187 | def _init_backend(backend_string: str) -> Path | None: 188 | is_alias, backend_dir, package_name = parse_backend_string(backend_string) 189 | if is_alias and not backend_dir: 190 | # avoid picking wrong backend by import_name only 191 | raise BackendError(f'{backend_string} is not installed') 192 | if backend_dir: 193 | sys.path.insert(0, str(backend_dir)) 194 | ytdl.init(package_name) 195 | return backend_dir 196 | 197 | 198 | def _autodetect_backend() -> tuple[str, Path | None]: 199 | candidates = tuple(get_known_backends()) 200 | for candidate in candidates: 201 | try: 202 | return candidate, _init_backend(candidate) 203 | except DLPlusException: 204 | pass 205 | raise AutodetectFailed(candidates) 206 | 207 | 208 | def init_backend(backend_string: str | None = None) -> BackendInfo: 209 | if backend_string is None: 210 | backend_string = ConfigValue.Backend.AUTODETECT 211 | alias: str | None 212 | if backend_string == ConfigValue.Backend.AUTODETECT: 213 | alias, backend_dir = _autodetect_backend() 214 | else: 215 | backend_dir = _init_backend(backend_string) 216 | backend = get_known_backend(backend_string) 217 | if backend is not None: 218 | alias = backend_string 219 | else: 220 | alias = None 221 | ytdl_module = ytdl.get_ytdl_module() 222 | path = Path(ytdl_module.__path__[0]) 223 | is_managed = _is_managed(path) 224 | if is_managed: 225 | metadata = load_metadata(backend_dir) 226 | else: 227 | metadata = None 228 | return BackendInfo( 229 | alias=alias, 230 | import_name=ytdl_module.__name__, 231 | version=ytdl.import_from('version', '__version__'), 232 | path=path, 233 | is_managed=is_managed, 234 | metadata=metadata, 235 | ) 236 | -------------------------------------------------------------------------------- /src/dl_plus/config.py: -------------------------------------------------------------------------------- 1 | import enum 2 | import itertools 3 | import os 4 | import shlex 5 | from configparser import ConfigParser 6 | from pathlib import Path 7 | from types import MappingProxyType 8 | from typing import List, Mapping, Optional, Union 9 | 10 | from dl_plus import deprecated 11 | 12 | from .exceptions import DLPlusException 13 | 14 | 15 | _IS_WIN = os.name == 'nt' 16 | 17 | 18 | class _StrEnum(str, enum.Enum): 19 | # enum.StrEnum is available since Python 3.11 20 | 21 | def __str__(self) -> str: 22 | return self.value 23 | 24 | 25 | class _EnvironVariable: 26 | __slots__ = ('names',) 27 | 28 | def __init__(self, *names: str) -> None: 29 | self.names = names 30 | 31 | def __get__(self, instance: Optional['_Environ'], owner) -> Optional[str]: 32 | if instance is None: 33 | return None 34 | for name in self.names: 35 | value = instance.get(name) 36 | if value is not None: 37 | return value 38 | return None 39 | 40 | 41 | class _Environ: 42 | CONFIG_HOME = _EnvironVariable('DL_PLUS_CONFIG_HOME', 'DL_PLUS_HOME') 43 | DATA_HOME = _EnvironVariable('DL_PLUS_DATA_HOME', 'DL_PLUS_HOME') 44 | CONFIG = _EnvironVariable('DL_PLUS_CONFIG') 45 | BACKEND = _EnvironVariable('DL_PLUS_BACKEND') 46 | 47 | def __init__(self, environ: Mapping[str, str]) -> None: 48 | self._environ = MappingProxyType(environ) 49 | 50 | def __getitem__(self, __key: str, /) -> str: 51 | return self._environ[__key] 52 | 53 | def __contains__(self, __key: str, /) -> bool: 54 | return __key in self._environ 55 | 56 | def get(self, __key: str, /) -> Optional[str]: 57 | return self._environ.get(__key) 58 | 59 | 60 | _environ = _Environ(os.environ) 61 | 62 | 63 | class Section(_StrEnum): 64 | MAIN = 'main' 65 | EXTRACTORS = 'extractors' 66 | BACKEND_OPTIONS = 'backend-options' 67 | 68 | DEPRECATED_EXTRACTORS = 'extractors.enable' 69 | 70 | 71 | class Option(_StrEnum): 72 | BACKEND = 'backend' 73 | 74 | 75 | class ConfigValue: 76 | 77 | class Backend(_StrEnum): 78 | AUTODETECT = ':autodetect:' 79 | 80 | class Extractor(_StrEnum): 81 | BUILTINS = ':builtins:' 82 | PLUGINS = ':plugins:' 83 | GENERIC = 'generic' 84 | 85 | 86 | DEFAULT_CONFIG = f""" 87 | [{Section.MAIN}] 88 | {Option.BACKEND} = {ConfigValue.Backend.AUTODETECT} 89 | 90 | [{Section.EXTRACTORS}] 91 | {ConfigValue.Extractor.PLUGINS} 92 | {ConfigValue.Extractor.BUILTINS} 93 | {ConfigValue.Extractor.GENERIC} 94 | """ 95 | 96 | 97 | class ConfigError(DLPlusException): 98 | 99 | pass 100 | 101 | 102 | def _get_win_app_data() -> Path: 103 | if app_data := _environ.get('AppData'): 104 | return Path(app_data) 105 | return Path.home() / 'AppData' / 'Roaming' 106 | 107 | 108 | _config_home: Optional[Path] = None 109 | 110 | 111 | def get_config_home() -> Path: 112 | global _config_home 113 | if _config_home: 114 | return _config_home 115 | path_from_env = _environ.CONFIG_HOME 116 | if path_from_env: 117 | _config_home = Path(path_from_env).resolve() 118 | return _config_home 119 | if _IS_WIN: 120 | parent = _get_win_app_data() 121 | else: 122 | xdg_config_home = _environ.get('XDG_CONFIG_HOME') 123 | if xdg_config_home: 124 | parent = Path(xdg_config_home) 125 | else: 126 | parent = Path.home() / '.config' 127 | _config_home = (parent / 'dl-plus').resolve() 128 | return _config_home 129 | 130 | 131 | def get_config_path(path: Union[Path, str, None] = None) -> Optional[Path]: 132 | is_default_path = False 133 | if not path: 134 | path = _environ.CONFIG 135 | if not path: 136 | path = get_config_home() / 'config.ini' 137 | is_default_path = True 138 | if isinstance(path, str): 139 | path = Path(path) 140 | path = path.resolve() 141 | if path.is_file(): 142 | return path 143 | if is_default_path: 144 | return None 145 | raise ConfigError(f'failed to get config path: {path} is not a file') 146 | 147 | 148 | class _Config(ConfigParser): 149 | 150 | def __init__(self) -> None: 151 | super().__init__( 152 | allow_no_value=True, 153 | delimiters=('=',), 154 | comment_prefixes=('#', ';'), 155 | inline_comment_prefixes=None, 156 | strict=True, 157 | empty_lines_in_values=False, 158 | default_section=None, 159 | interpolation=None, 160 | ) 161 | 162 | 163 | class _ConfigOptionProxy: 164 | 165 | __slots__ = ('section', 'option') 166 | 167 | def __init__(self, section: str, option: str) -> None: 168 | self.section = section 169 | self.option = option 170 | 171 | def __get__(self, instance: 'Config', owner): 172 | if instance is None: 173 | return self 174 | return instance.get(self.section, self.option) 175 | 176 | def __set__(self, instance: 'Config', value: str): 177 | instance.set(self.section, self.option, value) 178 | 179 | 180 | class Config(_Config): 181 | 182 | _UPDATE_SECTIONS = (Section.MAIN,) 183 | _REPLACE_SECTIONS = (Section.EXTRACTORS, Section.BACKEND_OPTIONS) 184 | 185 | def __init__(self) -> None: 186 | super().__init__() 187 | self.read_string(DEFAULT_CONFIG) 188 | 189 | def load( 190 | self, path: Union[Path, str, bool, None] = None, environ: bool = True, 191 | ) -> None: 192 | if path is True: 193 | path = None 194 | if path is not False: 195 | self.load_from_file(path) 196 | if environ: 197 | self.load_from_environ() 198 | 199 | def load_from_file(self, path: Union[Path, str, None] = None) -> None: 200 | _path = get_config_path(path) 201 | if not _path: 202 | return 203 | config = _Config() 204 | try: 205 | with open(_path) as fobj: 206 | config.read_file(fobj) 207 | except (OSError, ValueError) as exc: 208 | raise ConfigError(f'failed to load config: {exc}') from exc 209 | self._process_deprecated_extractors_section(config) 210 | for section in self._UPDATE_SECTIONS: 211 | self._update_section(section, config, replace=False) 212 | for section in self._REPLACE_SECTIONS: 213 | self._update_section(section, config, replace=True) 214 | 215 | def _process_deprecated_extractors_section(self, config: _Config) -> None: 216 | try: 217 | deprecated_extractors = config[Section.DEPRECATED_EXTRACTORS] 218 | except KeyError: 219 | return 220 | # deprecated in 0.7 221 | deprecated.warn( 222 | 'deprecated config section name: ' 223 | 'rename [extractors.enable] to [extractors]' 224 | ) 225 | if not config.has_section(Section.EXTRACTORS): 226 | config[Section.EXTRACTORS] = deprecated_extractors 227 | 228 | def load_from_environ(self) -> None: 229 | if backend := _environ.BACKEND: 230 | self.backend = backend 231 | 232 | def _update_section( 233 | self, name: str, config: _Config, replace: bool, 234 | ) -> None: 235 | if not config.has_section(name): 236 | return 237 | if not self.has_section(name): 238 | self.add_section(name) 239 | section = self[name] 240 | if replace: 241 | section.clear() 242 | section.update(config[name]) 243 | 244 | backend = _ConfigOptionProxy(Section.MAIN, Option.BACKEND) 245 | 246 | @property 247 | def extractors(self) -> List[str]: 248 | return self.options(Section.EXTRACTORS) 249 | 250 | @property 251 | def backend_options(self) -> Optional[List[str]]: 252 | if not self.has_section(Section.BACKEND_OPTIONS): 253 | return None 254 | return list(itertools.chain.from_iterable( 255 | map(shlex.split, self.options(Section.BACKEND_OPTIONS)))) 256 | 257 | 258 | _data_home: Optional[Path] = None 259 | 260 | 261 | def get_data_home() -> Path: 262 | global _data_home 263 | if _data_home: 264 | return _data_home 265 | path_from_env = _environ.DATA_HOME 266 | if path_from_env: 267 | _data_home = Path(path_from_env).resolve() 268 | return _data_home 269 | if _IS_WIN: 270 | parent = _get_win_app_data() 271 | else: 272 | xdg_data_home = _environ.get('XDG_DATA_HOME') 273 | if xdg_data_home: 274 | parent = Path(xdg_data_home) 275 | else: 276 | parent = Path.home() / '.local' / 'share' 277 | _data_home = (parent / 'dl-plus').resolve() 278 | 279 | # deprecated in 0.7 280 | if not _IS_WIN and not _data_home.exists(): 281 | use_config_home_as_data_home = False 282 | config_home = get_config_home() 283 | if (config_home / 'backends').exists(): 284 | deprecated.warn( 285 | "deprecated backends location: move 'backends' directory " 286 | f'from {config_home} to {_data_home}' 287 | ) 288 | use_config_home_as_data_home = True 289 | if (config_home / 'extractors').exists(): 290 | deprecated.warn( 291 | "deprecated extractor plugins location: move 'extractors' " 292 | f'directory from {config_home} to {_data_home}' 293 | ) 294 | use_config_home_as_data_home = True 295 | if use_config_home_as_data_home: 296 | _data_home = config_home 297 | 298 | return _data_home 299 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## 0.10.1 4 | 5 | ### Fixes 6 | 7 | * Fix compatibility with `yt-dlp` 2025.03.21+ 8 | 9 | ## 0.10.0 10 | 11 | ### Features 12 | 13 | * Backend `extras` support. `yt-dlp` is now installed with `default`, `curl-cffi`, `secretstorage` extra dependencies by default (configurable via `backends.ini`). 14 | * **(CLI)** Proper alias support. 15 | 16 | ### Fixes 17 | 18 | * Fix backend autodetect inconsistency. Previously, `dl-plus ` and `dl-plus --cmd backend ` could pick different backends if unmanaged backends were present. 19 | 20 | ## 0.9.0 21 | 22 | ### Breaking Changes 23 | 24 | * Dropped support for Python 3.8. 25 | 26 | ### Features 27 | 28 | * **(CLI)** New management commands were added — `backend list`, `backend uninstall NAME`, `extractor list`, `extractor uninstall NAME`. 29 | * Added support for Python 3.13. 30 | 31 | ### Fixes 32 | 33 | * **(CLI)** Fix `backend update` `PermissionError` on Windows. 34 | * **(CLI)** Don't try to validate a backend name if not passed. Fixes `TypeError` on `backend update` command. 35 | 36 | ## 0.8.0 37 | 38 | ### Features 39 | 40 | * **(Config)** Backend configuration file. A new config file `$DL_PLUS_CONFIG_HOME/backends.ini` can be used to extend/override the predefined list of backends. 41 | * **(CLI)** Two new commands were added — `config show main` and `config show backends` — to display `config.ini` and `backends.ini`, respectively. 42 | * A new backend/extractor installer based on `pip`. Unlike previously used `builtin` installer, `pip` installer is able to install dependencies. The new installer is used if `pip` is available, the legacy `builtin` installer is used otherwise. 43 | 44 | ### Improvements 45 | 46 | * **(Extractor API)** `yt-dlp`'s `_sort_format` warnings are supressed when possible. 47 | * **(CLI)** Backend name validation in backend–related commands. 48 | 49 | ## 0.7.0 50 | 51 | ### Breaking Changes 52 | 53 | * **(Config)** The `[extractors.enable]` section was renamed to `[extractors]`. The old name is still supported but deprecated. It is strongly recommended to update the config. 54 | * On \*nix (more specifically, non-Windows) systems the default location for managed backends and extractor plugins was changed from `$XDG_CONFIG_HOME/dl-plus` to `$XDG_DATA_HOME/dl-plus`. The old location is still supported but deprecated. It is strongly recommended to either move the data to the new location or set the `DL_PLUS_DATA_HOME`/`DL_PLUS_HOME` environment variable to keep the old location. 55 | 56 | ### Features 57 | 58 | * Added support for Python 3.12. 59 | * **(Env)** Added two new environment variables: `DL_PLUS_CONFIG_HOME` and `DL_PLUS_DATA_HOME`. These variables have higher priority than `DL_PLUS_HOME`. 60 | `DL_PLUS_CONFIG_HOME` is a directory where the config file is stored. `DL_PLUS_DATA_HOME` is a directory where managed backends and extractor plugins are stored. 61 | * **(Env)** Added a new environment variable `DL_PLUS_BACKEND`, which has higher priority than the config setting, but lower than the command line argument. The variable can be used, for example, to temporarily override the backend setting, especially if it is not possible/hard to do via the command line argument, e.g., `DL_PLUS_BACKEND=youtube-dl mpv https://...` can be used instead of `mpv --ytdl-raw-options=backend=youtube-dl mpv https://...`. 62 | 63 | ## 0.6.0 64 | 65 | ### Breaking Changes 66 | 67 | * Dropped support for Python 3.6, 3.7. 68 | * **(CLI)** The `backend install` `NAME` argument is now required. `youtube-dl` is no longer the default backend to install. 69 | * **(CLI)** The `backend info` `--backend` option was replaced by a positional argument (`backend info --backend=yt-dlp` → `backend info yt-dlp`). 70 | 71 | ### Features 72 | 73 | * Added support for Python 3.10, 3.11. 74 | * Backend autodetection. If the `backend` option is set to `:autodetect:`, `dl-plus` tries to initialize any known backend in the following order: `yt-dlp`, `youtube-dl`, `youtube-dlc`. 75 | * **(Config)** The default value of the `backend` option was changed from `youtube-dl` to `:autodetect:`. 76 | * **(CLI)** New commands: `backend update` and `extractor update`. 77 | * **(Extractor API)** Added an implementation of `Extractor._match_valid_url()` (introduced in `yt-dlp`) for backends that do not yet support it. 78 | 79 | ### Changes 80 | 81 | * **(CLI)** The `backend install` and `extractor install` commands no longer update backends/extractors if they are already installed. `backend update`/`extractor update` should be used in that case. 82 | * **(Config)** The default order of extractors was changed to `:plugins:`, `:builtins:`, `generic`. It was not possible to override any built–in extractor with an extractor provided by a plugin when `:builtins:` had a higher priority than `:plugins:`. 83 | 84 | ### Deprecations 85 | 86 | * **(Extractor API)** `Extractor.dlp_match()` is deprecated, `Extractor._match_valid_url()` should be used instead. 87 | 88 | ### Fixes 89 | 90 | * Added a workaround for lazy extractors. 91 | * **(Extractor API)** `Extractor.dlp_match()` is now compatible with `yt-dlp`. 92 | 93 | ### Improvements 94 | 95 | * **(CLI)** Improved compat mode detection. All known backends are checked now, not only `youtube-dl`. 96 | 97 | ## 0.5.0 98 | 99 | ### Features 100 | 101 | * Extractor plugins management. It is now possible to install extractor plugins using `dl-plus` itself. Plugins are installed into the `dl-plus` config directory. The format of the command is as follows: `dl-plus --cmd extractor install NAME [VERSION]`. 102 | * Configuration via environment variables: 103 | - `DL_PLUS_HOME` — the directory where the default config (`config.ini`), backends and extractors are stored. The default value is `$XDG_CONFIG_HOME/dl-plus`/`%APPDATA%\dl-plus`. 104 | - `DL_PLUS_CONFIG` — the path of the config file. The default value is `$DL_PLUS_HOME/config.ini`. 105 | 106 | ## 0.4.0 107 | 108 | ### Features 109 | 110 | * Backend management commands. It is now possible to install backends using `dl-plus` itself. Backends are installed into the `dl-plus` config directory, therefore, they are not visible to/do not interfere with other packages installed into the same Python environment. 111 | - `dl-plus --cmd backend install [NAME [VERSION]]` installs the backend package into the `dl-plus` config directory. 112 | - `dl-plus --cmd backend info` prints information about the configured backend. 113 | * A new optional `[backend-options]` config section. The section has exactly the same format as the `youtube-dl` config (one can think of it as the `youtube-dl` config embedded into the `dl-plus` one). If this section is present, even if it is empty, it overrides `youtube-dl` own config(s) (`--ignore-config` is used internally). 114 | * `youtube-dl`–compatible mode. When `dl-plus` is run as `youtube-dl` (e.g., via a symlink), it disables all additional command line options. The config file is still processed. 115 | 116 | ### Improvements 117 | 118 | * `-U`/`--update` is now handled by `dl-plus` itself rather than passing it to the underlying backend. For now, it says that this feature is not yet implemented (and it is not). 119 | 120 | ## 0.3.0 121 | 122 | ### Features 123 | 124 | #### Extractor API 125 | 126 | * Two new attributes were added: `DLP_BASE_URL` and `DLP_REL_URL`. If set, they are used to compute the `_VALID_URL` value. 127 | * A new `dlp_match` method was added. It returns a `re.Match` object for a given URL. 128 | 129 | ### Improvements 130 | 131 | * Remove duplicates when expanding the extractor list from the config (`[extractors.enable]`) or the command line arguments (`--extractor`). For example, `-E twitch:vod -E twitch` will no longer include the `twitch:vod` extractor twice. 132 | 133 | ### Internal Changes 134 | 135 | * `dl_plus.extractor.Extractor` and `dl_plus.extractor.ExtractorError` are now loaded lazily. It is now possible to import modules from the `dl_plus.extractor` package without first initializing `dl_plus.ytdl`. 136 | 137 | ## 0.2.0 138 | 139 | ### Features 140 | 141 | * Configurable `youtube-dl`–compatible backends. `youtube-dl` is not a hardcoded dependency anymore, `dl-plus` can now work with any _compatible_ package (that is, any _compatible_ fork), even if its import path is different). As a result, `dl-plus` no longer installs `youtube-dl` automatically. The backend can be configured using the `--backend` command line option or the `backend` option in the `[main]` section on the config file (see below). The value is an import path of the backend (e.g., `youtube_dl` for `youtube-dl`). 142 | * Configuration file support. The config provides used-defined default values for the command line options (that is, the config options have lower predecence than the command line ones). The default config location is `$XDG_CONFIG_HOME/dl-plus/config.ini` (*nix) or `%APPDATA%\dl-plus\config.ini` (Windows). 143 | 144 | ### Changes 145 | 146 | * The generic extractor is no longer included in the `:builtins:` list. That is, one should use `-E :builtins: -E generic` to get **all** built-in extractors. 147 | * Built-in extactors are now grouped by their names splitted by colons. For example, `-E twitch` is now expanded to a list of all `twitch:*` extractors (previously, one should specify each `twitch:*` extractor manually: `-E twitch:stream -E twitch:vod -E ...`). 148 | 149 | ### Fixes 150 | 151 | * `youtube-dl` config processing was restored. The config was simply ignored in the previous version. 152 | 153 | ### Improvements 154 | 155 | * The `--extractor` and `--force-generic-extractor` command-line options are mutually exclusive now. 156 | * Added the `--dlp-version` command-line option to check `dl-plus` version (the `--version` option shows the version of the backend). 157 | 158 | ## 0.1.0 159 | 160 | The first public release. 161 | -------------------------------------------------------------------------------- /src/dl_plus/cli/commands/base.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import shutil 4 | import sys 5 | from argparse import Namespace 6 | from collections.abc import Iterable 7 | from functools import cached_property 8 | from pathlib import Path 9 | from textwrap import dedent 10 | from typing import ( 11 | TYPE_CHECKING, ClassVar, Dict, List, NoReturn, Optional, Sequence, Tuple, 12 | Type, Union, 13 | ) 14 | 15 | from dl_plus.config import Config, ConfigError, get_config_path 16 | from dl_plus.exceptions import DLPlusException 17 | from dl_plus.pypi import PyPIClient, Wheel, WheelInstaller, load_metadata 18 | 19 | 20 | if TYPE_CHECKING: 21 | from dl_plus.cli.args import Arg, ArgGroup, ExclusiveArgGroup 22 | from dl_plus.pypi import Metadata 23 | 24 | 25 | CommandOrGroup = Union['Command', 'CommandGroup'] 26 | 27 | 28 | class _CommandBase: 29 | name: ClassVar[str] 30 | short_description: ClassVar[Optional[str]] = None 31 | long_description: ClassVar[Optional[str]] = None 32 | arguments: ClassVar[ 33 | Tuple[Union[Arg, ArgGroup, ExclusiveArgGroup], ...]] = () 34 | 35 | parent: ClassVar[Optional[Type[CommandGroup]]] = None 36 | 37 | _parents: ClassVar[Tuple[Type[CommandGroup], ...]] 38 | 39 | def __init_subclass__(cls): 40 | dct = cls.__dict__ 41 | if 'name' not in dct: 42 | setattr(cls, 'name', dct['__module__'].rpartition('.')[-1]) 43 | long_description = dct.get('long_description') 44 | if long_description: 45 | long_description = dedent(long_description).rstrip() + '\n' 46 | setattr(cls, 'long_description', long_description) 47 | 48 | @classmethod 49 | def get_parents( 50 | cls, *, include_root: bool = True, 51 | ) -> Tuple[Type[CommandGroup], ...]: 52 | if '_parents' not in cls.__dict__: 53 | parents = [] 54 | parent = cls.parent 55 | assert parent 56 | while parent: 57 | parents.append(parent) 58 | parent = parent.parent 59 | cls._parents = tuple(reversed(parents)) 60 | if include_root: 61 | return cls._parents 62 | return cls._parents[1:] 63 | 64 | def get_command_path(self, *, include_command: bool = True) -> List[str]: 65 | path = [parent.name for parent in self.get_parents(include_root=False)] 66 | if include_command: 67 | path.append(self.name) 68 | return path 69 | 70 | 71 | class Command(_CommandBase): 72 | args: Namespace 73 | 74 | def __init__(self, args: Namespace) -> None: 75 | self.args = args 76 | self.init() 77 | 78 | def init(self) -> None: 79 | pass 80 | 81 | def run(self) -> None: 82 | raise NotImplementedError 83 | 84 | @cached_property 85 | def config_path(self) -> Optional[Path]: 86 | try: 87 | return get_config_path(getattr(self.args, 'dlp_config', None)) 88 | except ConfigError: 89 | return None 90 | 91 | @cached_property 92 | def config(self) -> Config: 93 | config = Config() 94 | config_file: Union[Path, bool, None] 95 | if getattr(self.args, 'no_dlp_config', False): 96 | config_file = False 97 | else: 98 | config_file = self.config_path 99 | config.load(config_file) 100 | return config 101 | 102 | def die(self, message: str) -> NoReturn: 103 | raise CommandError(message) 104 | 105 | print = print 106 | 107 | def confirm(self, message: str) -> bool: 108 | try: 109 | assume_yes = self.args.assume_yes 110 | except AttributeError: 111 | assume_yes = None 112 | if assume_yes: 113 | return True 114 | prompt = f'{message} [Y/n]' 115 | if sys.stdin.isatty(): 116 | return input(f'{prompt} ').lower() in ['y', 'yes'] 117 | self.print(prompt) 118 | if assume_yes is None: 119 | self.print('Non-interactive mode, assuming no') 120 | return False 121 | self.die( 122 | 'non-interactive mode, use `--assume-yes` for automatic ' 123 | 'confirmation' 124 | ) 125 | 126 | 127 | class CommandGroup(_CommandBase): 128 | commands: ClassVar[Tuple[Type[CommandOrGroup], ...]] = () 129 | 130 | _commands: ClassVar[Dict[str, Type[CommandOrGroup]]] 131 | 132 | def __init_subclass__(cls): 133 | super().__init_subclass__() 134 | _commands: Dict[str, Type[CommandOrGroup]] = {} 135 | for command_or_group in cls.commands: 136 | assert not command_or_group.parent 137 | command_or_group.parent = cls 138 | _commands[command_or_group.name] = command_or_group 139 | cls._commands = _commands 140 | 141 | @classmethod 142 | def get_command(cls, command_path: Sequence[str]) -> Type[Command]: 143 | path_len = len(command_path) 144 | assert path_len 145 | group = cls 146 | for index, name in enumerate(command_path, 1): 147 | command_or_group = group._commands[name] 148 | if index == path_len: 149 | assert issubclass(command_or_group, Command) 150 | return command_or_group 151 | assert issubclass(command_or_group, CommandGroup) 152 | group = command_or_group 153 | raise AssertionError('the last path item must be a Command') 154 | 155 | 156 | class CommandError(DLPlusException): 157 | 158 | pass 159 | 160 | 161 | class BaseInstallUpdateCommand(Command): 162 | client: PyPIClient 163 | wheel_installer: WheelInstaller 164 | 165 | def get_package_dir(self) -> Path: 166 | raise NotImplementedError 167 | 168 | def get_short_name(self) -> str: 169 | """Return short name used for logging, command examples, etc.""" 170 | raise NotImplementedError 171 | 172 | def get_extras(self) -> Optional[Iterable[str]]: 173 | return None 174 | 175 | def init(self) -> None: 176 | self.client = PyPIClient() 177 | self.wheel_installer = WheelInstaller() 178 | 179 | def load_installed_metadata(self, package_dir: Path) -> Optional[Metadata]: 180 | if not package_dir.exists(): 181 | return None 182 | return load_metadata(package_dir) 183 | 184 | 185 | class BaseInstallCommand(BaseInstallUpdateCommand): 186 | 187 | def get_project_name_version_tuple(self) -> Tuple[str, Optional[str]]: 188 | raise NotImplementedError 189 | 190 | def get_force_flag(self) -> bool: 191 | return False 192 | 193 | def run(self): 194 | name, version = self.get_project_name_version_tuple() 195 | wheel = self.client.fetch_wheel_info(name, version) 196 | self.print(f'Found remote version: {wheel.name} {wheel.version}') 197 | 198 | package_dir = self.get_package_dir() 199 | installed_metadata = self.load_installed_metadata(package_dir) 200 | if not installed_metadata: 201 | self.install(wheel, package_dir) 202 | return 203 | 204 | self.print( 205 | f'Found installed version: {installed_metadata.name} ' 206 | f'{installed_metadata.version}' 207 | ) 208 | is_latest_installed = installed_metadata.version == wheel.version 209 | 210 | if not version: 211 | short_name = self.get_short_name() 212 | command_prefix = ' '.join( 213 | self.get_command_path(include_command=False)) 214 | self.print(f'{short_name} is already installed') 215 | if not is_latest_installed: 216 | self.print( 217 | f'Use `{command_prefix} update {short_name}` ' 218 | f'to update to the latest version' 219 | ) 220 | self.print( 221 | f'Use `{command_prefix} install {short_name} VERSION` ' 222 | f'to install a specific version' 223 | ) 224 | return 225 | 226 | if is_latest_installed: 227 | self.print('The same version is already installed') 228 | if not self.get_force_flag(): 229 | self.print('Use `--force` to reinstall') 230 | return 231 | self.print('Forcing installation') 232 | 233 | self.install(wheel, package_dir) 234 | 235 | def install(self, wheel: Wheel, package_dir: Path) -> None: 236 | self.print('Installing') 237 | self.print(f'Using {self.wheel_installer.identifier} installer') 238 | self.wheel_installer.install(wheel, package_dir, self.get_extras()) 239 | self.print('Installed') 240 | 241 | 242 | class BaseUpdateCommand(BaseInstallUpdateCommand): 243 | 244 | def get_project_name(self) -> str: 245 | raise NotImplementedError 246 | 247 | def run(self): 248 | name = self.get_project_name() 249 | wheel = self.client.fetch_wheel_info(name) 250 | self.print(f'Found remote version: {wheel.name} {wheel.version}') 251 | 252 | package_dir = self.get_package_dir() 253 | installed_metadata = self.load_installed_metadata(package_dir) 254 | if not installed_metadata: 255 | command_prefix = ' '.join( 256 | self.get_command_path(include_command=False)) 257 | short_name = self.get_short_name() 258 | self.die( 259 | f'{short_name} is not installed, use ' 260 | f'`{command_prefix} install {short_name} [VERSION]` to install' 261 | ) 262 | 263 | self.print( 264 | f'Found installed version: {installed_metadata.name} ' 265 | f'{installed_metadata.version}' 266 | ) 267 | if installed_metadata.version == wheel.version: 268 | self.print('The latest version is already installed') 269 | return 270 | 271 | self.update(wheel, package_dir) 272 | 273 | def update(self, wheel: Wheel, package_dir: Path) -> None: 274 | self.print('Updating') 275 | self.print(f'Using {self.wheel_installer.identifier} installer') 276 | self.wheel_installer.install(wheel, package_dir, self.get_extras()) 277 | self.print('Updated') 278 | 279 | 280 | class BaseUninstallCommand(Command): 281 | 282 | def get_package_dir(self) -> Path: 283 | raise NotImplementedError 284 | 285 | def get_short_name(self) -> str: 286 | """Return short name used for logging, command examples, etc.""" 287 | raise NotImplementedError 288 | 289 | def run(self): 290 | package_dir = self.get_package_dir() 291 | short_name = self.get_short_name() 292 | if not package_dir.exists(): 293 | self.die(f'{short_name} is not installed') 294 | if self.confirm(f'Uninstall {short_name}?'): 295 | self.uninstall(package_dir) 296 | else: 297 | self.print('Aborted') 298 | 299 | def uninstall(self, package_dir: Path) -> None: 300 | shutil.rmtree(package_dir) 301 | self.print('Unistalled') 302 | --------------------------------------------------------------------------------