├── ctfdl ├── __init__.py ├── cli │ ├── __init__.py │ ├── helpers.py │ └── main.py ├── common │ ├── console.py │ ├── version.py │ ├── __init__.py │ ├── archiver.py │ ├── logging.py │ ├── format_output.py │ └── updates.py ├── core │ ├── __init__.py │ ├── models.py │ ├── config.py │ └── events.py ├── resources │ └── templates │ │ ├── challenge │ │ ├── _components │ │ │ ├── json.jinja │ │ │ └── readme.jinja │ │ └── variants │ │ │ ├── json.yaml │ │ │ ├── default.yaml │ │ │ └── minimal.yaml │ │ ├── folder_structure │ │ ├── flat.jinja │ │ └── default.jinja │ │ └── index │ │ ├── json.jinja │ │ └── grouped.jinja ├── challenges │ ├── client.py │ ├── entry.py │ └── downloader.py ├── rendering │ ├── metadata_loader.py │ ├── context.py │ ├── variant_loader.py │ ├── renderers.py │ ├── engine.py │ └── inspector.py └── ui │ ├── messages.py │ └── rich_handler.py ├── docs ├── requirements.txt ├── index.md ├── usage.md └── templates.md ├── .github ├── dependabot.yml └── workflows │ ├── release.yml │ └── ci.yaml ├── .readthedocs.yml ├── tests └── test_cli.py ├── .pre-commit-config.yaml ├── mkdocs.yml ├── LICENSE ├── pyproject.toml ├── README.md └── .gitignore /ctfdl/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ctfdl/cli/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | mkdocs 2 | mkdocs-material 3 | -------------------------------------------------------------------------------- /ctfdl/common/console.py: -------------------------------------------------------------------------------- 1 | from rich.console import Console 2 | 3 | console = Console() 4 | -------------------------------------------------------------------------------- /ctfdl/core/__init__.py: -------------------------------------------------------------------------------- 1 | from .config import ExportConfig 2 | from .events import EventEmitter 3 | 4 | __all__ = ["EventEmitter", "ExportConfig"] 5 | -------------------------------------------------------------------------------- /ctfdl/resources/templates/challenge/_components/json.jinja: -------------------------------------------------------------------------------- 1 | {# 2 | description: Challenge data. 3 | prettify: true 4 | #} 5 | {{ challenge | tojson }} -------------------------------------------------------------------------------- /ctfdl/resources/templates/folder_structure/flat.jinja: -------------------------------------------------------------------------------- 1 | {# description: Organizes challenges into a flat directory structured #} 2 | {{ challenge.name | slugify }} -------------------------------------------------------------------------------- /ctfdl/resources/templates/challenge/variants/json.yaml: -------------------------------------------------------------------------------- 1 | name: json 2 | description: Outputs challenge data as JSON files. 3 | components: 4 | - file: challenge.json 5 | template: json.jinja 6 | -------------------------------------------------------------------------------- /ctfdl/resources/templates/challenge/variants/default.yaml: -------------------------------------------------------------------------------- 1 | name: default 2 | description: Challenge template including README 3 | components: 4 | - file: README.md 5 | template: readme.jinja 6 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "pip" 4 | directory: "/" 5 | schedule: 6 | interval: "weekly" 7 | versioning-strategy: "increase-if-necessary" 8 | -------------------------------------------------------------------------------- /ctfdl/resources/templates/challenge/variants/minimal.yaml: -------------------------------------------------------------------------------- 1 | name: minimal 2 | description: Simple template with only a README file for quick overview. 3 | components: 4 | - file: README.md 5 | template: readme.jinja 6 | -------------------------------------------------------------------------------- /ctfdl/resources/templates/index/json.jinja: -------------------------------------------------------------------------------- 1 | {# 2 | output_file: index.json 3 | description: JSON index of challenges 4 | prettify: true 5 | #} 6 | { 7 | "challenges": {{ challenges | map(attribute="data") | list | tojson }} 8 | } -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | build: 4 | os: ubuntu-24.04 5 | tools: 6 | python: "3.10" 7 | 8 | python: 9 | install: 10 | - requirements: docs/requirements.txt 11 | 12 | mkdocs: 13 | configuration: mkdocs.yml 14 | -------------------------------------------------------------------------------- /ctfdl/resources/templates/folder_structure/default.jinja: -------------------------------------------------------------------------------- 1 | {# description: Organizes by category and challenge name #} 2 | {{ challenge.category | slugify }}/{% if challenge.subcategory %}{{ challenge.subcategory | slugify }}/{% endif %}{{ challenge.name | slugify }} -------------------------------------------------------------------------------- /tests/test_cli.py: -------------------------------------------------------------------------------- 1 | from typer.testing import CliRunner 2 | 3 | from ctfdl.cli.main import app 4 | 5 | runner = CliRunner() 6 | 7 | 8 | def test_help_command_runs(): 9 | result = runner.invoke(app, ["--help"]) 10 | assert result.exit_code == 0 11 | assert "Usage:" in result.output 12 | -------------------------------------------------------------------------------- /ctfdl/common/version.py: -------------------------------------------------------------------------------- 1 | from importlib.metadata import version 2 | 3 | 4 | def show_version(): 5 | from rich.console import Console 6 | 7 | try: 8 | __version__ = version("ctf-dl") 9 | except Exception: 10 | __version__ = "dev" 11 | 12 | Console().print(f"📦 [bold]ctf-dl[/bold] version: [green]{__version__}[/green]") 13 | -------------------------------------------------------------------------------- /ctfdl/common/__init__.py: -------------------------------------------------------------------------------- 1 | from .archiver import zip_output_folder 2 | from .format_output import format_output 3 | from .logging import setup_logging_with_rich 4 | from .updates import check_updates 5 | from .version import show_version 6 | 7 | __all__ = [ 8 | "zip_output_folder", 9 | "format_output", 10 | "setup_logging_with_rich", 11 | "check_updates", 12 | "show_version", 13 | ] 14 | -------------------------------------------------------------------------------- /ctfdl/challenges/client.py: -------------------------------------------------------------------------------- 1 | from ctfbridge import create_client 2 | 3 | 4 | async def get_authenticated_client(url: str, username=None, password=None, token=None): 5 | client = await create_client(url) 6 | 7 | if username and password: 8 | await client.auth.login(username=username, password=password) 9 | elif token: 10 | await client.auth.login(token=token) 11 | else: 12 | pass 13 | 14 | return client 15 | -------------------------------------------------------------------------------- /ctfdl/core/models.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | from ctfbridge.models.challenge import Challenge 4 | from pydantic import BaseModel, Field 5 | 6 | 7 | class ChallengeEntry(BaseModel): 8 | data: Challenge = Field(..., description="The CTFBridge Challenge object") 9 | path: Path = Field(..., description="Path to the challenge's directory") 10 | updated: bool = Field(default=False, description="If the challenge was updated instead of new") 11 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/astral-sh/ruff-pre-commit 3 | rev: v0.12.12 4 | hooks: 5 | - id: ruff-check 6 | args: [--fix] 7 | - id: ruff-format 8 | - repo: https://github.com/asottile/pyupgrade 9 | rev: v3.20.0 10 | hooks: 11 | - id: pyupgrade 12 | args: ["--py312-plus"] 13 | - repo: https://github.com/codespell-project/codespell 14 | rev: v2.4.1 15 | hooks: 16 | - id: codespell 17 | -------------------------------------------------------------------------------- /ctfdl/common/archiver.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | from pathlib import Path 3 | 4 | from rich.console import Console 5 | 6 | console = Console() 7 | 8 | 9 | def zip_output_folder(output_dir: Path, archive_name="ctf-export"): 10 | parent_dir = output_dir.parent 11 | archive_path = shutil.make_archive( 12 | archive_name, "zip", root_dir=parent_dir, base_dir=output_dir.name 13 | ) 14 | console.print(f"🗂️ [green]Output saved to:[/] [bold underline]{archive_path}[/]") 15 | shutil.rmtree(parent_dir) 16 | -------------------------------------------------------------------------------- /ctfdl/common/logging.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from ctfdl.common.console import console 4 | 5 | 6 | def setup_logging_with_rich(debug: bool = False): 7 | from rich.logging import RichHandler 8 | 9 | level = logging.DEBUG if debug else logging.ERROR 10 | 11 | logging.basicConfig( 12 | level=level, 13 | format="%(message)s", 14 | datefmt="[%X]", 15 | handlers=[RichHandler(rich_tracebacks=True, markup=True, console=console)], 16 | ) 17 | logging.getLogger("ctfdl").setLevel(level) 18 | -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | # Welcome to ctf-dl Documentation 2 | 3 | **ctf-dl** is a flexible, fast, and scriptable tool for downloading CTF challenges from multiple platforms. 4 | 5 | This documentation covers: 6 | 7 | * Installation and setup 8 | * Full CLI reference 9 | * Output customization via templates 10 | * Developer and contribution guides 11 | 12 | --- 13 | 14 | ## 📦 Installation 15 | 16 | ```bash 17 | pip install ctf-dl 18 | ``` 19 | 20 | For development: 21 | 22 | ```bash 23 | git clone https://github.com/bjornmorten/ctf-dl.git 24 | cd ctf-dl 25 | pip install -e . 26 | ``` 27 | 28 | -------------------------------------------------------------------------------- /ctfdl/common/format_output.py: -------------------------------------------------------------------------------- 1 | import json 2 | from pathlib import Path 3 | 4 | import mdformat 5 | 6 | 7 | def format_output(text: str, output_file: str | Path, prettify: bool = False) -> str: 8 | if not prettify: 9 | return text 10 | 11 | ext = Path(output_file).suffix.lower() 12 | 13 | if ext == ".md": 14 | return mdformat.text(text, extensions={"tables"}) 15 | elif ext == ".json": 16 | try: 17 | obj = json.loads(text) 18 | return json.dumps(obj, indent=2, ensure_ascii=False) 19 | except Exception: 20 | return text 21 | 22 | return text 23 | -------------------------------------------------------------------------------- /ctfdl/rendering/metadata_loader.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import re 3 | from pathlib import Path 4 | 5 | import yaml 6 | 7 | logger = logging.getLogger(__name__) 8 | 9 | 10 | def parse_template_metadata(template_path: Path) -> dict: 11 | metadata = {} 12 | try: 13 | content = template_path.read_text(encoding="utf-8") 14 | 15 | # Match a Jinja comment block at the very start 16 | m = re.match(r"^\s*\{#(.*?)#\}", content, re.DOTALL) 17 | if m: 18 | comment_text = m.group(1).strip() 19 | metadata = yaml.safe_load(comment_text) or {} 20 | except Exception as e: 21 | logger.exception(f"Failed to parse metadata from {template_path}: {e}") 22 | 23 | return metadata 24 | -------------------------------------------------------------------------------- /ctfdl/rendering/context.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | from ctfdl.rendering.engine import TemplateEngine 4 | 5 | 6 | class TemplateEngineContext: 7 | _instance: TemplateEngine | None = None 8 | 9 | @classmethod 10 | def initialize(cls, user_template_dir: Path | None, builtin_template_dir: Path): 11 | if cls._instance is None: 12 | cls._instance = TemplateEngine(user_template_dir, builtin_template_dir) 13 | 14 | @classmethod 15 | def get(cls) -> TemplateEngine: 16 | if cls._instance is None: 17 | raise RuntimeError("TemplateEngineContext not initialized.") 18 | return cls._instance 19 | 20 | @classmethod 21 | def reset(cls): 22 | cls._instance = None 23 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Publish to PyPI 2 | 3 | on: 4 | push: 5 | tags: 6 | - "v*.*.*" 7 | 8 | jobs: 9 | publish: 10 | runs-on: ubuntu-latest 11 | 12 | steps: 13 | - uses: actions/checkout@v4 14 | 15 | - name: Set up Python 16 | uses: actions/setup-python@v5 17 | with: 18 | python-version: "3.11" 19 | 20 | - name: Install build tools 21 | run: | 22 | pip install build twine 23 | 24 | - name: Build the package 25 | run: python -m build 26 | 27 | - name: Publish to PyPI 28 | env: 29 | TWINE_USERNAME: __token__ 30 | TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} 31 | run: twine upload dist/* 32 | 33 | -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: ctf-dl 2 | repo_url: https://github.com/bjornmorten/ctf-dl 3 | repo_name: ctf-dl 4 | copyright: Copyright © 2025 bjornmorten 5 | 6 | theme: 7 | name: material 8 | features: 9 | - navigation.instant 10 | - navigation.sections 11 | - navigation.top 12 | palette: 13 | scheme: slate 14 | primary: blue 15 | accent: blue 16 | 17 | extra: 18 | generator: false 19 | social: 20 | - icon: fontawesome/brands/github 21 | link: https://github.com/bjornmorten/ctf-dl/ 22 | - icon: fontawesome/brands/python 23 | link: https://pypi.org/project/ctf-dl/ 24 | 25 | nav: 26 | - Home: index.md 27 | - Usage Guide: usage.md 28 | - Templates: templates.md 29 | 30 | plugins: 31 | - search 32 | -------------------------------------------------------------------------------- /ctfdl/resources/templates/index/grouped.jinja: -------------------------------------------------------------------------------- 1 | {# 2 | output_file: README.md 3 | description: Markdown file with challenges grouped by categories 4 | prettify: true 5 | #} 6 | # Challenge Index 7 | 8 | {% set grouped = {} %} 9 | {% for c in challenges %}{% set _ = grouped.setdefault(c.data.category, []).append(c) %}{% endfor %} 10 | 11 | {% set show_points = challenges | selectattr("data.value", "ne", None) | list | length > 0 %} 12 | 13 | {% for category, items in grouped.items() %} 14 | ## {{ category }} 15 | 16 | | Name {% if show_points %}| Points {% endif %}| Path | 17 | |------{% if show_points %}|--------{% endif %}|------| 18 | {% for c in items %} 19 | | {{ c.data.name }} {% if show_points %}| {{ c.data.value if c.data.value is not none else "" }} {% endif %}| [Link]({{ c.path }}) | 20 | {% endfor %} 21 | 22 | {% endfor %} -------------------------------------------------------------------------------- /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | name: CI 2 | on: 3 | push: 4 | pull_request: 5 | 6 | permissions: 7 | contents: read 8 | 9 | jobs: 10 | lint: 11 | name: Lint 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v4 15 | - uses: actions/setup-python@v5 16 | with: 17 | python-version: '3.12' 18 | - run: pip install ruff 19 | - run: ruff check . 20 | 21 | test: 22 | name: Tests 23 | runs-on: ubuntu-latest 24 | needs: lint 25 | strategy: 26 | matrix: 27 | python-version: ["3.10", "3.12"] 28 | 29 | steps: 30 | - uses: actions/checkout@v4 31 | - uses: actions/setup-python@v5 32 | with: 33 | python-version: ${{ matrix.python-version }} 34 | 35 | - name: Install dependencies 36 | run: pip install -e ".[dev]" 37 | 38 | - name: Run tests 39 | run: pytest 40 | 41 | -------------------------------------------------------------------------------- /ctfdl/resources/templates/challenge/_components/readme.jinja: -------------------------------------------------------------------------------- 1 | {# 2 | description: Challenge overview. 3 | prettify: true 4 | #} 5 | # {{ challenge.name }} 6 | 7 | **Category:** {{ challenge.category }} 8 | 9 | {% if challenge.value is not none %} 10 | **Points:** {{ challenge.value }} 11 | {% endif %} 12 | 13 | ## Description 14 | 15 | {% if challenge.description %} 16 | {{ challenge.description }} 17 | {% else %} 18 | _No description_ 19 | {% endif %} 20 | 21 | {% if challenge.services %} 22 | ## Services 23 | 24 | {% for service in challenge.services %} 25 | - {% if service.url %} 26 | [{{ service.url }}]({{ service.url }}) 27 | {% elif service.raw %} 28 | `{{ service.raw }}` 29 | {% elif service.host and service.port %} 30 | `nc {{ service.host }} {{ service.port }}` 31 | {% else %} 32 | (service details not available) 33 | {% endif %} 34 | {% endfor %} 35 | 36 | {% endif %} 37 | 38 | {% if challenge.attachments %} 39 | ## Attachments 40 | 41 | {% for attachment in challenge.attachments %} 42 | - [{{ attachment.name }}](files/{{ attachment.name }}) 43 | {% endfor %} 44 | 45 | {% endif %} -------------------------------------------------------------------------------- /ctfdl/core/config.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | from pydantic import BaseModel, Field 4 | 5 | 6 | class ExportConfig(BaseModel): 7 | url: str = Field(..., description="Base URL of the CTF platform") 8 | output: Path = Field(default=Path("challenges"), description="Output folder") 9 | 10 | token: str | None = None 11 | username: str | None = None 12 | password: str | None = None 13 | cookie: Path | None = None 14 | 15 | # Templating 16 | template_dir: Path | None = None 17 | variant_name: str = "default" 18 | folder_template_name: str = "default" 19 | index_template_name: str | None = "grouped" 20 | no_index: bool = False 21 | 22 | # Filters 23 | categories: list[str] | None = None 24 | min_points: int | None = None 25 | max_points: int | None = None 26 | solved: bool = False 27 | unsolved: bool = False 28 | # status... 29 | 30 | # Behavior 31 | update: bool = False 32 | no_attachments: bool = False 33 | parallel: int = 30 34 | list_templates: bool = False 35 | zip_output: bool = False 36 | debug: bool = False 37 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2025 bjornmorten 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /ctfdl/rendering/variant_loader.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from typing import Any 3 | 4 | import yaml 5 | 6 | 7 | class VariantLoader: 8 | def __init__(self, user_template_dir: Path | None, builtin_template_dir: Path): 9 | self.user_dir = user_template_dir 10 | self.builtin_dir = builtin_template_dir 11 | 12 | def resolve_variant(self, name: str) -> dict[str, Any]: 13 | variant_path = self._variant_path(name) 14 | if not variant_path.exists(): 15 | raise FileNotFoundError(f"Variant '{name}' not found at {variant_path}") 16 | 17 | variant = self._load_yaml(variant_path) 18 | 19 | if "extends" in variant: 20 | base = self.resolve_variant(variant["extends"]) 21 | merged = { 22 | "name": variant.get("name", base.get("name", name)), 23 | "components": variant.get("components", base.get("components", [])), 24 | } 25 | return merged 26 | 27 | return variant 28 | 29 | def _load_yaml(self, path: Path) -> dict[str, Any]: 30 | return yaml.safe_load(path.read_text(encoding="utf-8")) 31 | 32 | def _variant_path(self, name: str) -> Path: 33 | root = ( 34 | self.user_dir 35 | if self.user_dir and (self.user_dir / "challenge/variants").exists() 36 | else self.builtin_dir 37 | ) 38 | return root / "challenge/variants" / f"{name}.yaml" 39 | -------------------------------------------------------------------------------- /docs/usage.md: -------------------------------------------------------------------------------- 1 | # 📦 Usage Examples 2 | 3 | Here are common real-world examples of using `ctf-dl` to fetch and organize CTF challenges from supported platforms. 4 | 5 | --- 6 | 7 | ## 🔓 Basic Usage (Token) 8 | 9 | ```bash 10 | ctf-dl https://demo.ctfd.io --token ABC123 11 | ``` 12 | 13 | --- 14 | 15 | ## 🗂 Custom Output Folder 16 | 17 | ```bash 18 | ctf-dl https://demo.ctfd.io --token ABC123 --output ~/ctf-2025 19 | ``` 20 | 21 | --- 22 | 23 | ## 🎯 Filter by Category 24 | 25 | ```bash 26 | ctf-dl https://demo.ctfd.io --token ABC123 --categories Web Crypto 27 | ``` 28 | 29 | --- 30 | 31 | ## 📉 Filter by Points 32 | 33 | ```bash 34 | ctf-dl https://demo.ctfd.io --token ABC123 --min-points 100 --max-points 300 35 | ``` 36 | 37 | --- 38 | 39 | ## ✅ Only Solved Challenges 40 | 41 | ```bash 42 | ctf-dl https://demo.ctfd.io --token ABC123 --solved 43 | ``` 44 | 45 | --- 46 | 47 | ## 🚫 Skip Attachments 48 | 49 | ```bash 50 | ctf-dl https://demo.ctfd.io --token ABC123 --no-attachments 51 | ``` 52 | 53 | --- 54 | 55 | ## 🔁 Update Mode (Skip Existing) 56 | 57 | ```bash 58 | ctf-dl https://demo.ctfd.io --token ABC123 --update 59 | ``` 60 | 61 | --- 62 | 63 | ## 🗜 Zip Output After Download 64 | 65 | ```bash 66 | ctf-dl https://demo.ctfd.io --token ABC123 --zip 67 | ``` 68 | 69 | --- 70 | 71 | ## 🧩 Use a Custom Template 72 | 73 | ```bash 74 | ctf-dl https://demo.ctfd.io --token ABC123 --template json 75 | ``` 76 | 77 | --- 78 | 79 | ## 🔍 List All Available Templates 80 | 81 | ```bash 82 | ctf-dl --list-templates 83 | ``` 84 | 85 | -------------------------------------------------------------------------------- /ctfdl/core/events.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import logging 3 | from collections import defaultdict 4 | from collections.abc import Callable 5 | from typing import Any 6 | 7 | logger = logging.getLogger(__name__) 8 | 9 | 10 | class EventEmitter: 11 | """A simple event emitter class for decoupling components.""" 12 | 13 | def __init__(self): 14 | """Initializes the EventEmitter with a dictionary to hold listeners.""" 15 | self._listeners: defaultdict[str, list[Callable]] = defaultdict(list) 16 | 17 | def on(self, event_name: str, listener: Callable[..., Any]): 18 | """ 19 | Registers a listener function to be called when an event is emitted. 20 | 21 | Args: 22 | event_name: The name of the event to listen for. 23 | listener: The function to be called when the event occurs. 24 | """ 25 | self._listeners[event_name].append(listener) 26 | 27 | async def emit(self, event_name: str, *args: Any, **kwargs: Any): 28 | """ 29 | Emits an event, calling all registered listeners for that event. 30 | Awaits listeners that are coroutines. 31 | """ 32 | if event_name not in self._listeners: 33 | return 34 | 35 | for listener in self._listeners[event_name]: 36 | try: 37 | if asyncio.iscoroutinefunction(listener): 38 | await listener(*args, **kwargs) 39 | else: 40 | listener(*args, **kwargs) 41 | except Exception as e: 42 | logger.exception(f"Error in event listener for '{event_name}': {e}") 43 | -------------------------------------------------------------------------------- /ctfdl/common/updates.py: -------------------------------------------------------------------------------- 1 | from importlib.metadata import PackageNotFoundError, version 2 | 3 | import httpx 4 | from rich.console import Console 5 | 6 | console = Console() 7 | 8 | 9 | def check_updates(): 10 | packages_to_check = ["ctf-dl", "ctfbridge"] 11 | outdated = [] 12 | 13 | def get_latest_version(pkg): 14 | try: 15 | resp = httpx.get(f"https://pypi.org/pypi/{pkg}/json", timeout=5) 16 | resp.raise_for_status() 17 | return resp.json()["info"]["version"] 18 | except Exception as e: 19 | console.print(f"⚠️ Failed to fetch version for [yellow]{pkg}[/]: {e}") 20 | return None 21 | 22 | def compare_versions(pkg): 23 | try: 24 | installed = version(pkg) 25 | except PackageNotFoundError: 26 | console.print(f"❌ [red]{pkg}[/] is not installed.") 27 | return 28 | 29 | latest = get_latest_version(pkg) 30 | if not latest: 31 | return 32 | 33 | if installed != latest: 34 | console.print( 35 | f"📦 [yellow]{pkg}[/]: update available → [red]{installed}[/] → [green]{latest}[/]" 36 | ) 37 | outdated.append(pkg) 38 | else: 39 | console.print(f"✅ {pkg} is up to date ([green]{installed}[/])") 40 | 41 | console.print("🔍 Checking for updates...\n") 42 | for pkg in packages_to_check: 43 | compare_versions(pkg) 44 | 45 | if outdated: 46 | upgrade_cmd = "pip install --upgrade " + " ".join(outdated) 47 | console.print(f"\n🚀 To upgrade, run:\n[bold]{upgrade_cmd}[/bold]") 48 | else: 49 | console.print("\n🎉 All packages are up to date.") 50 | -------------------------------------------------------------------------------- /ctfdl/challenges/entry.py: -------------------------------------------------------------------------------- 1 | import tempfile 2 | from pathlib import Path 3 | 4 | from ctfdl.challenges.downloader import download_challenges 5 | from ctfdl.common.archiver import zip_output_folder 6 | from ctfdl.common.logging import setup_logging_with_rich 7 | from ctfdl.core.config import ExportConfig 8 | from ctfdl.core.events import EventEmitter 9 | from ctfdl.rendering.context import TemplateEngineContext 10 | from ctfdl.ui.rich_handler import RichConsoleHandler 11 | 12 | 13 | async def run_export(config: ExportConfig): 14 | setup_logging_with_rich(debug=config.debug) 15 | 16 | TemplateEngineContext.initialize( 17 | config.template_dir, Path(__file__).parent.parent / "resources" / "templates" 18 | ) 19 | 20 | if config.list_templates: 21 | TemplateEngineContext.get().list_templates() 22 | return 23 | 24 | emitter = EventEmitter() 25 | 26 | RichConsoleHandler(emitter) 27 | 28 | temp_dir = Path(tempfile.mkdtemp()) if config.zip_output else None 29 | output_dir = (temp_dir / "ctf-export") if temp_dir else config.output 30 | config.output = output_dir 31 | 32 | output_dir.mkdir(parents=True, exist_ok=True) 33 | 34 | try: 35 | success, index_data = await download_challenges(config, emitter) 36 | except Exception as e: 37 | await emitter.emit("download_fail", str(e)) 38 | raise SystemExit(1) 39 | 40 | if success: 41 | await emitter.emit("download_success") 42 | 43 | if not config.no_index: 44 | TemplateEngineContext.get().render_index( 45 | template_name=config.index_template_name or "grouped", 46 | challenges=index_data, 47 | output_path=output_dir / "index.md", 48 | ) 49 | 50 | if config.zip_output: 51 | zip_output_folder(output_dir, archive_name="ctf-export") 52 | -------------------------------------------------------------------------------- /ctfdl/rendering/renderers.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | from ctfbridge.models.challenge import Challenge as CTFBridgeChallenge 4 | from jinja2 import Environment 5 | 6 | from ctfdl.common.format_output import format_output 7 | from ctfdl.core.models import ChallengeEntry 8 | 9 | 10 | class BaseRenderer: 11 | """Base renderer with shared formatting and file writing logic.""" 12 | 13 | def _apply_formatting_and_write(self, rendered: str, output_path: Path, config: dict): 14 | """Format rendered content and write to disk.""" 15 | rendered = format_output( 16 | rendered, 17 | output_path, 18 | prettify=config.get("prettify", False), 19 | ) 20 | output_path.parent.mkdir(parents=True, exist_ok=True) 21 | output_path.write_text(rendered, encoding="utf-8") 22 | 23 | 24 | class ChallengeRenderer(BaseRenderer): 25 | """Renders individual challenge.""" 26 | 27 | def render(self, template, config: dict, challenge: CTFBridgeChallenge, output_dir: Path): 28 | rendered = template.render(challenge=challenge.model_dump()) 29 | output_path = output_dir / config["output_file"] 30 | self._apply_formatting_and_write(rendered, output_path, config) 31 | 32 | 33 | class FolderRenderer: 34 | """Renders the folder structure path for a challenge.""" 35 | 36 | def __init__(self, env: Environment): 37 | self.env = env 38 | 39 | def render(self, template, challenge: CTFBridgeChallenge) -> str: 40 | return template.render(challenge=challenge.model_dump()) 41 | 42 | 43 | class IndexRenderer(BaseRenderer): 44 | """Renders the global challenge index.""" 45 | 46 | def render(self, template, config: dict, challenges: list[ChallengeEntry], output_path: Path): 47 | rendered = template.render(challenges=[challenge.model_dump() for challenge in challenges]) 48 | 49 | final_path = output_path.parent / config.get("output_file", output_path.name) 50 | 51 | self._apply_formatting_and_write(rendered, final_path, config) 52 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "ctf-dl" 3 | dynamic = ["version"] 4 | description = "Command-line tool to download CTF challenges" 5 | authors = [ 6 | { name="bjornmorten", email="bjornmdev@proton.me" } 7 | ] 8 | license = { text = "MIT" } 9 | readme = "README.md" 10 | requires-python = ">=3.10" 11 | keywords = ["ctf", "ctf-tools"] 12 | classifiers = [ 13 | "Programming Language :: Python :: 3", 14 | "Programming Language :: Python :: 3.10", 15 | "Programming Language :: Python :: 3.11", 16 | "Programming Language :: Python :: 3.12", 17 | "License :: OSI Approved :: MIT License", 18 | "Operating System :: OS Independent" 19 | ] 20 | 21 | dependencies = [ 22 | "ctfbridge>=0.8.3", 23 | "httpx>=0.28.0", 24 | "typer>=0.15.3", 25 | "rich>=14.0.0", 26 | "Jinja2>=3.1.6", 27 | "PyYAML>=6.0.0", 28 | "python-slugify[unidecode]>=8.0.4", 29 | "mdformat==0.7.22", 30 | "mdformat_tables==1.0.0" 31 | ] 32 | 33 | [project.optional-dependencies] 34 | dev = [ 35 | "pytest>=8.0.0", 36 | "pytest-mock>=3.15.0", 37 | "ruff>=0.13.0", 38 | ] 39 | 40 | [project.urls] 41 | Homepage = "https://github.com/bjornmorten/ctf-dl/" 42 | Repository = "https://github.com/bjornmorten/ctf-dl/" 43 | Issues = "https://github.com/bjornmorten/ctf-dl/issues" 44 | 45 | [build-system] 46 | requires = ["setuptools>=61.0", "wheel", "setuptools-scm>=8"] 47 | build-backend = "setuptools.build_meta" 48 | 49 | [tool.setuptools] 50 | include-package-data = true 51 | packages = ["ctfdl"] 52 | 53 | [tool.setuptools.package-data] 54 | "ctfdl" = ["resources/**"] 55 | 56 | [tool.setuptools_scm] 57 | version_scheme = "post-release" 58 | local_scheme = "no-local-version" 59 | 60 | [tool.ruff] 61 | line-length = 100 62 | target-version = "py310" 63 | 64 | [tool.ruff.lint] 65 | extend-select = [ 66 | "T20", 67 | "I", 68 | "UP", 69 | "C4", 70 | "ISC", 71 | "LOG", 72 | "ICN", 73 | "SIM", 74 | "TID", 75 | "TCH", 76 | "TC", 77 | "PTH", 78 | "S", 79 | "DTZ", 80 | "N", 81 | ] 82 | 83 | [tool.ruff.lint.per-file-ignores] 84 | "tests/**/*" = ["S"] 85 | 86 | [project.scripts] 87 | ctf-dl = "ctfdl.cli.main:app" 88 | -------------------------------------------------------------------------------- /ctfdl/cli/helpers.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import typer 4 | 5 | from ctfdl.common.updates import check_updates 6 | from ctfdl.common.version import show_version 7 | from ctfdl.core.config import ExportConfig 8 | from ctfdl.rendering.inspector import list_available_templates 9 | 10 | 11 | def resolve_output_format(name: str) -> tuple[str, str, str]: 12 | output_format_map = { 13 | "json": ("json", "json", "flat"), 14 | "markdown": ("default", "grouped", "default"), 15 | "minimal": ("minimal", "grouped", "default"), 16 | } 17 | if name.lower() not in output_format_map: 18 | raise ValueError(f"Unknown output format: {name}") 19 | return output_format_map[name.lower()] 20 | 21 | 22 | def build_export_config(args: dict) -> ExportConfig: 23 | return ExportConfig( 24 | url=args["url"], 25 | output=Path(args["output"]), 26 | token=args["token"], 27 | username=args["username"], 28 | password=args["password"], 29 | cookie=Path(args["cookie"]) if args["cookie"] else None, 30 | template_dir=Path(args["template_dir"]) if args["template_dir"] else None, 31 | variant_name=args["variant_name"], 32 | folder_template_name=args["folder_template_name"], 33 | index_template_name=args["index_template_name"], 34 | no_index=args["no_index"], 35 | categories=args["categories"], 36 | min_points=args["min_points"], 37 | max_points=args["max_points"], 38 | status=args["status"], 39 | update=args["update"], 40 | no_attachments=args["no_attachments"], 41 | parallel=args["parallel"], 42 | list_templates=args["list_templates"], 43 | zip_output=args["zip_output"], 44 | debug=args["debug"], 45 | ) 46 | 47 | 48 | def handle_version(): 49 | show_version() 50 | raise typer.Exit() 51 | 52 | 53 | def handle_check_update(): 54 | check_updates() 55 | raise typer.Exit() 56 | 57 | 58 | def handle_list_templates(template_dir): 59 | list_available_templates( 60 | Path(template_dir) if template_dir else Path(), 61 | Path(__file__).parent.parent / "templates", 62 | ) 63 | raise typer.Exit() 64 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | ctf-dl 3 |

4 | 5 |

Fast and flexible challenge downloader for all major CTF platforms

6 | 7 |

8 | PyPI 9 | License 10 |

11 | 12 |

13 | Features • 14 | Install • 15 | Quickstart • 16 | Examples • 17 | License 18 |

19 | 20 | 21 | ## 🔧 Features 22 | 23 | - 🔽 **Download all challenges** from supported CTFs 24 | - 🎯 **Apply filters** by category, points, or solve status 25 | - 🗂️ **Organize challenges** with customizable Jinja2 templates 26 | - 🌐 **Supports** CTFd, rCTF, GZCTF, HTB, EPT, Berg, CryptoHack, pwn.college, and pwnable.{tw,kr,xyz} via [ctfbridge](https://github.com/bjornmorten/ctfbridge) 27 | 28 | 29 | ## 📦 Installation 30 | 31 | Run directly with [uv](https://github.com/astral-sh/uv): 32 | 33 | ```bash 34 | uvx ctf-dl 35 | ``` 36 | 37 | Or install permanently with pip: 38 | 39 | ```bash 40 | pip install ctf-dl 41 | ``` 42 | 43 | ## 🚀 Quickstart 44 | 45 | ```bash 46 | ctf-dl https://demo.ctfd.io -u user -p password 47 | ``` 48 | 49 | ## 💡 Examples 50 | 51 | ```bash 52 | # Download all challenges 53 | ctf-dl https://ctf.example.com 54 | 55 | # Specify output directory 56 | ctf-dl https://ctf.example.com -o example-ctf/ 57 | 58 | # Filter by categories 59 | ctf-dl https://ctf.example.com --categories Web Crypto 60 | 61 | # Overwrite existing challenges 62 | ctf-dl https://ctf.example.com --update 63 | 64 | # Compress output 65 | ctf-dl https://ctf.example.com --zip 66 | 67 | # Use JSON output format 68 | ctf-dl https://ctf.example.com --output-format json 69 | 70 | # List available templates 71 | ctf-dl --list-templates 72 | 73 | ``` 74 | 75 | ## 📁 Default Output Structure 76 | 77 | ``` 78 | challenges/ 79 | ├── README.md 80 | ├── pwn/ 81 | │ ├── rsa-beginner/ 82 | │ │ ├── README.md 83 | │ │ └── files/ 84 | │ │ ├── chal.py 85 | │ │ └── output.txt 86 | ├── web/ 87 | │ ├── sql-injection/ 88 | │ │ ├── README.md 89 | │ │ └── files/ 90 | │ │ └── app.py 91 | ``` 92 | 93 | ## 🤝 Contributing 94 | 95 | Contributions are welcome! See [ctfbridge](https://github.com/bjornmorten/ctfbridge) regarding platform support, or open an issue or pull request to improve **ctf-dl** itself. 96 | 97 | ## 🪪 License 98 | 99 | MIT License © 2025 [bjornmorten](https://github.com/bjornmorten) 100 | -------------------------------------------------------------------------------- /ctfdl/rendering/engine.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | from ctfbridge.models.challenge import Challenge as CTFBridgeChallenge 4 | from jinja2 import ChoiceLoader, Environment, FileSystemLoader, TemplateNotFound 5 | from slugify import slugify 6 | 7 | from ctfdl.core.models import ChallengeEntry 8 | from ctfdl.rendering.inspector import list_available_templates, validate_template_dir 9 | from ctfdl.rendering.metadata_loader import parse_template_metadata 10 | from ctfdl.rendering.renderers import ChallengeRenderer, FolderRenderer, IndexRenderer 11 | from ctfdl.rendering.variant_loader import VariantLoader 12 | 13 | 14 | class TemplateEngine: 15 | def __init__( 16 | self, 17 | user_template_dir: Path | None, 18 | builtin_template_dir: Path, 19 | ): 20 | self.user_template_dir = user_template_dir 21 | self.builtin_template_dir = builtin_template_dir 22 | 23 | loaders = [] 24 | if user_template_dir: 25 | loaders.append(FileSystemLoader(str(user_template_dir))) 26 | loaders.append(FileSystemLoader(str(builtin_template_dir))) 27 | 28 | self.env = Environment( 29 | loader=ChoiceLoader(loaders), 30 | trim_blocks=True, 31 | lstrip_blocks=True, 32 | autoescape=True, 33 | ) 34 | self.env.filters["slugify"] = slugify 35 | 36 | self.variant_loader = VariantLoader(user_template_dir, builtin_template_dir) 37 | self.challenge_renderer = ChallengeRenderer() 38 | self.folder_renderer = FolderRenderer(self.env) 39 | self.index_renderer = IndexRenderer() 40 | 41 | def _load_with_metadata(self, template_file: str) -> tuple: 42 | try: 43 | template = self.env.get_template(template_file) 44 | except TemplateNotFound: 45 | raise FileNotFoundError(f"Template '{template_file}' not found.") 46 | 47 | source, filename, _ = self.env.loader.get_source(self.env, template_file) 48 | metadata = parse_template_metadata(Path(filename)) 49 | 50 | return template, metadata 51 | 52 | def render_challenge(self, variant_name: str, challenge: CTFBridgeChallenge, output_dir: Path): 53 | variant = self.variant_loader.resolve_variant(variant_name) 54 | for comp in variant["components"]: 55 | template_file = f"challenge/_components/{comp['template']}" 56 | template, config = self._load_with_metadata(template_file) 57 | config["output_file"] = comp["file"] 58 | self.challenge_renderer.render(template, config, challenge, output_dir) 59 | 60 | def render_path(self, template_name: str, challenge: CTFBridgeChallenge) -> str: 61 | template_file = f"folder_structure/{template_name}.jinja" 62 | template, _ = self._load_with_metadata(template_file) 63 | return self.folder_renderer.render(template, challenge) 64 | 65 | def render_index(self, template_name: str, challenges: list[ChallengeEntry], output_path: Path): 66 | template_file = f"index/{template_name}.jinja" 67 | template, config = self._load_with_metadata(template_file) 68 | self.index_renderer.render(template, config, challenges, output_path) 69 | 70 | def validate(self) -> list: 71 | return validate_template_dir(self.user_template_dir or self.builtin_template_dir, self.env) 72 | 73 | def list_templates(self) -> None: 74 | list_available_templates(self.user_template_dir or Path(), self.builtin_template_dir) 75 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # UV 98 | # Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | #uv.lock 102 | 103 | # poetry 104 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 105 | # This is especially recommended for binary packages to ensure reproducibility, and is more 106 | # commonly ignored for libraries. 107 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 108 | #poetry.lock 109 | 110 | # pdm 111 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 112 | #pdm.lock 113 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 114 | # in version control. 115 | # https://pdm.fming.dev/latest/usage/project/#working-with-version-control 116 | .pdm.toml 117 | .pdm-python 118 | .pdm-build/ 119 | 120 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 121 | __pypackages__/ 122 | 123 | # Celery stuff 124 | celerybeat-schedule 125 | celerybeat.pid 126 | 127 | # SageMath parsed files 128 | *.sage.py 129 | 130 | # Environments 131 | .env 132 | .venv 133 | env/ 134 | venv/ 135 | ENV/ 136 | env.bak/ 137 | venv.bak/ 138 | 139 | # Spyder project settings 140 | .spyderproject 141 | .spyproject 142 | 143 | # Rope project settings 144 | .ropeproject 145 | 146 | # mkdocs documentation 147 | /site 148 | 149 | # mypy 150 | .mypy_cache/ 151 | .dmypy.json 152 | dmypy.json 153 | 154 | # Pyre type checker 155 | .pyre/ 156 | 157 | # pytype static type analyzer 158 | .pytype/ 159 | 160 | # Cython debug symbols 161 | cython_debug/ 162 | 163 | # PyCharm 164 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 165 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 166 | # and can be added to the global gitignore or merged into this file. For a more nuclear 167 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 168 | #.idea/ 169 | 170 | # Ruff stuff: 171 | .ruff_cache/ 172 | 173 | # PyPI configuration file 174 | .pypirc 175 | 176 | # Challenges output 177 | /challenges 178 | ctf-export.zip 179 | 180 | # 181 | uv.lock 182 | 183 | # 184 | *.swp 185 | -------------------------------------------------------------------------------- /ctfdl/rendering/inspector.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import yaml 4 | from jinja2 import Environment, TemplateNotFound 5 | from rich.console import Console 6 | from rich.table import Table 7 | 8 | from ctfdl.rendering.metadata_loader import parse_template_metadata 9 | 10 | 11 | def validate_template_dir(template_dir: Path, env: Environment) -> list[str]: 12 | errors = [] 13 | variant_dir = template_dir / "challenge/variants" 14 | 15 | if not variant_dir.exists(): 16 | return [f"Variant directory not found: {variant_dir}"] 17 | 18 | for file in variant_dir.glob("*.yaml"): 19 | try: 20 | data = yaml.safe_load(file.read_text(encoding="utf-8")) 21 | for comp in data.get("components", []): 22 | path = f"challenge/_components/{comp['template']}" 23 | try: 24 | env.get_template(path) 25 | except TemplateNotFound: 26 | errors.append(f"Missing component template: {path} (from {file.name})") 27 | except Exception as e: 28 | errors.append(f"Failed to parse {file.name}: {e}") 29 | 30 | return errors 31 | 32 | 33 | def list_available_templates(user_template_dir: Path, builtin_template_dir: Path) -> None: 34 | console = Console() 35 | all_sources = [("User", user_template_dir), ("Built-in", builtin_template_dir)] 36 | 37 | def gather_variants(template_dir): 38 | entries = [] 39 | for file in sorted((template_dir / "challenge/variants").glob("*.yaml")): 40 | name = file.stem 41 | description = "" 42 | try: 43 | data = yaml.safe_load(file.read_text(encoding="utf-8")) 44 | description = data.get("description", "") 45 | except Exception: 46 | description = "[red](invalid or unreadable)[/red]" 47 | entries.append((name, description)) 48 | return entries 49 | 50 | def gather_templates(template_dir, subpath, suffix): 51 | files = [] 52 | folder = template_dir / subpath 53 | if folder.exists(): 54 | for f in sorted(folder.glob(f"*{suffix}")): 55 | name = f.name.replace(suffix, "") 56 | desc = parse_template_metadata(f).get("description", "No description") 57 | files.append((name, desc)) 58 | return files 59 | 60 | printed = set() 61 | 62 | for label, source in all_sources: 63 | if not source.exists(): 64 | continue 65 | 66 | # Check if there are any templates 67 | variants = gather_variants(source) 68 | folders = gather_templates(source, "folder_structure", ".jinja") 69 | indexes = gather_templates(source, "index", ".jinja") 70 | 71 | if not (variants or folders or indexes): 72 | continue 73 | 74 | console.rule(f"[bold blue]{label} Templates") 75 | 76 | if variants: 77 | table = Table( 78 | title="[not italic]Challenge Variants", 79 | title_justify="left", 80 | show_header=True, 81 | header_style="bold magenta", 82 | title_style="bold", 83 | ) 84 | table.add_column("Name", style="cyan", no_wrap=True) 85 | table.add_column("Description", style="white") 86 | for name, desc in variants: 87 | if ("variant", name) not in printed: 88 | table.add_row(name, desc) 89 | printed.add(("variant", name)) 90 | console.print(table) 91 | 92 | if folders: 93 | table = Table( 94 | title="[not italic]Folder Structure Templates", 95 | title_justify="left", 96 | show_header=True, 97 | header_style="bold green", 98 | title_style="bold", 99 | ) 100 | table.add_column("Name", style="cyan") 101 | table.add_column("Description", style="white") 102 | for name, desc in folders: 103 | if ("folder", name) not in printed: 104 | table.add_row(name, desc) 105 | printed.add(("folder", name)) 106 | console.print(table) 107 | 108 | if indexes: 109 | table = Table( 110 | title="[not italic]Index Templates", 111 | title_justify="left", 112 | show_header=True, 113 | header_style="bold yellow", 114 | title_style="bold", 115 | ) 116 | table.add_column("Name", style="cyan") 117 | table.add_column("Description", style="white") 118 | for name, desc in indexes: 119 | if ("index", name) not in printed: 120 | table.add_row(name, desc) 121 | printed.add(("index", name)) 122 | console.print(table) 123 | -------------------------------------------------------------------------------- /ctfdl/ui/messages.py: -------------------------------------------------------------------------------- 1 | from rich.console import Console 2 | 3 | _default_console = Console(log_path=False, log_time=False) 4 | 5 | # ===== Basic Notifications ===== 6 | 7 | 8 | def info(msg: str, console: Console = _default_console): 9 | console.print(f"🔍 [cyan]{msg}[/]") 10 | 11 | 12 | def success(msg: str, console: Console = _default_console): 13 | console.print(f"✅ [green]{msg}[/]") 14 | 15 | 16 | def warning(msg: str, console: Console = _default_console): 17 | console.print(f"⚠️ [yellow]{msg}[/]") 18 | 19 | 20 | def error(msg: str, console: Console = _default_console): 21 | console.print(f"❌ [bold red]{msg}[/]") 22 | 23 | 24 | def debug(msg: str, console: Console = _default_console): 25 | console.print(f"[dim]{msg}[/]") 26 | 27 | 28 | # ===== Download Progress ===== 29 | 30 | 31 | def connecting(url: str, console: Console = _default_console): 32 | return console.status( 33 | f"[bold blue]️Connecting to CTF platform: [bold magenta]{url}[/]", 34 | spinner="dots", 35 | ) 36 | 37 | 38 | def connected(console: Console = _default_console): 39 | success("Connection established") 40 | 41 | 42 | def connection_failed(error_message: str, console: Console = _default_console): 43 | error(f"Connection failed: {error_message}") 44 | 45 | 46 | def authentication_required(console: Console = _default_console): 47 | console.print("🔒 [bold red]Authentication required[/bold red]\n") 48 | console.print("[yellow]Please provide authentication credentials.[/yellow]") 49 | console.print(" [cyan]--token[/cyan] YOUR_TOKEN") 50 | console.print(" or") 51 | console.print(" [cyan]--username[/cyan] USERNAME [cyan]--password[/cyan] PASSWORD\n") 52 | 53 | 54 | def no_challenges_found(console: Console = _default_console): 55 | error("There are no challenges to download...", console) 56 | 57 | 58 | def challenges_found(count: int, console: Console = _default_console): 59 | console.print(f"📦 Found [bold]{count} challenges[/] to download:\n") 60 | 61 | 62 | def downloaded_challenge(name: str, category: str, console: Console = _default_console): 63 | console.print(f"✅ Downloaded: [green]{name}[/] ([cyan]{category}[/])") 64 | 65 | 66 | def failed_challenge(name: str, reason: str, console: Console = _default_console): 67 | console.print(f"❌ [bold red]ERROR:[/] Failed [green]{name}[/]: {reason}") 68 | 69 | 70 | def download_success_new(count: int, console: Console = _default_console): 71 | console.print(f"🎉 [bold green]{count} challenges downloaded successfully![/bold green]") 72 | 73 | 74 | def download_success_skipped_all(count: int, console: Console = _default_console): 75 | console.print( 76 | f"⏩ [bold yellow]All {count} challenges were skipped.[/bold yellow]\n" 77 | f" Use [cyan]--update[/cyan] to re-download " 78 | f"[dim](this will overwrite existing files)[/dim]." 79 | ) 80 | 81 | 82 | def download_success_updated_all(count: int, console: Console = _default_console): 83 | console.print(f"🔄 [bold green]All {count} challenges were successfully updated![/bold green]") 84 | 85 | 86 | def download_success_summary( 87 | downloaded: int, updated: int, skipped: int, console: Console = _default_console 88 | ): 89 | console.print("🎉 [bold green]Download summary:[/bold green]") 90 | if downloaded: 91 | console.print(f" ✅ {downloaded} new challenges downloaded") 92 | if updated: 93 | console.print(f" 🔄 {updated} challenges updated") 94 | if skipped: 95 | console.print(f" ⏩ {skipped} challenges skipped") 96 | 97 | 98 | def zipped_output(path: str, console: Console = _default_console): 99 | console.print(f"🗂️ [green]Output saved to:[/] [bold underline]{path}[/]") 100 | 101 | 102 | # ===== Version and Update ===== 103 | 104 | 105 | def version_output(version: str, console: Console = _default_console): 106 | console.print(f"📦 [bold]ctf-dl[/bold] version: [green]{version}[/green]") 107 | 108 | 109 | def update_available(pkg: str, installed: str, latest: str, console: Console = _default_console): 110 | console.print( 111 | f"📦 [yellow]{pkg}[/]: update available → [red]{installed}[/] → [green]{latest}[/]" 112 | ) 113 | 114 | 115 | def up_to_date(pkg: str, version: str, console: Console = _default_console): 116 | console.print(f"✅ {pkg} is up to date ([green]{version}[/])") 117 | 118 | 119 | def update_failed(pkg: str, reason: str, console: Console = _default_console): 120 | console.print(f"⚠️ Failed to fetch version for [yellow]{pkg}[/]: {reason}") 121 | 122 | 123 | def not_installed(pkg: str, console: Console = _default_console): 124 | error(f"{pkg} is not installed.") 125 | 126 | 127 | def upgrade_tip(cmd: str, console: Console = _default_console): 128 | console.print(f"\n🚀 To upgrade, run:\n[bold]{cmd}[/bold]") 129 | 130 | 131 | # ===== Templates ===== 132 | 133 | 134 | def list_templates_header(name: str, console: Console = _default_console): 135 | console.print(f"\n📂 Available {name} Templates:") 136 | 137 | 138 | def list_template_item(name: str, console: Console = _default_console): 139 | console.print(f"- {name}") 140 | 141 | 142 | # ===== Context Manager ===== 143 | 144 | 145 | def spinner_status(message: str, console: Console = _default_console): 146 | return console.status(message, spinner="dots") 147 | -------------------------------------------------------------------------------- /ctfdl/challenges/downloader.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from pathlib import Path 3 | 4 | from ctfbridge.base.client import CTFClient 5 | from ctfbridge.exceptions import ( 6 | LoginError, 7 | MissingAuthMethodError, 8 | NotAuthenticatedError, 9 | UnknownBaseURLError, 10 | UnknownPlatformError, 11 | ) 12 | from ctfbridge.models.challenge import Challenge, ProgressData 13 | 14 | from ctfdl.challenges.client import get_authenticated_client 15 | from ctfdl.core import EventEmitter, ExportConfig 16 | from ctfdl.core.models import ChallengeEntry 17 | from ctfdl.rendering.context import TemplateEngineContext 18 | from ctfdl.rendering.engine import TemplateEngine 19 | 20 | 21 | async def download_challenges(config: ExportConfig, emitter: EventEmitter) -> tuple[bool, list]: 22 | try: 23 | await emitter.emit("connect_start", url=config.url) 24 | client = await get_authenticated_client( 25 | config.url, config.username, config.password, config.token 26 | ) 27 | await emitter.emit("connect_success") 28 | except UnknownPlatformError: 29 | await emitter.emit( 30 | "connect_fail", 31 | reason="Unsupported platform. You may suggest adding support here: https://github.com/bjornmorten/ctfbridge/issues", 32 | ) 33 | return False, [] 34 | except UnknownBaseURLError: 35 | await emitter.emit( 36 | "connect_fail", 37 | reason=( 38 | "Platform was identified, but base URL could not be determined. " 39 | "If you believe this is an error, you may open an issue here: " 40 | "https://github.com/bjornmorten/ctfbridge/issues" 41 | ), 42 | ) 43 | return False, [] 44 | except LoginError: 45 | await emitter.emit("connect_fail", reason="Authentication failed") 46 | return False, [] 47 | except MissingAuthMethodError: 48 | await emitter.emit("connect_fail", reason="Invalid authentication type") 49 | return False, [] 50 | 51 | challenges_iterator = client.challenges.iter_all( 52 | categories=config.categories, 53 | min_points=config.min_points, 54 | max_points=config.max_points, 55 | solved=True if config.solved else False if config.unsolved else None, 56 | detailed=True, 57 | enrich=True, 58 | ) 59 | 60 | template_engine = TemplateEngineContext.get() 61 | output_dir = config.output 62 | output_dir.mkdir(parents=True, exist_ok=True) 63 | all_challenges_data = [] 64 | 65 | sem = asyncio.Semaphore(config.parallel) 66 | tasks = [] 67 | challenge_count = 0 68 | 69 | async def process(chal: Challenge): 70 | try: 71 | await emitter.emit("challenge_start", challenge=chal) 72 | 73 | entry = await process_challenge( 74 | client, 75 | emitter, 76 | chal, 77 | template_engine, 78 | config, 79 | output_dir, 80 | ) 81 | if entry: 82 | all_challenges_data.append(entry) 83 | await emitter.emit("challenge_success", challenge=chal) 84 | except Exception as e: 85 | await emitter.emit("challenge_fail", challenge=chal, reason=str(e)) 86 | finally: 87 | await emitter.emit("challenge_complete", challenge=chal) 88 | 89 | async def worker(chal: Challenge): 90 | async with sem: 91 | await process(chal) 92 | 93 | await emitter.emit("fetch_start") 94 | 95 | try: 96 | started = False 97 | async for chal in challenges_iterator: 98 | if not started: 99 | await emitter.emit("download_start") 100 | started = True 101 | 102 | challenge_count += 1 103 | task = asyncio.create_task(worker(chal)) 104 | tasks.append(task) 105 | except NotAuthenticatedError: 106 | await emitter.emit("authentication_required") 107 | return False, [] 108 | 109 | if challenge_count == 0: 110 | await emitter.emit("no_challenges_found") 111 | await emitter.emit("download_complete") 112 | return False, [] 113 | 114 | await asyncio.gather(*tasks) 115 | 116 | await emitter.emit("download_complete") 117 | return True, all_challenges_data 118 | 119 | 120 | async def process_challenge( 121 | client: CTFClient, 122 | emitter: EventEmitter, 123 | chal: Challenge, 124 | template_engine: TemplateEngine, 125 | config: ExportConfig, 126 | output_dir: Path, 127 | ): 128 | rel_path_str = template_engine.render_path(config.folder_template_name, chal) 129 | chal_folder = output_dir / rel_path_str 130 | 131 | existed_before = chal_folder.exists() 132 | 133 | if existed_before and not config.update: 134 | await emitter.emit("challenge_skipped", challenge=chal) 135 | return 136 | 137 | async def progress_callback(pd: ProgressData): 138 | await emitter.emit("attachment_progress", progress_data=pd, challenge=chal) 139 | 140 | chal_folder.mkdir(parents=True, exist_ok=True) 141 | if not config.no_attachments and chal.attachments: 142 | files_dir = chal_folder / "files" 143 | files_dir.mkdir(exist_ok=True) 144 | chal = await client.attachments.download_all( 145 | chal, 146 | save_dir=str(files_dir), 147 | progress=progress_callback, 148 | concurrency=config.parallel, 149 | ) 150 | template_engine.render_challenge(config.variant_name, chal, chal_folder) 151 | 152 | await emitter.emit("challenge_downloaded", challenge=chal, updated=existed_before) 153 | 154 | return ChallengeEntry( 155 | data=chal, 156 | path=Path(rel_path_str), 157 | updated=existed_before, 158 | ) 159 | -------------------------------------------------------------------------------- /docs/templates.md: -------------------------------------------------------------------------------- 1 | # 🧩 Template System Documentation for `ctf-dl` 2 | 3 | This guide explains the templating system in `ctf-dl`, including how to define, extend, and use templates to control the output of downloaded Capture The Flag (CTF) challenges. 4 | 5 | --- 6 | 7 | ## 🎯 Overview 8 | 9 | The template system is designed to be **modular, extensible, and easy to customize**. It enables users to: 10 | 11 | * Format individual challenge outputs (e.g., README, solve scripts) 12 | * Customize folder structures 13 | * Generate global challenge indexes 14 | * Override or extend default templates without modifying the core tool 15 | 16 | --- 17 | 18 | ## 📁 Template Types 19 | 20 | Templates are grouped into three functional categories: 21 | 22 | | Template Type | Description | CLI Flag | 23 | | -------------------- | ------------------------------------------------ | ------------------- | 24 | | **Challenge** | Controls per-challenge files like README, solves | `--template` | 25 | | **Folder Structure** | Determines folder layout for challenges | `--folder-template` | 26 | | **Index** | Renders a challenge overview or summary | `--index-template` | 27 | 28 | --- 29 | 30 | ## 🧱 Directory Layout 31 | 32 | A custom template directory should follow this structure: 33 | 34 | ``` 35 | my-templates/ 36 | ├── challenge/ 37 | │ ├── _components/ 38 | │ │ ├── readme.jinja 39 | │ │ └── solve.py.jinja 40 | │ └── variants/ 41 | │ ├── minimal.yaml 42 | │ └── writeup.yaml 43 | ├── folder_structure/ 44 | │ └── default.path.jinja 45 | └── index/ 46 | └── grouped.md.jinja 47 | ``` 48 | 49 | Specify your template root with: 50 | 51 | ```bash 52 | ctf-dl --template-dir ./my-templates 53 | ``` 54 | 55 | --- 56 | 57 | ## 📄 Challenge Variants 58 | 59 | Variants describe which files should be rendered for each challenge, using a YAML config: 60 | 61 | ```yaml 62 | name: writeup 63 | components: 64 | - file: README.md 65 | template: readme.jinja 66 | - file: solve/solve.py 67 | template: solve.py.jinja 68 | ``` 69 | 70 | Use with: 71 | 72 | ```bash 73 | ctf-dl --template writeup --template-dir ./my-templates 74 | ``` 75 | 76 | ### 🔁 Extending Variants 77 | 78 | You can use `extends:` to inherit from another variant: 79 | 80 | ```yaml 81 | extends: base 82 | ``` 83 | 84 | * `components` are inherited unless explicitly overridden. 85 | 86 | --- 87 | 88 | ## 🧩 Component Templates 89 | 90 | Component templates are stored under `challenge/_components/`. Each Jinja file defines the content for a specific output file: 91 | 92 | ```jinja 93 | # {{ challenge.name }} 94 | 95 | **Points:** {{ challenge.value }} 96 | **Category:** {{ challenge.category }} 97 | 98 | {{ challenge.description }} 99 | ``` 100 | 101 | ### Supported Variables 102 | 103 | * `challenge`: Core metadata (name, category, value, etc.) 104 | 105 | Use Jinja’s `{% include %}` or `{% extends %}` to reuse logic between templates. 106 | 107 | ### 🔄 Overriding Built-in Components 108 | 109 | User-defined templates can override built-in components seamlessly. The system uses a layered loading approach: 110 | 111 | 1. If a component (e.g., `readme.jinja`) exists in the user’s template directory, it will be used. 112 | 2. If it does not exist, the system will fall back to the built-in component in the tool’s internal templates. 113 | 114 | This makes it easy to: 115 | 116 | * Fully replace any default template 117 | * Override only selected parts (e.g., just `solve.py.jinja`) 118 | * Extend base templates using Jinja’s `{% extends %}` syntax: 119 | 120 | ```jinja 121 | {# user-defined writeup.jinja #} 122 | {% extends "readme.jinja" %} 123 | 124 | {% block extra %} 125 | ## Additional Notes 126 | - Add any writeup-specific notes here. 127 | {% endblock %} 128 | ``` 129 | 130 | Built-in and user-defined templates are resolved using a Jinja2 `ChoiceLoader`, ensuring full compatibility. 131 | 132 | --- 133 | 134 | ## 🗂 Folder Structure Templates 135 | 136 | Folder templates determine how each challenge's directory is named and organized. 137 | 138 | Example (`default.path.jinja`): 139 | 140 | ```jinja 141 | {{ challenge.category | slugify }}/{{ challenge.name | slugify }} 142 | ``` 143 | 144 | Apply with: 145 | 146 | ```bash 147 | ctf-dl --folder-template default 148 | ``` 149 | 150 | --- 151 | 152 | ## 🧾 Index Templates 153 | 154 | Index templates render an overview (e.g., `index.md`) listing all downloaded challenges. 155 | 156 | Example (`grouped.md.jinja`): 157 | 158 | ```jinja 159 | # Challenge Index 160 | 161 | {% set grouped = {} %} 162 | {% for c in challenges %}{% set _ = grouped.setdefault(c.category, []).append(c) %}{% endfor %} 163 | 164 | {% for category, items in grouped.items() %} 165 | ## {{ category }} 166 | | Name | Points | Solved | Path | 167 | |------|--------|--------|------| 168 | {% for c in items %} 169 | | {{ c.name }} | {{ c.value }} | {{ "✅" if c.solved else "❌" }} | [Link]({{ c.path }}) | 170 | {% endfor %} 171 | {% endfor %} 172 | ``` 173 | 174 | Apply with: 175 | 176 | ```bash 177 | ctf-dl --index-template grouped 178 | ``` 179 | 180 | --- 181 | 182 | ## ⚙️ Template Resolution 183 | 184 | Templates are resolved using a layered strategy: 185 | 186 | 1. **User-provided templates** (`--template-dir`) 187 | 2. **Built-in defaults** (packaged with `ctf-dl`) 188 | 189 | The engine uses a Jinja2 `ChoiceLoader`: 190 | 191 | * User templates take precedence 192 | * Built-ins serve as fallback 193 | * Includes and extends work seamlessly across both 194 | 195 | This ensures that users can override just what they need, while still benefiting from the system's built-in defaults. 196 | 197 | --- 198 | 199 | ## 🧰 Developer Tools 200 | 201 | ### 🚀 Initialize a Template Directory 202 | 203 | ```bash 204 | ctf-dl init-template ./my-templates 205 | ``` 206 | 207 | Creates a scaffold with sample components, variants, and helpful comments. 208 | 209 | ### ✅ Validate a Template Directory 210 | 211 | ```bash 212 | ctf-dl validate-template --template-dir ./my-templates 213 | ``` 214 | 215 | Checks for: 216 | 217 | * Valid YAML structure 218 | * Referenced `.jinja` files 219 | * Jinja syntax errors 220 | 221 | --- 222 | 223 | ## ✅ Best Practices 224 | 225 | * 🧩 Reuse shared blocks in `_components/` 226 | * 🪄 Use `extends:` to avoid repeating boilerplate 227 | * 🧼 Use slugified folder names to avoid filesystem issues 228 | * 🧪 Validate templates before running exports 229 | * 🧠 Override only the templates you want to change 230 | -------------------------------------------------------------------------------- /ctfdl/cli/main.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import getpass 3 | import sys 4 | from enum import Enum 5 | 6 | import typer 7 | from rich.console import Console 8 | 9 | from ctfdl.cli.helpers import ( 10 | build_export_config, 11 | handle_check_update, 12 | handle_list_templates, 13 | handle_version, 14 | resolve_output_format, 15 | ) 16 | 17 | 18 | class ChallengeStatus(str, Enum): 19 | all = "all" 20 | solved = "solved" 21 | unsolved = "unsolved" 22 | 23 | 24 | console = Console(log_path=False) 25 | app = typer.Typer( 26 | add_completion=False, 27 | no_args_is_help=False, 28 | invoke_without_command=True, 29 | context_settings={ 30 | "help_option_names": ["-h", "--help"], 31 | "allow_extra_args": False, 32 | "ignore_unknown_options": False, 33 | "token_normalize_func": lambda x: x, 34 | }, 35 | ) 36 | 37 | 38 | @app.command(name=None) 39 | def cli( 40 | version: bool = typer.Option( 41 | False, 42 | "--version", 43 | is_eager=True, 44 | help="Show version and exit", 45 | rich_help_panel="Options", 46 | ), 47 | check_update: bool = typer.Option( 48 | False, 49 | "--check-update", 50 | is_eager=True, 51 | help="Check for updates", 52 | rich_help_panel="Options", 53 | ), 54 | debug: bool = typer.Option( 55 | False, "--debug", "-d", help="Enable debug logging", rich_help_panel="Options" 56 | ), 57 | url: str | None = typer.Argument( 58 | None, 59 | help="URL of the CTF instance (e.g., https://ctf.example.com)", 60 | show_default=False, 61 | ), 62 | output: str | None = typer.Option( 63 | "challenges", 64 | "--output", 65 | "-o", 66 | help="Output directory to save challenges", 67 | rich_help_panel="Output", 68 | ), 69 | zip_output: bool = typer.Option( 70 | False, 71 | "--zip", 72 | "-z", 73 | help="Compress output folder after download", 74 | rich_help_panel="Output", 75 | ), 76 | output_format: str | None = typer.Option( 77 | None, 78 | "--output-format", 79 | "-f", 80 | help="Preset output format (json, markdown, minimal)", 81 | rich_help_panel="Output", 82 | ), 83 | template_dir: str | None = typer.Option( 84 | None, 85 | "--template-dir", 86 | help="Directory containing custom templates", 87 | rich_help_panel="Templating", 88 | ), 89 | variant_name: str = typer.Option( 90 | "default", 91 | "--template", 92 | help="Challenge template variant to use", 93 | rich_help_panel="Templating", 94 | ), 95 | folder_template_name: str = typer.Option( 96 | "default", 97 | "--folder-template", 98 | help="Template for folder structure", 99 | rich_help_panel="Templating", 100 | ), 101 | index_template_name: str | None = typer.Option( 102 | "grouped", 103 | "--index-template", 104 | help="Template for challenge index", 105 | rich_help_panel="Templating", 106 | ), 107 | no_index: bool = typer.Option( 108 | False, 109 | "--no-index", 110 | help="Do not generate an index file", 111 | rich_help_panel="Templating", 112 | ), 113 | list_templates: bool = typer.Option( 114 | False, 115 | "--list-templates", 116 | help="List available templates and exit", 117 | rich_help_panel="Templating", 118 | ), 119 | token: str | None = typer.Option( 120 | None, 121 | "--token", 122 | "-t", 123 | help="Authentication token", 124 | rich_help_panel="Authentication", 125 | ), 126 | username: str | None = typer.Option( 127 | None, 128 | "--username", 129 | "-u", 130 | help="Login username", 131 | rich_help_panel="Authentication", 132 | ), 133 | password: str | None = typer.Option( 134 | None, 135 | "--password", 136 | "-p", 137 | help="Login password", 138 | rich_help_panel="Authentication", 139 | ), 140 | cookie: str | None = typer.Option( 141 | None, 142 | "--cookie", 143 | "-c", 144 | help="Path to cookie/session file", 145 | rich_help_panel="Authentication", 146 | ), 147 | categories: list[str] | None = typer.Option( 148 | None, 149 | "--categories", 150 | help="Only download specified categories", 151 | rich_help_panel="Filters", 152 | ), 153 | min_points: int | None = typer.Option( 154 | None, "--min-points", help="Minimum challenge points", rich_help_panel="Filters" 155 | ), 156 | max_points: int | None = typer.Option( 157 | None, "--max-points", help="Maximum challenge points", rich_help_panel="Filters" 158 | ), 159 | status: ChallengeStatus = typer.Option( 160 | ChallengeStatus.all, 161 | "--status", 162 | case_sensitive=False, 163 | help="Filter challenges by their completion status", 164 | rich_help_panel="Filters", 165 | ), 166 | update: bool = typer.Option( 167 | False, 168 | "--update", 169 | help="Update existing challenges instead of skipping them (overwrites existing files)", 170 | rich_help_panel="Behavior", 171 | ), 172 | no_attachments: bool = typer.Option( 173 | False, 174 | "--no-attachments", 175 | help="Do not download attachments", 176 | rich_help_panel="Behavior", 177 | ), 178 | parallel: int = typer.Option( 179 | 10, 180 | "--parallel", 181 | help="Number of parallel downloads", 182 | rich_help_panel="Behavior", 183 | ), 184 | ): 185 | if version: 186 | handle_version() 187 | 188 | if check_update: 189 | handle_check_update() 190 | 191 | if list_templates: 192 | handle_list_templates(template_dir) 193 | 194 | if url is None: 195 | raise typer.BadParameter("Missing required argument: URL") 196 | 197 | if username and not password: 198 | if sys.stdin.isatty(): 199 | password = getpass.getpass("Password: ") 200 | else: 201 | typer.secho("Error: password required but not provided", fg=typer.colors.RED) 202 | raise typer.Exit(code=1) 203 | 204 | if output_format: 205 | try: 206 | variant_name, index_template_name, folder_template_name = resolve_output_format( 207 | output_format 208 | ) 209 | except ValueError as e: 210 | raise typer.BadParameter(str(e)) 211 | 212 | config = build_export_config(locals()) 213 | 214 | from ctfdl.challenges.entry import run_export 215 | 216 | asyncio.run(run_export(config)) 217 | 218 | 219 | if __name__ == "__main__": 220 | app() 221 | -------------------------------------------------------------------------------- /ctfdl/ui/rich_handler.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | from ctfbridge.models.challenge import Challenge, ProgressData 4 | from rich.live import Live 5 | from rich.progress import Progress, ProgressColumn, SpinnerColumn, TextColumn 6 | from rich.progress_bar import ProgressBar 7 | from rich.table import Table 8 | from rich.text import Text 9 | from rich.tree import Tree 10 | 11 | import ctfdl.ui.messages as console_utils 12 | from ctfdl.common.console import console 13 | from ctfdl.core.events import EventEmitter 14 | 15 | 16 | def handles(event_name: str): 17 | """Decorator to mark a method as an event handler for `event_name`.""" 18 | 19 | def decorator(func): 20 | func._event_name = event_name 21 | return func 22 | 23 | return decorator 24 | 25 | 26 | class AdaptiveTimeColumn(ProgressColumn): 27 | def render(self, task): 28 | elapsed = int(task.elapsed or 0) 29 | if elapsed < 60: 30 | text = f"{elapsed}s" 31 | else: 32 | minutes, seconds = divmod(elapsed, 60) 33 | text = f"{minutes}m {seconds:02d}s" 34 | return Text(text, style="yellow") 35 | 36 | 37 | class RichConsoleHandler: 38 | def __init__(self, emitter: EventEmitter): 39 | self._console = console 40 | self._progress = Progress( 41 | SpinnerColumn(), 42 | TextColumn("[bold blue]{task.description}"), 43 | TextColumn("[green]({task.completed} downloaded)"), 44 | AdaptiveTimeColumn(), 45 | console=self._console, 46 | ) 47 | self._tree = Tree(self._progress) 48 | self._live = Live( 49 | self._tree, 50 | console=self._console, 51 | refresh_per_second=10, 52 | transient=True, 53 | ) 54 | 55 | self._main_task_id = None 56 | self._category_nodes: dict[str, dict] = {} 57 | self._challenge_nodes: dict[str, dict] = {} 58 | self._attachment_nodes: dict[str, any] = {} 59 | self._lock = asyncio.Lock() 60 | 61 | self._stats = {"downloaded": 0, "updated": 0, "skipped": 0} 62 | 63 | # Register handles 64 | for attr_name in dir(self): 65 | fn = getattr(self, attr_name) 66 | if callable(fn) and hasattr(fn, "_event_name"): 67 | emitter.on(fn._event_name, fn) 68 | 69 | # ===== Connection ===== 70 | 71 | @handles("connect_start") 72 | def on_connect_start(self, url: str): 73 | self._live.start() 74 | self._main_task_id = self._progress.add_task( 75 | description=f"Connecting to {url}...", total=None 76 | ) 77 | 78 | @handles("connect_success") 79 | def on_connect_success(self): 80 | self._progress.update(self._main_task_id, description="Connection established") 81 | 82 | @handles("connect_fail") 83 | def on_connect_fail(self, reason: str): 84 | if self._live.is_started: 85 | self._live.stop() 86 | console_utils.connection_failed(reason, console=self._console) 87 | 88 | @handles("authentication_required") 89 | def on_authentication_required(self): 90 | if self._live.is_started: 91 | self._live.stop() 92 | console_utils.authentication_required(console=self._console) 93 | 94 | # ===== Download Lifecycle ===== 95 | 96 | @handles("fetch_start") 97 | def on_fetch_start(self): 98 | self._progress.update(self._main_task_id, description="Fetching challenge list") 99 | 100 | @handles("download_start") 101 | def on_download_start(self): 102 | self._progress.update(self._main_task_id, description="Downloading challenges") 103 | 104 | @handles("no_challenges_found") 105 | def on_no_challenges_found(self): 106 | if self._main_task_id: 107 | self._progress.update(self._main_task_id, description="No challenges found") 108 | console_utils.no_challenges_found(console=self._console) 109 | 110 | @handles("download_fail") 111 | def on_download_fail(self, msg: str): 112 | if self._live.is_started: 113 | self._live.stop() 114 | console_utils.error(msg) 115 | 116 | @handles("download_success") 117 | def on_download_success(self): 118 | if self._live.is_started: 119 | self._live.stop() 120 | 121 | downloaded = self._stats["downloaded"] 122 | updated = self._stats["updated"] 123 | skipped = self._stats["skipped"] 124 | total = downloaded + updated + skipped 125 | 126 | if updated == 0 and skipped == 0: 127 | console_utils.download_success_new(downloaded, console=self._console) 128 | elif skipped == total: 129 | console_utils.download_success_skipped_all(skipped, console=self._console) 130 | elif updated == total: 131 | console_utils.download_success_updated_all(updated, console=self._console) 132 | else: 133 | console_utils.download_success_summary( 134 | downloaded, updated, skipped, console=self._console 135 | ) 136 | 137 | @handles("download_complete") 138 | def on_download_complete(self): 139 | if self._live.is_started: 140 | self._live.stop() 141 | 142 | # ===== Per-Challenge ===== 143 | 144 | @handles("challenge_skipped") 145 | def on_challenge_skipped(self, challenge: Challenge): 146 | self._stats["skipped"] += 1 147 | 148 | @handles("challenge_start") 149 | async def on_challenge_start(self, challenge: Challenge): 150 | async with self._lock: 151 | if challenge.category not in self._category_nodes: 152 | node = self._tree.add(f"📁 [bold cyan]{challenge.category}[/bold cyan]") 153 | self._category_nodes[challenge.category] = {"node": node, "count": 0} 154 | category_info = self._category_nodes[challenge.category] 155 | parent_node = category_info["node"] 156 | category_info["count"] += 1 157 | challenge_node = parent_node.add(f"📂 [bold]{challenge.name}[/bold]") 158 | self._challenge_nodes[challenge.name] = { 159 | "node": challenge_node, 160 | "parent": parent_node, 161 | } 162 | 163 | @handles("challenge_fail") 164 | def on_challenge_fail(self, challenge: Challenge, reason: str): 165 | console_utils.failed_challenge(challenge.name, reason, console=self._console) 166 | 167 | @handles("challenge_complete") 168 | async def on_challenge_complete(self, challenge: Challenge): 169 | async with self._lock: 170 | if challenge.name in self._challenge_nodes: 171 | node_info = self._challenge_nodes.pop(challenge.name) 172 | challenge_node = node_info["node"] 173 | parent_node = node_info["parent"] 174 | if parent_node and challenge_node in parent_node.children: 175 | parent_node.children.remove(challenge_node) 176 | 177 | if challenge.category in self._category_nodes: 178 | category_info = self._category_nodes[challenge.category] 179 | category_info["count"] -= 1 180 | if category_info["count"] == 0: 181 | category_node_to_remove = category_info["node"] 182 | if category_node_to_remove in self._tree.children: 183 | self._tree.children.remove(category_node_to_remove) 184 | del self._category_nodes[challenge.category] 185 | 186 | if self._main_task_id is not None: 187 | self._progress.update(self._main_task_id, advance=1) 188 | 189 | @handles("challenge_downloaded") 190 | def on_challenge_downloaded(self, challenge: Challenge, updated: bool = False): 191 | if updated: 192 | self._stats["updated"] += 1 193 | else: 194 | self._stats["downloaded"] += 1 195 | 196 | # ===== Attachments ===== 197 | 198 | @handles("attachment_progress") 199 | async def on_attachment_progress(self, progress_data: ProgressData, challenge: Challenge): 200 | pd = progress_data 201 | attachment_id = str(pd.attachment.download_info) 202 | 203 | async with self._lock: 204 | challenge_node_info = self._challenge_nodes.get(challenge.name) 205 | if not challenge_node_info: 206 | return 207 | challenge_node = challenge_node_info["node"] 208 | attachment_node = self._attachment_nodes.get(attachment_id) 209 | 210 | progress_bar = ProgressBar( 211 | total=pd.total_bytes, completed=pd.downloaded_bytes, width=30 212 | ) 213 | grid = Table.grid(expand=False) 214 | grid.add_row( 215 | f"📄 {pd.attachment.name} ", 216 | progress_bar, 217 | f" [yellow]{pd.percentage:.2f}%[/yellow]", 218 | ) 219 | 220 | if attachment_node is None: 221 | new_node = challenge_node.add(grid) 222 | self._attachment_nodes[attachment_id] = new_node 223 | else: 224 | attachment_node.label = grid 225 | 226 | if pd.downloaded_bytes == pd.total_bytes: 227 | if attachment_node and attachment_node in challenge_node.children: 228 | challenge_node.children.remove(attachment_node) 229 | self._attachment_nodes.pop(attachment_id, None) 230 | --------------------------------------------------------------------------------