├── src └── kiwixseeder │ ├── __about__.py │ ├── __main__.py │ ├── utils.py │ ├── forever.py │ ├── qbittorrent.py │ ├── download.py │ ├── context.py │ ├── library.py │ ├── runner.py │ └── entrypoint.py ├── tests └── test_stub.py ├── kiwixseeder.config.yml ├── gen-password.py ├── .github ├── stale.yml ├── workflows │ ├── DockerCD.yaml │ ├── QA.yml │ └── Tests.yml └── FUNDING.yml ├── get-pbkdf2.py ├── Dockerfile ├── entrypoint.sh ├── .gitignore ├── tasks.py ├── seeder-start-restart.sh ├── pyproject.toml ├── README.md └── LICENSE /src/kiwixseeder/__about__.py: -------------------------------------------------------------------------------- 1 | __version__ = "1.0.0" 2 | -------------------------------------------------------------------------------- /src/kiwixseeder/__main__.py: -------------------------------------------------------------------------------- 1 | from kiwixseeder.entrypoint import entrypoint 2 | 3 | entrypoint() 4 | -------------------------------------------------------------------------------- /tests/test_stub.py: -------------------------------------------------------------------------------- 1 | from kiwixseeder.entrypoint import prepare_context 2 | 3 | 4 | def test_no_args(): 5 | prepare_context([]) 6 | from kiwixseeder.context import Context 7 | 8 | context = Context.get() 9 | assert not context.all_good 10 | -------------------------------------------------------------------------------- /kiwixseeder.config.yml: -------------------------------------------------------------------------------- 1 | # yamllint disable rule:line-length 2 | # yamllint disable rule:indentation 3 | # yamllint disable rule:comments-indentation 4 | # too many spelling things, spell-checker: disable 5 | --- 6 | - module-name: 'iso639' 7 | data-files: 8 | dirs: 9 | - 'data' 10 | -------------------------------------------------------------------------------- /gen-password.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """command line to generate a short (8chars) password from alphanum""" 4 | 5 | import secrets 6 | import string 7 | import sys 8 | 9 | 10 | def gen_password() -> str: 11 | alphabet = string.ascii_letters + string.digits 12 | return "".join(secrets.choice(alphabet) for _ in range(8)) 13 | 14 | 15 | def main() -> int: 16 | print(gen_password()) # noqa: T201 17 | return 0 18 | 19 | 20 | if __name__ == "__main__": 21 | sys.exit(main()) 22 | -------------------------------------------------------------------------------- /.github/stale.yml: -------------------------------------------------------------------------------- 1 | daysUntilClose: false 2 | staleLabel: stale 3 | 4 | issues: 5 | daysUntilStale: 60 6 | markComment: > 7 | This issue has been automatically marked as stale because it has not had 8 | recent activity. It will be now be reviewed manually. Thank you 9 | for your contributions. 10 | 11 | pulls: 12 | daysUntilStale: 7 13 | markComment: > 14 | This pull request has been automatically marked as stale because it has not had 15 | recent activity. It will be now be reviewed manually. Thank you 16 | for your contributions. 17 | 18 | -------------------------------------------------------------------------------- /.github/workflows/DockerCD.yaml: -------------------------------------------------------------------------------- 1 | name: Docker CD 2 | 3 | on: 4 | push: 5 | branches: 6 | - 'main' 7 | workflow_dispatch: 8 | 9 | jobs: 10 | 11 | bittorrent-seeder: 12 | name: Deploy Seeder Image 13 | runs-on: ubuntu-22.04 14 | steps: 15 | - uses: actions/checkout@v3.4.0 16 | - name: Publish Docker Image 17 | uses: openzim/docker-publish-action@v10 18 | with: 19 | image-name: kiwix/bittorrent-seeder 20 | on-master: latest 21 | restrict-to: kiwix/seeder 22 | registries: ghcr.io 23 | credentials: 24 | GHCRIO_USERNAME=${{ secrets.GHCR_USERNAME }} 25 | GHCRIO_TOKEN=${{ secrets.GHCR_TOKEN }} 26 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: kiwix # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] 4 | patreon: # Replace with a single Patreon username 5 | open_collective: # Replace with a single Open Collective username 6 | ko_fi: # Replace with a single Ko-fi username 7 | tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry 9 | liberapay: # Replace with a single Liberapay username 10 | issuehunt: # Replace with a single IssueHunt username 11 | otechie: # Replace with a single Otechie username 12 | custom: # https://kiwix.org/support-us/ 13 | -------------------------------------------------------------------------------- /.github/workflows/QA.yml: -------------------------------------------------------------------------------- 1 | name: QA 2 | 3 | on: 4 | pull_request: 5 | push: 6 | branches: 7 | - main 8 | 9 | jobs: 10 | check-qa: 11 | runs-on: ubuntu-24.04 12 | 13 | steps: 14 | - uses: actions/checkout@v4 15 | 16 | - name: Set up Python 17 | uses: actions/setup-python@v5 18 | with: 19 | python-version-file: pyproject.toml 20 | architecture: x64 21 | 22 | - name: Install dependencies (and project) 23 | run: | 24 | pip install -U pip 25 | pip install -e .[lint,scripts,test,check] 26 | 27 | - name: Check black formatting 28 | run: inv lint-black 29 | 30 | - name: Check ruff 31 | run: inv lint-ruff 32 | 33 | - name: Check pyright 34 | run: inv check-pyright 35 | -------------------------------------------------------------------------------- /get-pbkdf2.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ command line to generate a base64-encoded PBKDF2 version of a password 4 | 5 | similar to how qBittorrent stores web-ui password in its config file. 6 | Format is salt:HMAC""" 7 | 8 | import base64 9 | import hashlib 10 | import secrets 11 | import sys 12 | 13 | 14 | def asb64(data: bytes) -> str: 15 | return base64.b64encode(data).decode("ASCII") 16 | 17 | 18 | def get_pbkdf2_for(password: str) -> str: 19 | salt = secrets.token_bytes(16) 20 | hmac = hashlib.pbkdf2_hmac( 21 | hash_name="sha512", 22 | password=password.encode("UTF-8"), 23 | salt=salt, 24 | iterations=100000, 25 | ) 26 | return f"{asb64(salt)}:{asb64(hmac)}" 27 | 28 | 29 | def main() -> int: 30 | if len(sys.argv) != 2: # noqa: PLR2004 31 | print(f"Usage: {sys.argv[0]} CLEAR_PASSWORD") # noqa: T201 32 | return 1 33 | print(get_pbkdf2_for(sys.argv[1])) # noqa: T201 34 | return 0 35 | 36 | 37 | if __name__ == "__main__": 38 | sys.exit(main()) 39 | -------------------------------------------------------------------------------- /.github/workflows/Tests.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: 4 | pull_request: 5 | push: 6 | branches: 7 | - main 8 | 9 | jobs: 10 | run-tests: 11 | strategy: 12 | matrix: 13 | os: [ubuntu-24.04] 14 | python: ["3.12", "3.13"] 15 | runs-on: ${{ matrix.os }} 16 | 17 | steps: 18 | - uses: actions/checkout@v4 19 | 20 | - name: Set up Python ${{ matrix.python }} 21 | uses: actions/setup-python@v5 22 | with: 23 | python-version: ${{ matrix.python }} 24 | architecture: x64 25 | 26 | - name: Install dependencies (and project) 27 | run: | 28 | pip install -U pip 29 | pip install -e .[test,scripts] 30 | 31 | - name: Run the tests 32 | run: inv coverage --args "-vvv" 33 | 34 | - name: Upload coverage report to codecov 35 | if: matrix.python == '3.12' 36 | uses: codecov/codecov-action@v4 37 | with: 38 | fail_ci_if_error: true 39 | token: ${{ secrets.CODECOV_TOKEN }} 40 | 41 | build_python: 42 | runs-on: ubuntu-24.04 43 | steps: 44 | - uses: actions/checkout@v4 45 | 46 | - name: Set up Python 47 | uses: actions/setup-python@v5 48 | with: 49 | python-version-file: pyproject.toml 50 | architecture: x64 51 | 52 | - name: Ensure we can build Python targets 53 | run: | 54 | pip install -U pip build 55 | python3 -m build --sdist --wheel 56 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.12-slim-bookworm 2 | LABEL org.opencontainers.image.source=https://github.com/kiwix/seeder 3 | 4 | ENV SHELL=bash 5 | 6 | RUN set -e \ 7 | && apt-get update \ 8 | && apt-get install -y --no-install-recommends \ 9 | dumb-init curl apt-transport-https ca-certificates gnupg2 \ 10 | # the daemon with webui \ 11 | && curl -L -o /usr/bin/qbittorrent-nox https://github.com/userdocs/qbittorrent-nox-static/releases/download/release-5.0.4_v2.0.11/x86_64-qbittorrent-nox \ 12 | && chmod +x /usr/bin/qbittorrent-nox \ 13 | # for convenience (qBittorrent-cli) 14 | && curl -L https://dl.cloudsmith.io/public/qbittorrent-cli/qbittorrent-cli/gpg.F8756541ADDA2B7D.key | apt-key add - \ 15 | && curl -L -o /etc/apt/sources.list.d/qbittorrent-cli.list https://repos.fedarovich.com/debian/bookworm/qbittorrent-cli.list \ 16 | && apt-get update \ 17 | && apt-get install -y --no-install-recommends qbittorrent-cli 18 | 19 | ENV NO_QBT="" 20 | ENV QBT_TORRENTING_PORT=6901 21 | ENV QBT_HOST=localhost 22 | ENV QBT_SCHEME=http 23 | ENV QBT_PORT=80 24 | ENV QBT_USERNAME=admin 25 | ENV QBT_PASSWORD= 26 | ENV WEBUI_ADDRESS=0.0.0.0 27 | 28 | ENV QBT_MAX_CONNECTIONS=500 29 | ENV QBT_MAX_CONNECTIONS_PER_TORRENT=100 30 | ENV QBT_MAX_UPLOADS=20 31 | ENV QBT_MAX_UPLOADS_PER_TORRENT=5 32 | ENV QBT_MAX_ACTIVE_CHECKING_TORRENTS=1 33 | 34 | # pyproject.toml and its dependencies 35 | COPY README.md /src/ 36 | COPY pyproject.toml README.md tasks.py /src/ 37 | COPY src/kiwixseeder/__about__.py /src/src/kiwixseeder/__about__.py 38 | # install python dependencies 39 | RUN pip install --no-cache-dir --break-system-packages /src/ 40 | 41 | COPY src/ /src/src 42 | RUN set -e \ 43 | && pip install --break-system-packages /src/ \ 44 | && kiwix-seeder --help 45 | 46 | COPY entrypoint.sh /usr/local/bin/entrypoint 47 | COPY gen-password.py /usr/local/bin/gen-password 48 | COPY get-pbkdf2.py /usr/local/bin/get-pbkdf2 49 | 50 | EXPOSE 80 51 | EXPOSE 6901 52 | VOLUME /root/.config/qBittorrent 53 | VOLUME /root/.local/share/qBittorrent 54 | VOLUME /data 55 | WORKDIR /data 56 | 57 | ENTRYPOINT ["/usr/bin/dumb-init", "--", "/usr/local/bin/entrypoint"] 58 | CMD ["kiwix-seeder-loop"] 59 | -------------------------------------------------------------------------------- /src/kiwixseeder/utils.py: -------------------------------------------------------------------------------- 1 | import time 2 | from dataclasses import dataclass 3 | 4 | import humanfriendly 5 | 6 | 7 | def format_size(value: int) -> str: 8 | """human friendly size in binary""" 9 | return humanfriendly.format_size(value, binary=True) 10 | 11 | 12 | def format_duration(value: float) -> str: 13 | """human friendly duration""" 14 | return humanfriendly.format_timespan(value) 15 | 16 | 17 | def sleep_nonblocking(seconds: int | float): 18 | for _ in range(0, int(seconds)): 19 | time.sleep(1) 20 | 21 | 22 | nd = 0 23 | 24 | 25 | @dataclass(kw_only=True) 26 | class SizeRange: 27 | """Size Range calculator ensuring min and max are usable (both optional)""" 28 | 29 | minimum: int = nd 30 | maximum: int = nd 31 | 32 | def is_valid(self) -> bool: 33 | """whether range is usable or not""" 34 | if self.minimum == self.maximum == nd: 35 | return True 36 | # maximum is either not set or positive 37 | if self.maximum != nd: 38 | return max(self.maximum, 0) >= max(self.minimum, 0) 39 | return True 40 | 41 | def is_above_min(self, value: int) -> bool: 42 | """whether value is greater-or-equal than our minimum""" 43 | return value >= max(self.minimum, 0) 44 | 45 | def is_below_max(self, value: int) -> bool: 46 | """whether value is lower-or-equal than our maximum""" 47 | return self.maximum == nd or value <= self.maximum 48 | 49 | def match(self, value: int) -> bool: 50 | """whether value is within the bounds of the range""" 51 | # not valid, not matching. 52 | if not self.is_valid(): 53 | return False 54 | # no bound, always OK 55 | if self.minimum == self.maximum == nd: 56 | return True 57 | return self.is_above_min(value) and self.is_below_max(value) 58 | 59 | def __str__(self) -> str: 60 | if not self.is_valid(): 61 | return f"Invalid range: min={self.minimum}, max={self.maximum}" 62 | if self.minimum == self.maximum == nd: 63 | return "all" 64 | if self.minimum == self.maximum: 65 | return f"exactly {format_size(self.maximum)}" 66 | if self.minimum == nd: 67 | return f"below {format_size(self.maximum)}" 68 | if self.maximum == nd: 69 | return f"above {format_size(self.minimum)}" 70 | return f"between {format_size(self.minimum)} and {format_size(self.maximum)}" 71 | -------------------------------------------------------------------------------- /src/kiwixseeder/forever.py: -------------------------------------------------------------------------------- 1 | """ forever running assistant to kiwix-seeder 2 | 3 | Provides a long-living process for Docker usage (as CMD) that periodically 4 | launches kiwix-seeder. 5 | 6 | Entirely driven by ENV (DEBUG, SLEEP_INTERVAL), it launches the regular kiwix-seeder 7 | as a subprocess to prevent any failure in it from breaking the loop""" 8 | 9 | import signal 10 | import subprocess 11 | import sys 12 | import time 13 | from types import FrameType 14 | 15 | from kiwixseeder.context import RC_INSUFFICIENT_STORAGE, RC_NOFILTER, Context 16 | from kiwixseeder.utils import format_duration 17 | 18 | Context.setup_logger() 19 | logger = Context.logger 20 | 21 | 22 | def main(args: list[str]) -> int: 23 | logger.info("[forever] Starting kiwix-seeder runner") 24 | 25 | exit_requested: bool = False 26 | 27 | def exit_gracefully(signum: int, frame: FrameType | None): # noqa: ARG001 28 | exit_requested = True # noqa: F841 # pyright: ignore [reportUnusedVariable] 29 | logger.info( 30 | f"[forever] Received {signal.Signals(signum).name}/{signum}. Exiting" 31 | ) 32 | sys.exit(-signum) 33 | 34 | signal.signal(signal.SIGTERM, exit_gracefully) 35 | signal.signal(signal.SIGINT, exit_gracefully) 36 | signal.signal(signal.SIGQUIT, exit_gracefully) 37 | 38 | first_launch = True 39 | while not exit_requested: 40 | # clear the OPDS cache on forever start 41 | # but only set it for subsequent calls if requested. 42 | # clearing on start makes most sense 43 | if first_launch and "--clear-opds" not in args and "-C" not in args: 44 | args += ["--clear-opds"] 45 | first_launch = False 46 | 47 | ps = subprocess.run(["/usr/bin/env", "kiwix-seeder", *args], check=False) 48 | 49 | if ps.returncode in (RC_NOFILTER, RC_INSUFFICIENT_STORAGE): 50 | logger.info("OK, there's a config issue here. Exiting forever loop") 51 | return ps.returncode 52 | 53 | if ps.returncode < 0: 54 | return ps.returncode 55 | 56 | if exit_requested: 57 | return 0 58 | 59 | logger.info(f"Sleeping for {format_duration(Context.sleep_interval)}…") 60 | for _ in range(0, int(Context.sleep_interval)): 61 | time.sleep(1) 62 | if exit_requested: 63 | break 64 | 65 | return 0 66 | 67 | 68 | def entrypoint(): 69 | sys.exit(main(sys.argv[1:])) 70 | 71 | 72 | if __name__ == "__main__": 73 | entrypoint() 74 | -------------------------------------------------------------------------------- /entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [[ "${WEBUI_SSL}" = "y" ]]; then 4 | export QBT_INSECURE=y 5 | export QBT_SCHEME=https 6 | fi 7 | 8 | function configure_qbt { 9 | 10 | QBT_HOST="${QBT_HOST:-localhost}" 11 | QBT_PORT="${QBT_PORT:-80}" 12 | QBT_USERNAME="${QBT_USERNAME:-admin}" 13 | 14 | # WEBUI_SSL=y enables SSL on WEB_UI and --insecure on kiwix-seeder 15 | if [[ "${WEBUI_SSL}" = "y" ]]; then 16 | websslvalue="true" 17 | webuischeme="https" 18 | else 19 | webuischeme="http" 20 | websslvalue="false" 21 | fi 22 | 23 | # configure qbittorrent-cli (qbt) 24 | if [[ "${WEBUI_SSL}" = "y" ]]; then 25 | qbt network settings --ignore-certificate-errors TRUE 26 | fi 27 | qbt settings set url "${webuischeme}://${QBT_HOST}:${QBT_PORT}/" 28 | qbt settings set username "${QBT_USERNAME}" 29 | echo "${QBT_PASSWORD}" | qbt settings set password -y 30 | 31 | QBT_CONFIG_FILE=/root/.config/qBittorrent/qBittorrent.conf 32 | if [ -f "$QBT_CONFIG_FILE" ] ; then 33 | echo "Found existing qBittorrent config file at $QBT_CONFIG_FILE" 34 | echo "Assuming persistent installation ; skipping configuration." 35 | return 36 | fi 37 | 38 | QBT_TORRENTING_PORT="${QBT_TORRENTING_PORT:-6901}" 39 | 40 | QBT_MAX_CONNECTIONS="${QBT_MAX_CONNECTIONS:-500}" 41 | QBT_MAX_CONNECTIONS_PER_TORRENT="${QBT_MAX_CONNECTIONS_PER_TORRENT:-100}" 42 | QBT_MAX_UPLOADS="${QBT_MAX_UPLOADS:-20}" 43 | QBT_MAX_UPLOADS_PER_TORRENT="${QBT_MAX_UPLOADS_PER_TORRENT:-5}" 44 | QBT_MAX_ACTIVE_CHECKING_TORRENTS="${QBT_MAX_ACTIVE_CHECKING_TORRENTS:-1}" 45 | 46 | if [ "x${QBT_PASSWORD}" = "x" ]; then 47 | QBT_PASSWORD=$(gen-password) 48 | echo "Generated web-ui password: ${QBT_PASSWORD}" 49 | fi 50 | PKBF2_PASSWORD=$(get-pbkdf2 "${QBT_PASSWORD}") 51 | 52 | mkdir -p $(dirname $QBT_CONFIG_FILE) 53 | 54 | cat < $QBT_CONFIG_FILE 55 | [BitTorrent] 56 | MergeTrackersEnabled=true 57 | Session\DefaultSavePath=/data/files 58 | Session\AddExtensionToIncompleteFiles=true 59 | Session\MaxConnections=${QBT_MAX_CONNECTIONS} 60 | Session\MaxConnectionsPerTorrent=${QBT_MAX_CONNECTIONS_PER_TORRENT} 61 | Session\MaxUploads=${QBT_MAX_UPLOADS} 62 | Session\MaxUploadsPerTorrent=${QBT_MAX_UPLOADS_PER_TORRENT} 63 | Session\Port=${QBT_TORRENTING_PORT} 64 | Session\Preallocation=true 65 | Session\QueueingSystemEnabled=false 66 | Session\SSL\Port=30154 67 | Session\MaxActiveCheckingTorrents=${QBT_MAX_ACTIVE_CHECKING_TORRENTS} 68 | 69 | [LegalNotice] 70 | Accepted=true 71 | 72 | [Meta] 73 | MigrationVersion=8 74 | 75 | [Preferences] 76 | General\Locale=en 77 | WebUI\Enabled=true 78 | WebUI\HTTPS\Enabled=${websslvalue} 79 | WebUI\HTTPS\CertificatePath=${WEBUI_SSL_CERT} 80 | WebUI\HTTPS\KeyPath=${WEBUI_SSL_KEY} 81 | WebUI\Address=${WEBUI_ADDRESS} 82 | WebUI\Port=${QBT_PORT} 83 | WebUI\Username=${QBT_USERNAME} 84 | WebUI\Password_PBKDF2="@ByteArray(${PKBF2_PASSWORD})" 85 | WebUI\LocalHostAuth=true 86 | WebUI\HostHeaderValidation=false 87 | WebUI\CSRFProtection=false 88 | 89 | [Core] 90 | AutoDeleteAddedTorrentFile=Always 91 | 92 | [Application] 93 | FileLogger\Age=5 94 | FileLogger\AgeType=0 95 | FileLogger\Backup=true 96 | FileLogger\DeleteOld=true 97 | FileLogger\Enabled=true 98 | FileLogger\MaxSizeBytes=1048576 99 | FileLogger\Path=/data/log 100 | GUI\Notifications\TorrentAdded=false 101 | 102 | EOF 103 | } 104 | 105 | 106 | if [ "x${NO_QBT}" = "x" ]; then 107 | configure_qbt 108 | echo "Starting a qbittorrent-nox process (set NO_QBT if you dont want to)" 109 | /usr/bin/qbittorrent-nox --daemon 110 | 111 | # give a few seconds to qBittorrent to start before staring the loop 112 | # as the loop has no fancy retry mechanism ATM and will wait sleep_interval (long) 113 | # before re-trying 114 | sleep 3 115 | fi 116 | 117 | 118 | exec "$@" 119 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # UV 98 | # Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | #uv.lock 102 | 103 | # poetry 104 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 105 | # This is especially recommended for binary packages to ensure reproducibility, and is more 106 | # commonly ignored for libraries. 107 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 108 | #poetry.lock 109 | 110 | # pdm 111 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 112 | #pdm.lock 113 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 114 | # in version control. 115 | # https://pdm.fming.dev/latest/usage/project/#working-with-version-control 116 | .pdm.toml 117 | .pdm-python 118 | .pdm-build/ 119 | 120 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 121 | __pypackages__/ 122 | 123 | # Celery stuff 124 | celerybeat-schedule 125 | celerybeat.pid 126 | 127 | # SageMath parsed files 128 | *.sage.py 129 | 130 | # Environments 131 | .env 132 | .venv 133 | env/ 134 | venv/ 135 | ENV/ 136 | env.bak/ 137 | venv.bak/ 138 | 139 | # Spyder project settings 140 | .spyderproject 141 | .spyproject 142 | 143 | # Rope project settings 144 | .ropeproject 145 | 146 | # mkdocs documentation 147 | /site 148 | 149 | # mypy 150 | .mypy_cache/ 151 | .dmypy.json 152 | dmypy.json 153 | 154 | # Pyre type checker 155 | .pyre/ 156 | 157 | # pytype static type analyzer 158 | .pytype/ 159 | 160 | # Cython debug symbols 161 | cython_debug/ 162 | 163 | # PyCharm 164 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 165 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 166 | # and can be added to the global gitignore or merged into this file. For a more nuclear 167 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 168 | #.idea/ 169 | 170 | # PyPI configuration file 171 | .pypirc 172 | 173 | # locally built binary 174 | /kiwix-seeder_darwin_* 175 | /kiwix-seeder_linux_* 176 | /kiwix-seeder_win32_* 177 | -------------------------------------------------------------------------------- /src/kiwixseeder/qbittorrent.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import time 3 | from dataclasses import dataclass 4 | from typing import Self 5 | 6 | import qbittorrentapi 7 | 8 | from kiwixseeder.context import QBT_CAT_NAME, Context 9 | from kiwixseeder.download import get_btih_from_url 10 | from kiwixseeder.library import Book 11 | from kiwixseeder.utils import format_size 12 | 13 | context = Context.get() 14 | client = context.qbt 15 | logger = context.logger 16 | 17 | 18 | @dataclass(kw_only=True) 19 | class TorrentInfo: 20 | """Custom backend-agnostic torrent info""" 21 | 22 | btih: str 23 | filename: str 24 | added_on: datetime.datetime 25 | size: int 26 | 27 | @classmethod 28 | def from_torrentdictionary(cls, tdict: qbittorrentapi.TorrentDictionary) -> Self: 29 | return cls( 30 | btih=tdict.properties.hash, 31 | filename=tdict.properties.name, 32 | added_on=datetime.datetime.fromtimestamp( 33 | tdict.properties.addition_date, tz=datetime.UTC 34 | ), 35 | size=tdict.properties.total_size, 36 | ) 37 | 38 | def __str__(self) -> str: 39 | return ( 40 | f"{self.filename} @ {self.btih} " # noqa: RUF001 41 | f"({format_size(self.size)})" 42 | ) 43 | 44 | 45 | class TorrentManager: 46 | 47 | def __init__(self) -> None: 48 | # maps {ident: str} to {btih: str} 49 | self.btihs: dict[str, str] = {} 50 | 51 | def is_connected(self) -> tuple[bool, str | Exception]: 52 | """whether qbittorrent is reachable and either version or exception""" 53 | try: 54 | return True, client.app_version() 55 | except Exception as exc: 56 | return False, exc 57 | 58 | def setup(self): 59 | # ensure we have our category 60 | if QBT_CAT_NAME not in client.torrent_categories.categories: 61 | client.torrent_categories.create_category(name=QBT_CAT_NAME) 62 | 63 | def reload(self): 64 | """read torrents list from qbittorrent""" 65 | self.btihs.clear() 66 | for torrent in client.torrents.info(category=QBT_CAT_NAME): 67 | self.btihs[torrent.properties.hash] = torrent.properties.name 68 | 69 | @property 70 | def nb_torrents(self) -> int: 71 | return len(self.btihs) 72 | 73 | def add(self, book: Book) -> bool: 74 | btih = self.add_url(url=book.torrent_url, btih=book.btih) 75 | if not btih: 76 | return False 77 | if book.btih != btih: 78 | book.btih = btih 79 | return True 80 | 81 | def add_url(self, url: str, btih: str | None) -> str: 82 | # upload_limit 83 | # download_limit 84 | # save_path 85 | # ratio_limit 86 | # seeding_time_limit 87 | # download_path 88 | try: 89 | btih = btih or get_btih_from_url(url) 90 | if client.torrents.add( 91 | urls=url, category=QBT_CAT_NAME 92 | ) == "Ok." and self.get_or_none(btih, with_patience=True): 93 | return btih 94 | raise OSError(f"Failed to add torrent for {url}") 95 | finally: 96 | self.reload() 97 | 98 | def get(self, ident: str) -> TorrentInfo: 99 | """Torrent dict from its hash""" 100 | return TorrentInfo.from_torrentdictionary( 101 | client.torrents.info(torrent_hashes=ident)[0] 102 | ) 103 | 104 | def get_or_none( 105 | self, ident: str, *, with_patience: bool = False 106 | ) -> TorrentInfo | None: 107 | """Torrent dict from its hash or None""" 108 | if with_patience: 109 | attempts = 100 110 | duration = 0.1 111 | else: 112 | attempts = 1 113 | duration = 0 114 | while attempts: 115 | attempts -= 1 116 | try: 117 | return self.get(ident) 118 | except IndexError: 119 | time.sleep(duration) 120 | continue 121 | return None 122 | 123 | def remove(self, ident: str) -> bool: 124 | """Remove a single torrent (if present) via its hash""" 125 | try: 126 | client.torrents.delete(torrent_hashes=ident, delete_files=True) 127 | finally: 128 | self.reload() 129 | return ident not in self.btihs 130 | 131 | @property 132 | def total_size(self) -> int: 133 | """total size of our torrents""" 134 | self.reload() 135 | return sum(self.get(btih).size for btih in self.btihs) 136 | -------------------------------------------------------------------------------- /tasks.py: -------------------------------------------------------------------------------- 1 | # pyright: strict, reportUntypedFunctionDecorator=false 2 | import os 3 | import pathlib 4 | import platform 5 | import shlex 6 | import sys 7 | 8 | from invoke.context import Context 9 | from invoke.tasks import task # pyright: ignore [reportUnknownVariableType] 10 | 11 | from kiwixseeder.__about__ import __version__ 12 | 13 | use_pty = not os.getenv("CI", "") 14 | 15 | 16 | @task(optional=["args"], help={"args": "pytest additional arguments"}) 17 | def test(ctx: Context, args: str = ""): 18 | """run tests (without coverage)""" 19 | ctx.run(f"pytest {args}", pty=use_pty) 20 | 21 | 22 | @task(optional=["args"], help={"args": "pytest additional arguments"}) 23 | def test_cov(ctx: Context, args: str = ""): 24 | """run test vith coverage""" 25 | ctx.run(f"coverage run -m pytest {args}", pty=use_pty) 26 | 27 | 28 | @task(optional=["html"], help={"html": "flag to export html report"}) 29 | def report_cov(ctx: Context, *, html: bool = False): 30 | """report coverage""" 31 | ctx.run("coverage combine", warn=True, pty=use_pty) 32 | ctx.run("coverage report --show-missing", pty=use_pty) 33 | ctx.run("coverage xml", pty=use_pty) 34 | if html: 35 | ctx.run("coverage html", pty=use_pty) 36 | 37 | 38 | @task( 39 | optional=["args", "html"], 40 | help={ 41 | "args": "pytest additional arguments", 42 | "html": "flag to export html report", 43 | }, 44 | ) 45 | def coverage(ctx: Context, args: str = "", *, html: bool = False): 46 | """run tests and report coverage""" 47 | test_cov(ctx, args=args) 48 | report_cov(ctx, html=html) 49 | 50 | 51 | @task(optional=["args"], help={"args": "black additional arguments"}) 52 | def lint_black(ctx: Context, args: str = "."): 53 | args = args or "." # needed for hatch script 54 | ctx.run("black --version", pty=use_pty) 55 | ctx.run(f"black --check --diff {args}", pty=use_pty) 56 | 57 | 58 | @task(optional=["args"], help={"args": "ruff additional arguments"}) 59 | def lint_ruff(ctx: Context, args: str = "."): 60 | args = args or "." # needed for hatch script 61 | ctx.run("ruff --version", pty=use_pty) 62 | ctx.run(f"ruff check {args}", pty=use_pty) 63 | 64 | 65 | @task( 66 | optional=["args"], 67 | help={ 68 | "args": "linting tools (black, ruff) additional arguments, typically a path", 69 | }, 70 | ) 71 | def lintall(ctx: Context, args: str = "."): 72 | """Check linting""" 73 | args = args or "." # needed for hatch script 74 | lint_black(ctx, args) 75 | lint_ruff(ctx, args) 76 | 77 | 78 | @task(optional=["args"], help={"args": "check tools (pyright) additional arguments"}) 79 | def check_pyright(ctx: Context, args: str = ""): 80 | """check static types with pyright""" 81 | ctx.run("pyright --version") 82 | ctx.run(f"pyright {args}", pty=use_pty) 83 | 84 | 85 | @task(optional=["args"], help={"args": "check tools (pyright) additional arguments"}) 86 | def checkall(ctx: Context, args: str = ""): 87 | """check static types""" 88 | check_pyright(ctx, args) 89 | 90 | 91 | @task(optional=["args"], help={"args": "black additional arguments"}) 92 | def fix_black(ctx: Context, args: str = "."): 93 | """fix black formatting""" 94 | args = args or "." # needed for hatch script 95 | ctx.run(f"black {args}", pty=use_pty) 96 | 97 | 98 | @task(optional=["args"], help={"args": "ruff additional arguments"}) 99 | def fix_ruff(ctx: Context, args: str = "."): 100 | """fix all ruff rules""" 101 | args = args or "." # needed for hatch script 102 | ctx.run(f"ruff check --fix {args}", pty=use_pty) 103 | 104 | 105 | @task( 106 | optional=["args"], 107 | help={ 108 | "args": "linting tools (black, ruff) additional arguments, typically a path", 109 | }, 110 | ) 111 | def fixall(ctx: Context, args: str = "."): 112 | """Fix everything automatically""" 113 | args = args or "." # needed for hatch script 114 | fix_black(ctx, args) 115 | fix_ruff(ctx, args) 116 | lintall(ctx, args) 117 | 118 | 119 | @task( 120 | optional=["filename", "compress"], 121 | help={ 122 | "filename": "output filename or fullname for the output binary", 123 | "no-compress": "dont zstd-compress binary (faster startup on macOS)", 124 | }, 125 | ) 126 | def binary(ctx: Context, filename: str = "", *, no_compress: bool = False): 127 | """build a standalone binary executable with nuitka""" 128 | fpath = ( 129 | pathlib.Path( 130 | filename 131 | or f"kiwix-seeder_{platform.system().lower()}" 132 | f"_{__version__}{'-nc' if no_compress else ''}" 133 | ) 134 | .expanduser() 135 | .resolve() 136 | ) 137 | fpath.parent.mkdir(parents=True, exist_ok=True) 138 | pyexe = shlex.quote(sys.executable) 139 | 140 | command = [ 141 | str(pyexe), 142 | "-m", 143 | "nuitka", 144 | "--onefile", 145 | "--python-flag=no_site,no_asserts,no_docstrings", 146 | "--include-package=kiwixseeder", 147 | "--user-package-configuration-file=kiwixseeder.config.yml", 148 | "--show-modules", 149 | "--warn-implicit-exceptions", 150 | "--warn-unusual-code", 151 | "--assume-yes-for-downloads", 152 | f'--output-dir="{fpath.parent!s}"', 153 | f'--output-filename="{fpath.name}"', 154 | "--remove-output", 155 | "--no-progressbar", 156 | ] 157 | if no_compress: 158 | command.append("--onefile-no-compression") 159 | command.append("src/kiwixseeder/") 160 | ctx.run(" ".join(command)) 161 | -------------------------------------------------------------------------------- /seeder-start-restart.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Start-restart script for Kiwix-seeder 4 | # 5 | # Assumes an optional (yet recommended) config file in /etc/seeder.config 6 | # to overwrite the following variables. 7 | 8 | CONTAINER_NAME="seeder" # name of docker container 9 | IMAGE="ghcr.io/kiwix/bittorrent-seeder:latest" # docker image to use 10 | 11 | DATA_PATH=$(pwd)/kiwix-seeder # path to store ZIM files (and incomplete .!qB ones) in (there's no hierarchy) 12 | MAX_STORAGE="10GiB" # maximum disk-space to use 13 | SLEEP_INTERVAL="1d" # how long to pause in-between catalog checks 14 | DEBUG="" # whether to print debug logs (set to 1 to enable) 15 | SEED_WHOLE_CATALOG= # whether to continue if filters (or lack of) end up seeding the whole catalog (prevents accidental no-filter launch) 16 | PERSIST_QBT_CONFIG= # `y` to reuse (ie. not overwrite) an existing qBittorrent config. This means changes to config via WebUI persists for instance. 17 | 18 | # 19 | # FILTERS 20 | # - use glob-patterns 21 | # - split using pipe (|) 22 | # See usage (kiwix-seeder --help) for details 23 | FILENAMES="" # matching filename (filepath actually) 24 | LANGUAGES="" # matching language (ISO-639-3 language codes) 25 | CATEGORIES="" # matching Category 26 | FLAVOURS="" # matching Flavour metadata (mini, nopic, maxi) 27 | TAGS="" # containing following tag(s) 28 | TITLES="" # matching Title metadata 29 | DESCRIPTIONS="" # matching Description metadata 30 | AUTHORS="" # matching Creator metadata 31 | PUBLISHERS="" # matching Publisher metadata 32 | MIN_SIZE="" # only ZIM larger than 33 | MAX_SIZE="" # only ZIM smaller than 34 | 35 | # the following applies to those using the in-container qBittorrent 36 | # If using a remote qBittorrent instance, see NO_DEAMON below 37 | QBT_TORRENTING_PORT=6901 # port to use for bittorrent. **MUST** be manually opened and forwarded to this host's IP as uPNP would not work accross docker 38 | QBT_PASSWORD="" # qBittorrent WebUI password. If empty, one will be gen and printed 39 | WEBUI_PORT=8000 # port on this host to map to the qBittorrent WebUI (so you can monitor it). Empty to not expose Web UI 40 | WEBUI_SSL= # `y` to make webui https instead of http 41 | WEBUI_SSL_CERT=localhost.crt # filename (inside $DATA_PATH/certs) to the certificate to use for webui (if ssl enabled) 42 | WEBUI_SSL_KEY=localhost.key # filename (inside $DATA_PATH/certs) to the certificate key to use for webui (if ssl enabled) 43 | # qBittorrent connection settings (defaults copied from qBittorrent) 44 | QBT_MAX_CONNECTIONS=500 45 | QBT_MAX_CONNECTIONS_PER_TORRENT=100 46 | QBT_MAX_UPLOADS=20 47 | QBT_MAX_UPLOADS_PER_TORRENT=5 48 | QBT_MAX_ACTIVE_CHECKING_TORRENTS=1 49 | # END OF CONFIG 50 | 51 | if [ -f ~/.config/seeder.config ]; then 52 | source ~/.config/seeder.config 53 | elif [ -f /etc/seeder.config ]; then 54 | source /etc/seeder.config 55 | fi 56 | 57 | # already running? 58 | docker ps |grep $CONTAINER_NAME |awk '{print $1}' | while read line ; do 59 | echo ">stopping seeder container $line" 60 | docker stop $line 61 | echo ">removing seeder container $line" 62 | docker rm $line 63 | done 64 | 65 | docker stop $CONTAINER_NAME 66 | docker rm --force $CONTAINER_NAME 67 | 68 | echo ">pulling image $IMAGE…" 69 | docker pull $IMAGE 70 | 71 | docker_run_opts= 72 | if [[ "${PERSIST_QBT_CONFIG}" = "y" ]]; then 73 | docker_run_opts+=" -v ${DATA_PATH}/qbittorrent-config:/root/.config/qBittorrent" 74 | fi 75 | 76 | if [[ "${WEBUI_PORT}x" != "x" ]]; then 77 | docker_run_opts+=" -p ${WEBUI_PORT}:80" 78 | fi 79 | 80 | echo ">starting seeder container" 81 | docker run \ 82 | --name $CONTAINER_NAME \ 83 | -v $DATA_PATH:/data \ 84 | -v $DATA_PATH/cache:/root/.config/kiwix-seeder \ 85 | -v $DATA_PATH/qbittorrent-cache:/root/.cache/qBittorrent \ 86 | -v $DATA_PATH/qbittorrent-share:/root/.local/share/qBittorrent \ 87 | -v $DATA_PATH/certs:/certs \ 88 | $docker_run_opts \ 89 | -p $QBT_TORRENTING_PORT:$QBT_TORRENTING_PORT \ 90 | -p $QBT_TORRENTING_PORT:$QBT_TORRENTING_PORT/udp \ 91 | -e DEBUG="${DEBUG}" \ 92 | -e SEED_WHOLE_CATALOG="${SEED_WHOLE_CATALOG}" \ 93 | -e FILENAMES="${FILENAMES}" \ 94 | -e LANGUAGES="${LANGUAGES}" \ 95 | -e CATEGORIES="${CATEGORIES}" \ 96 | -e FLAVOURS="${FLAVOURS}" \ 97 | -e TAGS="${TAGS}" \ 98 | -e AUTHORS="${AUTHORS}" \ 99 | -e PUBLISHERS="${PUBLISHERS}" \ 100 | -e MIN_SIZE="${MIN_SIZE}" \ 101 | -e MAX_SIZE="${MAX_SIZE}" \ 102 | -e QBT_PASSWORD="${QBT_PASSWORD}" \ 103 | -e QBT_TORRENTING_PORT="${QBT_TORRENTING_PORT}" \ 104 | -e QBT_MAX_CONNECTIONS="${QBT_MAX_CONNECTIONS}" \ 105 | -e QBT_MAX_CONNECTIONS_PER_TORRENT="${QBT_MAX_CONNECTIONS_PER_TORRENT}" \ 106 | -e QBT_MAX_UPLOADS="${QBT_MAX_UPLOADS}" \ 107 | -e QBT_MAX_UPLOADS_PER_TORRENT="${QBT_MAX_UPLOADS_PER_TORRENT}" \ 108 | -e QBT_MAX_ACTIVE_CHECKING_TORRENTS="${QBT_MAX_ACTIVE_CHECKING_TORRENTS}" \ 109 | -e MAX_STORAGE="${MAX_STORAGE}" \ 110 | -e SLEEP_INTERVAL="${SLEEP_INTERVAL}" \ 111 | -e WEBUI_SSL="${WEBUI_SSL}" \ 112 | -e WEBUI_SSL_CERT="/certs/${WEBUI_SSL_CERT}" \ 113 | -e WEBUI_SSL_KEY="/certs/${WEBUI_SSL_KEY}" \ 114 | --restart unless-stopped \ 115 | --detach \ 116 | -it \ 117 | $IMAGE 118 | -------------------------------------------------------------------------------- /src/kiwixseeder/download.py: -------------------------------------------------------------------------------- 1 | import io 2 | import re 3 | from http import HTTPStatus 4 | from pathlib import Path 5 | from urllib.parse import urlparse 6 | 7 | import requests 8 | import requests.adapters 9 | from urllib3.util.retry import Retry 10 | 11 | from kiwixseeder.context import Context 12 | 13 | session = requests.Session() 14 | # basic urllib retry mechanism. 15 | # Sleep (seconds): {backoff factor} * (2 ** ({number of total retries} - 1)) 16 | # https://docs.descarteslabs.com/_modules/urllib3/util/retry.html 17 | retries = Retry( 18 | total=10, # Total number of retries to allow. Takes precedence over other counts. 19 | connect=5, # How many connection-related errors to retry on 20 | read=5, # How many times to retry on read errors 21 | redirect=20, # How many redirects to perform. (to avoid infinite redirect loops) 22 | status=3, # How many times to retry on bad status codes 23 | other=0, # How many times to retry on other errors 24 | allowed_methods=None, # Set of HTTP verbs that we should retry on (False is all) 25 | status_forcelist=[ 26 | 413, 27 | 429, 28 | 500, 29 | 502, 30 | 503, 31 | 504, 32 | ], # Set of integer HTTP status we should force a retry on 33 | backoff_factor=30, # backoff factor to apply between attempts after the second try, 34 | backoff_max=1800.0, # allow up-to 30mn backoff (default 2mn) 35 | raise_on_redirect=False, # raise MaxRetryError instead of 3xx response 36 | raise_on_status=False, # raise on Bad Status or response 37 | respect_retry_after_header=True, # respect Retry-After header (status_forcelist) 38 | ) 39 | session.mount("http", requests.adapters.HTTPAdapter(max_retries=retries)) 40 | 41 | 42 | def get_online_rsc_size(url: str) -> int: 43 | """size (Content-Length) from url if specified, -1 otherwise (-2 on errors)""" 44 | try: 45 | resp = session.head(url, allow_redirects=True, timeout=60) 46 | # some servers dont offer HEAD 47 | if resp.status_code != HTTPStatus.OK: 48 | resp = session.get( 49 | url, 50 | allow_redirects=True, 51 | timeout=60, 52 | stream=True, 53 | headers={"Accept-Encoding": "identity"}, 54 | ) 55 | resp.raise_for_status() 56 | return int(resp.headers.get("Content-Length") or -1) 57 | except Exception: 58 | return -2 59 | 60 | 61 | def url_is_working(url: str) -> bool: 62 | """whether URL currently returns HTTP 200. Use to rule out 404 quickly""" 63 | resp = session.get(url, allow_redirects=True, timeout=60, stream=True) 64 | return resp.status_code == HTTPStatus.OK 65 | 66 | 67 | def get_payload_from( 68 | url: str, no_more_than: int = Context.max_direct_online_resource_payload_size 69 | ) -> bytes: 70 | """Retrieved content from an URL 71 | 72 | Limited in order to prevent download bomb. 73 | 74 | Parameters: 75 | url: URL to retrieve payload from (follows redirects) 76 | no_more_than: number of bytes to consider too much and fail at 77 | 78 | Raises: 79 | OSError: Should declared or retrieved size exceed no_more_than 80 | RequestException: HTTP or other error in requests 81 | ConnectionError: connection issues 82 | Timeout: ReadTimeout or request timeout""" 83 | size = get_online_rsc_size(url) 84 | if no_more_than and size > no_more_than: 85 | raise OSError(f"URL content is larger than {no_more_than!s}") 86 | 87 | resp = session.get(url, stream=True, allow_redirects=True, timeout=60) 88 | resp.raise_for_status() 89 | downloaded = 0 90 | payload = io.BytesIO() 91 | for data in resp.iter_content(2**30): 92 | downloaded += len(data) 93 | if no_more_than and downloaded > no_more_than: 94 | raise OSError(f"URL content is larger than {no_more_than!s}") 95 | payload.write(data) 96 | payload.seek(0) 97 | return payload.getvalue() 98 | 99 | 100 | def read_mirrorbrain_hash_from(url: str) -> str: 101 | """hashes from mirrorbrain-like (or raw) URL (checksums, btih) 102 | 103 | Format can be the raw digest or digest and filename: 104 | 9e92449ce93115e8d85e29e8e584dece wikipedia_ab_all_maxi_2024-02.zim 105 | 106 | Parameters: 107 | url: URL to read from. eg: download.kiwix.org/x/y/z.zim.sha1 108 | 109 | Raises: 110 | OSError: Should declared or retrieved size exceed no_more_than 111 | RequestException: HTTP or other error in requests 112 | ConnectionError: connection issues 113 | Timeout: ReadTimeout or request timeout 114 | UnicodeDecodeError: content cannot be decoded into ASCII 115 | UnicodeEncodeError: content cannot be encoded into UTF-8 116 | IndexError: content is empty or malformed 117 | """ 118 | return ( 119 | get_payload_from(url, no_more_than=2 * 2**10) 120 | .decode("UTF-8") 121 | .strip() 122 | .split(maxsplit=1)[0] 123 | .encode("UTF-8") 124 | .decode("ASCII") 125 | ) 126 | 127 | 128 | def get_btih_from_url(url: str) -> str: 129 | uri = urlparse(url) 130 | if uri.netloc != urlparse(Context.download_url).netloc: 131 | raise ValueError(f"btih from URL is reserved to {Context.download_url}") 132 | if not uri.path.endswith(".torrent"): 133 | raise ValueError( 134 | f"btih from URL is only for {Context.download_url}'s .torrent endpoint" 135 | ) 136 | # btih is 40-len but endpoint sends filename as well 137 | return read_mirrorbrain_hash_from(re.sub(r".torrent$", r".btih", url)) 138 | 139 | 140 | def get_pathname_from_url(url: str) -> Path: 141 | """Path of target file (a ZIM) from a torrent URL on DOWNLOAD_URL""" 142 | uri = urlparse(url) 143 | if uri.netloc != urlparse(Context.download_url).netloc: 144 | raise ValueError(f"path from URL is reserved to {Context.download_url}") 145 | if not uri.path.endswith(".torrent"): 146 | raise ValueError( 147 | f"path from URL is only for {Context.download_url}'s .torrent endpoint" 148 | ) 149 | return Path(re.sub(r".torrent$", r"", uri.path)) 150 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["hatchling"] 3 | build-backend = "hatchling.build" 4 | 5 | [project] 6 | name = "kiwix-seeder" 7 | requires-python = ">=3.12" 8 | description = "Bittorrent seeder for Kiwix ZIM Catalog" 9 | readme = "README.md" 10 | dependencies = [ 11 | "requests==2.32.3", 12 | "humanfriendly==10.0", 13 | "iso639-lang==2.5.1", 14 | "xmltodict==0.14.2", 15 | "qbittorrent-api==2024.12.71", 16 | "rich-argparse==1.6.0", 17 | ] 18 | authors = [ 19 | { name = "Kiwix", email = "dev@kiwix.org" }, 20 | ] 21 | keywords = ["kiwix", "bittorrent", "torrent", "seed"] 22 | license = {text = "GPL-3.0-or-later"} 23 | classifiers = [ 24 | "Programming Language :: Python :: 3.12", 25 | "Programming Language :: Python :: 3.13", 26 | "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", 27 | "Operating System :: POSIX", 28 | ] 29 | dynamic = ["version"] 30 | 31 | [project.optional-dependencies] 32 | scripts = [ 33 | "invoke==2.2.0", 34 | ] 35 | lint = [ 36 | "black==24.10.0", 37 | "ruff==0.8.4", 38 | ] 39 | check = [ 40 | "pyright==1.1.393", 41 | ] 42 | test = [ 43 | "pytest==8.3.4", 44 | "coverage==7.6.10", 45 | ] 46 | build = [ 47 | "nuitka==2.5.9", 48 | ] 49 | dev = [ 50 | "pre-commit==4.0.1", 51 | "ipython==8.31.0", 52 | "kiwix-seeder[scripts]", 53 | "kiwix-seeder[lint]", 54 | "kiwix-seeder[test]", 55 | "kiwix-seeder[check]", 56 | ] 57 | 58 | [project.scripts] 59 | kiwix-seeder = "kiwixseeder.entrypoint:entrypoint" 60 | kiwix-seeder-loop = "kiwixseeder.forever:entrypoint" 61 | 62 | [tool.hatch.version] 63 | path = "src/kiwixseeder/__about__.py" 64 | 65 | [tool.hatch.build] 66 | exclude = [ 67 | "/.github", 68 | ] 69 | 70 | [tool.hatch.build.targets.wheel] 71 | packages = ["src/kiwixseeder"] 72 | 73 | [tool.hatch.envs.default] 74 | features = ["dev"] 75 | 76 | [tool.hatch.envs.test] 77 | features = ["scripts", "test"] 78 | 79 | [[tool.hatch.envs.test.matrix]] 80 | python = ["3.12", "3.13"] 81 | 82 | [tool.hatch.envs.test.scripts] 83 | run = "inv test --args '{args}'" 84 | run-cov = "inv test-cov --args '{args}'" 85 | report-cov = "inv report-cov" 86 | coverage = "inv coverage --args '{args}'" 87 | html = "inv coverage --html --args '{args}'" 88 | 89 | [tool.hatch.envs.lint] 90 | template = "lint" 91 | skip-install = false 92 | features = ["scripts", "lint"] 93 | 94 | [tool.hatch.envs.lint.scripts] 95 | black = "inv lint-black --args '{args}'" 96 | ruff = "inv lint-ruff --args '{args}'" 97 | all = "inv lintall --args '{args}'" 98 | fix-black = "inv fix-black --args '{args}'" 99 | fix-ruff = "inv fix-ruff --args '{args}'" 100 | fixall = "inv fixall --args '{args}'" 101 | 102 | [tool.hatch.envs.check] 103 | features = ["scripts", "check"] 104 | 105 | [tool.hatch.envs.check.scripts] 106 | pyright = "inv check-pyright --args '{args}'" 107 | all = "inv checkall --args '{args}'" 108 | 109 | [tool.hatch.envs.build] 110 | features = ["scripts", "build"] 111 | 112 | [tool.hatch.envs.build.scripts] 113 | binary = "inv binary {args}" 114 | 115 | [tool.black] 116 | line-length = 88 117 | target-version = ['py312'] 118 | 119 | [tool.ruff] 120 | target-version = "py312" 121 | line-length = 88 122 | src = ["src"] 123 | 124 | [tool.ruff.lint] 125 | select = [ 126 | "A", # flake8-builtins 127 | # "ANN", # flake8-annotations 128 | "ARG", # flake8-unused-arguments 129 | # "ASYNC", # flake8-async 130 | "B", # flake8-bugbear 131 | # "BLE", # flake8-blind-except 132 | "C4", # flake8-comprehensions 133 | "C90", # mccabe 134 | # "COM", # flake8-commas 135 | # "D", # pydocstyle 136 | # "DJ", # flake8-django 137 | "DTZ", # flake8-datetimez 138 | "E", # pycodestyle (default) 139 | "EM", # flake8-errmsg 140 | # "ERA", # eradicate 141 | # "EXE", # flake8-executable 142 | "F", # Pyflakes (default) 143 | # "FA", # flake8-future-annotations 144 | "FBT", # flake8-boolean-trap 145 | # "FLY", # flynt 146 | # "G", # flake8-logging-format 147 | "I", # isort 148 | "ICN", # flake8-import-conventions 149 | # "INP", # flake8-no-pep420 150 | # "INT", # flake8-gettext 151 | "ISC", # flake8-implicit-str-concat 152 | "N", # pep8-naming 153 | # "NPY", # NumPy-specific rules 154 | # "PD", # pandas-vet 155 | # "PGH", # pygrep-hooks 156 | # "PIE", # flake8-pie 157 | # "PL", # Pylint 158 | "PLC", # Pylint: Convention 159 | "PLE", # Pylint: Error 160 | "PLR", # Pylint: Refactor 161 | "PLW", # Pylint: Warning 162 | # "PT", # flake8-pytest-style 163 | # "PTH", # flake8-use-pathlib 164 | # "PYI", # flake8-pyi 165 | "Q", # flake8-quotes 166 | # "RET", # flake8-return 167 | # "RSE", # flake8-raise 168 | "RUF", # Ruff-specific rules 169 | "S", # flake8-bandit 170 | # "SIM", # flake8-simplify 171 | # "SLF", # flake8-self 172 | "T10", # flake8-debugger 173 | "T20", # flake8-print 174 | # "TCH", # flake8-type-checking 175 | # "TD", # flake8-todos 176 | "TID", # flake8-tidy-imports 177 | # "TRY", # tryceratops 178 | "UP", # pyupgrade 179 | "W", # pycodestyle 180 | "YTT", # flake8-2020 181 | ] 182 | ignore = [ 183 | # Allow non-abstract empty methods in abstract base classes 184 | "B027", 185 | # Remove flake8-errmsg since we consider they bloat the code and provide limited value 186 | "EM", 187 | # Allow boolean positional values in function calls, like `dict.get(... True)` 188 | "FBT003", 189 | # Ignore checks for possible passwords 190 | "S105", "S106", "S107", 191 | # Ignore warnings on subprocess.run / popen 192 | "S603", 193 | # Ignore complexity 194 | "C901", "PLR0911", "PLR0912", "PLR0913", "PLR0915", 195 | # implicit string concatenation 196 | "ISC001", 197 | ] 198 | unfixable = [ 199 | # Don't touch unused imports 200 | "F401", 201 | ] 202 | 203 | [tool.ruff.lint.isort] 204 | known-first-party = ["kiwixseeder"] 205 | 206 | [tool.ruff.lint.flake8-bugbear] 207 | # add exceptions to B008 for fastapi. 208 | extend-immutable-calls = ["fastapi.Depends", "fastapi.Query"] 209 | 210 | [tool.ruff.lint.flake8-tidy-imports] 211 | ban-relative-imports = "all" 212 | 213 | [tool.ruff.lint.per-file-ignores] 214 | # Tests can use magic values, assertions, and relative imports 215 | "tests/**/*" = ["PLR2004", "S101", "TID252"] 216 | 217 | [tool.pytest.ini_options] 218 | minversion = "7.3" 219 | testpaths = ["tests"] 220 | pythonpath = [".", "src"] 221 | 222 | [tool.coverage.paths] 223 | kiwixseeder = ["src/kiwixseeder"] 224 | tests = ["tests"] 225 | 226 | [tool.coverage.run] 227 | source_pkgs = ["kiwixseeder"] 228 | branch = true 229 | parallel = true 230 | omit = [ 231 | "src/kiwixseeder/__about__.py", 232 | ] 233 | 234 | [tool.coverage.report] 235 | exclude_lines = [ 236 | "no cov", 237 | "if __name__ == .__main__.:", 238 | "if TYPE_CHECKING:", 239 | ] 240 | 241 | [tool.pyright] 242 | include = ["src", "tests", "tasks.py"] 243 | exclude = [".env/**", ".venv/**"] 244 | extraPaths = ["src"] 245 | pythonVersion = "3.12" 246 | typeCheckingMode="strict" 247 | disableBytesTypePromotions = true 248 | -------------------------------------------------------------------------------- /src/kiwixseeder/context.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import platform 4 | from dataclasses import dataclass, field 5 | from pathlib import Path 6 | from typing import Any, Self 7 | from urllib.parse import ParseResult, urlparse 8 | 9 | import humanfriendly 10 | import qbittorrentapi 11 | 12 | from kiwixseeder.utils import SizeRange 13 | 14 | 15 | def set_from_env(name: str) -> set[str]: 16 | """set() from ENV""" 17 | return {entry for entry in (os.getenv(name) or "").split("|") if entry} 18 | 19 | 20 | NAME = "kiwix-seeder" # must be filesystem-friendly (technical) 21 | CLI_NAME = "kiwix-seeder" 22 | HUMAN_NAME = "Kiwix Seeder" 23 | QBT_CAT_NAME = "kiwix-seeder" # name of category to group our torrents in 24 | RC_NOFILTER = 32 # exit-code when user has no filter and did not confirm 25 | RC_INSUFFICIENT_STORAGE = 30 # exit-code when store is not enough for selection 26 | 27 | CATALOG_URL = os.getenv("CATALOG_URL", "https://library.kiwix.org/catalog/v2") 28 | DOWNLOAD_URL = os.getenv("DOWNLOAD_URL", "https://download.kiwix.org") 29 | 30 | DEFAULT_QBT_USERNAME: str | None = os.getenv("QBT_USERNAME") 31 | DEFAULT_QBT_PASSWORD: str | None = os.getenv("QBT_PASSWORD") 32 | DEFAULT_QBT_SCHEME: str = os.getenv("QBT_SCHEME") or "http" 33 | DEFAULT_QBT_HOST: str = os.getenv("QBT_HOST") or "localhost" 34 | DEFAULT_QBT_PORT: int = int(os.getenv("QBT_PORT") or "8080") 35 | DEFAULT_QBT_INSECURE: bool = bool(os.getenv("QBT_INSECURE")) 36 | DEFAULT_MAX_STORAGE: int = humanfriendly.parse_size(os.getenv("MAX_STORAGE") or "10GiB") 37 | DEFAULT_KEEP_DURATION: float = humanfriendly.parse_timespan( 38 | os.getenv("KEEP_DURATION") or "12w" 39 | ) 40 | DEFAULT_SLEEP_INTERVAL: float = humanfriendly.parse_timespan( 41 | os.getenv("SLEEP_INTERVAL") or "1d" 42 | ) 43 | 44 | DEFAULT_BATCH_SIZE: int = int(os.getenv("BATCH_SIZE") or "100") 45 | DEFAULT_BATCH_INTERVAL: float = humanfriendly.parse_timespan( 46 | os.getenv("BATCH_INTERVAL") or "1m" 47 | ) 48 | 49 | DEFAULT_FILTER_FILENAMES: set[str] = set_from_env("FILENAMES") 50 | DEFAULT_FILTER_LANGUAGES: set[str] = set_from_env("LANGUAGES") 51 | DEFAULT_FILTER_CATEGORIES: set[str] = set_from_env("CATEGORIES") 52 | DEFAULT_FILTER_FLAVOURS: set[str] = set_from_env("FLAVOURS") 53 | DEFAULT_FILTER_TITLES: set[str] = set_from_env("TITLES") 54 | DEFAULT_FILTER_DESCRIPTIONS: set[str] = set_from_env("DESCRIPTIONS") 55 | DEFAULT_FILTER_TAGS: set[str] = set_from_env("TAGS") 56 | DEFAULT_FILTER_AUTHORS: set[str] = set_from_env("AUTHORS") 57 | DEFAULT_FILTER_PUBLISHERS: set[str] = set_from_env("PUBLISHERS") 58 | try: 59 | min_size: int = humanfriendly.parse_size(os.getenv("MIN_SIZE") or "0") 60 | except humanfriendly.InvalidSize: 61 | min_size: int = 0 62 | try: 63 | max_size: int = humanfriendly.parse_size(os.getenv("MAX_SIZE") or "") 64 | except humanfriendly.InvalidSize: 65 | max_size: int = 0 66 | DEFAULT_FILTER_FILESIZES: SizeRange = SizeRange(minimum=min_size, maximum=max_size) 67 | DEFAULT_DEBUG: bool = bool(os.getenv("DEBUG")) 68 | SEED_WHOLE_CATALOG: bool = bool(os.getenv("SEED_WHOLE_CATALOG")) 69 | DEFAULT_CLEAR_OPDS: bool = bool(os.getenv("CLEAR_OPDS")) 70 | 71 | # avoid debug-level logs of 3rd party deps 72 | for module in ("urllib3", "qbittorrentapi.request"): 73 | logging.getLogger(module).setLevel(logging.INFO) 74 | 75 | 76 | @dataclass(kw_only=True) 77 | class QbtConnection: 78 | """Abstraction over qBittorrent Connection 79 | 80 | Supports input as URI or individual parts and exposes them""" 81 | 82 | username: str | None 83 | password: str | None 84 | scheme: str 85 | host: str 86 | port: int 87 | 88 | @classmethod 89 | def using(cls, string: str) -> Self: 90 | """Init from a qbt-schemed URI""" 91 | uri = urlparse(string) 92 | if uri.scheme not in ("http", "https"): 93 | raise ValueError(f"Malformed HHTP(s) URL: {string}") 94 | return cls( 95 | username=uri.username, 96 | password=uri.password, 97 | scheme=uri.scheme, 98 | host=uri.hostname or "localhost", 99 | port=uri.port or 80, 100 | ) 101 | 102 | def __str__(self) -> str: 103 | return ParseResult( 104 | scheme=self.scheme, 105 | netloc=f"{self.username or ''}" 106 | f"{':' if self.password else ''}{self.password or ''}" 107 | f"@{self.host}:{self.port}", 108 | path="", 109 | params="", 110 | query="", 111 | fragment="", 112 | ).geturl() 113 | 114 | 115 | DEFAULT_QBT_CONN = str( 116 | QbtConnection.using(str(os.getenv("QBT_URL"))) 117 | if os.getenv("QBT_URL") 118 | else QbtConnection( 119 | username=DEFAULT_QBT_USERNAME, 120 | password=DEFAULT_QBT_PASSWORD, 121 | scheme=DEFAULT_QBT_SCHEME, 122 | host=DEFAULT_QBT_HOST, 123 | port=DEFAULT_QBT_PORT, 124 | ) 125 | ) 126 | 127 | 128 | @dataclass(kw_only=True) 129 | class Context: 130 | 131 | # singleton instance 132 | _instance: "Context | None" = None 133 | 134 | # debug flag 135 | debug: bool = DEFAULT_DEBUG 136 | 137 | dry_run: bool = False 138 | 139 | # forever mode: how much to sleep in-between runs 140 | sleep_interval: float = DEFAULT_SLEEP_INTERVAL 141 | 142 | is_mac: bool = platform.system() == "Darwin" 143 | is_win: bool = platform.system() == "Windows" 144 | is_nix: bool = platform.system() not in ("Darwin", "Windows") 145 | 146 | batch_size: int = DEFAULT_BATCH_SIZE 147 | batch_interval: float = DEFAULT_BATCH_INTERVAL 148 | 149 | catalog_url: str = CATALOG_URL 150 | download_url: str = DOWNLOAD_URL 151 | qbt: qbittorrentapi.Client 152 | qbt_insecure: bool = DEFAULT_QBT_INSECURE 153 | 154 | # filters 155 | filenames: set[str] = field(default_factory=lambda: DEFAULT_FILTER_FILENAMES) 156 | languages: set[str] = field(default_factory=lambda: DEFAULT_FILTER_LANGUAGES) 157 | categories: set[str] = field(default_factory=lambda: DEFAULT_FILTER_CATEGORIES) 158 | flavours: set[str] = field(default_factory=lambda: DEFAULT_FILTER_FLAVOURS) 159 | titles: set[str] = field(default_factory=lambda: DEFAULT_FILTER_TITLES) 160 | descriptions: set[str] = field(default_factory=lambda: DEFAULT_FILTER_DESCRIPTIONS) 161 | tags: set[str] = field(default_factory=lambda: DEFAULT_FILTER_TAGS) 162 | authors: set[str] = field(default_factory=lambda: DEFAULT_FILTER_AUTHORS) 163 | publishers: set[str] = field(default_factory=lambda: DEFAULT_FILTER_PUBLISHERS) 164 | filesizes: SizeRange = field(default_factory=lambda: DEFAULT_FILTER_FILESIZES) 165 | 166 | # general options 167 | max_storage: int = DEFAULT_MAX_STORAGE 168 | keep_for: float = DEFAULT_KEEP_DURATION 169 | all_good: bool = SEED_WHOLE_CATALOG 170 | clear_opds: bool = DEFAULT_CLEAR_OPDS 171 | 172 | logger: logging.Logger = logging.getLogger(NAME) # noqa: RUF009 173 | max_direct_online_resource_payload_size: int = 2048 174 | 175 | @classmethod 176 | def setup(cls, **kwargs: Any): 177 | if cls._instance: 178 | raise OSError("Already inited Context") 179 | cls._instance = cls(**kwargs) 180 | cls.setup_logger() 181 | 182 | @classmethod 183 | def setup_logger(cls): 184 | debug = cls._instance.debug if cls._instance else cls.debug 185 | if cls._instance: 186 | cls._instance.logger.setLevel(logging.DEBUG if debug else logging.INFO) 187 | else: 188 | cls.logger.setLevel(logging.DEBUG if debug else logging.INFO) 189 | logging.basicConfig( 190 | level=logging.DEBUG if debug else logging.INFO, 191 | format="%(asctime)s %(levelname)s | %(message)s", 192 | ) 193 | 194 | @classmethod 195 | def get(cls) -> "Context": 196 | if not cls._instance: 197 | raise OSError("Uninitialized context") # pragma: no cover 198 | return cls._instance 199 | 200 | @staticmethod 201 | def get_cache_path(fname: str) -> Path: 202 | """Path to save/read cache from/to""" 203 | xdg_cache_home = os.getenv("XDG_CACHE_HOME") 204 | # favor this env on any platform 205 | if xdg_cache_home: 206 | return Path(xdg_cache_home) / fname 207 | if Context.is_mac: 208 | return Path.home() / "Library" / "Caches" / NAME / fname 209 | if Context.is_win: 210 | return Path(os.getenv("APPDATA", "C:")) / NAME / fname 211 | return Path.home() / ".config" / NAME / fname 212 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Kiwix Seeder 2 | 3 | `kiwix-seeder` is a simple tool that allows one to manage a Bittorrent seeder for Kiwix Catalog's ZIMs effortlessly. 4 | 5 | [![CodeFactor](https://www.codefactor.io/repository/github/kiwix/seeder/badge)](https://www.codefactor.io/repository/github/kiwix/seeder) 6 | [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0) 7 | [![codecov](https://codecov.io/gh/kiwix/seeder/branch/main/graph/badge.svg)](https://codecov.io/gh/kiwix/seeder) 8 | ![PyPI - Python Version](https://img.shields.io/badge/python-3.12+-blue) 9 | [![Docker](https://ghcr-badge.egpl.dev/kiwix/bittorrent-seeder/latest_tag?label=docker&ignore=)](https://ghcr.io/kiwix/bittorrent-seeder/) 10 | 11 | It is composed of a script that runs periodically and which consists mostly in: 12 | 13 | - Downloading the Kiwix OPDS Catalog 14 | - Matching its entries with your defined filters 15 | - Communicating with your qBittorrent instance (via HTTP) 16 | - Removing unwanted (not matching or out of Catalog) ZIMs from qBittorrent and filesystem 17 | - Adding new matching ZIM to qBittorrent 18 | 19 | Its goal is thus to command the qBittorrent instance to download new torrents (any 20 | new ZIM in the Catalog matching the filters) and remove old ones (previously 21 | added torrents that dont match current filters or are not in Catalog anymore) 22 | 23 | **Key features:** 24 | 25 | - Very easy to use 26 | - Very flexible filters so you can precisely select what to download and seed 27 | - Compatible with your existing qBittorrent (doesn't mess with your stuff) 28 | 29 | ## Usage 30 | 31 | > [!CAUTION] 32 | > The parameters/config passed to `kiwix-seeder` is an indication of the new requested state. 33 | > Say you were using it and are seeding 20 torrents, if you relaunch it with filters that match only a single ZIM, **it won't add this ZIM to your list**, it will **remove all the others** (see `--keep` below) and then add it (it replaces everything based on the passed filters). 34 | > 35 | > Use `--dry-run` option to work on your filters 36 | 37 | ### Standalone version 38 | 39 | This version depends on a reachable qBittorrent instance. You call it to update your qBittorrent's list of ZIMs to seed. 40 | 41 | ```sh 42 | ❯ export QBT_URL="http://admin:mypass@nas.local:8080" 43 | ❯ kiwix-seeder --lang bam --max-storage 1GB 44 | ``` 45 | 46 | ### Docker version 47 | 48 | The Docker version includes qBittorrent so it's meant to run forever (uses `--daemon`). 49 | 50 | The `seeder-start-restart` assistant script starts it from your config file and periodically runs the `kiwix-seeder` script to manage the list of torrents. 51 | 52 | ```sh 53 | # start the long-lasting container 54 | ❯ seeder-start-restart 55 | 56 | # stop the container 57 | ❯ docker stop seeder 58 | ``` 59 | 60 | You can also run it without the helper script using image `ghcr.io/kiwix/bittorrent-seeder:latest`. You'll have to sort out volume mounting, port forwarding and configuration. Use the helper script as documentation! 61 | 62 | #### Monitoring 63 | 64 | You can monitor what's hapenning via Docker logs 65 | 66 | ```sh 67 | ❯ docker logs -n 10 -f seeder 68 | 2025-01-17 11:52:57,324 INFO | 1279. Added openZIM:wikipedia_lt_all:maxi @ 2024-06-14 (2.2 GiB) 69 | 2025-01-17 11:53:01,852 INFO | 1280. Added openZIM:wikipedia_lt_all:mini @ 2024-06-13 (294.03 MiB) 70 | 2025-01-17 11:53:06,376 INFO | 1281. Added openZIM:wikipedia_lt_all:nopic @ 2024-06-14 (669.17 MiB) 71 | 2025-01-17 11:53:10,772 INFO | 1282. Added openZIM:wikisource_lt_all:maxi @ 2024-06-16 (7.85 MiB) 72 | 2025-01-17 11:53:15,370 INFO | 1283. Added openZIM:wikisource_lt_all:nopic @ 2024-06-16 (7.13 MiB) 73 | 2025-01-17 11:53:19,845 INFO | 1284. Added openZIM:wiktionary_lt_all:maxi @ 2024-05-11 (704.35 MiB) 74 | 2025-01-17 11:53:24,353 INFO | 1285. Added openZIM:wiktionary_lt_all:nopic @ 2024-05-11 (687.87 MiB) 75 | 2025-01-17 11:53:29,308 INFO | 1286. Added openZIM:wikibooks_lt_all:nopic @ 2024-06-26 (100.94 MiB) 76 | 2025-01-17 11:53:34,094 INFO | 1287. Added openZIM:wikibooks_lt_all:maxi @ 2024-06-26 (107.22 MiB) 77 | 2025-01-17 11:53:38,544 INFO | 1288. Added openZIM:wikiquote_lt_all:nopic @ 2024-06-16 (6.12 MiB) 78 | ``` 79 | 80 | You can also query information from the qBittorrent instance using bundled qbittorrent-cli (`qbt` binary) 81 | 82 | ```sh 83 | ❯ docker exec -it seeder qbt global info 84 | Download speed: 382,456,016 bytes/s 85 | Downloaded data: 1,407,643,708,964 bytes 86 | Download speed limit: 0 bytes/s 87 | Upload speed: 13,972,949 bytes/s 88 | Uploaded data: 551,473,671,864 bytes 89 | Upload speed limit: 0 bytes/s 90 | DHT nodes: 366 91 | Connection status: Connected 92 | ``` 93 | 94 | ## Installation 95 | 96 | There are two main ways to use it; choose what's best for you: 97 | 98 | | Mode | Target | Reason | 99 | | --- | -------| --- | 100 | | Standalone Binary | If you already have a running qBittorrent instance. | Lightweight and flexible | 101 | | Docker Image | All in one docker image that runs both the script and qBittorrent. | Simplest | 102 | 103 | The Docker version obviously depends on Docker being installed and running. 104 | 105 | ### Docker version 106 | 107 | This version is intended for those who want a setup-and-forget solution. It comes with qbittorrent. There's a good number of options but it's tailored for a kiwix-seeder only usage so it's not as flexible (although you can change any settings via the WebUI) 108 | 109 | ```sh 110 | # 1. Make sure Docker is installed, running and you have rights over it 111 | ❯ docker ps 112 | 113 | # 1. Create a config file, mentioning where to store torrents and ZIM into. 114 | ❯ mkdir -p /data/seeder 115 | ❯ cat < /etc/seeder.config 116 | 117 | # where to store all data (ZIMs, cache, qBittorrent profile) 118 | DATA_PATH=/data/seeder 119 | 120 | # webui password used by 121 | # - script to communicate with qBittorrent 122 | # - user (you) via remote webui (see WEBUI_PORT below) 123 | QBT_PASSWORD="Choose this one" 124 | 125 | # BT port to use/announce. 126 | # **must** be manually forwarded on your Internet-connected router to your local IP 127 | # /!\ uPNP cannot work accross docker routing so it cannot automatically work 128 | QBT_TORRENTING_PORT=6901 129 | 130 | # port on host to map webui (for remote access, optional) 131 | WEBUI_PORT=8080 132 | 133 | # max storage size to use in DATA_PATH for ZIMs (stops if reached) 134 | MAX_STORAGE="10TiB" 135 | 136 | # interval between kiwix-seeder invocations (catalog refresh, ZIM addition/removal) 137 | SLEEP_INTERVAL="1d" 138 | EOF 139 | 140 | # 2. Download the helper script 141 | ❯ curl -L -o /usr/local/bin/seeder-start-restart https://github.com/kiwix/container-images/raw/refs/heads/main/bittorrent-seeder/seeder-start-restart.sh 142 | ❯ chmod +x /usr/local/bin/seeder-start-restart 143 | ``` 144 | 145 | That's it. You can now start it as show above (`seeder-start-restart`). 146 | 147 | If you created the config file in a different place, edit the helper script to load it properly. 148 | 149 | 150 | ### Standalone binary 151 | 152 | > [!IMPORTANT] 153 | > Standalone version requires you to run and configure qBittorrent yourself (see below) 154 | 155 | Simply download and invoke it 156 | 157 | ```sh 158 | ❯ curl -o /usr/local/bin/kiwix-seeder https://TBD && chmod +x /usr/local/bin/kiwix-seeder 159 | 160 | # set your qBittorrent URL so you can use kiwix-seeder binary without passing your credentials 161 | ❯ export QBT_URL="http://admin:mypass@nas.local:8080" 162 | ``` 163 | 164 | #### qBittorrent requirements 165 | 166 | - qBittorrent must be running when using `kiwix-seeder`. Once invocation of kiwix-server has completed, qBittorrent can be stopped/started at your convenience. 167 | - WebUI must be enabled and configured (see below) 168 | - The machine running `kiwix-seeder` must be able to communicate with qBittorrent WebUI URL. 169 | 170 | Check that you can make an HTTP request from `kiwix-seeder` machine to qBittorent URL using curl to ensure WebUI is working, reachable and the credentials are correct: 171 | 172 | 173 | ```sh 174 | ❯ curl -X POST -d 'username=XXXX&password=XXXX' ${QBT_URL}/api/v2/auth/login 175 | Ok. 176 | ``` 177 | 178 | Make sure your Bittorrent settings are working (port for incoming connection) otherwise this will be quite useless. 179 | 180 | ##### Enabling WebUI 181 | 182 | If you are using the Desktop version of qBittorrent, go to the *Options* panel then select *WebUI* on the sidebar. Then you need to enable *Web User Interface*. 183 | Make sure you know the address, port and credentials to use. 184 | 185 | ## Uninstalling 186 | 187 | Getting rid of the torrents/ZIM is easy because all torrents are within category `kiwix-seeder` and can be done by tweaking the configuration: 188 | 189 | - Set a filter that matchs nothing (`--max-file-size 1b`) 190 | - Set the keep-period very low (`--keep 1m`) 191 | 192 | Then when running, `kiwix-seeder` will remove all the torrents and their associated files. 193 | 194 | You can also do it outside of the tool, using qBittorrent UI or WebUI. Simply right-click on the `kiwix-seeder` category and select *Remove torrents*. You'll be prompted to confirm and whether you want to delete the associated files. 195 | 196 | 197 | If you're using the Docker version, stop it and maybe remove the container, image and your config file. 198 | 199 | ## Configuration 200 | 201 | See the `kiwix-seeder` usage for details on the options 202 | 203 | ```sh 204 | kiwix-seeder --help 205 | ``` 206 | 207 | If using the Docker version, check the first lines of the `seeder-start-restart` script for exposed variables. 208 | 209 | -------------------------------------------------------------------------------- /src/kiwixseeder/library.py: -------------------------------------------------------------------------------- 1 | import collections 2 | import datetime 3 | import re 4 | import urllib.parse 5 | from collections.abc import Generator 6 | from dataclasses import asdict, dataclass, field 7 | from pathlib import Path 8 | from typing import Any, ClassVar 9 | from uuid import UUID 10 | 11 | import iso639 12 | import xmltodict 13 | from iso639.exceptions import DeprecatedLanguageValue, InvalidLanguageValue 14 | 15 | from kiwixseeder.context import Context 16 | from kiwixseeder.download import get_btih_from_url, session 17 | from kiwixseeder.utils import format_size 18 | 19 | ETAG_CACHE_FILE = "OPDS.etag" 20 | BTIH_CACHE_FOLDER = "zim-btih-maps" 21 | context = Context.get() 22 | logger = context.logger 23 | 24 | 25 | def to_human_id(name: str, publisher: str | None = "", flavour: str | None = "") -> str: 26 | """periodless exchange identifier for ZIM Title""" 27 | publisher = publisher or "openZIM" 28 | flavour = flavour or "" 29 | return f"{publisher}:{name}:{flavour}" 30 | 31 | 32 | class BookBtihMapper: 33 | """Disk-cached mapping of Book UUID to BT Info Hash 34 | 35 | Required since btih is not a Catalog metadata but necessary to reconcile 36 | torrents with books uniquely""" 37 | 38 | # maps {uuid: str} to {btih: str} 39 | data: ClassVar[dict[str, str]] = {} 40 | last_read: datetime.datetime = datetime.datetime(2000, 1, 1, tzinfo=datetime.UTC) 41 | 42 | @classmethod 43 | def read(cls, *, force: bool = False): 44 | now = datetime.datetime.now(tz=datetime.UTC) 45 | if not force and cls.last_read + datetime.timedelta(60) >= now: 46 | return 47 | folder = context.get_cache_path("zim-btih-maps") 48 | folder.mkdir(parents=True, exist_ok=True) 49 | data = { 50 | fpath.name.split(":", 1)[0]: fpath.name.split(":", 1)[1] 51 | for fpath in folder.iterdir() 52 | if ":" in fpath.name 53 | } 54 | cls.last_read = now 55 | cls.data = data 56 | 57 | @classmethod 58 | def write(cls): 59 | folder = context.get_cache_path(BTIH_CACHE_FOLDER) 60 | folder.mkdir(parents=True, exist_ok=True) 61 | for uuid, btih in cls.data: 62 | folder.joinpath(f"{uuid}:{btih}").touch() 63 | 64 | @classmethod 65 | def get(cls, uuid: UUID) -> str | None: 66 | cls.read() 67 | return cls.data.get(uuid.hex) 68 | 69 | @classmethod 70 | def add(cls, uuid: UUID, btih: str): 71 | uuids = uuid.hex 72 | if uuids in cls.data: 73 | return 74 | cls.data[uuids] = btih 75 | folder = context.get_cache_path(BTIH_CACHE_FOLDER) 76 | folder.mkdir(parents=True, exist_ok=True) 77 | folder.joinpath(f"{uuids}:{btih}").touch() 78 | 79 | 80 | @dataclass(kw_only=True) 81 | class Book: 82 | uuid: UUID 83 | ident: str 84 | name: str 85 | title: str 86 | description: str 87 | author: str 88 | publisher: str 89 | langs_iso639_1: list[str] = field(default_factory=list) 90 | langs_iso639_3: list[str] 91 | tags: list[str] 92 | flavour: str 93 | size: int 94 | url: str 95 | illustration_relpath: str 96 | version: str 97 | last_seen_on: datetime.datetime 98 | _btih: str 99 | 100 | def __post_init__(self): 101 | for lang in list(self.langs_iso639_3): 102 | value: str = lang 103 | try: 104 | value = iso639.Lang(lang).pt1 105 | # skip language if code is invalid or deprecated 106 | except ( 107 | DeprecatedLanguageValue, 108 | InvalidLanguageValue, 109 | ): 110 | self.langs_iso639_3.remove(lang) 111 | continue 112 | self.langs_iso639_1.append(value) 113 | 114 | # fallback to eng if no valid code was supplied 115 | if not self.langs_iso639_3: 116 | self.langs_iso639_3.append("eng") 117 | if not self.langs_iso639_1: 118 | self.langs_iso639_1.append("en") 119 | 120 | @property 121 | def category(self) -> str: 122 | try: 123 | return next( 124 | tag.split(":", 1)[1] 125 | for tag in self.tags 126 | if tag.startswith("_category:") 127 | ) 128 | except StopIteration: 129 | return "" 130 | 131 | @property 132 | def filepath(self) -> Path: 133 | return Path(urllib.parse.urlparse(self.url).path) 134 | 135 | @property 136 | def filename(self) -> str: 137 | return Path(urllib.parse.urlparse(self.url).path).name 138 | 139 | @property 140 | def torrent_url(self) -> str: 141 | return f"{self.url}.torrent" 142 | 143 | @property 144 | def lang_codes(self) -> list[str]: 145 | return self.langs_iso639_3 146 | 147 | @property 148 | def lang_code(self) -> str: 149 | return self.langs_iso639_3[0] 150 | 151 | @property 152 | def language(self) -> iso639.Lang: 153 | return iso639.Lang(self.lang_code) 154 | 155 | @property 156 | def btih(self) -> str: 157 | if not self._btih: 158 | if btih := BookBtihMapper.get(self.uuid): 159 | self._btih = btih 160 | else: 161 | # use setter so it gets cached 162 | self.btih = get_btih_from_url(self.torrent_url) 163 | return self._btih 164 | 165 | @btih.setter 166 | def btih(self, value: str): 167 | BookBtihMapper.add(self.uuid, value) 168 | self._btih = value 169 | 170 | def to_dict(self) -> dict[str, Any]: 171 | return asdict(self) 172 | 173 | def __str__(self) -> str: 174 | return ( 175 | f"{self.ident} @ {self.version} " # noqa: RUF001 176 | f"({format_size(self.size)})" 177 | ) 178 | 179 | 180 | def read_etag_from_cache() -> str: 181 | fpath = context.get_cache_path(ETAG_CACHE_FILE) 182 | fpath.parent.mkdir(parents=True, exist_ok=True) 183 | try: 184 | return fpath.read_text().strip() 185 | except Exception: 186 | return "" 187 | 188 | 189 | def write_etag_to_cache(value: str): 190 | fpath = context.get_cache_path(ETAG_CACHE_FILE) 191 | fpath.parent.mkdir(parents=True, exist_ok=True) 192 | fpath.write_text(value) 193 | 194 | 195 | def query_etag() -> str: 196 | try: 197 | resp = session.head( 198 | f"{context.catalog_url}/entries", params={"count": "-1"}, timeout=30 199 | ) 200 | return resp.headers.get("etag") or "" 201 | except Exception: 202 | ... 203 | return "" 204 | 205 | 206 | class Catalog: 207 | def __init__(self): 208 | # list of Book by ident 209 | self._books: dict[str, Book] = {} 210 | # list of book-idents by language (ISO-639-1) 211 | self._by_langs: dict[str, list[str]] = {} 212 | BookBtihMapper.read(force=True) 213 | self.etag: str = read_etag_from_cache() 214 | 215 | def __contains__(self, ident: str) -> bool: 216 | return ident in self.get_all_ids() 217 | 218 | @property 219 | def all_books(self) -> Generator[Book, None, None]: 220 | self.ensure_fresh() 221 | yield from self._books.values() 222 | 223 | @property 224 | def nb_books(self) -> int: 225 | self.ensure_fresh() 226 | return len(self._books) 227 | 228 | @property 229 | def languages(self) -> collections.OrderedDict[str, str]: 230 | overrides = { 231 | "ina": "Interlingua", 232 | } 233 | return collections.OrderedDict( 234 | sorted( 235 | [ 236 | ( 237 | lang, 238 | overrides.get(lang, iso639.Lang(lang).name), 239 | ) 240 | for lang in self._by_langs.keys() 241 | ], 242 | key=lambda x: x[1], 243 | ) 244 | ) 245 | 246 | def get(self, ident: str) -> Book: 247 | self.ensure_fresh() 248 | return self._books[ident] 249 | 250 | def get_or_none(self, ident: str) -> Book | None: 251 | self.ensure_fresh() 252 | return self._books.get(ident) 253 | 254 | def get_all_ids(self) -> Generator[str, None, None]: 255 | self.ensure_fresh() 256 | yield from self._books.keys() 257 | 258 | def get_for_lang(self, lang_code: str) -> Generator[Book, str, None]: 259 | self.ensure_fresh() 260 | for ident in self._by_langs.get(lang_code, []): 261 | yield self.get(ident) 262 | 263 | def reset(self): 264 | self._books.clear() 265 | self._by_langs.clear() 266 | self.updated_on: datetime.datetime = datetime.datetime( 267 | 1970, 1, 1, tzinfo=datetime.UTC 268 | ) 269 | 270 | def ensure_fresh(self): 271 | """make sure catalog is loaded""" 272 | if not self._books: 273 | self.do_refresh() 274 | 275 | def do_refresh(self): 276 | logger.debug(f"refreshing catalog via {context.catalog_url}") 277 | books: dict[str, Book] = {} 278 | langs: dict[str, list[str]] = {} 279 | try: 280 | resp = session.get( 281 | f"{context.catalog_url}/entries", params={"count": "-1"}, timeout=30 282 | ) 283 | resp.raise_for_status() 284 | self.etag = resp.headers.get("etag") or "" 285 | fetched_on = datetime.datetime.now(datetime.UTC) 286 | catalog = xmltodict.parse(resp.content) 287 | if "feed" not in catalog: 288 | raise ValueError("Malformed OPDS response") 289 | if not int(catalog["feed"]["totalResults"]): 290 | raise OSError("Catalog has no entry; probably misbehaving") 291 | for entry in catalog["feed"]["entry"]: 292 | if not entry.get("name"): 293 | logger.warning(f"Skipping entry without name: {entry}") 294 | continue 295 | 296 | links = {link["@type"]: link for link in entry["link"]} 297 | version = datetime.datetime.fromisoformat( 298 | re.sub(r"[A-Z]$", "", entry["updated"]) 299 | ).strftime("%Y-%m-%d") 300 | flavour = entry.get("flavour") or "" 301 | publisher = entry.get("publisher", {}).get("name") or "" 302 | author = entry.get("author", {}).get("name") or "" 303 | ident = to_human_id( 304 | name=entry["name"], 305 | publisher=publisher, 306 | flavour=flavour, 307 | ) 308 | if not links.get("image/png;width=48;height=48;scale=1"): 309 | logger.warning(f"Book has no illustration: {ident}") 310 | 311 | uuid = UUID(entry["id"]) 312 | books[ident] = Book( 313 | uuid=uuid, 314 | ident=ident, 315 | name=entry["name"], 316 | title=entry["title"], 317 | author=author, 318 | publisher=publisher, 319 | description=entry["summary"], 320 | langs_iso639_3=list(set(entry["language"].split(","))) or ["eng"], 321 | tags=list(set(entry["tags"].split(";"))), 322 | flavour=flavour, 323 | size=int(links["application/x-zim"]["@length"]), 324 | url=re.sub(r".meta4$", "", links["application/x-zim"]["@href"]), 325 | illustration_relpath=links.get( 326 | "image/png;width=48;height=48;scale=1", {} 327 | ).get("@href", ""), 328 | version=version, 329 | last_seen_on=fetched_on, 330 | _btih=BookBtihMapper.get(uuid) or "", 331 | ) 332 | except Exception as exc: 333 | logger.error(f"Unable to load catalog from OPDS: {exc}") 334 | # only fail refresh if we have no previous catalog to use 335 | if not self._books: 336 | raise exc 337 | else: 338 | # re-order alphabetically by language then title 339 | books = collections.OrderedDict( 340 | sorted( 341 | ((ident, book) for ident, book in books.items()), 342 | key=lambda item: (item[1].language.name, item[1].title), 343 | ) 344 | ) 345 | for ident in books.keys(): 346 | for lang in books[ident].lang_codes: 347 | if lang not in langs: 348 | langs[lang] = [] 349 | langs[lang].append(ident) 350 | self._books = books 351 | self._by_langs = langs 352 | self.updated_on = datetime.datetime.now(datetime.UTC) 353 | logger.debug(f"refreshed on {self.updated_on}") 354 | -------------------------------------------------------------------------------- /src/kiwixseeder/runner.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import fnmatch 3 | 4 | from kiwixseeder.context import ( 5 | QBT_CAT_NAME, 6 | RC_INSUFFICIENT_STORAGE, 7 | RC_NOFILTER, 8 | Context, 9 | ) 10 | from kiwixseeder.library import Book, Catalog, query_etag, write_etag_to_cache 11 | from kiwixseeder.qbittorrent import TorrentManager 12 | from kiwixseeder.utils import format_duration, format_size, sleep_nonblocking 13 | 14 | context = Context.get() 15 | logger = context.logger 16 | 17 | 18 | class Runner: 19 | 20 | def __init__(self) -> None: 21 | self.exit_requested: bool = False 22 | self.now = datetime.datetime.now(datetime.UTC) 23 | self.manager: TorrentManager = TorrentManager() 24 | self.catalog: Catalog = Catalog() 25 | self.books: list[Book] = [] 26 | self.banner: str = "[dry-mode] " if context.dry_run else "" 27 | 28 | def stop(self): 29 | self.exit_requested = True 30 | 31 | def run(self) -> int: 32 | stop_after_filters: bool = False 33 | 34 | self.display_filters() 35 | try: 36 | self.connect_to_backend() 37 | except Exception as exc: 38 | if context.dry_run: 39 | logger.warning( 40 | f"{self.banner}Unable to connect to qBittorrent. " 41 | "We'll will stop after filters" 42 | ) 43 | stop_after_filters = True 44 | else: 45 | raise exc 46 | 47 | # clear OPDS cache if requested 48 | if context.clear_opds: 49 | self.catalog.etag = "" 50 | write_etag_to_cache("") 51 | 52 | if self.fetch_catalog() and not context.dry_run: 53 | logger.info("Catalog has not changed since last run, exiting.") 54 | return 0 55 | catalog_size = self.catalog.nb_books 56 | self.reduce_catalog() 57 | 58 | # make sure it's not an accidental no-param call 59 | books_size = sum(book.size for book in self.books) 60 | if len(self.books) == catalog_size and not context.all_good: 61 | logger.warning( 62 | f"{self.banner}You requesting seeding {len(self.books)} torrents " 63 | f"accounting for {format_size(books_size)}. " 64 | ) 65 | if ( 66 | not context.dry_run 67 | and input("Do you want to continue? Y/[N] ").upper() != "Y" 68 | ): 69 | logger.info("OK, exiting.") 70 | return RC_NOFILTER 71 | 72 | if stop_after_filters: 73 | return 0 74 | 75 | # read existing torrents from qbt 76 | self.manager.reload() 77 | logger.info( 78 | f"{self.banner}There are {self.manager.nb_torrents} torrents " 79 | f"in {QBT_CAT_NAME}" 80 | ) 81 | for btih in self.manager.btihs: 82 | logger.debug(f"* {self.manager.get(btih)!s}") 83 | 84 | self.remove_outdated_torrents() 85 | self.reconcile_books_and_torrents() 86 | 87 | if self.ensure_storage(): 88 | return RC_INSUFFICIENT_STORAGE 89 | 90 | self.add_books() 91 | 92 | logger.info(f"{QBT_CAT_NAME} has {self.manager.nb_torrents} torrents") 93 | return 0 94 | 95 | def display_filters(self): 96 | logger.info(f"{self.banner}Starting super-seeder with filters:") 97 | logger.info(f"Filenames: {', '.join(context.filenames) or 'all'}") 98 | logger.info(f"Languages: {', '.join(context.languages) or 'all'}") 99 | logger.info(f"Categories: {', '.join(context.categories) or 'all'}") 100 | logger.info(f"Flavours: {', '.join(context.flavours) or 'all'}") 101 | logger.info(f"Tags: {', '.join(context.tags) or 'all'}") 102 | logger.info(f"Authors: {', '.join(context.authors) or 'all'}") 103 | logger.info(f"Publishers: {', '.join(context.publishers) or 'all'}") 104 | logger.info(f"Size: {context.filesizes!s}") 105 | 106 | if not context.filesizes.is_valid(): 107 | raise ValueError("Invalid filters combination: sizes") 108 | 109 | def connect_to_backend(self): 110 | logger.info("Checking qBittorrent connection…") 111 | succeeded, vers_or_exc = self.manager.is_connected() 112 | if not succeeded and isinstance(vers_or_exc, BaseException): 113 | raise OSError( 114 | f"Unable to connect to qBittorrent: {vers_or_exc!s}" 115 | ) from vers_or_exc 116 | logger.info(f"> Connected to qBittorrent {vers_or_exc} ; fetching data…") 117 | 118 | self.manager.setup() 119 | 120 | def fetch_catalog(self): 121 | logger.info("Fetching catalog…") 122 | etag = query_etag() 123 | # resources online is same as last time 124 | if etag and self.catalog.etag and etag == self.catalog.etag: 125 | return True 126 | self.catalog.ensure_fresh() 127 | if not context.dry_run: 128 | write_etag_to_cache(self.catalog.etag) 129 | logger.info(f"Catalog contains {self.catalog.nb_books} ZIMs") 130 | 131 | def reduce_catalog(self): 132 | # build books with our filters 133 | self.books = list(filter(self.matches, self.catalog.all_books)) 134 | 135 | # drop catalog (we dont need any of it anymore) 136 | self.catalog.reset() 137 | 138 | logger.info(f"\033[0;32mFilters matches {len(self.books)} ZIMs\033[0m") 139 | if len(self.books) <= 15: # noqa: PLR2004 140 | for book in self.books: 141 | logger.debug(f"* {book!s}") 142 | 143 | def remove_outdated_torrents(self): 144 | if not self.manager.btihs: 145 | return 146 | 147 | logger.info("Checking for existing torrents removal…") 148 | 149 | # reconciling existing torrents and books 150 | unselected_books = list(self.manager.btihs.keys()) 151 | for book in self.books: 152 | btihs = [ 153 | btih 154 | for btih, fname in self.manager.btihs.items() 155 | # having condition on name first is important and it allows 156 | # us to only compare on btih if name matches. 157 | # we cant direclty compare on btih as it would require getting the 158 | # btih of all books otherwise which is resource intensive as it 159 | # requires an HTTP GET for each 160 | if fname == book.filename and book.btih == btih 161 | ] 162 | if btihs: 163 | book.btih = btihs[0] 164 | unselected_books.remove(book.btih) 165 | 166 | # keep those that are within --keep duration 167 | keep_until = self.now - datetime.timedelta(seconds=context.keep_for) 168 | for btih in unselected_books: 169 | if self.manager.get(btih).added_on <= keep_until: 170 | unselected_books.remove(btih) 171 | 172 | if not len(unselected_books): 173 | logger.info("> None") 174 | return 175 | 176 | logger.info( 177 | f"{self.banner}Removing {len(unselected_books)} outdated torrents " 178 | "(not in catalog, over --keep)…" 179 | ) 180 | for btih in unselected_books: 181 | logger.info(f"- {self.manager.get(btih)!s}") 182 | if context.dry_run: 183 | continue 184 | if not self.manager.remove(btih): 185 | logger.error(f"Failed to remove {btih}") 186 | 187 | def ensure_storage(self): 188 | torrents_size = self.manager.total_size 189 | books_size = sum(book.size for book in self.books) 190 | total_size = torrents_size + books_size 191 | logger.info(f"{self.banner}Checking overall storage needs:") 192 | logger.debug(f"- Existing torrents: {format_size(torrents_size)}") 193 | logger.debug(f"- Requested new torrents: {format_size(books_size)}") 194 | logger.info( 195 | f"- Total torrents: {format_size(total_size)} " 196 | f"{'>' if torrents_size > context.max_storage else '<='} " 197 | f"{format_size(context.max_storage)} (max storage)" 198 | ) 199 | 200 | if total_size > context.max_storage: 201 | logger.error("Total size exceeds max-storage") 202 | return True 203 | 204 | def reconcile_books_and_torrents(self): 205 | logger.info( 206 | "Reconciling books and torrents (may require btih endpoint requests)" 207 | ) 208 | self.books = [ 209 | book for book in self.books if book.btih not in self.manager.btihs 210 | ] 211 | 212 | def add_books(self): 213 | logger.info(f"{self.banner}Adding {len(self.books)} torrents…") 214 | for num, book in enumerate(self.books): 215 | if context.dry_run: 216 | logger.info(f"{num}. Would add {book!s}") 217 | continue 218 | if self.manager.add(book): 219 | logger.info(f"{num}. Added {book!s}") 220 | else: 221 | logger.error(f"Failed to add {book!s}") 222 | 223 | if ( 224 | len(self.books) > context.batch_size 225 | and num < len(self.books) - 1 226 | and (num + 1) % context.batch_size == 0 227 | ): 228 | logger.info( 229 | f"Pausing for {format_duration(context.batch_interval)} " 230 | f"after {context.batch_size} additions, to let qBittorrent breath" 231 | ) 232 | sleep_nonblocking(context.batch_interval) 233 | 234 | def matches_filename(self, book: Book) -> bool: 235 | if not context.filenames: 236 | return True 237 | 238 | for pattern in context.filenames: 239 | if book.filepath.match(pattern.lower()): 240 | return True 241 | return False 242 | 243 | def matches_lang(self, book: Book) -> bool: 244 | if not context.languages: 245 | return True 246 | for lang_code in context.languages: 247 | if lang_code.lower() in book.lang_codes: 248 | return True 249 | return False 250 | 251 | def matches_category(self, book: Book) -> bool: 252 | if not context.categories: 253 | return True 254 | for category_pattern in context.categories: 255 | if fnmatch.fnmatch(book.category.lower(), category_pattern.lower()): 256 | return True 257 | return False 258 | 259 | def matches_flavour(self, book: Book) -> bool: 260 | if not context.flavours: 261 | return True 262 | for flavour in context.flavours: 263 | if flavour.lower() in book.flavour.lower(): 264 | return True 265 | return False 266 | 267 | def matches_title(self, book: Book) -> bool: 268 | if not context.titles: 269 | return True 270 | for title_pattern in context.titles: 271 | if fnmatch.fnmatch(book.title.lower(), title_pattern.lower()): 272 | return True 273 | return False 274 | 275 | def matches_description(self, book: Book) -> bool: 276 | if not context.descriptions: 277 | return True 278 | for description_pattern in context.descriptions: 279 | if fnmatch.fnmatch(book.description.lower(), description_pattern.lower()): 280 | return True 281 | return False 282 | 283 | def matches_tag(self, book: Book) -> bool: 284 | if not context.tags: 285 | return True 286 | for tag_pattern in context.tags: 287 | for tag in book.tags: 288 | if fnmatch.fnmatch(tag.lower(), tag_pattern.lower()): 289 | return True 290 | return False 291 | 292 | def matches_author(self, book: Book) -> bool: 293 | if not context.authors: 294 | return True 295 | for author_pattern in context.authors: 296 | if fnmatch.fnmatch(book.author.lower(), author_pattern.lower()): 297 | return True 298 | return False 299 | 300 | def matches_publisher(self, book: Book) -> bool: 301 | if not context.publishers: 302 | return True 303 | for publisher_pattern in context.publishers: 304 | if fnmatch.fnmatch(book.publisher.lower(), publisher_pattern.lower()): 305 | return True 306 | return False 307 | 308 | def matches_size(self, book: Book) -> bool: 309 | return context.filesizes.match(book.size) 310 | 311 | def matches(self, book: Book) -> bool: 312 | for value in ( 313 | "filename", 314 | "lang", 315 | "category", 316 | "flavour", 317 | "title", 318 | "description", 319 | "tag", 320 | "author", 321 | "publisher", 322 | "size", 323 | ): 324 | if not getattr(self, f"matches_{value}")(book): 325 | return False 326 | return True 327 | -------------------------------------------------------------------------------- /src/kiwixseeder/entrypoint.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import signal 3 | import sys 4 | from types import FrameType 5 | 6 | import humanfriendly 7 | import qbittorrentapi 8 | from rich.markdown import ( # pyright: ignore [reportMissingImports] 9 | Markdown, # pyright: ignore [reportUnknownVariableType] 10 | ) 11 | from rich_argparse import ( # pyright: ignore [reportMissingImports] 12 | RichHelpFormatter, # pyright: ignore [reportUnknownVariableType] 13 | ) 14 | 15 | from kiwixseeder.__about__ import __version__ 16 | from kiwixseeder.context import ( 17 | CLI_NAME, 18 | DEFAULT_FILTER_FILESIZES, 19 | DEFAULT_KEEP_DURATION, 20 | DEFAULT_MAX_STORAGE, 21 | DEFAULT_QBT_CONN, 22 | Context, 23 | QbtConnection, 24 | ) 25 | from kiwixseeder.utils import SizeRange, format_duration, format_size 26 | 27 | logger = Context.logger 28 | 29 | epilog = """ 30 | --- 31 | ## Filtering 32 | 33 | Inputs are all *filters* meaning to reduce the number of matching ZIM to seed. 34 | This means that not entering any filters matches the whole Catalog (3,000+ ZIMs). 35 | Because seeding everything requires a lot of disk space, if you don't input any 36 | filter, you'll be prompted to confirm (use `--all-good` to silence it) 37 | 38 | Filters combines as: 39 | 40 | - If filter is not set or empty, it matches. 41 | - Evaluating a filter (filename, category, etc), if any value matches, it matches. 42 | - Evaluating all filters, if any filter (filaname, category, etc), if set and not \ 43 | matching, it does not match. 44 | 45 | _Examples:_ 46 | 47 | > `--filename 'mali*'` matches a single ZIM: 48 | 49 | - `openZIM:mali-pour-les-nuls_fr_all:@2024-09-06 (61.54 MiB)` 50 | 51 | > `--filename 'mali*' --filename 'eleda*'` matches two ZIMs: 52 | 53 | - `openZIM:mali-pour-les-nuls_fr_all:@2024-09-06 (61.54 MiB)` 54 | - `openZIM:eleda.education_fr_fo-offline:@2023-10-29 (50.03 MiB)` 55 | 56 | > `--filename 'mali*' --filename 'eleda*' --lang fra` matches two ZIMs: 57 | 58 | - `openZIM:mali-pour-les-nuls_fr_all:@2024-09-06 (61.54 MiB)` 59 | - `openZIM:eleda.education_fr_fo-offline:@2023-10-29 (50.03 MiB)` 60 | 61 | > `--filename 'mali*' --filename 'eleda*' --lang bam` matches zero ZIM 62 | 63 | 64 | ## Glob-patterns 65 | 66 | - filename-patterns are absolute (root is `/`) 67 | - root starts below zim folder (so `/zim` for instance) 68 | - `**` means any directory tree 69 | - `*` Matches any number of non-separator characters, including zero. 70 | - `[seq]` Matches one character in seq. 71 | - `[!seq]` Matches one character not in seq. 72 | 73 | Sample requests: 74 | 75 | - `wikipedia/*`: All in wikipedia folder 76 | - `wikipedia_fr_*` All wikipedia with `fr` lang (See --lang as well) 77 | - `*_maxi_*` All maxi ones 78 | - `wikipedia_fr_all_nopic_*` specific 79 | 80 | See https://docs.python.org/3/library/pathlib.html#pattern-language 81 | 82 | 83 | ## Keeping ZIMs 84 | 85 | `--keep` allows you to keep seeding ZIMs after those drop out of the Catalog. 86 | The Catalog only exposes the latest version of a *Title*. Most *Titles* are redone \ 87 | monthly.\n 88 | The webseeds (HTTP mirrors) only keep at most 2 versions of a *Title*. 89 | This option expects a duration with a period suffix(`d`: day, `w`: week, `y`: year) 90 | 91 | ⚠️ Without `--keep`, torrents (and ZIMs) are removed as soon as they are not 92 | matched by your filters or leave the Catalog. 93 | 94 | ## Samples 95 | 96 | | Sample | Request | 97 | | -------------------------------- | --------------------------------------------------| 98 | | All Wikimedia ones in Bamanankan | `--lang bam --category 'wiki*'` | 99 | | All medicine selections | `--filename '*_medicine_*'` | 100 | | All “Public” TED | `--category ted --title '*public*'` | 101 | 102 | """ 103 | 104 | 105 | def prepare_context(raw_args: list[str]) -> None: 106 | parser = argparse.ArgumentParser( 107 | prog=CLI_NAME, 108 | description="Automates a qBitottorrent instance to seed " 109 | "all or part of the Kiwix Catalog", 110 | formatter_class=RichHelpFormatter, # pyright: ignore [reportUnknownArgumentType] 111 | epilog=Markdown( # pyright: ignore [reportArgumentType] 112 | epilog, style="argparse.text" 113 | ), # pyright: ignore [reportUnknownArgumentType] 114 | ) 115 | 116 | parser.add_argument( 117 | "--qbt", 118 | dest="qbt_url", 119 | help="qBittorrent connection string. " 120 | "Format: http://{user}:{password}@{host}:{port}. " 121 | "Can be set via QBT_URL or parts via QBT_USER, QBT_PASSWORD, " 122 | f"QBT_HOST and QBT_PORT. Defaults to {DEFAULT_QBT_CONN}", 123 | type=str, 124 | default=DEFAULT_QBT_CONN, 125 | required=False, 126 | ) 127 | 128 | parser.add_argument( 129 | "--version", 130 | help="Display scraper version and exit", 131 | action="version", 132 | version=__version__, 133 | ) 134 | 135 | parser.add_argument( 136 | "--debug", 137 | dest="debug", 138 | help="Enable debug-level logs", 139 | default=Context.debug, 140 | action="store_true", 141 | ) 142 | 143 | parser.add_argument( 144 | "-C", 145 | "--clear-opds", 146 | dest="clear_opds", 147 | help=f"Clear OPDS Cache. Use it to when relaunching {CLI_NAME} " 148 | "while the Kiwix Catalog has not yet been refreshed", 149 | default=Context.clear_opds, 150 | action="store_true", 151 | ) 152 | 153 | parser.add_argument( 154 | "-k", 155 | "--insecure", 156 | dest="qbt_insecure", 157 | help="Disable SSL verification while communicating with qBittorrent.", 158 | default=Context.qbt_insecure, 159 | action="store_true", 160 | ) 161 | 162 | parser.add_argument( 163 | "--dry-run", 164 | dest="dry_run", 165 | help="Dry-run mode: no torrent will be added/removed to/from qBittorrent. " 166 | "Use it to test filters matching", 167 | default=Context.dry_run, 168 | action="store_true", 169 | ) 170 | 171 | parser.add_argument( 172 | "--all-good", 173 | dest="all_good", 174 | action="store_true", 175 | help="Continue even if your filters " 176 | "didnt filter anything out (thousands of torrents)", 177 | required=False, 178 | ) 179 | 180 | parser.add_argument( 181 | "--filename", 182 | dest="filenames", 183 | action="append", 184 | help="Only seed ZIMs matching this folder/filename pattern.\n" 185 | "Can be used multiple times.\n" 186 | "glob-pattern accepted.", 187 | type=str, 188 | default=[], 189 | required=False, 190 | ) 191 | 192 | parser.add_argument( 193 | "--lang", 194 | dest="languages", 195 | action="append", 196 | help="Only seed ZIMs for this ISO-639-3 language code.\n" 197 | "Can be used multiple times.", 198 | type=str, 199 | default=[], 200 | required=False, 201 | ) 202 | 203 | parser.add_argument( 204 | "--category", 205 | dest="categories", 206 | action="append", 207 | help="Only seed ZIMs in this category.\nCan be used multiple times." 208 | "\nglob-pattern accepted.", 209 | type=str, 210 | default=[], 211 | required=False, 212 | ) 213 | 214 | parser.add_argument( 215 | "--flavour", 216 | dest="flavours", 217 | action="append", 218 | help="Only seed ZIMs of this flavour.\nCan be used multiple times.\n" 219 | "You cant filter those without a flavour at the moment.", 220 | choices=["mini", "nopic", "maxi"], 221 | type=str, 222 | default=[], 223 | required=False, 224 | ) 225 | 226 | parser.add_argument( 227 | "--title", 228 | dest="titles", 229 | action="append", 230 | help="Only seed ZIMs matching this Title metadata pattern.\n" 231 | "Can be used multiple times.\n" 232 | "glob-pattern accepted.", 233 | type=str, 234 | default=[], 235 | required=False, 236 | ) 237 | 238 | parser.add_argument( 239 | "--description", 240 | dest="descriptions", 241 | action="append", 242 | help="Only seed ZIMs matching this Description metadata pattern.\n" 243 | "Can be used multiple times.\n" 244 | "glob-pattern accepted.", 245 | type=str, 246 | default=[], 247 | required=False, 248 | ) 249 | 250 | parser.add_argument( 251 | "--tags", 252 | dest="tags", 253 | action="append", 254 | help="Only seed ZIMs with this tag.\nCan be used multiple times." 255 | "\nglob-pattern accepted.", 256 | type=str, 257 | default=[], 258 | required=False, 259 | ) 260 | 261 | parser.add_argument( 262 | "--author", 263 | dest="authors", 264 | action="append", 265 | help="Only seed ZIMs created by this one.\nCan be used multiple times." 266 | "\nglob-pattern accepted.", 267 | type=str, 268 | default=[], 269 | required=False, 270 | ) 271 | 272 | parser.add_argument( 273 | "--publisher", 274 | dest="publishers", 275 | action="append", 276 | help="Only seed ZIMs published by this one.\nCan be used multiple times." 277 | "\nglob-pattern accepted.", 278 | type=str, 279 | default=[], 280 | required=False, 281 | ) 282 | 283 | parser.add_argument( 284 | "--min-file-size", 285 | dest="min_size", 286 | help="Only seed ZIMs at least this size. Input is parsed for suffix", 287 | type=str, 288 | default=format_size(DEFAULT_FILTER_FILESIZES.minimum), 289 | required=False, 290 | ) 291 | 292 | parser.add_argument( 293 | "--max-file-size", 294 | dest="max_size", 295 | help="Only seed ZIMs at most this size. Input is parsed for suffix", 296 | type=str, 297 | default=format_size(DEFAULT_FILTER_FILESIZES.maximum), 298 | required=False, 299 | ) 300 | 301 | parser.add_argument( 302 | "--max-storage", 303 | dest="max_storage", 304 | help="Overall seeder storage. " 305 | "Removes older torrents if new ones require additional disk space. " 306 | f"Defaults to {format_size(DEFAULT_MAX_STORAGE)}", 307 | type=str, 308 | default=format_size(DEFAULT_MAX_STORAGE), 309 | required=False, 310 | ) 311 | 312 | parser.add_argument( 313 | "--keep", 314 | dest="keep_for", 315 | help="Duration for which to keep an already-added torrent " 316 | "once it dropped out of the Catalog. Duration is computed from added date. " 317 | "Use duration prefixes (d for days, w for weeks, y for years). " 318 | f"Defaults to {format_duration(DEFAULT_KEEP_DURATION)}", 319 | type=str, 320 | default=format_duration(DEFAULT_KEEP_DURATION), 321 | required=False, 322 | ) 323 | 324 | args = parser.parse_args(raw_args) 325 | 326 | # ignore unset values in order to not override Context defaults 327 | args_dict = {key: value for key, value in args._get_kwargs() if value} 328 | 329 | # de-dup list of strings and cast to set 330 | for key in ("folder_prefixes", "categories", "tags", "scrapers"): 331 | if key in args_dict: 332 | args_dict[key] = set(args_dict[key]) 333 | 334 | # size-range 335 | min_size: int = ( 336 | -1 if args.min_size is None else humanfriendly.parse_size(args.min_size) 337 | ) 338 | max_size: int = ( 339 | -1 if args.max_size is None else humanfriendly.parse_size(args.max_size) 340 | ) 341 | args_dict.update({"filesizes": SizeRange(minimum=min_size, maximum=max_size)}) 342 | for key in ("min_size", "max_size"): 343 | if key in args_dict: 344 | del args_dict[key] 345 | 346 | # storage 347 | args_dict["max_storage"] = humanfriendly.parse_size(args_dict["max_storage"]) 348 | 349 | # keep duration 350 | args_dict["keep_for"] = humanfriendly.parse_timespan(args_dict["keep_for"]) 351 | 352 | # qbittorrent client 353 | conn = QbtConnection.using(args_dict["qbt_url"]) 354 | args_dict["qbt"] = ( 355 | qbittorrentapi.Client( # pyright: ignore [reportUnknownMemberType] 356 | host=f"{conn.scheme}://{conn.host}", 357 | port=conn.port, 358 | username=conn.username, 359 | password=conn.password, 360 | VERIFY_WEBUI_CERTIFICATE=not args_dict.get( 361 | "qbt_insecure", parser.get_default("qbt_insecure") 362 | ), 363 | ) 364 | ) 365 | del args_dict["qbt_url"] 366 | 367 | Context.setup(**args_dict) 368 | 369 | 370 | def main() -> int: 371 | try: 372 | prepare_context(sys.argv[1:]) 373 | # late import as to have an initialized Context 374 | from kiwixseeder.runner import Runner 375 | 376 | runner = Runner() 377 | 378 | def exit_gracefully(signum: int, frame: FrameType | None): # noqa: ARG001 379 | print("\n", flush=True) # noqa: T201 380 | logger.info(f"Received {signal.Signals(signum).name}/{signum}. Exiting") 381 | runner.stop() 382 | 383 | signal.signal(signal.SIGTERM, exit_gracefully) 384 | signal.signal(signal.SIGINT, exit_gracefully) 385 | signal.signal(signal.SIGQUIT, exit_gracefully) 386 | 387 | return runner.run() 388 | except Exception as exc: 389 | logger.error(f"General failure: {exc!s}") 390 | logger.exception(exc) 391 | return 1 392 | 393 | 394 | def entrypoint(): 395 | sys.exit(main()) 396 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU General Public License is a free, copyleft license for 11 | software and other kinds of works. 12 | 13 | The licenses for most software and other practical works are designed 14 | to take away your freedom to share and change the works. By contrast, 15 | the GNU General Public License is intended to guarantee your freedom to 16 | share and change all versions of a program--to make sure it remains free 17 | software for all its users. We, the Free Software Foundation, use the 18 | GNU General Public License for most of our software; it applies also to 19 | any other work released this way by its authors. You can apply it to 20 | your programs, too. 21 | 22 | When we speak of free software, we are referring to freedom, not 23 | price. Our General Public Licenses are designed to make sure that you 24 | have the freedom to distribute copies of free software (and charge for 25 | them if you wish), that you receive source code or can get it if you 26 | want it, that you can change the software or use pieces of it in new 27 | free programs, and that you know you can do these things. 28 | 29 | To protect your rights, we need to prevent others from denying you 30 | these rights or asking you to surrender the rights. Therefore, you have 31 | certain responsibilities if you distribute copies of the software, or if 32 | you modify it: responsibilities to respect the freedom of others. 33 | 34 | For example, if you distribute copies of such a program, whether 35 | gratis or for a fee, you must pass on to the recipients the same 36 | freedoms that you received. You must make sure that they, too, receive 37 | or can get the source code. And you must show them these terms so they 38 | know their rights. 39 | 40 | Developers that use the GNU GPL protect your rights with two steps: 41 | (1) assert copyright on the software, and (2) offer you this License 42 | giving you legal permission to copy, distribute and/or modify it. 43 | 44 | For the developers' and authors' protection, the GPL clearly explains 45 | that there is no warranty for this free software. For both users' and 46 | authors' sake, the GPL requires that modified versions be marked as 47 | changed, so that their problems will not be attributed erroneously to 48 | authors of previous versions. 49 | 50 | Some devices are designed to deny users access to install or run 51 | modified versions of the software inside them, although the manufacturer 52 | can do so. This is fundamentally incompatible with the aim of 53 | protecting users' freedom to change the software. The systematic 54 | pattern of such abuse occurs in the area of products for individuals to 55 | use, which is precisely where it is most unacceptable. Therefore, we 56 | have designed this version of the GPL to prohibit the practice for those 57 | products. If such problems arise substantially in other domains, we 58 | stand ready to extend this provision to those domains in future versions 59 | of the GPL, as needed to protect the freedom of users. 60 | 61 | Finally, every program is threatened constantly by software patents. 62 | States should not allow patents to restrict development and use of 63 | software on general-purpose computers, but in those that do, we wish to 64 | avoid the special danger that patents applied to a free program could 65 | make it effectively proprietary. To prevent this, the GPL assures that 66 | patents cannot be used to render the program non-free. 67 | 68 | The precise terms and conditions for copying, distribution and 69 | modification follow. 70 | 71 | TERMS AND CONDITIONS 72 | 73 | 0. Definitions. 74 | 75 | "This License" refers to version 3 of the GNU General Public License. 76 | 77 | "Copyright" also means copyright-like laws that apply to other kinds of 78 | works, such as semiconductor masks. 79 | 80 | "The Program" refers to any copyrightable work licensed under this 81 | License. Each licensee is addressed as "you". "Licensees" and 82 | "recipients" may be individuals or organizations. 83 | 84 | To "modify" a work means to copy from or adapt all or part of the work 85 | in a fashion requiring copyright permission, other than the making of an 86 | exact copy. The resulting work is called a "modified version" of the 87 | earlier work or a work "based on" the earlier work. 88 | 89 | A "covered work" means either the unmodified Program or a work based 90 | on the Program. 91 | 92 | To "propagate" a work means to do anything with it that, without 93 | permission, would make you directly or secondarily liable for 94 | infringement under applicable copyright law, except executing it on a 95 | computer or modifying a private copy. Propagation includes copying, 96 | distribution (with or without modification), making available to the 97 | public, and in some countries other activities as well. 98 | 99 | To "convey" a work means any kind of propagation that enables other 100 | parties to make or receive copies. Mere interaction with a user through 101 | a computer network, with no transfer of a copy, is not conveying. 102 | 103 | An interactive user interface displays "Appropriate Legal Notices" 104 | to the extent that it includes a convenient and prominently visible 105 | feature that (1) displays an appropriate copyright notice, and (2) 106 | tells the user that there is no warranty for the work (except to the 107 | extent that warranties are provided), that licensees may convey the 108 | work under this License, and how to view a copy of this License. If 109 | the interface presents a list of user commands or options, such as a 110 | menu, a prominent item in the list meets this criterion. 111 | 112 | 1. Source Code. 113 | 114 | The "source code" for a work means the preferred form of the work 115 | for making modifications to it. "Object code" means any non-source 116 | form of a work. 117 | 118 | A "Standard Interface" means an interface that either is an official 119 | standard defined by a recognized standards body, or, in the case of 120 | interfaces specified for a particular programming language, one that 121 | is widely used among developers working in that language. 122 | 123 | The "System Libraries" of an executable work include anything, other 124 | than the work as a whole, that (a) is included in the normal form of 125 | packaging a Major Component, but which is not part of that Major 126 | Component, and (b) serves only to enable use of the work with that 127 | Major Component, or to implement a Standard Interface for which an 128 | implementation is available to the public in source code form. A 129 | "Major Component", in this context, means a major essential component 130 | (kernel, window system, and so on) of the specific operating system 131 | (if any) on which the executable work runs, or a compiler used to 132 | produce the work, or an object code interpreter used to run it. 133 | 134 | The "Corresponding Source" for a work in object code form means all 135 | the source code needed to generate, install, and (for an executable 136 | work) run the object code and to modify the work, including scripts to 137 | control those activities. However, it does not include the work's 138 | System Libraries, or general-purpose tools or generally available free 139 | programs which are used unmodified in performing those activities but 140 | which are not part of the work. For example, Corresponding Source 141 | includes interface definition files associated with source files for 142 | the work, and the source code for shared libraries and dynamically 143 | linked subprograms that the work is specifically designed to require, 144 | such as by intimate data communication or control flow between those 145 | subprograms and other parts of the work. 146 | 147 | The Corresponding Source need not include anything that users 148 | can regenerate automatically from other parts of the Corresponding 149 | Source. 150 | 151 | The Corresponding Source for a work in source code form is that 152 | same work. 153 | 154 | 2. Basic Permissions. 155 | 156 | All rights granted under this License are granted for the term of 157 | copyright on the Program, and are irrevocable provided the stated 158 | conditions are met. This License explicitly affirms your unlimited 159 | permission to run the unmodified Program. The output from running a 160 | covered work is covered by this License only if the output, given its 161 | content, constitutes a covered work. This License acknowledges your 162 | rights of fair use or other equivalent, as provided by copyright law. 163 | 164 | You may make, run and propagate covered works that you do not 165 | convey, without conditions so long as your license otherwise remains 166 | in force. You may convey covered works to others for the sole purpose 167 | of having them make modifications exclusively for you, or provide you 168 | with facilities for running those works, provided that you comply with 169 | the terms of this License in conveying all material for which you do 170 | not control copyright. Those thus making or running the covered works 171 | for you must do so exclusively on your behalf, under your direction 172 | and control, on terms that prohibit them from making any copies of 173 | your copyrighted material outside their relationship with you. 174 | 175 | Conveying under any other circumstances is permitted solely under 176 | the conditions stated below. Sublicensing is not allowed; section 10 177 | makes it unnecessary. 178 | 179 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 180 | 181 | No covered work shall be deemed part of an effective technological 182 | measure under any applicable law fulfilling obligations under article 183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 184 | similar laws prohibiting or restricting circumvention of such 185 | measures. 186 | 187 | When you convey a covered work, you waive any legal power to forbid 188 | circumvention of technological measures to the extent such circumvention 189 | is effected by exercising rights under this License with respect to 190 | the covered work, and you disclaim any intention to limit operation or 191 | modification of the work as a means of enforcing, against the work's 192 | users, your or third parties' legal rights to forbid circumvention of 193 | technological measures. 194 | 195 | 4. Conveying Verbatim Copies. 196 | 197 | You may convey verbatim copies of the Program's source code as you 198 | receive it, in any medium, provided that you conspicuously and 199 | appropriately publish on each copy an appropriate copyright notice; 200 | keep intact all notices stating that this License and any 201 | non-permissive terms added in accord with section 7 apply to the code; 202 | keep intact all notices of the absence of any warranty; and give all 203 | recipients a copy of this License along with the Program. 204 | 205 | You may charge any price or no price for each copy that you convey, 206 | and you may offer support or warranty protection for a fee. 207 | 208 | 5. Conveying Modified Source Versions. 209 | 210 | You may convey a work based on the Program, or the modifications to 211 | produce it from the Program, in the form of source code under the 212 | terms of section 4, provided that you also meet all of these conditions: 213 | 214 | a) The work must carry prominent notices stating that you modified 215 | it, and giving a relevant date. 216 | 217 | b) The work must carry prominent notices stating that it is 218 | released under this License and any conditions added under section 219 | 7. This requirement modifies the requirement in section 4 to 220 | "keep intact all notices". 221 | 222 | c) You must license the entire work, as a whole, under this 223 | License to anyone who comes into possession of a copy. This 224 | License will therefore apply, along with any applicable section 7 225 | additional terms, to the whole of the work, and all its parts, 226 | regardless of how they are packaged. This License gives no 227 | permission to license the work in any other way, but it does not 228 | invalidate such permission if you have separately received it. 229 | 230 | d) If the work has interactive user interfaces, each must display 231 | Appropriate Legal Notices; however, if the Program has interactive 232 | interfaces that do not display Appropriate Legal Notices, your 233 | work need not make them do so. 234 | 235 | A compilation of a covered work with other separate and independent 236 | works, which are not by their nature extensions of the covered work, 237 | and which are not combined with it such as to form a larger program, 238 | in or on a volume of a storage or distribution medium, is called an 239 | "aggregate" if the compilation and its resulting copyright are not 240 | used to limit the access or legal rights of the compilation's users 241 | beyond what the individual works permit. Inclusion of a covered work 242 | in an aggregate does not cause this License to apply to the other 243 | parts of the aggregate. 244 | 245 | 6. Conveying Non-Source Forms. 246 | 247 | You may convey a covered work in object code form under the terms 248 | of sections 4 and 5, provided that you also convey the 249 | machine-readable Corresponding Source under the terms of this License, 250 | in one of these ways: 251 | 252 | a) Convey the object code in, or embodied in, a physical product 253 | (including a physical distribution medium), accompanied by the 254 | Corresponding Source fixed on a durable physical medium 255 | customarily used for software interchange. 256 | 257 | b) Convey the object code in, or embodied in, a physical product 258 | (including a physical distribution medium), accompanied by a 259 | written offer, valid for at least three years and valid for as 260 | long as you offer spare parts or customer support for that product 261 | model, to give anyone who possesses the object code either (1) a 262 | copy of the Corresponding Source for all the software in the 263 | product that is covered by this License, on a durable physical 264 | medium customarily used for software interchange, for a price no 265 | more than your reasonable cost of physically performing this 266 | conveying of source, or (2) access to copy the 267 | Corresponding Source from a network server at no charge. 268 | 269 | c) Convey individual copies of the object code with a copy of the 270 | written offer to provide the Corresponding Source. This 271 | alternative is allowed only occasionally and noncommercially, and 272 | only if you received the object code with such an offer, in accord 273 | with subsection 6b. 274 | 275 | d) Convey the object code by offering access from a designated 276 | place (gratis or for a charge), and offer equivalent access to the 277 | Corresponding Source in the same way through the same place at no 278 | further charge. You need not require recipients to copy the 279 | Corresponding Source along with the object code. If the place to 280 | copy the object code is a network server, the Corresponding Source 281 | may be on a different server (operated by you or a third party) 282 | that supports equivalent copying facilities, provided you maintain 283 | clear directions next to the object code saying where to find the 284 | Corresponding Source. Regardless of what server hosts the 285 | Corresponding Source, you remain obligated to ensure that it is 286 | available for as long as needed to satisfy these requirements. 287 | 288 | e) Convey the object code using peer-to-peer transmission, provided 289 | you inform other peers where the object code and Corresponding 290 | Source of the work are being offered to the general public at no 291 | charge under subsection 6d. 292 | 293 | A separable portion of the object code, whose source code is excluded 294 | from the Corresponding Source as a System Library, need not be 295 | included in conveying the object code work. 296 | 297 | A "User Product" is either (1) a "consumer product", which means any 298 | tangible personal property which is normally used for personal, family, 299 | or household purposes, or (2) anything designed or sold for incorporation 300 | into a dwelling. In determining whether a product is a consumer product, 301 | doubtful cases shall be resolved in favor of coverage. For a particular 302 | product received by a particular user, "normally used" refers to a 303 | typical or common use of that class of product, regardless of the status 304 | of the particular user or of the way in which the particular user 305 | actually uses, or expects or is expected to use, the product. A product 306 | is a consumer product regardless of whether the product has substantial 307 | commercial, industrial or non-consumer uses, unless such uses represent 308 | the only significant mode of use of the product. 309 | 310 | "Installation Information" for a User Product means any methods, 311 | procedures, authorization keys, or other information required to install 312 | and execute modified versions of a covered work in that User Product from 313 | a modified version of its Corresponding Source. The information must 314 | suffice to ensure that the continued functioning of the modified object 315 | code is in no case prevented or interfered with solely because 316 | modification has been made. 317 | 318 | If you convey an object code work under this section in, or with, or 319 | specifically for use in, a User Product, and the conveying occurs as 320 | part of a transaction in which the right of possession and use of the 321 | User Product is transferred to the recipient in perpetuity or for a 322 | fixed term (regardless of how the transaction is characterized), the 323 | Corresponding Source conveyed under this section must be accompanied 324 | by the Installation Information. But this requirement does not apply 325 | if neither you nor any third party retains the ability to install 326 | modified object code on the User Product (for example, the work has 327 | been installed in ROM). 328 | 329 | The requirement to provide Installation Information does not include a 330 | requirement to continue to provide support service, warranty, or updates 331 | for a work that has been modified or installed by the recipient, or for 332 | the User Product in which it has been modified or installed. Access to a 333 | network may be denied when the modification itself materially and 334 | adversely affects the operation of the network or violates the rules and 335 | protocols for communication across the network. 336 | 337 | Corresponding Source conveyed, and Installation Information provided, 338 | in accord with this section must be in a format that is publicly 339 | documented (and with an implementation available to the public in 340 | source code form), and must require no special password or key for 341 | unpacking, reading or copying. 342 | 343 | 7. Additional Terms. 344 | 345 | "Additional permissions" are terms that supplement the terms of this 346 | License by making exceptions from one or more of its conditions. 347 | Additional permissions that are applicable to the entire Program shall 348 | be treated as though they were included in this License, to the extent 349 | that they are valid under applicable law. If additional permissions 350 | apply only to part of the Program, that part may be used separately 351 | under those permissions, but the entire Program remains governed by 352 | this License without regard to the additional permissions. 353 | 354 | When you convey a copy of a covered work, you may at your option 355 | remove any additional permissions from that copy, or from any part of 356 | it. (Additional permissions may be written to require their own 357 | removal in certain cases when you modify the work.) You may place 358 | additional permissions on material, added by you to a covered work, 359 | for which you have or can give appropriate copyright permission. 360 | 361 | Notwithstanding any other provision of this License, for material you 362 | add to a covered work, you may (if authorized by the copyright holders of 363 | that material) supplement the terms of this License with terms: 364 | 365 | a) Disclaiming warranty or limiting liability differently from the 366 | terms of sections 15 and 16 of this License; or 367 | 368 | b) Requiring preservation of specified reasonable legal notices or 369 | author attributions in that material or in the Appropriate Legal 370 | Notices displayed by works containing it; or 371 | 372 | c) Prohibiting misrepresentation of the origin of that material, or 373 | requiring that modified versions of such material be marked in 374 | reasonable ways as different from the original version; or 375 | 376 | d) Limiting the use for publicity purposes of names of licensors or 377 | authors of the material; or 378 | 379 | e) Declining to grant rights under trademark law for use of some 380 | trade names, trademarks, or service marks; or 381 | 382 | f) Requiring indemnification of licensors and authors of that 383 | material by anyone who conveys the material (or modified versions of 384 | it) with contractual assumptions of liability to the recipient, for 385 | any liability that these contractual assumptions directly impose on 386 | those licensors and authors. 387 | 388 | All other non-permissive additional terms are considered "further 389 | restrictions" within the meaning of section 10. If the Program as you 390 | received it, or any part of it, contains a notice stating that it is 391 | governed by this License along with a term that is a further 392 | restriction, you may remove that term. If a license document contains 393 | a further restriction but permits relicensing or conveying under this 394 | License, you may add to a covered work material governed by the terms 395 | of that license document, provided that the further restriction does 396 | not survive such relicensing or conveying. 397 | 398 | If you add terms to a covered work in accord with this section, you 399 | must place, in the relevant source files, a statement of the 400 | additional terms that apply to those files, or a notice indicating 401 | where to find the applicable terms. 402 | 403 | Additional terms, permissive or non-permissive, may be stated in the 404 | form of a separately written license, or stated as exceptions; 405 | the above requirements apply either way. 406 | 407 | 8. Termination. 408 | 409 | You may not propagate or modify a covered work except as expressly 410 | provided under this License. Any attempt otherwise to propagate or 411 | modify it is void, and will automatically terminate your rights under 412 | this License (including any patent licenses granted under the third 413 | paragraph of section 11). 414 | 415 | However, if you cease all violation of this License, then your 416 | license from a particular copyright holder is reinstated (a) 417 | provisionally, unless and until the copyright holder explicitly and 418 | finally terminates your license, and (b) permanently, if the copyright 419 | holder fails to notify you of the violation by some reasonable means 420 | prior to 60 days after the cessation. 421 | 422 | Moreover, your license from a particular copyright holder is 423 | reinstated permanently if the copyright holder notifies you of the 424 | violation by some reasonable means, this is the first time you have 425 | received notice of violation of this License (for any work) from that 426 | copyright holder, and you cure the violation prior to 30 days after 427 | your receipt of the notice. 428 | 429 | Termination of your rights under this section does not terminate the 430 | licenses of parties who have received copies or rights from you under 431 | this License. If your rights have been terminated and not permanently 432 | reinstated, you do not qualify to receive new licenses for the same 433 | material under section 10. 434 | 435 | 9. Acceptance Not Required for Having Copies. 436 | 437 | You are not required to accept this License in order to receive or 438 | run a copy of the Program. Ancillary propagation of a covered work 439 | occurring solely as a consequence of using peer-to-peer transmission 440 | to receive a copy likewise does not require acceptance. However, 441 | nothing other than this License grants you permission to propagate or 442 | modify any covered work. These actions infringe copyright if you do 443 | not accept this License. Therefore, by modifying or propagating a 444 | covered work, you indicate your acceptance of this License to do so. 445 | 446 | 10. Automatic Licensing of Downstream Recipients. 447 | 448 | Each time you convey a covered work, the recipient automatically 449 | receives a license from the original licensors, to run, modify and 450 | propagate that work, subject to this License. You are not responsible 451 | for enforcing compliance by third parties with this License. 452 | 453 | An "entity transaction" is a transaction transferring control of an 454 | organization, or substantially all assets of one, or subdividing an 455 | organization, or merging organizations. If propagation of a covered 456 | work results from an entity transaction, each party to that 457 | transaction who receives a copy of the work also receives whatever 458 | licenses to the work the party's predecessor in interest had or could 459 | give under the previous paragraph, plus a right to possession of the 460 | Corresponding Source of the work from the predecessor in interest, if 461 | the predecessor has it or can get it with reasonable efforts. 462 | 463 | You may not impose any further restrictions on the exercise of the 464 | rights granted or affirmed under this License. For example, you may 465 | not impose a license fee, royalty, or other charge for exercise of 466 | rights granted under this License, and you may not initiate litigation 467 | (including a cross-claim or counterclaim in a lawsuit) alleging that 468 | any patent claim is infringed by making, using, selling, offering for 469 | sale, or importing the Program or any portion of it. 470 | 471 | 11. Patents. 472 | 473 | A "contributor" is a copyright holder who authorizes use under this 474 | License of the Program or a work on which the Program is based. The 475 | work thus licensed is called the contributor's "contributor version". 476 | 477 | A contributor's "essential patent claims" are all patent claims 478 | owned or controlled by the contributor, whether already acquired or 479 | hereafter acquired, that would be infringed by some manner, permitted 480 | by this License, of making, using, or selling its contributor version, 481 | but do not include claims that would be infringed only as a 482 | consequence of further modification of the contributor version. For 483 | purposes of this definition, "control" includes the right to grant 484 | patent sublicenses in a manner consistent with the requirements of 485 | this License. 486 | 487 | Each contributor grants you a non-exclusive, worldwide, royalty-free 488 | patent license under the contributor's essential patent claims, to 489 | make, use, sell, offer for sale, import and otherwise run, modify and 490 | propagate the contents of its contributor version. 491 | 492 | In the following three paragraphs, a "patent license" is any express 493 | agreement or commitment, however denominated, not to enforce a patent 494 | (such as an express permission to practice a patent or covenant not to 495 | sue for patent infringement). To "grant" such a patent license to a 496 | party means to make such an agreement or commitment not to enforce a 497 | patent against the party. 498 | 499 | If you convey a covered work, knowingly relying on a patent license, 500 | and the Corresponding Source of the work is not available for anyone 501 | to copy, free of charge and under the terms of this License, through a 502 | publicly available network server or other readily accessible means, 503 | then you must either (1) cause the Corresponding Source to be so 504 | available, or (2) arrange to deprive yourself of the benefit of the 505 | patent license for this particular work, or (3) arrange, in a manner 506 | consistent with the requirements of this License, to extend the patent 507 | license to downstream recipients. "Knowingly relying" means you have 508 | actual knowledge that, but for the patent license, your conveying the 509 | covered work in a country, or your recipient's use of the covered work 510 | in a country, would infringe one or more identifiable patents in that 511 | country that you have reason to believe are valid. 512 | 513 | If, pursuant to or in connection with a single transaction or 514 | arrangement, you convey, or propagate by procuring conveyance of, a 515 | covered work, and grant a patent license to some of the parties 516 | receiving the covered work authorizing them to use, propagate, modify 517 | or convey a specific copy of the covered work, then the patent license 518 | you grant is automatically extended to all recipients of the covered 519 | work and works based on it. 520 | 521 | A patent license is "discriminatory" if it does not include within 522 | the scope of its coverage, prohibits the exercise of, or is 523 | conditioned on the non-exercise of one or more of the rights that are 524 | specifically granted under this License. You may not convey a covered 525 | work if you are a party to an arrangement with a third party that is 526 | in the business of distributing software, under which you make payment 527 | to the third party based on the extent of your activity of conveying 528 | the work, and under which the third party grants, to any of the 529 | parties who would receive the covered work from you, a discriminatory 530 | patent license (a) in connection with copies of the covered work 531 | conveyed by you (or copies made from those copies), or (b) primarily 532 | for and in connection with specific products or compilations that 533 | contain the covered work, unless you entered into that arrangement, 534 | or that patent license was granted, prior to 28 March 2007. 535 | 536 | Nothing in this License shall be construed as excluding or limiting 537 | any implied license or other defenses to infringement that may 538 | otherwise be available to you under applicable patent law. 539 | 540 | 12. No Surrender of Others' Freedom. 541 | 542 | If conditions are imposed on you (whether by court order, agreement or 543 | otherwise) that contradict the conditions of this License, they do not 544 | excuse you from the conditions of this License. If you cannot convey a 545 | covered work so as to satisfy simultaneously your obligations under this 546 | License and any other pertinent obligations, then as a consequence you may 547 | not convey it at all. For example, if you agree to terms that obligate you 548 | to collect a royalty for further conveying from those to whom you convey 549 | the Program, the only way you could satisfy both those terms and this 550 | License would be to refrain entirely from conveying the Program. 551 | 552 | 13. Use with the GNU Affero General Public License. 553 | 554 | Notwithstanding any other provision of this License, you have 555 | permission to link or combine any covered work with a work licensed 556 | under version 3 of the GNU Affero General Public License into a single 557 | combined work, and to convey the resulting work. The terms of this 558 | License will continue to apply to the part which is the covered work, 559 | but the special requirements of the GNU Affero General Public License, 560 | section 13, concerning interaction through a network will apply to the 561 | combination as such. 562 | 563 | 14. Revised Versions of this License. 564 | 565 | The Free Software Foundation may publish revised and/or new versions of 566 | the GNU General Public License from time to time. Such new versions will 567 | be similar in spirit to the present version, but may differ in detail to 568 | address new problems or concerns. 569 | 570 | Each version is given a distinguishing version number. If the 571 | Program specifies that a certain numbered version of the GNU General 572 | Public License "or any later version" applies to it, you have the 573 | option of following the terms and conditions either of that numbered 574 | version or of any later version published by the Free Software 575 | Foundation. If the Program does not specify a version number of the 576 | GNU General Public License, you may choose any version ever published 577 | by the Free Software Foundation. 578 | 579 | If the Program specifies that a proxy can decide which future 580 | versions of the GNU General Public License can be used, that proxy's 581 | public statement of acceptance of a version permanently authorizes you 582 | to choose that version for the Program. 583 | 584 | Later license versions may give you additional or different 585 | permissions. However, no additional obligations are imposed on any 586 | author or copyright holder as a result of your choosing to follow a 587 | later version. 588 | 589 | 15. Disclaimer of Warranty. 590 | 591 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 592 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 596 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 597 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 599 | 600 | 16. Limitation of Liability. 601 | 602 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 610 | SUCH DAMAGES. 611 | 612 | 17. Interpretation of Sections 15 and 16. 613 | 614 | If the disclaimer of warranty and limitation of liability provided 615 | above cannot be given local legal effect according to their terms, 616 | reviewing courts shall apply local law that most closely approximates 617 | an absolute waiver of all civil liability in connection with the 618 | Program, unless a warranty or assumption of liability accompanies a 619 | copy of the Program in return for a fee. 620 | 621 | END OF TERMS AND CONDITIONS 622 | 623 | How to Apply These Terms to Your New Programs 624 | 625 | If you develop a new program, and you want it to be of the greatest 626 | possible use to the public, the best way to achieve this is to make it 627 | free software which everyone can redistribute and change under these terms. 628 | 629 | To do so, attach the following notices to the program. It is safest 630 | to attach them to the start of each source file to most effectively 631 | state the exclusion of warranty; and each file should have at least 632 | the "copyright" line and a pointer to where the full notice is found. 633 | 634 | 635 | Copyright (C) 636 | 637 | This program is free software: you can redistribute it and/or modify 638 | it under the terms of the GNU General Public License as published by 639 | the Free Software Foundation, either version 3 of the License, or 640 | (at your option) any later version. 641 | 642 | This program is distributed in the hope that it will be useful, 643 | but WITHOUT ANY WARRANTY; without even the implied warranty of 644 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 645 | GNU General Public License for more details. 646 | 647 | You should have received a copy of the GNU General Public License 648 | along with this program. If not, see . 649 | 650 | Also add information on how to contact you by electronic and paper mail. 651 | 652 | If the program does terminal interaction, make it output a short 653 | notice like this when it starts in an interactive mode: 654 | 655 | Copyright (C) 656 | This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 657 | This is free software, and you are welcome to redistribute it 658 | under certain conditions; type `show c' for details. 659 | 660 | The hypothetical commands `show w' and `show c' should show the appropriate 661 | parts of the General Public License. Of course, your program's commands 662 | might be different; for a GUI interface, you would use an "about box". 663 | 664 | You should also get your employer (if you work as a programmer) or school, 665 | if any, to sign a "copyright disclaimer" for the program, if necessary. 666 | For more information on this, and how to apply and follow the GNU GPL, see 667 | . 668 | 669 | The GNU General Public License does not permit incorporating your program 670 | into proprietary programs. If your program is a subroutine library, you 671 | may consider it more useful to permit linking proprietary applications with 672 | the library. If this is what you want to do, use the GNU Lesser General 673 | Public License instead of this License. But first, please read 674 | . 675 | --------------------------------------------------------------------------------