├── .github ├── dependabot.yml └── workflows │ ├── codespell.yml │ ├── dummy-portageq.sh │ ├── pre-commit-detect-outdated.yml │ ├── pre-commit.yml │ └── run-tests.yml ├── .gitignore ├── .pre-commit-config.yaml ├── MANIFEST.in ├── Makefile ├── README.md ├── fetchcommandwrapper ├── __init__.py ├── __main__.py └── version.py ├── make.conf ├── ruff.toml └── setup.py /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | 4 | - package-ecosystem: "github-actions" 5 | commit-message: 6 | include: "scope" 7 | prefix: "Actions" 8 | directory: "/" 9 | labels: 10 | - "enhancement" 11 | schedule: 12 | interval: "weekly" 13 | -------------------------------------------------------------------------------- /.github/workflows/codespell.yml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2024 Sebastian Pipping 2 | # Licensed under GPL v3 or later 3 | 4 | name: Enforce codespell-clean spelling 5 | 6 | on: 7 | pull_request: 8 | push: 9 | schedule: 10 | - cron: '0 2 * * 5' # Every Friday at 2am 11 | workflow_dispatch: 12 | 13 | # Drop permissions to minimum, for security 14 | permissions: 15 | contents: read 16 | 17 | jobs: 18 | codespell: 19 | name: Enforce codespell-clean spelling 20 | runs-on: ubuntu-22.04 21 | steps: 22 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 23 | - uses: codespell-project/actions-codespell@406322ec52dd7b488e48c1c4b82e2a8b3a1bf630 # v2.1 24 | -------------------------------------------------------------------------------- /.github/workflows/dummy-portageq.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | mirrors=( 3 | http://ftp.spline.inf.fu-berlin.de/mirrors/gentoo/ 4 | http://ftp-stud.hs-esslingen.de/pub/Mirrors/gentoo/ 5 | http://ftp.uni-erlangen.de/pub/mirrors/gentoo/ 6 | http://ftp.halifax.rwth-aachen.de/gentoo/ 7 | http://linux.rz.ruhr-uni-bochum.de/download/gentoo-mirror/ 8 | ftp://ftp.wh2.tu-dresden.de/pub/mirrors/gentoo/ 9 | ftp://sunsite.informatik.rwth-aachen.de/pub/Linux/gentoo/ 10 | ftp://ftp.tu-clausthal.de/pub/linux/gentoo/ 11 | ) 12 | echo "${mirrors[@]}" 13 | -------------------------------------------------------------------------------- /.github/workflows/pre-commit-detect-outdated.yml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2024 Sebastian Pipping 2 | # Licensed under GPL v3 or later 3 | 4 | name: Detect outdated pre-commit hooks 5 | 6 | on: 7 | schedule: 8 | - cron: '0 16 * * 5' # Every Friday 4pm 9 | workflow_dispatch: 10 | 11 | # NOTE: This will drop all permissions from GITHUB_TOKEN except metadata read, 12 | # and then (re)add the ones listed below: 13 | permissions: 14 | contents: write 15 | pull-requests: write 16 | 17 | jobs: 18 | pre_commit_detect_outdated: 19 | name: Detect outdated pre-commit hooks 20 | runs-on: ubuntu-latest 21 | steps: 22 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 23 | 24 | - name: Set up Python 3.13 25 | uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 26 | with: 27 | python-version: 3.13 28 | 29 | - name: Install pre-commit 30 | run: |- 31 | pip install \ 32 | --disable-pip-version-check \ 33 | --no-warn-script-location \ 34 | --user \ 35 | pre-commit 36 | echo "PATH=${HOME}/.local/bin:${PATH}" >> "${GITHUB_ENV}" 37 | 38 | - name: Check for outdated hooks 39 | run: |- 40 | pre-commit autoupdate 41 | git diff -- .pre-commit-config.yaml 42 | 43 | - name: Create pull request from changes (if any) 44 | id: create-pull-request 45 | uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8 46 | with: 47 | author: 'pre-commit ' 48 | base: master 49 | body: |- 50 | For your consideration. 51 | 52 | :warning: Please **CLOSE AND RE-OPEN** this pull request so that [further workflow runs get triggered](https://github.com/peter-evans/create-pull-request/blob/main/docs/concepts-guidelines.md#triggering-further-workflow-runs) for this pull request. 53 | branch: precommit-autoupdate 54 | commit-message: "pre-commit: Autoupdate" 55 | delete-branch: true 56 | draft: true 57 | labels: enhancement 58 | title: "pre-commit: Autoupdate" 59 | 60 | - name: Log pull request URL 61 | if: "${{ steps.create-pull-request.outputs.pull-request-url }}" 62 | run: | 63 | echo "Pull request URL is: ${{ steps.create-pull-request.outputs.pull-request-url }}" 64 | -------------------------------------------------------------------------------- /.github/workflows/pre-commit.yml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2024 Sebastian Pipping 2 | # Licensed under GPL v3 or later 3 | 4 | name: Run pre-commit 5 | 6 | # Drop permissions to minimum, for security 7 | permissions: 8 | contents: read 9 | 10 | on: 11 | pull_request: 12 | push: 13 | schedule: 14 | - cron: '0 3 * * 5' # Every Friday at 3am 15 | workflow_dispatch: 16 | 17 | jobs: 18 | pre-commit: 19 | name: Run pre-commit 20 | runs-on: ubuntu-latest 21 | steps: 22 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 23 | - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 24 | with: 25 | python-version: 3.13 26 | - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 27 | -------------------------------------------------------------------------------- /.github/workflows/run-tests.yml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2024 Sebastian Pipping 2 | # Licensed under GPL v3 or later 3 | 4 | name: Run the test suite 5 | 6 | # Drop permissions to minimum, for security 7 | permissions: 8 | contents: read 9 | 10 | on: 11 | pull_request: 12 | push: 13 | schedule: 14 | - cron: '0 3 * * 5' # Every Friday at 3am 15 | workflow_dispatch: 16 | 17 | jobs: 18 | run-tests: 19 | name: Run the test suite 20 | strategy: 21 | matrix: 22 | python-version: [3.9, 3.13] # no particular need for in-between versions 23 | runs-on: ubuntu-latest 24 | steps: 25 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 26 | - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 27 | with: 28 | python-version: ${{ matrix.python-version }} 29 | - name: Install dependencies 30 | run: | 31 | sudo apt-get update 32 | sudo apt-get install --no-install-recommends --yes -V \ 33 | aria2 34 | 35 | - name: Install fetchcommandwrapper 36 | run: | 37 | set -x -u 38 | pip3 install -e . 39 | echo "${HOME}/.local/bin" >> "${GITHUB_PATH}" 40 | 41 | - name: Install dummy portageq 42 | run: | 43 | sudo cp -v .github/workflows/dummy-portageq.sh /usr/bin/portageq 44 | sudo chmod a+x /usr/bin/portageq 45 | 46 | - name: Run smoke tests 47 | run: | 48 | set -x 49 | 50 | python3 --version 51 | head -n1 "$(type -P fetchcommandwrapper)" 52 | 53 | fetchcommandwrapper --help 54 | fetchcommandwrapper --version 55 | 56 | args=( 57 | --link-speed 100000 58 | http://ftp.spline.inf.fu-berlin.de/mirrors/gentoo/distfiles/4f/isomaster-1.3.17.tar.bz2 59 | ./ 60 | isomaster-1.3.17.tar.bz2 61 | ) 62 | 63 | fetchcommandwrapper --fresh "${args[@]}" 64 | fetchcommandwrapper --continue "${args[@]}" 65 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /dist 2 | /MANIFEST 3 | *.pyc 4 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2024 Sebastian Pipping 2 | # Licensed under GPL v3 or later 3 | 4 | repos: 5 | - repo: https://github.com/pre-commit/pre-commit-hooks 6 | rev: v5.0.0 7 | hooks: 8 | - id: check-merge-conflict 9 | - id: check-toml 10 | - id: check-yaml 11 | - id: end-of-file-fixer 12 | - id: trailing-whitespace 13 | 14 | - repo: https://github.com/astral-sh/ruff-pre-commit 15 | rev: v0.11.12 16 | hooks: 17 | - id: ruff 18 | args: 19 | - --fix 20 | - id: ruff-format 21 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include make.conf 2 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | dist: 2 | rm -f MANIFEST 3 | python3 setup.py sdist 4 | 5 | clean: 6 | find -type f -name '*.pyc' -delete 7 | 8 | .PHONY: clean dist 9 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # About 2 | 3 | **fetchcommandwrapper** combines 4 | download tool [aria2](https://aria2.github.io/) 5 | with variable `GENTOO_MIRRORS` 6 | to speed up distfile downloads of [Portage](https://wiki.gentoo.org/wiki/Portage) 7 | by downloading from multiple mirrors at the same time. 8 | **fetchcommandwrapper** integrates with Portage 9 | through variables `FETCHCOMMAND` (and `RESUMECOMMAND`), hence the name. 10 | 11 | 12 | # Installation 13 | 14 | ## System-wide installation in Gentoo 15 | 16 | ```console 17 | # sudo emerge -av app-portage/fetchcommandwrapper 18 | ``` 19 | You can then append line `source /usr/share/fetchcommandwrapper/make.conf` 20 | to file `/etc/portage/make.conf` to ease integration with Portage. 21 | 22 | 23 | ## Development from a Git clone 24 | 25 | ```console 26 | # pip install --user --editable . 27 | ``` 28 | -------------------------------------------------------------------------------- /fetchcommandwrapper/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hartwork/fetchcommandwrapper/fca607b44789c43966aa57abc208af26ee2eb349/fetchcommandwrapper/__init__.py -------------------------------------------------------------------------------- /fetchcommandwrapper/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (C) 2010-2017 Sebastian Pipping 3 | # Copyright (C) 2010 Andrew Karpow 4 | # Licensed under GPL v3 or later 5 | 6 | import os 7 | import sys 8 | from signal import SIGINT 9 | from textwrap import dedent 10 | 11 | MAX_STREAMS = 5 12 | ARIA2_COMMAND = "/usr/bin/aria2c" 13 | 14 | 15 | def print_greeting(): 16 | from fetchcommandwrapper.version import VERSION_STR # noqa: I001 17 | 18 | print("fetchcommandwrapper %s" % VERSION_STR) 19 | print() 20 | 21 | 22 | def parse_parameters(): 23 | USAGE = "\n %(prog)s [OPTIONS] URI DISTDIR FILE [-- [ARG ..]]" 24 | 25 | import argparse # noqa: I001 26 | from fetchcommandwrapper.version import VERSION_STR # noqa: I001 27 | 28 | parser = argparse.ArgumentParser(prog="fetchcommandwrapper", usage=USAGE) 29 | parser.add_argument("--version", action="version", version=VERSION_STR) 30 | 31 | group = parser.add_mutually_exclusive_group() 32 | group.add_argument( 33 | "-c", 34 | "--continue", 35 | action="store_true", 36 | dest="continue_flag", 37 | help="continue previous download", 38 | ) 39 | group.add_argument( 40 | "--fresh", 41 | action="store_false", 42 | dest="continue_flag", 43 | default=False, 44 | help="do not continue previous download (default)", 45 | ) 46 | parser.add_argument( 47 | "--link-speed", 48 | type=int, 49 | metavar="BYTES", 50 | dest="link_speed_bytes", 51 | help="specify link speed (bytes per second). enables dropping of slow connections.", 52 | ) 53 | parser.add_argument("uri", metavar="URI", help=argparse.SUPPRESS) 54 | parser.add_argument("distdir", metavar="DISTDIR", help=argparse.SUPPRESS) 55 | parser.add_argument("file_basename", metavar="FILE", help=argparse.SUPPRESS) 56 | 57 | opts, opts.argv_extra = parser.parse_known_args() 58 | 59 | opts.distdir = opts.distdir.rstrip("/") 60 | 61 | opts.file_fullpath = os.path.join(opts.distdir, opts.file_basename) 62 | 63 | return opts 64 | 65 | 66 | def invoke_wget(opts): 67 | args = ["/usr/bin/wget", "-O", opts.file_fullpath] 68 | args.append("--tries=5") 69 | args.append("--timeout=60") 70 | args.append("--passive-ftp") 71 | if opts.continue_flag: 72 | args.append("--continue") 73 | args.extend(opts.argv_extra) 74 | args.append(opts.uri) 75 | 76 | # Invoke wget 77 | print("Running... # %s" % " ".join(args)) 78 | import subprocess 79 | 80 | return subprocess.call(args) 81 | 82 | 83 | def print_invocation_details(opts): 84 | print("URI = %s" % opts.uri) 85 | print("DISTDIR = %s" % opts.distdir) 86 | print("FILE = %s" % opts.file_basename) 87 | print() 88 | 89 | 90 | def gentoo_mirrors(): 91 | import subprocess 92 | 93 | p = subprocess.Popen( 94 | ["/usr/bin/portageq", "gentoo_mirrors"], stdout=subprocess.PIPE, stderr=subprocess.PIPE 95 | ) 96 | (out, err) = p.communicate() 97 | if err: 98 | print("ERROR %s" % err.decode("UTF-8"), file=sys.stderr) 99 | sys.exit(1) 100 | return out.decode("UTF-8").rstrip("\n").split(" ") 101 | 102 | 103 | def supported(uri): 104 | return uri.startswith("http://") or uri.startswith("https://") or uri.startswith("ftp://") 105 | 106 | 107 | def print_mirror_details(supported_mirror_uris): 108 | print("Involved mirrors:") 109 | print("\n".join(" - " + e for e in supported_mirror_uris)) 110 | print(" (%d mirrors)" % len(supported_mirror_uris)) 111 | print() 112 | 113 | if len(supported_mirror_uris) < MAX_STREAMS: 114 | print( 115 | dedent( 116 | """\ 117 | WARNING: Please specify at least %d URIs in GENTOO_MIRRORS. 118 | The more the better.' 119 | """ 120 | % MAX_STREAMS 121 | ), 122 | file=sys.stderr, 123 | ) 124 | 125 | 126 | def make_final_uris(uri, supported_mirror_uris): 127 | final_uris = [ 128 | uri, 129 | ] 130 | mirrors_involved = False 131 | 132 | if not uri.endswith("/distfiles/layout.conf"): 133 | for i, mirror_uri in enumerate(supported_mirror_uris): 134 | if uri.startswith(mirror_uri): 135 | if i != 0: 136 | # Portage calls us for each mirror URI. Therefore we need 137 | # to error out on all but the first one, so we try each 138 | # mirrror once, at most. 139 | # This happens, when a file is not mirrored, e.g. with 140 | # sunrise ebuilds. 141 | print("ERROR: All Gentoo mirrors tried already, exiting.", file=sys.stderr) 142 | sys.exit(1) 143 | 144 | mirrors_involved = True 145 | 146 | local_part = uri[len(mirror_uri) :] 147 | final_uris = [e + local_part for e in supported_mirror_uris] 148 | import random 149 | 150 | random.shuffle(final_uris) 151 | break 152 | 153 | return final_uris, mirrors_involved 154 | 155 | 156 | def invoke_aria2(opts, final_uris): 157 | # Compose call arguments 158 | wanted_connections = min(MAX_STREAMS, len(final_uris)) 159 | if opts.link_speed_bytes and (len(final_uris) > MAX_STREAMS): 160 | drop_slow_links = True 161 | else: 162 | drop_slow_links = False 163 | 164 | if len(final_uris) > 1: 165 | print( 166 | "Targeting %d random connections, additional %d for backup" 167 | % (wanted_connections, max(0, len(final_uris) - MAX_STREAMS)) 168 | ) 169 | print() 170 | 171 | args = [ARIA2_COMMAND, "-d", opts.distdir, "-o", opts.file_basename] 172 | if drop_slow_links: 173 | wanted_minimum_link_speed = int(opts.link_speed_bytes / wanted_connections / 3) 174 | args.append("--lowest-speed-limit=%s" % wanted_minimum_link_speed) 175 | if opts.continue_flag: 176 | args.append("--continue") 177 | args.append("--allow-overwrite=true") 178 | args.append("--max-tries=5") 179 | args.append("--max-file-not-found=2") 180 | args.append("--user-agent=Wget/1.19.1") 181 | args.append("--split=%d" % wanted_connections) 182 | args.append("--max-connection-per-server=1") 183 | args.append("--uri-selector=inorder") 184 | args.append("--follow-metalink=mem") # e.g. GNOME servers with mirrorbrain 185 | args.extend(opts.argv_extra) 186 | args.extend(final_uris) 187 | 188 | # Invoke aria2 189 | print("Running... # %s" % " ".join(args)) 190 | import subprocess 191 | 192 | return subprocess.call(args) 193 | 194 | 195 | def _inner_main(): 196 | opts = parse_parameters() 197 | 198 | if not os.path.exists(ARIA2_COMMAND): 199 | print("ERROR: net-misc/aria2 not installed, falling back to net-misc/wget", file=sys.stderr) 200 | ret = invoke_wget(opts) 201 | sys.exit(ret) 202 | 203 | if not os.path.isdir(opts.distdir): 204 | print('ERROR: Path "%s" not a directory' % opts.distdir, file=sys.stderr) 205 | sys.exit(1) 206 | 207 | if opts.continue_flag: 208 | if not os.path.isfile(opts.file_fullpath): 209 | print('ERROR: Path "%s" not an existing file' % opts.file_fullpath, file=sys.stderr) 210 | sys.exit(1) 211 | 212 | supported_mirror_uris = [e for e in gentoo_mirrors() if supported(e)] 213 | 214 | final_uris, mirrors_involved = make_final_uris(opts.uri, supported_mirror_uris) 215 | 216 | print_greeting() 217 | print_invocation_details(opts) 218 | 219 | if mirrors_involved: 220 | print_mirror_details(supported_mirror_uris) 221 | 222 | ret = invoke_aria2(opts, final_uris) 223 | sys.exit(ret) 224 | 225 | 226 | def main(): 227 | try: 228 | _inner_main() 229 | except KeyboardInterrupt: 230 | sys.exit(128 + SIGINT) 231 | 232 | 233 | if __name__ == "__main__": 234 | main() 235 | -------------------------------------------------------------------------------- /fetchcommandwrapper/version.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2010-2020 Sebastian Pipping 2 | # Licensed under GPL v3 or later 3 | 4 | VERSION = (0, 8, 4) 5 | VERSION_STR = ".".join(str(e) for e in VERSION) 6 | -------------------------------------------------------------------------------- /make.conf: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2010-2017 Sebastian Pipping 2 | # Licensed under GPL v3 or later 3 | 4 | ### Append "source /usr/share/fetchcommandwrapper/make.conf" 5 | ### to /etc/portage/make.conf to integrate fetchcommandwrapper. 6 | ### 7 | ### Specify additional parameters _before_ that using 8 | ### FETCHCOMMANDWRAPPER_OPTIONS in /etc/make.conf 9 | 10 | FETCHCOMMAND="/usr/bin/fetchcommandwrapper ${FETCHCOMMANDWRAPPER_OPTIONS} \${URI} \${DISTDIR} \${FILE} -- ${FETCHCOMMANDWRAPPER_EXTRA}" 11 | RESUMECOMMAND="/usr/bin/fetchcommandwrapper ${FETCHCOMMANDWRAPPER_OPTIONS} --continue \${URI} \${DISTDIR} \${FILE} -- ${FETCHCOMMANDWRAPPER_EXTRA}" 12 | -------------------------------------------------------------------------------- /ruff.toml: -------------------------------------------------------------------------------- 1 | indent-width = 4 2 | line-length = 100 3 | target-version = "py39" 4 | 5 | [lint] 6 | select = [ 7 | "E", # pycodestyle 8 | "F", # Pyflakes + flake8 9 | "I", # isort 10 | "UP", # pyupgrade 11 | "W", # pycodestyle 12 | ] 13 | ignore = [ 14 | "UP031", # Use format specifiers instead of percent format; TODO drop 15 | ] 16 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (C) 2010 Sebastian Pipping 3 | # Licensed under GPL v3 or later 4 | 5 | import sys 6 | 7 | from setuptools import find_packages, setup 8 | 9 | sys.path.insert(0, "modules") 10 | from fetchcommandwrapper.version import VERSION_STR 11 | 12 | setup( 13 | name="fetchcommandwrapper", 14 | description="Wrapper around Aria2 for portage's FETCHCOMMAND variable", 15 | long_description=open("README.md").read(), 16 | long_description_content_type="text/markdown", 17 | license="GPL v3 or later", 18 | version=VERSION_STR, 19 | url="https://github.com/hartwork/fetchcommandwrapper", 20 | author="Sebastian Pipping", 21 | author_email="sping@gentoo.org", 22 | python_requires=">=3.9", 23 | setup_requires=[ 24 | "setuptools>=38.6.0", # for long_description_content_type 25 | ], 26 | packages=find_packages(), 27 | entry_points={ 28 | "console_scripts": [ 29 | "fetchcommandwrapper = fetchcommandwrapper.__main__:main", 30 | ], 31 | }, 32 | data_files=[ 33 | ( 34 | "share/fetchcommandwrapper", 35 | [ 36 | "make.conf", 37 | ], 38 | ), 39 | ], 40 | classifiers=[ 41 | "Development Status :: 4 - Beta", 42 | "License :: OSI Approved", 43 | "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", 44 | "Intended Audience :: End Users/Desktop", 45 | "Intended Audience :: System Administrators", 46 | "Programming Language :: Python", 47 | "Programming Language :: Python :: 3", 48 | "Programming Language :: Python :: 3.9", 49 | "Programming Language :: Python :: 3.10", 50 | "Programming Language :: Python :: 3.11", 51 | "Programming Language :: Python :: 3.12", 52 | "Programming Language :: Python :: 3.13", 53 | "Programming Language :: Python :: 3 :: Only", 54 | "Topic :: Internet", 55 | "Topic :: Internet :: File Transfer Protocol (FTP)", 56 | "Topic :: Internet :: WWW/HTTP", 57 | ], 58 | ) 59 | --------------------------------------------------------------------------------