├── .github ├── FUNDING.yml └── workflows │ ├── jekyll-gh-pages.yml │ └── tests.yml ├── .gitignore ├── LICENSE ├── README.md ├── abx_pkg ├── __init__.py ├── admin.py ├── apps.py ├── base_types.py ├── binary.py ├── binprovider.py ├── binprovider_ansible.py ├── binprovider_apt.py ├── binprovider_brew.py ├── binprovider_npm.py ├── binprovider_pip.py ├── binprovider_pyinfra.py ├── models.py ├── semver.py ├── settings.py ├── shallowbinary.py ├── tests.py └── views.py ├── bin └── publish.sh ├── django_example_project ├── README.md ├── manage.py └── project │ ├── __init__.py │ ├── admin.py │ ├── asgi.py │ ├── migrations │ ├── 0001_initial.py │ ├── 0002_alter_dependency_options_dependency_min_version.py │ └── __init__.py │ ├── models.py │ ├── settings.py │ ├── urls.py │ ├── views.py │ └── wsgi.py ├── pyproject.toml ├── tests.py └── uv.lock /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: ["ArchiveBox", "pirate"] 4 | custom: ["https://donate.archivebox.io/"] 5 | -------------------------------------------------------------------------------- /.github/workflows/jekyll-gh-pages.yml: -------------------------------------------------------------------------------- 1 | # Sample workflow for building and deploying a Jekyll site to GitHub Pages 2 | name: Deploy Jekyll with GitHub Pages dependencies preinstalled 3 | 4 | on: 5 | # Runs on pushes targeting the default branch 6 | push: 7 | branches: ["main"] 8 | 9 | # Allows you to run this workflow manually from the Actions tab 10 | workflow_dispatch: 11 | 12 | # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages 13 | permissions: 14 | contents: read 15 | pages: write 16 | id-token: write 17 | 18 | # Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. 19 | # However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. 20 | concurrency: 21 | group: "pages" 22 | cancel-in-progress: false 23 | 24 | jobs: 25 | # Build job 26 | build: 27 | runs-on: ubuntu-latest 28 | steps: 29 | - name: Checkout 30 | uses: actions/checkout@v4 31 | - name: Setup Pages 32 | uses: actions/configure-pages@v5 33 | - name: Build with Jekyll 34 | uses: actions/jekyll-build-pages@v1 35 | with: 36 | source: ./ 37 | destination: ./_site 38 | - name: Upload artifact 39 | uses: actions/upload-pages-artifact@v3 40 | 41 | # Deployment job 42 | deploy: 43 | environment: 44 | name: github-pages 45 | url: ${{ steps.deployment.outputs.page_url }} 46 | runs-on: ubuntu-latest 47 | needs: build 48 | steps: 49 | - name: Deploy to GitHub Pages 50 | id: deployment 51 | uses: actions/deploy-pages@v4 52 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: Run Tests 2 | 3 | on: 4 | push: 5 | branches: [ "main" ] 6 | pull_request: 7 | branches: [ "main" ] 8 | workflow_dispatch: 9 | 10 | permissions: 11 | contents: read 12 | 13 | jobs: 14 | build: 15 | runs-on: ${{ matrix.os }} 16 | strategy: 17 | fail-fast: true 18 | matrix: 19 | python_version: ['3.10', '3.11', '3.12'] 20 | os: [ubuntu-latest, macOS-latest] # TODO: windows-latest 21 | 22 | steps: 23 | - uses: actions/checkout@v4 24 | 25 | - name: Install uv 26 | uses: astral-sh/setup-uv@v3 27 | with: 28 | enable-cache: true 29 | cache-dependency-glob: "uv.lock" 30 | 31 | - name: Setup venv and install pip dependencies 32 | run: | 33 | uv venv \ 34 | && uv sync \ 35 | && uv pip install pip \ 36 | && echo "/home/linuxbrew/.linuxbrew/bin" >> "$GITHUB_PATH" 37 | 38 | - name: Run tests 39 | run: | 40 | source .venv/bin/activate \ 41 | && python tests.py 42 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pypackages__/ 2 | __pycache__/ 3 | .mypy_cache/ 4 | *.py[cod] 5 | dist/ 6 | .pdm-build 7 | .pdm-python 8 | 9 | venv/ 10 | .venv/ 11 | 12 | .vscode 13 | *.code-workspace 14 | .env 15 | *.log 16 | .DS_Store 17 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Nick Sweeting 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

abx-pkg         📦 apt  brew  pip  npm  ₊₊₊
Simple Python interfaces for package managers + installed binaries.

2 |
3 | 4 | [![PyPI][pypi-badge]][pypi] 5 | [![Python Version][version-badge]][pypi] 6 | [![Django Version][django-badge]][pypi] 7 | [![GitHub][licence-badge]][licence] 8 | [![GitHub Last Commit][repo-badge]][repo] 9 | 10 | 11 |
12 | 13 | **It's an ORM for your package managers, providing a nice python types for packages + installers.** 14 | 15 | **This is a [Python library](https://pypi.org/project/abx-pkg/) for installing & managing packages locally with a variety of package managers.** 16 | It's designed for when `requirements.txt` isn't enough, and you have to detect or install dependencies at runtime. It's great for installing and managing MCP servers and their dependencies at runtime. 17 | 18 | 19 | ```python 20 | pip install abx-pkg 21 | 22 | python 23 | >>> from abx_pkg import Binary, NpmProvider 24 | 25 | >>> curl = Binary('curl').load() 26 | >>> print(curl.abspath, curl.version, curl.exec(cmd=['--version'])) 27 | /usr/bin/curl 7.81.0 curl 7.81.0 (x86_64-apple-darwin23.0) libcurl/7.81.0 ... 28 | 29 | >>> NpmProvider().install('puppeteer') 30 | ``` 31 | 32 | 33 | > ✨ Built with [`pydantic`](https://pydantic-docs.helpmanual.io/) v2 for strong static typing guarantees and json import/export compatibility 34 | > 📦 Provides consistent cross-platform interfaces for dependency resolution & installation at runtime 35 | > 🌈 Integrates with [`django`](https://docs.djangoproject.com/en/5.0/) >= 4.0, [`django-ninja`](https://django-ninja.dev/), and OpenAPI + [`django-jsonform`](https://django-jsonform.readthedocs.io/) out-of-the-box 36 | > 🦄 Lets you use the builtin `abx-pkg` methods, or [`pyinfra`](https://github.com/pyinfra-dev/pyinfra) / [`ansible`](https://github.com/ansible/ansible) for the install operations 37 | 38 | Built by ArchiveBox to install & auto-update our extractor dependencies at runtime (chrome, wget, curl, etc.) on `macOS`/`Linux`/`Docker`. 39 | 40 |
41 | 42 | > [!WARNING] 43 | > This is `BETA` software, the API is mostly stable but there may be minor changes later on. 44 | 45 | 46 | **Source Code**: [https://github.com/ArchiveBox/abx-pkg/](https://github.com/ArchiveBox/abx-pkg/) 47 | 48 | **Documentation**: [https://github.com/ArchiveBox/abx-pkg/blob/main/README.md](https://github.com/ArchiveBox/abx-pkg/blob/main/README.md) 49 | 50 |
51 | 52 | ```python 53 | from abx_pkg import * 54 | 55 | apt, brew, pip, npm, env = AptProvider(), BrewProvider(), PipProvider(), NpmProvider(), EnvProvider() 56 | 57 | dependencies = [ 58 | Binary(name='curl', binproviders=[env, apt, brew]), 59 | Binary(name='wget', binproviders=[env, apt, brew]), 60 | Binary(name='yt-dlp', binproviders=[env, pip, apt, brew]), 61 | Binary(name='playwright', binproviders=[env, pip, npm]), 62 | Binary(name='puppeteer', binproviders=[env, npm]), 63 | ] 64 | for binary in dependencies: 65 | binary = binary.load_or_install() 66 | 67 | print(binary.abspath, binary.version, binary.binprovider, binary.is_valid, binary.sha256) 68 | # Path('/usr/bin/curl') SemVer('7.81.0') AptProvider() True abc134... 69 | 70 | binary.exec(cmd=['--version']) # curl 7.81.0 (x86_64-apple-darwin23.0) libcurl/7.81.0 ... 71 | ``` 72 | 73 | ```python 74 | from pydantic import InstanceOf 75 | from abx_pkg import Binary, BinProvider, BrewProvider, EnvProvider 76 | 77 | # you can also define binaries as classes, making them usable for type checking 78 | class CurlBinary(Binary): 79 | name: str = 'curl' 80 | binproviders: list[InstanceOf[BinProvider]] = [BrewProvider(), EnvProvider()] 81 | 82 | curl = CurlBinary().install() 83 | assert isinstance(curl, CurlBinary) # CurlBinary is a unique type you can use in annotations now 84 | print(curl.abspath, curl.version, curl.binprovider, curl.is_valid) # Path('/opt/homebrew/bin/curl') SemVer('8.4.0') BrewProvider() True 85 | curl.exec(cmd=['--version']) # curl 8.4.0 (x86_64-apple-darwin23.0) libcurl/8.4.0 ... 86 | ``` 87 | 88 | ```python 89 | from abx_pkg import Binary, EnvProvider, PipProvider 90 | 91 | # We also provide direct package manager (aka BinProvider) APIs 92 | apt = AptProvider() 93 | apt.install('wget') 94 | print(apt.PATH, apt.get_abspaths('wget'), apt.get_version('wget')) 95 | 96 | # even if packages are installed by tools we don't control (e.g. pyinfra/ansible/puppet/etc.) 97 | from pyinfra.operations import apt 98 | apt.packages(name="Install ffmpeg", packages=['ffmpeg'], _sudo=True) 99 | 100 | # our Binary API provides a nice type-checkable, validated, serializable handle 101 | ffmpeg = Binary(name='ffmpeg').load() 102 | print(ffmpeg) # name=ffmpeg abspath=/usr/bin/ffmpeg version=3.3.0 is_valid=True binprovider=apt ... 103 | print(ffmpeg.abspaths) # show all the ffmpeg binaries found in $PATH (in case theres more than one available) 104 | print(ffmpeg.model_dump()) # ... everything can also be dumped/loaded as json-ready dict 105 | print(ffmpeg.model_json_schema()) # ... OpenAPI-ready JSON schema showing all available fields 106 | ``` 107 | 108 | ### Supported Package Managers 109 | 110 | **So far it supports `installing`/`finding installed`/~~`updating`/`removing`~~ packages on `Linux`/`macOS` with:** 111 | 112 | - `apt` (Ubuntu/Debian/etc.) 113 | - `brew` (macOS/Linux) 114 | - `pip` (Linux/macOS/Windows) 115 | - `npm` (Linux/macOS/Windows) 116 | - `env` (looks for existing version of binary in user's `$PATH` at runtime) 117 | - `vendor` (you can bundle vendored copies of packages you depend on within your source) 118 | 119 | *Planned:* `docker`, `cargo`, `nix`, `apk`, `go get`, `gem`, `pkg`, *and more using `ansible`/[`pyinfra`](https://github.com/pyinfra-dev/pyinfra)...* 120 | 121 | --- 122 | 123 | 124 | ## Usage 125 | 126 | ```bash 127 | pip install abx-pkg 128 | ``` 129 | 130 | ### [`BinProvider`](https://github.com/ArchiveBox/abx-pkg/blob/main/abx_pkg/binprovider.py#:~:text=class%20BinProvider) 131 | 132 | **Implementations: `EnvProvider`, `AptProvider`, `BrewProvider`, `PipProvider`, `NpmProvider`** 133 | 134 | This type represents a "provider of binaries", e.g. a package manager like `apt`/`pip`/`npm`, or `env` (which finds binaries in your `$PATH`). 135 | 136 | `BinProvider`s implement the following interface: 137 | * `.INSTALLER_BIN -> /opt/homebrew/bin/brew` provider's pkg manager location 138 | * `.PATH -> PATHStr('/opt/homebrew/bin:/usr/local/bin:...')` where provider stores bins 139 | * `get_packages(bin_name: str) -> InstallArgs(['curl', 'libcurl4', '...])` find pkg dependencies for a bin 140 | - `install(bin_name: str)` install a bin using binprovider to install needed packages 141 | - `load(bin_name: str)` find an existing installed binary 142 | - `load_or_install(bin_name: str)` `->` `Binary` find existing / install if needed 143 | - `get_version(bin_name: str) -> SemVer('1.0.0')` get currently installed version 144 | - `get_abspath(bin_name: str) -> Path('/absolute/path/to/bin')` get installed bin abspath 145 | * `get_abspaths(bin_name: str) -> [Path('/opt/homebrew/bin/curl'), Path('/other/paths/to/curl'), ...]` get all matching bins found 146 | * `get_sha256(bin_name: str) -> str` get sha256 hash hexdigest of the binary 147 | 148 | 149 | ```python 150 | import platform 151 | from typing import List 152 | from abx_pkg import EnvProvider, PipProvider, AptProvider, BrewProvider 153 | 154 | ### Example: Finding an existing install of bash using the system $PATH environment 155 | env = EnvProvider() 156 | bash = env.load(bin_name='bash') # Binary('bash', provider=env) 157 | print(bash.abspath) # Path('/opt/homebrew/bin/bash') 158 | print(bash.version) # SemVer('5.2.26') 159 | bash.exec(['-c', 'echo hi']) # hi 160 | 161 | ### Example: Installing curl using the apt package manager 162 | apt = AptProvider() 163 | curl = apt.install(bin_name='curl') # Binary('curl', provider=apt) 164 | print(curl.abspath) # Path('/usr/bin/curl') 165 | print(curl.version) # SemVer('8.4.0') 166 | print(curl.sha256) # 9fd780521c97365f94c90724d80a889097ae1eeb2ffce67b87869cb7e79688ec 167 | curl.exec(['--version']) # curl 7.81.0 (x86_64-pc-linux-gnu) libcurl/7.81.0 ... 168 | 169 | ### Example: Finding/Installing django with pip (w/ customized binpath resolution behavior) 170 | pip = PipProvider( 171 | abspath_handler={'*': lambda self, bin_name, **context: inspect.getfile(bin_name)}, # use python inspect to get path instead of os.which 172 | ) 173 | django_bin = pip.load_or_install('django') # Binary('django', provider=pip) 174 | print(django_bin.abspath) # Path('/usr/lib/python3.10/site-packages/django/__init__.py') 175 | print(django_bin.version) # SemVer('5.0.2') 176 | ``` 177 | 178 | ### [`Binary`](https://github.com/ArchiveBox/abx-pkg/blob/main/abx_pkg/binary.py#:~:text=class%20Binary) 179 | 180 | This type represents a single binary dependency aka a package (e.g. `wget`, `curl`, `ffmpeg`, etc.). 181 | It can define one or more `BinProvider`s that it supports, along with overrides to customize the behavior for each. 182 | 183 | `Binary`s implement the following interface: 184 | - `load()`, `install()`, `load_or_install()` `->` `Binary` 185 | - `binprovider: InstanceOf[BinProvider]` 186 | - `abspath: Path` 187 | - `abspaths: list[Path]` 188 | - `version: SemVer` 189 | - `sha256: str` 190 | 191 | ```python 192 | from abx_pkg import BinProvider, Binary, BinProviderName, BinName, ProviderLookupDict, SemVer 193 | 194 | class CustomBrewProvider(BrewProvider): 195 | name: str = 'custom_brew' 196 | 197 | def get_macos_packages(self, bin_name: str, **context) -> list[str]: 198 | extra_packages_lookup_table = json.load(Path('macos_packages.json')) 199 | return extra_packages_lookup_table.get(platform.machine(), [bin_name]) 200 | 201 | 202 | ### Example: Create a re-usable class defining a binary and its providers 203 | class YtdlpBinary(Binary): 204 | name: BinName = 'ytdlp' 205 | description: str = 'YT-DLP (Replacement for YouTube-DL) Media Downloader' 206 | 207 | binproviders_supported: list[BinProvider] = [EnvProvider(), PipProvider(), AptProvider(), CustomBrewProvider()] 208 | 209 | # customize installed package names for specific package managers 210 | provider_overrides: dict[BinProviderName, ProviderLookupDict] = { 211 | 'pip': {'packages': ['yt-dlp[default,curl-cffi]']}, # can use literal values (packages -> list[str], version -> SemVer, abspath -> Path, install -> str log) 212 | 'apt': {'packages': lambda: ['yt-dlp', 'ffmpeg']}, # also accepts any pure Callable that returns a list of packages 213 | 'brew': {'packages': 'self.get_macos_packages'}, # also accepts string reference to function on self (where self is the BinProvider) 214 | } 215 | 216 | 217 | ytdlp = YtdlpBinary().load_or_install() 218 | print(ytdlp.binprovider) # BrewProvider(...) 219 | print(ytdlp.abspath) # Path('/opt/homebrew/bin/yt-dlp') 220 | print(ytdlp.abspaths) # [Path('/opt/homebrew/bin/yt-dlp'), Path('/usr/local/bin/yt-dlp')] 221 | print(ytdlp.version) # SemVer('2024.4.9') 222 | print(ytdlp.sha256) # 46c3518cfa788090c42e379971485f56d007a6ce366dafb0556134ca724d6a36 223 | print(ytdlp.is_valid) # True 224 | ``` 225 | 226 | ```python 227 | from abx_pkg import BinProvider, Binary, BinProviderName, BinName, ProviderLookupDict, SemVer 228 | 229 | #### Example: Create a binary that uses Podman if available, or Docker otherwise 230 | class DockerBinary(Binary): 231 | name: BinName = 'docker' 232 | 233 | binproviders_supported: list[BinProvider] = [EnvProvider(), AptProvider()] 234 | 235 | provider_overrides: dict[BinProviderName, ProviderLookupDict] = { 236 | 'env': { 237 | # example: prefer podman if installed (falling back to docker) 238 | 'abspath': lambda: os.which('podman') or os.which('docker') or os.which('docker-ce'), 239 | }, 240 | 'apt': { 241 | # example: vary installed package name based on your CPU architecture 242 | 'packages': { 243 | 'amd64': ['docker'], 244 | 'armv7l': ['docker-ce'], 245 | 'arm64': ['docker-ce'], 246 | }.get(platform.machine(), 'docker'), 247 | }, 248 | } 249 | 250 | docker = DockerBinary().load_or_install() 251 | print(docker.binprovider) # EnvProvider() 252 | print(docker.abspath) # Path('/usr/local/bin/podman') 253 | print(docker.abspaths) # [Path('/usr/local/bin/podman'), Path('/opt/homebrew/bin/podman')] 254 | print(docker.version) # SemVer('6.0.2') 255 | print(docker.is_valid) # True 256 | 257 | # You can also pass **kwargs to override properties at runtime, 258 | # e.g. if you want to force the abspath to be at a specific path: 259 | custom_docker = DockerBinary(abspath='~/custom/bin/podman').load() 260 | print(custom_docker.name) # 'docker' 261 | print(custom_docker.binprovider) # EnvProvider() 262 | print(custom_docker.abspath) # Path('/Users/example/custom/bin/podman') 263 | print(custom_docker.version) # SemVer('5.0.2') 264 | print(custom_docker.is_valid) # True 265 | ``` 266 | 267 | ### [`SemVer`](https://github.com/ArchiveBox/abx-pkg/blob/main/abx_pkg/semver.py#:~:text=class%20SemVer) 268 | 269 | ```python 270 | from abx_pkg import SemVer 271 | 272 | ### Example: Use the SemVer type directly for parsing & verifying version strings 273 | SemVer.parse('Google Chrome 124.0.6367.208+beta_234. 234.234.123') # SemVer(124, 0, 6367') 274 | SemVer.parse('2024.04.05) # SemVer(2024, 4, 5) 275 | SemVer.parse('1.9+beta') # SemVer(1, 9, 0) 276 | str(SemVer(1, 9, 0)) # '1.9.0' 277 | ``` 278 |
279 | 280 | > These types are all meant to be used library-style to make writing your own apps easier. 281 | > e.g. you can use it to build things like: [`playwright install --with-deps`](https://playwright.dev/docs/browsers#install-system-dependencies)) 282 | 283 | 284 |
285 | 286 | --- 287 | --- 288 | 289 |
290 |
291 | 292 | 293 | ## Django Usage 294 | 295 | With a few more packages, you get type-checked Django fields & forms that support `BinProvider` and `Binary`. 296 | 297 | > [!TIP] 298 | > For the full Django experience, we recommend installing these 3 excellent packages: 299 | > - [`django-admin-data-views`](https://github.com/MrThearMan/django-admin-data-views) 300 | > - [`django-pydantic-field`](https://github.com/surenkov/django-pydantic-field) 301 | > - [`django-jsonform`](https://django-jsonform.readthedocs.io/) 302 | > `pip install abx-pkg django-admin-data-views django-pydantic-field django-jsonform` 303 | 304 |
305 | 306 | ### Django Model Usage: Store `BinProvider` and `Binary` entries in your model fields 307 | 308 | ```bash 309 | pip install django-pydantic-field 310 | ``` 311 | 312 | *Fore more info see the [`django-pydantic-field`](https://github.com/surenkov/django-pydantic-field) docs...* 313 | 314 | Example Django `models.py` showing how to store `Binary` and `BinProvider` instances in DB fields: 315 | ```python 316 | from typing import List 317 | from django.db import models 318 | from pydantic import InstanceOf 319 | from abx_pkg import BinProvider, Binary, SemVer 320 | from django_pydantic_field import SchemaField 321 | 322 | class InstalledBinary(models.Model): 323 | name = models.CharField(max_length=63) 324 | binary: Binary = SchemaField() 325 | binproviders: list[InstanceOf[BinProvider]] = SchemaField(default=[]) 326 | version: SemVer = SchemaField(default=(0,0,1)) 327 | ``` 328 | 329 | And here's how to save a `Binary` using the example model: 330 | ```python 331 | # find existing curl Binary in $PATH 332 | curl = Binary(name='curl').load() 333 | 334 | # save it to the DB using our new model 335 | obj = InstalledBinary( 336 | name='curl', 337 | binary=curl, # store Binary/BinProvider/SemVer values directly in fields 338 | binproviders=[env], # no need for manual JSON serialization / schema checking 339 | min_version=SemVer('6.5.0'), 340 | ) 341 | obj.save() 342 | ``` 343 | 344 | When fetching it back from the DB, the `Binary` field is auto-deserialized / immediately usable: 345 | ``` 346 | obj = InstalledBinary.objects.get(name='curl') # everything is transparently serialized to/from the DB, 347 | # and is ready to go immediately after querying: 348 | assert obj.binary.abspath == curl.abspath 349 | print(obj.binary.abspath) # Path('/usr/local/bin/curl') 350 | obj.binary.exec(['--version']) # curl 7.81.0 (x86_64-apple-darwin23.0) libcurl/7.81.0 ... 351 | ``` 352 | *For a full example see our provided [`django_example_project/`](https://github.com/ArchiveBox/abx-pkg/tree/main/django_example_project)...* 353 | 354 |
355 | 356 | ### Django Admin Usage: Display `Binary` objects nicely in the Admin UI 357 | 358 | Django Admin binaries list viewDjango Admin binaries detail view 359 | 360 | ```bash 361 | pip install abx-pkg django-admin-data-views 362 | ``` 363 | *For more info see the [`django-admin-data-views`](https://github.com/MrThearMan/django-admin-data-views) docs...* 364 | 365 | Then add this to your `settings.py`: 366 | ```python 367 | INSTALLED_APPS = [ 368 | # ... 369 | 'admin_data_views' 370 | 'abx_pkg' 371 | # ... 372 | ] 373 | 374 | # point these to a function that gets the list of all binaries / a single binary 375 | ABX_PKG_GET_ALL_BINARIES = 'abx_pkg.views.get_all_binaries' 376 | ABX_PKG_GET_BINARY = 'abx_pkg.views.get_binary' 377 | 378 | ADMIN_DATA_VIEWS = { 379 | "NAME": "Environment", 380 | "URLS": [ 381 | { 382 | "route": "binaries/", 383 | "view": "abx_pkg.views.binaries_list_view", 384 | "name": "binaries", 385 | "items": { 386 | "route": "/", 387 | "view": "abx_pkg.views.binary_detail_view", 388 | "name": "binary", 389 | }, 390 | }, 391 | # Coming soon: binprovider_list_view + binprovider_detail_view ... 392 | ], 393 | } 394 | ``` 395 | *For a full example see our provided [`django_example_project/`](https://github.com/ArchiveBox/abx-pkg/tree/main/django_example_project)...* 396 | 397 |
398 | Note: If you override the default site admin, you must register the views manually... 399 |

400 | admin.py: 401 |
402 | 

403 | class YourSiteAdmin(admin.AdminSite):
404 |     """Your customized version of admin.AdminSite"""
405 |     ...
406 | 

407 | custom_admin = YourSiteAdmin()
408 | custom_admin.register(get_user_model())
409 | ...
410 | from abx_pkg.admin import register_admin_views
411 | register_admin_views(custom_admin)
412 |
413 |
414 | 415 |
416 | 417 | ### ~~Django Admin Usage: JSONFormWidget for editing `BinProvider` and `Binary` data~~ 418 | 419 |
Expand to see more... 420 | 421 | 422 | 423 | > [!IMPORTANT] 424 | > This feature is coming soon but is blocked on a few issues being fixed first: 425 | > - https://github.com/surenkov/django-pydantic-field/issues/64 426 | > - https://github.com/surenkov/django-pydantic-field/issues/65 427 | > - https://github.com/surenkov/django-pydantic-field/issues/66 428 | 429 | ~~Install `django-jsonform` to get auto-generated Forms for editing BinProvider, Binary, etc. data~~ 430 | ```bash 431 | pip install django-pydantic-field django-jsonform 432 | ``` 433 | *For more info see the [`django-jsonform`](https://django-jsonform.readthedocs.io/) docs...* 434 | 435 | `admin.py`: 436 | ```python 437 | from django.contrib import admin 438 | from django_jsonform.widgets import JSONFormWidget 439 | from django_pydantic_field.v2.fields import PydanticSchemaField 440 | 441 | class MyModelAdmin(admin.ModelAdmin): 442 | formfield_overrides = {PydanticSchemaField: {"widget": JSONFormWidget}} 443 | 444 | admin.site.register(MyModel, MyModelAdmin) 445 | ``` 446 | 447 |
448 | 449 | *For a full example see our provided [`django_example_project/`](https://github.com/ArchiveBox/abx-pkg/tree/main/django_example_project)...* 450 | 451 |
452 | 453 | --- 454 | 455 |
456 | 457 | 458 | ## Examples 459 | 460 | ### Advanced: Implement your own package manager behavior by subclassing BinProvider 461 | 462 | ```python 463 | from subprocess import run, PIPE 464 | 465 | from abx_pkg import BinProvider, BinProviderName, BinName, SemVer 466 | 467 | class CargoProvider(BinProvider): 468 | name: BinProviderName = 'cargo' 469 | 470 | def on_setup_paths(self): 471 | if '~/.cargo/bin' not in sys.path: 472 | sys.path.append('~/.cargo/bin') 473 | 474 | def on_install(self, bin_name: BinName, **context): 475 | packages = self.on_get_packages(bin_name) 476 | installer_process = run(['cargo', 'install', *packages.split(' ')], capture_output = True, text=True) 477 | assert installer_process.returncode == 0 478 | 479 | def on_get_packages(self, bin_name: BinName, **context) -> InstallArgs: 480 | # optionally remap bin_names to strings passed to installer 481 | # e.g. 'yt-dlp' -> ['yt-dlp, 'ffmpeg', 'libcffi', 'libaac'] 482 | return [bin_name] 483 | 484 | def on_get_abspath(self, bin_name: BinName, **context) -> Path | None: 485 | self.on_setup_paths() 486 | return Path(os.which(bin_name)) 487 | 488 | def on_get_version(self, bin_name: BinName, **context) -> SemVer | None: 489 | self.on_setup_paths() 490 | return SemVer(run([bin_name, '--version'], stdout=PIPE).stdout.decode()) 491 | 492 | cargo = CargoProvider() 493 | rg = cargo.install(bin_name='ripgrep') 494 | print(rg.binprovider) # CargoProvider() 495 | print(rg.version) # SemVer(14, 1, 0) 496 | ``` 497 | 498 | 499 |
500 | 501 | --- 502 | 503 |
504 | 505 | *Note:* this package used to be called `pydantic-pkgr`, it was renamed to `abx-pkg` on 2024-11-12. 506 | 507 | ### TODO 508 | 509 | - [x] Implement initial basic support for `apt`, `brew`, and `pip` 510 | - [x] Provide editability and actions via Django Admin UI using [`django-pydantic-field`](https://github.com/surenkov/django-pydantic-field) and [`django-jsonform`](https://django-jsonform.readthedocs.io/en/latest/) 511 | - [ ] Implement `update` and `remove` actions on BinProviders 512 | - [ ] Add `preinstall` and `postinstall` hooks for things like adding `apt` sources and running cleanup scripts 513 | - [ ] Implement more package managers (`cargo`, `gem`, `go get`, `ppm`, `nix`, `docker`, etc.) 514 | - [ ] Add `Binary.min_version` that affects `.is_valid` based on whether it meets minimum `SemVer` threshold 515 | 516 | 517 | ### Other Packages We Like 518 | 519 | - https://github.com/MrThearMan/django-signal-webhooks 520 | - https://github.com/MrThearMan/django-admin-data-views 521 | - https://github.com/lazybird/django-solo 522 | - https://github.com/joshourisman/django-pydantic-settings 523 | - https://github.com/surenkov/django-pydantic-field 524 | - https://github.com/jordaneremieff/djantic 525 | 526 | [coverage-badge]: https://coveralls.io/repos/github/ArchiveBox/abx-pkg/badge.svg?branch=main 527 | [status-badge]: https://img.shields.io/github/actions/workflow/status/ArchiveBox/abx-pkg/test.yml?branch=main 528 | [pypi-badge]: https://img.shields.io/pypi/v/abx-pkg?v=1 529 | [licence-badge]: https://img.shields.io/github/license/ArchiveBox/abx-pkg?v=1 530 | [repo-badge]: https://img.shields.io/github/last-commit/ArchiveBox/abx-pkg?v=1 531 | [issues-badge]: https://img.shields.io/github/issues-raw/ArchiveBox/abx-pkg?v=1 532 | [version-badge]: https://img.shields.io/pypi/pyversions/abx-pkg?v=1 533 | [downloads-badge]: https://img.shields.io/pypi/dm/abx-pkg?v=1 534 | [django-badge]: https://img.shields.io/pypi/djversions/abx-pkg?v=1 535 | 536 | [coverage]: https://coveralls.io/github/ArchiveBox/abx-pkg?branch=main 537 | [status]: https://github.com/ArchiveBox/abx-pkg/actions/workflows/test.yml 538 | [pypi]: https://pypi.org/project/abx-pkg 539 | [licence]: https://github.com/ArchiveBox/abx-pkg/blob/main/LICENSE 540 | [repo]: https://github.com/ArchiveBox/abx-pkg/commits/main 541 | [issues]: https://github.com/ArchiveBox/abx-pkg/issues 542 | -------------------------------------------------------------------------------- /abx_pkg/__init__.py: -------------------------------------------------------------------------------- 1 | __package__ = "abx_pkg" 2 | 3 | from .base_types import ( 4 | BinName, 5 | InstallArgs, 6 | PATHStr, 7 | HostBinPath, 8 | HostExistsPath, 9 | BinDirPath, 10 | BinProviderName, 11 | bin_name, 12 | bin_abspath, 13 | bin_abspaths, 14 | func_takes_args_or_kwargs, 15 | ) 16 | from .semver import SemVer, bin_version 17 | from .shallowbinary import ShallowBinary 18 | from .binprovider import ( 19 | BinProvider, 20 | EnvProvider, 21 | OPERATING_SYSTEM, 22 | DEFAULT_PATH, 23 | DEFAULT_ENV_PATH, 24 | PYTHON_BIN_DIR, 25 | BinProviderOverrides, 26 | BinaryOverrides, 27 | ProviderFuncReturnValue, 28 | HandlerType, 29 | HandlerValue, 30 | HandlerDict, 31 | HandlerReturnValue, 32 | ) 33 | from .binary import Binary 34 | 35 | from .binprovider_apt import AptProvider 36 | from .binprovider_brew import BrewProvider 37 | from .binprovider_pip import PipProvider 38 | from .binprovider_npm import NpmProvider 39 | from .binprovider_ansible import AnsibleProvider 40 | from .binprovider_pyinfra import PyinfraProvider 41 | 42 | ALL_PROVIDERS = [ 43 | EnvProvider, 44 | AptProvider, 45 | BrewProvider, 46 | PipProvider, 47 | NpmProvider, 48 | AnsibleProvider, 49 | PyinfraProvider, 50 | ] 51 | ALL_PROVIDER_NAMES = [provider.__fields__['name'].default for provider in ALL_PROVIDERS] 52 | ALL_PROVIDER_CLASSES = [provider.__class__.__name__ for provider in ALL_PROVIDERS] 53 | 54 | 55 | __all__ = [ 56 | # Main types 57 | "BinProvider", 58 | "Binary", 59 | "SemVer", 60 | "ShallowBinary", 61 | 62 | # Helper Types 63 | "BinName", 64 | "InstallArgs", 65 | "PATHStr", 66 | "BinDirPath", 67 | "HostBinPath", 68 | "HostExistsPath", 69 | "BinProviderName", 70 | 71 | # Override types 72 | "BinProviderOverrides", 73 | "BinaryOverrides", 74 | "ProviderFuncReturnValue", 75 | "HandlerType", 76 | "HandlerValue", 77 | "HandlerDict", 78 | "HandlerReturnValue", 79 | 80 | # Validator Functions 81 | "bin_version", 82 | "bin_name", 83 | "bin_abspath", 84 | "bin_abspaths", 85 | "func_takes_args_or_kwargs", 86 | 87 | # Globals 88 | "OPERATING_SYSTEM", 89 | "DEFAULT_PATH", 90 | "DEFAULT_ENV_PATH", 91 | "PYTHON_BIN_DIR", 92 | 93 | # BinProviders 94 | *ALL_PROVIDER_CLASSES, 95 | ] 96 | -------------------------------------------------------------------------------- /abx_pkg/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | 3 | 4 | 5 | 6 | def register_admin_views(admin_site: admin.AdminSite): 7 | """register the django-admin-data-views defined in settings.ADMIN_DATA_VIEWS""" 8 | 9 | from admin_data_views.admin import get_app_list, admin_data_index_view, get_admin_data_urls, get_urls 10 | 11 | CustomAdminCls = admin_site.__class__ 12 | 13 | admin_site.get_app_list = get_app_list.__get__(admin_site, CustomAdminCls) 14 | admin_site.admin_data_index_view = admin_data_index_view.__get__(admin_site, CustomAdminCls) 15 | admin_site.get_admin_data_urls = get_admin_data_urls.__get__(admin_site, CustomAdminCls) 16 | admin_site.get_urls = get_urls(admin_site.get_urls).__get__(admin_site, CustomAdminCls) 17 | 18 | return admin_site 19 | 20 | 21 | register_admin_views(admin.site) 22 | 23 | # if you've implemented a custom admin site, you should call this funciton on your site 24 | 25 | # class YourSiteAdmin(admin.AdminSite): 26 | # ... 27 | # 28 | # custom_site = YourSiteAdmin() 29 | # 30 | # register_admin_views(custom_site) 31 | -------------------------------------------------------------------------------- /abx_pkg/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class AbxPkgConfig(AppConfig): 5 | default_auto_field = 'django.db.models.BigAutoField' 6 | name = 'abx_pkg' 7 | -------------------------------------------------------------------------------- /abx_pkg/base_types.py: -------------------------------------------------------------------------------- 1 | __package__ = "abx_pkg" 2 | 3 | import os 4 | import shutil 5 | 6 | from pathlib import Path 7 | from typing import List, Tuple, Callable, Any, Annotated 8 | 9 | from pydantic import TypeAdapter, AfterValidator, BeforeValidator, ValidationError 10 | 11 | def validate_binprovider_name(name: str) -> str: 12 | assert 1 < len(name) < 16, 'BinProvider names must be between 1 and 16 characters long' 13 | assert name.replace('_', '').isalnum(), 'BinProvider names can only contain a-Z0-9 and underscores' 14 | assert name[0].isalpha(), 'BinProvider names must start with a letter' 15 | return name 16 | 17 | BinProviderName = Annotated[str, AfterValidator(validate_binprovider_name)] 18 | # in practice this is essentially BinProviderName: Literal['env', 'pip', 'apt', 'brew', 'npm', 'vendor'] 19 | # but because users can create their own BinProviders we cant restrict it to a preset list of literal names 20 | 21 | 22 | 23 | def validate_bin_dir(path: Path) -> Path: 24 | path = path.expanduser().absolute() 25 | assert path.resolve() 26 | assert os.path.isdir(path) and os.access(path, os.R_OK), f'path entries to add to $PATH must be absolute paths to directories {dir}' 27 | return path 28 | 29 | BinDirPath = Annotated[Path, AfterValidator(validate_bin_dir)] 30 | 31 | def validate_PATH(PATH: str | List[str]) -> str: 32 | paths = PATH.split(':') if isinstance(PATH, str) else list(PATH) 33 | assert all(Path(bin_dir) for bin_dir in paths) 34 | return ':'.join(paths).strip(':') 35 | 36 | PATHStr = Annotated[str, BeforeValidator(validate_PATH)] 37 | 38 | def func_takes_args_or_kwargs(lambda_func: Callable[..., Any]) -> bool: 39 | """returns True if a lambda func takes args/kwargs of any kind, otherwise false if it's pure/argless""" 40 | code = lambda_func.__code__ 41 | has_args = code.co_argcount > 0 42 | has_varargs = code.co_flags & 0x04 != 0 43 | has_varkw = code.co_flags & 0x08 != 0 44 | return has_args or has_varargs or has_varkw 45 | 46 | 47 | # @validate_call 48 | def bin_name(bin_path_or_name: str | Path) -> str: 49 | """ 50 | - wget -> wget 51 | - /usr/bin/wget -> wget 52 | - ~/bin/wget -> wget 53 | - ~/.local/bin/wget -> wget 54 | - @postlight/parser -> @postlight/parser 55 | - @postlight/parser@2.2.3 -> @postlight/parser 56 | - yt-dlp==2024.05.09 -> yt-dlp 57 | - postlight/parser^2.2.3 -> postlight/parser 58 | - @postlight/parser@^2.2.3 -> @postlight/parser 59 | """ 60 | str_bin_name = str(bin_path_or_name).split('^', 1)[0].split('=', 1)[0].split('>', 1)[0].split('<', 1)[0] 61 | if str_bin_name.startswith('@'): 62 | # @postlight/parser@^2.2.3 -> @postlight/parser 63 | str_bin_name = '@' + str_bin_name[1:].split('@', 1)[0] 64 | else: 65 | str_bin_name = str_bin_name.split('@', 1)[0] 66 | 67 | assert len(str_bin_name) > 0, 'Binary names must be non-empty' 68 | name = Path(str_bin_name).name if str_bin_name[0] in ('.', '/', '~') else str_bin_name 69 | assert 1 <= len(name) < 64, 'Binary names must be between 1 and 63 characters long' 70 | assert name.replace('-', '').replace('_', '').replace('.', '').replace(' ', '').replace('@', '').replace('/', '').isalnum(), ( 71 | f'Binary name can only contain a-Z0-9-_.@/ and spaces: {name}') 72 | assert name.replace('@', '')[0].isalpha(), 'Binary names must start with a letter or @' 73 | # print('PARSING BIN NAME', bin_path_or_name, '->', name) 74 | return name 75 | 76 | BinName = Annotated[str, AfterValidator(bin_name)] 77 | 78 | # @validate_call 79 | def path_is_file(path: Path | str) -> Path: 80 | path = Path(path) if isinstance(path, str) else path 81 | assert os.path.isfile(path) and os.access(path, os.R_OK), f'Path is not a file or we dont have permission to read it: {path}' 82 | return path 83 | 84 | HostExistsPath = Annotated[Path, AfterValidator(path_is_file)] 85 | 86 | # @validate_call 87 | def path_is_executable(path: HostExistsPath) -> HostExistsPath: 88 | assert os.path.isfile(path) and os.access(path, os.X_OK), f'Path is not executable (fix by running chmod +x {path})' 89 | return path 90 | 91 | # @validate_call 92 | def path_is_script(path: HostExistsPath) -> HostExistsPath: 93 | SCRIPT_EXTENSIONS = ('.py', '.js', '.sh') 94 | assert path.suffix.lower() in SCRIPT_EXTENSIONS, 'Path is not a script (does not end in {})'.format(', '.join(SCRIPT_EXTENSIONS)) 95 | return path 96 | 97 | HostExecutablePath = Annotated[HostExistsPath, AfterValidator(path_is_executable)] 98 | 99 | # @validate_call 100 | def path_is_abspath(path: Path) -> Path: 101 | path = path.expanduser().absolute() # resolve ~/ -> /home/ HostBinPath | None: 112 | assert bin_path_or_name 113 | if PATH is None: 114 | PATH = os.environ.get('PATH', '/bin') 115 | if PATH: 116 | PATH = str(PATH) 117 | else: 118 | return None 119 | 120 | if str(bin_path_or_name).startswith('/'): 121 | # already a path, get its absolute form 122 | abspath = Path(bin_path_or_name).expanduser().absolute() 123 | else: 124 | # not a path yet, get path using shutil.which 125 | binpath = shutil.which(bin_path_or_name, mode=os.X_OK, path=PATH) 126 | # print(bin_path_or_name, PATH.split(':'), binpath, 'GOPINGNGN') 127 | if not binpath: 128 | # some bins dont show up with shutil.which (e.g. django-admin.py) 129 | for path in PATH.split(':'): 130 | bin_dir = Path(path) 131 | # print('BIN_DIR', bin_dir, bin_dir.is_dir()) 132 | if not (os.path.isdir(bin_dir) and os.access(bin_dir, os.R_OK)): 133 | # raise Exception(f'Found invalid dir in $PATH: {bin_dir}') 134 | continue 135 | bin_file = bin_dir / bin_path_or_name 136 | # print(bin_file, path, bin_file.exists(), bin_file.is_file(), bin_file.is_symlink()) 137 | if os.path.isfile(bin_file) and os.access(bin_file, os.R_OK): 138 | return bin_file 139 | 140 | return None 141 | # print(binpath, PATH) 142 | if str(Path(binpath).parent) not in PATH: 143 | # print('WARNING, found bin but not in PATH', binpath, PATH) 144 | # found bin but it was outside our search $PATH 145 | return None 146 | abspath = Path(binpath).expanduser().absolute() 147 | 148 | try: 149 | return TypeAdapter(HostBinPath).validate_python(abspath) 150 | except ValidationError: 151 | return None 152 | 153 | # @validate_call 154 | def bin_abspaths(bin_path_or_name: BinName | Path, PATH: PATHStr | None=None) -> List[HostBinPath]: 155 | assert bin_path_or_name 156 | 157 | PATH = PATH or os.environ.get('PATH', '/bin') 158 | abspaths = [] 159 | 160 | if str(bin_path_or_name).startswith('/'): 161 | # already a path, get its absolute form 162 | abspaths.append(Path(bin_path_or_name).expanduser().absolute()) 163 | else: 164 | # not a path yet, get path using shutil.which 165 | for path in PATH.split(':'): 166 | binpath = shutil.which(bin_path_or_name, mode=os.X_OK, path=path) 167 | if binpath and str(Path(binpath).parent) in PATH: 168 | abspaths.append(binpath) 169 | 170 | try: 171 | return TypeAdapter(List[HostBinPath]).validate_python(abspaths) 172 | except ValidationError: 173 | return [] 174 | 175 | 176 | 177 | 178 | ################## Types ############################################## 179 | 180 | UNKNOWN_SHA256 = 'unknown' 181 | 182 | def is_valid_sha256(sha256: str) -> str: 183 | if sha256 == UNKNOWN_SHA256: 184 | return sha256 185 | assert len(sha256) == 64 186 | assert sha256.isalnum() 187 | return sha256 188 | 189 | Sha256 = Annotated[str, AfterValidator(is_valid_sha256)] 190 | 191 | def is_valid_install_args(install_args: List[str] | Tuple[str, ...] | str) -> Tuple[str, ...]: 192 | """Make sure a string is a valid install string for a package manager, e.g. ['yt-dlp', 'ffmpeg']""" 193 | if isinstance(install_args, str): 194 | install_args = [install_args] 195 | assert install_args 196 | assert all(len(arg) for arg in install_args) 197 | return tuple(install_args) 198 | 199 | def is_name_of_method_on_self(method_name: str) -> str: 200 | assert method_name.startswith('self.') and method_name.replace('.', '').replace('_', '').isalnum() 201 | return method_name 202 | 203 | InstallArgs = Annotated[Tuple[str, ...] | List[str], AfterValidator(is_valid_install_args)] 204 | 205 | SelfMethodName = Annotated[str, AfterValidator(is_name_of_method_on_self)] 206 | 207 | 208 | -------------------------------------------------------------------------------- /abx_pkg/binary.py: -------------------------------------------------------------------------------- 1 | __package__ = 'abx_pkg' 2 | 3 | from typing import Any, Optional, Dict, List 4 | from typing_extensions import Self 5 | 6 | from pydantic import Field, model_validator, computed_field, field_validator, validate_call, field_serializer, ConfigDict, InstanceOf 7 | 8 | from .semver import SemVer 9 | from .shallowbinary import ShallowBinary 10 | from .binprovider import BinProvider, EnvProvider, BinaryOverrides 11 | from .base_types import ( 12 | BinName, 13 | bin_abspath, 14 | bin_abspaths, 15 | HostBinPath, 16 | BinProviderName, 17 | PATHStr, 18 | Sha256, 19 | ) 20 | 21 | DEFAULT_PROVIDER = EnvProvider() 22 | 23 | 24 | class Binary(ShallowBinary): 25 | model_config = ConfigDict(extra='allow', populate_by_name=True, validate_defaults=True, validate_assignment=True, from_attributes=True, revalidate_instances='always', arbitrary_types_allowed=True) 26 | 27 | name: BinName = '' 28 | description: str = '' 29 | 30 | binproviders_supported: List[InstanceOf[BinProvider]] = Field(default_factory=lambda : [DEFAULT_PROVIDER], alias='binproviders') 31 | overrides: BinaryOverrides = Field(default_factory=dict) 32 | 33 | loaded_binprovider: Optional[InstanceOf[BinProvider]] = Field(default=None, alias='binprovider') 34 | loaded_abspath: Optional[HostBinPath] = Field(default=None, alias='abspath') 35 | loaded_version: Optional[SemVer] = Field(default=None, alias='version') 36 | loaded_sha256: Optional[Sha256] = Field(default=None, alias='sha256') 37 | 38 | # bin_filename: see below 39 | # is_executable: see below 40 | # is_script 41 | # is_valid: see below 42 | 43 | 44 | @model_validator(mode='after') 45 | def validate(self): 46 | # assert self.name, 'Binary.name must not be empty' 47 | # self.description = self.description or self.name 48 | 49 | assert self.binproviders_supported, f'No providers were given for package {self.name}' 50 | 51 | # pull in any overrides from the binproviders 52 | for binprovider in self.binproviders_supported: 53 | overrides_for_bin = binprovider.overrides.get(self.name, {}) 54 | if overrides_for_bin: 55 | self.overrides[binprovider.name] = { 56 | **overrides_for_bin, 57 | **self.overrides.get(binprovider.name, {}), 58 | } 59 | return self 60 | 61 | @field_validator('loaded_abspath', mode='before') 62 | def parse_abspath(cls, value: Any) -> Optional[HostBinPath]: 63 | return bin_abspath(value) if value else None 64 | 65 | @field_validator('loaded_version', mode='before') 66 | def parse_version(cls, value: Any) -> Optional[SemVer]: 67 | return SemVer(value) if value else None 68 | 69 | @field_serializer('overrides', when_used='json') 70 | def serialize_overrides(self, overrides: BinaryOverrides) -> Dict[BinProviderName, Dict[str, str]]: 71 | return { 72 | binprovider_name: { 73 | handler_type: str(handler_value) 74 | for handler_type, handler_value in binprovider_overrides.items() 75 | } 76 | for binprovider_name, binprovider_overrides in overrides.items() 77 | } 78 | 79 | @computed_field 80 | @property 81 | def loaded_abspaths(self) -> Dict[BinProviderName, List[HostBinPath]]: 82 | if not self.loaded_abspath: 83 | # binary has not been loaded yet 84 | return {} 85 | 86 | all_bin_abspaths = {self.loaded_binprovider.name: [self.loaded_abspath]} if self.loaded_binprovider else {} 87 | for binprovider in self.binproviders_supported: 88 | if not binprovider.PATH: 89 | # print('skipping provider', binprovider.name, binprovider.PATH) 90 | continue 91 | for abspath in bin_abspaths(self.name, PATH=binprovider.PATH): 92 | existing = all_bin_abspaths.get(binprovider.name, []) 93 | if abspath not in existing: 94 | all_bin_abspaths[binprovider.name] = [ 95 | *existing, 96 | abspath, 97 | ] 98 | return all_bin_abspaths 99 | 100 | 101 | @computed_field 102 | @property 103 | def loaded_bin_dirs(self) -> Dict[BinProviderName, PATHStr]: 104 | return { 105 | provider_name: ':'.join([str(bin_abspath.parent) for bin_abspath in bin_abspaths]) 106 | for provider_name, bin_abspaths in self.loaded_abspaths.items() 107 | } 108 | 109 | @computed_field 110 | @property 111 | def python_name(self) -> str: 112 | return self.name.replace('-', '_').replace('.', '_') 113 | 114 | # @validate_call 115 | def get_binprovider(self, binprovider_name: BinProviderName, **extra_overrides) -> InstanceOf[BinProvider]: 116 | for binprovider in self.binproviders_supported: 117 | if binprovider.name == binprovider_name: 118 | overrides_for_binprovider = { 119 | self.name: self.overrides.get(binprovider_name, {}) 120 | } 121 | return binprovider.get_provider_with_overrides(overrides=overrides_for_binprovider, **extra_overrides) 122 | 123 | raise KeyError(f'{binprovider_name} is not a supported BinProvider for Binary(name={self.name})') 124 | 125 | @validate_call 126 | def install(self, binproviders: Optional[List[BinProviderName]]=None, **extra_overrides) -> Self: 127 | assert self.name, f'No binary name was provided! {self}' 128 | 129 | if binproviders is not None and len(list(binproviders)) == 0: 130 | return self 131 | 132 | 133 | inner_exc = Exception('No providers were available') 134 | errors = {} 135 | for binprovider in self.binproviders_supported: 136 | if binproviders and (binprovider.name not in binproviders): 137 | continue 138 | 139 | try: 140 | provider = self.get_binprovider(binprovider_name=binprovider.name, **extra_overrides) 141 | 142 | installed_bin = provider.install(self.name) 143 | if installed_bin is not None and installed_bin.loaded_abspath: 144 | # print('INSTALLED', self.name, installed_bin) 145 | return self.__class__(**{ 146 | **self.model_dump(), 147 | **installed_bin.model_dump(exclude={'binproviders_supported'}), 148 | 'loaded_binprovider': provider, 149 | 'binproviders_supported': self.binproviders_supported, 150 | 'overrides': self.overrides, 151 | }) 152 | except Exception as err: 153 | # print(err) 154 | # raise 155 | inner_exc = err 156 | errors[binprovider.name] = str(err) 157 | 158 | provider_names = ', '.join(binproviders or [p.name for p in self.binproviders_supported]) 159 | raise Exception(f'None of the configured providers ({provider_names}) were able to install binary: {self.name} ERRORS={errors}') from inner_exc 160 | 161 | @validate_call 162 | def load(self, binproviders: Optional[List[BinProviderName]]=None, nocache=False, **extra_overrides) -> Self: 163 | assert self.name, f'No binary name was provided! {self}' 164 | 165 | # if we're already loaded, skip loading 166 | if self.is_valid: 167 | return self 168 | 169 | # if binproviders list is passed but it's empty, skip loading 170 | if binproviders is not None and len(list(binproviders)) == 0: 171 | return self 172 | 173 | inner_exc = Exception('No providers were available') 174 | errors = {} 175 | for binprovider in self.binproviders_supported: 176 | if binproviders and binprovider.name not in binproviders: 177 | continue 178 | 179 | try: 180 | provider = self.get_binprovider(binprovider_name=binprovider.name, **extra_overrides) 181 | 182 | installed_bin = provider.load(self.name, nocache=nocache) 183 | if installed_bin is not None and installed_bin.loaded_abspath: 184 | # print('LOADED', binprovider, self.name, installed_bin) 185 | return self.__class__(**{ 186 | **self.model_dump(), 187 | **installed_bin.model_dump(exclude={'binproviders_supported'}), 188 | 'loaded_binprovider': provider, 189 | 'binproviders_supported': self.binproviders_supported, 190 | 'overrides': self.overrides, 191 | }) 192 | else: 193 | continue 194 | except Exception as err: 195 | # print(err) 196 | inner_exc = err 197 | errors[binprovider.name] = str(err) 198 | 199 | provider_names = ', '.join(binproviders or [p.name for p in self.binproviders_supported]) 200 | raise Exception(f'None of the configured providers ({provider_names}) were able to load binary: {self.name} ERRORS={errors}') from inner_exc 201 | 202 | @validate_call 203 | def load_or_install(self, binproviders: Optional[List[BinProviderName]]=None, nocache: bool=False, **extra_overrides) -> Self: 204 | assert self.name, f'No binary name was provided! {self}' 205 | 206 | if self.is_valid: 207 | return self 208 | 209 | if binproviders is not None and len(list(binproviders)) == 0: 210 | return self 211 | 212 | inner_exc = Exception('No providers were available') 213 | errors = {} 214 | for binprovider in self.binproviders_supported: 215 | if binproviders and binprovider.name not in binproviders: 216 | continue 217 | 218 | try: 219 | provider = self.get_binprovider(binprovider_name=binprovider.name, **extra_overrides) 220 | 221 | installed_bin = provider.load_or_install(self.name, nocache=nocache) 222 | if installed_bin is not None and installed_bin.loaded_abspath: 223 | # print('LOADED_OR_INSTALLED', self.name, installed_bin) 224 | return self.__class__(**{ 225 | **self.model_dump(), 226 | **installed_bin.model_dump(exclude={'binproviders_supported'}), 227 | 'loaded_binprovider': provider, 228 | 'binproviders_supported': self.binproviders_supported, 229 | 'overrides': self.overrides, 230 | }) 231 | else: 232 | continue 233 | except Exception as err: 234 | # print(err) 235 | inner_exc = err 236 | errors[binprovider.name] = str(err) 237 | continue 238 | 239 | provider_names = ', '.join(binproviders or [p.name for p in self.binproviders_supported]) 240 | raise Exception(f'None of the configured providers ({provider_names}) were able to find or install binary: {self.name} ERRORS={errors}') from inner_exc 241 | -------------------------------------------------------------------------------- /abx_pkg/binprovider.py: -------------------------------------------------------------------------------- 1 | __package__ = "abx_pkg" 2 | 3 | import os 4 | import sys 5 | import pwd 6 | import shutil 7 | import hashlib 8 | import platform 9 | import subprocess 10 | import functools 11 | 12 | from typing import Callable, Optional, Iterable, List, cast, final, Dict, Any, Tuple, Literal, Protocol, runtime_checkable 13 | 14 | from typing_extensions import Self, TypedDict 15 | from pathlib import Path 16 | 17 | from pydantic_core import ValidationError 18 | from pydantic import BaseModel, Field, TypeAdapter, validate_call, ConfigDict, InstanceOf, computed_field, model_validator 19 | 20 | from .semver import SemVer 21 | from .base_types import ( 22 | BinName, 23 | BinDirPath, 24 | HostBinPath, 25 | BinProviderName, 26 | PATHStr, 27 | InstallArgs, 28 | Sha256, 29 | SelfMethodName, 30 | UNKNOWN_SHA256, 31 | bin_name, 32 | path_is_executable, 33 | path_is_script, 34 | bin_abspath, 35 | bin_abspaths, 36 | func_takes_args_or_kwargs, 37 | ) 38 | 39 | ################## GLOBALS ########################################## 40 | 41 | OPERATING_SYSTEM = platform.system().lower() 42 | DEFAULT_PATH = "/home/linuxbrew/.linuxbrew/bin:/opt/homebrew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" 43 | DEFAULT_ENV_PATH = os.environ.get("PATH", DEFAULT_PATH) 44 | PYTHON_BIN_DIR = str(Path(sys.executable).parent) 45 | 46 | if PYTHON_BIN_DIR not in DEFAULT_ENV_PATH: 47 | DEFAULT_ENV_PATH = PYTHON_BIN_DIR + ":" + DEFAULT_ENV_PATH 48 | 49 | UNKNOWN_ABSPATH = Path('/usr/bin/true') 50 | UNKNOWN_VERSION = cast(SemVer, SemVer.parse('999.999.999')) 51 | 52 | ################## VALIDATORS ####################################### 53 | 54 | NEVER_CACHE = ( 55 | None, 56 | UNKNOWN_ABSPATH, 57 | UNKNOWN_VERSION, 58 | UNKNOWN_SHA256, 59 | ) 60 | 61 | def binprovider_cache(binprovider_method): 62 | """cache non-null return values for BinProvider methods on the BinProvider instance""" 63 | 64 | method_name = binprovider_method.__name__ 65 | 66 | @functools.wraps(binprovider_method) 67 | def cached_function(self, bin_name: BinName, **kwargs): 68 | self._cache = self._cache or {} 69 | self._cache[method_name] = self._cache.get(method_name, {}) 70 | method_cache = self._cache[method_name] 71 | 72 | if bin_name in method_cache and not kwargs.get('nocache'): 73 | # print('USING CACHED VALUE:', f'{self.__class__.__name__}.{method_name}({bin_name}, {kwargs}) -> {method_cache[bin_name]}') 74 | return method_cache[bin_name] 75 | 76 | return_value = binprovider_method(self, bin_name, **kwargs) 77 | 78 | if return_value and return_value not in NEVER_CACHE: 79 | self._cache[method_name][bin_name] = return_value 80 | return return_value 81 | 82 | cached_function.__name__ = f'{method_name}_cached' 83 | 84 | return cached_function 85 | 86 | 87 | 88 | class ShallowBinary(BaseModel): 89 | """ 90 | Shallow version of Binary used as a return type for BinProvider methods (e.g. load_or_install()). 91 | (doesn't implement full Binary interface, but can be used to populate a full loaded Binary instance) 92 | """ 93 | 94 | model_config = ConfigDict(extra="forbid", populate_by_name=True, validate_defaults=True, validate_assignment=False, from_attributes=True, arbitrary_types_allowed=True) 95 | 96 | name: BinName = "" 97 | description: str = "" 98 | 99 | binproviders_supported: List[InstanceOf["BinProvider"]] = Field(default_factory=list, alias="binproviders") 100 | overrides: 'BinaryOverrides' = Field(default_factory=dict) 101 | 102 | loaded_binprovider: InstanceOf["BinProvider"] = Field(alias="binprovider") 103 | loaded_abspath: HostBinPath = Field(alias="abspath") 104 | loaded_version: SemVer = Field(alias="version") 105 | loaded_sha256: Sha256 = Field(alias="sha256") 106 | 107 | def __getattr__(self, item): 108 | """Allow accessing fields as attributes by both field name and alias name""" 109 | for field, meta in self.model_fields.items(): 110 | if meta.alias == item: 111 | return getattr(self, field) 112 | return super().__getattr__(item) 113 | 114 | @model_validator(mode="after") 115 | def validate(self) -> Self: 116 | self.description = self.description or self.name 117 | return self 118 | 119 | @computed_field # type: ignore[misc] # see mypy issue #1362 120 | @property 121 | def bin_filename(self) -> BinName: 122 | if self.is_script: 123 | # e.g. '.../Python.framework/Versions/3.11/lib/python3.11/sqlite3/__init__.py' -> sqlite 124 | name = self.name 125 | elif self.loaded_abspath: 126 | # e.g. '/opt/homebrew/bin/wget' -> wget 127 | name = bin_name(self.loaded_abspath) 128 | else: 129 | # e.g. 'ytdlp' -> 'yt-dlp' 130 | name = bin_name(self.name) 131 | return name 132 | 133 | @computed_field # type: ignore[misc] # see mypy issue #1362 134 | @property 135 | def is_executable(self) -> bool: 136 | try: 137 | assert self.loaded_abspath and path_is_executable(self.loaded_abspath) 138 | return True 139 | except (ValidationError, AssertionError): 140 | return False 141 | 142 | @computed_field # type: ignore[misc] # see mypy issue #1362 143 | @property 144 | def is_script(self) -> bool: 145 | try: 146 | assert self.loaded_abspath and path_is_script(self.loaded_abspath) 147 | return True 148 | except (ValidationError, AssertionError): 149 | return False 150 | 151 | @computed_field # type: ignore[misc] # see mypy issue #1362 152 | @property 153 | def is_valid(self) -> bool: 154 | return bool(self.name and self.loaded_abspath and self.loaded_version and (self.is_executable or self.is_script)) 155 | 156 | @computed_field 157 | @property 158 | def bin_dir(self) -> BinDirPath | None: 159 | if not self.loaded_abspath: 160 | return None 161 | return TypeAdapter(BinDirPath).validate_python(self.loaded_abspath.parent) 162 | 163 | @computed_field 164 | @property 165 | def loaded_respath(self) -> HostBinPath | None: 166 | return self.loaded_abspath and self.loaded_abspath.resolve() 167 | 168 | # @validate_call 169 | def exec( 170 | self, bin_name: BinName | HostBinPath | None = None, cmd: Iterable[str | Path | int | float | bool] = (), cwd: str | Path = ".", quiet=False, **kwargs 171 | ) -> subprocess.CompletedProcess: 172 | bin_name = str(bin_name or self.loaded_abspath or self.name) 173 | if bin_name == self.name: 174 | assert self.loaded_abspath, "Binary must have a loaded_abspath, make sure to load_or_install() first" 175 | assert self.loaded_version, "Binary must have a loaded_version, make sure to load_or_install() first" 176 | assert os.path.isdir(cwd) and os.access(cwd, os.R_OK), f"cwd must be a valid, accessible directory: {cwd}" 177 | cmd = [str(bin_name), *(str(arg) for arg in cmd)] 178 | if not quiet: 179 | print('$', ' '.join(cmd), file=sys.stderr) 180 | return subprocess.run(cmd, capture_output=True, text=True, cwd=str(cwd), **kwargs) 181 | 182 | 183 | DEFAULT_OVERRIDES = { 184 | '*': { 185 | 'version': 'self.default_version_handler', 186 | 'abspath': 'self.default_abspath_handler', 187 | 'packages': 'self.default_packages_handler', 188 | 'install': 'self.default_install_handler', 189 | }, 190 | } 191 | 192 | 193 | class BinProvider(BaseModel): 194 | model_config = ConfigDict(extra='forbid', populate_by_name=True, validate_defaults=True, validate_assignment=False, from_attributes=True, revalidate_instances='always', arbitrary_types_allowed=True) 195 | name: BinProviderName = '' 196 | 197 | PATH: PATHStr = Field(default=str(Path(sys.executable).parent), repr=False) # e.g. '/opt/homebrew/bin:/opt/archivebox/bin' 198 | INSTALLER_BIN: BinName = 'env' 199 | 200 | euid: Optional[int] = None 201 | 202 | overrides: 'BinProviderOverrides' = Field(default=DEFAULT_OVERRIDES, repr=False, exclude=True) 203 | 204 | _dry_run: bool = False 205 | _install_timeout: int = 120 206 | _version_timeout: int = 10 207 | _cache: Dict[str, Dict[str, Any]] | None = None 208 | _INSTALLER_BIN_ABSPATH: HostBinPath | None = None # speed optimization only, faster to cache the abspath than to recompute it on every access 209 | _INSTALLER_BINARY: ShallowBinary | None = None # speed optimization only, faster to cache the binary than to recompute it on every access 210 | 211 | def __eq__(self, other: Any) -> bool: 212 | try: 213 | return dict(self) == dict(other) # only compare pydantic fields, ignores classvars/@properties/@cached_properties/_fields/etc. 214 | except Exception: 215 | return False 216 | 217 | @property 218 | def EUID(self) -> int: 219 | """ 220 | Detect the user (UID) to run as when executing this binprovider's INSTALLER_BIN 221 | e.g. homebrew should never be run as root, we can tell which user to run it as by looking at who owns its binary 222 | apt should always be run as root, pip should be run as the user that owns the venv, etc. 223 | """ 224 | 225 | # use user-provided value if one is set 226 | if self.euid is not None: 227 | return self.euid 228 | 229 | # fallback to owner of installer binary 230 | try: 231 | installer_bin = self.INSTALLER_BIN_ABSPATH 232 | if installer_bin: 233 | return os.stat(installer_bin).st_uid 234 | except Exception: 235 | # INSTALLER_BIN_ABSPATH is not always availabe (e.g. at import time, or if it dynamically changes) 236 | pass 237 | 238 | # fallback to current user 239 | return os.geteuid() 240 | 241 | 242 | @computed_field 243 | @property 244 | def INSTALLER_BIN_ABSPATH(self) -> HostBinPath | None: 245 | """Actual absolute path of the underlying package manager (e.g. /usr/local/bin/npm)""" 246 | if self._INSTALLER_BIN_ABSPATH: 247 | # return cached value if we have one 248 | return self._INSTALLER_BIN_ABSPATH 249 | 250 | abspath = bin_abspath(self.INSTALLER_BIN, PATH=self.PATH) or bin_abspath(self.INSTALLER_BIN) # find self.INSTALLER_BIN abspath using environment path 251 | if not abspath: 252 | # underlying package manager not found on this host, return None 253 | return None 254 | 255 | valid_abspath = TypeAdapter(HostBinPath).validate_python(abspath) 256 | if valid_abspath: 257 | # if we found a valid abspath, cache it 258 | self._INSTALLER_BIN_ABSPATH = valid_abspath 259 | return valid_abspath 260 | 261 | @property 262 | def INSTALLER_BINARY(self) -> ShallowBinary | None: 263 | """Get the loaded binary for this binprovider's INSTALLER_BIN""" 264 | 265 | if self._INSTALLER_BINARY: 266 | # return cached value if we have one 267 | return self._INSTALLER_BINARY 268 | 269 | abspath = self.INSTALLER_BIN_ABSPATH 270 | if not abspath: 271 | return None 272 | 273 | try: 274 | # try loading it from the BinProvider's own PATH (e.g. ~/test/.venv/bin/pip) 275 | loaded_bin = self.load(bin_name=self.INSTALLER_BIN) 276 | if loaded_bin: 277 | self._INSTALLER_BINARY = loaded_bin 278 | return loaded_bin 279 | except Exception: 280 | pass 281 | 282 | env = EnvProvider() 283 | try: 284 | # try loading it from the env provider (e.g. /opt/homebrew/bin/pip) 285 | loaded_bin = env.load(bin_name=self.INSTALLER_BIN) 286 | if loaded_bin: 287 | self._INSTALLER_BINARY = loaded_bin 288 | return loaded_bin 289 | except Exception: 290 | pass 291 | 292 | version = UNKNOWN_VERSION 293 | sha256 = UNKNOWN_SHA256 294 | 295 | return ShallowBinary( 296 | name=self.INSTALLER_BIN, 297 | abspath=abspath, 298 | binprovider=env, 299 | version=version, 300 | sha256=sha256, 301 | ) 302 | 303 | @computed_field 304 | @property 305 | def is_valid(self) -> bool: 306 | return bool(self.INSTALLER_BIN_ABSPATH) 307 | 308 | @final 309 | # @validate_call(config={'arbitrary_types_allowed': True}) 310 | def get_provider_with_overrides(self, overrides: Optional['BinProviderOverrides']=None, dry_run: bool=False, install_timeout: int | None=None, version_timeout: int | None=None) -> Self: 311 | # created an updated copy of the BinProvider with the overrides applied, then get the handlers on it. 312 | # important to do this so that any subsequent calls to handler functions down the call chain 313 | # still have access to the overrides, we don't have to have to pass them down as args all the way down the stack 314 | 315 | updated_binprovider: Self = self.model_copy() 316 | 317 | # main binary-specific overrides for [abspath, version, packages, install] 318 | overrides = overrides or {} 319 | 320 | # extra overrides that are also configurable, can add more in the future as-needed for tunable options 321 | updated_binprovider._dry_run = dry_run 322 | updated_binprovider._install_timeout = install_timeout or self._install_timeout 323 | updated_binprovider._version_timeout = version_timeout or self._version_timeout 324 | 325 | # overrides = { 326 | # 'wget': { 327 | # 'packages': lambda: ['wget'], 328 | # 'abspath': lambda: shutil.which('wget'), 329 | # 'version': lambda: SemVer.parse(os.system('wget --version')), 330 | # 'install': lambda: os.system('brew install wget'), 331 | # }, 332 | # } 333 | for binname, bin_overrides in overrides.items(): 334 | updated_binprovider.overrides[binname] = { 335 | **updated_binprovider.overrides.get(binname, {}), 336 | **bin_overrides, 337 | } 338 | 339 | return updated_binprovider 340 | 341 | 342 | # @validate_call 343 | def _get_handler_for_action(self, bin_name: BinName, handler_type: 'HandlerType') -> Callable[..., 'HandlerReturnValue']: 344 | """ 345 | Get the handler func for a given key + Dict of handler callbacks + fallback default handler. 346 | e.g. _get_handler_for_action(bin_name='yt-dlp', 'install', default_handler=self.default_install_handler, ...) -> Callable 347 | """ 348 | 349 | # e.g. {'yt-dlp': {'install': 'self.default_install_handler'}} -> 'self.default_install_handler' 350 | handler: HandlerValue = ( 351 | self.overrides.get(bin_name, {}).get(handler_type) 352 | or self.overrides.get('*', {}).get(handler_type) 353 | ) 354 | # print('getting handler for action', bin_name, handler_type, handler_func) 355 | assert handler, f'BinProvider(name={self.name}) has no {handler_type} handler implemented for Binary(name={bin_name})' 356 | 357 | # if handler_func is already a callable, return it directly 358 | if isinstance(handler, Callable): 359 | handler_func: Callable[..., HandlerReturnValue] = handler 360 | return handler_func 361 | 362 | # if handler_func is string reference to a function on self, swap it for the actual function 363 | elif isinstance(handler, str) and (handler.startswith('self.') or handler.startswith('BinProvider.')): 364 | # special case, allow dotted path references to methods on self (where self refers to the BinProvider) 365 | handler_method: Callable[..., HandlerReturnValue] = getattr(self, handler.split('self.', 1)[-1]) 366 | return handler_method 367 | 368 | # if handler_func is any other value, treat is as a literal and return a func that provides the literal 369 | literal_value = TypeAdapter(HandlerReturnValue).validate_python(handler) 370 | handler_func: Callable[..., HandlerReturnValue] = lambda: literal_value # noqa: E731 371 | return handler_func 372 | 373 | # @validate_call 374 | def _call_handler_for_action(self, bin_name: BinName, handler_type: 'HandlerType', **kwargs) -> 'HandlerReturnValue': 375 | handler_func: Callable[..., HandlerReturnValue] = self._get_handler_for_action( 376 | bin_name=bin_name, # e.g. 'yt-dlp', or 'wget', etc. 377 | handler_type=handler_type, # e.g. abspath, version, packages, install 378 | ) 379 | 380 | # def timeout_handler(signum, frame): 381 | # raise TimeoutError(f'{self.__class__.__name__} Timeout while running {handler_type} for Binary {bin_name}') 382 | 383 | # signal ONLY WORKS IN MAIN THREAD, not a viable solution for timeout enforcement! breaks in prod 384 | # signal.signal(signal.SIGALRM, handler=timeout_handler) 385 | # signal.alarm(timeout) 386 | try: 387 | if not func_takes_args_or_kwargs(handler_func): 388 | # if it's a pure argless lambda/func, dont pass bin_path and other **kwargs 389 | handler_func_without_args = cast(Callable[[], HandlerReturnValue], handler_func) 390 | return handler_func_without_args() 391 | 392 | handler_func = cast(Callable[..., HandlerReturnValue], handler_func) 393 | if hasattr(handler_func, '__self__'): 394 | # func is already a method bound to self, just call it directly 395 | return handler_func(bin_name, **kwargs) 396 | else: 397 | # func is not bound to anything, pass BinProvider as first arg 398 | return handler_func(self, bin_name, **kwargs) 399 | except TimeoutError: 400 | raise 401 | # finally: 402 | # signal.alarm(0) 403 | 404 | 405 | # DEFAULT HANDLERS, override these in subclasses as needed: 406 | 407 | # @validate_call 408 | def default_abspath_handler(self, bin_name: BinName | HostBinPath, **context) -> 'AbspathFuncReturnValue': # aka str | Path | None 409 | # print(f'[*] {self.__class__.__name__}: Getting abspath for {bin_name}...') 410 | 411 | if not self.PATH: 412 | return None 413 | 414 | return bin_abspath(bin_name, PATH=self.PATH) 415 | 416 | # @validate_call 417 | def default_version_handler(self, bin_name: BinName, abspath: Optional[HostBinPath]=None, **context) -> 'VersionFuncReturnValue': # aka List[str] | Tuple[str, ...] 418 | 419 | abspath = abspath or self.get_abspath(bin_name, quiet=True) 420 | if not abspath: 421 | return None 422 | 423 | # print(f'[*] {self.__class__.__name__}: Getting version for {bin_name}...') 424 | 425 | validation_err = None 426 | 427 | # Attempt 1: $ --version 428 | dash_dash_version_result = self.exec(bin_name=abspath, cmd=['--version'], timeout=self._version_timeout, quiet=True) 429 | dash_dash_version_out = dash_dash_version_result.stdout.strip() 430 | try: 431 | version = SemVer.parse(dash_dash_version_out) 432 | assert version, f"Could not parse version from $ {bin_name} --version: {dash_dash_version_result.stdout}\n{dash_dash_version_result.stderr}\n".strip() 433 | return version 434 | except (ValidationError, AssertionError) as err: 435 | validation_err = err 436 | 437 | # Attempt 2: $ -version 438 | dash_version_out = self.exec(bin_name=abspath, cmd=["-version"], timeout=self._version_timeout, quiet=True).stdout.strip() 439 | try: 440 | version = SemVer.parse(dash_version_out) 441 | assert version, f"Could not parse version from $ {bin_name} -version: {dash_version_out}".strip() 442 | return version 443 | except (ValidationError, AssertionError) as err: 444 | validation_err = validation_err or err 445 | 446 | # Attempt 3: $ -v 447 | dash_v_out = self.exec(bin_name=abspath, cmd=["-v"], timeout=self._version_timeout, quiet=True).stdout.strip() 448 | try: 449 | version = SemVer.parse(dash_v_out) 450 | assert version, f"Could not parse version from $ {bin_name} -v: {dash_v_out}".strip() 451 | return version 452 | except (ValidationError, AssertionError) as err: 453 | validation_err = validation_err or err 454 | 455 | raise ValueError( 456 | f"Unable to find {bin_name} version from {bin_name} --version, -version or -v output\n{dash_dash_version_out or dash_version_out or dash_v_out}".strip() 457 | ) from validation_err 458 | 459 | # @validate_call 460 | def default_packages_handler(self, bin_name: BinName, **context) -> 'PackagesFuncReturnValue': # aka List[str] aka InstallArgs 461 | # print(f'[*] {self.__class__.__name__}: Getting install command for {bin_name}') 462 | # ... install command calculation logic here 463 | return [bin_name] 464 | 465 | # @validate_call 466 | def default_install_handler(self, bin_name: BinName, packages: Optional[InstallArgs]=None, **context) -> 'InstallFuncReturnValue': # aka str 467 | self.setup() 468 | packages = packages or self.get_packages(bin_name) 469 | if not self.INSTALLER_BIN_ABSPATH: 470 | raise Exception(f'{self.name} install method is not available on this host ({self.INSTALLER_BIN} not found in $PATH)') 471 | 472 | # print(f'[*] {self.__class__.__name__}: Installing {bin_name}: {self.INSTALLER_BIN_ABSPATH} {packages}') 473 | 474 | # ... override the default install logic here ... 475 | 476 | # proc = self.exec(bin_name=self.INSTALLER_BIN_ABSPATH, cmd=['install', *packages], timeout=self._install_timeout) 477 | # if not proc.returncode == 0: 478 | # print(proc.stdout.strip()) 479 | # print(proc.stderr.strip()) 480 | # raise Exception(f'{self.name} Failed to install {bin_name}: {proc.stderr.strip()}\n{proc.stdout.strip()}') 481 | 482 | return f'{self.name} BinProvider does not implement any install method' 483 | 484 | 485 | def setup_PATH(self) -> None: 486 | for path in reversed(self.PATH.split(':')): 487 | if path not in sys.path: 488 | sys.path.insert(0, path) # e.g. /opt/archivebox/bin:/bin:/usr/local/bin:... 489 | 490 | # @validate_call 491 | def exec(self, bin_name: BinName | HostBinPath, cmd: Iterable[str | Path | int | float | bool]=(), cwd: Path | str='.', quiet=False, **kwargs) -> subprocess.CompletedProcess: 492 | if shutil.which(str(bin_name)): 493 | bin_abspath = bin_name 494 | else: 495 | bin_abspath = self.get_abspath(str(bin_name)) 496 | assert bin_abspath, f'BinProvider {self.name} cannot execute bin_name {bin_name} because it could not find its abspath. (Did {self.__class__.__name__}.load_or_install({bin_name}) fail?)' 497 | assert os.access(cwd, os.R_OK) and os.path.isdir(cwd), f'cwd must be a valid, accessible directory: {cwd}' 498 | cmd = [str(bin_abspath), *(str(arg) for arg in cmd)] 499 | if not quiet: 500 | prefix = 'DRY RUN: $' if self._dry_run else '$' 501 | print(prefix, ' '.join(cmd), file=sys.stderr) 502 | 503 | # https://stackoverflow.com/a/6037494/2156113 504 | # copy env and modify it to run the subprocess as the the designated user 505 | env = kwargs.get('env', {}) or os.environ.copy() 506 | pw_record = pwd.getpwuid(self.EUID) 507 | run_as_uid = pw_record.pw_uid 508 | run_as_gid = pw_record.pw_gid 509 | # update environment variables so that subprocesses dont try to write to /root home directory 510 | # for things like cache dirs, logs, etc. npm/pip/etc. often try to write to $HOME 511 | env['PWD'] = str(cwd) 512 | env['HOME'] = pw_record.pw_dir 513 | env['LOGNAME'] = pw_record.pw_name 514 | env['USER'] = pw_record.pw_name 515 | 516 | def drop_privileges(): 517 | try: 518 | os.setuid(run_as_uid) 519 | os.setgid(run_as_gid) 520 | except Exception: 521 | pass 522 | 523 | if self._dry_run: 524 | return subprocess.CompletedProcess(cmd, 0, '', 'skipped (dry run)') 525 | 526 | return subprocess.run(cmd, capture_output=True, text=True, cwd=str(cwd), env=env, preexec_fn=drop_privileges, **kwargs) 527 | 528 | 529 | # CALLING API, DONT OVERRIDE THESE: 530 | 531 | @final 532 | @binprovider_cache 533 | # @validate_call 534 | def get_abspaths(self, bin_name: BinName, nocache: bool=False) -> List[HostBinPath]: 535 | return bin_abspaths(bin_name, PATH=self.PATH) 536 | 537 | @final 538 | @binprovider_cache 539 | # @validate_call 540 | def get_sha256(self, bin_name: BinName, abspath: Optional[HostBinPath]=None, nocache: bool=False) -> Sha256 | None: 541 | """Get the sha256 hash of the binary at the given abspath (or equivalent hash of the underlying package)""" 542 | 543 | abspath = abspath or self.get_abspath(bin_name, nocache=nocache) 544 | if not abspath or not os.access(abspath, os.R_OK): 545 | return None 546 | 547 | if sys.version_info >= (3, 11): 548 | with open(abspath, "rb", buffering=0) as f: 549 | return TypeAdapter(Sha256).validate_python(hashlib.file_digest(f, 'sha256').hexdigest()) 550 | 551 | hash_sha256 = hashlib.sha256() 552 | with open(abspath, "rb") as f: 553 | for chunk in iter(lambda: f.read(4096), b""): 554 | hash_sha256.update(chunk) 555 | return TypeAdapter(Sha256).validate_python(hash_sha256.hexdigest()) 556 | 557 | @final 558 | @binprovider_cache 559 | # @validate_call 560 | def get_abspath(self, bin_name: BinName, quiet: bool=False, nocache: bool=False) -> HostBinPath | None: 561 | self.setup_PATH() 562 | abspath = None 563 | try: 564 | abspath = cast(AbspathFuncReturnValue, self._call_handler_for_action(bin_name=bin_name, handler_type='abspath')) 565 | except Exception: 566 | if not quiet: 567 | raise 568 | if not abspath: 569 | return None 570 | result = TypeAdapter(HostBinPath).validate_python(abspath) 571 | return result 572 | 573 | @final 574 | @binprovider_cache 575 | # @validate_call 576 | def get_version(self, bin_name: BinName, abspath: Optional[HostBinPath]=None, quiet: bool=False, nocache: bool=False) -> SemVer | None: 577 | version = None 578 | try: 579 | version = cast(VersionFuncReturnValue, self._call_handler_for_action(bin_name=bin_name, handler_type='version', abspath=abspath)) 580 | except Exception: 581 | if not quiet: 582 | raise 583 | 584 | if not version: 585 | return None 586 | 587 | if not isinstance(version, SemVer): 588 | version = SemVer.parse(version) 589 | 590 | return version 591 | 592 | @final 593 | @binprovider_cache 594 | # @validate_call 595 | def get_packages(self, bin_name: BinName, quiet: bool=False, nocache: bool=False) -> InstallArgs: 596 | packages = None 597 | try: 598 | packages = cast(PackagesFuncReturnValue, self._call_handler_for_action(bin_name=bin_name, handler_type='packages')) 599 | except Exception: 600 | if not quiet: 601 | raise 602 | 603 | if not packages: 604 | packages = [bin_name] 605 | result = TypeAdapter(InstallArgs).validate_python(packages) 606 | return result 607 | 608 | def setup(self) -> None: 609 | """Override this to do any setup steps needed before installing packaged (e.g. create a venv, init an npm prefix, etc.)""" 610 | pass 611 | 612 | @final 613 | @binprovider_cache 614 | @validate_call 615 | def install(self, bin_name: BinName, quiet: bool=False, nocache: bool=False) -> ShallowBinary | None: 616 | self.setup() 617 | 618 | packages = self.get_packages(bin_name, quiet=quiet, nocache=nocache) 619 | 620 | self.setup_PATH() 621 | install_log = None 622 | try: 623 | install_log = cast(InstallFuncReturnValue, self._call_handler_for_action(bin_name=bin_name, handler_type='install', packages=packages)) 624 | except Exception as err: 625 | install_log = f'{self.__class__.__name__} Failed to install {bin_name}, got {err.__class__.__name__}: {err}' 626 | if not quiet: 627 | raise 628 | 629 | if self._dry_run: 630 | # return fake ShallowBinary if we're just doing a dry run 631 | # no point trying to get real abspath or version if nothing was actually installed 632 | return ShallowBinary( 633 | name=bin_name, 634 | binprovider=self, 635 | abspath=Path(shutil.which(bin_name) or UNKNOWN_ABSPATH), 636 | version=cast(SemVer, UNKNOWN_VERSION), 637 | sha256=UNKNOWN_SHA256, binproviders=[self], 638 | ) 639 | 640 | installed_abspath = self.get_abspath(bin_name, quiet=True, nocache=nocache) 641 | if not quiet: 642 | assert installed_abspath, f'{self.__class__.__name__} Unable to find abspath for {bin_name} after installing. PATH={self.PATH} LOG={install_log}' 643 | 644 | installed_version = self.get_version(bin_name, abspath=installed_abspath, quiet=True, nocache=nocache) 645 | if not quiet: 646 | assert installed_version, f'{self.__class__.__name__} Unable to find version for {bin_name} after installing. ABSPATH={installed_abspath} LOG={install_log}' 647 | 648 | sha256 = self.get_sha256(bin_name, abspath=installed_abspath, nocache=nocache) or UNKNOWN_SHA256 649 | 650 | if (installed_abspath and installed_version): 651 | # installed binary is valid and ready to use 652 | result = ShallowBinary( 653 | name=bin_name, 654 | binprovider=self, 655 | abspath=installed_abspath, 656 | version=installed_version, 657 | sha256=sha256, 658 | binproviders=[self], 659 | ) 660 | else: 661 | result = None 662 | 663 | return result 664 | 665 | @final 666 | @validate_call 667 | def load(self, bin_name: BinName, quiet: bool=True, nocache: bool=False) -> ShallowBinary | None: 668 | installed_abspath = self.get_abspath(bin_name, quiet=quiet, nocache=nocache) 669 | if not installed_abspath: 670 | return None 671 | 672 | installed_version = self.get_version(bin_name, abspath=installed_abspath, quiet=quiet, nocache=nocache) 673 | if not installed_version: 674 | return None 675 | 676 | sha256 = self.get_sha256(bin_name, abspath=installed_abspath) or UNKNOWN_SHA256 # not ideal to store UNKNOWN_SHA256but it's better than nothing and this value isnt critical 677 | 678 | return ShallowBinary( 679 | name=bin_name, 680 | binprovider=self, 681 | abspath=installed_abspath, 682 | version=installed_version, 683 | sha256=sha256, 684 | binproviders=[self], 685 | ) 686 | 687 | @final 688 | @validate_call 689 | def load_or_install(self, bin_name: BinName, quiet: bool=False, nocache: bool=False) -> ShallowBinary | None: 690 | installed = self.load(bin_name=bin_name, quiet=True, nocache=nocache) 691 | if not installed: 692 | installed = self.install(bin_name=bin_name, quiet=quiet, nocache=nocache) 693 | return installed 694 | 695 | 696 | 697 | class EnvProvider(BinProvider): 698 | name: BinProviderName = 'env' 699 | INSTALLER_BIN: BinName = 'which' 700 | PATH: PATHStr = DEFAULT_ENV_PATH # add dir containing python to $PATH 701 | 702 | overrides: 'BinProviderOverrides' = { 703 | '*': { 704 | **BinProvider.model_fields['overrides'].default['*'], 705 | 'install': 'self.install_noop', 706 | }, 707 | 'python': { 708 | 'abspath': Path(sys.executable), 709 | 'version': '{}.{}.{}'.format(*sys.version_info[:3]), 710 | }, 711 | } 712 | 713 | def install_noop(self, bin_name: BinName, packages: Optional[InstallArgs]=None, **context) -> str: 714 | """The env BinProvider is ready-only and does not install any packages, so this is a no-op""" 715 | return 'env is ready-only and just checks for existing binaries in $PATH' 716 | 717 | ############################################################################################################ 718 | 719 | 720 | 721 | AbspathFuncReturnValue = str | HostBinPath | None 722 | VersionFuncReturnValue = str | Tuple[int, ...] | Tuple[str, ...] | SemVer | None # SemVer is a subclass of NamedTuple 723 | PackagesFuncReturnValue = List[str] | Tuple[str, ...] | str | InstallArgs | None 724 | InstallFuncReturnValue = str | None 725 | ProviderFuncReturnValue = AbspathFuncReturnValue | VersionFuncReturnValue | PackagesFuncReturnValue | InstallFuncReturnValue 726 | 727 | @runtime_checkable 728 | class AbspathFuncWithArgs(Protocol): 729 | def __call__(_self, binprovider: 'BinProvider', bin_name: BinName, **context) -> 'AbspathFuncReturnValue': 730 | ... 731 | 732 | @runtime_checkable 733 | class VersionFuncWithArgs(Protocol): 734 | def __call__(_self, binprovider: 'BinProvider', bin_name: BinName, **context) -> 'VersionFuncReturnValue': 735 | ... 736 | 737 | @runtime_checkable 738 | class PackagesFuncWithArgs(Protocol): 739 | def __call__(_self, binprovider: 'BinProvider', bin_name: BinName, **context) -> 'PackagesFuncReturnValue': 740 | ... 741 | 742 | @runtime_checkable 743 | class InstallFuncWithArgs(Protocol): 744 | def __call__(_self, binprovider: 'BinProvider', bin_name: BinName, **context) -> 'InstallFuncReturnValue': 745 | ... 746 | 747 | AbspathFuncWithNoArgs = Callable[[], AbspathFuncReturnValue] 748 | VersionFuncWithNoArgs = Callable[[], VersionFuncReturnValue] 749 | PackagesFuncWithNoArgs = Callable[[], PackagesFuncReturnValue] 750 | InstallFuncWithNoArgs = Callable[[], InstallFuncReturnValue] 751 | 752 | AbspathHandlerValue = SelfMethodName | AbspathFuncWithNoArgs | AbspathFuncWithArgs | AbspathFuncReturnValue 753 | VersionHandlerValue = SelfMethodName | VersionFuncWithNoArgs | VersionFuncWithArgs | VersionFuncReturnValue 754 | PackagesHandlerValue = SelfMethodName | PackagesFuncWithNoArgs | PackagesFuncWithArgs | PackagesFuncReturnValue 755 | InstallHandlerValue = SelfMethodName | InstallFuncWithNoArgs | InstallFuncWithArgs | InstallFuncReturnValue 756 | 757 | HandlerType = Literal['abspath', 'version', 'packages', 'install'] 758 | HandlerValue = AbspathHandlerValue | VersionHandlerValue | PackagesHandlerValue | InstallHandlerValue 759 | HandlerReturnValue = AbspathFuncReturnValue | VersionFuncReturnValue | PackagesFuncReturnValue | InstallFuncReturnValue 760 | 761 | class HandlerDict(TypedDict, total=False): 762 | abspath: AbspathHandlerValue 763 | version: VersionHandlerValue 764 | packages: PackagesHandlerValue 765 | install: InstallHandlerValue 766 | 767 | # Binary.overrides map BinProviderName:HandlerType:Handler {'brew': {'packages': [...]}} 768 | BinaryOverrides = Dict[BinProviderName, HandlerDict] 769 | 770 | # BinProvider.overrides map BinName:HandlerType:Handler {'wget': {'packages': [...]}} 771 | BinProviderOverrides = Dict[BinName | Literal['*'], HandlerDict] 772 | 773 | -------------------------------------------------------------------------------- /abx_pkg/binprovider_ansible.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | __package__ = 'abx_pkg' 3 | 4 | import os 5 | import sys 6 | import tempfile 7 | from pathlib import Path 8 | from typing import Optional 9 | 10 | from .base_types import BinProviderName, PATHStr, BinName, InstallArgs 11 | from .binprovider import BinProvider, OPERATING_SYSTEM, DEFAULT_PATH 12 | 13 | 14 | ANSIBLE_INSTALLED = False 15 | ANSIBLE_IMPORT_ERROR = None 16 | try: 17 | from ansible_runner import Runner, RunnerConfig 18 | ANSIBLE_INSTALLED = True 19 | except ImportError as err: 20 | ANSIBLE_IMPORT_ERROR = err 21 | 22 | 23 | ANSIBLE_INSTALL_PLAYBOOK_TEMPLATE = """ 24 | --- 25 | - name: Install system packages 26 | hosts: localhost 27 | gather_facts: false 28 | tasks: 29 | - name: 'Install system packages: {pkg_names}' 30 | {installer_module}: 31 | name: "{{{{item}}}}" 32 | state: {state} 33 | loop: {pkg_names} 34 | """ 35 | 36 | 37 | def ansible_package_install(pkg_names: str | InstallArgs, playbook_template=ANSIBLE_INSTALL_PLAYBOOK_TEMPLATE, installer_module='auto', state='present', quiet=True) -> str: 38 | if not ANSIBLE_INSTALLED: 39 | raise RuntimeError("Ansible is not installed! To fix:\n pip install ansible ansible-runner") from ANSIBLE_IMPORT_ERROR 40 | 41 | if isinstance(pkg_names, str): 42 | pkg_names = pkg_names.split(' ') 43 | else: 44 | pkg_names = list(pkg_names) 45 | 46 | if installer_module == "auto": 47 | if OPERATING_SYSTEM == 'darwin': 48 | # macOS: Use homebrew 49 | playbook = playbook_template.format(pkg_names=pkg_names, state=state, installer_module="community.general.homebrew") 50 | else: 51 | # Linux: Use Ansible catchall that autodetects apt/yum/pkg/nix/etc. 52 | playbook = playbook_template.format(pkg_names=pkg_names, state=state, installer_module="ansible.builtin.package") 53 | else: 54 | # Custom installer module 55 | playbook = playbook_template.format(pkg_names=pkg_names, state=state, installer_module="ansible.builtin.package") 56 | 57 | 58 | # create a temporary directory using the context manager 59 | with tempfile.TemporaryDirectory() as temp_dir: 60 | ansible_home = Path(temp_dir) / 'tmp' 61 | ansible_home.mkdir(exist_ok=True) 62 | 63 | playbook_path = Path(temp_dir) / 'install_playbook.yml' 64 | playbook_path.write_text(playbook) 65 | 66 | # run the playbook using ansible-runner 67 | os.environ["ANSIBLE_INVENTORY_UNPARSED_WARNING"] = "False" 68 | os.environ["ANSIBLE_LOCALHOST_WARNING"] = "False" 69 | os.environ["ANSIBLE_HOME"] = str(ansible_home) 70 | rc = RunnerConfig( 71 | private_data_dir=temp_dir, 72 | playbook=str(playbook_path), 73 | rotate_artifacts=50000, 74 | host_pattern="localhost", 75 | quiet=quiet, 76 | ) 77 | rc.prepare() 78 | r = Runner(config=rc) 79 | r.run() 80 | succeeded = r.status == "successful" 81 | result_text = f'Installing {pkg_names} on {OPERATING_SYSTEM} using Ansible {installer_module} {["failed", "succeeded"][succeeded]}:{r.stdout.read()}\n{r.stderr.read()}'.strip() 82 | 83 | # check for succes/failure 84 | if succeeded: 85 | return result_text 86 | else: 87 | if "Permission denied" in result_text: 88 | raise PermissionError( 89 | f"Installing {pkg_names} failed! Need to be root to use package manager (retry with sudo, or install manually)" 90 | ) 91 | raise Exception(f"Installing {pkg_names} failed! (retry with sudo, or install manually)\n{result_text}") 92 | 93 | 94 | class AnsibleProvider(BinProvider): 95 | name: BinProviderName = "ansible" 96 | INSTALLER_BIN: BinName = "ansible" 97 | PATH: PATHStr = os.environ.get("PATH", DEFAULT_PATH) 98 | 99 | ansible_installer_module: str = 'auto' # e.g. community.general.homebrew, ansible.builtin.apt, etc. 100 | ansible_playbook_template: str = ANSIBLE_INSTALL_PLAYBOOK_TEMPLATE 101 | 102 | 103 | def default_install_handler(self, bin_name: str, packages: Optional[InstallArgs] = None, **context) -> str: 104 | packages = packages or self.get_packages(bin_name) 105 | 106 | if not self.INSTALLER_BIN_ABSPATH: 107 | raise Exception(f"{self.__class__.__name__}.INSTALLER_BIN is not available on this host: {self.INSTALLER_BIN}") 108 | 109 | return ansible_package_install( 110 | pkg_names=packages, 111 | quiet=True, 112 | playbook_template=self.ansible_playbook_template, 113 | installer_module=self.ansible_installer_module, 114 | ) 115 | 116 | 117 | if __name__ == "__main__": 118 | result = ansible = AnsibleProvider() 119 | 120 | if len(sys.argv) > 1: 121 | result = func = getattr(ansible, sys.argv[1]) # e.g. install 122 | 123 | if len(sys.argv) > 2: 124 | result = func(sys.argv[2]) # e.g. install ffmpeg 125 | 126 | print(result) 127 | -------------------------------------------------------------------------------- /abx_pkg/binprovider_apt.py: -------------------------------------------------------------------------------- 1 | 2 | #!/usr/bin/env python 3 | __package__ = "abx_pkg" 4 | 5 | import sys 6 | import time 7 | import shutil 8 | from typing import Optional 9 | 10 | from pydantic import model_validator, TypeAdapter 11 | 12 | from .base_types import BinProviderName, PATHStr, BinName, InstallArgs 13 | from .binprovider import BinProvider 14 | 15 | _LAST_UPDATE_CHECK = None 16 | UPDATE_CHECK_INTERVAL = 60 * 60 * 24 # 1 day 17 | 18 | 19 | class AptProvider(BinProvider): 20 | name: BinProviderName = "apt" 21 | INSTALLER_BIN: BinName = "apt-get" 22 | 23 | PATH: PATHStr = "" 24 | 25 | euid: Optional[int] = 0 # always run apt as root 26 | 27 | @model_validator(mode="after") 28 | def load_PATH_from_dpkg_install_location(self): 29 | dpkg_abspath = shutil.which("dpkg") 30 | if (not self.INSTALLER_BIN_ABSPATH) or not dpkg_abspath or not self.is_valid: 31 | # package manager is not available on this host 32 | # self.PATH: PATHStr = '' 33 | # self.INSTALLER_BIN_ABSPATH = None 34 | return self 35 | 36 | PATH = self.PATH 37 | dpkg_install_dirs = self.exec(bin_name=dpkg_abspath, cmd=["-L", "bash"], quiet=True).stdout.strip().split("\n") 38 | dpkg_bin_dirs = [path for path in dpkg_install_dirs if path.endswith("/bin")] 39 | for bin_dir in dpkg_bin_dirs: 40 | if str(bin_dir) not in PATH: 41 | PATH = ":".join([str(bin_dir), *PATH.split(":")]) 42 | self.PATH = TypeAdapter(PATHStr).validate_python(PATH) 43 | return self 44 | 45 | def default_install_handler(self, bin_name: BinName, packages: Optional[InstallArgs] = None, **context) -> str: 46 | global _LAST_UPDATE_CHECK 47 | 48 | packages = packages or self.get_packages(bin_name) 49 | 50 | if not (self.INSTALLER_BIN_ABSPATH and shutil.which("dpkg")): 51 | raise Exception(f"{self.__class__.__name__}.INSTALLER_BIN is not available on this host: {self.INSTALLER_BIN}") 52 | 53 | # print(f'[*] {self.__class__.__name__}: Installing {bin_name}: {self.INSTALLER_BIN} install {packages}') 54 | 55 | # Attempt 1: Try installing with Pyinfra 56 | from .binprovider_pyinfra import PYINFRA_INSTALLED, pyinfra_package_install 57 | 58 | if PYINFRA_INSTALLED: 59 | return pyinfra_package_install([bin_name], installer_module="operations.apt.packages") 60 | 61 | # Attempt 2: Try installing with Ansible 62 | from .binprovider_ansible import ANSIBLE_INSTALLED, ansible_package_install 63 | 64 | if ANSIBLE_INSTALLED: 65 | return ansible_package_install([bin_name], installer_module="ansible.builtin.apt") 66 | 67 | # Attempt 3: Fallback to installing manually by calling apt in shell 68 | if not _LAST_UPDATE_CHECK or (time.time() - _LAST_UPDATE_CHECK) > UPDATE_CHECK_INTERVAL: 69 | # only update if we haven't checked in the last day 70 | self.exec(bin_name=self.INSTALLER_BIN_ABSPATH, cmd=["update", "-qq"]) 71 | _LAST_UPDATE_CHECK = time.time() 72 | 73 | proc = self.exec(bin_name=self.INSTALLER_BIN_ABSPATH, cmd=["install", "-y", "-qq", "--no-install-recommends", *packages]) 74 | if proc.returncode != 0: 75 | print(proc.stdout.strip()) 76 | print(proc.stderr.strip()) 77 | raise Exception(f"{self.__class__.__name__} install got returncode {proc.returncode} while installing {packages}: {packages}") 78 | 79 | return proc.stderr.strip() + "\n" + proc.stdout.strip() 80 | return f"Installed {packages} succesfully." 81 | 82 | 83 | if __name__ == "__main__": 84 | result = apt = AptProvider() 85 | 86 | if len(sys.argv) > 1: 87 | result = func = getattr(apt, sys.argv[1]) # e.g. install 88 | 89 | if len(sys.argv) > 2: 90 | result = func(sys.argv[2]) # e.g. install ffmpeg 91 | 92 | print(result) 93 | -------------------------------------------------------------------------------- /abx_pkg/binprovider_brew.py: -------------------------------------------------------------------------------- 1 | 2 | #!/usr/bin/env python3 3 | __package__ = "abx_pkg" 4 | 5 | import os 6 | import sys 7 | import time 8 | import platform 9 | from typing import Optional 10 | from pathlib import Path 11 | 12 | from pydantic import model_validator, TypeAdapter 13 | 14 | from .base_types import BinProviderName, PATHStr, BinName, InstallArgs, HostBinPath, bin_abspath 15 | from .semver import SemVer 16 | from .binprovider import BinProvider 17 | 18 | OS = platform.system().lower() 19 | 20 | NEW_MACOS_DIR = Path('/opt/homebrew/bin') 21 | OLD_MACOS_DIR = Path('/usr/local/bin') 22 | DEFAULT_MACOS_DIR = NEW_MACOS_DIR if platform.machine() == 'arm64' else OLD_MACOS_DIR 23 | DEFAULT_LINUX_DIR = Path('/home/linuxbrew/.linuxbrew/bin') 24 | GUESSED_BREW_PREFIX = DEFAULT_MACOS_DIR if OS == 'darwin' else DEFAULT_LINUX_DIR 25 | 26 | _LAST_UPDATE_CHECK = None 27 | UPDATE_CHECK_INTERVAL = 60 * 60 * 24 # 1 day 28 | 29 | 30 | class BrewProvider(BinProvider): 31 | name: BinProviderName = "brew" 32 | INSTALLER_BIN: BinName = "brew" 33 | 34 | PATH: PATHStr = f"{DEFAULT_LINUX_DIR}:{NEW_MACOS_DIR}:{OLD_MACOS_DIR}" 35 | 36 | brew_prefix: Path = GUESSED_BREW_PREFIX 37 | 38 | @model_validator(mode="after") 39 | def load_PATH(self): 40 | if not self.INSTALLER_BIN_ABSPATH: 41 | # brew is not availabe on this host 42 | self.PATH: PATHStr = "" 43 | return self 44 | 45 | PATHs = set(self.PATH.split(':')) 46 | 47 | if OS == 'darwin' and os.path.isdir(DEFAULT_MACOS_DIR) and os.access(DEFAULT_MACOS_DIR, os.R_OK): 48 | PATHs.add(str(DEFAULT_MACOS_DIR)) 49 | self.brew_prefix = DEFAULT_MACOS_DIR / "bin" 50 | if OS != 'darwin' and os.path.isdir(DEFAULT_LINUX_DIR) and os.access(DEFAULT_LINUX_DIR, os.R_OK): 51 | PATHs.add(str(DEFAULT_LINUX_DIR)) 52 | self.brew_prefix = DEFAULT_LINUX_DIR / "bin" 53 | 54 | if not PATHs: 55 | # if we cant autodetect the paths, run brew --prefix to get the path manually (very slow) 56 | self.brew_prefix = Path(self.exec(bin_name=self.INSTALLER_BIN_ABSPATH, cmd=["--prefix"]).stdout.strip()) 57 | PATHs.add(str(self.brew_prefix / "bin")) 58 | 59 | self.PATH = TypeAdapter(PATHStr).validate_python(':'.join(PATHs)) 60 | return self 61 | 62 | def default_install_handler(self, bin_name: str, packages: Optional[InstallArgs] = None, **context) -> str: 63 | global _LAST_UPDATE_CHECK 64 | 65 | packages = packages or self.get_packages(bin_name) 66 | 67 | if not self.INSTALLER_BIN_ABSPATH: 68 | raise Exception(f"{self.__class__.__name__}.INSTALLER_BIN is not available on this host: {self.INSTALLER_BIN}") 69 | 70 | # print(f'[*] {self.__class__.__name__}: Installing {bin_name}: {self.INSTALLER_BIN_ABSPATH} install {packages}') 71 | 72 | # Attempt 1: Try installing with Pyinfra 73 | from .binprovider_pyinfra import PYINFRA_INSTALLED, pyinfra_package_install 74 | 75 | if PYINFRA_INSTALLED: 76 | return pyinfra_package_install((bin_name,), installer_module="operations.brew.packages") 77 | 78 | # Attempt 2: Try installing with Ansible 79 | from .binprovider_ansible import ANSIBLE_INSTALLED, ansible_package_install 80 | 81 | if ANSIBLE_INSTALLED: 82 | return ansible_package_install(bin_name, installer_module="community.general.homebrew") 83 | 84 | # Attempt 3: Fallback to installing manually by calling brew in shell 85 | 86 | if not _LAST_UPDATE_CHECK or (time.time() - _LAST_UPDATE_CHECK) > UPDATE_CHECK_INTERVAL: 87 | # only update if we haven't checked in the last day 88 | self.exec(bin_name=self.INSTALLER_BIN_ABSPATH, cmd=["update"]) 89 | _LAST_UPDATE_CHECK = time.time() 90 | 91 | proc = self.exec(bin_name=self.INSTALLER_BIN_ABSPATH, cmd=["install", *packages]) 92 | if proc.returncode != 0: 93 | print(proc.stdout.strip()) 94 | print(proc.stderr.strip()) 95 | raise Exception(f"{self.__class__.__name__} install got returncode {proc.returncode} while installing {packages}: {packages}") 96 | 97 | return proc.stderr.strip() + "\n" + proc.stdout.strip() 98 | 99 | def default_abspath_handler(self, bin_name: BinName | HostBinPath, **context) -> HostBinPath | None: 100 | # print(f'[*] {self.__class__.__name__}: Getting abspath for {bin_name}...') 101 | 102 | if not self.PATH: 103 | return None 104 | 105 | # not all brew-installed binaries are symlinked into the default bin dir (e.g. curl) 106 | # because it might conflict with a system binary of the same name (e.g. /usr/bin/curl) 107 | # so we need to check for the binary in the namespaced opt dir and Cellar paths as well 108 | extra_path = self.PATH.replace('/bin', f'/opt/{bin_name}/bin') # e.g. /opt/homebrew/opt/curl/bin/curl 109 | search_paths = f'{self.PATH}:{extra_path}' 110 | 111 | # add unlinked Cellar paths,e.g. /opt/homebrew/Cellar/curl/8.10.1/bin 112 | cellar_paths = ':'.join(str(path) for path in (self.brew_prefix / 'Cellar' / bin_name).glob('*/bin')) 113 | if cellar_paths: 114 | search_paths += ':' + cellar_paths 115 | 116 | abspath = bin_abspath(bin_name, PATH=search_paths) 117 | if abspath: 118 | return abspath 119 | 120 | if not self.INSTALLER_BIN_ABSPATH: 121 | return None 122 | 123 | # This code works but theres no need, the method above is much faster: 124 | 125 | # # try checking filesystem or using brew list to get the Cellar bin path (faster than brew info) 126 | # for package in (self.get_packages(str(bin_name)) or [str(bin_name)]): 127 | # try: 128 | # paths = self.exec(bin_name=self.INSTALLER_BIN_ABSPATH, cmd=[ 129 | # 'list', 130 | # '--formulae', 131 | # package, 132 | # ], timeout=self._version_timeout, quiet=True).stdout.strip().split('\n') 133 | # # /opt/homebrew/Cellar/curl/8.10.1/bin/curl 134 | # # /opt/homebrew/Cellar/curl/8.10.1/bin/curl-config 135 | # # /opt/homebrew/Cellar/curl/8.10.1/include/curl/ (12 files) 136 | # return [line for line in paths if '/Cellar/' in line and line.endswith(f'/bin/{bin_name}')][0].strip() 137 | # except Exception: 138 | # pass 139 | 140 | # # fallback to using brew info to get the Cellar bin path 141 | # for package in (self.get_packages(str(bin_name)) or [str(bin_name)]): 142 | # try: 143 | # info_lines = self.exec(bin_name=self.INSTALLER_BIN_ABSPATH, cmd=[ 144 | # 'info', 145 | # '--quiet', 146 | # package, 147 | # ], timeout=self._version_timeout, quiet=True).stdout.strip().split('\n') 148 | # # /opt/homebrew/Cellar/curl/8.10.0 (530 files, 4MB) 149 | # cellar_path = [line for line in info_lines if '/Cellar/' in line][0].rsplit(' (', 1)[0] 150 | # abspath = bin_abspath(bin_name, PATH=f'{cellar_path}/bin') 151 | # if abspath: 152 | # return abspath 153 | # except Exception: 154 | # pass 155 | # return None 156 | 157 | 158 | def default_version_handler(self, bin_name: BinName, abspath: Optional[HostBinPath]=None, **context) -> SemVer | None: 159 | # print(f'[*] {self.__class__.__name__}: Getting version for {bin_name}...') 160 | 161 | # shortcut: if we already have the Cellar abspath, extract the version from it 162 | if abspath and '/Cellar/' in str(abspath): 163 | # /opt/homebrew/Cellar/curl/8.10.1/bin/curl -> 8.10.1 164 | version = str(abspath).rsplit(f'/bin/{bin_name}', 1)[0].rsplit('/', 1)[-1] 165 | if version: 166 | try: 167 | return SemVer.parse(version) 168 | except ValueError: 169 | pass 170 | 171 | # fallback to running $ --version 172 | try: 173 | version = super().default_version_handler(bin_name, abspath=abspath, **context) 174 | if version: 175 | return SemVer.parse(version) 176 | except ValueError: 177 | pass 178 | 179 | if not self.INSTALLER_BIN_ABSPATH: 180 | return None 181 | 182 | # fallback to using brew list to get the package version (faster than brew info) 183 | for package in (self.get_packages(str(bin_name)) or [str(bin_name)]): 184 | try: 185 | paths = self.exec(bin_name=self.INSTALLER_BIN_ABSPATH, cmd=[ 186 | 'list', 187 | '--formulae', 188 | package, 189 | ], timeout=self._version_timeout, quiet=True).stdout.strip().split('\n') 190 | # /opt/homebrew/Cellar/curl/8.10.1/bin/curl 191 | cellar_abspath = [line for line in paths if '/Cellar/' in line and line.endswith(f'/bin/{bin_name}')][0].strip() 192 | # /opt/homebrew/Cellar/curl/8.10.1/bin/curl -> 8.10.1 193 | version = cellar_abspath.rsplit(f'/bin/{bin_name}', 1)[0].rsplit('/', 1)[-1] 194 | if version: 195 | return SemVer.parse(version) 196 | except Exception: 197 | pass 198 | 199 | # fallback to using brew info to get the version (slowest method of all) 200 | packages = self.get_packages(str(bin_name)) or [str(bin_name)] 201 | main_package = packages[0] # assume first package in list is the main one 202 | try: 203 | version_str = self.exec(bin_name=self.INSTALLER_BIN_ABSPATH, cmd=[ 204 | 'info', 205 | '--quiet', 206 | main_package, 207 | ], quiet=True, timeout=self._version_timeout).stdout.strip().split('\n')[0] 208 | # ==> curl: stable 8.10.1 (bottled), HEAD [keg-only] 209 | return SemVer.parse(version_str) 210 | except Exception: 211 | return None 212 | 213 | return None 214 | 215 | if __name__ == "__main__": 216 | # Usage: 217 | # ./binprovider_brew.py load yt-dlp 218 | # ./binprovider_brew.py install pip 219 | # ./binprovider_brew.py get_version pip 220 | # ./binprovider_brew.py get_abspath pip 221 | result = brew = BrewProvider() 222 | 223 | if len(sys.argv) > 1: 224 | result = func = getattr(brew, sys.argv[1]) # e.g. install 225 | 226 | if len(sys.argv) > 2: 227 | result = func(sys.argv[2]) # e.g. install ffmpeg 228 | 229 | print(result) 230 | -------------------------------------------------------------------------------- /abx_pkg/binprovider_npm.py: -------------------------------------------------------------------------------- 1 | 2 | #!/usr/bin/env python3 3 | 4 | __package__ = "abx_pkg" 5 | 6 | import os 7 | import sys 8 | import json 9 | import tempfile 10 | 11 | from pathlib import Path 12 | from typing import Optional, List 13 | from typing_extensions import Self 14 | 15 | from pydantic import model_validator, TypeAdapter, computed_field 16 | from platformdirs import user_cache_path 17 | 18 | from .base_types import BinProviderName, PATHStr, BinName, InstallArgs, HostBinPath, bin_abspath 19 | from .semver import SemVer 20 | from .binprovider import BinProvider 21 | 22 | # Cache these values globally because they never change at runtime 23 | _CACHED_GLOBAL_NPM_PREFIX: Path | None = None 24 | _CACHED_HOME_DIR: Path = Path('~').expanduser().absolute() 25 | 26 | 27 | USER_CACHE_PATH = Path(tempfile.gettempdir()) / 'npm-cache' 28 | try: 29 | user_cache_path = user_cache_path(appname='npm', appauthor='abx-pkg', ensure_exists=True) 30 | if os.access(user_cache_path, os.W_OK): 31 | USER_CACHE_PATH = user_cache_path 32 | except Exception: 33 | pass 34 | 35 | 36 | class NpmProvider(BinProvider): 37 | name: BinProviderName = 'npm' 38 | INSTALLER_BIN: BinName = 'npm' 39 | 40 | PATH: PATHStr = '' 41 | 42 | npm_prefix: Optional[Path] = None # None = -g global, otherwise it's a path 43 | 44 | cache_dir: Path = USER_CACHE_PATH 45 | cache_arg: str = f'--cache={cache_dir}' 46 | 47 | npm_install_args: List[str] = ['--force', '--no-audit', '--no-fund', '--loglevel=error'] 48 | 49 | _CACHED_LOCAL_NPM_PREFIX: Path | None = None 50 | 51 | @computed_field 52 | @property 53 | def is_valid(self) -> bool: 54 | """False if npm_prefix is not created yet or if npm binary is not found in PATH""" 55 | if self.npm_prefix: 56 | npm_bin_dir = self.npm_prefix / 'node_modules' / '.bin' 57 | npm_bin_dir_exists = (os.path.isdir(npm_bin_dir) and os.access(npm_bin_dir, os.R_OK)) 58 | if not npm_bin_dir_exists: 59 | return False 60 | 61 | return bool(self.INSTALLER_BIN_ABSPATH) 62 | 63 | @model_validator(mode='after') 64 | def detect_euid_to_use(self) -> Self: 65 | """Detect the user (UID) to run as when executing npm (should be same as the user that owns the npm_prefix dir)""" 66 | if self.euid is None: 67 | # try dropping to the owner of the npm prefix dir if it exists 68 | if self.npm_prefix and os.path.isdir(self.npm_prefix): 69 | self.euid = os.stat(self.npm_prefix).st_uid 70 | 71 | # try dropping to the owner of the npm binary if it's not root 72 | installer_bin = self.INSTALLER_BIN_ABSPATH 73 | if installer_bin: 74 | self.euid = self.euid or os.stat(installer_bin).st_uid 75 | 76 | # fallback to the currently running user 77 | self.euid = self.euid or os.geteuid() 78 | 79 | return self 80 | 81 | @model_validator(mode='after') 82 | def load_PATH_from_npm_prefix(self) -> Self: 83 | self.PATH = self._load_PATH() 84 | return self 85 | 86 | def _load_PATH(self) -> str: 87 | PATH = self.PATH 88 | npm_bin_dirs: set[Path] = set() 89 | global _CACHED_GLOBAL_NPM_PREFIX 90 | 91 | if self.npm_prefix: 92 | # restrict PATH to only use npm prefix 93 | npm_bin_dirs = {self.npm_prefix / 'node_modules/.bin'} 94 | 95 | if self.INSTALLER_BIN_ABSPATH: 96 | # find all local and global npm PATHs 97 | npm_local_dir = self._CACHED_LOCAL_NPM_PREFIX or self.exec(bin_name=self.INSTALLER_BIN_ABSPATH, cmd=['prefix'], quiet=True).stdout.strip() 98 | self._CACHED_LOCAL_NPM_PREFIX = npm_local_dir 99 | 100 | # start at npm_local_dir and walk up to $HOME (or /), finding all npm bin dirs along the way 101 | search_dir = Path(npm_local_dir) 102 | stop_if_reached = [str(Path('/')), str(_CACHED_HOME_DIR)] 103 | num_hops, max_hops = 0, 6 104 | while num_hops < max_hops and str(search_dir) not in stop_if_reached: 105 | try: 106 | npm_bin_dirs.add(list(search_dir.glob('node_modules/.bin'))[0]) 107 | break 108 | except (IndexError, OSError, Exception): 109 | # could happen becuase we dont have permission to access the parent dir, or it's been moved, or many other weird edge cases... 110 | pass 111 | search_dir = search_dir.parent 112 | num_hops += 1 113 | 114 | npm_global_dir = _CACHED_GLOBAL_NPM_PREFIX or self.exec(bin_name=self.INSTALLER_BIN_ABSPATH, cmd=['prefix', '-g'], quiet=True).stdout.strip() + '/bin' # /opt/homebrew/bin 115 | _CACHED_GLOBAL_NPM_PREFIX = npm_global_dir 116 | npm_bin_dirs.add(npm_global_dir) 117 | 118 | for bin_dir in npm_bin_dirs: 119 | if str(bin_dir) not in PATH: 120 | PATH = ':'.join([*PATH.split(':'), str(bin_dir)]) 121 | return TypeAdapter(PATHStr).validate_python(PATH) 122 | 123 | def setup(self) -> None: 124 | """create npm install prefix and node_modules_dir if needed""" 125 | if not self.PATH or not self._CACHED_LOCAL_NPM_PREFIX: 126 | self.PATH = self._load_PATH() 127 | 128 | try: 129 | self.cache_dir.mkdir(parents=True, exist_ok=True) 130 | os.system(f'chown {self.EUID} "{self.cache_dir}"') 131 | os.system(f'chmod 777 "{self.cache_dir}"') # allow all users to share cache dir 132 | except Exception: 133 | self.cache_arg = '--no-cache' 134 | 135 | if self.npm_prefix: 136 | (self.npm_prefix / 'node_modules/.bin').mkdir(parents=True, exist_ok=True) 137 | 138 | def default_install_handler(self, bin_name: str, packages: Optional[InstallArgs]=None, **context) -> str: 139 | self.setup() 140 | 141 | packages = packages or self.get_packages(bin_name) 142 | if not self.INSTALLER_BIN_ABSPATH: 143 | raise Exception(f'{self.__class__.__name__} install method is not available on this host ({self.INSTALLER_BIN} not found in $PATH)') 144 | 145 | # print(f'[*] {self.__class__.__name__}: Installing {bin_name}: {self.INSTALLER_BIN_ABSPATH} install {packages}') 146 | 147 | install_args = [*self.npm_install_args, self.cache_arg] 148 | if self.npm_prefix: 149 | install_args.append(f'--prefix={self.npm_prefix}') 150 | else: 151 | install_args.append('--global') 152 | 153 | proc = self.exec(bin_name=self.INSTALLER_BIN_ABSPATH, cmd=[ 154 | "install", 155 | *install_args, 156 | *packages, 157 | ]) 158 | 159 | if proc.returncode != 0: 160 | print(proc.stdout.strip()) 161 | print(proc.stderr.strip()) 162 | raise Exception(f'{self.__class__.__name__}: install got returncode {proc.returncode} while installing {packages}: {packages}') 163 | 164 | return (proc.stderr.strip() + '\n' + proc.stdout.strip()).strip() 165 | 166 | def default_abspath_handler(self, bin_name: BinName, **context) -> HostBinPath | None: 167 | # print(self.__class__.__name__, 'on_get_abspath', bin_name) 168 | 169 | # try searching for the bin_name in BinProvider.PATH first (fastest) 170 | try: 171 | abspath = super().default_abspath_handler(bin_name, **context) 172 | if abspath: 173 | return TypeAdapter(HostBinPath).validate_python(abspath) 174 | except Exception: 175 | pass 176 | 177 | if not self.INSTALLER_BIN_ABSPATH: 178 | return None 179 | 180 | # fallback to using npm show to get alternate binary names based on the package, then try to find those in BinProvider.PATH 181 | try: 182 | packages = self.get_packages(str(bin_name)) or [str(bin_name)] 183 | main_package = packages[0] # assume first package in list is the main one 184 | output_lines = self.exec(bin_name=self.INSTALLER_BIN_ABSPATH, cmd=[ 185 | 'show', 186 | '--json', 187 | main_package, 188 | ], timeout=self._version_timeout, quiet=True).stdout.strip().split('\n') 189 | # { ... 190 | # "version": "2.2.3", 191 | # "bin": { 192 | # "mercury-parser": "cli.js", 193 | # "postlight-parser": "cli.js" 194 | # }, 195 | # ... 196 | # } 197 | alt_bin_names = json.loads(output_lines[0])['bin'].keys() 198 | for alt_bin_name in alt_bin_names: 199 | abspath = bin_abspath(alt_bin_name, PATH=self.PATH) 200 | if abspath: 201 | return TypeAdapter(HostBinPath).validate_python(abspath) 202 | except Exception: 203 | pass 204 | return None 205 | 206 | def default_version_handler(self, bin_name: BinName, abspath: Optional[HostBinPath]=None, **context) -> SemVer | None: 207 | # print(f'[*] {self.__class__.__name__}: Getting version for {bin_name}...') 208 | try: 209 | version = super().default_version_handler(bin_name, abspath, **context) 210 | if version: 211 | return SemVer.parse(version) 212 | except ValueError: 213 | pass 214 | 215 | if not self.INSTALLER_BIN_ABSPATH: 216 | return None 217 | 218 | # fallback to using npm list to get the installed package version 219 | try: 220 | packages = self.get_packages(str(bin_name), **context) or [str(bin_name)] 221 | main_package = packages[0] # assume first package in list is the main one 222 | 223 | # remove the package version if it exists "@postslight/parser@^1.2.3" -> "@postlight/parser" 224 | if main_package[0] == '@': 225 | package = '@' + main_package[1:].split('@', 1)[0] 226 | else: 227 | package = main_package.split('@', 1)[0] 228 | 229 | # npm list --depth=0 --json --prefix= "@postlight/parser" 230 | # (dont use 'npm info @postlight/parser version', it shows *any* availabe version, not installed version) 231 | json_output = self.exec(bin_name=self.INSTALLER_BIN_ABSPATH, cmd=[ 232 | 'list', 233 | f'--prefix={self.npm_prefix}' if self.npm_prefix else '--global', 234 | '--depth=0', 235 | '--json', 236 | package, 237 | ], timeout=self._version_timeout, quiet=True).stdout.strip() 238 | # { 239 | # "name": "lib", 240 | # "dependencies": { 241 | # "@postlight/parser": { 242 | # "version": "2.2.3", 243 | # "overridden": false 244 | # } 245 | # } 246 | # } 247 | version_str = json.loads(json_output)['dependencies'][package]['version'] 248 | return SemVer.parse(version_str) 249 | except Exception: 250 | raise 251 | return None 252 | 253 | if __name__ == "__main__": 254 | # Usage: 255 | # ./binprovider_npm.py load @postlight/parser 256 | # ./binprovider_npm.py install @postlight/parser 257 | # ./binprovider_npm.py get_version @postlight/parser 258 | # ./binprovider_npm.py get_abspath @postlight/parser 259 | result = npm = NpmProvider() 260 | 261 | if len(sys.argv) > 1: 262 | result = func = getattr(npm, sys.argv[1]) # e.g. install 263 | 264 | if len(sys.argv) > 2: 265 | result = func(sys.argv[2]) # e.g. install ffmpeg 266 | 267 | print(result) 268 | -------------------------------------------------------------------------------- /abx_pkg/binprovider_pip.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | __package__ = "abx_pkg" 4 | 5 | import os 6 | import sys 7 | import site 8 | import shutil 9 | import sysconfig 10 | import subprocess 11 | import tempfile 12 | from platformdirs import user_cache_path 13 | 14 | from pathlib import Path 15 | from typing import Optional, List, Set 16 | from typing_extensions import Self 17 | from pydantic import model_validator, TypeAdapter, computed_field 18 | 19 | from .base_types import BinProviderName, PATHStr, BinName, InstallArgs, HostBinPath, bin_abspath, bin_abspaths 20 | from .semver import SemVer 21 | from .binprovider import BinProvider, DEFAULT_ENV_PATH 22 | 23 | ACTIVE_VENV = os.getenv('VIRTUAL_ENV', None) 24 | _CACHED_GLOBAL_PIP_BIN_DIRS: Set[str] | None = None 25 | 26 | 27 | USER_CACHE_PATH = Path(tempfile.gettempdir()) / 'pip-cache' 28 | try: 29 | user_cache_path = user_cache_path(appname='pip', appauthor='abx-pkg', ensure_exists=True) 30 | if os.access(user_cache_path, os.W_OK): 31 | USER_CACHE_PATH = user_cache_path 32 | except Exception: 33 | pass 34 | 35 | 36 | class PipProvider(BinProvider): 37 | name: BinProviderName = "pip" 38 | INSTALLER_BIN: BinName = "pip" 39 | 40 | PATH: PATHStr = '' 41 | 42 | pip_venv: Optional[Path] = None # None = system site-packages (user or global), otherwise it's a path e.g. DATA_DIR/lib/pip/venv 43 | 44 | cache_dir: Path = USER_CACHE_PATH 45 | cache_arg: str = f'--cache-dir={cache_dir}' 46 | 47 | pip_install_args: List[str] = ["--no-input", "--disable-pip-version-check", "--quiet"] # extra args for pip install ... e.g. --upgrade 48 | 49 | _INSTALLER_BIN_ABSPATH: HostBinPath | None = None # speed optimization only, faster to cache the abspath than to recompute it on every access 50 | 51 | @computed_field 52 | @property 53 | def is_valid(self) -> bool: 54 | """False if pip_venv is not created yet or if pip binary is not found in PATH""" 55 | if self.pip_venv: 56 | venv_pip_path = self.pip_venv / "bin" / "python" 57 | venv_pip_binary_exists = (os.path.isfile(venv_pip_path) and os.access(venv_pip_path, os.X_OK)) 58 | if not venv_pip_binary_exists: 59 | return False 60 | 61 | return bool(self.INSTALLER_BIN_ABSPATH) 62 | 63 | @computed_field 64 | @property 65 | def INSTALLER_BIN_ABSPATH(self) -> HostBinPath | None: 66 | """Actual absolute path of the underlying package manager (e.g. /usr/local/bin/npm)""" 67 | if self._INSTALLER_BIN_ABSPATH: 68 | # return cached value if we have one 69 | return self._INSTALLER_BIN_ABSPATH 70 | 71 | abspath = None 72 | 73 | if self.pip_venv: 74 | assert self.INSTALLER_BIN != 'pipx', "Cannot use pipx with pip_venv" 75 | 76 | # use venv pip 77 | venv_pip_path = self.pip_venv / "bin" / self.INSTALLER_BIN 78 | if os.path.isfile(venv_pip_path) and os.access(venv_pip_path, os.R_OK) and os.access(venv_pip_path, os.X_OK): 79 | abspath = str(venv_pip_path) 80 | else: 81 | # use system pip 82 | relpath = bin_abspath(self.INSTALLER_BIN, PATH=DEFAULT_ENV_PATH) or shutil.which(self.INSTALLER_BIN) 83 | abspath = relpath and Path(relpath).resolve() # find self.INSTALLER_BIN abspath using environment path 84 | 85 | if not abspath: 86 | # underlying package manager not found on this host, return None 87 | return None 88 | valid_abspath = TypeAdapter(HostBinPath).validate_python(abspath) 89 | if valid_abspath: 90 | # if we found a valid abspath, cache it 91 | self._INSTALLER_BIN_ABSPATH = valid_abspath 92 | return valid_abspath 93 | 94 | @model_validator(mode='after') 95 | def detect_euid_to_use(self) -> Self: 96 | """Detect the user (UID) to run as when executing pip (should be same as the user that owns the pip_venv dir)""" 97 | 98 | if self.euid is None: 99 | # try dropping to the owner of the npm prefix dir if it exists 100 | if self.pip_venv and os.path.isdir(self.pip_venv): 101 | self.euid = os.stat(self.pip_venv).st_uid 102 | 103 | # try dropping to the owner of the npm binary if it's not root 104 | installer_bin = self.INSTALLER_BIN_ABSPATH 105 | if installer_bin: 106 | self.euid = self.euid or os.stat(installer_bin).st_uid 107 | 108 | # fallback to the currently running user 109 | self.euid = self.euid or os.geteuid() 110 | 111 | return self 112 | 113 | @model_validator(mode="after") 114 | def load_PATH_from_pip_sitepackages(self) -> Self: 115 | """Assemble PATH from pip_venv, pipx, or autodetected global python system site-packages and user site-packages""" 116 | global _CACHED_GLOBAL_PIP_BIN_DIRS 117 | PATH = self.PATH 118 | 119 | pip_bin_dirs = set() 120 | 121 | if self.pip_venv: 122 | # restrict PATH to only use venv bin path 123 | pip_bin_dirs = {str(self.pip_venv / "bin")} 124 | 125 | elif self.INSTALLER_BIN == "pipx": 126 | # restrict PATH to only use global pipx bin path 127 | pipx_abspath = self.INSTALLER_BIN_ABSPATH 128 | if pipx_abspath: 129 | proc = self.exec(bin_name=pipx_abspath, cmd=["environment"], quiet=True, timeout=self._version_timeout) # run $ pipx environment 130 | if proc.returncode == 0: 131 | PIPX_BIN_DIR = proc.stdout.strip().split("PIPX_BIN_DIR=")[-1].split("\n", 1)[0] 132 | pip_bin_dirs = {PIPX_BIN_DIR} 133 | else: 134 | # autodetect global system python paths 135 | 136 | if _CACHED_GLOBAL_PIP_BIN_DIRS: 137 | pip_bin_dirs = _CACHED_GLOBAL_PIP_BIN_DIRS.copy() 138 | else: 139 | pip_bin_dirs = { 140 | * ( 141 | str(Path(sitepackage_dir).parent.parent.parent / "bin") # /opt/homebrew/opt/python@3.11/Frameworks/Python.framework/Versions/3.11/bin 142 | for sitepackage_dir in site.getsitepackages() 143 | ), 144 | str(Path(site.getusersitepackages()).parent.parent.parent / "bin"), # /Users/squash/Library/Python/3.9/bin 145 | sysconfig.get_path("scripts"), # /opt/homebrew/bin 146 | str(Path(sys.executable).resolve().parent), # /opt/homebrew/Cellar/python@3.11/3.11.9/Frameworks/Python.framework/Versions/3.11/bin 147 | } 148 | 149 | # find every python installed in the system PATH and add their parent path, as that's where its corresponding pip will link global bins 150 | for abspath in bin_abspaths("python", PATH=DEFAULT_ENV_PATH): # ~/Library/Frameworks/Python.framework/Versions/3.10/bin 151 | pip_bin_dirs.add(str(abspath.parent)) 152 | for abspath in bin_abspaths("python3", PATH=DEFAULT_ENV_PATH): # /usr/local/bin or anywhere else we see python3 in $PATH 153 | pip_bin_dirs.add(str(abspath.parent)) 154 | 155 | _CACHED_GLOBAL_PIP_BIN_DIRS = pip_bin_dirs.copy() 156 | 157 | # remove any active venv from PATH because we're trying to only get the global system python paths 158 | if ACTIVE_VENV: 159 | pip_bin_dirs.remove(f"{ACTIVE_VENV}/bin") 160 | 161 | for bin_dir in pip_bin_dirs: 162 | if bin_dir not in PATH: 163 | PATH = ":".join([*PATH.split(":"), bin_dir]) 164 | self.PATH = TypeAdapter(PATHStr).validate_python(PATH) 165 | return self 166 | 167 | def setup(self): 168 | """create pip venv dir if needed""" 169 | try: 170 | self.cache_dir.mkdir(parents=True, exist_ok=True) 171 | os.system(f'chown {self.EUID} "{self.cache_dir}" 2>/dev/null') # try to ensure cache dir is writable by EUID 172 | os.system(f'chmod 777 "{self.cache_dir}" 2>/dev/null') # allow all users to share cache dir 173 | except Exception: 174 | self.cache_arg = '--no-cache-dir' 175 | 176 | if self.pip_venv: 177 | self._pip_setup_venv(self.pip_venv) 178 | 179 | def _pip_setup_venv(self, pip_venv: Path): 180 | pip_venv.parent.mkdir(parents=True, exist_ok=True) 181 | 182 | # create new venv in pip_venv if it doesnt exist 183 | venv_pip_path = pip_venv / "bin" / "python" 184 | venv_pip_binary_exists = (os.path.isfile(venv_pip_path) and os.access(venv_pip_path, os.X_OK)) 185 | if not venv_pip_binary_exists: 186 | import venv 187 | 188 | venv.create( 189 | str(pip_venv), 190 | system_site_packages=False, 191 | clear=True, 192 | symlinks=True, 193 | with_pip=True, 194 | upgrade_deps=True, 195 | ) 196 | assert os.path.isfile(venv_pip_path) and os.access(venv_pip_path, os.X_OK), f'could not find pip inside venv after creating it: {pip_venv}' 197 | self.exec(bin_name=venv_pip_path, cmd=["install", self.cache_arg, "--upgrade", "pip", "setuptools"]) # setuptools is not installed by default after python >= 3.12 198 | 199 | def _pip_show(self, bin_name: BinName, packages: Optional[InstallArgs] = None) -> List[str]: 200 | pip_abspath = self.INSTALLER_BIN_ABSPATH 201 | if not pip_abspath: 202 | raise Exception( 203 | f"{self.__class__.__name__} install method is not available on this host ({self.INSTALLER_BIN} not found in $PATH)" 204 | ) 205 | 206 | packages = packages or self.get_packages(str(bin_name)) or [str(bin_name)] 207 | main_package = packages[0] # assume first package in list is the main one 208 | output_lines = self.exec(bin_name=pip_abspath, cmd=[ 209 | 'show', 210 | '--no-input', 211 | main_package, 212 | ], timeout=self._version_timeout, quiet=True).stdout.strip().split('\n') 213 | return output_lines 214 | 215 | def _pip_install(self, packages: InstallArgs) -> subprocess.CompletedProcess: 216 | pip_abspath = self.INSTALLER_BIN_ABSPATH 217 | if not pip_abspath: 218 | raise Exception( 219 | f"{self.__class__.__name__} install method is not available on this host ({self.INSTALLER_BIN} not found in $PATH)" 220 | ) 221 | 222 | return self.exec(bin_name=pip_abspath, cmd=[ 223 | 'install', 224 | '--no-input', 225 | self.cache_arg, 226 | *self.pip_install_args, 227 | *packages, 228 | ]) 229 | 230 | 231 | def default_install_handler(self, bin_name: str, packages: Optional[InstallArgs] = None, **context) -> str: 232 | if self.pip_venv: 233 | self.setup() 234 | 235 | packages = packages or self.get_packages(bin_name) 236 | 237 | # print(f'[*] {self.__class__.__name__}: Installing {bin_name}: {self.INSTALLER_BIN_ABSPATH} install {packages}') 238 | 239 | # pip install --no-input --cache-dir= 240 | proc = self._pip_install(packages) 241 | 242 | if proc.returncode != 0: 243 | print(proc.stdout.strip()) 244 | print(proc.stderr.strip()) 245 | raise Exception(f"{self.__class__.__name__}: install got returncode {proc.returncode} while installing {packages}: {packages}") 246 | 247 | return proc.stderr.strip() + "\n" + proc.stdout.strip() 248 | 249 | def default_abspath_handler(self, bin_name: BinName, **context) -> HostBinPath | None: 250 | 251 | # try searching for the bin_name in BinProvider.PATH first (fastest) 252 | try: 253 | abspath = super().default_abspath_handler(bin_name, **context) 254 | if abspath: 255 | return TypeAdapter(HostBinPath).validate_python(abspath) 256 | except ValueError: 257 | pass 258 | 259 | # fallback to using pip show to get the site-packages bin path 260 | output_lines = self._pip_show(bin_name) 261 | # For more information, please refer to 262 | # Location: /Volumes/NVME/Users/squash/Library/Python/3.11/lib/python/site-packages 263 | # Requires: brotli, certifi, mutagen, pycryptodomex, requests, urllib3, websockets 264 | # Required-by: 265 | try: 266 | location = [line for line in output_lines if line.startswith('Location: ')][0].split('Location: ', 1)[-1] 267 | except IndexError: 268 | return None 269 | PATH = str(Path(location).parent.parent.parent / 'bin') 270 | abspath = bin_abspath(str(bin_name), PATH=PATH) 271 | if abspath: 272 | return TypeAdapter(HostBinPath).validate_python(abspath) 273 | else: 274 | return None 275 | 276 | def default_version_handler(self, bin_name: BinName, abspath: Optional[HostBinPath]=None, **context) -> SemVer | None: 277 | # print(f'[*] {self.__class__.__name__}: Getting version for {bin_name}...') 278 | 279 | # try running --version first (fastest) 280 | try: 281 | version = super().default_version_handler(bin_name, abspath, **context) 282 | if version: 283 | return SemVer.parse(version) 284 | except ValueError: 285 | pass 286 | 287 | # fallback to using pip show to get the version (slower) 288 | output_lines = self._pip_show(bin_name) 289 | # Name: yt-dlp 290 | # Version: 1.3.0 291 | # Location: /Volumes/NVME/Users/squash/Library/Python/3.11/lib/python/site-packages 292 | try: 293 | version_str = [line for line in output_lines if line.startswith('Version: ')][0].split('Version: ', 1)[-1] 294 | return SemVer.parse(version_str) 295 | except Exception: 296 | return None 297 | 298 | 299 | if __name__ == "__main__": 300 | # Usage: 301 | # ./binprovider_pip.py load yt-dlp 302 | # ./binprovider_pip.py install pip 303 | # ./binprovider_pip.py get_version pip 304 | # ./binprovider_pip.py get_abspath pip 305 | result = pip = PipProvider() 306 | 307 | if len(sys.argv) > 1: 308 | result = func = getattr(pip, sys.argv[1]) # e.g. install 309 | 310 | if len(sys.argv) > 2: 311 | result = func(sys.argv[2]) # e.g. install ffmpeg 312 | 313 | print(result) 314 | -------------------------------------------------------------------------------- /abx_pkg/binprovider_pyinfra.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | __package__ = 'abx_pkg' 3 | 4 | import os 5 | import sys 6 | import shutil 7 | from pathlib import Path 8 | 9 | from typing import Optional, Dict, Any 10 | 11 | from .base_types import BinProviderName, PATHStr, BinName, InstallArgs 12 | from .binprovider import BinProvider, OPERATING_SYSTEM, DEFAULT_PATH 13 | 14 | PYINFRA_INSTALLED = False 15 | PYINFRA_IMPORT_ERROR = None 16 | try: 17 | # from pyinfra import host 18 | from pyinfra import operations # noqa: F401 19 | from pyinfra.api import Config, Inventory, State 20 | from pyinfra.api.connect import connect_all 21 | from pyinfra.api.operation import add_op 22 | from pyinfra.api.operations import run_ops 23 | from pyinfra.api.exceptions import PyinfraError 24 | 25 | PYINFRA_INSTALLED = True 26 | except ImportError as err: 27 | PYINFRA_IMPORT_ERROR = err 28 | pass 29 | 30 | 31 | 32 | 33 | def pyinfra_package_install(pkg_names: InstallArgs, installer_module: str = "auto", installer_extra_kwargs: Optional[Dict[str, Any]] = None) -> str: 34 | if not PYINFRA_INSTALLED: 35 | raise RuntimeError("Pyinfra is not installed! To fix:\n pip install pyinfra") from PYINFRA_IMPORT_ERROR 36 | 37 | config = Config() 38 | inventory = Inventory((["@local"], {})) 39 | state = State(inventory=inventory, config=config) 40 | 41 | if isinstance(pkg_names, str): 42 | pkg_names = pkg_names.split(' ') # type: ignore 43 | 44 | connect_all(state) 45 | 46 | _sudo_user = None 47 | if installer_module == 'auto': 48 | is_macos = OPERATING_SYSTEM == "darwin" 49 | if is_macos: 50 | installer_module = 'operations.brew.packages' 51 | try: 52 | _sudo_user = Path(shutil.which('brew')).stat().st_uid 53 | except Exception: 54 | pass 55 | else: 56 | installer_module = 'operations.server.packages' 57 | else: 58 | # TODO: non-stock pyinfra modules from other libraries? 59 | assert installer_module.startswith('operations.') 60 | 61 | try: 62 | installer_module_op = eval(installer_module) 63 | except Exception as err: 64 | raise RuntimeError(f'Failed to import pyinfra installer_module {installer_module}: {err.__class__.__name__}') from err 65 | 66 | result = add_op( 67 | state, 68 | installer_module_op, 69 | name=f"Install system packages: {pkg_names}", 70 | packages=pkg_names, 71 | _sudo_user=_sudo_user, 72 | **(installer_extra_kwargs or {}), 73 | ) 74 | 75 | succeeded = False 76 | try: 77 | run_ops(state) 78 | succeeded = True 79 | except PyinfraError: 80 | succeeded = False 81 | 82 | result = result[state.inventory.hosts["@local"]] 83 | result_text = f'Installing {pkg_names} on {OPERATING_SYSTEM} using Pyinfra {installer_module} {["failed", "succeeded"][succeeded]}\n{result.stdout}\n{result.stderr}'.strip() 84 | 85 | if succeeded: 86 | return result_text 87 | 88 | if "Permission denied" in result_text: 89 | raise PermissionError( 90 | f"Installing {pkg_names} failed! Need to be root to use package manager (retry with sudo, or install manually)" 91 | ) 92 | raise Exception(f"Installing {pkg_names} failed! (retry with sudo, or install manually)\n{result_text}") 93 | 94 | 95 | 96 | class PyinfraProvider(BinProvider): 97 | name: BinProviderName = "pyinfra" 98 | INSTALLER_BIN: BinName = "pyinfra" 99 | PATH: PATHStr = os.environ.get("PATH", DEFAULT_PATH) 100 | 101 | pyinfra_installer_module: str = 'auto' # e.g. operations.apt.packages, operations.server.packages, etc. 102 | pyinfra_installer_kwargs: Dict[str, Any] = {} 103 | 104 | 105 | def default_install_handler(self, bin_name: str, packages: Optional[InstallArgs] = None, **context) -> str: 106 | packages = packages or self.get_packages(bin_name) 107 | 108 | return pyinfra_package_install( 109 | pkg_names=packages, 110 | installer_module=self.pyinfra_installer_module, 111 | installer_extra_kwargs=self.pyinfra_installer_kwargs, 112 | ) 113 | 114 | 115 | if __name__ == "__main__": 116 | result = pyinfra = PyinfraProvider() 117 | 118 | if len(sys.argv) > 1: 119 | result = func = getattr(pyinfra, sys.argv[1]) # e.g. install 120 | 121 | if len(sys.argv) > 2: 122 | result = func(sys.argv[2]) # e.g. install ffmpeg 123 | 124 | print(result) 125 | -------------------------------------------------------------------------------- /abx_pkg/models.py: -------------------------------------------------------------------------------- 1 | # pip install django-pydantic-field 2 | 3 | ### EXAMPLE USAGE 4 | # 5 | # from django.db import models 6 | # from django_pydantic_field import SchemaField 7 | # 8 | # from abx_pkg import BinProvider, EnvProvider, Binary 9 | # 10 | # DEFAULT_PROVIDER = EnvProvider() 11 | # 12 | # class MyModel(models.Model): 13 | # ... 14 | # 15 | # # SchemaField supports storing a single BinProvider/Binary in a field... 16 | # favorite_binprovider: BinProvider = SchemaField(default=DEFAULT_PROVIDER) 17 | # 18 | # # ... or inside a collection type like list[...] dict[...] 19 | # optional_binaries: list[Binary] = SchemaField(default=[]) 20 | # 21 | # curl = Binary(name='curl', providers=[DEFAULT_PROVIDER]).load() 22 | # 23 | # obj = MyModel(optional_binaries=[curl]) 24 | # obj.save() 25 | # 26 | # assert obj.favorite_binprovider == DEFAULT_PROVIDER 27 | # assert obj.optional_binaries[0].provider == DEFAULT_PROVIDER 28 | -------------------------------------------------------------------------------- /abx_pkg/semver.py: -------------------------------------------------------------------------------- 1 | __package__ = 'abx_pkg' 2 | 3 | import re 4 | import subprocess 5 | from collections import namedtuple 6 | 7 | from typing import Any, Optional, TYPE_CHECKING 8 | 9 | from pydantic_core import ValidationError 10 | from pydantic import validate_call 11 | 12 | from .base_types import HostBinPath 13 | 14 | 15 | def is_semver_str(semver: Any) -> bool: 16 | if isinstance(semver, str): 17 | return (semver.count('.') == 2 and semver.replace('.', '').isdigit()) 18 | return False 19 | 20 | def semver_to_str(semver: tuple[int, int, int] | str) -> str: 21 | if isinstance(semver, (list, tuple)): 22 | return '.'.join(str(chunk) for chunk in semver) 23 | if is_semver_str(semver): 24 | return semver 25 | raise ValidationError('Tried to convert invalid SemVer: {}'.format(semver)) 26 | 27 | 28 | SemVerTuple = namedtuple('SemVerTuple', ('major', 'minor', 'patch'), defaults=(0, 0, 0)) 29 | SemVerParsableTypes = str | tuple[str | int, ...] | list[str | int] 30 | 31 | class SemVer(SemVerTuple): 32 | major: int 33 | minor: int = 0 34 | patch: int = 0 35 | 36 | if TYPE_CHECKING: 37 | full_text: str | None = '' 38 | 39 | def __new__(cls, *args, full_text=None, **kwargs): 40 | # '1.1.1' 41 | if len(args) == 1 and is_semver_str(args[0]): 42 | result = SemVer.parse(args[0]) 43 | 44 | # ('1', '2', '3') 45 | elif len(args) == 1 and isinstance(args[0], (tuple, list)): 46 | result = SemVer.parse(args[0]) 47 | 48 | # (1, '2', None) 49 | elif not all(isinstance(arg, (int, type(None))) for arg in args): 50 | result = SemVer.parse(args) 51 | 52 | # (None) 53 | elif all(chunk in ('', 0, None) for chunk in (*args, *kwargs.values())): 54 | result = None 55 | 56 | # 1, 2, 3 57 | else: 58 | result = SemVerTuple.__new__(cls, *args, **kwargs) 59 | 60 | if result is not None: 61 | # add first line as extra hidden metadata so it can be logged without having to re-run version cmd 62 | result.full_text = full_text or str(result) 63 | return result 64 | 65 | @classmethod 66 | def parse(cls, version_stdout: SemVerParsableTypes) -> Optional['SemVer']: 67 | """ 68 | parses a version tag string formatted like into (major, minor, patch) ints 69 | 'Google Chrome 124.0.6367.208' -> (124, 0, 6367) 70 | 'GNU Wget 1.24.5 built on darwin23.2.0.' -> (1, 24, 5) 71 | 'curl 8.4.0 (x86_64-apple-darwin23.0) ...' -> (8, 4, 0) 72 | '2024.04.09' -> (2024, 4, 9) 73 | 74 | """ 75 | # print('INITIAL_VALUE', type(version_stdout).__name__, version_stdout) 76 | 77 | if isinstance(version_stdout, (tuple, list)): 78 | version_stdout = '.'.join(str(chunk) for chunk in version_stdout) 79 | elif isinstance(version_stdout, bytes): 80 | version_stdout = version_stdout.decode() 81 | elif not isinstance(version_stdout, str): 82 | version_stdout = str(version_stdout) 83 | 84 | # no text to work with, return None immediately 85 | if not version_stdout.strip(): 86 | # raise Exception('Tried to parse semver from empty version output (is binary installed and available?)') 87 | return None 88 | 89 | just_numbers = lambda col: '.'.join([chunk for chunk in re.split(r'[\D]', col.lower().strip('v'), maxsplit=10) if chunk.isdigit()][:3]) # split on any non-num character e.g. 5.2.26(1)-release -> ['5', '2', '26', '1', '', '', ...] 90 | contains_semver = lambda col: ( 91 | col.count('.') in (1, 2, 3) 92 | and all(chunk.isdigit() for chunk in col.split('.')[:3]) # first 3 chunks can only be nums 93 | ) 94 | 95 | full_text = version_stdout.split('\n')[0].strip() 96 | first_line_columns = full_text.split()[:5] 97 | version_columns = list(filter(contains_semver, map(just_numbers, first_line_columns))) 98 | 99 | # could not find any column of first line that looks like a version number, despite there being some text 100 | if not version_columns: 101 | # raise Exception('Failed to parse semver from version command output: {}'.format(' '.join(first_line_columns))) 102 | return None 103 | 104 | # take first col containing a semver, and truncate it to 3 chunks (e.g. 2024.04.09.91) -> (2024, 04, 09) 105 | first_version_tuple = version_columns[0].split('.', 3)[:3] 106 | 107 | # print('FINAL_VALUE', first_version_tuple) 108 | 109 | return cls(*(int(chunk) for chunk in first_version_tuple), full_text=full_text) 110 | 111 | def __str__(self): 112 | return '.'.join(str(chunk) for chunk in self) 113 | 114 | 115 | # Not needed as long as we dont stray any further from a basic NamedTuple 116 | # if we start overloading more methods or it becomes a fully custom type, then we probably need this: 117 | # @classmethod 118 | # def __get_pydantic_core_schema__(cls, source: Type[Any], handler: GetCoreSchemaHandler) -> core_schema.CoreSchema: 119 | # default_schema = handler(source) 120 | # return core_schema.no_info_after_validator_function( 121 | # cls.parse, 122 | # default_schema, 123 | # serialization=core_schema.plain_serializer_function_ser_schema( 124 | # lambda semver: str(semver), 125 | # info_arg=False, 126 | # return_schema=core_schema.str_schema(), 127 | # ), 128 | # ) 129 | 130 | 131 | # @validate_call 132 | def bin_version(bin_path: HostBinPath, args=("--version",)) -> SemVer | None: 133 | return SemVer(subprocess.run([str(bin_path), *args], stdout=subprocess.PIPE, text=True).stdout.strip()) 134 | -------------------------------------------------------------------------------- /abx_pkg/settings.py: -------------------------------------------------------------------------------- 1 | from typing import Callable 2 | 3 | from django.conf import settings 4 | from django.utils.module_loading import import_string 5 | 6 | 7 | ABX_PKG_GET_ALL_BINARIES = getattr(settings, 'ABX_PKG_GET_ALL_BINARIES', 'abx_pkg.views.get_all_binaries') 8 | ABX_PKG_GET_BINARY = getattr(settings, 'ABX_PKG_GET_BINARY', 'abx_pkg.views.get_binary') 9 | 10 | 11 | if isinstance(ABX_PKG_GET_ALL_BINARIES, str): 12 | get_all_abx_pkg_binaries = import_string(ABX_PKG_GET_ALL_BINARIES) 13 | elif isinstance(ABX_PKG_GET_ALL_BINARIES, Callable): 14 | get_all_abx_pkg_binaries = ABX_PKG_GET_ALL_BINARIES 15 | else: 16 | raise ValueError('ABX_PKG_GET_ALL_BINARIES must be a function or dotted import path to a function') 17 | 18 | if isinstance(ABX_PKG_GET_BINARY, str): 19 | get_abx_pkg_binary = import_string(ABX_PKG_GET_BINARY) 20 | elif isinstance(ABX_PKG_GET_BINARY, Callable): 21 | get_abx_pkg_binary = ABX_PKG_GET_BINARY 22 | else: 23 | raise ValueError('ABX_PKG_GET_BINARY must be a function or dotted import path to a function') 24 | 25 | 26 | -------------------------------------------------------------------------------- /abx_pkg/shallowbinary.py: -------------------------------------------------------------------------------- 1 | __package__ = 'abx_pkg' 2 | 3 | 4 | # Unfortunately it must be kept in the same file as BinProvider because of the circular type reference between them 5 | from .binprovider import ShallowBinary # noqa: F401 6 | -------------------------------------------------------------------------------- /abx_pkg/tests.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | # Create your tests here. 4 | -------------------------------------------------------------------------------- /abx_pkg/views.py: -------------------------------------------------------------------------------- 1 | # pip install django-admin-data-views 2 | 3 | from django.http import HttpRequest 4 | from django.utils.html import mark_safe 5 | 6 | from admin_data_views.typing import TableContext, ItemContext 7 | from admin_data_views.utils import render_with_table_view, render_with_item_view, ItemLink 8 | 9 | from .binary import Binary 10 | 11 | 12 | def get_all_binaries() -> list[Binary]: 13 | """Override this function implement getting the list of binaries to render""" 14 | return [] 15 | 16 | def get_binary(name: str) -> Binary: 17 | """Override this function implement getting the list of binaries to render""" 18 | 19 | from . import settings 20 | 21 | for binary in settings.ABX_PKG_GET_ALL_BINARIES(): 22 | if binary.name == key: 23 | return binary 24 | return None 25 | 26 | 27 | 28 | @render_with_table_view 29 | def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext: 30 | 31 | assert request.user.is_superuser, 'Must be a superuser to view configuration settings.' 32 | 33 | from . import settings 34 | 35 | rows = { 36 | "Binary": [], 37 | "Found Version": [], 38 | "Provided By": [], 39 | "Found Abspath": [], 40 | "Overrides": [], 41 | "Description": [], 42 | } 43 | 44 | for binary in settings.get_all_abx_pkg_binaries(): 45 | binary = binary.load_or_install() 46 | 47 | rows['Binary'].append(ItemLink(binary.name, key=binary.name)) 48 | rows['Found Version'].append(binary.loaded_version) 49 | rows['Provided By'].append(binary.loaded_binprovider) 50 | rows['Found Abspath'].append(binary.loaded_abspath) 51 | rows['Overrides'].append(str(binary.overrides)) 52 | rows['Description'].append(binary.description) 53 | 54 | return TableContext( 55 | title="Binaries", 56 | table=rows, 57 | ) 58 | 59 | @render_with_item_view 60 | def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext: 61 | 62 | assert request.user.is_superuser, 'Must be a superuser to view configuration settings.' 63 | 64 | from . import settings 65 | 66 | binary = settings.get_abx_pkg_binary(key) 67 | 68 | assert binary, f'Could not find a binary matching the specified name: {key}' 69 | 70 | binary = binary.load_or_install() 71 | 72 | return ItemContext( 73 | slug=key, 74 | title=key, 75 | data=[ 76 | { 77 | "name": binary.name, 78 | "description": binary.description, 79 | "fields": { 80 | 'binprovider': binary.loaded_provider, 81 | 'abspath': binary.loaded_abspath, 82 | 'version': binary.loaded_version, 83 | 'is_script': binary.is_script, 84 | 'is_executable': binary.is_executable, 85 | 'is_valid': binary.is_valid, 86 | 'overrides': str(binary.overrides), 87 | 'providers': str(binary.binproviders_supported), 88 | }, 89 | "help_texts": { 90 | # TODO 91 | }, 92 | }, 93 | ], 94 | ) 95 | -------------------------------------------------------------------------------- /bin/publish.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [[ "$1" == "" ]]; then 4 | echo "Usage: $0 " 5 | exit 1 6 | fi 7 | 8 | echo "Bumping to version $1" 9 | 10 | rm -Rf dist 2>/dev/null 11 | nano pyproject.toml 12 | git add -p . 13 | git commit -m "bump version to $1" 14 | git tag -f -a "v$1" -m "v$1" 15 | git push origin 16 | git push origin --tags -f 17 | uv lock 18 | uv sync 19 | uv build 20 | uv publish 21 | -------------------------------------------------------------------------------- /django_example_project/README.md: -------------------------------------------------------------------------------- 1 | # Example Django Project 2 | 3 | 4 | ```bash 5 | git clone 'https://github.com/ArchiveBox/abx-pkg' 6 | cd abx-pkg 7 | 8 | pip install -e . # install abx_pkg from source 9 | 10 | cd django_example_project/ # then go into the demo project dir 11 | 12 | ./manage.py makemigrations # create any migrations if needed 13 | ./manage.py migrate # then run them to create the demo sqlite db 14 | ./manage.py createsuperuser # create an admin user to test out the UI 15 | 16 | ./manage.py runserver # then open http://127.0.0.1:8000/admin/ 17 | ``` 18 | -------------------------------------------------------------------------------- /django_example_project/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """Django's command-line utility for administrative tasks.""" 3 | import os 4 | import sys 5 | 6 | 7 | def main(): 8 | """Run administrative tasks.""" 9 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'project.settings') 10 | try: 11 | from django.core.management import execute_from_command_line 12 | except ImportError as exc: 13 | raise ImportError( 14 | "Couldn't import Django. Are you sure it's installed and " 15 | "available on your PYTHONPATH environment variable? Did you " 16 | "forget to activate a virtual environment?" 17 | ) from exc 18 | execute_from_command_line(sys.argv) 19 | 20 | 21 | if __name__ == '__main__': 22 | main() 23 | -------------------------------------------------------------------------------- /django_example_project/project/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ArchiveBox/abx-pkg/cf4a4f23ee57720186e8f907e32c75bed729035a/django_example_project/project/__init__.py -------------------------------------------------------------------------------- /django_example_project/project/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | 3 | from django_jsonform.widgets import JSONFormWidget 4 | from django_pydantic_field.v2.fields import PydanticSchemaField 5 | 6 | from project.models import Dependency 7 | 8 | 9 | def patch_schema_for_jsonform(schema): 10 | """recursively patch a schema dictionary in-place to fix any missing properties/keys on objects""" 11 | 12 | # base case: schema is type: "object" with no properties/keys 13 | if schema.get('type') == 'object' and not ('properties' in schema or 'keys' in schema): 14 | if 'default' in schema and isinstance(schema['default'], dict): 15 | schema['properties'] = { 16 | key: {"type": "string", "default": value} 17 | for key, value in schema['default'].items() 18 | } 19 | else: 20 | schema['properties'] = {} 21 | elif schema.get('type') == 'array' and not ('items' in schema): 22 | if 'default' in schema and isinstance(schema['default'], (tuple, list)): 23 | schema['items'] = {'type': 'string', 'default': schema['default']} 24 | else: 25 | schema['items'] = {'type': 'string', 'default': []} 26 | 27 | # recursive case: iterate through all values and process any sub-objects 28 | for key, value in schema.items(): 29 | if isinstance(value, dict): 30 | patch_schema_for_jsonform(value) 31 | 32 | 33 | 34 | class PatchedJSONFormWidget(JSONFormWidget): 35 | def get_schema(self): 36 | self.schema = super().get_schema() 37 | patch_schema_for_jsonform(self.schema) 38 | return self.schema 39 | 40 | 41 | 42 | class DependencyAdmin(admin.ModelAdmin): 43 | formfield_overrides = {PydanticSchemaField: {"widget": PatchedJSONFormWidget}} 44 | 45 | admin.site.register(Dependency, DependencyAdmin) 46 | -------------------------------------------------------------------------------- /django_example_project/project/asgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | ASGI config for project project. 3 | 4 | It exposes the ASGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/5.0/howto/deployment/asgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.asgi import get_asgi_application 13 | 14 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'project.settings') 15 | 16 | application = get_asgi_application() 17 | -------------------------------------------------------------------------------- /django_example_project/project/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 5.0.6 on 2024-05-21 00:31 2 | 3 | import django.core.serializers.json 4 | import django_pydantic_field.compat.django 5 | import django_pydantic_field.fields 6 | import pydantic_pkgr.binary 7 | import pydantic_pkgr.binprovider 8 | from django.db import migrations, models 9 | 10 | 11 | class Migration(migrations.Migration): 12 | 13 | initial = True 14 | 15 | dependencies = [ 16 | ] 17 | 18 | operations = [ 19 | migrations.CreateModel( 20 | name='Dependency', 21 | fields=[ 22 | ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 23 | ('label', models.CharField(max_length=63)), 24 | ('default_binprovider', django_pydantic_field.fields.PydanticSchemaField(config=None, default={'name': 'env'}, encoder=django.core.serializers.json.DjangoJSONEncoder, schema=pydantic_pkgr.binprovider.BinProvider)), 25 | ('binaries', django_pydantic_field.fields.PydanticSchemaField(config=None, default=[], encoder=django.core.serializers.json.DjangoJSONEncoder, schema=django_pydantic_field.compat.django.GenericContainer(list, (pydantic_pkgr.binary.Binary,)))), 26 | ], 27 | ), 28 | ] 29 | -------------------------------------------------------------------------------- /django_example_project/project/migrations/0002_alter_dependency_options_dependency_min_version.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 5.0.6 on 2024-05-21 02:39 2 | 3 | import django.core.serializers.json 4 | import django_pydantic_field.fields 5 | import pydantic_pkgr.semver 6 | from django.db import migrations 7 | 8 | 9 | class Migration(migrations.Migration): 10 | 11 | dependencies = [ 12 | ('project', '0001_initial'), 13 | ] 14 | 15 | operations = [ 16 | migrations.AlterModelOptions( 17 | name='dependency', 18 | options={'verbose_name_plural': 'Dependencies'}, 19 | ), 20 | migrations.AddField( 21 | model_name='dependency', 22 | name='min_version', 23 | field=django_pydantic_field.fields.PydanticSchemaField(config=None, default=[0, 0, 1], encoder=django.core.serializers.json.DjangoJSONEncoder, schema=pydantic_pkgr.semver.SemVer), 24 | ), 25 | ] 26 | -------------------------------------------------------------------------------- /django_example_project/project/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ArchiveBox/abx-pkg/cf4a4f23ee57720186e8f907e32c75bed729035a/django_example_project/project/migrations/__init__.py -------------------------------------------------------------------------------- /django_example_project/project/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | from django_pydantic_field import SchemaField 3 | 4 | from abx_pkg import BinProvider, EnvProvider, Binary, SemVer 5 | 6 | 7 | DEFAULT_PROVIDER = EnvProvider() 8 | 9 | 10 | class Dependency(models.Model): 11 | """Example model implementing fields that contain BinProvider and Binary data""" 12 | 13 | label = models.CharField(max_length=63) 14 | 15 | default_binprovider: BinProvider = SchemaField(default=DEFAULT_PROVIDER) 16 | 17 | binaries: list[Binary] = SchemaField(default=[]) 18 | 19 | min_version: SemVer = SchemaField(default=(0,0,1)) 20 | 21 | class Meta: 22 | verbose_name_plural = 'Dependencies' 23 | -------------------------------------------------------------------------------- /django_example_project/project/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django settings for project project. 3 | 4 | Generated by 'django-admin startproject' using Django 5.0.6. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/5.0/topics/settings/ 8 | 9 | For the full list of settings and their values, see 10 | https://docs.djangoproject.com/en/5.0/ref/settings/ 11 | """ 12 | 13 | from pathlib import Path 14 | 15 | # Build paths inside the project like this: BASE_DIR / 'subdir'. 16 | BASE_DIR = Path(__file__).resolve().parent.parent 17 | 18 | 19 | # Quick-start development settings - unsuitable for production 20 | # See https://docs.djangoproject.com/en/5.0/howto/deployment/checklist/ 21 | 22 | # SECURITY WARNING: keep the secret key used in production secret! 23 | SECRET_KEY = 'django-insecure-uppz@=qfk)lw+neoz3x)xq&vr715i@9c9-@yi=)!trr12rnb0+' 24 | 25 | # SECURITY WARNING: don't run with debug turned on in production! 26 | DEBUG = True 27 | 28 | ALLOWED_HOSTS = [] 29 | 30 | 31 | # Application definition 32 | 33 | INSTALLED_APPS = [ 34 | 'django.contrib.admin', 35 | 'django.contrib.auth', 36 | 'django.contrib.contenttypes', 37 | 'django.contrib.sessions', 38 | 'django.contrib.messages', 39 | 'django.contrib.staticfiles', 40 | 41 | 'admin_data_views', 42 | 'django_jsonform', 43 | 44 | 'abx_pkg', 45 | 46 | 'project', 47 | ] 48 | 49 | MIDDLEWARE = [ 50 | 'django.middleware.security.SecurityMiddleware', 51 | 'django.contrib.sessions.middleware.SessionMiddleware', 52 | 'django.middleware.common.CommonMiddleware', 53 | 'django.middleware.csrf.CsrfViewMiddleware', 54 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 55 | 'django.contrib.messages.middleware.MessageMiddleware', 56 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 57 | ] 58 | 59 | ROOT_URLCONF = 'project.urls' 60 | 61 | TEMPLATES = [ 62 | { 63 | 'BACKEND': 'django.template.backends.django.DjangoTemplates', 64 | 'DIRS': [], 65 | 'APP_DIRS': True, 66 | 'OPTIONS': { 67 | 'context_processors': [ 68 | 'django.template.context_processors.debug', 69 | 'django.template.context_processors.request', 70 | 'django.contrib.auth.context_processors.auth', 71 | 'django.contrib.messages.context_processors.messages', 72 | ], 73 | }, 74 | }, 75 | ] 76 | 77 | WSGI_APPLICATION = 'project.wsgi.application' 78 | 79 | 80 | # Database 81 | # https://docs.djangoproject.com/en/5.0/ref/settings/#databases 82 | 83 | DATABASES = { 84 | 'default': { 85 | 'ENGINE': 'django.db.backends.sqlite3', 86 | 'NAME': BASE_DIR / 'db.sqlite3', 87 | } 88 | } 89 | 90 | 91 | # Password validation 92 | # https://docs.djangoproject.com/en/5.0/ref/settings/#auth-password-validators 93 | 94 | AUTH_PASSWORD_VALIDATORS = [ 95 | { 96 | 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', 97 | }, 98 | { 99 | 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', 100 | }, 101 | { 102 | 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', 103 | }, 104 | { 105 | 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', 106 | }, 107 | ] 108 | 109 | 110 | # Internationalization 111 | # https://docs.djangoproject.com/en/5.0/topics/i18n/ 112 | 113 | LANGUAGE_CODE = 'en-us' 114 | 115 | TIME_ZONE = 'UTC' 116 | 117 | USE_I18N = True 118 | 119 | USE_TZ = True 120 | 121 | 122 | # Static files (CSS, JavaScript, Images) 123 | # https://docs.djangoproject.com/en/5.0/howto/static-files/ 124 | 125 | STATIC_URL = 'static/' 126 | 127 | # Default primary key field type 128 | # https://docs.djangoproject.com/en/5.0/ref/settings/#default-auto-field 129 | 130 | DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' 131 | 132 | 133 | ABX_PKG_GET_ALL_BINARIES = 'project.views.get_all_binaries' 134 | ABX_PKG_GET_BINARY = 'project.views.get_binary' 135 | 136 | 137 | ADMIN_DATA_VIEWS = { 138 | "NAME": "Environment", 139 | "URLS": [ 140 | { 141 | "route": "binaries/", 142 | "view": "abx_pkg.views.binaries_list_view", 143 | "name": "binaries", 144 | "items": { 145 | "route": "/", 146 | "view": "abx_pkg.views.binary_detail_view", 147 | "name": "binary", 148 | }, 149 | }, 150 | # Coming soon: binprovider_list_view + binprovider_detail_view ... 151 | ], 152 | } 153 | -------------------------------------------------------------------------------- /django_example_project/project/urls.py: -------------------------------------------------------------------------------- 1 | """ 2 | URL configuration for project project. 3 | 4 | The `urlpatterns` list routes URLs to views. For more information please see: 5 | https://docs.djangoproject.com/en/5.0/topics/http/urls/ 6 | Examples: 7 | Function views 8 | 1. Add an import: from my_app import views 9 | 2. Add a URL to urlpatterns: path('', views.home, name='home') 10 | Class-based views 11 | 1. Add an import: from other_app.views import Home 12 | 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') 13 | Including another URLconf 14 | 1. Import the include() function: from django.urls import include, path 15 | 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) 16 | """ 17 | from django.contrib import admin 18 | from django.urls import path 19 | 20 | urlpatterns = [ 21 | path('admin/', admin.site.urls), 22 | ] 23 | -------------------------------------------------------------------------------- /django_example_project/project/views.py: -------------------------------------------------------------------------------- 1 | from abx_pkg.binary import Binary 2 | 3 | 4 | def get_all_binaries() -> list[Binary]: 5 | """Override this function implement getting the list of binaries to render""" 6 | return [ 7 | Binary(name='bash'), 8 | Binary(name='python'), 9 | Binary(name='brew'), 10 | Binary(name='git'), 11 | ] 12 | 13 | def get_binary(name: str) -> Binary: 14 | """Override this function implement getting the list of binaries to render""" 15 | 16 | from abx_pkg import settings 17 | 18 | for binary in settings.get_all_abx_pkg_binaries(): 19 | if binary.name == name: 20 | return binary 21 | return None 22 | -------------------------------------------------------------------------------- /django_example_project/project/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for project project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/5.0/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.wsgi import get_wsgi_application 13 | 14 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'project.settings') 15 | 16 | application = get_wsgi_application() 17 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "abx-pkg" 3 | version = "0.6.0" 4 | description = "System package manager interfaces with Python type hints" 5 | authors = [ 6 | {name = "Nick Sweeting", email = "abx-pkg-pyproject-toml@sweeting.me"}, 7 | ] 8 | requires-python = ">=3.10" 9 | license = {text = "MIT"} 10 | readme = "README.md" 11 | keywords = [ 12 | "pydantic", 13 | "packagemanager", 14 | "apt", 15 | "brew", 16 | "pip", 17 | "system", 18 | "dependencies", 19 | ] 20 | classifiers = [ 21 | "Programming Language :: Python", 22 | "Programming Language :: Python :: 3", 23 | "Programming Language :: Python :: 3 :: Only", 24 | "Programming Language :: Python :: 3.10", 25 | "Programming Language :: Python :: 3.11", 26 | "Programming Language :: Python :: 3.12", 27 | "Framework :: Django", 28 | "Framework :: Django :: 4.0", 29 | "Framework :: Django :: 4.1", 30 | "Framework :: Django :: 4.2", 31 | "Framework :: Django :: 5.0", 32 | "Framework :: Django :: 5.1", 33 | "Environment :: Web Environment", 34 | "Intended Audience :: Developers", 35 | "Natural Language :: English", 36 | ] 37 | 38 | dependencies = [ 39 | "pip>=24.3.1", 40 | "typing-extensions>=4.11.0", 41 | "platformdirs>=4.3.6", 42 | "pydantic>=2.7.1", 43 | "pydantic-core>=2.18.2", 44 | ] 45 | 46 | [project.optional-dependencies] 47 | pyinfra = [ 48 | 'pyinfra>=2.6.1', 49 | ] 50 | ansible = [ 51 | 'ansible>=10.5.0', 52 | 'ansible-core>=2.17.5', 53 | 'ansible-runner>=2.4.0', 54 | ] 55 | all = [ 56 | "abx-pkg[pyinfra,ansible]", 57 | ] 58 | 59 | [build-system] 60 | requires = ["hatchling"] 61 | build-backend = "hatchling.build" 62 | 63 | [tool.uv] 64 | dev-dependencies = [ 65 | "mypy>=1.10.0", 66 | "pyright", 67 | "django>=4.0", 68 | "django-stubs>=5.0.0", 69 | "django-admin-data-views>=0.3.1", 70 | "django-pydantic-field>=0.3.9", 71 | "django-jsonform>=2.22.0", 72 | ] 73 | 74 | [tool.mypy] 75 | mypy_path = "abx_pkg" 76 | python_version = "3.10" 77 | warn_return_any = "True" 78 | warn_unused_configs = "True" 79 | plugins = [ 80 | "mypy_django_plugin.main", 81 | ] 82 | 83 | [tool.pyright] 84 | include = ["abx_pkg"] 85 | exclude = [ 86 | "**/node_modules", 87 | "**/__pycache__", 88 | "**/migrations", 89 | ] 90 | reportMissingImports = true 91 | reportMissingTypeStubs = false 92 | pythonVersion = "3.10" 93 | pythonPlatform = "Linux" 94 | 95 | 96 | [project.urls] 97 | Homepage = "https://github.com/ArchiveBox/abx-pkg" 98 | Source = "https://github.com/ArchiveBox/abx-pkg" 99 | Documentation = "https://github.com/ArchiveBox/abx-pkg" 100 | "Bug Tracker" = "https://github.com/ArchiveBox/abx-pkg/issues" 101 | Changelog = "https://github.com/ArchiveBox/abx-pkg/releases" 102 | Donate = "https://github.com/ArchiveBox/ArchiveBox/wiki/Donations" 103 | -------------------------------------------------------------------------------- /tests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | import sys 5 | import shutil 6 | import unittest 7 | import subprocess 8 | from io import StringIO 9 | from unittest import mock 10 | from pathlib import Path 11 | 12 | # from rich import print 13 | 14 | from abx_pkg import ( 15 | BinProvider, EnvProvider, Binary, SemVer, BinProviderOverrides, 16 | PipProvider, NpmProvider, AptProvider, BrewProvider, 17 | ) 18 | 19 | 20 | class TestSemVer(unittest.TestCase): 21 | 22 | def test_parsing(self): 23 | self.assertEqual(SemVer(None), None) 24 | self.assertEqual(SemVer(''), None) 25 | self.assertEqual(SemVer.parse(''), None) 26 | self.assertEqual(SemVer(1), (1, 0, 0)) 27 | self.assertEqual(SemVer(1, 2), (1, 2, 0)) 28 | self.assertEqual(SemVer('1.2+234234'), (1, 2, 234234)) 29 | self.assertEqual(SemVer('1.2+beta'), (1, 2, 0)) 30 | self.assertEqual(SemVer('1.2.4(1)+beta'), (1, 2, 4)) 31 | self.assertEqual(SemVer('1.2+beta(3)'), (1, 2, 3)) 32 | self.assertEqual(SemVer('1.2+6-be1ta(4)'), (1, 2, 6)) 33 | self.assertEqual(SemVer('1.2 curl(8)beta-4'), (1, 2, 0)) 34 | self.assertEqual(SemVer('1.2+curl(8)beta-4'), (1, 2, 8)) 35 | self.assertEqual(SemVer((1, 2, 3)), (1, 2, 3)) 36 | self.assertEqual(getattr(SemVer((1, 2, 3)), 'full_text'), '1.2.3') 37 | self.assertEqual(SemVer(('1', '2', '3')), (1, 2, 3)) 38 | self.assertEqual(SemVer.parse('5.6.7'), (5, 6, 7)) 39 | self.assertEqual(SemVer.parse('124.0.6367.208'), (124, 0, 6367)) 40 | self.assertEqual(SemVer.parse('Google Chrome 124.1+234.234'), (124, 1, 234)) 41 | self.assertEqual(SemVer.parse('Google Ch1rome 124.0.6367.208'), (124, 0, 6367)) 42 | self.assertEqual(SemVer.parse('Google Chrome 124.0.6367.208+beta_234. 234.234.123\n123.456.324'), (124, 0, 6367)) 43 | self.assertEqual(getattr(SemVer.parse('Google Chrome 124.0.6367.208+beta_234. 234.234.123\n123.456.324'), 'full_text'), 'Google Chrome 124.0.6367.208+beta_234. 234.234.123') 44 | self.assertEqual(SemVer.parse('Google Chrome'), None) 45 | 46 | 47 | class TestBinProvider(unittest.TestCase): 48 | 49 | def test_python_env(self): 50 | provider = EnvProvider() 51 | 52 | python_bin = provider.load('python') 53 | self.assertEqual(python_bin, provider.load_or_install('python')) 54 | 55 | self.assertEqual(python_bin.loaded_version, SemVer('{}.{}.{}'.format(*sys.version_info[:3]))) 56 | self.assertEqual(python_bin.loaded_abspath, Path(sys.executable).absolute()) 57 | self.assertEqual(python_bin.loaded_respath, Path(sys.executable).resolve()) 58 | self.assertTrue(python_bin.is_valid) 59 | self.assertTrue(python_bin.is_executable) 60 | self.assertFalse(python_bin.is_script) 61 | self.assertTrue(bool(str(python_bin))) # easy way to make sure serializing doesnt throw an error 62 | self.assertEqual(str(python_bin.loaded_binprovider.INSTALLER_BINARY.abspath), str(shutil.which('which'))) 63 | 64 | 65 | def test_bash_env(self): 66 | envprovider = EnvProvider() 67 | 68 | SYS_BASH_VERSION = subprocess.check_output('bash --version', shell=True, text=True).split('\n')[0] 69 | 70 | bash_bin = envprovider.load_or_install('bash') 71 | self.assertEqual(bash_bin.loaded_version, SemVer(SYS_BASH_VERSION)) 72 | self.assertGreater(bash_bin.loaded_version, SemVer('3.0.0')) 73 | self.assertEqual(bash_bin.loaded_abspath, Path(shutil.which('bash'))) 74 | self.assertTrue(bash_bin.is_valid) 75 | self.assertTrue(bash_bin.is_executable) 76 | self.assertFalse(bash_bin.is_script) 77 | self.assertTrue(bool(str(bash_bin))) # easy way to make sure serializing doesnt throw an error 78 | 79 | def test_overrides(self): 80 | 81 | class TestRecord: 82 | called_default_abspath_getter = False 83 | called_default_version_getter = False 84 | called_default_packages_getter = False 85 | called_custom_install_handler = False 86 | 87 | def custom_version_getter(): 88 | return '1.2.3' 89 | 90 | def custom_abspath_getter(self, bin_name, **context): 91 | assert self.__class__.__name__ == 'CustomProvider' 92 | return '/usr/bin/true' 93 | 94 | class CustomProvider(BinProvider): 95 | name: str = 'custom' 96 | 97 | overrides: BinProviderOverrides = { 98 | '*': { 99 | 'abspath': 'self.default_abspath_getter', # test staticmethod referenced via dotted notation on self. 100 | 'packages': 'self.default_packages_getter', # test classmethod referenced via dotted notation on self. 101 | 'version': 'self.default_version_getter', # test normal method referenced via dotted notation on self. 102 | 'install': None, # test intentionally nulled handler 103 | }, 104 | 'somebin': { 105 | 'abspath': custom_abspath_getter, # test external func that takes self, bin_name, and **context 106 | 'version': custom_version_getter, # test external func that takes no args 107 | 'packages': ['literal', 'return', 'value'], # test literal return value 108 | }, 109 | 'abc': { 110 | 'packages': 'self.alternate_packages_getter', # test classmethod that overrules default handler 111 | }, 112 | } 113 | 114 | @staticmethod 115 | def default_abspath_getter(): 116 | TestRecord.called_default_abspath_getter = True 117 | return '/bin/bash' 118 | 119 | @classmethod 120 | def default_packages_getter(cls, bin_name: str, **context): 121 | TestRecord.called_default_packages_getter = True 122 | return None 123 | 124 | def default_version_getter(self, bin_name: str, **context): 125 | TestRecord.called_default_version_getter = True 126 | return '999.999.999' 127 | 128 | 129 | @classmethod 130 | def alternate_packages_getter(cls, bin_name: str, **context): 131 | TestRecord.called_default_packages_getter = True 132 | return ['abc', 'def'] 133 | 134 | def on_install(self, bin_name: str, **context): 135 | raise NotImplementedError('whattt') 136 | 137 | provider = CustomProvider() 138 | provider._dry_run = True 139 | 140 | self.assertFalse(TestRecord.called_default_abspath_getter) 141 | self.assertFalse(TestRecord.called_default_version_getter) 142 | self.assertFalse(TestRecord.called_default_packages_getter) 143 | self.assertFalse(TestRecord.called_custom_install_handler) 144 | 145 | # test default abspath getter 146 | self.assertEqual(provider.get_abspath('doesnotexist'), Path('/bin/bash')) 147 | self.assertTrue(TestRecord.called_default_abspath_getter) 148 | 149 | # test custom abspath getter 150 | self.assertEqual(provider.get_abspath('somebin'), Path('/usr/bin/true')) # test that Callable getter that takes self, bin_name, **context works + result is auto-cast to Path 151 | 152 | # test default version getter 153 | self.assertEqual(provider.get_version('doesnotexist'), SemVer('999.999.999')) # test that normal 'self.some_method' dot referenced getter works and result is auto-cast to SemVer 154 | self.assertTrue(TestRecord.called_default_version_getter) 155 | 156 | # test custom version getter 157 | self.assertEqual(provider.get_version('somebin'), SemVer('1.2.3')) # test that remote Callable func getter that takes no args works and str result is auto-cast to SemVer 158 | 159 | # test default packages getter 160 | self.assertEqual(provider.get_packages('doesnotexist'), ('doesnotexist',)) # test that it fallsback to [bin_name] by default if getter returns None 161 | self.assertTrue(TestRecord.called_default_packages_getter) 162 | self.assertEqual(provider.get_packages('abc'), ('abc', 'def')) # test that classmethod getter funcs work 163 | 164 | # test custom packages getter 165 | self.assertEqual(provider.get_packages('somebin'), ('literal', 'return', 'value')) # test that literal return values in overrides work 166 | 167 | # test install handler 168 | exc = None 169 | try: 170 | provider.install('doesnotexist') 171 | except Exception as err: 172 | exc = err 173 | self.assertIsInstance(exc, AssertionError) 174 | self.assertTrue('BinProvider(name=custom) has no install handler implemented for Binary(name=doesnotexist)' in str(exc)) 175 | 176 | 177 | class TestBinary(unittest.TestCase): 178 | 179 | def test_python_bin(self): 180 | envprovider = EnvProvider() 181 | 182 | python_bin = Binary(name='python', binproviders=[envprovider]) 183 | 184 | self.assertIsNone(python_bin.loaded_binprovider) 185 | self.assertIsNone(python_bin.loaded_abspath) 186 | self.assertIsNone(python_bin.loaded_version) 187 | 188 | python_bin = python_bin.load() 189 | 190 | shallow_bin = envprovider.load_or_install('python') 191 | assert shallow_bin and python_bin.loaded_binprovider 192 | self.assertEqual(python_bin.loaded_binprovider, shallow_bin.loaded_binprovider) 193 | self.assertEqual(python_bin.loaded_abspath, shallow_bin.loaded_abspath) 194 | self.assertEqual(python_bin.loaded_version, shallow_bin.loaded_version) 195 | self.assertEqual(python_bin.loaded_sha256, shallow_bin.loaded_sha256) 196 | 197 | self.assertEqual(python_bin.loaded_version, SemVer('{}.{}.{}'.format(*sys.version_info[:3]))) 198 | self.assertEqual(python_bin.loaded_abspath, Path(sys.executable).absolute()) 199 | self.assertEqual(python_bin.loaded_respath, Path(sys.executable).resolve()) 200 | self.assertTrue(python_bin.is_valid) 201 | self.assertTrue(python_bin.is_executable) 202 | self.assertFalse(python_bin.is_script) 203 | self.assertTrue(bool(str(python_bin))) # easy way to make sure serializing doesnt throw an error 204 | 205 | 206 | def flatten(xss): 207 | return [x for xs in xss for x in xs] 208 | 209 | class InstallTest(unittest.TestCase): 210 | 211 | def install_with_binprovider(self, provider, binary): 212 | 213 | binary_bin = binary.load_or_install() 214 | provider_bin = provider.load_or_install(bin_name=binary.name) 215 | # print(binary_bin, binary_bin.bin_dir, binary_bin.loaded_abspath) 216 | # print('\n'.join(f'{provider}={path}' for provider, path in binary.loaded_abspaths.items()), '\n') 217 | # print() 218 | try: 219 | self.assertEqual(binary_bin.loaded_binprovider, provider_bin.loaded_binprovider) 220 | except AssertionError: 221 | print('binary_bin', dict(binary_bin.loaded_binprovider)) 222 | print('provider_bin', dict(provider_bin.loaded_binprovider)) 223 | raise 224 | self.assertEqual(binary_bin.loaded_abspath, provider_bin.loaded_abspath) 225 | self.assertEqual(binary_bin.loaded_version, provider_bin.loaded_version) 226 | self.assertEqual(binary_bin.loaded_sha256, provider_bin.loaded_sha256) 227 | 228 | self.assertIn(binary_bin.loaded_abspath, flatten(binary_bin.loaded_abspaths.values())) 229 | self.assertIn(str(binary_bin.bin_dir), flatten(PATH.split(':') for PATH in binary_bin.loaded_bin_dirs.values())) 230 | 231 | PATH = provider.PATH 232 | bin_abspath = shutil.which(binary.name, path=PATH) 233 | assert bin_abspath, f'Could not find {binary.name} in PATH={PATH}' 234 | VERSION = SemVer.parse(subprocess.check_output(f'{bin_abspath} --version', shell=True, text=True)) 235 | ABSPATH = Path(bin_abspath).absolute().resolve() 236 | 237 | self.assertEqual(binary_bin.loaded_version, VERSION) 238 | self.assertIn(binary_bin.loaded_abspath, provider.get_abspaths(binary_bin.name)) 239 | self.assertEqual(binary_bin.loaded_respath, ABSPATH) 240 | self.assertTrue(binary_bin.is_valid) 241 | self.assertTrue(binary_bin.is_executable) 242 | self.assertFalse(binary_bin.is_script) 243 | self.assertTrue(bool(str(binary_bin))) # easy way to make sure serializing doesnt throw an error 244 | # print(provider.PATH) 245 | # print() 246 | # print() 247 | # print(binary.bin_filename, binary.bin_dir, binary.loaded_abspaths) 248 | # print() 249 | # print() 250 | # print(provider.name, 'PATH=', provider.PATH, 'ABSPATHS=', provider.get_abspaths(bin_name=binary_bin.name)) 251 | return provider_bin 252 | 253 | def test_env_provider(self): 254 | provider = EnvProvider() 255 | binary = Binary(name='wget', binproviders=[provider]).load() 256 | self.install_with_binprovider(provider, binary) 257 | 258 | def test_pip_provider(self): 259 | # pipprovider = PipProvider() 260 | pipprovider = PipProvider(pip_venv=os.environ.get('VIRTUAL_ENV', None)) 261 | # print('PIP BINPROVIDER', pipprovider.INSTALLER_BIN_ABSPATH, 'PATH=', pipprovider.PATH) 262 | binary = Binary(name='yt-dlp', binproviders=[pipprovider]) 263 | self.install_with_binprovider(pipprovider, binary) 264 | 265 | def test_npm_provider(self): 266 | npmprovider = NpmProvider() 267 | # print(provider.PATH) 268 | binary = Binary(name='tsx', binproviders=[npmprovider]) 269 | self.install_with_binprovider(npmprovider, binary) 270 | 271 | @mock.patch("sys.stderr") 272 | @mock.patch("subprocess.run", return_value=subprocess.CompletedProcess(args=[], returncode=0, stdout='', stderr='')) 273 | def test_dry_run_doesnt_exec(self, mock_run, _mock_stderr): 274 | pipprovider = PipProvider().get_provider_with_overrides(dry_run=True) 275 | pipprovider.install(bin_name='doesnotexist') 276 | mock_run.assert_not_called() 277 | 278 | @mock.patch("sys.stderr", new_callable=StringIO) 279 | def test_dry_run_prints_stderr(self, mock_stderr): 280 | pipprovider = PipProvider() 281 | binary = Binary(name='doesnotexist', binproviders=[pipprovider]) 282 | binary.install(dry_run=True) 283 | 284 | self.assertIn('DRY RUN', mock_stderr.getvalue()) 285 | 286 | def test_brew_provider(self): 287 | # print(provider.PATH) 288 | os.environ['HOMEBREW_NO_AUTO_UPDATE'] = 'True' 289 | os.environ['HOMEBREW_NO_INSTALL_CLEANUP'] = 'True' 290 | os.environ['HOMEBREW_NO_ENV_HINTS'] = 'True' 291 | 292 | is_on_windows = sys.platform.lower().startswith('win') or os.name == 'nt' 293 | is_on_macos = 'darwin' in sys.platform.lower() 294 | is_on_linux = 'linux' in sys.platform.lower() 295 | has_brew = shutil.which('brew') 296 | # has_apt = shutil.which('dpkg') is not None 297 | 298 | provider = BrewProvider() 299 | if has_brew: 300 | self.assertTrue(provider.PATH) 301 | self.assertTrue(provider.is_valid) 302 | else: 303 | # print('SHOULD NOT HAVE BREW, but got', provider.INSTALLER_BIN_ABSPATH, 'PATH=', provider.PATH) 304 | self.assertFalse(provider.is_valid) 305 | 306 | exception = None 307 | result = None 308 | try: 309 | binary = Binary(name='wget', binproviders=[provider]) 310 | result = self.install_with_binprovider(provider, binary) 311 | except Exception as err: 312 | exception = err 313 | 314 | 315 | if is_on_macos or (is_on_linux and has_brew): 316 | self.assertTrue(has_brew) 317 | if exception: 318 | raise exception 319 | self.assertIsNone(exception) 320 | self.assertTrue(result) 321 | elif is_on_windows or (is_on_linux and not has_brew): 322 | self.assertFalse(has_brew) 323 | self.assertIsInstance(exception, Exception) 324 | self.assertFalse(result) 325 | else: 326 | raise exception 327 | 328 | 329 | def test_apt_provider(self): 330 | is_on_windows = sys.platform.startswith('win') or os.name == 'nt' 331 | is_on_macos = 'darwin' in sys.platform 332 | is_on_linux = 'linux' in sys.platform 333 | # has_brew = shutil.which('brew') is not None 334 | has_apt = shutil.which('apt-get') is not None 335 | 336 | 337 | exception = None 338 | result = None 339 | provider = AptProvider() 340 | if has_apt: 341 | self.assertTrue(provider.PATH) 342 | else: 343 | self.assertFalse(provider.PATH) 344 | try: 345 | # print(provider.PATH) 346 | binary = Binary(name='wget', binproviders=[provider]) 347 | result = self.install_with_binprovider(provider, binary) 348 | except Exception as err: 349 | exception = err 350 | 351 | 352 | if is_on_linux: 353 | self.assertTrue(has_apt) 354 | if exception: 355 | raise exception 356 | self.assertIsNone(exception) 357 | self.assertTrue(result) 358 | elif is_on_windows or is_on_macos: 359 | self.assertFalse(has_apt) 360 | self.assertIsInstance(exception, Exception) 361 | self.assertFalse(result) 362 | else: 363 | raise exception 364 | 365 | 366 | if __name__ == '__main__': 367 | unittest.main() 368 | --------------------------------------------------------------------------------