├── .github └── workflows │ └── python.yml ├── .gitignore ├── .mypy.ini ├── AUTHORS ├── LICENSE ├── MANIFEST.in ├── NEWS ├── README.md ├── RELEASE ├── autoformat.sh ├── data ├── services │ ├── debspawn-clear-caches.service │ └── debspawn-clear-caches.timer └── tmpfiles.d │ └── debspawn.conf ├── debspawn.py ├── debspawn ├── __init__.py ├── aptcache.py ├── build.py ├── cli.py ├── config.py ├── dsrun ├── injectpkg.py ├── maintain.py ├── nspawn.py ├── osbase.py └── utils │ ├── __init__.py │ ├── command.py │ ├── env.py │ ├── log.py │ ├── misc.py │ └── zstd_tar.py ├── docs ├── __init__.py ├── assemble_man.py ├── debspawn-build.1.xml ├── debspawn-create.1.xml ├── debspawn-delete.1.xml ├── debspawn-list.1.xml ├── debspawn-login.1.xml ├── debspawn-maintain.1.xml ├── debspawn-run.1.xml ├── debspawn-update.1.xml └── debspawn.1.xml ├── install-sysdata.py ├── lint.sh ├── pyproject.toml ├── setup.cfg ├── setup.py └── tests ├── __init__.py ├── conftest.py ├── test_cud.py └── test_utils.py /.github/workflows/python.yml: -------------------------------------------------------------------------------- 1 | name: Build & Test 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | build: 11 | 12 | runs-on: ubuntu-latest 13 | strategy: 14 | matrix: 15 | python-version: [ '3.11', 16 | '3.12', 17 | '3.13' ] 18 | 19 | name: Python ${{ matrix.python-version }} 20 | steps: 21 | - uses: actions/checkout@v3 22 | - name: Set up Python ${{ matrix.python-version }} 23 | uses: actions/setup-python@v3 24 | with: 25 | python-version: ${{ matrix.python-version }} 26 | 27 | - name: Update cache 28 | run: sudo apt-get update -qq 29 | 30 | - name: Install system prerequisites 31 | run: sudo apt-get install -yq 32 | xsltproc 33 | docbook-xsl 34 | docbook-xml 35 | zstd 36 | systemd 37 | systemd-container 38 | debootstrap 39 | 40 | - name: Upgrading pip 41 | run: python -m pip install --upgrade pip 42 | 43 | - name: Install dependencies 44 | run: python -m pip install 45 | setuptools 46 | tomlkit 47 | pkgconfig 48 | flake8 49 | pytest 50 | pylint 51 | mypy 52 | isort 53 | black 54 | 55 | - name: Build & Install 56 | run: | 57 | ./setup.py build 58 | ./setup.py install --root=/tmp 59 | rm -rf build/ 60 | 61 | - name: Test 62 | run: | 63 | sudo $(which python3) -m pytest 64 | rm -rf build/ 65 | 66 | - name: Lint (flake8) 67 | run: | 68 | python -m flake8 ./ --statistics 69 | python -m flake8 debspawn/dsrun --statistics 70 | 71 | - name: Lint (pylint) 72 | run: | 73 | python -m pylint -f colorized ./debspawn 74 | python -m pylint -f colorized ./debspawn/dsrun 75 | python -m pylint -f colorized ./tests ./data 76 | python -m pylint -f colorized setup.py install-sysdata.py 77 | 78 | - name: Lint (mypy) 79 | run: | 80 | python -m mypy --install-types --non-interactive . 81 | python -m mypy ./debspawn/dsrun 82 | 83 | - name: Lint (isort) 84 | run: isort --diff . 85 | 86 | - name: Lint (black) 87 | run: black --diff . 88 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | -------------------------------------------------------------------------------- /.mypy.ini: -------------------------------------------------------------------------------- 1 | [mypy] 2 | show_column_numbers = True 3 | pretty = True 4 | 5 | strict_optional = False 6 | ignore_missing_imports = True 7 | 8 | warn_redundant_casts = True 9 | warn_unused_ignores = True 10 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Matthias Klumpp 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU LESSER GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | 9 | This version of the GNU Lesser General Public License incorporates 10 | the terms and conditions of version 3 of the GNU General Public 11 | License, supplemented by the additional permissions listed below. 12 | 13 | 0. Additional Definitions. 14 | 15 | As used herein, "this License" refers to version 3 of the GNU Lesser 16 | General Public License, and the "GNU GPL" refers to version 3 of the GNU 17 | General Public License. 18 | 19 | "The Library" refers to a covered work governed by this License, 20 | other than an Application or a Combined Work as defined below. 21 | 22 | An "Application" is any work that makes use of an interface provided 23 | by the Library, but which is not otherwise based on the Library. 24 | Defining a subclass of a class defined by the Library is deemed a mode 25 | of using an interface provided by the Library. 26 | 27 | A "Combined Work" is a work produced by combining or linking an 28 | Application with the Library. The particular version of the Library 29 | with which the Combined Work was made is also called the "Linked 30 | Version". 31 | 32 | The "Minimal Corresponding Source" for a Combined Work means the 33 | Corresponding Source for the Combined Work, excluding any source code 34 | for portions of the Combined Work that, considered in isolation, are 35 | based on the Application, and not on the Linked Version. 36 | 37 | The "Corresponding Application Code" for a Combined Work means the 38 | object code and/or source code for the Application, including any data 39 | and utility programs needed for reproducing the Combined Work from the 40 | Application, but excluding the System Libraries of the Combined Work. 41 | 42 | 1. Exception to Section 3 of the GNU GPL. 43 | 44 | You may convey a covered work under sections 3 and 4 of this License 45 | without being bound by section 3 of the GNU GPL. 46 | 47 | 2. Conveying Modified Versions. 48 | 49 | If you modify a copy of the Library, and, in your modifications, a 50 | facility refers to a function or data to be supplied by an Application 51 | that uses the facility (other than as an argument passed when the 52 | facility is invoked), then you may convey a copy of the modified 53 | version: 54 | 55 | a) under this License, provided that you make a good faith effort to 56 | ensure that, in the event an Application does not supply the 57 | function or data, the facility still operates, and performs 58 | whatever part of its purpose remains meaningful, or 59 | 60 | b) under the GNU GPL, with none of the additional permissions of 61 | this License applicable to that copy. 62 | 63 | 3. Object Code Incorporating Material from Library Header Files. 64 | 65 | The object code form of an Application may incorporate material from 66 | a header file that is part of the Library. You may convey such object 67 | code under terms of your choice, provided that, if the incorporated 68 | material is not limited to numerical parameters, data structure 69 | layouts and accessors, or small macros, inline functions and templates 70 | (ten or fewer lines in length), you do both of the following: 71 | 72 | a) Give prominent notice with each copy of the object code that the 73 | Library is used in it and that the Library and its use are 74 | covered by this License. 75 | 76 | b) Accompany the object code with a copy of the GNU GPL and this license 77 | document. 78 | 79 | 4. Combined Works. 80 | 81 | You may convey a Combined Work under terms of your choice that, 82 | taken together, effectively do not restrict modification of the 83 | portions of the Library contained in the Combined Work and reverse 84 | engineering for debugging such modifications, if you also do each of 85 | the following: 86 | 87 | a) Give prominent notice with each copy of the Combined Work that 88 | the Library is used in it and that the Library and its use are 89 | covered by this License. 90 | 91 | b) Accompany the Combined Work with a copy of the GNU GPL and this license 92 | document. 93 | 94 | c) For a Combined Work that displays copyright notices during 95 | execution, include the copyright notice for the Library among 96 | these notices, as well as a reference directing the user to the 97 | copies of the GNU GPL and this license document. 98 | 99 | d) Do one of the following: 100 | 101 | 0) Convey the Minimal Corresponding Source under the terms of this 102 | License, and the Corresponding Application Code in a form 103 | suitable for, and under terms that permit, the user to 104 | recombine or relink the Application with a modified version of 105 | the Linked Version to produce a modified Combined Work, in the 106 | manner specified by section 6 of the GNU GPL for conveying 107 | Corresponding Source. 108 | 109 | 1) Use a suitable shared library mechanism for linking with the 110 | Library. A suitable mechanism is one that (a) uses at run time 111 | a copy of the Library already present on the user's computer 112 | system, and (b) will operate properly with a modified version 113 | of the Library that is interface-compatible with the Linked 114 | Version. 115 | 116 | e) Provide Installation Information, but only if you would otherwise 117 | be required to provide such information under section 6 of the 118 | GNU GPL, and only to the extent that such information is 119 | necessary to install and execute a modified version of the 120 | Combined Work produced by recombining or relinking the 121 | Application with a modified version of the Linked Version. (If 122 | you use option 4d0, the Installation Information must accompany 123 | the Minimal Corresponding Source and Corresponding Application 124 | Code. If you use option 4d1, you must provide the Installation 125 | Information in the manner specified by section 6 of the GNU GPL 126 | for conveying Corresponding Source.) 127 | 128 | 5. Combined Libraries. 129 | 130 | You may place library facilities that are a work based on the 131 | Library side by side in a single library together with other library 132 | facilities that are not Applications and are not covered by this 133 | License, and convey such a combined library under terms of your 134 | choice, if you do both of the following: 135 | 136 | a) Accompany the combined library with a copy of the same work based 137 | on the Library, uncombined with any other library facilities, 138 | conveyed under the terms of this License. 139 | 140 | b) Give prominent notice with the combined library that part of it 141 | is a work based on the Library, and explaining where to find the 142 | accompanying uncombined form of the same work. 143 | 144 | 6. Revised Versions of the GNU Lesser General Public License. 145 | 146 | The Free Software Foundation may publish revised and/or new versions 147 | of the GNU Lesser General Public License from time to time. Such new 148 | versions will be similar in spirit to the present version, but may 149 | differ in detail to address new problems or concerns. 150 | 151 | Each version is given a distinguishing version number. If the 152 | Library as you received it specifies that a certain numbered version 153 | of the GNU Lesser General Public License "or any later version" 154 | applies to it, you have the option of following the terms and 155 | conditions either of that published version or of any later version 156 | published by the Free Software Foundation. If the Library as you 157 | received it does not specify a version number of the GNU Lesser 158 | General Public License, you may choose any version of the GNU Lesser 159 | General Public License ever published by the Free Software Foundation. 160 | 161 | If the Library as you received it specifies that a proxy can decide 162 | whether future versions of the GNU Lesser General Public License shall 163 | apply, that proxy's public statement of acceptance of any version is 164 | permanent authorization for you to choose that version for the 165 | Library. 166 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include debspawn/dsrun 2 | include install-sysdata.py 3 | include docs/* 4 | include data/services/* 5 | include data/tmpfiles.d/* 6 | -------------------------------------------------------------------------------- /NEWS: -------------------------------------------------------------------------------- 1 | Version 0.6.4 2 | ~~~~~~~~~~~~~ 3 | Released: 2024-02-20 4 | 5 | Bugfixes: 6 | * Ensure nspawn never tries to bindmount /etc/localtime 7 | * Fix Lintian check permission error 8 | 9 | Version 0.6.3 10 | ~~~~~~~~~~~~~ 11 | Released: 2023-12-22 12 | 13 | Bugfixes: 14 | * Ensure containers have a dummy /etc/hosts file (Matthias Klumpp) 15 | * Work around nspawn wanting to ensure its user-home exists (Matthias Klumpp) 16 | * dsrun: Use eatmydata a bit more sparingly (Matthias Klumpp) 17 | * Ensure passwd is available in the build environment (Matthias Klumpp) 18 | * Ensure locally injected packages are always preferred (Matthias Klumpp) 19 | * Fix debsign error on source only builds (Maykel Moya) 20 | 21 | Version 0.6.2 22 | ~~~~~~~~~~~~~ 23 | Released: 2023-05-28 24 | 25 | Features: 26 | * Support Python 3.11 27 | 28 | Bugfixes: 29 | * Allow APT more choices when installing build-deps from partial suites 30 | * Set BaseSuite for image recreation if image has a custom name 31 | * Make APT consider all suites equally for dependency resolution 32 | 33 | Version 0.6.1 34 | ~~~~~~~~~~~~~ 35 | Released: 2023-01-02 36 | 37 | Features: 38 | * Make container APT configuration more sbuild-like 39 | 40 | Bugfixes: 41 | * docs: Update positional argument info in manual pages 42 | * Use useradd instead of adduser 43 | * Don't force a suite when packages are injected 44 | 45 | Version 0.6.0 46 | ~~~~~~~~~~~~~ 47 | Released: 2022-10-02 48 | 49 | Features: 50 | * Allow containers that run a custom command to be booted as well 51 | * Add configuration option to disable package caching 52 | * Add configuration option to set a different bootstrap tool 53 | * Make artifact storage in interactive mode an explicit action 54 | 55 | Bugfixes: 56 | * Fix pyproject.toml license/author fields 57 | * Don't use deprecated Lintian argument 58 | * Use tomlkit instead of deprecated toml 59 | 60 | Version 0.5.2 61 | ~~~~~~~~~~~~~ 62 | Released: 2022-02-22 63 | 64 | Features: 65 | * Format source code with Black 66 | * Allow to boot a container for interactive logins 67 | 68 | Bugfixes: 69 | * Set suite explicitly when resolving build-deps 70 | * Do not include APT package caches in tarballs 71 | 72 | Version 0.5.1 73 | ~~~~~~~~~~~~~ 74 | Released: 2021-11-08 75 | 76 | Notes: 77 | * This release changes the default bootstrap variant to `buildd`, which may result 78 | in users needing to pass `--variant=none` to build with existing images, or change 79 | the default via Debspawn's global settings. 80 | * The image name and suite name have been decoupled, so users can now give images 81 | arbitrary names and create multiple ones for different purposes. 82 | 83 | Features: 84 | * Allow custom container image names, decoupling them from being suite-based 85 | * Propagate proxy settings through to APT, debootstrap and nspawn 86 | * Default to the 'buildd' bootstrap variant 87 | * Make update-all command work with custom image names 88 | * Add global config option for default bootstrap variant 89 | 90 | Bugfixes: 91 | * Give access to /boot as well if read-kmods is passed 92 | * run: Copy build directory by default, instead of bindmounting it 93 | * run: Retrieve artifacts the same way as regular build artifacts 94 | * Unmount any bindmounds when cleaning up temporary directories 95 | * man: Document the SyscallFilter config option 96 | * man: Clarify new image name / suite relations in ds-create manual page 97 | 98 | Version 0.5.0 99 | ~~~~~~~~~~~~~ 100 | Released: 2021-06-04 101 | 102 | Features: 103 | * First release also available on PyPI! 104 | * maintain: Add new flag to print status information 105 | * maintain: status: Include debootstrap version in reports 106 | * docs: Document the `maintain` subcommand 107 | * Install systemd timer to clear all caches monthly 108 | * Unconditionally save buildlog 109 | 110 | Bugfixes: 111 | * Rework how external system files are installed 112 | * Include extra data in manifest as well 113 | * Fix image creation if resolv.conf is a symlink 114 | 115 | Version 0.4.2 116 | ~~~~~~~~~~~~~ 117 | Released: 2021-05-24 118 | 119 | Features: 120 | * Add "maintain" subcommand to migrate or reset settings & state 121 | * Configure APT to not install recommends by default (deb: #987312) 122 | * Retry apt updates a few times to protect against bad mirrors 123 | * Add tmpfiles.d snippet to manage debspawn's temporary directory 124 | * Allow defining custom environment variables for package builds (deb: #986967) 125 | * Add maintenance action to update all images 126 | 127 | Bugfixes: 128 | * Interpret EOF as "No" in interactive override question 129 | * Implement privileged device access properly 130 | * Move images to the right default location 131 | * Don't try to bindmound KVM if it doesn't exist 132 | * Use dpkg --print-architecture to determine arch (deb: #987547) 133 | * run: Mount builddir in initialization step 134 | * Don't register any of our nspawn containers by default 135 | * Check system encoding properly (deb: #982793) 136 | * Atomically and safely copy files into unsafe environments 137 | * Run builds as user with a random free UID (deb: #989049) 138 | 139 | Contributors: 140 | Helmut Grohne, Matthias Klumpp 141 | 142 | Version 0.4.1 143 | ~~~~~~~~~~~~~ 144 | Released: 2020-12-22 145 | 146 | Features: 147 | * README, debspawn.1: document config file (Gordon Ball) 148 | * Install lintian after build (Harlan Lieberman-Berg) 149 | * Allow custom scripts to cache their prepared images for faster builds (Matthias Klumpp) 150 | * Allow running fully privileged containers (Matthias Klumpp) 151 | * Make global config file use TOML, update documentation (Matthias Klumpp) 152 | 153 | Bugfixes: 154 | * Pass --console nspawn flag only if our systemd version is high enough (Matthias Klumpp) 155 | * Enforce the suite name of the env we built in for changes files (Matthias Klumpp) 156 | * Add extra suites to sources even if base suite is equal to image suite (Matthias Klumpp) 157 | * Have nspawn recreate container machine-id each time (Matthias Klumpp) 158 | * cli: Safeguard against cases where we have flags but no subcommands (Matthias Klumpp) 159 | * Disable syscall filter for some groups by default (Matthias Klumpp) 160 | 161 | Version 0.4.0 162 | ~~~~~~~~~~~~~ 163 | Released: 2020-01-20 164 | 165 | Features: 166 | * Implement an interactive build mode 167 | * Store a copy of the build log by default 168 | * Allow copying back changes in interactive mode 169 | * Use a bit of color in errors and warnings, if possible 170 | * Update manual pages 171 | * Permit recreation of images, instead of just updating them 172 | 173 | Bugfixes: 174 | * Move dsrun helper into the package itself 175 | * Drop some unwanted files from /dev before creating OS tarballs 176 | * Remove d/files file if it's created by Debspawn pre-build 177 | * Interactive mode and build logs are mutually exclusive for now 178 | * Add MANIFEST file 179 | 180 | Version 0.3.0 181 | ~~~~~~~~~~~~~ 182 | Released: 2020-01-06 183 | 184 | Features: 185 | * Allow to override temporary directory path explicitly in config 186 | * Allow full sources.list customization at image creation time 187 | * Add initial test infrastructure 188 | * Allow 'b' shorthand for the 'build' subparser (Mo Zhou) 189 | * Allow turning on d/rules clean on the host, disable it by default 190 | * Allow selected environment variables to survive auto-sudo 191 | * Implement way to run Lintian as part of the build 192 | * Print pretty error message if configuration JSON is broken 193 | * Prefer hardlinks over copies when creating the APT package cache 194 | * Implement support for injecting packages 195 | * docs: Add a note about how to inject packages 196 | * Only install minimal Python in containers 197 | * Harmonize project name (= Debspawn spelling everywhere) 198 | * Add command to list installed container image details 199 | * Update sbuild replacement note 200 | 201 | Bugfixes: 202 | * Ensure we have absolute paths for debspawn run 203 | * Don't fail running command without build/artifacts directory 204 | * Build packages with epochs correctly when building from source-dir 205 | * Sign packages with an epoch correctly 206 | * Change HOME when dropping privileges 207 | * Don't install arch-indep build-deps on arch-only builds 208 | * Shorten nspawn machine name when hostname is exceptionally long 209 | * tests: Test container updates 210 | * Ensure all data lands in its intended directories when installing 211 | 212 | Version 0.2.1 213 | ~~~~~~~~~~~~~ 214 | Released: 2019-01-10 215 | 216 | Features: 217 | * Allow giving the container extra capabilities easily for custom commands 218 | * Allow giving the container permission to access the host's /dev 219 | * Allow creating an image with a suite and base-suite 220 | 221 | Version 0.2.0 222 | ~~~~~~~~~~~~~ 223 | Released: 2018-08-28 224 | 225 | Features: 226 | * Allow specifying enabled archive components at image creation time 227 | * Support printing the program version to stdout 228 | * Allow diverting the maintainer address 229 | * Prepare container for arbitrary run action similarly to package build 230 | * Support more build-only choices 231 | * Print some basic system info to the log 232 | * Log some basic disk space stats before/after build 233 | 234 | Bugfixes: 235 | * random.choices is only available since Python 3.6, replace it 236 | * Enforce dsrun to be installed in a location were we can find it 237 | * Ensure we don't try to link journals 238 | * Force new configuration by default, not old one 239 | * Set environment shell 240 | 241 | Version 0.1.0 242 | ~~~~~~~~~~~~~ 243 | Released: 2018-08-20 244 | 245 | Notes: 246 | * Initial release 247 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Debspawn 2 | 3 | [![Build & Test](https://github.com/lkhq/debspawn/actions/workflows/python.yml/badge.svg)](https://github.com/lkhq/debspawn/actions/workflows/python.yml) 4 | 5 | Debspawn is a tool to build Debian packages in an isolated environment. Unlike similar tools like `sbuild` 6 | or `pbuilder`, `debspawn` uses `systemd-nspawn` instead of plain chroots to manage the isolated environment. 7 | This allows Debspawn to isolate builds from the host system much more via containers. It also allows 8 | for more advanced features to manage builds, for example setting resource limits for individual builds. 9 | 10 | Please keep in mind that Debspawn is *not* a security feature! While it provides a lot of isolation from the 11 | host system, you should not run arbitrary untrusted code with it. The usual warnings for all container technology 12 | apply here. 13 | 14 | Debspawn also allows one to run arbitrary custom commands in its environment. This is used by the Laniakea[1] Spark workers 15 | to execute a variety of non-package builds and QA actions in the same environment in which we usually build packages. 16 | 17 | Debspawn was created to be simple to use in automation as well as by humans. It should both be easily usable on large 18 | build farms with good integration with a job runner, as well as on a personal workstation by a human user (to reproduce 19 | builds done elsewhere, or to develop a Debian package). 20 | Due to that, the most common operations are as easily accessible as possible and should require zero configuration 21 | by default. Additionally, `debspawn` will always try to do the right thing automatically before resorting to a flag 22 | that the user has to set. 23 | Options which change the build environment are - with one exception - not made available intentionally, so 24 | achieving reproducible builds is easier. 25 | See the FAQ below for more details. 26 | 27 | [1]: https://github.com/lkhq/laniakea 28 | 29 | ## Usage 30 | 31 | ### Installing Debspawn 32 | 33 | #### Via the Debian package 34 | 35 | On Debian/Ubuntu, simply run 36 | ```bash 37 | sudo apt install debspawn 38 | ``` 39 | to start using Debspawn. 40 | 41 | #### Via PyPI 42 | 43 | > **⚠ WARNING: Careful when installing via PyPI!** 44 | > While we do ship `debspawn` on PyPI, installing it via `pip` will not install certain system services 45 | > to automate cache cleanup and temp data cleanup. In addition to that, all manual pages will be missing. 46 | > This is due to intentional limitations of Python packages installed via pip. 47 | 48 | If you want to install Debspawn via PyPI anyway, you can use `pip install debspawn`. 49 | You can decide to install the system data files manually later by running the `install-sysdata.py` script from the 50 | Git repository and adjusting the `debspawn` binary path in the installed systemd units, if you want to. 51 | 52 | #### Via the Git repository 53 | 54 | Clone the Git repository, install the (build and runtime) dependencies of `debspawn`: 55 | ```bash 56 | sudo apt install xsltproc docbook-xsl python3-setuptools zstd systemd-container debootstrap 57 | ``` 58 | 59 | You can the run `debspawn.py` directly from the Git repository, or choose to install it: 60 | ```bash 61 | sudo python setup.py install --root=/ 62 | ``` 63 | 64 | Debspawn requires at least Python 3.9 on the host system, and Python 3.5 in the container. 65 | We try to keep the dependency footprint of this tool as small as possible, so it is not planned to raise that 66 | requirement or add any more dependencies anytime soon (especially not for the minimum Python version used in a container). 67 | 68 | ### On superuser permission 69 | 70 | If `sudo` is available on the system, `debspawn` will automatically request root permission 71 | when it needs it, there is no need to run it as root explicitly. 72 | If it can not obtain privileges, `debspawn` will exit with the appropriate error message. 73 | 74 | ### Creating a new image 75 | 76 | You can easily create images for any suite that has a script in `debootstrap`. For Debian Unstable for example: 77 | ```bash 78 | $ debspawn create sid 79 | ``` 80 | This will create a Debian Sid (unstable) image for the current system architecture. 81 | 82 | To create an image for testing Ubuntu builds: 83 | ```bash 84 | $ debspawn create --arch=i386 cosmic 85 | ``` 86 | This creates an `i386` image for Ubuntu 18.10. If you want to use a different mirror than set by default, pass it 87 | with the `--mirror` option. 88 | 89 | ### Refreshing an image 90 | 91 | Just run `debspawn update` and give the details of the base image that should be updated: 92 | ```bash 93 | $ debspawn update sid 94 | $ debspawn update --arch=i386 cosmic 95 | ``` 96 | 97 | This will update the base image contents and perform other maintenance actions. 98 | 99 | ### Building a package 100 | 101 | You can build a package from its source directory, or just by passing a plain `.dsc` file to `debspawn`. If the result 102 | should be automatically signed, the `--sign` flag needs to be passed too: 103 | ```bash 104 | $ cd ~/packages/hello 105 | $ debspawn build sid --sign 106 | 107 | $ debspawn build --arch=i386 cosmic ./hello_2.10-1.dsc 108 | ``` 109 | 110 | Build results are by default returned in `/var/lib/debspawn/results/`. 111 | 112 | If you need to inject other local packages as build dependencies, place `deb` files in `/var/lib/debspawn/injected-pkgs` 113 | (or other location set in the config file). 114 | 115 | ### Building a package - with git-buildpackage 116 | 117 | You can use a command like this to build your project with gbp and Debspawn: 118 | ```bash 119 | $ gbp buildpackage --git-builder='debspawn build sid --sign' 120 | ``` 121 | 122 | You might also want to add `--results-dir=..` to the debspawn arguments to get the resulting artifacts in the directory 123 | to which the package repository was originally exported. 124 | 125 | ### Manual interactive-shell action 126 | 127 | If you want to, you can log into the container environment and either play around in 128 | ephemeral mode with no persistent changes, or pass `--persistent` to `debspawn` so all changes are permanently saved: 129 | ```bash 130 | $ debspawn login sid 131 | 132 | # Attention! This may alter the build environment! 133 | $ debspawn login --persistent sid 134 | ``` 135 | 136 | ### Deleting a container image 137 | 138 | At some point, you may want to permanently remove a container image again, for example because the 139 | release it was built for went end of life. 140 | This is easily done as well: 141 | ```bash 142 | $ debspawn delete sid 143 | $ debspawn delete --arch=i386 cosmic 144 | ``` 145 | 146 | ### Running arbitrary commands 147 | 148 | This is achieved with the `debspawn run` command and is a bit more involved. Refer to its manual page 149 | and help output for more information on how to use it: `man debspawn run`. 150 | 151 | ### Global configuration 152 | 153 | Debspawn will read a global configuration file from `/etc/debspawn/global.toml`, or a configuration file in a location 154 | specified by the `--config` flag. 155 | If a configuration file is specified on the command line, the global file is ignored completely rather than merged. 156 | 157 | The config is a TOML file containing any of the following (all optional) keys: 158 | 159 | * `OSRootsDir`: directory for os images (`/var/lib/debspawn/images/`) 160 | * `ResultsDir`: directory for build artifacts (`/var/lib/debspawn/results/`) 161 | * `APTCacheDir`: directory for debspawn's own package cache (`/var/lib/debspawn/aptcache/`) 162 | * `InjectedPkgsDir`: packages placed in this directory will be available as dependencies for builds (`/var/lib/debspawn/injected-pkgs/`) 163 | * `TempDir`: temporary directory used for running containers (`/var/tmp/debspawn/`) 164 | * `AllowUnsafePermissions`: allow usage of riskier container permissions, such as binding the host `/dev` and `/proc` into the container (`false`) 165 | 166 | ## FAQ 167 | 168 | #### Why use systemd-nspawn? Why not $other_container_manager? 169 | 170 | Systemd-nspawn is a very lightweight container solution readily available without much (or any) setup on all Linux 171 | systems that are using systemd. It does not need any background daemon and while it does not provide a lot of features, 172 | it fits the relatively simple usecase of building in an isolated environment perfectly. 173 | 174 | 175 | #### Do I need to set up apt-cacher-ng to use this efficiently? 176 | 177 | No - while `apt-cacher-ng` is generally a useful tool, it is not required for efficient use of `debspawn`. 178 | `debspawn` will cache downloaded packages between runs fully automatically, so packages only get downloaded when 179 | they have not been retrieved before. 180 | 181 | 182 | #### Is the build environment the same as sbuild? 183 | 184 | No, unfortunately. Due to the different technology used, there are subtle differences between sbuild chroots and 185 | `debspawn` containers. 186 | The differences should - for the most part - not have any impact on package builds, and any such occurrence is highly 187 | likely a bug in the package's build process. If you think it is not, please file a bug against Debspawn. We try to be 188 | as close to sbuild's default environment as possible. 189 | 190 | One way the build environment differs from Debian's default sbuild setup intentionally is in its consistent use of unicode. 191 | By default, `debspawn` will ensure that unicode is always available and used. If you do not want this behavior, you can pass 192 | the `--no-unicode` flag to `debspawn` to disable unicode in the tool itself and in the build environment. 193 | 194 | 195 | #### Will this replace sbuild? 196 | 197 | Not in the foreseeable future on Debian itself. 198 | Sbuild is a proven tool that works well for Debian and supports other OSes than Linux, while `debspawn` is Linux-only, 199 | a thing that will not change due to its use of systemd. 200 | However, Laniakea-using derivatives such as PureOS use the tool for building all packages and for constructing other 201 | build environments to e.g. build disk images. 202 | 203 | 204 | #### What is the relation of this project with Laniakea? 205 | 206 | The Laniakea job runner uses `debspawn` for a bunch of tasks and the integration with the Laniakea system is generally 207 | quite tight. 208 | Of course you can use `debspawn` without Laniakea and integrate it with any tool you want. Debspawn will always be usable 209 | without Laniakea automation. 210 | 211 | 212 | #### This tool is really fast! What is the secret? 213 | 214 | Surprisingly, building packages with `debspawn` is often a bit faster than using `pbuilder` and `sbuild` with their 215 | default settings. 216 | The speed gain comes in large part from the internal use of the Zstandard compression algorithm for container base images. 217 | Zstd allows for fast decompression of the tarballs, which is exactly why it was chosen (LZ4 would be even faster, 218 | but Zstd actually is a good compromise between compression ration and speed). This shaves off a few seconds of time 219 | for each build that is used on base image decompression. 220 | Additionally, Debspawn uses `eatmydata` to disable fsync & co. by default in a few places, improving the time it takes 221 | to set up the build environment by quite a bit as well. 222 | If you want, you can configure other tools to make use of the same methods as well and see if they run faster. 223 | There's nothing new or unusually clever here at all! 224 | -------------------------------------------------------------------------------- /RELEASE: -------------------------------------------------------------------------------- 1 | Debspawn Release Notes 2 | 3 | 1. Write NEWS entries for Debspawn in the same format as usual. 4 | 5 | git shortlog v0.6.4.. | grep -i -v trivial | grep -v Merge > NEWS.new 6 | 7 | -------------------------------------------------------------------------------- 8 | Version 0.6.5 9 | ~~~~~~~~~~~~~ 10 | Released: 2024-xx-xx 11 | 12 | Notes: 13 | 14 | Features: 15 | 16 | Bugfixes: 17 | -------------------------------------------------------------------------------- 18 | 19 | 2. Commit changes in Git: 20 | 21 | git commit -a -m "Release version 0.6.5" 22 | git tag -s -f -m "Release 0.6.5" v0.6.5 23 | git push --tags 24 | git push 25 | 26 | 3. Upload to PyPI: 27 | 28 | python setup.py sdist 29 | twine upload dist/* 30 | 31 | 4. Do post release version bump in `RELEASE` and `debspawn/__init__.py` 32 | 33 | 5. Commit trivial changes: 34 | 35 | git commit -a -m "trivial: post release version bump" 36 | git push 37 | -------------------------------------------------------------------------------- /autoformat.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | BASEDIR=$(dirname "$0") 5 | cd $BASEDIR 6 | 7 | echo "=== ISort ===" 8 | python -m isort . 9 | python -m isort ./debspawn/dsrun 10 | 11 | echo "=== Black ===" 12 | python -m black . 13 | -------------------------------------------------------------------------------- /data/services/debspawn-clear-caches.service: -------------------------------------------------------------------------------- 1 | # This file is part of debspawn. 2 | # 3 | # Debspawn is free software: you can redistribute it and/or modify 4 | # it under the terms of the GNU Lesser General Public License as published by 5 | # the Free Software Foundation, either version 3 of the license, or 6 | # (at your option) any later version. 7 | 8 | [Unit] 9 | Description=Clear debspawn caches 10 | Wants=debspawn-clear-caches.timer 11 | 12 | [Service] 13 | Type=oneshot 14 | ExecStart=@PREFIX@/bin/debspawn maintain --clear-caches 15 | 16 | PrivateTmp=true 17 | PrivateDevices=true 18 | PrivateNetwork=true 19 | -------------------------------------------------------------------------------- /data/services/debspawn-clear-caches.timer: -------------------------------------------------------------------------------- 1 | # This file is part of debspawn. 2 | # 3 | # Debspawn is free software: you can redistribute it and/or modify 4 | # it under the terms of the GNU Lesser General Public License as published by 5 | # the Free Software Foundation, either version 3 of the license, or 6 | # (at your option) any later version. 7 | 8 | [Unit] 9 | Description=Clear all debspawn caches regularly 10 | 11 | [Timer] 12 | OnCalendar=monthly 13 | RandomizedDelaySec=12h 14 | AccuracySec=20min 15 | Persistent=true 16 | 17 | [Install] 18 | WantedBy=timers.target 19 | -------------------------------------------------------------------------------- /data/tmpfiles.d/debspawn.conf: -------------------------------------------------------------------------------- 1 | # This file is part of debspawn. 2 | # 3 | # Debspawn is free software: you can redistribute it and/or modify 4 | # it under the terms of the GNU Lesser General Public License as published by 5 | # the Free Software Foundation, either version 3 of the license, or 6 | # (at your option) any later version. 7 | 8 | # Ensure debspawn temporary directory is only owned by root and cleaned regularly 9 | D /var/tmp/debspawn 0755 root root 2d 10 | -------------------------------------------------------------------------------- /debspawn.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import os 4 | import sys 5 | 6 | from debspawn import cli 7 | 8 | thisfile = __file__ 9 | if not os.path.isabs(thisfile): 10 | thisfile = os.path.normpath(os.path.join(os.getcwd(), thisfile)) 11 | 12 | sys.exit(cli.run(thisfile, sys.argv[1:])) 13 | -------------------------------------------------------------------------------- /debspawn/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (C) 2018-2022 Matthias Klumpp 4 | # 5 | # Licensed under the GNU Lesser General Public License Version 3 6 | # 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Lesser General Public License as published by 9 | # the Free Software Foundation, either version 3 of the license, or 10 | # (at your option) any later version. 11 | # 12 | # This software is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Lesser General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Lesser General Public License 18 | # along with this software. If not, see . 19 | 20 | __appname__ = 'debspawn' 21 | __version__ = '0.6.5' 22 | -------------------------------------------------------------------------------- /debspawn/aptcache.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (C) 2018-2022 Matthias Klumpp 4 | # 5 | # Licensed under the GNU Lesser General Public License Version 3 6 | # 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Lesser General Public License as published by 9 | # the Free Software Foundation, either version 3 of the license, or 10 | # (at your option) any later version. 11 | # 12 | # This software is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Lesser General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Lesser General Public License 18 | # along with this software. If not, see . 19 | 20 | import os 21 | import shutil 22 | from glob import glob 23 | from pathlib import Path 24 | 25 | from .utils.misc import hardlink_or_copy 26 | 27 | 28 | class APTCache: 29 | ''' 30 | Manage cache of APT packages 31 | ''' 32 | 33 | def __init__(self, osbase): 34 | self._cache_dir = os.path.join(osbase.global_config.aptcache_dir, osbase.name) 35 | 36 | def merge_from_dir(self, tmp_cache_dir): 37 | ''' 38 | Merge in packages from a temporary cache 39 | ''' 40 | 41 | from random import choice 42 | from string import digits, ascii_lowercase 43 | 44 | Path(self._cache_dir).mkdir(parents=True, exist_ok=True) 45 | for pkg_fname in glob(os.path.join(tmp_cache_dir, '*.deb')): 46 | pkg_basename = os.path.basename(pkg_fname) 47 | pkg_cachepath = os.path.join(self._cache_dir, pkg_basename) 48 | 49 | if not os.path.isfile(pkg_cachepath): 50 | pkg_tmp_name = ( 51 | pkg_cachepath + '.tmp-' + ''.join(choice(ascii_lowercase + digits) for _ in range(8)) 52 | ) 53 | shutil.copy2(pkg_fname, pkg_tmp_name) 54 | try: 55 | os.rename(pkg_tmp_name, pkg_cachepath) 56 | except OSError: 57 | # maybe some other debspawn instance tried to add the package just now, 58 | # in that case we give up 59 | os.remove(pkg_tmp_name) 60 | 61 | def create_instance_cache(self, tmp_cache_dir): 62 | ''' 63 | Copy the cache to a temporary location for use in a new container instance. 64 | ''' 65 | 66 | Path(self._cache_dir).mkdir(parents=True, exist_ok=True) 67 | Path(tmp_cache_dir).mkdir(parents=True, exist_ok=True) 68 | 69 | for pkg_fname in glob(os.path.join(self._cache_dir, '*.deb')): 70 | pkg_cachepath = os.path.join(tmp_cache_dir, os.path.basename(pkg_fname)) 71 | 72 | if not os.path.isfile(pkg_cachepath): 73 | hardlink_or_copy(pkg_fname, pkg_cachepath) 74 | 75 | def clear(self): 76 | ''' 77 | Remove all cache contents. 78 | ''' 79 | 80 | Path(self._cache_dir).mkdir(parents=True, exist_ok=True) 81 | cache_size = len(glob(os.path.join(self._cache_dir, '*.deb'))) 82 | 83 | old_cache_dir = self._cache_dir.rstrip(os.sep) + '.old' 84 | os.rename(self._cache_dir, old_cache_dir) 85 | Path(self._cache_dir).mkdir(parents=True, exist_ok=True) 86 | 87 | shutil.rmtree(old_cache_dir) 88 | 89 | return cache_size 90 | 91 | def delete(self): 92 | ''' 93 | Remove cache completely - only useful when removing a base image completely. 94 | ''' 95 | shutil.rmtree(self._cache_dir) 96 | -------------------------------------------------------------------------------- /debspawn/build.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (C) 2018-2022 Matthias Klumpp 4 | # 5 | # Licensed under the GNU Lesser General Public License Version 3 6 | # 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Lesser General Public License as published by 9 | # the Free Software Foundation, either version 3 of the license, or 10 | # (at your option) any later version. 11 | # 12 | # This software is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Lesser General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Lesser General Public License 18 | # along with this software. If not, see . 19 | 20 | import os 21 | import platform 22 | import subprocess 23 | from glob import glob 24 | from collections.abc import Iterable 25 | 26 | from .nspawn import nspawn_run_persist, nspawn_run_helper_persist 27 | from .injectpkg import PackageInjector 28 | from .utils.env import ( 29 | ensure_root, 30 | get_tree_size, 31 | get_free_space, 32 | get_owner_uid_gid, 33 | switch_unprivileged, 34 | get_random_free_uid_gid, 35 | ) 36 | from .utils.log import ( 37 | input_bool, 38 | print_info, 39 | print_warn, 40 | print_error, 41 | print_bullet, 42 | print_header, 43 | print_section, 44 | capture_console_output, 45 | save_captured_console_output, 46 | ) 47 | from .utils.misc import ( 48 | cd, 49 | listify, 50 | temp_dir, 51 | safe_copy, 52 | format_filesize, 53 | version_noepoch, 54 | ) 55 | from .utils.command import safe_run 56 | 57 | 58 | class BuildError(Exception): 59 | """Package build failed with a generic error.""" 60 | 61 | 62 | def interact_with_build_environment( 63 | osbase, 64 | instance_dir, 65 | machine_name, 66 | *, 67 | pkg_dir_root, 68 | source_pkg_dir, 69 | aptcache_tmp, 70 | pkginjector, 71 | prev_exitcode, 72 | ) -> bool: 73 | '''Launch an interactive shell in the build environment''' 74 | 75 | # find the right directory to switch to 76 | pkg_dir = pkg_dir_root 77 | for f in glob(os.path.join(pkg_dir, '*')): 78 | if os.path.isdir(f): 79 | pkg_dir = f 80 | break 81 | 82 | print() 83 | print_info('Launching interactive shell in build environment.') 84 | if prev_exitcode != 0: 85 | print_info('The previous build step failed with exit code {}'.format(prev_exitcode)) 86 | else: 87 | print_info('The previous build step was successful.') 88 | print_info('Temporary location of package files on the host:\n => file://{}'.format(pkg_dir)) 89 | print_info('Press CTL+D to exit the interactive shell.') 90 | print() 91 | 92 | nspawn_flags = ['--bind={}:/srv/build/'.format(pkg_dir_root)] 93 | nspawn_run_persist( 94 | osbase, 95 | instance_dir, 96 | machine_name, 97 | chdir=os.path.join('/srv/build', os.path.basename(pkg_dir)), 98 | flags=nspawn_flags, 99 | tmp_apt_cache_dir=aptcache_tmp, 100 | pkginjector=pkginjector, 101 | syscall_filter=osbase.global_config.syscall_filter, 102 | verbose=True, 103 | ) 104 | 105 | print() 106 | copy_artifacts = input_bool( 107 | 'Should any generated build artifacts (binary/source packages, etc.) be saved?', default=False 108 | ) 109 | if copy_artifacts: 110 | print_bullet('Artifacts will be copied to the results directory.') 111 | else: 112 | print_bullet('Artifacts will not be kept.') 113 | 114 | if source_pkg_dir: 115 | copy_changes = input_bool( 116 | ( 117 | 'Should changes to the debian/ directory be copied back to the host?\n' 118 | 'This will OVERRIDE all changes made on files on the host.' 119 | ), 120 | default=False, 121 | ) 122 | 123 | if copy_changes: 124 | print_info('Cleaning up...') 125 | # clean the source tree. we intentionally ignore errors here. 126 | nspawn_run_persist( 127 | osbase, 128 | instance_dir, 129 | machine_name, 130 | chdir=os.path.join('/srv/build', os.path.basename(pkg_dir)), 131 | flags=nspawn_flags, 132 | command=['dpkg-buildpackage', '-T', 'clean'], 133 | tmp_apt_cache_dir=aptcache_tmp, 134 | pkginjector=pkginjector, 135 | ) 136 | 137 | print() 138 | print_info('Copying back changes...') 139 | known_files = {} 140 | dest_debian_dir = os.path.join(source_pkg_dir, 'debian') 141 | src_debian_dir = os.path.join(pkg_dir, 'debian') 142 | 143 | # get uid/gid of the user who invoked us 144 | o_uid, o_gid = get_owner_uid_gid() 145 | 146 | # collect list of existing packages 147 | for sdir, _, files in os.walk(dest_debian_dir): 148 | for f in files: 149 | fname = os.path.join(sdir, f) 150 | known_files[os.path.relpath(fname, dest_debian_dir)] = fname 151 | 152 | # walk through the source files, copying everything to the destination 153 | for sdir, _, files in os.walk(src_debian_dir): 154 | for f in files: 155 | fname = os.path.join(sdir, f) 156 | rel_fname = os.path.relpath(fname, src_debian_dir) 157 | dest_fname = os.path.normpath(os.path.join(dest_debian_dir, rel_fname)) 158 | dest_dir = os.path.dirname(dest_fname) 159 | if rel_fname in known_files: 160 | del known_files[rel_fname] 161 | 162 | if os.path.isdir(fname): 163 | print('New dir: {}'.format(rel_fname)) 164 | with switch_unprivileged(): 165 | os.makedirs(dest_fname, exist_ok=True) 166 | continue 167 | if not os.path.isdir(dest_dir): 168 | print('New dir: {}'.format(os.path.relpath(dest_dir, dest_debian_dir))) 169 | with switch_unprivileged(): 170 | os.makedirs(dest_dir, exist_ok=True) 171 | 172 | print('Copy: {}'.format(rel_fname)) 173 | safe_copy(fname, dest_fname) 174 | os.chown(dest_fname, o_uid, o_gid, follow_symlinks=False) 175 | 176 | for rel_fname, fname in known_files.items(): 177 | print('Delete: {}'.format(rel_fname)) 178 | os.remove(fname) 179 | print() 180 | else: 181 | print_bullet('Discarding build environment.') 182 | else: 183 | print_info('Can not copy back changes as original package directory is unknown.') 184 | 185 | return copy_artifacts 186 | 187 | 188 | def internal_execute_build( 189 | osbase, 190 | pkg_dir, 191 | build_only=None, 192 | *, 193 | qa_lintian=False, 194 | interact=False, 195 | source_pkg_dir=None, 196 | buildflags: list[str] = None, 197 | build_env: dict[str, str] = None, 198 | ): 199 | '''Perform the actual build on an extracted package directory''' 200 | assert not build_only or isinstance(build_only, str) 201 | if not pkg_dir: 202 | raise ValueError('Package directory is missing!') 203 | pkg_dir = os.path.normpath(pkg_dir) 204 | if not build_env: 205 | build_env = {} 206 | 207 | # get a fresh UID to give to our build user within the container 208 | builder_uid = get_random_free_uid_gid()[0] 209 | 210 | with osbase.new_instance() as (instance_dir, machine_name): 211 | # first, check basic requirements 212 | 213 | # instance dir and pkg dir are both temporary directories, so they 214 | # will be on the same filesystem configured as workspace for debspawn. 215 | # therefore we only check on directory. 216 | free_space = get_free_space(instance_dir) 217 | print_info('Free space in workspace: {}'.format(format_filesize(free_space))) 218 | 219 | # check for at least 512MiB - this is a ridiculously small amount, so the build will likely fail. 220 | # but with even less, even attempting a build is pointless. 221 | if (free_space / 2048) < 512: 222 | print_error('Not enough free space available in workspace.') 223 | return 8 224 | 225 | # prepare the build. At this point, we only run trusted code and the container 226 | # has network access 227 | with temp_dir('pkgsync-' + machine_name) as pkgsync_tmp: 228 | # create temporary locations set up and APT cache sharing and package injection 229 | aptcache_tmp = os.path.join(pkgsync_tmp, 'aptcache') 230 | pkginjector = PackageInjector(osbase) 231 | if pkginjector.has_injectables(): 232 | pkginjector.create_instance_repo(os.path.join(pkgsync_tmp, 'pkginject')) 233 | 234 | # set up the build environment 235 | nspawn_flags = ['--bind={}:/srv/build/'.format(pkg_dir)] 236 | prep_flags = ['--build-prepare'] 237 | 238 | # if we force a suite and have injected packages, the injected packages 239 | # will never be picked up. 240 | if not pkginjector.has_injectables(): 241 | prep_flags.extend(['--suite', osbase.suite]) 242 | 243 | if build_only == 'arch': 244 | prep_flags.append('--arch-only') 245 | r = nspawn_run_helper_persist( 246 | osbase, 247 | instance_dir, 248 | machine_name, 249 | prep_flags, 250 | '/srv', 251 | build_uid=builder_uid, 252 | nspawn_flags=nspawn_flags, 253 | tmp_apt_cache_dir=aptcache_tmp, 254 | pkginjector=pkginjector, 255 | ) 256 | if r != 0: 257 | print_error('Build environment setup failed.') 258 | return False 259 | 260 | # run the actual build. At this point, code is less trusted, and we disable network access. 261 | nspawn_flags = ['--bind={}:/srv/build/'.format(pkg_dir), '-u', 'builder', '--private-network'] 262 | helper_flags = ['--build-run'] 263 | helper_flags.extend(['--suite', osbase.suite]) 264 | if buildflags: 265 | helper_flags.append('--buildflags={}'.format(';'.join(buildflags))) 266 | r = nspawn_run_helper_persist( 267 | osbase, 268 | instance_dir, 269 | machine_name, 270 | helper_flags, 271 | '/srv', 272 | build_uid=builder_uid, 273 | nspawn_flags=nspawn_flags, 274 | tmp_apt_cache_dir=aptcache_tmp, 275 | pkginjector=pkginjector, 276 | env_vars=build_env, 277 | syscall_filter=osbase.global_config.syscall_filter, 278 | ) 279 | # exit, unless we are in interactive mode 280 | if r != 0 and not interact: 281 | return False 282 | 283 | if qa_lintian and r == 0: 284 | # running Lintian was requested, so do so. 285 | # we use Lintian from the container, so we validate with the validator from 286 | # the OS the package was actually built against 287 | nspawn_flags = ['--bind={}:/srv/build/'.format(pkg_dir)] 288 | r = nspawn_run_helper_persist( 289 | osbase, 290 | instance_dir, 291 | machine_name, 292 | ['--run-qa', '--lintian'], 293 | '/srv', 294 | build_uid=builder_uid, 295 | nspawn_flags=nspawn_flags, 296 | tmp_apt_cache_dir=aptcache_tmp, 297 | pkginjector=pkginjector, 298 | ) 299 | if r != 0: 300 | print_error('QA failed.') 301 | return False 302 | print() # extra blank line after Lintian output 303 | 304 | if interact: 305 | ri = interact_with_build_environment( 306 | osbase, 307 | instance_dir, 308 | machine_name, 309 | pkg_dir_root=pkg_dir, 310 | source_pkg_dir=source_pkg_dir, 311 | aptcache_tmp=aptcache_tmp, 312 | pkginjector=pkginjector, 313 | prev_exitcode=r, 314 | ) 315 | # if we exit with a non-True result, we stop here and don't proceed 316 | # with the next steps that save artifacts. 317 | if not ri: 318 | return False 319 | 320 | build_dir_size = get_tree_size(pkg_dir) 321 | print_info( 322 | 'This build required {} of dedicated disk space.'.format(format_filesize(build_dir_size)) 323 | ) 324 | 325 | return True 326 | 327 | 328 | def print_build_detail(osbase, pkgname, version): 329 | print_info('Package: {}'.format(pkgname)) 330 | print_info('Version: {}'.format(version)) 331 | print_info('Distribution: {}'.format(osbase.suite)) 332 | print_info('Architecture: {}'.format(osbase.arch)) 333 | print_info() 334 | 335 | 336 | def _read_source_package_details(): 337 | out, err, ret = safe_run(['dpkg-parsechangelog']) 338 | if ret != 0: 339 | raise BuildError('Running dpkg-parsechangelog failed: {}{}'.format(out, err)) 340 | 341 | pkg_sourcename = None 342 | pkg_version = None 343 | for line in out.split('\n'): 344 | if line.startswith('Source: '): 345 | pkg_sourcename = line[8:].strip() 346 | elif line.startswith('Version: '): 347 | pkg_version = line[9:].strip() 348 | 349 | if not pkg_sourcename or not pkg_version: 350 | print_error('Unable to determine source package name or source package version. Can not continue.') 351 | return None, None, None 352 | 353 | pkg_version_dsc = version_noepoch(pkg_version) 354 | dsc_fname = '{}_{}.dsc'.format(pkg_sourcename, pkg_version_dsc) 355 | 356 | return pkg_sourcename, pkg_version, dsc_fname 357 | 358 | 359 | def _get_build_flags(build_only=None, include_orig=False, maintainer=None, extra_flags: Iterable[str] = None): 360 | import shlex 361 | 362 | buildflags = [] 363 | extra_flags = listify(extra_flags) 364 | 365 | if build_only: 366 | if build_only == 'binary': 367 | buildflags.append('-b') 368 | elif build_only == 'arch': 369 | buildflags.append('-B') 370 | elif build_only == 'indep': 371 | buildflags.append('-A') 372 | elif build_only == 'source': 373 | buildflags.append('-S') 374 | else: 375 | print_error('Invalid build-only flag "{}". Can not continue.'.format(build_only)) 376 | return False, [] 377 | 378 | if include_orig: 379 | buildflags.append('-sa') 380 | if maintainer: 381 | buildflags.append('-m{}'.format(maintainer.replace(';', ','))) 382 | buildflags.append('-e{}'.format(maintainer.replace(';', ','))) 383 | for flag_raw in extra_flags: 384 | buildflags.extend(shlex.split(flag_raw)) 385 | 386 | return True, buildflags 387 | 388 | 389 | def _sign_result(results_dir, spkg_name, spkg_version, build_arch, build_only): 390 | print_section('Signing Package') 391 | spkg_version_noepoch = version_noepoch(spkg_version) 392 | sign_arch = 'source' if build_only == 'source' else build_arch 393 | changes_basename = '{}_{}_{}.changes'.format(spkg_name, spkg_version_noepoch, sign_arch) 394 | 395 | with switch_unprivileged(): 396 | proc = subprocess.run(['debsign', os.path.join(results_dir, changes_basename)], check=False) 397 | if proc.returncode != 0: 398 | print_error('Signing failed.') 399 | return False 400 | return True 401 | 402 | 403 | def _print_system_info(): 404 | from . import __version__ 405 | from .utils.misc import current_time_string 406 | 407 | print_info( 408 | 'debspawn {version} on {host} at {time}'.format( 409 | version=__version__, host=platform.node(), time=current_time_string() 410 | ) 411 | ) 412 | 413 | 414 | def build_from_directory( 415 | osbase, 416 | pkg_dir, 417 | *, 418 | sign=False, 419 | build_only=None, 420 | include_orig=False, 421 | maintainer=None, 422 | clean_source=False, 423 | qa_lintian=False, 424 | interact=False, 425 | log_build=True, 426 | extra_dpkg_flags: list[str] = None, 427 | build_env: dict[str, str] = None, 428 | ): 429 | ensure_root() 430 | osbase.ensure_exists() 431 | extra_dpkg_flags = listify(extra_dpkg_flags) 432 | 433 | if interact and log_build: 434 | print_warn('Build log and interactive mode can not be enabled at the same time. Disabling build log.') 435 | print() 436 | log_build = False 437 | 438 | # capture console output if we should log the build 439 | if log_build: 440 | capture_console_output() 441 | 442 | if not pkg_dir: 443 | pkg_dir = os.getcwd() 444 | pkg_dir = os.path.abspath(pkg_dir) 445 | 446 | r, buildflags = _get_build_flags(build_only, include_orig, maintainer, extra_dpkg_flags) 447 | if not r: 448 | return False 449 | 450 | _print_system_info() 451 | print_header('Package build (from directory)') 452 | 453 | print_section('Creating source package') 454 | with cd(pkg_dir): 455 | with switch_unprivileged(): 456 | deb_files_fname = os.path.join(pkg_dir, 'debian', 'files') 457 | if os.path.isfile(deb_files_fname): 458 | deb_files_fname = None # the file already existed, we don't need to clean it up later 459 | 460 | pkg_sourcename, pkg_version, dsc_fname = _read_source_package_details() 461 | if not pkg_sourcename: 462 | return False 463 | 464 | cmd = ['dpkg-buildpackage', '-S', '--no-sign'] 465 | # d/rules clean requires build dependencies installed if run on the host 466 | # we avoid that by default, unless explicitly requested 467 | if not clean_source: 468 | cmd.append('-nc') 469 | 470 | proc = subprocess.run(cmd, check=False) 471 | if proc.returncode != 0: 472 | return False 473 | 474 | # remove d/files file that was created when generating the source package. 475 | # we only clean up the file if it didn't exist prior to us running the command. 476 | if deb_files_fname: 477 | try: 478 | os.remove(deb_files_fname) 479 | except OSError: 480 | pass 481 | 482 | print_header('Package build') 483 | print_build_detail(osbase, pkg_sourcename, pkg_version) 484 | 485 | success = False 486 | with temp_dir(pkg_sourcename) as pkg_tmp_dir: 487 | with cd(pkg_tmp_dir): 488 | cmd = ['dpkg-source', '-x', os.path.join(pkg_dir, '..', dsc_fname)] 489 | proc = subprocess.run(cmd, check=False) 490 | if proc.returncode != 0: 491 | return False 492 | 493 | success = internal_execute_build( 494 | osbase, 495 | pkg_tmp_dir, 496 | build_only, 497 | qa_lintian=qa_lintian, 498 | interact=interact, 499 | source_pkg_dir=pkg_dir, 500 | buildflags=buildflags, 501 | build_env=build_env, 502 | ) 503 | 504 | # copy build results 505 | if success: 506 | osbase.retrieve_artifacts(pkg_tmp_dir) 507 | 508 | # save buildlog, if we generated one 509 | log_fname = os.path.join( 510 | osbase.results_dir, 511 | '{}_{}_{}.buildlog'.format(pkg_sourcename, version_noepoch(pkg_version), osbase.arch), 512 | ) 513 | save_captured_console_output(log_fname) 514 | 515 | # exit, there is nothing more to do if no package was built 516 | if not success: 517 | return False 518 | 519 | # sign the resulting package 520 | if sign: 521 | r = _sign_result(osbase.results_dir, pkg_sourcename, pkg_version, osbase.arch, build_only) 522 | if not r: 523 | return False 524 | 525 | print_info('Done.') 526 | 527 | return True 528 | 529 | 530 | def build_from_dsc( 531 | osbase, 532 | dsc_fname, 533 | *, 534 | sign=False, 535 | build_only=None, 536 | include_orig=False, 537 | maintainer=None, 538 | qa_lintian: bool = False, 539 | interact: bool = False, 540 | log_build: bool = True, 541 | extra_dpkg_flags: Iterable[str] = None, 542 | build_env: dict[str, str] = None, 543 | ): 544 | ensure_root() 545 | osbase.ensure_exists() 546 | extra_dpkg_flags = listify(extra_dpkg_flags) 547 | 548 | if interact and log_build: 549 | print_warn('Build log and interactive mode can not be enabled at the same time. Disabling build log.') 550 | print() 551 | log_build = False 552 | 553 | # capture console output if we should log the build 554 | if log_build: 555 | capture_console_output() 556 | 557 | r, buildflags = _get_build_flags(build_only, include_orig, maintainer, extra_dpkg_flags) 558 | if not r: 559 | return False 560 | 561 | _print_system_info() 562 | 563 | success = False 564 | dsc_fname = os.path.abspath(os.path.normpath(dsc_fname)) 565 | tmp_prefix = os.path.basename(dsc_fname).replace('.dsc', '').replace(' ', '-') 566 | with temp_dir(tmp_prefix) as pkg_tmp_dir: 567 | with cd(pkg_tmp_dir): 568 | cmd = ['dpkg-source', '-x', dsc_fname] 569 | proc = subprocess.run(cmd, check=False) 570 | if proc.returncode != 0: 571 | return False 572 | 573 | pkg_srcdir = None 574 | for f in glob('./*'): 575 | if os.path.isdir(f): 576 | pkg_srcdir = f 577 | break 578 | if not pkg_srcdir: 579 | print_error('Unable to find source directory of extracted package.') 580 | return False 581 | 582 | with cd(pkg_srcdir): 583 | pkg_sourcename, pkg_version, dsc_fname = _read_source_package_details() 584 | if not pkg_sourcename: 585 | return False 586 | 587 | print_header('Package build') 588 | print_build_detail(osbase, pkg_sourcename, pkg_version) 589 | 590 | success = internal_execute_build( 591 | osbase, 592 | pkg_tmp_dir, 593 | build_only, 594 | qa_lintian=qa_lintian, 595 | interact=interact, 596 | buildflags=buildflags, 597 | build_env=build_env, 598 | ) 599 | 600 | # copy build results 601 | if success: 602 | osbase.retrieve_artifacts(pkg_tmp_dir) 603 | 604 | # save buildlog, if we generated one 605 | log_fname = os.path.join( 606 | osbase.results_dir, 607 | '{}_{}_{}.buildlog'.format(pkg_sourcename, version_noepoch(pkg_version), osbase.arch), 608 | ) 609 | save_captured_console_output(log_fname) 610 | 611 | # build log is saved, but no artifacts are available, so there's nothing more to do 612 | if not success: 613 | return False 614 | 615 | # sign the resulting package 616 | if sign: 617 | r = _sign_result(osbase.results_dir, pkg_sourcename, pkg_version, osbase.arch, build_only) 618 | if not r: 619 | return False 620 | 621 | print_info('Done.') 622 | return True 623 | -------------------------------------------------------------------------------- /debspawn/config.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (C) 2018-2022 Matthias Klumpp 4 | # 5 | # Licensed under the GNU Lesser General Public License Version 3 6 | # 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Lesser General Public License as published by 9 | # the Free Software Foundation, either version 3 of the license, or 10 | # (at your option) any later version. 11 | # 12 | # This software is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Lesser General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Lesser General Public License 18 | # along with this software. If not, see . 19 | 20 | import os 21 | import sys 22 | 23 | import tomlkit 24 | 25 | thisfile = __file__ 26 | if not os.path.isabs(thisfile): 27 | thisfile = os.path.normpath(os.path.join(os.getcwd(), thisfile)) 28 | 29 | 30 | __all__ = ['GlobalConfig'] 31 | 32 | 33 | class GlobalConfig: 34 | ''' 35 | Global configuration singleton affecting all of Debspawn. 36 | ''' 37 | 38 | _instance = None 39 | 40 | class __GlobalConfig: 41 | def load(self, fname=None): 42 | if not fname: 43 | fname = '/etc/debspawn/global.toml' 44 | 45 | cdata = {} 46 | if os.path.isfile(fname): 47 | with open(fname) as f: 48 | try: 49 | cdata = tomlkit.load(f) 50 | except tomlkit.exceptions.ParseError as e: 51 | print( 52 | 'Unable to parse global configuration (global.toml): {}'.format(str(e)), 53 | file=sys.stderr, 54 | ) 55 | sys.exit(8) 56 | 57 | self._dsrun_path = os.path.normpath(os.path.join(thisfile, '..', 'dsrun')) 58 | if not os.path.isfile(self._dsrun_path): 59 | print( 60 | 'Debspawn is not set up properly: Unable to find file "{}". Can not continue.'.format( 61 | self._dsrun_path 62 | ), 63 | file=sys.stderr, 64 | ) 65 | sys.exit(4) 66 | 67 | self._osroots_dir = cdata.get('OSImagesDir', '/var/lib/debspawn/images/') 68 | self._results_dir = cdata.get('ResultsDir', '/var/lib/debspawn/results/') 69 | self._aptcache_dir = cdata.get('APTCacheDir', '/var/lib/debspawn/aptcache/') 70 | self._injected_pkgs_dir = cdata.get('InjectedPkgsDir', '/var/lib/debspawn/injected-pkgs/') 71 | self._temp_dir = cdata.get('TempDir', '/var/tmp/debspawn/') 72 | self._default_bootstrap_variant = cdata.get('DefaultBootstrapVariant', 'buildd') 73 | self._allow_unsafe_perms = cdata.get('AllowUnsafePermissions', False) 74 | self._cache_packages = bool(cdata.get('CachePackages', True)) 75 | self._bootstrap_tool = cdata.get('BootstrapTool', 'debootstrap') 76 | 77 | self._syscall_filter = cdata.get('SyscallFilter', 'compat') 78 | if self._syscall_filter == 'compat': 79 | # permit some system calls known to be needed by packages that sbuild & Co. 80 | # build without problems. 81 | self._syscall_filter = ['@memlock', '@pkey', '@clock', '@cpu-emulation'] 82 | elif self._syscall_filter == 'nspawn-default': 83 | # make no additional changes, so nspawn's built-in defaults are used 84 | self._syscall_filter = [] 85 | else: 86 | if type(self._syscall_filter) is not list: 87 | print( 88 | ( 89 | 'Configuration error (global.toml): Entry "SyscallFilter" needs to be either a ' 90 | 'string value ("compat" or "nspawn-default"), or a list of permissible ' 91 | 'system call names as listed by the syscall-filter command of systemd-analyze(1)' 92 | ), 93 | file=sys.stderr, 94 | ) 95 | sys.exit(8) 96 | 97 | @property 98 | def dsrun_path(self) -> str: 99 | return self._dsrun_path 100 | 101 | @dsrun_path.setter 102 | def dsrun_path(self, v): 103 | self._dsrun_path = v 104 | 105 | @property 106 | def osroots_dir(self) -> str: 107 | return self._osroots_dir 108 | 109 | @property 110 | def results_dir(self) -> str: 111 | return self._results_dir 112 | 113 | @property 114 | def aptcache_dir(self) -> str: 115 | return self._aptcache_dir 116 | 117 | @property 118 | def injected_pkgs_dir(self) -> str: 119 | return self._injected_pkgs_dir 120 | 121 | @property 122 | def temp_dir(self) -> str: 123 | return self._temp_dir 124 | 125 | @property 126 | def default_bootstrap_variant(self) -> str: 127 | return self._default_bootstrap_variant 128 | 129 | @property 130 | def syscall_filter(self) -> list: 131 | """Customize which syscalls should be filtered.""" 132 | return self._syscall_filter 133 | 134 | @property 135 | def allow_unsafe_perms(self) -> bool: 136 | """Whether usage of unsafe permissions is allowed.""" 137 | return self._allow_unsafe_perms 138 | 139 | @property 140 | def cache_packages(self) -> bool: 141 | """Whether APT packages should be cached by debspawn.""" 142 | return self._cache_packages 143 | 144 | @property 145 | def bootstrap_tool(self) -> str: 146 | """The chroot bootstrap tool that we should use.""" 147 | return self._bootstrap_tool 148 | 149 | def __init__(self, fname=None): 150 | if not GlobalConfig._instance: 151 | GlobalConfig._instance = GlobalConfig.__GlobalConfig() 152 | GlobalConfig._instance.load(fname) 153 | 154 | def __getattr__(self, name): 155 | return getattr(self._instance, name) 156 | -------------------------------------------------------------------------------- /debspawn/dsrun: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (C) 2018-2022 Matthias Klumpp 5 | # 6 | # SPDX-License-Identifier: LGPL-3.0-or-later 7 | 8 | # IMPORTANT: This file is placed within a Debspawn container. 9 | # The containers only contain a minimal set of packages, and only a reduced 10 | # installation of Python is available via the python3-minimal package. 11 | # This file must be self-contained and only depend on modules available 12 | # in that Python installation. 13 | # It must also not depend on any Python 3 feature introduced after version 3.5. 14 | # See /usr/share/doc/python3.*-minimal/README.Debian for a list of permitted 15 | # modules. 16 | # Additionally, the CLI API of this file should remain as stable as possible, 17 | # to not introduce odd behavior if a container image wasn't updated and is used 18 | # with a newer debspawn version. 19 | 20 | import os 21 | import pwd 22 | import sys 23 | import time 24 | import subprocess 25 | from glob import glob 26 | from argparse import ArgumentParser 27 | from contextlib import contextmanager 28 | 29 | # the user performing builds in the container 30 | BUILD_USER = 'builder' 31 | 32 | # the directory where we build a package 33 | BUILD_DIR = '/srv/build' 34 | 35 | # additional packages to be used when building 36 | EXTRAPKG_DIR = '/srv/extra-packages' 37 | 38 | # directory that may or may not be exist, but must never be written to 39 | INVALID_DIR = '/run/invalid' 40 | 41 | 42 | # 43 | # Globals 44 | # 45 | 46 | unicode_enabled = True 47 | color_enabled = True 48 | 49 | 50 | def run_command(cmd, env=None, *, check=True): 51 | if isinstance(cmd, str): 52 | cmd = cmd.split(' ') 53 | 54 | proc_env = env 55 | if proc_env: 56 | proc_env = os.environ.copy() 57 | proc_env.update(env) 58 | 59 | p = subprocess.run(cmd, env=proc_env, check=False) 60 | if p.returncode != 0: 61 | if check: 62 | print('Command `{}` failed.'.format(' '.join(cmd)), 63 | file=sys.stderr) 64 | sys.exit(p.returncode) 65 | else: 66 | return False 67 | return True 68 | 69 | 70 | def run_apt_command(cmd): 71 | if isinstance(cmd, str): 72 | cmd = cmd.split(' ') 73 | 74 | env = {'DEBIAN_FRONTEND': 'noninteractive'} 75 | apt_cmd = ['apt-get', 76 | '-uyq', 77 | '-o Dpkg::Options::="--force-confnew"'] 78 | apt_cmd.extend(cmd) 79 | 80 | if cmd == 'update': 81 | # retry an apt update a few times, to protect a bit against bad 82 | # network connections or a flaky mirror / internal build queue repo 83 | for i in range(0, 3): 84 | is_last = i == 2 85 | if run_command(apt_cmd, env, check=is_last): 86 | break 87 | print('APT update failed, retrying...') 88 | time.sleep(5) 89 | else: 90 | run_command(apt_cmd, env) 91 | 92 | 93 | def print_textbox(title, tl, hline, tr, vline, bl, br): 94 | def write_utf8(s): 95 | sys.stdout.buffer.write(s.encode('utf-8')) 96 | 97 | tlen = len(title) 98 | write_utf8('\n{}'.format(tl)) 99 | write_utf8(hline * (10 + tlen)) 100 | write_utf8('{}\n'.format(tr)) 101 | 102 | write_utf8('{} {}'.format(vline, title)) 103 | write_utf8(' ' * 8) 104 | write_utf8('{}\n'.format(vline)) 105 | 106 | write_utf8(bl) 107 | write_utf8(hline * (10 + tlen)) 108 | write_utf8('{}\n'.format(br)) 109 | 110 | sys.stdout.flush() 111 | 112 | 113 | def print_header(title): 114 | if unicode_enabled: 115 | print_textbox(title, '╔', '═', '╗', '║', '╚', '╝') 116 | else: 117 | print_textbox(title, '+', '═', '+', '|', '+', '+') 118 | 119 | 120 | def print_section(title): 121 | if unicode_enabled: 122 | print_textbox(title, '┌', '─', '┐', '│', '└', '┘') 123 | else: 124 | print_textbox(title, '+', '-', '+', '|', '+', '+') 125 | 126 | 127 | @contextmanager 128 | def eatmydata(): 129 | try: 130 | # FIXME: We just override the env vars here, appending to them would 131 | # be much cleaner. 132 | os.environ['LD_LIBRARY_PATH'] = '/usr/lib/libeatmydata' 133 | os.environ['LD_PRELOAD'] = 'libeatmydata.so' 134 | yield 135 | finally: 136 | del os.environ['LD_LIBRARY_PATH'] 137 | del os.environ['LD_PRELOAD'] 138 | 139 | 140 | def ensure_no_nonexistent_dirs(): 141 | nonexistent_dirs = ('/nonexistent', INVALID_DIR) 142 | for path in nonexistent_dirs: 143 | if os.path.exists(path): 144 | if os.geteuid() == 0: 145 | run_command('rm -r {}'.format(path)) 146 | continue 147 | 148 | if path == INVALID_DIR: 149 | # ensure invalid dir has no permissions 150 | try: 151 | os.chmod(INVALID_DIR, 0o000) 152 | except PermissionError: 153 | print('WARNING: Directory {} exists and is writable.'.format(INVALID_DIR), 154 | file=sys.stderr) 155 | else: 156 | print('WARNING: Directory {} exists and can not be removed!'.format(path), 157 | file=sys.stderr) 158 | 159 | 160 | def drop_privileges(): 161 | import grp 162 | 163 | if os.geteuid() != 0: 164 | return 165 | 166 | builder_gid = grp.getgrnam(BUILD_USER).gr_gid 167 | builder_uid = pwd.getpwnam(BUILD_USER).pw_uid 168 | os.setgroups([]) 169 | os.setgid(builder_gid) 170 | os.setuid(builder_uid) 171 | 172 | 173 | def update_container(builder_uid): 174 | with eatmydata(): 175 | run_apt_command('update') 176 | run_apt_command('full-upgrade') 177 | 178 | run_apt_command(['install', '--no-install-recommends', 179 | 'build-essential', 'dpkg-dev', 'fakeroot', 'eatmydata']) 180 | 181 | run_apt_command(['--purge', 'autoremove']) 182 | run_apt_command('clean') 183 | 184 | try: 185 | pwd.getpwnam(BUILD_USER) 186 | except KeyError: 187 | print('No "{}" user, creating it.'.format(BUILD_USER)) 188 | run_command('useradd -M -f -1 -d {} --uid {} {}' 189 | .format(INVALID_DIR, builder_uid, BUILD_USER)) 190 | 191 | run_command('mkdir -p /srv/build') 192 | run_command('chown {} /srv/build'.format(BUILD_USER)) 193 | 194 | # ensure the non existent directory is gone even if it was 195 | # created accidentally 196 | ensure_no_nonexistent_dirs() 197 | 198 | return True 199 | 200 | 201 | def prepare_run(): 202 | print_section('Preparing container') 203 | 204 | with eatmydata(): 205 | run_apt_command('update') 206 | run_apt_command('full-upgrade') 207 | 208 | return True 209 | 210 | 211 | def _generate_hashes(filename): 212 | import hashlib 213 | 214 | hash_md5 = hashlib.md5() 215 | hash_sha256 = hashlib.sha256() 216 | file_size = 0 217 | 218 | with open(filename, 'rb') as f: 219 | for chunk in iter(lambda: f.read(4096), b''): 220 | hash_md5.update(chunk) 221 | hash_sha256.update(chunk) 222 | file_size = f.tell() 223 | 224 | return hash_md5.hexdigest(), hash_sha256.hexdigest(), file_size 225 | 226 | 227 | def prepare_package_build(arch_only=False, suite=None): 228 | from datetime import datetime, timezone 229 | 230 | print_section('Preparing container for build') 231 | 232 | with eatmydata(): 233 | run_apt_command('update') 234 | run_apt_command('full-upgrade') 235 | run_apt_command(['install', '--no-install-recommends', 236 | 'build-essential', 'dpkg-dev', 'fakeroot']) 237 | 238 | # check if we have extra packages to register with APT 239 | if os.path.exists(EXTRAPKG_DIR) and os.path.isdir(EXTRAPKG_DIR): 240 | if os.listdir(EXTRAPKG_DIR): 241 | with eatmydata(): 242 | run_apt_command(['install', '--no-install-recommends', 'apt-utils']) 243 | print() 244 | print('Using injected packages as additional APT package source.') 245 | 246 | packages_index_fname = os.path.join(EXTRAPKG_DIR, 'Packages') 247 | os.chdir(EXTRAPKG_DIR) 248 | with open(packages_index_fname, 'wt') as f: 249 | proc = subprocess.Popen(['apt-ftparchive', 250 | 'packages', 251 | '.'], 252 | cwd=EXTRAPKG_DIR, 253 | stdout=f) 254 | ret = proc.wait() 255 | if ret != 0: 256 | print('ERROR: Unable to generate temporary APT repository for injected packages.', 257 | file=sys.stderr) 258 | sys.exit(2) 259 | 260 | with open(os.path.join(EXTRAPKG_DIR, 'Release'), 'wt') as f: 261 | release_tmpl = '''Archive: local-pkg-inject 262 | Origin: LocalInjected 263 | Label: LocalInjected 264 | Acquire-By-Hash: no 265 | Component: main 266 | Date: {date} 267 | MD5Sum: 268 | {md5_hash} {size} Packages 269 | SHA256: 270 | {sha256_hash} {size} Packages 271 | ''' 272 | md5_hash, sha256_hash, size = _generate_hashes(packages_index_fname) 273 | f.write(release_tmpl.format( 274 | date=datetime.now(timezone.utc).strftime('%a, %d %b %Y %H:%M:%S +0000'), 275 | md5_hash=md5_hash, 276 | sha256_hash=sha256_hash, 277 | size=size) 278 | ) 279 | 280 | with open('/etc/apt/sources.list', 'a') as f: 281 | f.write('deb [trusted=yes] file://{} ./\n'.format(EXTRAPKG_DIR)) 282 | 283 | with eatmydata(): 284 | # make APT aware of the new packages, update base packages if needed 285 | run_apt_command('update') 286 | run_apt_command('full-upgrade') 287 | 288 | # ensure we are in our build directory at this point 289 | os.chdir(BUILD_DIR) 290 | 291 | run_command('chown -R {} /srv/build'.format(BUILD_USER)) 292 | for f in glob('./*'): 293 | if os.path.isdir(f): 294 | os.chdir(f) 295 | break 296 | 297 | print_section('Installing package build-dependencies') 298 | with eatmydata(): 299 | cmd = ['build-dep'] 300 | if arch_only: 301 | cmd.append('--arch-only') 302 | cmd.append('./') 303 | run_apt_command(cmd) 304 | 305 | return True 306 | 307 | 308 | def build_package(buildflags=None, suite=None): 309 | drop_privileges() 310 | print_section('Build') 311 | 312 | os.chdir(BUILD_DIR) 313 | for f in glob('./*'): 314 | if os.path.isdir(f): 315 | os.chdir(f) 316 | break 317 | 318 | cmd = ['dpkg-buildpackage'] 319 | if suite: 320 | cmd.append('--changes-option=-DDistribution={}'.format(suite)) 321 | if buildflags: 322 | cmd.extend(buildflags) 323 | run_command(cmd) 324 | 325 | # run_command will exit the whole program if the command failed, 326 | # so we can return True here (everything went fine if we are here) 327 | return True 328 | 329 | 330 | def run_qatasks(qa_lintian=True): 331 | ''' Run QA tasks on a built package immediately after build (currently Lintian) ''' 332 | os.chdir(BUILD_DIR) 333 | for f in glob('./*'): 334 | if os.path.isdir(f): 335 | os.chdir(f) 336 | break 337 | 338 | if qa_lintian: 339 | print_section('QA: Prepare') 340 | 341 | if qa_lintian: 342 | # install Lintian if Lintian check was requested 343 | run_apt_command(['install', 'lintian']) 344 | 345 | print_section('QA: Lintian') 346 | 347 | drop_privileges() 348 | cmd = ['lintian', 349 | '-I', # infos by default 350 | '--pedantic', # pedantic hints by default, 351 | '--tag-display-limit', '0', # display all tags found (even if that may be a lot occasionally) 352 | ] 353 | run_command(cmd) 354 | 355 | # run_command will exit the whole program if the command failed, 356 | # so we can return True here (everything went fine if we are here) 357 | return True 358 | 359 | 360 | def setup_environment(builder_uid=None, use_color=True, use_unicode=True, *, is_update=False): 361 | os.environ['LANG'] = 'C.UTF-8' if use_unicode else 'C' 362 | os.environ['LC_ALL'] = 'C.UTF-8' if use_unicode else 'C' 363 | os.environ['HOME'] = '/nonexistent' 364 | 365 | os.environ['TERM'] = 'xterm-256color' if use_color else 'xterm-mono' 366 | os.environ['SHELL'] = '/bin/sh' 367 | 368 | del os.environ['LOGNAME'] 369 | 370 | # ensure no directories exists that shouldn't be there 371 | ensure_no_nonexistent_dirs() 372 | 373 | if builder_uid and builder_uid > 0 and os.geteuid() == 0: 374 | # we are root and got a UID to change the BUILD_USER to 375 | try: 376 | pwd.getpwnam(BUILD_USER) 377 | except KeyError: 378 | if not is_update: 379 | print('WARNING: No "{}" user found in this container!'.format(BUILD_USER), 380 | file=sys.stderr) 381 | return 382 | run_command('usermod -u {} {}'.format(builder_uid, BUILD_USER)) 383 | 384 | 385 | def main(): 386 | if not os.environ.get('container'): 387 | print('This helper script must be run in a systemd-nspawn container.') 388 | return 1 389 | 390 | parser = ArgumentParser(description='Debspawn helper script') 391 | 392 | parser.add_argument('--no-color', action='store_true', dest='no_color', 393 | help='Disable terminal colors.') 394 | parser.add_argument('--no-unicode', action='store_true', dest='no_unicode', 395 | help='Disable unicode support.') 396 | parser.add_argument('--buid', action='store', type=int, dest='builder_uid', 397 | help='Designated UID of the build user within the container.') 398 | 399 | parser.add_argument('--update', action='store_true', dest='update', 400 | help='Initialize the container.') 401 | parser.add_argument('--arch-only', action='store_true', dest='arch_only', default=None, 402 | help='Only get arch-dependent packages (used when satisfying build dependencies).') 403 | parser.add_argument('--build-prepare', action='store_true', dest='build_prepare', 404 | help='Prepare building a Debian package.') 405 | parser.add_argument('--build-run', action='store_true', dest='build_run', 406 | help='Build a Debian package.') 407 | parser.add_argument('--lintian', action='store_true', dest='qa_lintian', 408 | help='Run Lintian on the generated package.') 409 | parser.add_argument('--buildflags', action='store', dest='buildflags', default=None, 410 | help='Flags passed to dpkg-buildpackage.') 411 | parser.add_argument('--suite', action='store', dest='suite', default=None, 412 | help='The suite we are building for (may be inferred if not set).') 413 | parser.add_argument('--prepare-run', action='store_true', dest='prepare_run', 414 | help='Prepare container image for generic script run.') 415 | parser.add_argument('--run-qa', action='store_true', dest='run_qatasks', 416 | help='Run QA tasks (only Lintian currently) against a package.') 417 | 418 | options = parser.parse_args(sys.argv[1:]) 419 | 420 | # initialize environment defaults 421 | global unicode_enabled, color_enabled 422 | unicode_enabled = not options.no_unicode 423 | color_enabled = not options.no_color 424 | setup_environment(options.builder_uid, 425 | color_enabled, 426 | unicode_enabled, 427 | is_update=options.update) 428 | 429 | if options.update: 430 | r = update_container(options.builder_uid) 431 | if not r: 432 | return 2 433 | elif options.build_prepare: 434 | r = prepare_package_build(options.arch_only, options.suite) 435 | if not r: 436 | return 2 437 | elif options.build_run: 438 | buildflags = [] 439 | if options.buildflags: 440 | buildflags = [s.strip('\'" ') for s in options.buildflags.split(';')] 441 | r = build_package(buildflags, options.suite) 442 | if not r: 443 | return 2 444 | elif options.prepare_run: 445 | r = prepare_run() 446 | if not r: 447 | return 2 448 | elif options.run_qatasks: 449 | r = run_qatasks(qa_lintian=options.qa_lintian) 450 | if not r: 451 | return 2 452 | else: 453 | print('ERROR: No action specified.', file=sys.stderr) 454 | return 1 455 | 456 | return 0 457 | 458 | 459 | if __name__ == '__main__': 460 | sys.exit(main()) 461 | -------------------------------------------------------------------------------- /debspawn/injectpkg.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (C) 2019-2022 Matthias Klumpp 4 | # 5 | # Licensed under the GNU Lesser General Public License Version 3 6 | # 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Lesser General Public License as published by 9 | # the Free Software Foundation, either version 3 of the license, or 10 | # (at your option) any later version. 11 | # 12 | # This software is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Lesser General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Lesser General Public License 18 | # along with this software. If not, see . 19 | 20 | import os 21 | from glob import glob 22 | from pathlib import Path 23 | from contextlib import contextmanager 24 | 25 | from .utils import temp_dir, print_info, hardlink_or_copy 26 | 27 | 28 | class PackageInjector: 29 | ''' 30 | Inject packages from external sources into the container APT environment. 31 | ''' 32 | 33 | def __init__(self, osbase): 34 | self._pkgs_basedir = osbase.global_config.injected_pkgs_dir 35 | self._pkgs_specific_dir = os.path.join(self._pkgs_basedir, osbase.name) 36 | self._has_injectables = None 37 | self._instance_repo_dir = None 38 | 39 | def has_injectables(self): 40 | '''Return True if we actually have any packages ready to inject''' 41 | 42 | if type(self._has_injectables) is bool: 43 | return self._has_injectables 44 | 45 | if os.path.exists(self._pkgs_basedir) and os.path.isdir(self._pkgs_basedir): 46 | for f in os.listdir(self._pkgs_basedir): 47 | if os.path.isfile(os.path.join(self._pkgs_basedir, f)): 48 | self._has_injectables = True 49 | return True 50 | 51 | if os.path.exists(self._pkgs_specific_dir) and os.path.isdir(self._pkgs_specific_dir): 52 | for f in os.listdir(self._pkgs_specific_dir): 53 | if os.path.isfile(os.path.join(self._pkgs_specific_dir, f)): 54 | self._has_injectables = True 55 | return True 56 | 57 | self._has_injectables = False 58 | return False 59 | 60 | def create_instance_repo(self, tmp_repo_dir): 61 | ''' 62 | Create a temporary location where all injected packages for this container 63 | are copied to. 64 | ''' 65 | 66 | Path(self._pkgs_basedir).mkdir(parents=True, exist_ok=True) 67 | Path(tmp_repo_dir).mkdir(parents=True, exist_ok=True) 68 | 69 | print_info('Copying injected packages to instance location') 70 | self._instance_repo_dir = tmp_repo_dir 71 | 72 | # copy/link injected packages specific to this environment 73 | if os.path.isdir(self._pkgs_specific_dir): 74 | for pkg_fname in glob(os.path.join(self._pkgs_specific_dir, '*.deb')): 75 | pkg_path = os.path.join(tmp_repo_dir, os.path.basename(pkg_fname)) 76 | 77 | if not os.path.isfile(pkg_path): 78 | hardlink_or_copy(pkg_fname, pkg_path) 79 | 80 | # copy/link injected packages used by all environments 81 | for pkg_fname in glob(os.path.join(self._pkgs_basedir, '*.deb')): 82 | pkg_path = os.path.join(tmp_repo_dir, os.path.basename(pkg_fname)) 83 | 84 | if not os.path.isfile(pkg_path): 85 | hardlink_or_copy(pkg_fname, pkg_path) 86 | 87 | @property 88 | def instance_repo_dir(self) -> str: 89 | return self._instance_repo_dir 90 | 91 | 92 | @contextmanager 93 | def package_injector(osbase, machine_name=None): 94 | ''' 95 | Create a package injector as context manager and make 96 | it create a new temporary instance repo. 97 | ''' 98 | 99 | if not machine_name: 100 | from random import choice 101 | from string import digits, ascii_lowercase 102 | 103 | nid = ''.join(choice(ascii_lowercase + digits) for _ in range(4)) 104 | machine_name = '{}-{}'.format(osbase.name, nid) 105 | 106 | pi = PackageInjector(osbase) 107 | if not pi.has_injectables(): 108 | yield pi 109 | else: 110 | with temp_dir('pkginject-' + machine_name) as injectrepo_tmp: 111 | pi.create_instance_repo(injectrepo_tmp) 112 | yield pi 113 | -------------------------------------------------------------------------------- /debspawn/maintain.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (C) 2018-2022 Matthias Klumpp 4 | # 5 | # Licensed under the GNU Lesser General Public License Version 3 6 | # 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Lesser General Public License as published by 9 | # the Free Software Foundation, either version 3 of the license, or 10 | # (at your option) any later version. 11 | # 12 | # This software is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Lesser General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Lesser General Public License 18 | # along with this software. If not, see . 19 | 20 | import os 21 | import sys 22 | import json 23 | import shutil 24 | from glob import glob 25 | 26 | from .config import GlobalConfig 27 | from .osbase import OSBase 28 | from .utils.env import ensure_root 29 | from .utils.log import ( 30 | print_info, 31 | print_warn, 32 | print_error, 33 | print_bullet, 34 | print_section, 35 | print_bool_item, 36 | ) 37 | 38 | 39 | def ensure_rmtree_symattack_protection(): 40 | '''Exit the program with an error if rmtree does not protect against symlink attacks''' 41 | if not shutil.rmtree.avoids_symlink_attacks: 42 | print_error('Will not continue: rmtree does not run in symlink-attack protected mode.') 43 | sys.exit(1) 44 | 45 | 46 | def maintain_migrate(gconf: GlobalConfig): 47 | '''Migrate configuration from older versions of debspawn to the latest version''' 48 | 49 | ensure_root() 50 | 51 | # migrate old container images directory, if needed 52 | images_dir = gconf.osroots_dir 53 | if not os.path.isdir(images_dir): 54 | old_images_dir = '/var/lib/debspawn/containers' 55 | if os.path.isdir(old_images_dir): 56 | print_info('Migrating images directory...') 57 | shutil.move(old_images_dir, images_dir) 58 | 59 | 60 | def maintain_clear_caches(gconf: GlobalConfig): 61 | '''Delete all cache data for all images''' 62 | 63 | ensure_root() 64 | ensure_rmtree_symattack_protection() 65 | 66 | aptcache_dir = gconf.aptcache_dir 67 | if os.path.isdir(aptcache_dir): 68 | for cdir in glob(os.path.join(aptcache_dir, '*')): 69 | print_info('Removing APT cache for: {}'.format(os.path.basename(cdir))) 70 | if os.path.isdir(cdir): 71 | shutil.rmtree(cdir) 72 | else: 73 | os.remove(cdir) 74 | 75 | dcache_dir = os.path.join(gconf.osroots_dir, 'dcache') 76 | if os.path.isdir(dcache_dir): 77 | print_info('Removing image derivatives cache.') 78 | shutil.rmtree(dcache_dir) 79 | 80 | 81 | def maintain_purge(gconf: GlobalConfig, force: bool = False): 82 | '''Remove all images as well as any data associated with them''' 83 | 84 | ensure_root() 85 | ensure_rmtree_symattack_protection() 86 | 87 | if not force: 88 | print_warn( 89 | ( 90 | 'This action will delete ALL your images as well as their configuration, build results and other ' 91 | 'associated data and will clear all data from the directories you may have configured as default.' 92 | ) 93 | ) 94 | delete_all = False 95 | while True: 96 | try: 97 | in_res = input('Do you really want to continue? [y/N]: ') 98 | except EOFError: 99 | in_res = 'n' 100 | print() 101 | if not in_res: 102 | delete_all = False 103 | break 104 | elif in_res.lower() == 'y': 105 | delete_all = True 106 | break 107 | elif in_res.lower() == 'n': 108 | delete_all = False 109 | break 110 | 111 | if not delete_all: 112 | print_info('Purge action aborted.') 113 | return 114 | 115 | print_warn('Deleting all images, image configuration, build results and state data.') 116 | for sdir in [gconf.osroots_dir, gconf.results_dir, gconf.aptcache_dir, gconf.injected_pkgs_dir]: 117 | if not os.path.isdir(sdir): 118 | continue 119 | if sdir.startswith('/home/') or sdir.startswith('/usr/'): 120 | continue 121 | print_info('Purging: {}'.format(sdir)) 122 | for d in glob(os.path.join(sdir, '*')): 123 | if os.path.isdir(d): 124 | shutil.rmtree(d) 125 | else: 126 | os.remove(d) 127 | 128 | default_state_dir = '/var/lib/debspawn/' 129 | if os.path.isdir(default_state_dir): 130 | print_info('Removing: {}'.format(default_state_dir)) 131 | shutil.rmtree(default_state_dir) 132 | 133 | 134 | def maintain_update_all(gconf: GlobalConfig): 135 | '''Update all container images that we know.''' 136 | 137 | ensure_root() 138 | 139 | osroots_dir = gconf.osroots_dir 140 | tar_files = [] 141 | if os.path.isdir(osroots_dir): 142 | tar_files = list(glob(os.path.join(osroots_dir, '*.tar.zst'))) 143 | if not tar_files: 144 | print_info('No container base images have been found!') 145 | return 146 | 147 | failed_images = [] 148 | nodata_images = [] 149 | first_entry = True 150 | for tar_fname in tar_files: 151 | img_basepath = os.path.splitext(os.path.splitext(tar_fname)[0])[0] 152 | config_fname = img_basepath + '.json' 153 | imgid = os.path.basename(img_basepath) 154 | 155 | # read configuration data 156 | if not os.path.isfile(config_fname): 157 | nodata_images.append(imgid) 158 | continue 159 | 160 | with open(config_fname, 'rt') as f: 161 | cdata = json.loads(f.read()) 162 | 163 | if not first_entry: 164 | print() 165 | first_entry = False 166 | print_bullet('Update: {}'.format(imgid), indent=1, large=True) 167 | 168 | osbase = OSBase( 169 | gconf, 170 | cdata['Suite'], 171 | cdata['Architecture'], 172 | cdata.get('Variant'), 173 | custom_name=os.path.basename(img_basepath), 174 | ) 175 | r = osbase.update() 176 | if not r: 177 | print_error('Failed to update {}'.format(imgid)) 178 | failed_images.append(imgid) 179 | 180 | if nodata_images or failed_images: 181 | print() 182 | for imgid in nodata_images: 183 | print_warn('Could not auto-update image {}: Configuration data is missing.'.format(imgid)) 184 | if failed_images: 185 | print_error('Failed to update image(s): {}'.format(', '.join(failed_images))) 186 | sys.exit(1) 187 | 188 | 189 | def maintain_print_status(gconf: GlobalConfig): 190 | ''' 191 | Print status information about this Debspawn installation 192 | that may be useful for debugging issues. 193 | ''' 194 | import platform 195 | 196 | from . import __version__ 197 | from .nspawn import systemd_version, systemd_detect_virt 198 | from .osbase import bootstrap_tool_version, print_container_base_image_info 199 | 200 | print('Debspawn Status Report', end='') 201 | sys.stdout.flush() 202 | 203 | # read distribution information 204 | os_release = {} 205 | if os.path.exists('/etc/os-release'): 206 | with open('/etc/os-release') as f: 207 | for line in f: 208 | k, v = line.rstrip().split("=") 209 | os_release[k] = v.strip('"') 210 | 211 | print_section('Host System') 212 | print('OS:', os_release.get('NAME', 'Unknown'), os_release.get('VERSION', '')) 213 | print('Platform:', platform.platform(aliased=True)) 214 | print('Virtualization:', systemd_detect_virt()) 215 | print('Systemd-nspawn version:', systemd_version()) 216 | print('Bootstrap tool:', '{} {}'.format(gconf.bootstrap_tool, bootstrap_tool_version(gconf))) 217 | 218 | print_section('Container image list') 219 | print_container_base_image_info(gconf) 220 | 221 | print_section('Debspawn') 222 | print('Version:', __version__) 223 | print_bool_item( 224 | 'Tmpfiles.d configuration:', 225 | os.path.isfile('/usr/lib/tmpfiles.d/debspawn.conf'), 226 | text_true='installed', 227 | text_false='missing', 228 | ) 229 | print_bool_item( 230 | 'Monthly cache cleanup timer:', 231 | os.path.isfile('/lib/systemd/system/debspawn-clear-caches.timer'), 232 | text_true='available', 233 | text_false='missing', 234 | ) 235 | print_bool_item( 236 | 'Manual pages:', 237 | len(glob('/usr/share/man/man1/debspawn*.1.*')) >= 8, 238 | text_true='installed', 239 | text_false='missing', 240 | ) 241 | if not os.path.isfile('/etc/debspawn/global.toml'): 242 | print('Global configuration: default') 243 | else: 244 | print('Global configuration:') 245 | with open('/etc/debspawn/global.toml', 'r') as f: 246 | for line in f: 247 | print(' ', line) 248 | -------------------------------------------------------------------------------- /debspawn/nspawn.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (C) 2018-2022 Matthias Klumpp 4 | # 5 | # Licensed under the GNU Lesser General Public License Version 3 6 | # 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Lesser General Public License as published by 9 | # the Free Software Foundation, either version 3 of the license, or 10 | # (at your option) any later version. 11 | # 12 | # This software is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Lesser General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Lesser General Public License 18 | # along with this software. If not, see . 19 | 20 | import os 21 | import typing as T 22 | import platform 23 | import subprocess 24 | 25 | from .utils import ( 26 | safe_run, 27 | temp_dir, 28 | print_info, 29 | print_warn, 30 | print_error, 31 | run_forwarded, 32 | ) 33 | from .injectpkg import PackageInjector 34 | from .utils.env import unicode_allowed, colored_output_allowed 35 | from .utils.command import run_command 36 | 37 | __systemd_version = None 38 | 39 | 40 | def systemd_version(): 41 | global __systemd_version 42 | if __systemd_version: 43 | return __systemd_version 44 | 45 | __systemd_version = -1 46 | try: 47 | out, _, _ = safe_run(['systemd-nspawn', '--version']) 48 | parts = out.split(' ', 2) 49 | if len(parts) >= 2: 50 | __systemd_version = int(parts[1]) 51 | except Exception as e: 52 | print_warn('Unable to determine systemd version: {}'.format(e)) 53 | 54 | return __systemd_version 55 | 56 | 57 | def systemd_detect_virt(): 58 | vm_name = 'unknown' 59 | try: 60 | out, _, _ = run_command(['systemd-detect-virt']) 61 | vm_name = out.strip() 62 | except Exception as e: 63 | print_warn('Unable to determine current virtualization: {}'.format(e)) 64 | 65 | return vm_name 66 | 67 | 68 | def systemd_version_atleast(expected_version: int): 69 | v = systemd_version() 70 | # we always assume we are running the highest version, 71 | # if we failed to determine the right systemd version 72 | if v < 0: 73 | return True 74 | if v >= expected_version: 75 | return True 76 | return False 77 | 78 | 79 | def get_nspawn_personality(osbase): 80 | ''' 81 | Return the syszemd-nspawn container personality for the given combination 82 | of host architecture and base OS. 83 | This allows running x86 builds on amd64 machines. 84 | ''' 85 | import fnmatch 86 | 87 | if platform.machine() == 'x86_64' and fnmatch.filter([osbase.arch], 'i?86'): 88 | return 'x86' 89 | return None 90 | 91 | 92 | def _execute_sdnspawn( 93 | osbase, 94 | parameters, 95 | machine_name, 96 | *, 97 | boot: bool = False, 98 | allow_permissions: list[str] = None, 99 | syscall_filter: list[str] = None, 100 | env_vars: dict[str, str] = None, 101 | private_users: bool = False, 102 | nowait: bool = False, 103 | ) -> T.Union[subprocess.CompletedProcess, subprocess.Popen]: 104 | ''' 105 | Execute systemd-nspawn with the given parameters. 106 | Mess around with cgroups if necessary. 107 | ''' 108 | import sys 109 | 110 | if not allow_permissions: 111 | allow_permissions = [] 112 | if not syscall_filter: 113 | syscall_filter = [] 114 | if not env_vars: 115 | env_vars = {} 116 | 117 | capabilities = [] 118 | full_dev_access = False 119 | full_proc_access = False 120 | ro_kmods_access = False 121 | kvm_access = False 122 | all_privileges = False 123 | for perm in allow_permissions: 124 | perm = perm.lower() 125 | if perm.startswith('cap_') or perm == 'all': 126 | if perm == 'all': 127 | capabilities.append(perm) 128 | print_warn('Container retains all privileges.') 129 | all_privileges = True 130 | else: 131 | capabilities.append(perm.upper()) 132 | elif perm == 'full-dev': 133 | full_dev_access = True 134 | elif perm == 'full-proc': 135 | full_proc_access = True 136 | elif perm == 'read-kmods': 137 | ro_kmods_access = True 138 | elif perm == 'kvm': 139 | kvm_access = True 140 | else: 141 | print_info('Unknown allowed permission: {}'.format(perm)) 142 | 143 | if ( 144 | capabilities or full_dev_access or full_proc_access or kvm_access 145 | ) and not osbase.global_config.allow_unsafe_perms: 146 | print_error( 147 | 'Configuration does not permit usage of additional and potentially dangerous permissions. Exiting.' 148 | ) 149 | sys.exit(9) 150 | 151 | cmd = ['systemd-nspawn'] 152 | cmd.extend(['-M', machine_name]) 153 | if boot: 154 | # if we boot the container, we also register it with machinectl, otherwise 155 | # we run an unregistered container with the command as PID2 156 | cmd.append('-b') 157 | cmd.append('--notify-ready=yes') 158 | else: 159 | cmd.append('--register=no') 160 | cmd.append('-a') 161 | if private_users: 162 | cmd.append('-U') # User namespaces with --private-users=pick --private-users-chown, if possible 163 | 164 | # never try to bindmount /etc/localtime 165 | cmd.append('--timezone=copy') 166 | 167 | if full_dev_access: 168 | cmd.extend(['--bind', '/dev']) 169 | if systemd_version_atleast(244): 170 | cmd.append('--console=pipe') 171 | cmd.extend(['--property=DeviceAllow=block-* rw', '--property=DeviceAllow=char-* rw']) 172 | if kvm_access and not full_dev_access: 173 | if os.path.exists('/dev/kvm'): 174 | cmd.extend(['--bind', '/dev/kvm']) 175 | cmd.extend(['--property=DeviceAllow=/dev/kvm rw']) 176 | else: 177 | print_warn( 178 | 'Access to KVM requested, but /dev/kvm does not exist on the host. Is virtualization supported?' 179 | ) 180 | if full_proc_access: 181 | cmd.extend(['--bind', '/proc']) 182 | if not all_privileges: 183 | print_warn('Container has access to host /proc') 184 | if ro_kmods_access: 185 | cmd.extend(['--bind-ro', '/lib/modules/']) 186 | cmd.extend(['--bind-ro', '/boot/']) 187 | if capabilities: 188 | cmd.extend(['--capability', ','.join(capabilities)]) 189 | if syscall_filter: 190 | cmd.extend(['--system-call-filter', ' '.join(syscall_filter)]) 191 | 192 | for v_name, v_value in env_vars.items(): 193 | cmd.extend(['-E', '{}={}'.format(v_name, v_value)]) 194 | 195 | # add custom parameters 196 | cmd.extend(parameters) 197 | 198 | if nowait: 199 | return subprocess.Popen(cmd, shell=False, stdin=subprocess.DEVNULL) 200 | else: 201 | return run_forwarded(cmd) 202 | 203 | 204 | def nspawn_run_persist( 205 | osbase, 206 | base_dir, 207 | machine_name, 208 | chdir, 209 | command: T.Union[list[str], str] = None, 210 | flags: T.Union[list[str], str] = None, 211 | *, 212 | tmp_apt_cache_dir: str = None, 213 | pkginjector: PackageInjector = None, 214 | allowed: list[str] = None, 215 | syscall_filter: list[str] = None, 216 | env_vars: dict[str, str] = None, 217 | private_users: bool = False, 218 | boot: bool = False, 219 | verbose: bool = False, 220 | ): 221 | if isinstance(command, str): 222 | command = command.split(' ') 223 | elif not command: 224 | command = [] 225 | if isinstance(flags, str): 226 | flags = flags.split(' ') 227 | elif not flags: 228 | flags = [] 229 | 230 | personality = get_nspawn_personality(osbase) 231 | 232 | def run_nspawn_with_aptcache(aptcache_tmp_dir): 233 | params = [ 234 | '--chdir={}'.format(chdir), 235 | '--link-journal=no', 236 | ] 237 | if aptcache_tmp_dir: 238 | params.append('--bind={}:/var/cache/apt/archives/'.format(aptcache_tmp_dir)) 239 | if pkginjector and pkginjector.instance_repo_dir: 240 | params.append('--bind={}:/srv/extra-packages/'.format(pkginjector.instance_repo_dir)) 241 | 242 | if personality: 243 | params.append('--personality={}'.format(personality)) 244 | params.extend(flags) 245 | params.extend(['-{}D'.format('' if verbose else 'q'), base_dir]) 246 | 247 | # nspawn can not run a command in a booted container on its own 248 | if not boot: 249 | params.extend(command) 250 | sdns_nowait = boot and command 251 | 252 | # ensure the temporary apt cache is up-to-date 253 | if aptcache_tmp_dir: 254 | osbase.aptcache.create_instance_cache(aptcache_tmp_dir) 255 | 256 | # run command in container 257 | ns_proc = _execute_sdnspawn( 258 | osbase, 259 | params, 260 | machine_name, 261 | allow_permissions=allowed, 262 | syscall_filter=syscall_filter, 263 | env_vars=env_vars, 264 | private_users=private_users, 265 | boot=boot, 266 | nowait=sdns_nowait, 267 | ) 268 | 269 | if not sdns_nowait: 270 | ret = ns_proc.returncode 271 | else: 272 | try: 273 | import time 274 | 275 | # the container is (hopefully) running now, but let's check for that 276 | time_ac_start = time.time() 277 | container_booted = False 278 | while (time.time() - time_ac_start) < 60: 279 | scisr_out, _, _ = run_command( 280 | [ 281 | 'systemd-run', 282 | '-GP', 283 | '--wait', 284 | '-qM', 285 | machine_name, 286 | 'systemctl', 287 | 'is-system-running', 288 | ] 289 | ) 290 | 291 | # check if we are actually running, try again later if not 292 | if scisr_out.strip() in ('running', 'degraded'): 293 | print() 294 | container_booted = True 295 | break 296 | time.sleep(0.5) 297 | 298 | if container_booted: 299 | sdr_cmd = [ 300 | 'systemd-run', 301 | '-GP', 302 | '--wait', 303 | '-qM', 304 | machine_name, 305 | '--working-directory', 306 | chdir, 307 | ] + command 308 | proc = run_forwarded(sdr_cmd) 309 | ret = proc.returncode 310 | else: 311 | ret = 7 312 | print_error('Timed out while waiting for the container to boot.') 313 | finally: 314 | run_forwarded(['machinectl', 'poweroff', machine_name]) 315 | try: 316 | ns_proc.wait(30) 317 | except subprocess.TimeoutExpired: 318 | ns_proc.terminate() 319 | 320 | # archive APT cache, so future runs of this command are faster (unless disabled in configuration) 321 | if aptcache_tmp_dir: 322 | osbase.aptcache.merge_from_dir(aptcache_tmp_dir) 323 | 324 | return ret 325 | 326 | if not osbase.cache_packages: 327 | # APT package caching was explicitly disabled by the user 328 | ret = run_nspawn_with_aptcache(None) 329 | elif tmp_apt_cache_dir: 330 | # we will be reusing an externally provided temporary APT cache directory 331 | ret = run_nspawn_with_aptcache(tmp_apt_cache_dir) 332 | else: 333 | # we will create our own temporary APT cache dir 334 | with temp_dir('aptcache-' + machine_name) as aptcache_tmp: 335 | ret = run_nspawn_with_aptcache(aptcache_tmp) 336 | 337 | return ret 338 | 339 | 340 | def nspawn_run_ephemeral( 341 | osbase, 342 | base_dir, 343 | machine_name, 344 | chdir, 345 | command: T.Union[list[str], str] = None, 346 | flags: T.Union[list[str], str] = None, 347 | allowed: list[str] = None, 348 | syscall_filter: list[str] = None, 349 | env_vars: dict[str, str] = None, 350 | private_users: bool = False, 351 | boot: bool = False, 352 | ): 353 | if isinstance(command, str): 354 | command = command.split(' ') 355 | elif not command: 356 | command = [] 357 | if isinstance(flags, str): 358 | flags = flags.split(' ') 359 | elif not flags: 360 | flags = [] 361 | 362 | personality = get_nspawn_personality(osbase) 363 | 364 | params = ['--chdir={}'.format(chdir), '--link-journal=no'] 365 | if personality: 366 | params.append('--personality={}'.format(personality)) 367 | params.extend(flags) 368 | params.extend(['-qxD', base_dir]) 369 | params.extend(command) 370 | 371 | return _execute_sdnspawn( 372 | osbase, 373 | params, 374 | machine_name, 375 | allow_permissions=allowed, 376 | syscall_filter=syscall_filter, 377 | env_vars=env_vars, 378 | private_users=private_users, 379 | ).returncode 380 | 381 | 382 | def nspawn_make_helper_cmd(flags, build_uid: int): 383 | if isinstance(flags, str): 384 | flags = flags.split(' ') 385 | 386 | cmd = ['/usr/lib/debspawn/dsrun'] 387 | if not colored_output_allowed(): 388 | cmd.append('--no-color') 389 | if not unicode_allowed(): 390 | cmd.append('--no-unicode') 391 | if build_uid > 0: 392 | cmd.append('--buid={}'.format(build_uid)) 393 | 394 | cmd.extend(flags) 395 | return cmd 396 | 397 | 398 | def nspawn_run_helper_ephemeral( 399 | osbase, 400 | base_dir, 401 | machine_name, 402 | helper_flags, 403 | chdir='/tmp', 404 | *, 405 | build_uid: int, 406 | nspawn_flags: T.Union[list[str], str] = None, 407 | allowed: list[str] = None, 408 | env_vars: dict[str, str] = None, 409 | private_users: bool = False, 410 | ): 411 | cmd = nspawn_make_helper_cmd(helper_flags, build_uid) 412 | return nspawn_run_ephemeral( 413 | base_dir, 414 | machine_name, 415 | chdir, 416 | cmd, 417 | flags=nspawn_flags, 418 | allowed=allowed, 419 | env_vars=env_vars, 420 | private_users=private_users, 421 | ) 422 | 423 | 424 | def nspawn_run_helper_persist( 425 | osbase, 426 | base_dir, 427 | machine_name, 428 | helper_flags, 429 | chdir='/tmp', 430 | *, 431 | build_uid: int, 432 | nspawn_flags: T.Union[list[str], str] = None, 433 | tmp_apt_cache_dir=None, 434 | pkginjector=None, 435 | allowed: list[str] = None, 436 | syscall_filter: list[str] = None, 437 | env_vars: dict[str, str] = None, 438 | private_users: bool = False, 439 | ): 440 | cmd = nspawn_make_helper_cmd(helper_flags, build_uid) 441 | return nspawn_run_persist( 442 | osbase, 443 | base_dir, 444 | machine_name, 445 | chdir, 446 | cmd, 447 | nspawn_flags, 448 | tmp_apt_cache_dir=tmp_apt_cache_dir, 449 | pkginjector=pkginjector, 450 | allowed=allowed, 451 | syscall_filter=syscall_filter, 452 | env_vars=env_vars, 453 | private_users=private_users, 454 | ) 455 | -------------------------------------------------------------------------------- /debspawn/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the GNU Lesser General Public License Version 3 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU Lesser General Public License as published by 7 | # the Free Software Foundation, either version 3 of the license, or 8 | # (at your option) any later version. 9 | # 10 | # This software is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU Lesser General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU Lesser General Public License 16 | # along with this software. If not, see . 17 | 18 | from .env import unicode_allowed, colored_output_allowed 19 | from .log import print_info, print_warn, print_error, print_header, print_section 20 | from .misc import ( 21 | cd, 22 | listify, 23 | temp_dir, 24 | rmtree_mntsafe, 25 | systemd_escape, 26 | format_filesize, 27 | hardlink_or_copy, 28 | ) 29 | from .command import safe_run, run_forwarded 30 | 31 | __all__ = [ 32 | 'print_info', 33 | 'print_warn', 34 | 'print_error', 35 | 'print_header', 36 | 'print_section', 37 | 'colored_output_allowed', 38 | 'unicode_allowed', 39 | 'safe_run', 40 | 'run_forwarded', 41 | 'listify', 42 | 'temp_dir', 43 | 'cd', 44 | 'hardlink_or_copy', 45 | 'format_filesize', 46 | 'rmtree_mntsafe', 47 | 'systemd_escape', 48 | ] 49 | -------------------------------------------------------------------------------- /debspawn/utils/command.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (C) 2016-2022 Matthias Klumpp 4 | # Copyright (C) 2012-2013 Paul Tagliamonte 5 | # 6 | # Licensed under the GNU Lesser General Public License Version 3 7 | # 8 | # This program is free software: you can redistribute it and/or modify 9 | # it under the terms of the GNU Lesser General Public License as published by 10 | # the Free Software Foundation, either version 3 of the license, or 11 | # (at your option) any later version. 12 | # 13 | # This software is distributed in the hope that it will be useful, 14 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | # GNU Lesser General Public License for more details. 17 | # 18 | # You should have received a copy of the GNU Lesser General Public License 19 | # along with this software. If not, see . 20 | 21 | import sys 22 | import shlex 23 | import subprocess 24 | 25 | from .log import TwoStreamLogger 26 | 27 | 28 | class SubprocessError(Exception): 29 | def __init__(self, out, err, ret, cmd): 30 | self.out = out 31 | self.err = err 32 | self.ret = ret 33 | self.cmd = cmd 34 | super(SubprocessError, self).__init__('%s: %d\n%s' % (str(self.cmd), self.ret, str(self.err))) 35 | 36 | def __str__(self): 37 | return '%s: %d\n%s' % (str(self.cmd), self.ret, str(self.err)) 38 | 39 | 40 | # Input may be a byte string, a unicode string, or a file-like object 41 | def run_command(command, input=None): 42 | if not isinstance(command, list): 43 | command = shlex.split(command) 44 | 45 | if not input: 46 | input = None 47 | elif isinstance(input, str): 48 | input = input.encode('utf-8') 49 | elif not isinstance(input, bytes): 50 | input = input.read() 51 | 52 | try: 53 | pipe = subprocess.Popen( 54 | command, 55 | shell=False, 56 | stdin=subprocess.PIPE, 57 | stdout=subprocess.PIPE, 58 | stderr=subprocess.PIPE, 59 | ) 60 | except OSError: 61 | return (None, None, -1) 62 | 63 | (output, stderr) = pipe.communicate(input=input) 64 | (output, stderr) = (c.decode('utf-8', errors='ignore') for c in (output, stderr)) 65 | return (output, stderr, pipe.returncode) 66 | 67 | 68 | def safe_run(cmd, input=None, expected=0): 69 | if not isinstance(expected, tuple): 70 | expected = (expected,) 71 | 72 | out, err, ret = run_command(cmd, input=input) 73 | 74 | if ret not in expected: 75 | raise SubprocessError(out, err, ret, cmd) 76 | 77 | return out, err, ret 78 | 79 | 80 | def run_forwarded(command): 81 | ''' 82 | Run a command, forwarding all output to the current stdout as well as to 83 | our build-logger in case we have one set previously. 84 | ''' 85 | if not isinstance(command, list): 86 | command = shlex.split(command) 87 | 88 | if isinstance(sys.stdout, TwoStreamLogger): 89 | proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) 90 | # ensure output is written to our file as well as stdout (as sys.stdout may be a redirect) 91 | while True: 92 | line = proc.stdout.readline() 93 | if proc.poll() is not None: 94 | break 95 | sys.stdout.write(str(line, 'utf-8', 'replace')) 96 | return proc 97 | else: 98 | return subprocess.run(command, check=False) 99 | -------------------------------------------------------------------------------- /debspawn/utils/env.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (C) 2017-2022 Matthias Klumpp 4 | # 5 | # Licensed under the GNU Lesser General Public License Version 3 6 | # 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Lesser General Public License as published by 9 | # the Free Software Foundation, either version 3 of the license, or 10 | # (at your option) any later version. 11 | # 12 | # This software is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Lesser General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Lesser General Public License 18 | # along with this software. If not, see . 19 | 20 | import os 21 | import sys 22 | import shutil 23 | from contextlib import contextmanager 24 | 25 | _unicode_allowed = True # store whether we are allowed to use unicode 26 | _owner_uid = 0 # uid of the user on whose behalf we are running 27 | _owner_gid = 0 # gid of the user on whose behalf we are running 28 | 29 | 30 | def set_owning_user(user, group=None): 31 | ''' 32 | Set the user on whose behalf we are running. 33 | This is useful so we can drop privileges to 34 | the perticular user in many cases. 35 | ''' 36 | from grp import getgrnam 37 | from pwd import getpwnam, getpwuid 38 | 39 | if user.isdecimal(): 40 | uid = int(user) 41 | else: 42 | uid = getpwnam(user).pw_uid 43 | 44 | if not group: 45 | gid = getpwuid(uid).pw_gid 46 | elif group.isdecimal(): 47 | gid = int(group) 48 | else: 49 | gid = getgrnam(group).gr_gid 50 | 51 | global _owner_uid 52 | global _owner_gid 53 | _owner_uid = uid 54 | _owner_gid = gid 55 | 56 | 57 | def ensure_root(): 58 | ''' 59 | Ensure we are running as root and all code following 60 | this function is privileged. 61 | ''' 62 | if os.geteuid() == 0: 63 | return 64 | 65 | args = sys.argv.copy() 66 | 67 | owner_set = any(a.startswith('--owner=') for a in sys.argv) 68 | if owner_set: 69 | # we don't override an owner explicitly set by the user 70 | args = sys.argv.copy() 71 | else: 72 | args = [sys.argv[0]] 73 | 74 | # set flag to tell the new process who it can impersonate 75 | # for unprivileged actions. It it is root, just omit the flag. 76 | uid = os.getuid() 77 | gid = os.getgid() 78 | if uid != 0 or gid != 0: 79 | args.append('--owner={}:{}'.format(uid, gid)) 80 | args.extend(sys.argv[1:]) 81 | 82 | def filter_env_far(result, name): 83 | value = os.environ.get(name) 84 | if not value: 85 | return 86 | result.append('{}={}'.format(name, shlex.quote(value))) 87 | 88 | if shutil.which('sudo'): 89 | # Filter "good" environment variables that we want to have after running sudo. 90 | # Most of those are standard variables affecting debsign bahevior later, in case 91 | # the user has requested signing 92 | import shlex 93 | 94 | env = [] 95 | filter_env_far(env, 'DEBEMAIL') 96 | filter_env_far(env, 'DEBFULLNAME') 97 | filter_env_far(env, 'GPGKEY') 98 | filter_env_far(env, 'GPG_AGENT_INFO') 99 | filter_env_far(env, 'HTTP_PROXY') 100 | filter_env_far(env, 'HTTPS_PROXY') 101 | filter_env_far(env, 'http_proxy') 102 | filter_env_far(env, 'https_proxy') 103 | 104 | os.execvp("sudo", ["sudo"] + env + args) 105 | else: 106 | print('This command needs to be run as root.') 107 | sys.exit(1) 108 | 109 | 110 | @contextmanager 111 | def switch_unprivileged(): 112 | ''' 113 | Run actions using the unprivileged user ID 114 | on the behalf of which we are running. 115 | This is NOT a security feature! 116 | ''' 117 | import pwd 118 | 119 | if _owner_uid == 0 and _owner_gid == 0: 120 | # we can't really do much here, we have to run 121 | # as root, as we don't know an unprivileged user 122 | # to switch to 123 | 124 | yield 125 | else: 126 | orig_egid = os.getegid() 127 | orig_euid = os.geteuid() 128 | orig_home = os.environ.get('HOME') 129 | if not orig_home: 130 | orig_home = pwd.getpwuid(os.getuid()).pw_dir 131 | 132 | try: 133 | os.setegid(_owner_gid) 134 | os.seteuid(_owner_uid) 135 | os.environ['HOME'] = pwd.getpwuid(_owner_uid).pw_dir 136 | 137 | yield 138 | finally: 139 | os.setegid(orig_egid) 140 | os.seteuid(orig_euid) 141 | os.environ['HOME'] = orig_home 142 | 143 | 144 | def get_owner_uid_gid(): 145 | return _owner_uid, _owner_gid 146 | 147 | 148 | def get_random_free_uid_gid(): 149 | '''Get a random unused UID and GID for the current system.''' 150 | import pwd 151 | import random 152 | 153 | uid = 1000 154 | gid = 1000 155 | for pw in pwd.getpwall(): 156 | if pw.pw_name == 'nobody': 157 | continue 158 | if pw.pw_uid > uid: 159 | uid = pw.pw_uid 160 | if pw.pw_gid > gid: 161 | gid = pw.pw_gid 162 | # we can not use an extremely large number here, as otherwise the container's 163 | # lastlog/faillog will grow to insane sizes 164 | r = random.randint(100, 2048) 165 | return uid + r, gid + r 166 | 167 | 168 | def colored_output_allowed(): 169 | return (hasattr(sys.stdout, "isatty") and sys.stdout.isatty()) or ( 170 | 'TERM' in os.environ and os.environ['TERM'] == 'ANSI' 171 | ) 172 | 173 | 174 | def unicode_allowed(): 175 | return _unicode_allowed 176 | 177 | 178 | def set_unicode_allowed(val): 179 | global _unicode_allowed 180 | _unicode_allowed = val 181 | 182 | 183 | def get_free_space(path): 184 | ''' 185 | Return free space of :path 186 | ''' 187 | real_path = os.path.realpath(path) 188 | stat = os.statvfs(real_path) 189 | # get free space in MiB. 190 | free_space = float(stat.f_bsize * stat.f_bavail) 191 | return free_space 192 | 193 | 194 | def get_tree_size(path): 195 | ''' 196 | Return total size of files in path and subdirs. If 197 | is_dir() or stat() fails, print an error message to stderr 198 | and assume zero size (for example, file has been deleted). 199 | ''' 200 | total = 0 201 | for entry in os.scandir(path): 202 | try: 203 | is_dir = entry.is_dir(follow_symlinks=False) 204 | except OSError as error: 205 | print('Error calling is_dir():', error, file=sys.stderr) 206 | continue 207 | if is_dir: 208 | total += get_tree_size(entry.path) 209 | else: 210 | try: 211 | total += entry.stat(follow_symlinks=False).st_size 212 | except OSError as error: 213 | print('Error calling stat():', error, file=sys.stderr) 214 | return total 215 | -------------------------------------------------------------------------------- /debspawn/utils/log.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (C) 2017-2022 Matthias Klumpp 4 | # 5 | # Licensed under the GNU Lesser General Public License Version 3 6 | # 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Lesser General Public License as published by 9 | # the Free Software Foundation, either version 3 of the license, or 10 | # (at your option) any later version. 11 | # 12 | # This software is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Lesser General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Lesser General Public License 18 | # along with this software. If not, see . 19 | 20 | import os 21 | import re 22 | import sys 23 | 24 | from .env import unicode_allowed 25 | from .misc import safe_copy 26 | 27 | 28 | def console_supports_color(): 29 | ''' 30 | Returns True if the running system's terminal supports color, and False 31 | otherwise. 32 | ''' 33 | 34 | is_a_tty = hasattr(sys.stdout, 'isatty') and sys.stdout.isatty() 35 | return 'ANSICON' in os.environ or is_a_tty 36 | 37 | 38 | def print_textbox(title, tl, hline, tr, vline, bl, br): 39 | def write_utf8(s): 40 | sys.stdout.buffer.write(s.encode('utf-8')) 41 | 42 | tlen = len(title) 43 | write_utf8('\n{}'.format(tl)) 44 | write_utf8(hline * (10 + tlen)) 45 | write_utf8('{}\n'.format(tr)) 46 | 47 | write_utf8('{} {}'.format(vline, title)) 48 | write_utf8(' ' * 8) 49 | write_utf8('{}\n'.format(vline)) 50 | 51 | write_utf8(bl) 52 | write_utf8(hline * (10 + tlen)) 53 | write_utf8('{}\n'.format(br)) 54 | 55 | sys.stdout.flush() 56 | 57 | 58 | def print_header(title): 59 | if unicode_allowed(): 60 | print_textbox(title, '╔', '═', '╗', '║', '╚', '╝') 61 | else: 62 | print_textbox(title, '+', '═', '+', '|', '+', '+') 63 | 64 | 65 | def print_section(title): 66 | if unicode_allowed(): 67 | print_textbox(title, '┌', '─', '┐', '│', '└', '┘') 68 | else: 69 | print_textbox(title, '+', '-', '+', '|', '+', '+') 70 | 71 | 72 | def print_info(*arg): 73 | ''' 74 | Prints an information message and ensures that it shows up on 75 | stdout immediately. 76 | ''' 77 | print(*arg) 78 | sys.stdout.flush() 79 | 80 | 81 | def print_warn(*arg): 82 | ''' 83 | Prints an information message and ensures that it shows up on 84 | stdout immediately. 85 | ''' 86 | if console_supports_color(): 87 | print('\033[93m/!\\\033[0m', *arg) 88 | else: 89 | print('/!\\', *arg) 90 | sys.stdout.flush() 91 | 92 | 93 | def print_error(*arg): 94 | ''' 95 | Prints an information message and ensures that it shows up on 96 | stdout immediately. 97 | ''' 98 | if console_supports_color(): 99 | print('\033[91mERROR:\033[0m', *arg, file=sys.stderr) 100 | else: 101 | print('ERROR:', *arg, file=sys.stderr) 102 | sys.stderr.flush() 103 | 104 | 105 | def print_bullet(*arg, large: bool = False, indent: int = 0): 106 | ''' 107 | Prints a bullet point to the console, with a set 108 | indentation and style. 109 | ''' 110 | if unicode_allowed(): 111 | b = '●' if large else '•' 112 | else: 113 | b = '*' 114 | print((' ' * indent) + b, *arg) 115 | 116 | 117 | def print_bool_item(prefix: str, b: bool, text_true: str = 'yes', text_false: str = 'no'): 118 | ''' 119 | Prints a (colored, if possible) boolean item with a given prefix. 120 | ''' 121 | if console_supports_color(): 122 | s = '\033[92m{}\033[0m'.format(text_true) if b else '\033[91m{}\033[0m'.format(text_false) 123 | else: 124 | s = text_true if b else text_false 125 | if prefix: 126 | print(prefix, s) 127 | else: 128 | print(prefix) 129 | sys.stdout.flush() 130 | 131 | 132 | def input_bool(question_text, default=False) -> bool: 133 | """As user a Yes/No question.""" 134 | if default: 135 | default_info = '[Y/n]' 136 | else: 137 | default_info = '[y/N]' 138 | while True: 139 | try: 140 | in_str = input('{} {}:'.format(question_text, default_info)) 141 | except EOFError: 142 | return default 143 | if in_str == 'y' or in_str == 'Y': 144 | return True 145 | elif in_str == 'n' or in_str == 'N': 146 | return False 147 | elif not in_str: 148 | return default 149 | 150 | 151 | class TwoStreamLogger: 152 | ''' 153 | Permits logging messages to stdout/stderr as well as to a file. 154 | ''' 155 | 156 | class Buffer: 157 | def __init__(self, fstream, cstream): 158 | self._fstream = fstream 159 | self._cstream = cstream 160 | 161 | def write(self, message): 162 | self._fstream.write(str(message, 'utf-8', 'replace')) 163 | self._cstream.buffer.write(message) 164 | 165 | def __init__(self, fstream, cstream, fflush_always=False): 166 | self._fstream = fstream 167 | self._cstream = cstream 168 | self._fflush_always = fflush_always 169 | self._colorsub = re.compile('\x1b\\[(K|.*?m)') 170 | self.buffer = TwoStreamLogger.Buffer(fstream, cstream) 171 | 172 | def write(self, message): 173 | # write message to console 174 | self._cstream.write(message) 175 | if self._fflush_always: 176 | self.flush() 177 | 178 | # write message to file, stripping ANSI colors 179 | self._fstream.write(self._colorsub.sub('', message)) 180 | 181 | def flush(self): 182 | self._cstream.flush() 183 | self._fstream.flush() 184 | 185 | def copy_to(self, fname): 186 | self.flush() 187 | safe_copy(self._fstream.name, fname, preserve_mtime=False) 188 | 189 | def isatty(self): 190 | return self._cstream.isatty() 191 | 192 | 193 | def capture_console_output(): 194 | ''' 195 | Direct console output to a file as well 196 | as to the original stdout/stderr terminal. 197 | ''' 198 | from tempfile import NamedTemporaryFile 199 | 200 | logfile = NamedTemporaryFile(mode='a', prefix='ds_', suffix='.log') 201 | nstdout = TwoStreamLogger(logfile, sys.stdout) 202 | nstderr = TwoStreamLogger(logfile, sys.stderr, True) 203 | 204 | sys.stdout = nstdout 205 | sys.stderr = nstderr 206 | 207 | 208 | def save_captured_console_output(fname): 209 | from .env import get_owner_uid_gid 210 | 211 | if hasattr(sys.stdout, 'copy_to'): 212 | o_uid, o_gid = get_owner_uid_gid() 213 | sys.stdout.copy_to(fname) 214 | os.chown(fname, o_uid, o_gid) 215 | -------------------------------------------------------------------------------- /debspawn/utils/misc.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (C) 2017-2022 Matthias Klumpp 4 | # 5 | # Licensed under the GNU Lesser General Public License Version 3 6 | # 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Lesser General Public License as published by 9 | # the Free Software Foundation, either version 3 of the license, or 10 | # (at your option) any later version. 11 | # 12 | # This software is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Lesser General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Lesser General Public License 18 | # along with this software. If not, see . 19 | 20 | import os 21 | import sys 22 | import stat 23 | import fcntl 24 | import shutil 25 | import typing as T 26 | import subprocess 27 | from pathlib import Path 28 | from contextlib import contextmanager 29 | 30 | from ..config import GlobalConfig 31 | 32 | 33 | class MountError(Exception): 34 | """Error while dealing with mountpoints.""" 35 | 36 | 37 | def listify(item: T.Any): 38 | ''' 39 | Return a list of :item, unless :item already is a lit. 40 | ''' 41 | if not item: 42 | return [] 43 | return item if isinstance(item, list) else [item] 44 | 45 | 46 | @contextmanager 47 | def cd(where): 48 | ncwd = os.getcwd() 49 | try: 50 | yield os.chdir(where) 51 | finally: 52 | os.chdir(ncwd) 53 | 54 | 55 | def random_string(prefix: T.Optional[str] = None, count: int = 8): 56 | ''' 57 | Create a string of random alphanumeric characters of a given length, 58 | separated with a hyphen from an optional prefix. 59 | ''' 60 | 61 | from random import choice 62 | from string import digits, ascii_lowercase 63 | 64 | if count <= 0: 65 | count = 1 66 | rdm_id = ''.join(choice(ascii_lowercase + digits) for _ in range(count)) 67 | if prefix: 68 | return '{}-{}'.format(prefix, rdm_id) 69 | return rdm_id 70 | 71 | 72 | def systemd_escape(name: str) -> T.Optional[str]: 73 | '''Escape a string using systemd's escaping rules.''' 74 | from .command import run_command 75 | 76 | out, _, ret = run_command(['systemd-escape', name]) 77 | if ret != 0: 78 | return None 79 | return out.strip() 80 | 81 | 82 | @contextmanager 83 | def temp_dir(basename=None): 84 | '''Context manager for a temporary directory in debspawn's temp-dir location. 85 | 86 | This function will also ensure that we will not jump into possibly still 87 | bind-mounted directories upon deletion, and will unmount those directories 88 | instead. 89 | ''' 90 | 91 | dir_name = random_string(basename) 92 | temp_basedir = GlobalConfig().temp_dir 93 | if not temp_basedir: 94 | temp_basedir = '/var/tmp/debspawn/' 95 | 96 | tmp_path = os.path.join(temp_basedir, dir_name) 97 | Path(tmp_path).mkdir(parents=True, exist_ok=True) 98 | 99 | fd = os.open(tmp_path, os.O_RDONLY) 100 | # we hold a shared lock on the directory to prevent systemd-tmpfiles 101 | # from deleting it, just in case we are building something for days 102 | try: 103 | if fd > 0: 104 | fcntl.flock(fd, fcntl.LOCK_SH | fcntl.LOCK_NB) 105 | except (IOError, OSError): 106 | print('WARNING: Unable to lock temporary directory {}'.format(tmp_path), file=sys.stderr) 107 | sys.stderr.flush() 108 | 109 | try: 110 | yield tmp_path 111 | finally: 112 | try: 113 | fcntl.flock(fd, fcntl.LOCK_UN) 114 | rmtree_mntsafe(tmp_path) 115 | finally: 116 | if fd > 0: 117 | os.close(fd) 118 | 119 | 120 | def safe_copy(src, dst, *, preserve_mtime: bool = True): 121 | ''' 122 | Attempt to safely copy a file, by atomically replacing the destination and 123 | protecting against symlink attacks. 124 | ''' 125 | dst_tmp = random_string(dst + '.tmp') 126 | try: 127 | if preserve_mtime: 128 | shutil.copy2(src, dst_tmp) 129 | else: 130 | shutil.copy(src, dst_tmp) 131 | if os.path.islink(dst): 132 | os.remove(dst) 133 | os.replace(dst_tmp, dst) 134 | finally: 135 | try: 136 | os.remove(dst_tmp) 137 | except OSError: 138 | pass 139 | 140 | 141 | def maybe_remove(f): 142 | '''Delete a file if it exists, but do nothing if it doesn't.''' 143 | try: 144 | os.remove(f) 145 | except OSError: 146 | pass 147 | 148 | 149 | def format_filesize(num, suffix='B'): 150 | for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']: 151 | if abs(num) < 1024.0: 152 | return "%3.1f%s%s" % (num, unit, suffix) 153 | num /= 1024.0 154 | return "%.1f%s%s" % (num, 'Yi', suffix) 155 | 156 | 157 | def current_time_string(): 158 | '''Get the current time as human-readable string.''' 159 | 160 | from datetime import datetime, timezone 161 | 162 | utc_dt = datetime.now(timezone.utc) 163 | return utc_dt.astimezone().strftime('%Y-%m-%d %H:%M:%S UTC%z') 164 | 165 | 166 | def version_noepoch(version): 167 | '''Return version from :version without epoch.''' 168 | 169 | version_noe = version 170 | if ':' in version_noe: 171 | version_noe = version_noe.split(':', 1)[1] 172 | return version_noe 173 | 174 | 175 | def hardlink_or_copy(src, dst): 176 | '''Hardlink a file :src to :dst or copy the file in case linking is not possible''' 177 | 178 | try: 179 | os.link(src, dst) 180 | except (PermissionError, OSError): 181 | shutil.copy2(src, dst) 182 | 183 | 184 | def is_mountpoint(path) -> bool: 185 | '''Check if :path is a mountpoint. 186 | 187 | Unlike os.path.ismount, this function will also consider 188 | bindmountpoints. 189 | This function may be slow 190 | ''' 191 | 192 | if not os.path.exists(path): 193 | return False 194 | if os.path.ismount(path): 195 | return True 196 | 197 | ret = subprocess.run(['findmnt', '-M', str(path)], capture_output=True, check=False) 198 | if ret.returncode == 0: 199 | return True 200 | return False 201 | 202 | 203 | def bindmount(from_path, to_path): 204 | '''Create a bindmount point.''' 205 | 206 | cmd = ['mount', '--bind', from_path, to_path] 207 | ret = subprocess.run(cmd, capture_output=True, check=False) 208 | if ret.returncode != 0: 209 | raise MountError('Unable to create bindmount {} -> {}'.format(from_path, to_path)) 210 | 211 | 212 | def umount(path, lazy: bool = True): 213 | '''Try to unmount a path.''' 214 | 215 | cmd = ['umount'] 216 | if lazy: 217 | cmd.append('-l') 218 | cmd.append(path) 219 | ret = subprocess.run(cmd, capture_output=True, check=False) 220 | if ret.returncode != 0: 221 | raise MountError('Unable to umount path {}'.format(path)) 222 | 223 | # try again if the mountpoint is still there, as 224 | # overmounting may have happened 225 | if is_mountpoint(path): 226 | umount(path, lazy=lazy) 227 | 228 | 229 | def _rmtree_mntsafe_fd(topfd, path, onerror): 230 | try: 231 | with os.scandir(topfd) as scandir_it: 232 | entries = list(scandir_it) 233 | except OSError as err: 234 | err.filename = path 235 | onerror(os.scandir, path, sys.exc_info()) 236 | return 237 | for entry in entries: 238 | fullname = os.path.join(path, entry.name) 239 | try: 240 | is_dir = entry.is_dir(follow_symlinks=False) 241 | except OSError: 242 | is_dir = False 243 | else: 244 | if is_dir: 245 | try: 246 | orig_st = entry.stat(follow_symlinks=False) 247 | is_dir = stat.S_ISDIR(orig_st.st_mode) 248 | except OSError: 249 | onerror(os.lstat, fullname, sys.exc_info()) 250 | continue 251 | if is_dir: 252 | if is_mountpoint(fullname): 253 | try: 254 | umount(fullname) 255 | orig_st = os.stat(fullname, follow_symlinks=False) 256 | except Exception: 257 | onerror(umount, fullname, sys.exc_info()) 258 | continue 259 | 260 | try: 261 | dirfd = os.open(entry.name, os.O_RDONLY, dir_fd=topfd) 262 | except OSError: 263 | onerror(os.open, fullname, sys.exc_info()) 264 | else: 265 | try: 266 | if os.path.samestat(orig_st, os.fstat(dirfd)): 267 | _rmtree_mntsafe_fd(dirfd, fullname, onerror) 268 | try: 269 | os.rmdir(entry.name, dir_fd=topfd) 270 | except OSError: 271 | onerror(os.rmdir, fullname, sys.exc_info()) 272 | else: 273 | try: 274 | # This can only happen if someone replaces 275 | # a directory with a symlink after the call to 276 | # os.scandir or stat.S_ISDIR above. 277 | raise OSError('Cannot call rmtree on a symbolic link') 278 | except OSError: 279 | onerror(os.path.islink, fullname, sys.exc_info()) 280 | finally: 281 | os.close(dirfd) 282 | else: 283 | try: 284 | os.unlink(entry.name, dir_fd=topfd) 285 | except OSError: 286 | onerror(os.unlink, fullname, sys.exc_info()) 287 | 288 | 289 | def rmtree_mntsafe(path, ignore_errors=False, onerror=None): 290 | '''Recursively delete a directory tree, unmounting mount points if possible. 291 | This function is based on shutil.rmtree, but will not jump into mount points, 292 | but instead try to unmount them and if that fails leave them alone. 293 | This prevents data loss in case bindmounts were set carelessly. 294 | 295 | If ignore_errors is set, errors are ignored; otherwise, if onerror 296 | is set, it is called to handle the error with arguments (func, 297 | path, exc_info) where func is platform and implementation dependent; 298 | path is the argument to that function that caused it to fail; and 299 | exc_info is a tuple returned by sys.exc_info(). If ignore_errors 300 | is false and onerror is None, an exception is raised. 301 | ''' 302 | if ignore_errors: 303 | # pylint: disable=function-redefined 304 | def onerror(*args): 305 | pass 306 | 307 | elif onerror is None: 308 | # pylint: disable=misplaced-bare-raise 309 | def onerror(*args): 310 | raise 311 | 312 | # While the unsafe rmtree works fine on bytes, the fd based does not. 313 | if isinstance(path, bytes): 314 | path = os.fsdecode(path) 315 | 316 | if os.path.ismount(path): 317 | try: 318 | umount(path) 319 | except Exception: 320 | onerror(umount, path, sys.exc_info()) 321 | return 322 | 323 | # Note: To guard against symlink races, we use the standard 324 | # lstat()/open()/fstat() trick. 325 | try: 326 | orig_st = os.lstat(path) 327 | except Exception: 328 | onerror(os.lstat, path, sys.exc_info()) 329 | return 330 | try: 331 | fd = os.open(path, os.O_RDONLY) 332 | except Exception: 333 | onerror(os.open, path, sys.exc_info()) 334 | return 335 | try: 336 | if os.path.samestat(orig_st, os.fstat(fd)): 337 | _rmtree_mntsafe_fd(fd, path, onerror) 338 | try: 339 | os.rmdir(path) 340 | except OSError: 341 | onerror(os.rmdir, path, sys.exc_info()) 342 | else: 343 | try: 344 | # symlinks to directories are forbidden, see bug #1669 345 | raise OSError("Cannot call rmtree on a symbolic link") 346 | except OSError: 347 | onerror(os.path.islink, path, sys.exc_info()) 348 | finally: 349 | os.close(fd) 350 | -------------------------------------------------------------------------------- /debspawn/utils/zstd_tar.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (C) 2018-2022 Matthias Klumpp 4 | # 5 | # Licensed under the GNU Lesser General Public License Version 3 6 | # 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Lesser General Public License as published by 9 | # the Free Software Foundation, either version 3 of the license, or 10 | # (at your option) any later version. 11 | # 12 | # This software is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Lesser General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Lesser General Public License 18 | # along with this software. If not, see . 19 | 20 | import shutil 21 | 22 | from .command import run_command 23 | 24 | 25 | class TarError(Exception): 26 | """Generic error while using tar/zstd.""" 27 | 28 | 29 | def ensure_tar_zstd(): 30 | '''Check if the required binaries for compression are available''' 31 | 32 | if not shutil.which('zstd'): 33 | raise TarError( 34 | ( 35 | 'The "zsdt" binary was not found, we can not compress tarballs. ' 36 | 'Please install zstd to continue!' 37 | ) 38 | ) 39 | if not shutil.which('tar'): 40 | raise TarError( 41 | 'The "tar" binary was not found, we can not create tarballs. Please install tar to continue!' 42 | ) 43 | 44 | 45 | def compress_directory(dirname, tarname): 46 | '''Compress a directory to a given tarball''' 47 | 48 | cmd = ['tar', '-C', dirname, '-I', 'zstd', '-cf', tarname, '.'] 49 | 50 | out, err, ret = run_command(cmd) 51 | 52 | if ret != 0: 53 | raise TarError('Unable to create tarball "{}":\n{}{}'.format(tarname, out, err)) 54 | 55 | 56 | def decompress_tarball(tarname, dirname): 57 | '''Compress a directory to a given tarball''' 58 | 59 | cmd = ['tar', '-C', dirname, '-I', 'zstd', '-xf', tarname] 60 | 61 | out, err, ret = run_command(cmd) 62 | 63 | if ret != 0: 64 | raise TarError('Unable to decompress tarball "{}":\n{}{}'.format(tarname, out, err)) 65 | -------------------------------------------------------------------------------- /docs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lkhq/debspawn/756cdf76c6dc2db38527363ef98f42edf24e6504/docs/__init__.py -------------------------------------------------------------------------------- /docs/assemble_man.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (C) 2018-2022 Matthias Klumpp 5 | # 6 | # SPDX-License-Identifier: LGPL-3.0-or-later 7 | 8 | import os 9 | import sys 10 | from functools import reduce 11 | from subprocess import check_call 12 | from xml.sax.saxutils import escape as xml_escape 13 | 14 | sys.path.append("..") 15 | 16 | 17 | class DocbookEditor: 18 | def __init__(self): 19 | self._replacements = {} 20 | 21 | def add_substvar(self, name, replacement): 22 | self._replacements['@{}@'.format(name)] = replacement 23 | 24 | def register_command_flag_synopsis(self, actions, command_name): 25 | flags_text = '' 26 | flags_entries = '' 27 | for item in actions: 28 | options_text = xml_escape('|'.join(item.option_strings)) 29 | flags_text += '{}\n'.format(options_text) 30 | 31 | oid = item.option_strings[0] 32 | desc_text = None 33 | if oid == '-h': 34 | desc_text = 'Print brief help information about available commands.' 35 | if command_name != 'create': 36 | if oid == '--variant': 37 | desc_text = 'Set the variant of the selected image, that was used for bootstrapping.' 38 | elif oid == '-a': 39 | desc_text = 'The architecture of the base image that should be selected.' 40 | 41 | if not desc_text: 42 | desc_text = item.help 43 | desc_text = xml_escape(desc_text) 44 | 45 | if desc_text.startswith('CF|'): 46 | desc_text = desc_text[3:] 47 | desc_text = desc_text.replace('binary:', ':', 1) 48 | desc_text = desc_text.replace('arch:', ':', 1) 49 | desc_text = desc_text.replace('indep:', ':', 1) 50 | desc_text = desc_text.replace('source:', ':', 1) 51 | 52 | flags_entries += ''' 53 | {} 54 | 55 | 56 | {} 57 | 58 | 59 | '''.format( 60 | options_text, desc_text 61 | ) 62 | 63 | self.add_substvar('{}_FLAGS_SYNOPSIS'.format(command_name.upper()), flags_text) 64 | self.add_substvar('{}_FLAGS_ENTRIES'.format(command_name.upper()), flags_entries) 65 | 66 | def process_file(self, input_fname, output_fname): 67 | with open(input_fname, 'r') as f: 68 | template_content = f.read() 69 | 70 | result = reduce( 71 | lambda x, y: x.replace(y, self._replacements[y]), self._replacements, template_content 72 | ) 73 | 74 | with open(output_fname, 'w') as f: 75 | f.write(result) 76 | 77 | return output_fname 78 | 79 | 80 | def generate_docbook_pages(build_dir): 81 | from debspawn.cli import create_parser 82 | 83 | build_dir = os.path.abspath(build_dir) 84 | 85 | parser = create_parser() 86 | editor = DocbookEditor() 87 | editor.register_command_flag_synopsis(parser._get_optional_actions(), 'BASE') 88 | 89 | xml_manpages = [] 90 | xml_manpages.append(editor.process_file('docs/debspawn.1.xml', os.path.join(build_dir, 'debspawn.1.xml'))) 91 | 92 | for command, sp in parser._get_positional_actions()[0]._name_parser_map.items(): 93 | editor.register_command_flag_synopsis(sp._get_optional_actions(), command) 94 | template_fname = 'docs/debspawn-{}.1.xml'.format(command) 95 | if not os.path.isfile(template_fname): 96 | if command in ['ls', 'b']: 97 | continue # the ls and b shorthands need to manual page 98 | print('Manual page template {} is missing! Skipping it.'.format(template_fname)) 99 | continue 100 | 101 | xml_manpages.append( 102 | editor.process_file(template_fname, os.path.join(build_dir, os.path.basename(template_fname))) 103 | ) 104 | 105 | return xml_manpages 106 | 107 | 108 | def create_manpage(xml_src, out_dir): 109 | man_name = os.path.splitext(os.path.basename(xml_src))[0] 110 | out_fname = os.path.join(out_dir, man_name) 111 | 112 | print('Generating manual page {}'.format(man_name)) 113 | check_call( 114 | [ 115 | 'xsltproc', 116 | '--nonet', 117 | '--stringparam', 118 | 'man.output.quietly', 119 | '1', 120 | '--stringparam', 121 | 'funcsynopsis.style', 122 | 'ansi', 123 | '--stringparam', 124 | 'man.th.extra1.suppress', 125 | '1', 126 | '-o', 127 | out_fname, 128 | 'http://docbook.sourceforge.net/release/xsl/current/manpages/docbook.xsl', 129 | xml_src, 130 | ] 131 | ) 132 | return out_fname 133 | 134 | 135 | if __name__ == '__main__': 136 | generate_docbook_pages('/tmp') 137 | -------------------------------------------------------------------------------- /docs/debspawn-build.1.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 August, 2018"> 6 | 7 | 8 | 9 | ]> 10 | 11 | 12 | 13 | &command; 14 | 15 | 2018-2022 16 | Matthias Klumpp 17 | 18 | Debspawn 19 | &date; 20 | 21 | 22 | 23 | &pagename; 24 | 1 25 | 26 | 27 | &pagename; 28 | Build Debian packages in a container 29 | 30 | 31 | 32 | 33 | &command; 34 | @BUILD_FLAGS_SYNOPSIS@ 35 | SUITE 36 | DIR|DSC_FILE 37 | 38 | 39 | 40 | 41 | Description 42 | 43 | Build a Debian package from a directory or source package *.dsc file. 44 | debspawn will create a new container for the respective build using the base image specified, 45 | build the package and return build artifacts in the default output directory /var/lib/debspawn/results/ unless 46 | a different location was specified via the flag. 47 | 48 | 49 | Downloaded packages that are build dependencies are cached and will be reused on subsequent builds if possible. 50 | 51 | 52 | You can inject packages into the build environment that are not available in the preconfigured APT repositories by 53 | placing them in /var/lib/debspawn/injected-pkgs/${container-name}, or in 54 | /var/lib/debspawn/injected-pkgs/ to make a package available in all environments. 55 | Internally, debspawn will build a transient package repository with the respective packages and 56 | add it as a package source for APT. 57 | 58 | 59 | If you want to debug the package build process, you can pass the flag to debspawn. This 60 | will open an interactive root shell in the build environment post-build, no matter whether the build failed or succeeded. 61 | After investigating the issue / building the package manually, the shell can be exited and the user is asked whether debspawn 62 | should copy back the changes made in the packages' debian/ directory to the host to make them permanent. 63 | Please keep in mind that while interactive mode is enabled, no build log can be created. 64 | 65 | 66 | 67 | 68 | Examples 69 | 70 | You can build a package from its source directory, or just by passing a plain .dsc file to &command;. 71 | If the result should be automatically signed, the flag needs to be passed too: 72 | 73 | 74 | $ cd ~/packages/hello 75 | $ &command; sid --sign 76 | 77 | $ &command; --arch=i386 cosmic ./hello_2.10-1.dsc 78 | 79 | 80 | You can also build packages using git-buildpackage and debspawn. In this case the 81 | flag is also used to perform a Lintian static analysis check in the container after build: 82 | 83 | 84 | $ gbp buildpackage --git-builder='debspawn b sid --lintian --sign' 85 | 86 | 87 | To debug a build issue interactively, the flag can be used: 88 | 89 | 90 | $ &command; sid --interact 91 | 92 | 93 | 94 | 95 | Options 96 | 97 | 98 | @BUILD_FLAGS_ENTRIES@ 99 | 100 | 101 | 102 | 103 | Differences to sbuild 104 | 105 | On Debian, sbuild is the primary tool used for package building, which uses different technology. 106 | So naturally, the question is whether the sbuild build environments and the debspawn build environments 107 | are be identical or at least compatible. 108 | 109 | 110 | Due to the different technology used, there may be subtle differences between sbuild chroots and 111 | debspawn containers. The differences should not have any impact on package builds, and any such occurrence is 112 | highly likely a bug in the package's build process. 113 | If you think it is not, please file a bug against Debspawn. We try to be as close to sbuild's default environment as possible, but 114 | unfortunately can not make any guarantees. 115 | 116 | 117 | One way the build environment of debspawn differs from Debian's default sbuild setup intentionally is 118 | in its consistent use of unicode. By default, debspawn will ensure that unicode is always available and enabled. 119 | If you do not want this behavior, you can pass the flag to &command; to disable unicode in 120 | the tool itself and in the build environment. 121 | 122 | 123 | 124 | 125 | See Also 126 | debspawn-update(1), debspawn-create(1), dpkg-buildpackage(1). 127 | 128 | 129 | AUTHOR 130 | 131 | This manual page was written by Matthias Klumpp mak@debian.org. 132 | 133 | 134 | 135 | -------------------------------------------------------------------------------- /docs/debspawn-create.1.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 August, 2018"> 6 | 7 | 8 | 9 | ]> 10 | 11 | 12 | 13 | &command; 14 | 15 | 2018-2022 16 | Matthias Klumpp 17 | 18 | Debspawn 19 | &date; 20 | 21 | 22 | 23 | &pagename; 24 | 1 25 | 26 | 27 | &pagename; 28 | Create new container images 29 | 30 | 31 | 32 | 33 | &command; 34 | @CREATE_FLAGS_SYNOPSIS@ 35 | NAME 36 | 37 | 38 | 39 | 40 | Description 41 | 42 | Create a new base image for a suite known to debootstrap(1). The image will later be used to spawn 43 | ephemeral containers in which packages can be built. 44 | 45 | 46 | 47 | 48 | Examples 49 | 50 | You can easily create images for any suite that has a script in debootstrap. For example, to create a Debian Unstable image for 51 | your current machine architecture, you can use: 52 | 53 | 54 | $ &command; unstable 55 | 56 | 57 | A more advanced example, for building on Ubuntu 18.10 on the x86 architecture: 58 | 59 | 60 | $ &command; --arch=i386 cosmic 61 | 62 | 63 | The suite name is inferred from the container image name given as positional parameter. If it can not be inferred, 64 | you will need to pass the parameter with the primary suite name for this image. 65 | If a is passed and no is set, the image name will automatically 66 | be assumed to be for an overlay suite, which may not always be the desired result. 67 | 68 | 69 | 70 | 71 | Options 72 | 73 | 74 | 75 | NAME 76 | The name of the container image to create (usually the name of the suite). 77 | 78 | 79 | @CREATE_FLAGS_ENTRIES@ 80 | 81 | 82 | 83 | 84 | See Also 85 | debspawn-build(1), debootstrap(1), systemd-nspawn(1). 86 | 87 | 88 | AUTHOR 89 | 90 | This manual page was written by Matthias Klumpp mak@debian.org. 91 | 92 | 93 | 94 | -------------------------------------------------------------------------------- /docs/debspawn-delete.1.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 August, 2018"> 6 | 7 | 8 | 9 | ]> 10 | 11 | 12 | 13 | &command; 14 | 15 | 2018-2022 16 | Matthias Klumpp 17 | 18 | Debspawn 19 | &date; 20 | 21 | 22 | 23 | &pagename; 24 | 1 25 | 26 | 27 | &pagename; 28 | Remove a container image 29 | 30 | 31 | 32 | 33 | &command; 34 | @DELETE_FLAGS_SYNOPSIS@ 35 | NAME 36 | 37 | 38 | 39 | 40 | Description 41 | 42 | Remove an image known to debspawn and clear all data related to it. This explicitly includes 43 | any cached data, but does not include generated build artifacts that may still exist in the results directory. 44 | 45 | 46 | 47 | 48 | Options 49 | 50 | 51 | 52 | NAME 53 | The name of the container image to delete (usually a distribution suite name). 54 | 55 | 56 | @DELETE_FLAGS_ENTRIES@ 57 | 58 | 59 | 60 | 61 | See Also 62 | debspawn-create(1). 63 | 64 | 65 | AUTHOR 66 | 67 | This manual page was written by Matthias Klumpp mak@debian.org. 68 | 69 | 70 | 71 | -------------------------------------------------------------------------------- /docs/debspawn-list.1.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 August, 2018"> 6 | 7 | 8 | 9 | ]> 10 | 11 | 12 | 13 | &command; 14 | 15 | 2018-2022 16 | Matthias Klumpp 17 | 18 | Debspawn 19 | &date; 20 | 21 | 22 | 23 | &pagename; 24 | 1 25 | 26 | 27 | &pagename; 28 | List information about container images 29 | 30 | 31 | 32 | 33 | &command; 34 | @LIST_FLAGS_SYNOPSIS@ 35 | SUITE 36 | 37 | 38 | 39 | 40 | Description 41 | 42 | This command will list detailed information about all currently registered container images that Debspawn can use 43 | as build environments. 44 | 45 | 46 | 47 | 48 | Options 49 | 50 | 51 | @LIST_FLAGS_ENTRIES@ 52 | 53 | 54 | 55 | 56 | See Also 57 | debspawn-create(1), debspawn-update(1). 58 | 59 | 60 | AUTHOR 61 | 62 | This manual page was written by Matthias Klumpp mak@debian.org. 63 | 64 | 65 | 66 | -------------------------------------------------------------------------------- /docs/debspawn-login.1.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 August, 2018"> 6 | 7 | 8 | 9 | ]> 10 | 11 | 12 | 13 | &command; 14 | 15 | 2018-2022 16 | Matthias Klumpp 17 | 18 | Debspawn 19 | &date; 20 | 21 | 22 | 23 | &pagename; 24 | 1 25 | 26 | 27 | &pagename; 28 | Open interactive shell session in a container 29 | 30 | 31 | 32 | 33 | &command; 34 | @LOGIN_FLAGS_SYNOPSIS@ 35 | NAME 36 | 37 | 38 | 39 | 40 | Description 41 | 42 | This command enters an interactive shell session in a container that is normally used for building. 43 | This can be useful to inspect the build environment, or to manually customize the container image for 44 | special applications if the flag is set. 45 | 46 | 47 | 48 | 49 | Options 50 | 51 | 52 | 53 | NAME 54 | The name of the container image (usually a distribution suite name). 55 | 56 | 57 | @LOGIN_FLAGS_ENTRIES@ 58 | 59 | 60 | 61 | 62 | See Also 63 | debspawn(1), systemd-nspawn(1). 64 | 65 | 66 | AUTHOR 67 | 68 | This manual page was written by Matthias Klumpp mak@debian.org. 69 | 70 | 71 | 72 | -------------------------------------------------------------------------------- /docs/debspawn-maintain.1.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 August, 2018"> 6 | 7 | 8 | 9 | ]> 10 | 11 | 12 | 13 | &command; 14 | 15 | 2018-2022 16 | Matthias Klumpp 17 | 18 | Debspawn 19 | &date; 20 | 21 | 22 | 23 | &pagename; 24 | 1 25 | 26 | 27 | &pagename; 28 | Run various maintenance actions 29 | 30 | 31 | 32 | 33 | &command; 34 | @MAINTAIN_FLAGS_SYNOPSIS@ 35 | 36 | 37 | 38 | 39 | Description 40 | 41 | Perform various maintenance actions on debspawn. Actions this subcommand allows 42 | will affect generic settings of debspawn or all of its container images at once. 43 | It can also be used to display general, useful information about the system and 44 | debspawn installation to help with finding setup issues. 45 | 46 | 47 | 48 | 49 | Examples 50 | 51 | You can update all container images that debspawn knows of in one go: 52 | 53 | 54 | $ &command; --update-all 55 | 56 | 57 | If you want to get information about the current debspawn installation 58 | (useful when reporting an issue against it), the option will print a 59 | status summary and highlight issues: 60 | 61 | 62 | $ &command; --status 63 | 64 | 65 | You can clear all caches for all images to free up disk space (missing data will be downloaded 66 | or regenerated again when it is needed): 67 | 68 | 69 | $ &command; --clear-caches 70 | 71 | 72 | 73 | 74 | Options 75 | 76 | 77 | @MAINTAIN_FLAGS_ENTRIES@ 78 | 79 | 80 | 81 | 82 | See Also 83 | debspawn-build(1), debootstrap(1), systemd-nspawn(1). 84 | 85 | 86 | AUTHOR 87 | 88 | This manual page was written by Matthias Klumpp mak@debian.org. 89 | 90 | 91 | 92 | -------------------------------------------------------------------------------- /docs/debspawn-run.1.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 August, 2018"> 6 | 7 | 8 | 9 | ]> 10 | 11 | 12 | 13 | &command; 14 | 15 | 2018-2022 16 | Matthias Klumpp 17 | 18 | Debspawn 19 | &date; 20 | 21 | 22 | 23 | &pagename; 24 | 1 25 | 26 | 27 | &pagename; 28 | Run arbitrary commands in debspawn container session 29 | 30 | 31 | 32 | 33 | &command; 34 | @RUN_FLAGS_SYNOPSIS@ 35 | NAME 36 | COMMAND 37 | 38 | 39 | 40 | 41 | Description 42 | 43 | This subcommand allows you to run arbitrary commands in an ephemeral debspawn container, using 44 | the same environment that is normally used for building packages. 45 | 46 | 47 | &command; is explicitly designed to be used by other automation tools for custom applications, 48 | and usually you will want to use debspawn build instead to build Debian packages. 49 | 50 | 51 | 52 | 53 | Options 54 | 55 | 56 | 57 | NAME 58 | The name of the container image (usually a distribution suite name). 59 | 60 | 61 | COMMAND 62 | The command to run. 63 | 64 | 65 | @RUN_FLAGS_ENTRIES@ 66 | 67 | 68 | 69 | 70 | See Also 71 | debspawn-build(1). 72 | 73 | 74 | AUTHOR 75 | 76 | This manual page was written by Matthias Klumpp mak@debian.org. 77 | 78 | 79 | 80 | -------------------------------------------------------------------------------- /docs/debspawn-update.1.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 August, 2018"> 6 | 7 | 8 | 9 | ]> 10 | 11 | 12 | 13 | &command; 14 | 15 | 2018-2022 16 | Matthias Klumpp 17 | 18 | Debspawn 19 | &date; 20 | 21 | 22 | 23 | &pagename; 24 | 1 25 | 26 | 27 | &pagename; 28 | Update a container image 29 | 30 | 31 | 32 | 33 | &command; 34 | @UPDATE_FLAGS_SYNOPSIS@ 35 | NAME 36 | 37 | 38 | 39 | 40 | Description 41 | 42 | Update a container base image. This achieves the same thing as running apt update && apt full-upgrade on the base 43 | image and making the changes permanent. Additionally, &command; will prune all caches and ensure all required packages and 44 | scripts are installed in the container image. 45 | 46 | 47 | Running &command; on the images that are in use about once a week ensures builds will happen faster, due to less changes 48 | that have to be done prior to each build. 49 | 50 | 51 | 52 | 53 | Examples 54 | 55 | Updating images is easy, you just pass the same arguments you used for creating them, but use the update subcommand instead: 56 | 57 | 58 | $ &command; sid 59 | $ &command; --arch=i386 cosmic 60 | 61 | 62 | 63 | 64 | Options 65 | 66 | 67 | 68 | NAME 69 | The name of the container image (usually a distribution suite name). 70 | 71 | 72 | @UPDATE_FLAGS_ENTRIES@ 73 | 74 | 75 | 76 | 77 | See Also 78 | debspawn-create(1), debspawn-build(1). 79 | 80 | 81 | AUTHOR 82 | 83 | This manual page was written by Matthias Klumpp mak@debian.org. 84 | 85 | 86 | 87 | -------------------------------------------------------------------------------- /docs/debspawn.1.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 18 August, 2018"> 7 | 9 | 10 | ]> 11 | 12 | 13 | 14 | debspawn 15 | 16 | 2018-2022 17 | Matthias Klumpp 18 | 19 | Debspawn 20 | &date; 21 | 22 | 23 | 24 | debspawn 25 | 1 26 | 27 | 28 | &package; 29 | Build in nspawn containers 30 | 31 | 32 | 33 | 34 | &package; 35 | @BASE_FLAGS_SYNOPSIS@ 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | Description 44 | 45 | This manual page documents the &package; command. 46 | 47 | 48 | &package; is a tool to build Debian packages in an isolated environment, using nspawn containers. 49 | By using containers, Debspawn can isolate builds from the host system much better than a regular chroot could. 50 | It also allows for more advanced features to manage builds, for example setting resource limits for individual builds. 51 | 52 | 53 | Please keep in mind that Debspawn is not a security feature! While it provides a lot of isolation from the host system, you should not run arbitrary 54 | untrusted code with it. The usual warnings for all technology based on Linux containers apply here. 55 | See systemd-nspawn(1) for more information on the container solution Debspawn uses. 56 | 57 | 58 | Debspawn also allows one to run arbitrary custom commands in its environment. This is useful to execute a variety of non-package build and QA actions that 59 | make sense to be run in the same environment in which packages are usually built. 60 | 61 | 62 | For more information about the Debspawn project, you can visit its project page. 63 | 64 | 65 | 66 | 67 | Subcommands 68 | 69 | 70 | &package; actions are invoked via subcommands. Refer to their individual manual pages for further details. 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | Create a new container base image for a specific suite, architecture and variant. A custom mirror location can also be 80 | provided. For details, see debspawn-create(1). 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | List information about all container image that Debspawn knows on the current host. 90 | For details, see debspawn-list(1). 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | Delete a container base image and all data associated with it. 100 | For details, see debspawn-delete(1). 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | Update a container base image, ensuring all packages are up to date and the image is set up 110 | properly for use with debspawn. 111 | For details, see debspawn-update(1). 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | Build a Debian package in an isolated environment. 121 | For details, see debspawn-build(1). 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | Get an interactive shell session in a container. 131 | For details, see debspawn-login(1). 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | Run arbitrary commands in debspawn container session. This is primarily useful 141 | for using &package; to isolate non-package build processes. 142 | For details, see debspawn-run(1). 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | Flags 153 | 154 | 155 | @BASE_FLAGS_ENTRIES@ 156 | 157 | 158 | 159 | 160 | Configuration 161 | 162 | 163 | Configuration is read from an optional TOML file, located at /etc/debspawn/global.toml or a location specified 164 | with . Specifying a config file on the command line will skip loading of the global, system-wide configuration. 165 | 166 | 167 | 168 | The following keys are valid at the document root level, all are optional: 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | Location for stored container images. 177 | 178 | 179 | 180 | 181 | 182 | 183 | Default output directory for build artifacts on successful builds. 184 | 185 | 186 | 187 | 188 | 189 | 190 | Location for debspawn's package cache. 191 | 192 | 193 | 194 | 195 | 196 | 197 | 198 | Package files placed in the root of this directory are available to all containers 199 | to satisfy build dependencies, while ones placed in subdirectories with the OS image 200 | name (e.g. sid-arm64) will only be available to the specified container. 201 | 202 | 203 | 204 | 205 | 206 | 207 | 208 | Temporary data location (Default: /var/tmp/debspawn/). 209 | 210 | 211 | 212 | 213 | 214 | 215 | 216 | Set a default variant used for bootstrapping with debootstrap that gets 217 | used if no variant is explicitly set when creating a new image. 218 | Set to none to make "no variant" the default. 219 | (Default: buildd) 220 | 221 | 222 | 223 | 224 | 225 | 226 | 227 | 228 | Set the system call filter used by &package; containers. 229 | This will take a list of system call names or set names as described in the 230 | "System Call Filtering" section of systemd.exec(5). 231 | 232 | 233 | It also recognizes the special string-only values compat and 234 | nspawn-default, where compat will allow 235 | enough system calls to permit many builds and tests that would run in a 236 | regular sbuild(1) chroot to work with &package; 237 | as well. By setting nspawn-default, the more restrictive defaults 238 | of systemd-nspawn(1) are applied. 239 | (Default: compat) 240 | 241 | 242 | 243 | 244 | 245 | 246 | 247 | 248 | Boolean option. If set to true, unsafe options can be used 249 | for building software via &package; run, such as making the 250 | host's /dev and /proc filesystems 251 | available from within the container. See the --allow option 252 | of &package; run for more details. 253 | (Default: false) 254 | 255 | 256 | 257 | 258 | 259 | 260 | 261 | 262 | Boolean option. If set to false, &package; will not 263 | manage its own local cache of APT packages, but will instead always try to download 264 | them. It is only recommended to change this option if you are already running a separate APT 265 | package repository mirror or a caching proxy such as apt-cacher-ng(8). 266 | (Default: true) 267 | 268 | 269 | 270 | 271 | 272 | 273 | 274 | 275 | Set the bootstrap tool that should be used for bootstrapping new images. 276 | The tool should have an interface compatible with debootstrap(8). This option 277 | allows one to use alternative tools like mmdebstrap(1) with &package;. 278 | (Default: debootstrap) 279 | 280 | 281 | 282 | 283 | 284 | 285 | 286 | 287 | 288 | See Also 289 | dpkg-buildpackage(1), systemd-nspawn(1), sbuild(1). 290 | 291 | 292 | AUTHOR 293 | 294 | This manual page was written by Matthias Klumpp mak@debian.org. 295 | 296 | 297 | 298 | -------------------------------------------------------------------------------- /install-sysdata.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (C) 2018-2022 Matthias Klumpp 5 | # 6 | # SPDX-License-Identifier: LGPL-3.0-or-later 7 | 8 | # 9 | # This is a helper script to install additional configuration and documentation into 10 | # system locations, which Python's setuptools and pip will not usually let us install. 11 | # 12 | 13 | import os 14 | import sys 15 | import shutil 16 | from pathlib import Path 17 | from argparse import ArgumentParser 18 | from tempfile import TemporaryDirectory 19 | 20 | try: 21 | import pkgconfig 22 | except ImportError: 23 | print() 24 | print( 25 | ( 26 | 'Unable to import pkgconfig. Please install the module ' 27 | '(apt install python3-pkgconfig or pip install pkgconfig) ' 28 | 'to continue.' 29 | ) 30 | ) 31 | print() 32 | sys.exit(4) 33 | from docs.assemble_man import create_manpage, generate_docbook_pages 34 | 35 | 36 | class Installer: 37 | def __init__(self, root: str = None, prefix: str = None): 38 | if not root: 39 | root = os.environ.get('DESTDIR') 40 | if not root: 41 | root = '/' 42 | self.root = root 43 | 44 | if not prefix: 45 | prefix = '/usr/local' if self.root == '/' else '/usr' 46 | if prefix.startswith('/'): 47 | prefix = prefix[1:] 48 | self.prefix = prefix 49 | 50 | def install(self, src, dst, replace_vars=False): 51 | if dst.startswith('/'): 52 | dst = dst[1:] 53 | dst_full = os.path.join(self.root, dst, os.path.basename(src)) 54 | else: 55 | dst_full = os.path.join(self.root, self.prefix, dst, os.path.basename(src)) 56 | 57 | Path(os.path.dirname(dst_full)).mkdir(mode=0o755, parents=True, exist_ok=True) 58 | if replace_vars: 59 | with open(src, 'r') as f_src: 60 | with open(dst_full, 'w') as f_dst: 61 | for line in f_src: 62 | f_dst.write(line.replace('@PREFIX@', '/' + self.prefix)) 63 | else: 64 | shutil.copy(src, dst_full) 65 | os.chmod(dst_full, 0o644) 66 | print('{}\t\t{}'.format(os.path.basename(src), dst_full)) 67 | 68 | 69 | def chdir_to_source_root(): 70 | thisfile = __file__ 71 | if not os.path.isabs(thisfile): 72 | thisfile = os.path.normpath(os.path.join(os.getcwd(), thisfile)) 73 | os.chdir(os.path.dirname(thisfile)) 74 | 75 | 76 | def make_manpages(temp_dir): 77 | '''Build manual pages''' 78 | 79 | # check for xsltproc, we need it to build manual pages 80 | if not shutil.which('xsltproc'): 81 | print('The "xsltproc" binary was not found. Please install it to continue!') 82 | sys.exit(1) 83 | 84 | build_dir = os.path.join(temp_dir, 'docbook') 85 | Path(build_dir).mkdir(parents=True, exist_ok=True) 86 | pages = generate_docbook_pages(build_dir) 87 | man_files = [] 88 | for page in pages: 89 | man_files.append(create_manpage(page, temp_dir)) 90 | return man_files 91 | 92 | 93 | def install_data(temp_dir: str, root_dir: str, prefix_dir: str): 94 | chdir_to_source_root() 95 | 96 | print('Checking dependencies') 97 | if not pkgconfig.installed('systemd', '>= 240'): 98 | print('Systemd is not installed on this system. Please make systemd available to continue.') 99 | sys.exit(4) 100 | 101 | print('Generating manual pages') 102 | manpage_files = make_manpages(temp_dir) 103 | 104 | print('Installing data') 105 | inst = Installer(root_dir, prefix_dir) 106 | sd_tmpfiles_dir = pkgconfig.variables('systemd')['tmpfilesdir'] 107 | sd_system_unit_dir = pkgconfig.variables('systemd')['systemdsystemunitdir'] 108 | man_dir = os.path.join('share', 'man', 'man1') 109 | 110 | inst.install('data/tmpfiles.d/debspawn.conf', sd_tmpfiles_dir) 111 | inst.install('data/services/debspawn-clear-caches.timer', sd_system_unit_dir) 112 | inst.install('data/services/debspawn-clear-caches.service', sd_system_unit_dir, replace_vars=True) 113 | for mf in manpage_files: 114 | inst.install(mf, man_dir) 115 | 116 | 117 | def main(): 118 | parser = ArgumentParser(description='Debspawn system data installer') 119 | 120 | parser.add_argument( 121 | '--root', action='store', dest='root', default=None, help='Root directory to install into.' 122 | ) 123 | parser.add_argument( 124 | '--prefix', 125 | action='store', 126 | dest='prefix', 127 | default=None, 128 | help='Directory prefix (usually `/usr` or `/usr/local`).', 129 | ) 130 | 131 | options = parser.parse_args(sys.argv[1:]) 132 | with TemporaryDirectory(prefix='dsinstall-') as temp_dir: 133 | install_data(temp_dir, options.root, options.prefix) 134 | return 0 135 | 136 | 137 | if __name__ == '__main__': 138 | sys.exit(main()) 139 | -------------------------------------------------------------------------------- /lint.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | BASEDIR=$(dirname "$0") 5 | cd $BASEDIR 6 | 7 | echo "=== Flake8 ===" 8 | python -m flake8 ./ --statistics 9 | python -m flake8 debspawn/dsrun --statistics 10 | echo "✓" 11 | 12 | echo "=== Pylint ===" 13 | python -m pylint -f colorized ./debspawn 14 | python -m pylint -f colorized ./debspawn/dsrun 15 | python -m pylint -f colorized ./tests ./data 16 | python -m pylint -f colorized setup.py install-sysdata.py 17 | echo "✓" 18 | 19 | echo "=== MyPy ===" 20 | python -m mypy . 21 | python -m mypy ./debspawn/dsrun 22 | echo "✓" 23 | 24 | echo "=== Isort ===" 25 | isort --diff . 26 | echo "✓" 27 | 28 | echo "=== Black ===" 29 | black --diff . 30 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "debspawn" 3 | description = "Debian package builder and build helper using systemd-nspawn" 4 | authors = [ 5 | {name = "Matthias Klumpp", email = "matthias@tenstral.net"}, 6 | ] 7 | license = {text="LGPL-3.0-or-later"} 8 | readme = "README.md" 9 | requires-python = ">=3.9" 10 | dynamic = ['version'] 11 | 12 | dependencies = [ 13 | "tomlkit>=0.8", 14 | ] 15 | 16 | [project.urls] 17 | Documentation = "https://github.com/lkhq/debspawn" 18 | Source = "https://github.com/lkhq/debspawn" 19 | 20 | [build-system] 21 | requires = ["setuptools", "wheel", "pkgconfig"] 22 | build-backend = "setuptools.build_meta" 23 | 24 | [tool.pylint.master] 25 | 26 | [tool.pylint.format] 27 | max-line-length = 120 28 | 29 | [tool.pylint."messages control"] 30 | disable = [ 31 | 'C', 'R', 32 | 'fixme', 33 | 'unused-argument', 34 | 'global-statement', 35 | 'logging-format-interpolation', 36 | 'attribute-defined-outside-init', 37 | 'protected-access', 38 | 'broad-except', 39 | 'redefined-builtin', 40 | 'unspecified-encoding', 41 | ] 42 | 43 | [tool.pylint.reports] 44 | score = 'no' 45 | 46 | [tool.isort] 47 | py_version = 39 48 | profile = "black" 49 | multi_line_output = 3 50 | skip_gitignore = true 51 | length_sort = true 52 | atomic = true 53 | 54 | [tool.black] 55 | target-version = ['py39'] 56 | line-length = 110 57 | skip-string-normalization = true 58 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 120 3 | ignore = E203,W503 4 | 5 | [metadata] 6 | description_file = README.md 7 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import os 4 | import sys 5 | import shutil 6 | import platform 7 | from subprocess import check_call 8 | 9 | import tomllib 10 | from setuptools import setup 11 | from setuptools.command.install_scripts import install_scripts as install_scripts_orig 12 | 13 | sys.path.append(os.getcwd()) 14 | from debspawn import __appname__, __version__ # noqa: E402 15 | 16 | thisfile = __file__ 17 | if not os.path.isabs(thisfile): 18 | thisfile = os.path.normpath(os.path.join(os.getcwd(), thisfile)) 19 | source_root = os.path.dirname(thisfile) 20 | 21 | 22 | class install_scripts(install_scripts_orig): 23 | def _check_command(self, command): 24 | if not shutil.which(command): 25 | print( 26 | 'The "{}" binary was not found. Please install it to continue!'.format(command), 27 | file=sys.stderr, 28 | ) 29 | sys.exit(1) 30 | 31 | def _check_commands_available(self): 32 | '''Check if certain commands are available that debspawn needs to work.''' 33 | self._check_command('systemd-nspawn') 34 | self._check_command('findmnt') 35 | self._check_command('zstd') 36 | self._check_command('debootstrap') 37 | self._check_command('dpkg') 38 | 39 | def run(self): 40 | if platform.system() == 'Windows': 41 | super().run() 42 | return 43 | 44 | if not self.skip_build: 45 | self.run_command('build_scripts') 46 | self.outfiles = [] 47 | 48 | if self.dry_run: 49 | return 50 | 51 | # We want the files to be installed without a suffix on Unix 52 | self.mkpath(self.install_dir) 53 | for infile in self.get_inputs(): 54 | infile = os.path.basename(infile) 55 | in_built = os.path.join(self.build_dir, infile) 56 | in_stripped = infile[:-3] if infile.endswith('.py') else infile 57 | outfile = os.path.join(self.install_dir, in_stripped) 58 | # NOTE: Mode is preserved by default 59 | self.copy_file(in_built, outfile) 60 | self.outfiles.append(outfile) 61 | 62 | # try to install configuration snippets, manual pages and other external data 63 | bin_install_dir = str(self.install_dir) 64 | if '/usr/' in bin_install_dir: 65 | install_root = bin_install_dir.split('/usr/', 1)[0] 66 | prefix = '/usr/local' if '/usr/local/' in bin_install_dir else '/usr' 67 | sysdata_install_script = os.path.join(source_root, 'install-sysdata.py') 68 | if os.path.isfile(sysdata_install_script) and os.path.isdir(install_root): 69 | check_call( 70 | [sys.executable, sysdata_install_script, '--root', install_root, '--prefix', prefix] 71 | ) 72 | else: 73 | print('Unable to install externally managed data!', file=sys.stderr) 74 | else: 75 | print( 76 | ( 77 | '\n\n ------------------------\n' 78 | 'Unable to install external configuration and manual pages!\n' 79 | 'While these files are not essential to work with debspawn, they will improve how it runs ' 80 | 'or are useful as documentation. Please install these files manually by running the ' 81 | '`install-sysdata.py` script from debspawn\'s source directory manually as root.\n' 82 | 'Installing these external files is not possible when installing e.g. with pip. If `setup.py` is ' 83 | 'used directly we make an attempt to install the files, but this attempt has failed.' 84 | '\n ------------------------\n\n' 85 | ), 86 | file=sys.stderr, 87 | ) 88 | 89 | 90 | cmdclass = { 91 | 'install_scripts': install_scripts, 92 | } 93 | 94 | packages = [ 95 | 'debspawn', 96 | 'debspawn.utils', 97 | ] 98 | 99 | package_data = {'': ['debspawn/dsrun']} 100 | 101 | scripts = ['debspawn.py'] 102 | 103 | with open('pyproject.toml', 'rb') as f: 104 | pp_data = tomllib.load(f) 105 | install_requires = pp_data['project']['dependencies'] 106 | 107 | setup( 108 | name=__appname__, 109 | version=__version__, 110 | author="Matthias Klumpp", 111 | author_email="matthias@tenstral.net", 112 | description='Easily build Debian packages in systemd-nspawn containers', 113 | license="LGPL-3.0+", 114 | url="https://github.com/lkhq/debspawn", 115 | long_description=open(os.path.join(source_root, 'README.md')).read(), 116 | long_description_content_type='text/markdown', 117 | # 118 | python_requires='>=3.11', 119 | platforms=['any'], 120 | zip_safe=False, 121 | include_package_data=True, 122 | # 123 | packages=packages, 124 | cmdclass=cmdclass, 125 | package_data=package_data, 126 | scripts=scripts, 127 | install_requires=install_requires, 128 | ) 129 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (C) 2019-2020 Matthias Klumpp 4 | # 5 | # Licensed under the GNU Lesser General Public License Version 3 6 | # 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Lesser General Public License as published by 9 | # the Free Software Foundation, either version 3 of the license, or 10 | # (at your option) any later version. 11 | # 12 | # This software is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Lesser General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Lesser General Public License 18 | # along with this software. If not, see . 19 | 20 | import os 21 | import sys 22 | 23 | thisfile = __file__ 24 | if not os.path.isabs(thisfile): 25 | thisfile = os.path.normpath(os.path.join(os.getcwd(), thisfile)) 26 | source_root = os.path.normpath(os.path.join(os.path.dirname(thisfile), '..')) 27 | sys.path.append(os.path.normpath(source_root)) 28 | 29 | 30 | __all__ = ['source_root'] 31 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (C) 2019-2020 Matthias Klumpp 4 | # 5 | # Licensed under the GNU Lesser General Public License Version 3 6 | # 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Lesser General Public License as published by 9 | # the Free Software Foundation, either version 3 of the license, or 10 | # (at your option) any later version. 11 | # 12 | # This software is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Lesser General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Lesser General Public License 18 | # along with this software. If not, see . 19 | 20 | import os 21 | import sys 22 | 23 | import pytest 24 | 25 | # pylint: disable=redefined-outer-name 26 | 27 | 28 | @pytest.fixture(scope='session', autouse=True) 29 | def gconfig(): 30 | ''' 31 | Ensure the global config object is set up properly for unit-testing. 32 | ''' 33 | import shutil 34 | 35 | import debspawn.cli 36 | 37 | from . import source_root 38 | 39 | debspawn.cli.__mainfile = os.path.join(source_root, 'debspawn.py') 40 | 41 | class MockOptions: 42 | config = None 43 | no_unicode = False 44 | owner = None 45 | 46 | gconf = debspawn.cli.init_config(MockOptions()) 47 | 48 | test_tmp_dir = '/tmp/debspawn-test/' 49 | shutil.rmtree(test_tmp_dir, ignore_errors=True) 50 | os.makedirs(test_tmp_dir) 51 | 52 | gconf._instance._osroots_dir = os.path.join(test_tmp_dir, 'containers/') 53 | gconf._instance._results_dir = os.path.join(test_tmp_dir, 'results/') 54 | gconf._instance._aptcache_dir = os.path.join(test_tmp_dir, 'aptcache/') 55 | gconf._instance._injected_pkgs_dir = os.path.join(test_tmp_dir, 'injected-pkgs/') 56 | 57 | return gconf 58 | 59 | 60 | @pytest.fixture(scope='session', autouse=True) 61 | def ensure_root(): 62 | ''' 63 | Ensure we run with superuser permissions. 64 | ''' 65 | 66 | if os.geteuid() != 0: 67 | print('The testsuite has to be run with superuser permissions in order to create nspawn instances.') 68 | sys.exit(1) 69 | 70 | 71 | @pytest.fixture(scope='session') 72 | def build_arch(): 73 | ''' 74 | Retrieve the current architecture we should build packages for. 75 | ''' 76 | from debspawn.utils.command import safe_run 77 | 78 | out, _, ret = safe_run(['dpkg', '--print-architecture']) 79 | assert ret == 0 80 | 81 | arch = out.strip() 82 | if not arch: 83 | arch = 'amd64' # assume amd64 as default 84 | 85 | return arch 86 | 87 | 88 | @pytest.fixture(scope='session') 89 | def testing_container(gconfig, build_arch): 90 | ''' 91 | Create a container for Debian stable used for default tests 92 | ''' 93 | from debspawn.osbase import OSBase 94 | 95 | suite = 'stable' 96 | variant = 'minbase' 97 | components = ['main', 'contrib', 'non-free'] 98 | extra_suites = [] 99 | 100 | osbase = OSBase(gconfig, suite, build_arch, variant=variant, base_suite=None) 101 | r = osbase.create(None, components, extra_suites=extra_suites) 102 | assert r 103 | 104 | return (suite, build_arch, variant) 105 | -------------------------------------------------------------------------------- /tests/test_cud.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (C) 2019-2020 Matthias Klumpp 4 | # 5 | # Licensed under the GNU Lesser General Public License Version 3 6 | # 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Lesser General Public License as published by 9 | # the Free Software Foundation, either version 3 of the license, or 10 | # (at your option) any later version. 11 | # 12 | # This software is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Lesser General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Lesser General Public License 18 | # along with this software. If not, see . 19 | 20 | from debspawn.osbase import OSBase 21 | 22 | 23 | def test_container_create_delete(gconfig, testing_container): 24 | # the "default" container is created by a fixture. 25 | # what we actually want to do here in future is create and 26 | # delete containers with special settings 27 | pass 28 | 29 | 30 | def test_container_update(gconfig, testing_container): 31 | '''Update a container''' 32 | 33 | suite, arch, variant = testing_container 34 | osbase = OSBase(gconfig, suite, arch, variant) 35 | assert osbase.update() 36 | 37 | 38 | def test_container_recreate(gconfig, testing_container): 39 | '''Test recreating a container''' 40 | 41 | suite, arch, variant = testing_container 42 | osbase = OSBase(gconfig, suite, arch, variant) 43 | assert osbase.recreate() 44 | -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (C) 2019-2022 Matthias Klumpp 4 | # 5 | # Licensed under the GNU Lesser General Public License Version 3 6 | # 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Lesser General Public License as published by 9 | # the Free Software Foundation, either version 3 of the license, or 10 | # (at your option) any later version. 11 | # 12 | # This software is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Lesser General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Lesser General Public License 18 | # along with this software. If not, see . 19 | 20 | import os 21 | import tempfile 22 | 23 | from debspawn.utils.misc import umount, bindmount, is_mountpoint, rmtree_mntsafe 24 | 25 | 26 | def test_bindmount_umount(gconfig): 27 | with tempfile.TemporaryDirectory() as src_tmpdir1: 28 | with tempfile.TemporaryDirectory() as src_tmpdir2: 29 | with tempfile.TemporaryDirectory() as dest_tmpdir: 30 | bindmount(src_tmpdir1, dest_tmpdir) 31 | assert is_mountpoint(dest_tmpdir) 32 | 33 | bindmount(src_tmpdir2, dest_tmpdir) 34 | assert is_mountpoint(dest_tmpdir) 35 | 36 | # sanity check 37 | open(os.path.join(src_tmpdir2, 'test'), 'a').close() 38 | assert os.path.isfile(os.path.join(dest_tmpdir, 'test')) 39 | 40 | # umount is supposed to unmount everything, even overmounted directories 41 | umount(dest_tmpdir) 42 | assert not is_mountpoint(dest_tmpdir) 43 | 44 | 45 | def test_rmtree_mntsafe(gconfig): 46 | mnt_tmpdir = tempfile.TemporaryDirectory().name 47 | dest_tmpdir = tempfile.TemporaryDirectory().name 48 | 49 | # create directory structure and files to delete 50 | mp_dir = os.path.join(dest_tmpdir, 'subdir', 'mountpoint') 51 | mount_subdir = os.path.join(mnt_tmpdir, 'subdir_in_mount') 52 | os.makedirs(mp_dir) 53 | os.makedirs(mount_subdir) 54 | open(os.path.join(dest_tmpdir, 'file1.txt'), 'a').close() 55 | open(os.path.join(mp_dir, 'file_below_mountpoint.txt'), 'a').close() 56 | open(os.path.join(mnt_tmpdir, 'file_in_mount.txt'), 'a').close() 57 | open(os.path.join(mount_subdir, 'file_in_mount_subdir.txt'), 'a').close() 58 | 59 | # create bindmount 60 | bindmount(mnt_tmpdir, mp_dir) 61 | assert is_mountpoint(mp_dir) 62 | 63 | # try to delete the directory structure containing bindmounts 64 | rmtree_mntsafe(dest_tmpdir) 65 | 66 | # verify 67 | assert not os.path.exists(dest_tmpdir) 68 | assert os.path.isfile(os.path.join(mnt_tmpdir, 'file_in_mount.txt')) 69 | assert os.path.isfile(os.path.join(mount_subdir, 'file_in_mount_subdir.txt')) 70 | 71 | # cleanup mounted dir 72 | rmtree_mntsafe(mnt_tmpdir) 73 | assert not os.path.exists(mnt_tmpdir) 74 | --------------------------------------------------------------------------------