├── .coveragerc ├── .gitignore ├── .travis.yml ├── CHANGELOG.rst ├── LICENSE.txt ├── MANIFEST.in ├── Makefile ├── README.rst ├── constraints.txt ├── docs ├── api.rst ├── changelog.rst ├── comparisons.rst ├── conf.py ├── index.rst └── readme.rst ├── py2deb ├── __init__.py ├── cli.py ├── converter.py ├── hooks.py ├── namespaces.py ├── package.py ├── tests.py └── utils.py ├── requirements-checks.txt ├── requirements-tests.txt ├── requirements-travis.txt ├── requirements.txt ├── scripts ├── pypy.py ├── pypy.sh └── travis.sh ├── setup.cfg ├── setup.py └── tox.ini /.coveragerc: -------------------------------------------------------------------------------- 1 | # .coveragerc: Configuration file for coverage.py. 2 | # http://nedbatchelder.com/code/coverage/ 3 | 4 | [run] 5 | branch = true 6 | source = py2deb 7 | omit = py2deb/tests.py 8 | 9 | # vim: ft=dosini 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.egg-info 2 | *.pyc 3 | .coverage 4 | .tox/ 5 | __pycache__/ 6 | build/ 7 | dist/ 8 | docs/_build/ 9 | docs/_static/ 10 | docs/_templates/ 11 | docs/build 12 | htmlcov/ 13 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # We use tox for testing against Python interpreters provided by deadsnakes and 2 | # the PyPy PPA, also on Travis CI, because it's important for py2deb that the 3 | # Python interpreter was installed using a Debian package. We use the following 4 | # environment variables to accomplish this: 5 | # 6 | # - $PYTHON is set so that the Makefile uses the correct Python interpreter 7 | # when the virtual environment is created. 8 | # 9 | # - $TOXENV is set so that tox knows what needs to be tested. 10 | # 11 | # - Because 'language: generic' has nothing to do with Python no virtual 12 | # environment will be provided by Travis CI, which is good, because we 13 | # want to create and use our own virtual environment. 14 | # 15 | # Note that the build matrix entries are ordered (roughly) by decreasing build 16 | # time in order to minimize the total runtime of the build matrix. 17 | 18 | sudo: required 19 | dist: bionic 20 | language: generic 21 | matrix: 22 | include: 23 | - env: PYTHON=pypy3 TOXENV=pypy3 # 15m50s 24 | - env: PYTHON=pypy TOXENV=pypy # 6m57s 25 | - env: PYTHON=python3.7 TOXENV=py37 # 4m18s 26 | - env: PYTHON=python3.5 TOXENV=py35 # 4m12s 27 | - env: PYTHON=python3.6 TOXENV=py36 # 3m53s 28 | - env: PYTHON=python2.7 TOXENV=py27 # 3m38s 29 | install: 30 | - scripts/travis.sh 31 | - make install 32 | script: 33 | - make check 34 | - make tox 35 | after_success: 36 | - coveralls 37 | branches: 38 | except: 39 | - /^[0-9]/ 40 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2020 Peter Odding, Arjan Verwer and Paylogic International 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.rst 2 | include *.txt 3 | graft docs 4 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for py2deb. 2 | # 3 | # Author: Peter Odding 4 | # Last Change: August 6, 2020 5 | # URL: https://github.com/paylogic/py2deb 6 | 7 | PACKAGE_NAME = py2deb 8 | WORKON_HOME ?= $(HOME)/.virtualenvs 9 | VIRTUAL_ENV ?= $(WORKON_HOME)/$(PACKAGE_NAME) 10 | PYTHON ?= python3 11 | PATH := $(VIRTUAL_ENV)/bin:$(PATH) 12 | MAKE := $(MAKE) --no-print-directory 13 | SHELL = bash 14 | 15 | # Configure pip not to use binary wheels on PyPy 3, with the goal of 16 | # stabilizing Travis CI builds. My trials in getting this to work: 17 | # 18 | # 1. I struggled to work around several issues involving pip using wheels on 19 | # PyPy 3 but subsequently "crashing" (tracebacks) in one of several ways due 20 | # to the use of wheels. 21 | # 22 | # 2. After wasting quite a few more hours than I care to admit on (1) I decided 23 | # to preserve my sanity by "giving in" and just disabling wheels on PyPy 3 24 | # altogether. This seemed like a simple solution, but it wasn't 😇. 25 | # 26 | # 3. Unfortunately even (2) is not enough, because some of our requirements use 27 | # the "setup_requires" feature. This used to be handled by setuptools, but 28 | # nowadays it's handled by a nested pip process, and that will try to use 29 | # wheels even when the parent process received the "--no-binary=:all:" 30 | # command line option 😞. I guess this is a bug in the packaging ecosystem 31 | # (I assume pip) but that doesn't really help me at this point. 32 | # 33 | # 4. To work around (3) some of the "transitive build requirements" are listed 34 | # in requirements-tests.txt which means they're installed by the top level 35 | # pip process which does respect the "--no-binary=:all:" option. 36 | # 37 | # Here's an overview of my weekend of experimentation to find a way 38 | # to successfully set up a usable py2deb test environment for PyPy 3: 39 | # https://github.com/paylogic/py2deb/compare/4ab626b6582...affa7158560 40 | ifeq ($(findstring pypy3,$(PYTHON)),pypy3) 41 | NO_BINARY_OPTION := :all: 42 | else 43 | NO_BINARY_OPTION := :none: 44 | endif 45 | 46 | # Define how we run 'pip' in a single place (DRY). 47 | PIP_CMD := python -m pip 48 | 49 | # Define how we run 'pip install' in a single place (DRY). 50 | PIP_INSTALL_CMD := $(PIP_CMD) install \ 51 | --constraint=constraints.txt \ 52 | --no-binary=$(NO_BINARY_OPTION) 53 | 54 | default: 55 | @echo "Makefile for $(PACKAGE_NAME)" 56 | @echo 57 | @echo 'Usage:' 58 | @echo 59 | @echo ' make install install the package in a virtual environment' 60 | @echo ' make reset recreate the virtual environment' 61 | @echo ' make check check coding style (PEP-8, PEP-257)' 62 | @echo ' make test run the test suite, report coverage' 63 | @echo ' make tox run the tests on all Python versions' 64 | @echo ' make readme update usage in readme' 65 | @echo ' make docs update documentation using Sphinx' 66 | @echo ' make publish publish changes to GitHub/PyPI' 67 | @echo ' make clean cleanup all temporary files' 68 | @echo 69 | 70 | install: 71 | @test -d "$(VIRTUAL_ENV)" || mkdir -p "$(VIRTUAL_ENV)" 72 | @test -x "$(VIRTUAL_ENV)/bin/python" || virtualenv --python=$(PYTHON) "$(VIRTUAL_ENV)" 73 | ifeq ($(findstring pypy,$(PYTHON)),pypy) 74 | # Downgrade pip on PyPy in an attempt to avoid wheel incompatibilities. 75 | @source "$(VIRTUAL_ENV)/bin/activate" && $(PYTHON) scripts/pypy.py 76 | endif 77 | ifeq ($(TRAVIS), true) 78 | # Setuptools and wheel are build dependencies of cryptography. If we don't 79 | # install them before the main 'pip install' run the setup.py script of 80 | # cryptography attempts to take care of this on its own initiative which 81 | # fails on PyPy due to incompatibilities between pip and PyPy: 82 | # https://travis-ci.org/github/paylogic/py2deb/jobs/713379963 83 | @$(PIP_INSTALL_CMD) --upgrade 'setuptools >= 40.6.0' wheel 84 | @$(PIP_INSTALL_CMD) --requirement=requirements-travis.txt 85 | else 86 | @$(PIP_INSTALL_CMD) --requirement=requirements.txt 87 | @$(PIP_CMD) uninstall --yes $(PACKAGE_NAME) &>/dev/null || true 88 | @$(PIP_INSTALL_CMD) --no-deps --ignore-installed . 89 | endif 90 | 91 | reset: 92 | @$(MAKE) clean 93 | @rm -Rf "$(VIRTUAL_ENV)" 94 | @$(MAKE) install 95 | 96 | check: install 97 | @$(PIP_INSTALL_CMD) --upgrade --requirement=requirements-checks.txt 98 | @flake8 99 | 100 | test: install 101 | @$(PIP_INSTALL_CMD) --requirement=requirements-tests.txt 102 | @py.test --cov 103 | @coverage html 104 | @coverage report --fail-under=90 &>/dev/null 105 | 106 | tox: install 107 | @$(PIP_INSTALL_CMD) tox 108 | @tox 109 | 110 | readme: install 111 | @$(PIP_INSTALL_CMD) cogapp 112 | @cog.py -r README.rst 113 | 114 | docs: readme 115 | @$(PIP_INSTALL_CMD) sphinx 116 | @cd docs && sphinx-build -nWb html -d build/doctrees . build/html 117 | 118 | publish: install 119 | @git push origin && git push --tags origin 120 | @$(MAKE) clean 121 | @$(PIP_INSTALL_CMD) twine wheel 122 | @$(PYTHON) setup.py sdist bdist_wheel 123 | @twine upload dist/* 124 | @$(MAKE) clean 125 | 126 | clean: 127 | @rm -Rf *.egg .cache .coverage .tox build dist docs/build htmlcov 128 | @find -depth -type d -name __pycache__ -exec rm -Rf {} \; 129 | @find -type f -name '*.pyc' -delete 130 | 131 | .PHONY: default install reset check test tox readme docs publish clean 132 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | py2deb: Python to Debian package converter 2 | ========================================== 3 | 4 | .. image:: https://travis-ci.org/paylogic/py2deb.svg?branch=master 5 | :target: https://travis-ci.org/paylogic/py2deb 6 | 7 | .. image:: https://coveralls.io/repos/paylogic/py2deb/badge.svg?branch=master 8 | :target: https://coveralls.io/r/paylogic/py2deb?branch=master 9 | 10 | The Python package py2deb_ converts Python source distributions to Debian 11 | binary packages (the ones used for installation). It uses pip-accel_ (based on 12 | pip_) to download, unpack and compile Python packages. Because of this py2deb_ 13 | is compatible with the command line interface of the ``pip install`` command. 14 | For example you can specify packages to convert as command line arguments but 15 | you can also use `requirement files`_ if you want. 16 | 17 | During the conversion process dependencies are automatically taken into account 18 | and converted as well so you don't actually have to use requirement files 19 | including transitive dependencies. In fact you might prefer not explicitly 20 | listing your transitive dependencies in requirement files because py2deb_ will 21 | translate the version constraints of Python packages into Debian package 22 | relationships. 23 | 24 | The py2deb_ package is currently tested on CPython_ 2.7, 3.5, 3.6, 3.7 and 25 | PyPy_ 2 and 3. Unfortunately Python 3.8+ is not yet supported (see below). For 26 | usage instructions please refer to the documentation hosted on `Read The 27 | Docs`_. 28 | 29 | .. contents:: 30 | :local: 31 | 32 | Installation 33 | ------------ 34 | 35 | The py2deb_ package is available on PyPI_, so installation is very simple: 36 | 37 | .. code-block:: console 38 | 39 | $ pip install py2deb 40 | 41 | There are some system dependencies which you have to install as well: 42 | 43 | .. code-block:: console 44 | 45 | $ sudo apt-get install dpkg-dev fakeroot 46 | 47 | Optionally you can also install Lintian_ (which is not a hard dependency but 48 | more of a "nice to have"): 49 | 50 | .. code-block:: console 51 | 52 | $ sudo apt-get install lintian 53 | 54 | When Lintian is installed it will be run automatically to sanity check 55 | converted packages. This slows down the conversion process somewhat but can be 56 | very useful, especially when working on py2deb itself. Currently py2deb doesn't 57 | fail when Lintian reports errors, this is due to the unorthodox ways in which 58 | py2deb can be used. This may change in the future as py2deb becomes more 59 | mature. 60 | 61 | Usage 62 | ----- 63 | 64 | There are two ways to use the py2deb_ package: As the command line program 65 | ``py2deb`` and as a Python API. For details about the Python API please refer 66 | to the API documentation hosted on `Read the Docs`_. The command line interface 67 | is described below. 68 | 69 | Command line 70 | ~~~~~~~~~~~~ 71 | 72 | .. A DRY solution to avoid duplication of the `py2deb --help' text: 73 | .. 74 | .. [[[cog 75 | .. from humanfriendly.usage import inject_usage 76 | .. inject_usage('py2deb.cli') 77 | .. ]]] 78 | 79 | **Usage:** `py2deb [OPTIONS] ...` 80 | 81 | Convert Python packages to Debian packages according to the given 82 | command line options (see below). The command line arguments are the 83 | same as accepted by the "pip install" command because py2deb invokes 84 | pip during the conversion process. This means you can name the 85 | package(s) to convert on the command line but you can also use 86 | "requirement files" if you prefer. 87 | 88 | If you want to pass command line options to pip (e.g. because you want 89 | to use a custom index URL or a requirements file) then you will need 90 | to tell py2deb where the options for py2deb stop and the options for 91 | pip begin. In such cases you can use the following syntax: 92 | 93 | .. code-block:: sh 94 | 95 | $ py2deb -r /tmp -- -r requirements.txt 96 | 97 | So the "--" marker separates the py2deb options from the pip options. 98 | 99 | **Supported options:** 100 | 101 | .. csv-table:: 102 | :header: Option, Description 103 | :widths: 30, 70 104 | 105 | 106 | "``-c``, ``--config=FILENAME``","Load a configuration file. Because the command line arguments are processed 107 | in the given order, you have the choice and responsibility to decide if 108 | command line options override configuration file options or vice versa. 109 | Refer to the documentation for details on the configuration file format. 110 | 111 | The default configuration files /etc/py2deb.ini and ~/.py2deb.ini are 112 | automatically loaded if they exist. This happens before environment 113 | variables and command line options are processed. 114 | 115 | Can also be set using the environment variable ``$PY2DEB_CONFIG``." 116 | "``-r``, ``--repository=DIRECTORY``","Change the directory where \*.deb archives are stored. Defaults to 117 | the system wide temporary directory (which is usually /tmp). If 118 | this directory doesn't exist py2deb refuses to run. 119 | 120 | Can also be set using the environment variable ``$PY2DEB_REPOSITORY``." 121 | "``--use-system-package=PYTHON_PACKAGE_NAME,DEBIAN_PACKAGE_NAME``","Exclude a Python package (the name before the comma) from conversion and 122 | replace references to the Python package with a specific Debian package 123 | name. This allows you to use system packages for specific Python 124 | requirements." 125 | ``--name-prefix=PREFIX``,"Set the name prefix used during the name conversion from Python to 126 | Debian packages. Defaults to ""python"". The name prefix and package 127 | names are always delimited by a dash. 128 | 129 | Can also be set using the environment variable ``$PY2DEB_NAME_PREFIX``." 130 | ``--no-name-prefix=PYTHON_PACKAGE_NAME``,"Exclude a Python package from having the name prefix applied 131 | during the package name conversion. This is useful to avoid 132 | awkward repetitions." 133 | "``--rename=PYTHON_PACKAGE_NAME,DEBIAN_PACKAGE_NAME``","Override the package name conversion algorithm for the given pair 134 | of package names. Useful if you don't agree with the algorithm :-)" 135 | ``--install-prefix=DIRECTORY``,"Override the default system wide installation prefix. By setting 136 | this to anything other than ""/usr"" or ""/usr/local"" you change the 137 | way py2deb works. It will build packages with a file system layout 138 | similar to a Python virtual environment, except there will not be 139 | a Python executable: The packages are meant to be loaded by 140 | modifying Python's module search path. Refer to the documentation 141 | for details. 142 | 143 | Can also be set using the environment variable ``$PY2DEB_INSTALL_PREFIX``." 144 | "``--install-alternative=LINK,PATH``","Use Debian's ""update-alternatives"" system to add an executable 145 | that's installed in a custom installation prefix (see above) to 146 | the system wide executable search path. Refer to the documentation 147 | for details." 148 | ``--python-callback=EXPRESSION``,"Set a Python callback to be called during the conversion process. Refer to 149 | the documentation for details about the use of this feature and the syntax 150 | of ``EXPRESSION``. 151 | 152 | Can also be set using the environment variable ``$PY2DEB_CALLBACK``." 153 | ``--report-dependencies=FILENAME``,"Add the Debian relationships needed to depend on the converted 154 | package(s) to the given control file. If the control file already 155 | contains relationships the additional relationships will be added 156 | to the control file; they won't overwrite existing relationships." 157 | "``-y``, ``--yes``","Instruct pip-accel to automatically install build time dependencies 158 | where possible. Refer to the pip-accel documentation for details. 159 | 160 | Can also be set using the environment variable ``$PY2DEB_AUTO_INSTALL``." 161 | "``-v``, ``--verbose``",Make more noise :-). 162 | "``-h``, ``--help``",Show this message and exit. 163 | 164 | .. [[[end]]] 165 | 166 | Future improvements 167 | ------------------- 168 | 169 | The following sections list possible improvements to the project: 170 | 171 | .. contents:: 172 | :local: 173 | 174 | .. _Python 3.8+ compatibility: 175 | 176 | Python 3.8+ compatibility 177 | ~~~~~~~~~~~~~~~~~~~~~~~~~ 178 | 179 | The py2deb_ project builds on top of pip-accel_, which was developed between 180 | 2013 and 2015 on top of ``pip >= 7.0, < 7.2``. Since that time pip_ has grown 181 | enormously: At the time of writing (in August 2020) we're now at pip 20! 182 | 183 | None of the improvements made between pip 7-20 are available in pip-accel and 184 | py2deb and this has become somewhat of a glaring issue that plenty of users 185 | have run into (see `#17`_, `#18`_, `#27`_ and `#31`_). 186 | 187 | Known issues being caused by this include: 188 | 189 | - The old pip version prevents Python 3.8+ compatibility. 190 | 191 | - The old pip version doesn't know about ``python_requires`` metadata provided 192 | by PyPI and this forces users to maintain constraints files themselves, even 193 | though this shouldn't be necessary. 194 | 195 | - While pip-accel supports installation from wheels, it was never exposed via 196 | the Python API and so py2deb lacks support for converting wheels (it 197 | currently needs source distributions). 198 | 199 | The current state of affairs is best summarized in `this comment`_. I'm hoping 200 | to complete the upgrade to newer pip and pip-accel releases in the coming weeks 201 | (as of this writing in August 2020) but can't commit to a date. 202 | 203 | .. _this comment: https://github.com/paylogic/py2deb/issues/18#issuecomment-633848582 204 | 205 | Installation of system wide files 206 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 207 | 208 | Find a way to facilitate (explicit / opt-in) installation of system wide files 209 | (not related to Python per se) based on a Python distribution? This could 210 | significantly reduce the need for "wrapper packages" that basically just pull 211 | in packages converted by py2deb and drop a few configuration files into place. 212 | 213 | :Related issues: See issue `#7`_ for a related discussion. 214 | 215 | Conversion of binary wheels 216 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~ 217 | 218 | Investigate the feasability of supporting conversion of binary wheels. Slowly 219 | but surely the Python community seems to be gravitating towards (binary) wheels 220 | and once gravity has shifted we don't want to be left in the dust! 😉 221 | 222 | Full PEP-440 compatibility 223 | ~~~~~~~~~~~~~~~~~~~~~~~~~~ 224 | 225 | Dive into PEP-440_ and see if it can be fully supported? Then `this question on 226 | Reddit`_ can finally get a satisfying answer 🙂. 227 | 228 | Similar projects 229 | ---------------- 230 | 231 | There are several projects out there that share similarities with py2deb, for 232 | example I know of stdeb_, dh-virtualenv_ and fpm_. The documentation includes a 233 | fairly `detailed comparison`_ with each of these projects. 234 | 235 | Contact 236 | ------- 237 | 238 | The latest version of py2deb is available on PyPI_ and GitHub_. The 239 | documentation is hosted on `Read the Docs`_ and includes a changelog_. For 240 | questions, bug reports, suggestions, etc. please create an issue on GitHub_. 241 | 242 | License 243 | ------- 244 | 245 | This software is licensed under the `MIT license`_. 246 | 247 | © 2020 Peter Odding, Arjan Verwer and Paylogic International. 248 | 249 | .. External references: 250 | .. _CPython: https://en.wikipedia.org/wiki/CPython 251 | .. _GitHub: https://github.com/paylogic/py2deb 252 | .. _Lintian: http://en.wikipedia.org/wiki/Lintian 253 | .. _MIT license: http://en.wikipedia.org/wiki/MIT_License 254 | .. _PEP-440: https://www.python.org/dev/peps/pep-0440/ 255 | .. _Pillow: https://python-pillow.github.io/ 256 | .. _PyPI: https://pypi.org/project/py2deb 257 | .. _PyPy: https://en.wikipedia.org/wiki/PyPy 258 | .. _Read The Docs: https://py2deb.readthedocs.io 259 | .. _changelog: https://py2deb.readthedocs.io/en/latest/changelog.html 260 | .. _deb-pkg-tools: https://pypi.org/project/deb-pkg-tools 261 | .. _detailed comparison: https://py2deb.readthedocs.io/en/latest/comparisons.html 262 | .. _dh-virtualenv: https://github.com/spotify/dh-virtualenv 263 | .. _fpm: https://github.com/jordansissel/fpm 264 | .. _pip-accel: https://pypi.org/project/pip-accel 265 | .. _pip: https://pypi.org/project/pip 266 | .. _py2deb: https://pypi.org/project/py2deb 267 | .. _python-imaging: https://packages.debian.org/search?keywords=python-imaging 268 | .. _python-pil: https://packages.debian.org/search?keywords=python-pil 269 | .. _requirement files: http://www.pip-installer.org/en/latest/cookbook.html#requirements-files 270 | .. _stdeb: https://pypi.org/project/stdeb 271 | .. _this question on Reddit: https://www.reddit.com/r/Python/comments/2x7s17/py2deb_python_to_debian_package_converter/coxyyzu 272 | .. _#7: https://github.com/paylogic/py2deb/issues/7 273 | .. _#17: https://github.com/paylogic/py2deb/issues/17 274 | .. _#18: https://github.com/paylogic/py2deb/issues/18 275 | .. _#27: https://github.com/paylogic/py2deb/issues/27 276 | .. _#31: https://github.com/paylogic/py2deb/issues/31 277 | -------------------------------------------------------------------------------- /constraints.txt: -------------------------------------------------------------------------------- 1 | # This is a pip constraints file that is used to preserve Python 2.6 2 | # compatibility (on Travis CI). Why I'm still doing that in 2018 is 3 | # a good question, maybe simply to prove that I can :-P. 4 | 5 | # flake8 3.0.0 drops explicit support for Python 2.6: 6 | # http://flake8.pycqa.org/en/latest/release-notes/3.0.0.html 7 | flake8 < 3.0.0 ; python_version < '2.7' 8 | 9 | # flake8-docstrings 1.0.0 switches from pep257 to pydocstyle and I haven't been 10 | # able to find a combination of versions of flake8-docstrings and pydocstyle 11 | # that actually works on Python 2.6. Here's the changelog: 12 | # https://gitlab.com/pycqa/flake8-docstrings/blob/master/HISTORY.rst 13 | flake8-docstrings < 1.0.0 ; python_version < '2.7' 14 | 15 | # Constrain pip to the (very old but nonetheless) most recent release of pip 16 | # that pip-accel and consequently py2deb are compatible with. An upcoming 17 | # refactor is intended to finally resolve this awkward situation :-). 18 | pip >= 7.0, < 7.2 19 | 20 | # pycparser < 2.19 drops Python 2.6 compatibility: 21 | # https://github.com/eliben/pycparser/blob/master/CHANGES 22 | pycparser < 2.19 ; python_version < '2.7' 23 | 24 | # pydocstyle 4.0.0 drops Python 2 compatibility: 25 | # http://www.pydocstyle.org/en/5.0.2/release_notes.html#july-6th-2019 26 | pydocstyle < 4.0.0 ; python_version < '3.0' 27 | 28 | # pyflakes 2.0.0 drops Python 2.6 compatibility: 29 | # https://github.com/PyCQA/pyflakes/blob/master/NEWS.txt 30 | pyflakes < 2.0.0 ; python_version < '2.7' 31 | 32 | # pytest 3.3 drops Python 2.6 compatibility: 33 | # https://docs.pytest.org/en/latest/changelog.html#pytest-3-3-0-2017-11-23 34 | pytest < 3.3 ; python_version < '2.7' 35 | 36 | # setuptools 45 drops Python 2.7 compatibility: 37 | # https://setuptools.readthedocs.io/en/latest/history.html#v45-0-0 38 | setuptools < 45 39 | 40 | # tox 3.0.0 drops Python 2.6 compatibility: 41 | # https://tox.readthedocs.io/en/latest/changelog.html#v3-0-0-2018-04-02 42 | tox < 3.0.0 43 | 44 | # virtualenv release 16.0.0 drops Python 2.6 compatibility: 45 | # https://virtualenv.pypa.io/en/latest/changes/ 46 | # Gotcha: Because we use 'tox' running on Python 2.7 to create a Python 2.6 47 | # virtual environment, we can't constrain to python_version < '2.7'! 48 | virtualenv < 16.0.0 49 | 50 | # wheel 0.30.0 drops Python 2.6 compatibility: 51 | # https://pypi.org/project/wheel 52 | wheel < 0.30.0 ; python_version < '2.7' 53 | 54 | # zipp 2 drops Python 2 compatibility: 55 | # https://github.com/jaraco/zipp/issues/50 56 | zipp < 2 ; python_version < '3.0' 57 | 58 | # The following constraints are used by the py2deb test suite to make sure it 59 | # doesn't download incompatible packages. Nowadays pip mostly does this 60 | # automatically however the old version of pip that pip-accel depends on 61 | # doesn't have that feature yet, hence the need for explicit constraints. 62 | 63 | # cairocffi 1.0.0 (a dependency of WeasyPrint) drops Python 2 compatibility: 64 | # https://github.com/Kozea/cairocffi/blob/master/NEWS.rst#version-100 65 | cairocffi < 1.0.0 ; python_version < '3.0' 66 | 67 | # cssselect2 (a dependency of WeasyPrint) drops Python 2 compatibility: 68 | # https://github.com/Kozea/cssselect2/blob/master/CHANGES 69 | cssselect2 < 0.3.0 ; python_version < '3.0' 70 | 71 | # tinycss2 1.0.0 (a dependency of WeasyPrint) drops Python 2 compatibility: 72 | # https://tinycss2.readthedocs.io/en/latest/#version-1-0-0 73 | tinycss2 < 1.0.0 ; python_version < '3.0' 74 | -------------------------------------------------------------------------------- /docs/api.rst: -------------------------------------------------------------------------------- 1 | API documentation 2 | ================= 3 | 4 | The following documentation is based on the source code of version |release| of 5 | the `py2deb` package. The following modules are available: 6 | 7 | .. contents:: 8 | :local: 9 | 10 | :mod:`py2deb` 11 | ------------- 12 | 13 | .. automodule:: py2deb 14 | :members: 15 | 16 | :mod:`py2deb.cli` 17 | ----------------- 18 | 19 | .. automodule:: py2deb.cli 20 | :members: 21 | 22 | :mod:`py2deb.converter` 23 | ----------------------- 24 | 25 | .. automodule:: py2deb.converter 26 | :members: 27 | 28 | :mod:`py2deb.hooks` 29 | ------------------- 30 | 31 | .. automodule:: py2deb.hooks 32 | :members: 33 | 34 | :mod:`py2deb.namespaces` 35 | ------------------------ 36 | 37 | .. automodule:: py2deb.namespaces 38 | :members: 39 | 40 | :mod:`py2deb.package` 41 | --------------------- 42 | 43 | .. automodule:: py2deb.package 44 | :members: 45 | 46 | :mod:`py2deb.tests` 47 | ------------------- 48 | 49 | .. automodule:: py2deb.tests 50 | :members: 51 | 52 | :mod:`py2deb.utils` 53 | ------------------- 54 | 55 | .. automodule:: py2deb.utils 56 | :members: 57 | -------------------------------------------------------------------------------- /docs/changelog.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../CHANGELOG.rst 2 | -------------------------------------------------------------------------------- /docs/comparisons.rst: -------------------------------------------------------------------------------- 1 | Where does py2deb fit in? 2 | ========================= 3 | 4 | There are several projects out there that share similarities with py2deb, for 5 | example I know of stdeb_, dh-virtualenv_ and fpm_. For those who know these 6 | other projects already and are curious about where py2deb fits in, I would 7 | classify py2deb as a sort of pragmatic compromise between dh-virtualenv_ and 8 | stdeb_ (without the disadvantages that I see in both of these projects). 9 | 10 | Below I will attempt to provide a fair comparison between these projects. 11 | Please note that it is not my intention to discourage the use of any of these 12 | projects or to list just the down sides: They all have their place! Of course I 13 | do think `py2deb` has something to add to the mix, otherwise I wouldn't have 14 | created it :-). 15 | 16 | If you feel that another project should be discussed here or that an existing 17 | comparison is inaccurate then feel free to mention this on the `py2deb issue 18 | tracker`_. 19 | 20 | .. contents:: 21 | :local: 22 | 23 | The short comparison 24 | -------------------- 25 | 26 | In my research into `py2deb`, `stdeb` and `dh-virtualenv` I've come to a sort 27 | of realization about all of these projects that makes it fairly easy to 28 | differentiate them for those who have a passing familiarity with one or more of 29 | these projects: The projects can be placed on a spectrum ranging from very 30 | pragmatic (and dumb, to a certain extent :-) to very perfectionistic (and 31 | idealistic and fragile, to a certain extent). Based on my observations: 32 | 33 | - `dh-virtualenv` is a pragmatic solution to a concrete problem. It solves this 34 | single problem and seems to do so quite well. 35 | 36 | - `stdeb` is somewhat pragmatic in the sense that it tries to make the contents 37 | of the `Python Package Index`_ available to Debian based systems, but it is 38 | quite perfectionistic (idealistic) in how it goes about accomplishing this. 39 | When it works it results in fairly high quality conversions. 40 | 41 | - `py2deb` sits somewhere between `dh-virtualenv` and `stdeb`: 42 | 43 | - It allows complete requirement sets to be converted (similar to 44 | `dh-virtualenv`). 45 | 46 | - It converts requirement sets by generating individual binary packages 47 | (similar to `stdeb`). 48 | 49 | - It can convert requirement sets using a custom name and installation prefix 50 | to allow the same kind of isolation that `dh-virtualenv` provides. 51 | 52 | - It uses `dpkg-shlibdeps` to automatically track dependencies on system 53 | packages (inspired by `stdeb`). 54 | 55 | Comparison to stdeb 56 | ------------------- 57 | 58 | The stdeb_ program converts Python source distributions to Debian source 59 | packages which can then be compiled to Debian binary packages (optionally in a 60 | single call). 61 | 62 | Shared goals with stdeb 63 | ~~~~~~~~~~~~~~~~~~~~~~~ 64 | 65 | The `stdeb` and `py2deb` projects share very similar goals, in fact `py2deb` 66 | started out being based on `stdeb` but eventually reimplemented the required 67 | functionality on top of pip-accel_ and deb-pkg-tools_. The following goals are 68 | still shared between `stdeb` and `py2deb`: 69 | 70 | - Combine the power and ease of deployment of Debian packaging with the rich 71 | ecosystem of Python packages available on the `Python Package Index`_. 72 | 73 | - Provide users with a very easy way to take a Python package and convert it 74 | into a Debian binary package that is ready to install, without having to know 75 | the intricate details of the Debian packaging ecosystem. 76 | 77 | Notable differences with stdeb 78 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 79 | 80 | Although `py2deb` started out being a wrapper for `stdeb` the goals of the two 81 | projects have diverged quite a bit since then. Some notable differences: 82 | 83 | - `stdeb` starts by generating Debian source packages while `py2deb` generates 84 | Debian binary packages without intermediate Debian source packages: 85 | 86 | - `stdeb` works by converting a Python package to a Debian source package 87 | that uses the existing Debian Python packaging mechanisms. The Debian 88 | source package can then be compiled into a Debian binary package. These two 89 | actions can optionally be combined into a single invocation. `stdeb` is 90 | intended to generate Python Debian packages that comply to the Debian 91 | packaging policies as much as possible (this is my interpretation). 92 | 93 | - For example Python modules are installed in the pyshared_ directory so 94 | that multiple Python versions can use the modules. The advantages of this 95 | are clear, but the main disadvantage is that `stdeb` is sensitive to 96 | changes in Debian packaging infrastructure. For example it doesn't run on 97 | older versions of Ubuntu Linux (at one point this was a requirement for 98 | me). `py2deb` on the other hand is kind of dumb but works almost 99 | everywhere. 100 | 101 | - `py2deb` never generates Debian source packages, instead it generates 102 | Debian binary packages directly. This means `py2deb` doesn't use or 103 | integrate with the Debian Python packaging mechanisms. This was a conscious 104 | choice that I'll elaborate on further in the following point. 105 | 106 | - The main use case of `stdeb` is to convert individual Python packages to 107 | Debian packages that are installed system wide under the `python-*` name 108 | prefix. On the other hand `py2deb` always converts complete dependency sets 109 | (in fact `py2deb` started out as a wrapper for `stdeb` that just added the 110 | "please convert a complete dependency set for me" aspect). Some consequences 111 | of this: 112 | 113 | - `stdeb` works fine when converting a couple of Python packages individually 114 | but if you want to convert a large dependency set it quickly becomes hairy 115 | and fragile due to scripting of `stdeb`, conflicts with existing system 116 | packages and other reasons. If you want this process to run automatically 117 | and reliably without supervision then I personally wouldn't recommend 118 | `stdeb` - it has given me quite a few headaches because I was pushing 119 | `stdeb` way beyond its intended use case (my fault entirely, I'm not 120 | blaming the tool). 121 | 122 | - The larger the dependency set given to `py2deb`, the larger the risk that 123 | conflicts will occur between Python packages from the official repositories 124 | versus the packages converted by `py2deb`. This is why `py2deb` eventually 125 | stopped being based on `stdeb`: In order to add the ability to install 126 | converted packages under a custom name prefix and installation prefix. 127 | When used in this mode `py2deb` is something of a pragmatic compromise 128 | between `stdeb` and `dh-virtualenv`. 129 | 130 | Comparison to dh-virtualenv 131 | --------------------------- 132 | 133 | The dh-virtualenv_ tool provides helpers to easily create a Debian source 134 | package that takes a `pip requirements file`_ and builds a Python virtual 135 | environment that is then packaged as a Debian binary package. 136 | 137 | Shared goals with dh-virtualenv 138 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 139 | 140 | The following goals are shared between `dh-virtualenv` and `py2deb`: 141 | 142 | - Combine the power and ease of deployment of Debian packaging with the rich 143 | ecosystem of Python packages available on the `Python Package Index`_. 144 | 145 | - Easily deploy Python based applications with complex dependency sets which 146 | may conflict with system wide Python packages (`dh-virtualenv` always 147 | provides this isolation while `py2deb` provides the option but doesn't 148 | enforce it). 149 | 150 | Notable differences with dh-virtualenv 151 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 152 | 153 | The following notable differences can be observed: 154 | 155 | - `dh-virtualenv` requires creating a Debian source package in order to 156 | generate a Debian binary package while `py2deb` focuses exclusively on 157 | generating Debian binary packages. Both approaches are valid and have 158 | advantages and disadvantages: 159 | 160 | - The use of `dh-virtualenv` requires a certain amount of knowledge about how 161 | to create, manage and build Debian source packages. 162 | 163 | - The use of `py2deb` requires fairly little knowledge about Debian packaging 164 | and it specifically doesn't require any knowledge about Debian source 165 | packages. 166 | 167 | - `dh-virtualenv` includes a complete requirement set in a single binary 168 | package while `py2deb` converts each requirement individually (whether 169 | configured to use an isolated name space or not): 170 | 171 | - An advantage of `dh-virtualenv` here is that the generated ``*.deb`` is 172 | completely self contained. The disadvantage of this is that when you update 173 | only a few requirements in a large requirement set you get to rebuild, 174 | redownload and reinstall the complete requirement set anyway. 175 | 176 | - For `py2deb` the situation is the inverse: Generated binary packages are 177 | not self contained (each requirement gets a separate ``*.deb`` archive). 178 | This means that when only a few requirements in a large requirement set are 179 | updated only those requirements are rebuilt, redownloaded and reinstalled. 180 | 181 | Comparison to fpm 182 | ----------------- 183 | 184 | The fpm_ program is a generic package converter that supports multiple input 185 | formats (Python packages, Ruby packages, etc.) and multiple output formats 186 | (Debian binary packages, Red Hat binary packages, etc.). 187 | 188 | Shared goals with fpm 189 | ~~~~~~~~~~~~~~~~~~~~~ 190 | 191 | The `fpm` and `py2deb` projects in the end have very different goals but there 192 | is at least one shared goal: 193 | 194 | - Provide users with a very easy way to take a Python package and convert it 195 | into a Debian binary package that is ready to install, without having to know 196 | the intricate details of the Debian packaging ecosystem. 197 | 198 | Notable differences with fpm 199 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 200 | 201 | Here are some notable differences between `fpm` and `py2deb`: 202 | 203 | - `fpm` is a generic package converter while `py2deb` specializes in conversion 204 | of Python to Debian packages. This makes `fpm` more like a Swiss Army knife 205 | while `py2deb` has a very specialized use case for which it is actually 206 | specialized (`py2deb` is smarter about Python to Debian package conversion). 207 | 208 | - With `py2deb` it is very easy to convert packages using a custom name and 209 | installation prefix, allowing conversion of large/complex requirement sets 210 | that would inevitably conflict with Debian packages from official 211 | repositories (e.g. because of older or newer versions). 212 | 213 | - `py2deb` recognizes dependencies on system packages (libraries) and embeds 214 | them in the dependencies of the generated Debian packages. This is not so 215 | important when `py2deb` is used on the system where the converted packages 216 | will be installed (the dependencies will already have been installed, 217 | otherwise the package couldn't have been built and converted) but it's 218 | essential when the converted packages will be deployed to other systems. 219 | 220 | .. _deb-pkg-tools: https://pypi.org/project/deb-pkg-tools 221 | .. _dh-virtualenv: https://github.com/spotify/dh-virtualenv 222 | .. _fpm: https://github.com/jordansissel/fpm 223 | .. _pip requirements file: https://pip.pypa.io/en/latest/user_guide.html#requirements-files 224 | .. _pip-accel: https://github.com/paylogic/pip-accel 225 | .. _py2deb issue tracker: https://github.com/paylogic/py2deb/issues 226 | .. _pyshared: https://www.debian.org/doc/packaging-manuals/python-policy/ch-python.html#s-paths 227 | .. _Python Package Index: http://pypi.python.org/ 228 | .. _stdeb: https://github.com/astraw/stdeb 229 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | """ 2 | Documentation build configuration file for the `py2deb` package. 3 | 4 | This Python script contains the Sphinx configuration for building the 5 | documentation of the `py2deb` project. This file is execfile()d with the 6 | current directory set to its containing dir. 7 | """ 8 | 9 | import os 10 | import sys 11 | 12 | # Add the 'py2deb' source distribution's root directory to the module path. 13 | sys.path.insert(0, os.path.abspath(os.pardir)) 14 | 15 | # -- General configuration ----------------------------------------------------- 16 | 17 | # Sphinx extension module names. 18 | extensions = [ 19 | 'sphinx.ext.autodoc', 20 | 'sphinx.ext.doctest', 21 | 'sphinx.ext.intersphinx', 22 | 'sphinx.ext.viewcode', 23 | 'humanfriendly.sphinx', 24 | 'property_manager.sphinx', 25 | ] 26 | 27 | # Configuration for the `autodoc' extension. 28 | autodoc_member_order = 'bysource' 29 | 30 | # Paths that contain templates, relative to this directory. 31 | templates_path = ['templates'] 32 | 33 | # The suffix of source filenames. 34 | source_suffix = '.rst' 35 | 36 | # The master toctree document. 37 | master_doc = 'index' 38 | 39 | # General information about the project. 40 | project = u'py2deb' 41 | copyright = u'2020, Paylogic International (Arjan Verwer & Peter Odding)' 42 | 43 | # The version info for the project you're documenting, acts as replacement for 44 | # |version| and |release|, also used in various other places throughout the 45 | # built documents. 46 | 47 | # Find the package version and make it the release. 48 | from py2deb import __version__ as py2deb_version # noqa 49 | 50 | # The short X.Y version. 51 | version = '.'.join(py2deb_version.split('.')[:2]) 52 | 53 | # The full version, including alpha/beta/rc tags. 54 | release = py2deb_version 55 | 56 | # The language for content autogenerated by Sphinx. Refer to documentation 57 | # for a list of supported languages. 58 | language = 'en' 59 | 60 | # List of patterns, relative to source directory, that match files and 61 | # directories to ignore when looking for source files. 62 | exclude_patterns = ['build'] 63 | 64 | # If true, '()' will be appended to :func: etc. cross-reference text. 65 | add_function_parentheses = True 66 | 67 | # The name of the Pygments (syntax highlighting) style to use. 68 | pygments_style = 'sphinx' 69 | 70 | # Refer to the Python standard library. 71 | # From: http://twistedmatrix.com/trac/ticket/4582. 72 | intersphinx_mapping = { 73 | 'coloredlogs': ('https://coloredlogs.readthedocs.io/en/latest', None), 74 | 'debpkgtools': ('https://deb-pkg-tools.readthedocs.io/en/latest', None), 75 | 'executor': ('https://executor.readthedocs.io/en/latest', None), 76 | 'humanfriendly': ('https://humanfriendly.readthedocs.io/en/latest', None), 77 | 'pipaccel': ('https://pip-accel.readthedocs.io/en/latest', None), 78 | 'propertymanager': ('https://property-manager.readthedocs.io/en/latest', None), 79 | 'python2': ('https://docs.python.org/2', None), 80 | 'python3': ('https://docs.python.org/3', None), 81 | 'setuptools': ('https://setuptools.readthedocs.io/en/latest', None), 82 | } 83 | 84 | # -- Options for HTML output --------------------------------------------------- 85 | 86 | # The theme to use for HTML and HTML Help pages. See the documentation for 87 | # a list of builtin themes. 88 | html_theme = 'nature' 89 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | py2deb: Python to Debian package converter 2 | ========================================== 3 | 4 | Welcome to the documentation of py2deb version |release|! 5 | The following sections are available: 6 | 7 | .. contents:: 8 | :local: 9 | 10 | User documentation 11 | ------------------ 12 | 13 | The readme is the best place to start reading, it's targeted at all users and 14 | documents the command line interface: 15 | 16 | .. toctree:: 17 | readme.rst 18 | 19 | The comparison between py2deb and similar tools can help to determine if py2deb 20 | can be useful for you. 21 | 22 | .. toctree:: 23 | comparisons.rst 24 | 25 | API documentation 26 | ----------------- 27 | 28 | The following API documentation is automatically generated from the source code: 29 | 30 | .. toctree:: 31 | api.rst 32 | 33 | Change log 34 | ---------- 35 | 36 | The change log lists notable changes to the project: 37 | 38 | .. toctree:: 39 | changelog.rst 40 | -------------------------------------------------------------------------------- /docs/readme.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../README.rst 2 | -------------------------------------------------------------------------------- /py2deb/__init__.py: -------------------------------------------------------------------------------- 1 | # py2deb: Python to Debian package converter. 2 | # 3 | # Authors: 4 | # - Arjan Verwer 5 | # - Peter Odding 6 | # Last Change: August 5, 2020 7 | # URL: https://py2deb.readthedocs.io 8 | 9 | """ 10 | The top level :mod:`py2deb` module contains only a version number. 11 | 12 | .. data:: __version__ 13 | 14 | The version number of the :pypi:`pydeb` package (a string). 15 | """ 16 | 17 | # Semi-standard module versioning. 18 | __version__ = '5.0' 19 | -------------------------------------------------------------------------------- /py2deb/cli.py: -------------------------------------------------------------------------------- 1 | # Command line interface for the `py2deb' program. 2 | # 3 | # Author: Peter Odding 4 | # Last Change: May 22, 2017 5 | # URL: https://py2deb.readthedocs.io 6 | 7 | """ 8 | Usage: py2deb [OPTIONS] ... 9 | 10 | Convert Python packages to Debian packages according to the given 11 | command line options (see below). The command line arguments are the 12 | same as accepted by the `pip install' command because py2deb invokes 13 | pip during the conversion process. This means you can name the 14 | package(s) to convert on the command line but you can also use 15 | `requirement files' if you prefer. 16 | 17 | If you want to pass command line options to pip (e.g. because you want 18 | to use a custom index URL or a requirements file) then you will need 19 | to tell py2deb where the options for py2deb stop and the options for 20 | pip begin. In such cases you can use the following syntax: 21 | 22 | $ py2deb -r /tmp -- -r requirements.txt 23 | 24 | So the `--' marker separates the py2deb options from the pip options. 25 | 26 | Supported options: 27 | 28 | -c, --config=FILENAME 29 | 30 | Load a configuration file. Because the command line arguments are processed 31 | in the given order, you have the choice and responsibility to decide if 32 | command line options override configuration file options or vice versa. 33 | Refer to the documentation for details on the configuration file format. 34 | 35 | The default configuration files /etc/py2deb.ini and ~/.py2deb.ini are 36 | automatically loaded if they exist. This happens before environment 37 | variables and command line options are processed. 38 | 39 | Can also be set using the environment variable $PY2DEB_CONFIG. 40 | 41 | -r, --repository=DIRECTORY 42 | 43 | Change the directory where *.deb archives are stored. Defaults to 44 | the system wide temporary directory (which is usually /tmp). If 45 | this directory doesn't exist py2deb refuses to run. 46 | 47 | Can also be set using the environment variable $PY2DEB_REPOSITORY. 48 | 49 | --use-system-package=PYTHON_PACKAGE_NAME,DEBIAN_PACKAGE_NAME 50 | 51 | Exclude a Python package (the name before the comma) from conversion and 52 | replace references to the Python package with a specific Debian package 53 | name. This allows you to use system packages for specific Python 54 | requirements. 55 | 56 | --name-prefix=PREFIX 57 | 58 | Set the name prefix used during the name conversion from Python to 59 | Debian packages. Defaults to `python'. The name prefix and package 60 | names are always delimited by a dash. 61 | 62 | Can also be set using the environment variable $PY2DEB_NAME_PREFIX. 63 | 64 | --no-name-prefix=PYTHON_PACKAGE_NAME 65 | 66 | Exclude a Python package from having the name prefix applied 67 | during the package name conversion. This is useful to avoid 68 | awkward repetitions. 69 | 70 | --rename=PYTHON_PACKAGE_NAME,DEBIAN_PACKAGE_NAME 71 | 72 | Override the package name conversion algorithm for the given pair 73 | of package names. Useful if you don't agree with the algorithm :-) 74 | 75 | --install-prefix=DIRECTORY 76 | 77 | Override the default system wide installation prefix. By setting 78 | this to anything other than `/usr' or `/usr/local' you change the 79 | way py2deb works. It will build packages with a file system layout 80 | similar to a Python virtual environment, except there will not be 81 | a Python executable: The packages are meant to be loaded by 82 | modifying Python's module search path. Refer to the documentation 83 | for details. 84 | 85 | Can also be set using the environment variable $PY2DEB_INSTALL_PREFIX. 86 | 87 | --install-alternative=LINK,PATH 88 | 89 | Use Debian's `update-alternatives' system to add an executable 90 | that's installed in a custom installation prefix (see above) to 91 | the system wide executable search path. Refer to the documentation 92 | for details. 93 | 94 | --python-callback=EXPRESSION 95 | 96 | Set a Python callback to be called during the conversion process. Refer to 97 | the documentation for details about the use of this feature and the syntax 98 | of EXPRESSION. 99 | 100 | Can also be set using the environment variable $PY2DEB_CALLBACK. 101 | 102 | --report-dependencies=FILENAME 103 | 104 | Add the Debian relationships needed to depend on the converted 105 | package(s) to the given control file. If the control file already 106 | contains relationships the additional relationships will be added 107 | to the control file; they won't overwrite existing relationships. 108 | 109 | -y, --yes 110 | 111 | Instruct pip-accel to automatically install build time dependencies 112 | where possible. Refer to the pip-accel documentation for details. 113 | 114 | Can also be set using the environment variable $PY2DEB_AUTO_INSTALL. 115 | 116 | -v, --verbose 117 | 118 | Make more noise :-). 119 | 120 | -h, --help 121 | 122 | Show this message and exit. 123 | """ 124 | 125 | # Standard library modules. 126 | import getopt 127 | import logging 128 | import os 129 | import sys 130 | 131 | # External dependencies. 132 | import coloredlogs 133 | from deb_pkg_tools.control import patch_control_file 134 | from humanfriendly.terminal import usage, warning 135 | 136 | # Modules included in our package. 137 | from py2deb.converter import PackageConverter 138 | 139 | # Initialize a logger. 140 | logger = logging.getLogger(__name__) 141 | 142 | 143 | def main(): 144 | """Command line interface for the ``py2deb`` program.""" 145 | # Configure terminal output. 146 | coloredlogs.install() 147 | try: 148 | # Initialize a package converter. 149 | converter = PackageConverter() 150 | # Parse and validate the command line options. 151 | options, arguments = getopt.getopt(sys.argv[1:], 'c:r:yvh', [ 152 | 'config=', 'repository=', 'use-system-package=', 'name-prefix=', 153 | 'no-name-prefix=', 'rename=', 'install-prefix=', 154 | 'install-alternative=', 'python-callback=', 'report-dependencies=', 155 | 'yes', 'verbose', 'help', 156 | ]) 157 | control_file_to_update = None 158 | for option, value in options: 159 | if option in ('-c', '--config'): 160 | converter.load_configuration_file(value) 161 | elif option in ('-r', '--repository'): 162 | converter.set_repository(value) 163 | elif option == '--use-system-package': 164 | python_package_name, _, debian_package_name = value.partition(',') 165 | converter.use_system_package(python_package_name, debian_package_name) 166 | elif option == '--name-prefix': 167 | converter.set_name_prefix(value) 168 | elif option == '--no-name-prefix': 169 | converter.rename_package(value, value) 170 | elif option == '--rename': 171 | python_package_name, _, debian_package_name = value.partition(',') 172 | converter.rename_package(python_package_name, debian_package_name) 173 | elif option == '--install-prefix': 174 | converter.set_install_prefix(value) 175 | elif option == '--install-alternative': 176 | link, _, path = value.partition(',') 177 | converter.install_alternative(link, path) 178 | elif option == '--python-callback': 179 | converter.set_python_callback(value) 180 | elif option == '--report-dependencies': 181 | control_file_to_update = value 182 | if not os.path.isfile(control_file_to_update): 183 | msg = "The given control file doesn't exist! (%s)" 184 | raise Exception(msg % control_file_to_update) 185 | elif option in ('-y', '--yes'): 186 | converter.set_auto_install(True) 187 | elif option in ('-v', '--verbose'): 188 | coloredlogs.increase_verbosity() 189 | elif option in ('-h', '--help'): 190 | usage(__doc__) 191 | return 192 | else: 193 | assert False, "Unhandled option!" 194 | except Exception as e: 195 | warning("Failed to parse command line arguments: %s", e) 196 | sys.exit(1) 197 | # Convert the requested package(s). 198 | try: 199 | if arguments: 200 | archives, relationships = converter.convert(arguments) 201 | if relationships and control_file_to_update: 202 | patch_control_file(control_file_to_update, dict(depends=relationships)) 203 | else: 204 | usage(__doc__) 205 | except Exception: 206 | logger.exception("Caught an unhandled exception!") 207 | sys.exit(1) 208 | -------------------------------------------------------------------------------- /py2deb/converter.py: -------------------------------------------------------------------------------- 1 | # py2deb: Python to Debian package converter. 2 | # 3 | # Authors: 4 | # - Arjan Verwer 5 | # - Peter Odding 6 | # Last Change: August 6, 2020 7 | # URL: https://py2deb.readthedocs.io 8 | 9 | """ 10 | The :mod:`py2deb.converter` module contains the high level conversion logic. 11 | 12 | This module defines the :class:`PackageConverter` class which provides the 13 | intended way for external Python code to interface with `py2deb`. The separation 14 | between the :class:`PackageConverter` and :class:`.PackageToConvert` 15 | classes is somewhat crude (because neither class can work without the other) 16 | but the idea is to separate the high level conversion logic from the low level 17 | conversion logic. 18 | """ 19 | 20 | # Standard library modules. 21 | import importlib 22 | import logging 23 | import os 24 | import re 25 | import shutil 26 | import tempfile 27 | 28 | # External dependencies. 29 | from property_manager import PropertyManager, cached_property, lazy_property, mutable_property, set_property 30 | from deb_pkg_tools.cache import get_default_cache 31 | from deb_pkg_tools.checks import check_duplicate_files 32 | from deb_pkg_tools.utils import find_debian_architecture 33 | from humanfriendly import coerce_boolean 34 | from humanfriendly.text import compact 35 | from pip_accel import PipAccelerator 36 | from pip_accel.config import Config as PipAccelConfig 37 | from six.moves import configparser 38 | 39 | # Modules included in our package. 40 | from py2deb.utils import ( 41 | PackageRepository, 42 | convert_package_name, 43 | default_name_prefix, 44 | normalize_package_name, 45 | normalize_package_version, 46 | package_names_match, 47 | tokenize_version, 48 | ) 49 | from py2deb.package import PackageToConvert 50 | 51 | # Initialize a logger. 52 | logger = logging.getLogger(__name__) 53 | 54 | MACHINE_ARCHITECTURE_MAPPING = dict(i686='i386', x86_64='amd64', armv6l='armhf') 55 | """ 56 | Mapping of supported machine architectures (a dictionary). 57 | 58 | The keys are the names reported by :func:`os.uname()` and the values are 59 | machine architecture labels used in the Debian packaging system. 60 | """ 61 | 62 | 63 | class PackageConverter(PropertyManager): 64 | 65 | """The external interface of `py2deb`, the Python to Debian package converter.""" 66 | 67 | def __init__(self, load_configuration_files=True, load_environment_variables=True, **options): 68 | """ 69 | Initialize a Python to Debian package converter. 70 | 71 | :param load_configuration_files: 72 | 73 | When :data:`True` (the default) 74 | :func:`load_default_configuration_files()` is called automatically. 75 | 76 | :param load_environment_variables: 77 | 78 | When :data:`True` (the default) 79 | :func:`load_environment_variables()` is called automatically. 80 | 81 | :param options: 82 | 83 | Any keyword arguments are passed on to the initializer of the 84 | :class:`~property_manager.PropertyManager` class. 85 | """ 86 | # Initialize our superclass. 87 | super(PackageConverter, self).__init__(**options) 88 | # Initialize our internal state. 89 | self.pip_accel = PipAccelerator(PipAccelConfig()) 90 | if load_configuration_files: 91 | self.load_default_configuration_files() 92 | if load_environment_variables: 93 | self.load_environment_variables() 94 | 95 | @lazy_property 96 | def alternatives(self): 97 | """ 98 | The update-alternatives_ configuration (a set of tuples). 99 | 100 | The value of this property is a set of :class:`set` of tuples with two 101 | strings each (the strings passed to :func:`install_alternative()`). 102 | It's used by :func:`~py2deb.hooks.create_alternatives()` and 103 | :func:`~py2deb.hooks.cleanup_alternatives()` during installation and 104 | removal of the generated package. 105 | """ 106 | return set() 107 | 108 | @cached_property 109 | def debian_architecture(self): 110 | """ 111 | The Debian architecture of the current environment (a string). 112 | 113 | This logic was originally implemented in py2deb but has since been 114 | moved to :func:`deb_pkg_tools.utils.find_debian_architecture()`. 115 | This property remains as a convenient shortcut. 116 | """ 117 | return find_debian_architecture() 118 | 119 | @mutable_property 120 | def install_prefix(self): 121 | """ 122 | The installation prefix for converted packages (a string, defaults to ``/usr``). 123 | 124 | To generate system wide packages one of the installation prefixes 125 | ``/usr`` or ``/usr/local`` should be used. Setting this property to any 126 | other value will create packages using a "custom installation prefix" 127 | that's not included in :data:`sys.path` by default. 128 | 129 | The installation prefix directory doesn't have to exist on the system 130 | where the package is converted and will be automatically created on the 131 | system where the package is installed. 132 | 133 | .. versionadded:: 2.0 134 | 135 | Before his property became part of the documented and public API in 136 | release 2.0 the setter :func:`set_install_prefix()` was the only 137 | documented interface. The use of this setter is no longer required 138 | but still allowed. 139 | """ 140 | return '/usr' 141 | 142 | @mutable_property 143 | def lintian_enabled(self): 144 | """ 145 | :data:`True` to enable Lintian_, :data:`False` to disable it (defaults to :data:`True`). 146 | 147 | If this is :data:`True` then Lintian_ will automatically be run after 148 | each package is converted to sanity check the result. Any problems 149 | found by Lintian are information intended for the operator, that is to 150 | say they don't cause py2deb to fail. 151 | """ 152 | return True 153 | 154 | @lintian_enabled.setter 155 | def lintian_enabled(self, value): 156 | """Automatically coerce :attr:`lintian_enabled` to a boolean value.""" 157 | set_property(self, 'lintian_enabled', coerce_boolean(value)) 158 | 159 | @lazy_property 160 | def lintian_ignore(self): 161 | """A list of strings with Lintian tags to ignore.""" 162 | return [ 163 | 'binary-without-manpage', 164 | 'changelog-file-missing-in-native-package', 165 | 'debian-changelog-file-missing', 166 | 'embedded-javascript-library', 167 | 'extra-license-file', 168 | 'unknown-control-interpreter', 169 | 'unusual-control-interpreter', 170 | 'vcs-field-uses-unknown-uri-format', 171 | ] 172 | 173 | @lazy_property 174 | def name_mapping(self): 175 | """ 176 | Mapping of Python package names to Debian package names (a dictionary). 177 | 178 | The :attr:`name_mapping` property enables renaming of packages during 179 | the conversion process. The keys as well as the values of the 180 | dictionary are expected to be lowercased strings. 181 | 182 | .. versionadded:: 2.0 183 | 184 | Before his property became part of the documented and public API in 185 | release 2.0 the :func:`rename_package()` method was the only 186 | documented interface. The use of this setter is no longer required 187 | but still allowed. 188 | """ 189 | return {} 190 | 191 | @mutable_property 192 | def name_prefix(self): 193 | """ 194 | The name prefix for converted packages (a string). 195 | 196 | The default value of :attr:`name_prefix` depends on 197 | the Python interpreter that's used to run py2deb: 198 | 199 | - On Python 2 the default name prefix is ``python``. 200 | - On Python 3 the default name prefix is ``python3``. 201 | - On PyPy_ 2 the default name prefix is ``pypy``. 202 | - On PyPy_ 3 the default name prefix is ``pypy3``. 203 | 204 | When one of these default name prefixes is used, converted packages may 205 | conflict with system wide packages provided by Debian / Ubuntu. If this 206 | starts to bite then consider changing the name and installation prefix. 207 | 208 | .. versionadded:: 2.0 209 | 210 | Before his property became part of the documented and public API in 211 | release 2.0 the setter :func:`set_name_prefix()` was the only 212 | documented interface. The use of this setter is no longer required 213 | but still allowed. 214 | 215 | Release 2.0 introduced the alternative default name prefixes 216 | ``pypy`` and ``python3``. Before that release the default name 217 | prefix ``python`` was (erroneously) used for all interpreters. 218 | 219 | .. _PyPy: https://en.wikipedia.org/wiki/PyPy 220 | """ 221 | return default_name_prefix() 222 | 223 | @mutable_property 224 | def prerelease_workaround(self): 225 | """ 226 | Whether to enable the pre-release workaround in :func:`.normalize_package_version()` (a boolean). 227 | 228 | By setting this to :data:`False` converted version numbers will match 229 | those generated by py2deb 0.25 and earlier. Release 1.0 introduced the 230 | pre-release workaround and release 2.1 added the option to control 231 | backwards compatibility in this respect. 232 | """ 233 | return True 234 | 235 | @mutable_property 236 | def python_callback(self): 237 | """ 238 | An optional Python callback to be called during the conversion process (defaults to :data:`None`). 239 | 240 | You can set the value of :attr:`python_callback` to one of the following: 241 | 242 | 1. A callable object (to be provided by Python API callers). 243 | 244 | 2. A string containing the pathname of a Python script and the name of 245 | a callable, separated by a colon. The Python script will be loaded 246 | using :keyword:`exec`. 247 | 248 | 3. A string containing the "dotted path" of a Python module and the 249 | name of a callable, separated by a colon. The Python module will be 250 | loaded using :func:`importlib.import_module()`. 251 | 252 | 4. Any value that evaluates to :data:`False` will clear an existing 253 | callback (if any). 254 | 255 | The callback will be called at the very last step before the binary 256 | package's metadata and contents are packaged as a ``*.deb`` archive. 257 | This allows arbitrary manipulation of resulting binary packages, e.g. 258 | changing package metadata or files to be packaged. An example use case: 259 | 260 | - Consider a dependency set (group of related packages) that has 261 | previously been converted and deployed. 262 | 263 | - A new version of the dependency set switches from Python package A to 264 | Python package B, where the two Python packages contain conflicting 265 | files (installed in the same location). This could happen when 266 | switching to a project's fork. 267 | 268 | - A deployment of the new dependency set will conflict with existing 269 | installations due to "unrelated" packages (in the eyes of :man:`apt` 270 | and :man:`dpkg`) installing the same files. 271 | 272 | - By injecting a custom Python callback the user can mark package B as 273 | "replacing" and "breaking" package A. Refer to `section 7.6`_ of the 274 | Debian policy manual for details about the required binary control 275 | fields (hint: ``Replaces:`` and ``Breaks:``). 276 | 277 | .. warning:: The callback is responsible for not making changes that 278 | would break the installation of the converted dependency 279 | set! 280 | 281 | :raises: The following exceptions can be raised when you set this property: 282 | 283 | - :exc:`~exceptions.ValueError` when you set this to something 284 | that's not callable and cannot be converted to a callable. 285 | 286 | - :exc:`~exceptions.ImportError` when the expression contains 287 | a dotted path that cannot be imported. 288 | 289 | .. versionadded:: 2.0 290 | 291 | Before his property became part of the documented and public API in 292 | release 2.0 the setter :func:`set_python_callback()` was the only 293 | documented way to configure the callback. The use of this setter is 294 | no longer required but still allowed. 295 | 296 | .. _section 7.6: https://www.debian.org/doc/debian-policy/ch-relationships.html#s-replaces 297 | """ 298 | 299 | @python_callback.setter 300 | def python_callback(self, value): 301 | """Automatically coerce :attr:`python_callback` to a callable value.""" 302 | if value: 303 | # Python callers get to pass a callable directly. 304 | if not callable(value): 305 | expression = value 306 | # Otherwise we expect a string to parse (from a command line 307 | # argument, environment variable or configuration file). 308 | callback_path, _, callback_name = expression.partition(':') 309 | if os.path.isfile(callback_path): 310 | # Callback specified as Python script. 311 | script_name = os.path.basename(callback_path) 312 | if script_name.endswith('.py'): 313 | script_name, _ = os.path.splitext(script_name) 314 | environment = dict(__file__=callback_path, __name__=script_name) 315 | logger.debug("Loading Python callback from pathname: %s", callback_path) 316 | with open(callback_path) as handle: 317 | exec(handle.read(), environment) 318 | value = environment.get(callback_name) 319 | else: 320 | # Callback specified as `dotted path'. 321 | logger.debug("Loading Python callback from dotted path: %s", callback_path) 322 | module = importlib.import_module(callback_path) 323 | value = getattr(module, callback_name, None) 324 | if not callable(value): 325 | raise ValueError(compact(""" 326 | The Python callback expression {expr} didn't result in 327 | a valid callable! (result: {value}) 328 | """, expr=expression, value=value)) 329 | else: 330 | value = None 331 | set_property(self, 'python_callback', value) 332 | 333 | @mutable_property(cached=True) 334 | def repository(self): 335 | """ 336 | The directory where py2deb stores generated ``*.deb`` archives (a :class:`.PackageRepository` object). 337 | 338 | By default the system wide temporary files directory is used as the 339 | repository directory (usually this is ``/tmp``) but it's expected that 340 | most callers will want to change this. 341 | 342 | .. versionadded:: 2.0 343 | 344 | Before his property became part of the documented and public API in 345 | release 2.0 the :func:`set_repository()` method was the only 346 | documented interface. The use of this method is no longer required 347 | but still allowed. 348 | """ 349 | return PackageRepository(tempfile.gettempdir()) 350 | 351 | @repository.setter 352 | def repository(self, value): 353 | """Automatically coerce :attr:`repository` values.""" 354 | directory = os.path.abspath(value) 355 | if not os.path.isdir(directory): 356 | msg = "Repository directory doesn't exist! (%s)" 357 | raise ValueError(msg % directory) 358 | set_property(self, 'repository', PackageRepository(directory)) 359 | 360 | @lazy_property 361 | def scripts(self): 362 | """ 363 | Mapping of Python package names to shell commands (a dictionary). 364 | 365 | The keys of this dictionary are expected to be lowercased strings. 366 | 367 | .. versionadded:: 2.0 368 | 369 | Before his property became part of the documented and public API in 370 | release 2.0 the :func:`set_conversion_command()` method was the only 371 | documented interface. The use of this method is no longer required 372 | but still allowed. 373 | """ 374 | return {} 375 | 376 | @lazy_property 377 | def system_packages(self): 378 | """ 379 | Mapping of Python package names to Debian package names (a dictionary). 380 | 381 | The :attr:`system_packages` property enables Python packages in a 382 | requirement set to be excluded from the package conversion process. Any 383 | references to excluded packages are replaced with a reference to the 384 | corresponding system package. The keys as well as the values of the 385 | dictionary are expected to be lowercased strings. 386 | 387 | .. versionadded:: 2.0 388 | 389 | Before his property became part of the documented and public API in 390 | release 2.0 the :func:`use_system_package()` method was the only 391 | documented interface. The use of this method is no longer required 392 | but still allowed. 393 | """ 394 | return {} 395 | 396 | def install_alternative(self, link, path): 397 | r""" 398 | Install system wide link for program installed in custom installation prefix. 399 | 400 | Use Debian's update-alternatives_ system to add an executable that's 401 | installed in a custom installation prefix to the system wide executable 402 | search path using a symbolic link. 403 | 404 | :param link: The generic name for the master link (a string). This is 405 | the first argument passed to ``update-alternatives --install``. 406 | :param path: The alternative being introduced for the master link (a 407 | string). This is the third argument passed to 408 | ``update-alternatives --install``. 409 | :raises: :exc:`~exceptions.ValueError` when one of the paths is not 410 | provided (e.g. an empty string). 411 | 412 | If this is a bit vague, consider the following example: 413 | 414 | .. code-block:: sh 415 | 416 | $ py2deb --name-prefix=py2deb \ 417 | --no-name-prefix=py2deb \ 418 | --install-prefix=/usr/lib/py2deb \ 419 | --install-alternative=/usr/bin/py2deb,/usr/lib/py2deb/bin/py2deb \ 420 | py2deb==0.1 421 | 422 | This example will convert `py2deb` and its dependencies using a custom 423 | name prefix and a custom installation prefix which means the ``py2deb`` 424 | program is not available on the default executable search path. This is 425 | why update-alternatives_ is used to create a symbolic link 426 | ``/usr/bin/py2deb`` which points to the program inside the custom 427 | installation prefix. 428 | 429 | .. _update-alternatives: http://manpages.debian.org/cgi-bin/man.cgi?query=update-alternatives 430 | """ 431 | if not link: 432 | raise ValueError("Please provide a nonempty name for the master link!") 433 | if not path: 434 | raise ValueError("Please provide a nonempty name for the alternative being introduced!") 435 | self.alternatives.add((link, path)) 436 | 437 | def rename_package(self, python_package_name, debian_package_name): 438 | """ 439 | Override the package name conversion algorithm for the given pair of names. 440 | 441 | :param python_package_name: The name of a Python package 442 | as found on PyPI (a string). 443 | :param debian_package_name: The name of the converted 444 | Debian package (a string). 445 | :raises: :exc:`~exceptions.ValueError` when a package name is not 446 | provided (e.g. an empty string). 447 | """ 448 | if not python_package_name: 449 | raise ValueError("Please provide a nonempty Python package name!") 450 | if not debian_package_name: 451 | raise ValueError("Please provide a nonempty Debian package name!") 452 | self.name_mapping[python_package_name.lower()] = debian_package_name.lower() 453 | 454 | def set_auto_install(self, enabled): 455 | """ 456 | Enable or disable automatic installation of build time dependencies. 457 | 458 | :param enabled: Any value, evaluated using 459 | :func:`~humanfriendly.coerce_boolean()`. 460 | """ 461 | self.pip_accel.config.auto_install = coerce_boolean(enabled) 462 | 463 | def set_conversion_command(self, python_package_name, command): 464 | """ 465 | Set shell command to be executed during conversion process. 466 | 467 | :param python_package_name: The name of a Python package 468 | as found on PyPI (a string). 469 | :param command: The shell command to execute (a string). 470 | :raises: :exc:`~exceptions.ValueError` when the package name or 471 | command is not provided (e.g. an empty string). 472 | 473 | The shell command is executed in the directory containing the Python 474 | module(s) that are to be installed by the converted package. 475 | 476 | .. warning:: This functionality allows arbitrary manipulation of the 477 | Python modules to be installed by the converted package. 478 | It should clearly be considered a last resort, only for 479 | for fixing things like packaging issues with Python 480 | packages that you can't otherwise change. 481 | 482 | For example old versions of Fabric_ bundle a copy of Paramiko_. Most 483 | people will never notice this because Python package managers don't 484 | complain about this, they just blindly overwrite the files... Debian's 485 | packaging system is much more strict and will consider the converted 486 | Fabric and Paramiko packages as conflicting and thus broken. In this 487 | case you have two options: 488 | 489 | 1. Switch to a newer version of Fabric that no longer bundles Paramiko; 490 | 2. Use the conversion command ``rm -rf paramiko`` to convert Fabric 491 | (yes this is somewhat brute force :-). 492 | 493 | .. _Fabric: https://pypi.org/project/Fabric 494 | .. _Paramiko: https://pypi.org/project/paramiko 495 | """ 496 | if not python_package_name: 497 | raise ValueError("Please provide a nonempty Python package name!") 498 | if not command: 499 | raise ValueError("Please provide a nonempty shell command!") 500 | self.scripts[python_package_name.lower()] = command 501 | 502 | def set_install_prefix(self, directory): 503 | """ 504 | Set installation prefix to use during package conversion. 505 | 506 | The installation directory doesn't have to exist on the system where 507 | the package is converted. 508 | 509 | :param directory: The pathname of the directory where the converted 510 | packages should be installed (a string). 511 | :raises: :exc:`~exceptions.ValueError` when no installation prefix is 512 | provided (e.g. an empty string). 513 | """ 514 | if not directory: 515 | raise ValueError("Please provide a nonempty installation prefix!") 516 | self.install_prefix = directory 517 | 518 | def set_lintian_enabled(self, enabled): 519 | """ 520 | Enable or disable automatic Lintian_ checks after package building. 521 | 522 | :param enabled: Any value, evaluated using :func:`~humanfriendly.coerce_boolean()`. 523 | 524 | .. _Lintian: http://lintian.debian.org/ 525 | """ 526 | self.lintian_enabled = enabled 527 | 528 | def set_name_prefix(self, prefix): 529 | """ 530 | Set package name prefix to use during package conversion. 531 | 532 | :param prefix: The name prefix to use (a string). 533 | :raises: :exc:`~exceptions.ValueError` when no name prefix is 534 | provided (e.g. an empty string). 535 | """ 536 | if not prefix: 537 | raise ValueError("Please provide a nonempty name prefix!") 538 | self.name_prefix = prefix 539 | 540 | def set_python_callback(self, expression): 541 | """Set the value of :attr:`python_callback`.""" 542 | self.python_callback = expression 543 | 544 | def set_repository(self, directory): 545 | """ 546 | Set pathname of directory where `py2deb` stores converted packages. 547 | 548 | :param directory: The pathname of a directory (a string). 549 | :raises: :exc:`~exceptions.ValueError` when the directory doesn't 550 | exist. 551 | """ 552 | self.repository = directory 553 | 554 | def use_system_package(self, python_package_name, debian_package_name): 555 | """ 556 | Exclude a Python package from conversion. 557 | 558 | :param python_package_name: The name of a Python package 559 | as found on PyPI (a string). 560 | :param debian_package_name: The name of the Debian package that should 561 | be used to fulfill the dependency (a string). 562 | :raises: :exc:`~exceptions.ValueError` when a package name is not 563 | provided (e.g. an empty string). 564 | 565 | References to the Python package are replaced with a specific Debian 566 | package name. This allows you to use system packages for specific 567 | Python requirements. 568 | """ 569 | if not python_package_name: 570 | raise ValueError("Please provide a nonempty Python package name!") 571 | if not debian_package_name: 572 | raise ValueError("Please provide a nonempty Debian package name!") 573 | self.system_packages[python_package_name.lower()] = debian_package_name.lower() 574 | 575 | def load_environment_variables(self): 576 | """ 577 | Load configuration defaults from environment variables. 578 | 579 | The following environment variables are currently supported: 580 | 581 | - ``$PY2DEB_CONFIG`` 582 | - ``$PY2DEB_REPOSITORY`` 583 | - ``$PY2DEB_NAME_PREFIX`` 584 | - ``$PY2DEB_INSTALL_PREFIX`` 585 | - ``$PY2DEB_AUTO_INSTALL`` 586 | - ``$PY2DEB_LINTIAN`` 587 | """ 588 | for variable, setter in (('PY2DEB_CONFIG', self.load_configuration_file), 589 | ('PY2DEB_REPOSITORY', self.set_repository), 590 | ('PY2DEB_NAME_PREFIX', self.set_name_prefix), 591 | ('PY2DEB_INSTALL_PREFIX', self.set_install_prefix), 592 | ('PY2DEB_AUTO_INSTALL', self.set_auto_install), 593 | ('PY2DEB_LINTIAN', self.set_lintian_enabled), 594 | ('PY2DEB_CALLBACK', self.set_python_callback)): 595 | value = os.environ.get(variable) 596 | if value is not None: 597 | setter(value) 598 | 599 | def load_configuration_file(self, configuration_file): 600 | """ 601 | Load configuration defaults from a configuration file. 602 | 603 | :param configuration_file: The pathname of a configuration file (a 604 | string). 605 | :raises: :exc:`~exceptions.Exception` when the configuration file 606 | cannot be loaded. 607 | 608 | Below is an example of the available options, I assume that the mapping 609 | between the configuration options and the setters of 610 | :class:`PackageConverter` is fairly obvious (it should be :-). 611 | 612 | .. code-block:: ini 613 | 614 | # The `py2deb' section contains global options. 615 | [py2deb] 616 | repository = /tmp 617 | name-prefix = py2deb 618 | install-prefix = /usr/lib/py2deb 619 | auto-install = on 620 | lintian = on 621 | 622 | # The `alternatives' section contains instructions 623 | # for Debian's `update-alternatives' system. 624 | [alternatives] 625 | /usr/bin/py2deb = /usr/lib/py2deb/bin/py2deb 626 | 627 | # Sections starting with `package:' contain conversion options 628 | # specific to a package. 629 | [package:py2deb] 630 | no-name-prefix = true 631 | 632 | Note that the configuration options shown here are just examples, they 633 | are not the configuration defaults (they are what I use to convert 634 | `py2deb` itself). Package specific sections support the following 635 | options: 636 | 637 | **no-name-prefix**: 638 | A boolean indicating whether the configured name prefix should be 639 | applied or not. Understands ``true`` and ``false`` (``false`` is the 640 | default and you only need this option to change the default). 641 | 642 | **rename**: 643 | Gives an override for the package name conversion algorithm (refer to 644 | :func:`rename_package()` for details). 645 | 646 | **script**: 647 | Set a shell command to be executed during the conversion process 648 | (refer to :func:`set_conversion_command()` for details). 649 | """ 650 | # Load the configuration file. 651 | parser = configparser.RawConfigParser() 652 | configuration_file = os.path.expanduser(configuration_file) 653 | logger.debug("Loading configuration file: %s", configuration_file) 654 | files_loaded = parser.read(configuration_file) 655 | try: 656 | assert len(files_loaded) == 1 657 | assert os.path.samefile(configuration_file, files_loaded[0]) 658 | except Exception: 659 | msg = "Failed to load configuration file! (%s)" 660 | raise Exception(msg % configuration_file) 661 | # Apply the global settings in the configuration file. 662 | if parser.has_option('py2deb', 'repository'): 663 | self.set_repository(parser.get('py2deb', 'repository')) 664 | if parser.has_option('py2deb', 'name-prefix'): 665 | self.set_name_prefix(parser.get('py2deb', 'name-prefix')) 666 | if parser.has_option('py2deb', 'install-prefix'): 667 | self.set_install_prefix(parser.get('py2deb', 'install-prefix')) 668 | if parser.has_option('py2deb', 'auto-install'): 669 | self.set_auto_install(parser.get('py2deb', 'auto-install')) 670 | if parser.has_option('py2deb', 'lintian'): 671 | self.set_lintian_enabled(parser.get('py2deb', 'lintian')) 672 | if parser.has_option('py2deb', 'python-callback'): 673 | self.set_python_callback(parser.get('py2deb', 'python-callback')) 674 | # Apply the defined alternatives. 675 | if parser.has_section('alternatives'): 676 | for link, path in parser.items('alternatives'): 677 | self.install_alternative(link, path) 678 | # Apply any package specific settings. 679 | for section in parser.sections(): 680 | tag, _, package = section.partition(':') 681 | if tag == 'package': 682 | if parser.has_option(section, 'no-name-prefix'): 683 | if parser.getboolean(section, 'no-name-prefix'): 684 | self.rename_package(package, package) 685 | if parser.has_option(section, 'rename'): 686 | rename_to = parser.get(section, 'rename') 687 | self.rename_package(package, rename_to) 688 | if parser.has_option(section, 'script'): 689 | script = parser.get(section, 'script') 690 | self.set_conversion_command(package, script) 691 | 692 | def load_default_configuration_files(self): 693 | """ 694 | Load configuration options from default configuration files. 695 | 696 | The following default configuration file locations are checked: 697 | 698 | - ``/etc/py2deb.ini`` 699 | - ``~/.py2deb.ini`` 700 | 701 | :raises: :exc:`~exceptions.Exception` when a configuration file 702 | exists but cannot be loaded. 703 | """ 704 | for location in ('/etc/py2deb.ini', os.path.expanduser('~/.py2deb.ini')): 705 | if os.path.isfile(location): 706 | self.load_configuration_file(location) 707 | 708 | def convert(self, pip_install_arguments): 709 | """ 710 | Convert one or more Python packages to Debian packages. 711 | 712 | :param pip_install_arguments: The command line arguments to the ``pip 713 | install`` command. 714 | :returns: A tuple with two lists: 715 | 716 | 1. A list of strings containing the pathname(s) of the 717 | generated Debian package package archive(s). 718 | 719 | 2. A list of strings containing the Debian package 720 | relationship(s) required to depend on the converted 721 | package(s). 722 | :raises: :exc:`~deb_pkg_tools.checks.DuplicateFilesFound` if two 723 | converted package archives contain the same files (certainly 724 | not what you want within a set of dependencies). 725 | 726 | Here's an example of what's returned: 727 | 728 | >>> from py2deb.converter import PackageConverter 729 | >>> converter = PackageConverter() 730 | >>> archives, relationships = converter.convert(['py2deb']) 731 | >>> print(archives) 732 | ['/tmp/python-py2deb_0.18_all.deb'] 733 | >>> print(relationships) 734 | ['python-py2deb (=0.18)'] 735 | 736 | """ 737 | try: 738 | generated_archives = [] 739 | dependencies_to_report = [] 740 | # Download and unpack the requirement set and store the complete 741 | # set as an instance variable because transform_version() will need 742 | # it later on. 743 | self.packages_to_convert = list(self.get_source_distributions(pip_install_arguments)) 744 | # Convert packages that haven't been converted already. 745 | for package in self.packages_to_convert: 746 | # If the requirement is a 'direct' (non-transitive) requirement 747 | # it means the caller explicitly asked for this package to be 748 | # converted, so we add it to the list of converted dependencies 749 | # that we report to the caller once we've finished converting. 750 | if package.requirement.is_direct: 751 | dependencies_to_report.append('%s (= %s)' % (package.debian_name, package.debian_version)) 752 | if package.existing_archive: 753 | # If the same version of this package was converted in a 754 | # previous run we can save a lot of time by skipping it. 755 | logger.info("Package %s (%s) already converted: %s", 756 | package.python_name, package.python_version, 757 | package.existing_archive.filename) 758 | generated_archives.append(package.existing_archive) 759 | else: 760 | archive = package.convert() 761 | if not os.path.samefile(os.path.dirname(archive), self.repository.directory): 762 | shutil.move(archive, self.repository.directory) 763 | archive = os.path.join(self.repository.directory, os.path.basename(archive)) 764 | generated_archives.append(archive) 765 | # Use deb-pkg-tools to sanity check the generated package archives 766 | # for duplicate files. This should never occur but unfortunately 767 | # can happen because Python's packaging infrastructure is a lot 768 | # more `forgiving' in the sense of blindly overwriting files 769 | # installed by other packages ;-). 770 | if len(generated_archives) > 1: 771 | check_duplicate_files(generated_archives, cache=get_default_cache()) 772 | # Let the caller know which archives were generated (whether 773 | # previously or now) and how to depend on the converted packages. 774 | return generated_archives, sorted(dependencies_to_report) 775 | finally: 776 | # Always clean up temporary directories created by pip and pip-accel. 777 | self.pip_accel.cleanup_temporary_directories() 778 | 779 | def get_source_distributions(self, pip_install_arguments): 780 | """ 781 | Use :pypi:`pip-accel` to download and unpack Python source distributions. 782 | 783 | Retries several times if a download fails (so it doesn't fail 784 | immediately when a package index server returns a transient error). 785 | 786 | :param pip_install_arguments: 787 | 788 | The command line arguments to the ``pip install`` command (an 789 | iterable of strings). 790 | 791 | :returns: 792 | 793 | A generator of :class:`.PackageToConvert` objects. 794 | 795 | :raises: 796 | 797 | When downloading fails even after several retries this function 798 | raises ``pip.exceptions.DistributionNotFound``. This function can 799 | also raise other exceptions raised by :pypi:`pip` because it uses 800 | :pypi:`pip-accel` to call :pypi:`pip` (as a Python API). 801 | """ 802 | # We depend on `pip install --ignore-installed ...' so we can guarantee 803 | # that all of the packages specified by the caller are converted, 804 | # instead of only those not currently installed somewhere where pip can 805 | # see them (a poorly defined concept to begin with). 806 | arguments = ['--ignore-installed'] + list(pip_install_arguments) 807 | for requirement in self.pip_accel.get_requirements(arguments): 808 | if requirement.name.lower() not in self.system_packages: 809 | yield PackageToConvert(self, requirement) 810 | 811 | def transform_name(self, python_package_name, *extras): 812 | """ 813 | Transform Python package name to Debian package name. 814 | 815 | :param python_package_name: The name of a Python package 816 | as found on PyPI (a string). 817 | :param extras: Any extras requested to be included (a tuple of strings). 818 | :returns: The transformed name (a string). 819 | 820 | Examples: 821 | 822 | >>> from py2deb.converter import PackageConverter 823 | >>> converter = PackageConverter() 824 | >>> converter.transform_name('example') 825 | 'python-example' 826 | >>> converter.set_name_prefix('my-custom-prefix') 827 | >>> converter.transform_name('example') 828 | 'my-custom-prefix-example' 829 | >>> converter.set_name_prefix('some-web-app') 830 | >>> converter.transform_name('raven', 'flask') 831 | 'some-web-app-raven-flask' 832 | 833 | """ 834 | key = python_package_name.lower() 835 | # Check for a system package override by the caller. 836 | debian_package_name = self.system_packages.get(key) 837 | if debian_package_name: 838 | # We don't modify the names of system packages. 839 | return debian_package_name 840 | # Check for a package rename override by the caller. 841 | debian_package_name = self.name_mapping.get(key) 842 | if not debian_package_name: 843 | # No override. Make something up :-). 844 | debian_package_name = convert_package_name( 845 | python_package_name=python_package_name, 846 | name_prefix=self.name_prefix, 847 | extras=extras, 848 | ) 849 | # Always normalize the package name (even if it was given to us by the caller). 850 | return normalize_package_name(debian_package_name) 851 | 852 | def transform_version(self, package_to_convert, python_requirement_name, python_requirement_version): 853 | """ 854 | Transform a Python requirement version to a Debian version number. 855 | 856 | :param package_to_convert: The :class:`.PackageToConvert` whose 857 | requirement is being transformed. 858 | :param python_requirement_name: The name of a Python package 859 | as found on PyPI (a string). 860 | :param python_requirement_version: The required version of the 861 | Python package (a string). 862 | :returns: The transformed version (a string). 863 | 864 | This method is a wrapper for :func:`.normalize_package_version()` that 865 | takes care of one additional quirk to ensure compatibility with 866 | :pypi:`pip`. Explaining this quirk requires a bit of context: 867 | 868 | - When package A requires package B (via ``install_requires``) and 869 | package A absolutely pins the required version of package B using one 870 | or more trailing zeros (e.g. ``B==1.0.0``) but the actual version 871 | number of package B (embedded in the metadata of package B) contains 872 | less trailing zeros (e.g. ``1.0``) then :pypi:`pip` will not complain 873 | but silently fetch version ``1.0`` of package B to satisfy the 874 | requirement. 875 | 876 | - However this doesn't change the absolutely pinned version in the 877 | ``install_requires`` metadata of package A. 878 | 879 | - When py2deb converts the resulting requirement set, the dependency of 880 | package A is converted as ``B (= 1.0.0)``. The resulting packages 881 | will not be installable because :man:`apt` considers ``1.0`` to be 882 | different from ``1.0.0``. 883 | 884 | This method analyzes the requirement set to identify occurrences of 885 | this quirk and strip trailing zeros in ``install_requires`` metadata 886 | that would otherwise result in converted packages that cannot be 887 | installed. 888 | """ 889 | matching_packages = [ 890 | pkg for pkg in self.packages_to_convert 891 | if package_names_match(pkg.python_name, python_requirement_name) 892 | ] 893 | if len(matching_packages) > 1: 894 | # My assumption while writing this code is that this should never 895 | # happen. This check is to make sure that if it does happen it will 896 | # be noticed because the last thing I want is for this `hack' to 897 | # result in packages that are silently wrongly converted. 898 | normalized_name = normalize_package_name(python_requirement_name) 899 | num_matches = len(matching_packages) 900 | raise Exception(compact(""" 901 | Expected requirement set to contain exactly one Python package 902 | whose name can be normalized to {name} but encountered {count} 903 | packages instead! (matching packages: {matches}) 904 | """, name=normalized_name, count=num_matches, matches=matching_packages)) 905 | elif matching_packages: 906 | # Check whether the version number included in the requirement set 907 | # matches the version number in a package's requirements. 908 | requirement_to_convert = matching_packages[0] 909 | if python_requirement_version != requirement_to_convert.python_version: 910 | logger.debug("Checking whether to strip trailing zeros from required version ..") 911 | # Check whether the version numbers share the same prefix. 912 | required_version = tokenize_version(python_requirement_version) 913 | included_version = tokenize_version(requirement_to_convert.python_version) 914 | common_length = min(len(required_version), len(included_version)) 915 | required_prefix = required_version[:common_length] 916 | included_prefix = included_version[:common_length] 917 | prefixes_match = (required_prefix == included_prefix) 918 | logger.debug("Prefix of required version: %s", required_prefix) 919 | logger.debug("Prefix of included version: %s", included_prefix) 920 | logger.debug("Prefixes match? %s", prefixes_match) 921 | # Check if 1) only the required version has a suffix and 2) this 922 | # suffix consists only of trailing zeros. 923 | required_suffix = required_version[common_length:] 924 | included_suffix = included_version[common_length:] 925 | logger.debug("Suffix of required version: %s", required_suffix) 926 | logger.debug("Suffix of included version: %s", included_suffix) 927 | if prefixes_match and required_suffix and not included_suffix: 928 | # Check whether the suffix of the required version contains 929 | # only zeros, i.e. pip considers the version numbers the same 930 | # although apt would not agree. 931 | if all(re.match('^0+$', t) for t in required_suffix if t.isdigit()): 932 | modified_version = ''.join(required_prefix) 933 | logger.warning("Stripping superfluous trailing zeros from required" 934 | " version of %s required by %s! (%s -> %s)", 935 | python_requirement_name, package_to_convert.python_name, 936 | python_requirement_version, modified_version) 937 | python_requirement_version = modified_version 938 | return normalize_package_version(python_requirement_version, prerelease_workaround=self.prerelease_workaround) 939 | -------------------------------------------------------------------------------- /py2deb/hooks.py: -------------------------------------------------------------------------------- 1 | # py2deb: Python to Debian package converter. 2 | # 3 | # Author: Peter Odding 4 | # Last Change: August 6, 2020 5 | # URL: https://py2deb.readthedocs.io 6 | 7 | """ 8 | The :mod:`py2deb.hooks` module contains post-installation and pre-removal hooks. 9 | 10 | This module is a bit special in the sense that it is a part of the py2deb code 11 | base but it is embedded in the Debian binary packages generated by py2deb as a 12 | post-installation and pre-removal hook. 13 | 14 | Because this module is embedded in generated packages it can't use the external 15 | dependencies of the py2deb project, it needs to restrict itself to Python's 16 | standard library. 17 | 18 | My reasons for including this Python script as a "proper module" inside the 19 | py2deb project: 20 | 21 | - It encourages proper documentation of the functionality in this module, which 22 | enables users to read through the documentation without having to dive into 23 | py2deb's source code. 24 | 25 | - It makes it easier to unit test the individual functions in this script 26 | without jumping through too many hoops (I greatly value test suite coverage). 27 | 28 | The :func:`~py2deb.package.PackageToConvert.generate_maintainer_script()` 29 | method is responsible for converting this module into a post-installation or 30 | pre-removal script. It does so by reading this module's source code and 31 | appending a call to :func:`post_installation_hook()` or 32 | :func:`pre_removal_hook()` at the bottom. 33 | """ 34 | 35 | # Standard library modules. 36 | import errno 37 | import imp 38 | import json 39 | import logging 40 | import os 41 | import py_compile 42 | import subprocess 43 | 44 | # Detect whether the Python implementation we're running on supports PEP 3147. 45 | HAS_PEP_3147 = hasattr(imp, 'get_tag') 46 | 47 | # Initialize a logger. 48 | logger = logging.getLogger('py2deb.hooks') 49 | 50 | 51 | def post_installation_hook(package_name, alternatives, modules_directory, namespaces, namespace_style): 52 | """ 53 | Generic post-installation hook for packages generated by py2deb. 54 | 55 | :param package_name: 56 | 57 | The name of the system package (a string). 58 | 59 | :param alternatives: 60 | 61 | The relevant subset of values in 62 | :attr:`~py2deb.converter.PackageConverter.alternatives`. 63 | 64 | :param modules_directory: 65 | 66 | The absolute pathname of the directory where Python modules are installed 67 | (a string). 68 | 69 | :param namespaces: 70 | 71 | The namespaces used by the package (a list of tuples in the format 72 | generated by :attr:`~py2deb.package.PackageToConvert.namespaces`). 73 | 74 | :param namespace_style: 75 | 76 | The style of namespaces being used (one of the strings returned by 77 | :attr:`~py2deb.package.PackageToConvert.namespace_style`). 78 | 79 | Uses the following functions to implement everything py2deb needs from the 80 | post-installation maintainer script: 81 | 82 | - :func:`generate_bytecode_files()` 83 | - :func:`create_alternatives()` 84 | - :func:`initialize_namespaces()` 85 | """ 86 | initialize_logging() 87 | installed_files = find_installed_files(package_name) 88 | generate_bytecode_files(package_name, installed_files) 89 | create_alternatives(package_name, alternatives) 90 | initialize_namespaces(package_name, modules_directory, namespaces, namespace_style) 91 | 92 | 93 | def pre_removal_hook(package_name, alternatives, modules_directory, namespaces): 94 | """ 95 | Generic pre-removal hook for packages generated by py2deb. 96 | 97 | :param package_name: 98 | 99 | The name of the system package (a string). 100 | 101 | :param alternatives: 102 | 103 | The relevant subset of values in 104 | :attr:`~py2deb.converter.PackageConverter.alternatives`. 105 | 106 | :param modules_directory: 107 | 108 | The absolute pathname of the directory where Python modules are installed 109 | (a string). 110 | 111 | :param namespaces: 112 | 113 | The namespaces used by the package (a list of tuples in the format 114 | generated by :attr:`py2deb.package.PackageToConvert.namespaces`). 115 | 116 | Uses the following functions to implement everything py2deb needs from the 117 | pre-removal maintainer script: 118 | 119 | - :func:`cleanup_bytecode_files()` 120 | - :func:`cleanup_alternatives()` 121 | - :func:`cleanup_namespaces()` 122 | """ 123 | initialize_logging() 124 | installed_files = find_installed_files(package_name) 125 | cleanup_bytecode_files(package_name, installed_files) 126 | cleanup_alternatives(package_name, alternatives) 127 | cleanup_namespaces(package_name, modules_directory, namespaces) 128 | 129 | 130 | def initialize_logging(): 131 | """Initialize logging to the terminal and :man:`apt` log files.""" 132 | logging.basicConfig( 133 | level=logging.INFO, 134 | format='%(asctime)s %(name)s[%(process)d] %(levelname)s %(message)s', 135 | datefmt='%Y-%m-%d %H:%M:%S') 136 | 137 | 138 | def find_installed_files(package_name): 139 | """ 140 | Find the files installed by a Debian system package. 141 | 142 | :param package_name: The name of the system package (a string). 143 | :returns: A list of absolute filenames (strings). 144 | 145 | Uses the ``dpkg -L`` command. 146 | """ 147 | dpkg = subprocess.Popen(['dpkg', '-L', package_name], stdout=subprocess.PIPE, universal_newlines=True) 148 | stdout, stderr = dpkg.communicate() 149 | return stdout.splitlines() 150 | 151 | 152 | def generate_bytecode_files(package_name, installed_files): 153 | """ 154 | Generate Python byte code files for the ``*.py`` files installed by a package. 155 | 156 | :param package_name: 157 | 158 | The name of the system package (a string). 159 | 160 | :param installed_files: 161 | 162 | A list of strings with the absolute pathnames of installed files. 163 | 164 | Uses :func:`py_compile.compile()` to generate bytecode files. 165 | """ 166 | num_generated = 0 167 | for filename in installed_files: 168 | if filename.endswith('.py'): 169 | py_compile.compile(filename) 170 | num_generated += 1 171 | if num_generated > 0: 172 | logger.info("Generated %i Python bytecode file(s) for %s package.", num_generated, package_name) 173 | 174 | 175 | def cleanup_bytecode_files(package_name, installed_files): 176 | """ 177 | Cleanup Python byte code files generated when a package was installed. 178 | 179 | :param package_name: 180 | 181 | The name of the system package (a string). 182 | 183 | :param installed_files: 184 | 185 | A list of strings with the absolute pathnames of installed files. 186 | """ 187 | num_removed = cleanup_bytecode_helper(installed_files) 188 | if num_removed > 0: 189 | logger.info("Cleaned up %i Python bytecode file(s) for %s package.", num_removed, package_name) 190 | 191 | 192 | def cleanup_bytecode_helper(filenames): 193 | """ 194 | Cleanup Python byte code files. 195 | 196 | :param filenames: A list of strings with the absolute pathnames of installed files. 197 | :returns: The number of files that were removed (an integer). 198 | """ 199 | num_removed = 0 200 | for filename in filenames: 201 | if filename.endswith('.py'): 202 | for bytecode_file in find_bytecode_files(filename): 203 | os.unlink(bytecode_file) 204 | num_removed += 1 205 | if HAS_PEP_3147: 206 | remove_empty_directory(os.path.join(os.path.dirname(filename), '__pycache__')) 207 | return num_removed 208 | 209 | 210 | def remove_empty_directory(directory): 211 | """ 212 | Remove a directory if it is empty. 213 | 214 | :param directory: The pathname of the directory (a string). 215 | """ 216 | try: 217 | os.rmdir(directory) 218 | except OSError as e: 219 | if e.errno not in (errno.ENOTEMPTY, errno.ENOENT): 220 | raise 221 | 222 | 223 | def find_bytecode_files(python_file): 224 | """ 225 | Find the byte code file(s) generated from a Python file. 226 | 227 | :param python_file: The pathname of a ``*.py`` file (a string). 228 | :returns: A generator of pathnames (strings). 229 | 230 | Starting from Python 3.2 byte code files are written according to `PEP 231 | 3147`_ which also defines :func:`imp.cache_from_source()` to locate 232 | (optimized) byte code files. When this function is available it is used, 233 | when it's not available the corresponding ``*.pyc`` and/or ``*.pyo`` files 234 | are located manually by :func:`find_bytecode_files()`. 235 | 236 | .. _PEP 3147: https://www.python.org/dev/peps/pep-3147/ 237 | """ 238 | if HAS_PEP_3147: 239 | bytecode_file = imp.cache_from_source(python_file, True) 240 | if os.path.isfile(bytecode_file): 241 | yield bytecode_file 242 | optimized_bytecode_file = imp.cache_from_source(python_file, False) 243 | if os.path.isfile(optimized_bytecode_file): 244 | yield optimized_bytecode_file 245 | else: 246 | for suffix in ('c', 'o'): 247 | bytecode_file = python_file + suffix 248 | if os.path.isfile(bytecode_file): 249 | yield bytecode_file 250 | 251 | 252 | def create_alternatives(package_name, alternatives): 253 | """ 254 | Use :man:`update-alternatives` to install a global symbolic link to a program. 255 | 256 | :param package_name: 257 | 258 | The name of the system package (a string). 259 | 260 | :param alternatives: 261 | 262 | The relevant subset of values in 263 | :attr:`~py2deb.converter.PackageConverter.alternatives`. 264 | 265 | Install a program available inside the custom installation prefix in the 266 | system wide executable search path using the Debian alternatives system. 267 | """ 268 | for link, path in alternatives: 269 | name = os.path.basename(link) 270 | subprocess.call(['update-alternatives', '--install', link, name, path, '0']) 271 | 272 | 273 | def cleanup_alternatives(package_name, alternatives): 274 | """ 275 | Cleanup the alternatives that were previously installed by :func:`create_alternatives()`. 276 | 277 | :param package_name: 278 | 279 | The name of the system package (a string). 280 | 281 | :param alternatives: 282 | 283 | The relevant subset of values in 284 | :attr:`~py2deb.converter.PackageConverter.alternatives`. 285 | """ 286 | for link, path in alternatives: 287 | name = os.path.basename(link) 288 | subprocess.call(['update-alternatives', '--remove', name, path]) 289 | 290 | 291 | def initialize_namespaces(package_name, modules_directory, namespaces, namespace_style): 292 | """ 293 | Initialize Python `namespace packages`_ so they can be imported in the normal way. 294 | 295 | Both pkgutil-style and pkg_resources-style namespace packages are supported 296 | (although support for the former was added in 2020 whereas support for the 297 | latter has existed since 2015). 298 | 299 | :param package_name: 300 | 301 | The name of the system package (a string). 302 | 303 | :param modules_directory: 304 | 305 | The absolute pathname of the directory where Python modules are installed 306 | (a string). 307 | 308 | :param namespaces: 309 | 310 | The namespaces used by the package (a list of tuples in the format 311 | generated by :attr:`~py2deb.package.PackageToConvert.namespaces`). 312 | 313 | :param namespace_style: 314 | 315 | The style of namespaces being used (one of the strings returned by 316 | :attr:`~py2deb.package.PackageToConvert.namespace_style`). 317 | 318 | .. _namespace packages: https://packaging.python.org/guides/packaging-namespace-packages/ 319 | """ 320 | if namespaces: 321 | with NameSpaceReferenceCount(modules_directory) as reference_counts: 322 | for components in namespaces: 323 | package_directory = os.path.join(modules_directory, *components) 324 | logger.debug("Initializing namespace %s (%s) ..", '.'.join(components), package_directory) 325 | if not os.path.isdir(package_directory): 326 | os.makedirs(package_directory) 327 | package_file = os.path.join(package_directory, '__init__.py') 328 | with open(package_file, 'w') as handle: 329 | if namespace_style == 'pkgutil': 330 | handle.write("__path__ = __import__('pkgutil').extend_path(__path__, __name__)\n") 331 | elif namespace_style == 'setuptools': 332 | handle.write("__import__('pkg_resources').declare_namespace(__name__)\n") 333 | else: 334 | handle.write("# namespace package\n") 335 | reference_counts[components] += 1 336 | logger.info("Initialized %i namespaces for %s package.", len(namespaces), package_name) 337 | 338 | 339 | def cleanup_namespaces(package_name, modules_directory, namespaces): 340 | """ 341 | Clean up Python namespace packages previously initialized using :func:`initialize_namespaces()`. 342 | 343 | :param package_name: 344 | 345 | The name of the system package (a string). 346 | 347 | :param modules_directory: 348 | 349 | The absolute pathname of the directory where Python modules are 350 | installed (a string). 351 | 352 | :param namespaces: 353 | 354 | The namespaces used by the package (a list of tuples in the format 355 | generated by :attr:`~py2deb.package.PackageToConvert.namespaces`). 356 | """ 357 | if namespaces: 358 | with NameSpaceReferenceCount(modules_directory) as reference_counts: 359 | num_cleaned = 0 360 | for components in reversed(list(namespaces)): 361 | package_directory = os.path.join(modules_directory, *components) 362 | init_file = os.path.join(package_directory, '__init__.py') 363 | if reference_counts[components] > 1: 364 | logger.debug("Not yet de-initializing namespace %s (%s) ..", 365 | '.'.join(components), package_directory) 366 | elif reference_counts[components] == 1: 367 | logger.debug("De-initializing namespace %s (%s) ..", 368 | '.'.join(components), package_directory) 369 | cleanup_bytecode_helper([init_file]) 370 | os.unlink(init_file) 371 | remove_empty_directory(package_directory) 372 | num_cleaned += 1 373 | reference_counts[components] -= 1 374 | if num_cleaned > 0: 375 | logger.info("Cleaned up %i namespaces for %s package.", num_cleaned, package_name) 376 | 377 | 378 | class NameSpaceReferenceCount(dict): 379 | 380 | """Persistent reference counting for initialization of namespace packages.""" 381 | 382 | def __init__(self, modules_directory): 383 | """ 384 | Initialize a :class:`NameSpaceReferenceCount` object. 385 | 386 | :param modules_directory: 387 | 388 | The absolute pathname of the directory where Python modules are 389 | installed (a string). 390 | """ 391 | self.data_file = os.path.join(modules_directory, 'py2deb-namespaces.json') 392 | 393 | def __enter__(self): 394 | """Load the persistent data file (if it exists).""" 395 | if os.path.isfile(self.data_file): 396 | with open(self.data_file) as handle: 397 | self.update(json.load(handle)) 398 | return self 399 | 400 | def __exit__(self, exc_type=None, exc_value=None, traceback=None): 401 | """Save the persistent data file.""" 402 | if len(self) > 0: 403 | with open(self.data_file, 'w') as handle: 404 | json.dump(self, handle) 405 | elif os.path.isfile(self.data_file): 406 | os.unlink(self.data_file) 407 | 408 | def __getitem__(self, key): 409 | """Get the reference count of a namespace (defaults to zero).""" 410 | return dict.get(self, '.'.join(key), 0) 411 | 412 | def __setitem__(self, key, value): 413 | """Set the reference count of a namespace.""" 414 | key = '.'.join(key) 415 | if value > 0: 416 | dict.__setitem__(self, key, value) 417 | else: 418 | self.pop(key, None) 419 | -------------------------------------------------------------------------------- /py2deb/namespaces.py: -------------------------------------------------------------------------------- 1 | # py2deb: Python to Debian package converter. 2 | # 3 | # Author: Peter Odding 4 | # Last Change: August 5, 2020 5 | # URL: https://py2deb.readthedocs.io 6 | 7 | """ 8 | Python package namespace auto detection. 9 | 10 | This module is used by :pypi:`py2deb` to detect pkgutil-style `namespace 11 | packages`_ to enable special handling of the ``__init__.py`` files involved, 12 | because these would otherwise cause :man:`dpkg` file conflicts. 13 | 14 | .. note:: The ``__init__.py`` files that define pkgutil-style namespace 15 | packages can contain arbitrary Python code (including comments and 16 | with room for minor differences in coding style) which makes reliable 17 | identification harder than it should be. We use :func:`ast.parse()` 18 | to look for hints and only when we find enough hints do we consider a 19 | module to be part of a pkgutil-style namespace package. 20 | 21 | .. _namespace packages: https://packaging.python.org/guides/packaging-namespace-packages/ 22 | """ 23 | 24 | # Standard library modules. 25 | import ast 26 | import logging 27 | import os 28 | 29 | # Initialize a logger for this module. 30 | logger = logging.getLogger(__name__) 31 | 32 | # Public identifiers that require documentation. 33 | __all__ = ("find_pkgutil_namespaces", "find_pkgutil_ns_hints", "find_python_modules") 34 | 35 | 36 | def find_pkgutil_namespaces(directory): 37 | """ 38 | Find the pkgutil-style `namespace packages`_ in an unpacked Python distribution archive. 39 | 40 | :param directory: 41 | 42 | The pathname of a directory containing an unpacked Python distribution 43 | archive (a string). 44 | 45 | :returns: 46 | 47 | A generator of dictionaries similar to those returned by 48 | :func:`find_python_modules()`. 49 | 50 | This function combines :func:`find_python_modules()` and 51 | :func:`find_pkgutil_ns_hints()` to make it easy for callers 52 | to identify the namespace packages defined by an unpacked 53 | Python distribution archive. 54 | """ 55 | for details in find_python_modules(directory): 56 | logger.debug("Checking file for pkgutil-style namespace definition: %s", details['abspath']) 57 | try: 58 | with open(details['abspath']) as handle: 59 | contents = handle.read() 60 | # The intention of the following test is to start with a cheap test 61 | # to quickly disqualify large and irrelevant __init__.py files, 62 | # without having to parse their full AST. 63 | if "pkgutil" in contents: 64 | module = ast.parse(contents, filename=details['abspath']) 65 | hints = find_pkgutil_ns_hints(module) 66 | if len(hints) >= 5: 67 | yield details 68 | except Exception: 69 | logger.warning("Swallowing exception during pkgutil-style namespace analysis ..", exc_info=True) 70 | 71 | 72 | def find_pkgutil_ns_hints(tree): 73 | """ 74 | Analyze an AST for hints that we're dealing with a Python module that defines a pkgutil-style namespace package. 75 | 76 | :param tree: 77 | 78 | The result of :func:`ast.parse()` when run on a Python module (which is 79 | assumed to be an ``__init__.py`` file). 80 | 81 | :returns: 82 | 83 | A :class:`set` of strings where each string represents a hint (an 84 | indication) that we're dealing with a pkgutil-style namespace module. No 85 | single hint can definitely tell us, but a couple of unique hints taken 86 | together should provide a reasonable amount of confidence (at least this 87 | is the idea, how well this works in practice remains to be seen). 88 | """ 89 | hints = set() 90 | for node in ast.walk(tree): 91 | if isinstance(node, ast.Attribute): 92 | if node.attr == "extend_path": 93 | logger.debug("Found hint! ('extend_path' reference)") 94 | hints.add("extend_path") 95 | elif isinstance(node, ast.Import) and any(alias.name == "pkgutil" for alias in node.names): 96 | logger.debug("Found hint! (import pkg_util)") 97 | hints.update(("import", "pkgutil")) 98 | elif ( 99 | isinstance(node, ast.ImportFrom) 100 | and node.module == "pkgutil" 101 | and any(alias.name == "extend_path" for alias in node.names) 102 | ): 103 | logger.debug("Found hint! (from pkg_util import extend_path)") 104 | hints.update(("import", "pkgutil", "extend_path")) 105 | elif isinstance(node, ast.Name): 106 | if node.id == "extend_path": 107 | logger.debug("Found hint! ('extend_path' reference)") 108 | hints.add("extend_path") 109 | elif node.id == "pkgutil": 110 | logger.debug("Found hint! ('pkgutil' reference)") 111 | hints.add("pkgutil") 112 | elif node.id == "__import__": 113 | logger.debug("Found hint! ('__import__' reference)") 114 | hints.add("import") 115 | elif node.id == "__name__": 116 | logger.debug("Found hint! ('__name__' reference)") 117 | hints.add("__name__") 118 | elif node.id == "__path__": 119 | logger.debug("Found hint! ('__path__' reference)") 120 | hints.add("__path__") 121 | elif isinstance(node, ast.Str) and node.s in ("pkgutil", "extend_path"): 122 | logger.debug("Found hint! ('%s' string literal)", node.s) 123 | hints.add(node.s) 124 | return hints 125 | 126 | 127 | def find_python_modules(directory): 128 | """ 129 | Find the Python modules in an unpacked Python distribution archive. 130 | 131 | :param directory: 132 | 133 | The pathname of a directory containing an unpacked Python distribution 134 | archive (a string). 135 | 136 | :returns: A list of dictionaries with the following key/value pairs: 137 | 138 | - ``abspath`` gives the absolute pathname of a Python module (a string). 139 | - ``relpath`` gives the pathname of a Python module (a string) 140 | relative to the intended installation directory. 141 | - ``name`` gives the dotted name of a Python module (a string). 142 | 143 | This function works as follows: 144 | 145 | 1. Use :func:`os.walk()` to recursively search for ``__init__.py`` files in 146 | the directory given by the caller and collect the relative pathnames of 147 | the directories containing the ``__init__.py`` files. 148 | 149 | 2. Use :func:`os.path.commonprefix()` to determine the common prefix of the 150 | resulting directory pathnames. 151 | 152 | 3. Use :func:`os.path.split()` to partition the common prefix into an 153 | insignificant part (all but the final pathname component) and the 154 | significant part (the final pathname component). 155 | 156 | 4. Strip the insignificant part of the common prefix from the directory 157 | pathnames we collected in step 1. 158 | 159 | 5. Replace :data:`os.sep` occurrences with dots to convert (what remains 160 | of) the directory pathnames to "dotted paths". 161 | """ 162 | logger.debug("Searching for pkgutil-style namespace packages in %s ..", directory) 163 | # Find the relative pathnames of all __init__.py files (relative 164 | # to the root directory given to us by the caller). 165 | modules = [] 166 | for root, dirs, files in os.walk(directory): 167 | for filename in files: 168 | if filename == "__init__.py": 169 | abspath = os.path.join(root, filename) 170 | relpath = os.path.relpath(abspath, directory) 171 | # Ignore a top level "build" directory (generated by pip). 172 | path_segments = relpath.split(os.path.sep) 173 | if not (path_segments and path_segments[0] == 'build'): 174 | module_path, basename = os.path.split(relpath) 175 | modules.append({ 176 | 'abspath': abspath, 177 | 'relpath': relpath, 178 | 'name': module_path, 179 | }) 180 | logger.debug("Found modules defined using __init__.py files: %s", modules) 181 | # Determine the common prefix of the module paths. 182 | common_prefix = os.path.commonprefix([m['name'] for m in modules]) 183 | logger.debug("Determined common prefix: %s", common_prefix) 184 | # Separate the path segments at the start of the common prefix (which are 185 | # insignificant for our purposes) from the final path segment (which is 186 | # essential for our purposes). 187 | head, tail = os.path.split(common_prefix) 188 | if head and tail: 189 | logger.debug("Stripping insignificant part of prefix: [%s/]%s", head, tail) 190 | else: 191 | logger.debug("Common prefix has no insignificant part (nothing to strip).") 192 | # Prepare to strip the insignificant part of the common prefix (the 193 | # +1 is to strip the dot that leads up to the final path segment). 194 | strip_length = len(head) + 1 if head else 0 195 | # Translate (what remains of) the module pathnames to dotted names. 196 | for details in modules: 197 | if strip_length > 0: 198 | # Strip the insignificant part of the common prefix. 199 | details['name'] = details['name'][strip_length:] 200 | details['relpath'] = details['relpath'][strip_length:] 201 | # Translate pathnames to dotted names. 202 | details['name'] = details['name'].replace(os.sep, ".") 203 | # Share our results with the caller. 204 | yield details 205 | -------------------------------------------------------------------------------- /py2deb/package.py: -------------------------------------------------------------------------------- 1 | # py2deb: Python to Debian package converter. 2 | # 3 | # Authors: 4 | # - Arjan Verwer 5 | # - Peter Odding 6 | # Last Change: August 6, 2020 7 | # URL: https://py2deb.readthedocs.io 8 | 9 | """ 10 | The :mod:`py2deb.package` module contains the low level conversion logic. 11 | 12 | This module defines the :class:`PackageToConvert` class which implements the 13 | low level logic of converting a single Python package to a Debian package. The 14 | separation between the :class:`.PackageConverter` and :class:`PackageToConvert` 15 | classes is somewhat crude (because neither class can work without the other) 16 | but the idea is to separate the high level conversion logic from the low level 17 | conversion logic. 18 | """ 19 | 20 | # Standard library modules. 21 | import glob 22 | import logging 23 | import os 24 | import platform 25 | import re 26 | import sys 27 | import time 28 | 29 | # External dependencies. 30 | from deb_pkg_tools.control import merge_control_fields, unparse_control_fields 31 | from deb_pkg_tools.package import build_package, find_object_files, find_system_dependencies, strip_object_files 32 | from executor import execute 33 | from humanfriendly.text import concatenate, pluralize 34 | from pkg_resources import Requirement 35 | from pkginfo import UnpackedSDist 36 | from property_manager import PropertyManager, cached_property 37 | from six import BytesIO 38 | from six.moves import configparser 39 | 40 | # Modules included in our package. 41 | from py2deb.namespaces import find_pkgutil_namespaces 42 | from py2deb.utils import ( 43 | TemporaryDirectory, 44 | detect_python_script, 45 | embed_install_prefix, 46 | normalize_package_version, 47 | package_names_match, 48 | python_version, 49 | ) 50 | 51 | # Initialize a logger. 52 | logger = logging.getLogger(__name__) 53 | 54 | # The following installation prefixes are known to contain a `bin' directory 55 | # that's available on the default executable search path (the environment 56 | # variable $PATH). 57 | KNOWN_INSTALL_PREFIXES = ('/usr', '/usr/local') 58 | 59 | 60 | class PackageToConvert(PropertyManager): 61 | 62 | """ 63 | Abstraction for Python packages to be converted to Debian packages. 64 | 65 | Contains a :class:`pip_accel.req.Requirement` object, has a back 66 | reference to the :class:`.PackageConverter` and provides all of the 67 | Debian package metadata implied by the Python package metadata. 68 | """ 69 | 70 | def __init__(self, converter, requirement): 71 | """ 72 | Initialize a package to convert. 73 | 74 | :param converter: The :class:`.PackageConverter` that holds the user 75 | options and knows how to transform package names. 76 | :param requirement: A :class:`pip_accel.req.Requirement` object 77 | (created by :func:`~py2deb.converter.PackageConverter.get_source_distributions()`). 78 | """ 79 | self.converter = converter 80 | self.requirement = requirement 81 | 82 | @cached_property 83 | def debian_dependencies(self): 84 | """ 85 | Find Debian dependencies of Python package. 86 | 87 | Converts `Python version specifiers`_ to `Debian package 88 | relationships`_. 89 | 90 | :returns: A list with Debian package relationships (strings) in the 91 | format of the ``Depends:`` line of a Debian package 92 | ``control`` file. Based on :data:`python_requirements`. 93 | 94 | .. _Python version specifiers: http://www.python.org/dev/peps/pep-0440/#version-specifiers 95 | .. _Debian package relationships: https://www.debian.org/doc/debian-policy/ch-relationships.html 96 | """ 97 | dependencies = set() 98 | for requirement in self.python_requirements: 99 | debian_package_name = self.converter.transform_name(requirement.project_name, *requirement.extras) 100 | if requirement.specs: 101 | for constraint, version in requirement.specs: 102 | version = self.converter.transform_version(self, requirement.project_name, version) 103 | if version == 'dev': 104 | # Requirements like 'pytz > dev' (celery==3.1.16) don't 105 | # seem to really mean anything to pip (based on my 106 | # reading of the 1.4.x source code) but Debian will 107 | # definitely complain because version strings should 108 | # start with a digit. In this case we'll just fall 109 | # back to a dependency without a version specification 110 | # so we don't drop the dependency. 111 | dependencies.add(debian_package_name) 112 | elif constraint == '==': 113 | dependencies.add('%s (= %s)' % (debian_package_name, version)) 114 | elif constraint == '!=': 115 | values = (debian_package_name, version, debian_package_name, version) 116 | dependencies.add('%s (<< %s) | %s (>> %s)' % values) 117 | elif constraint == '<': 118 | dependencies.add('%s (<< %s)' % (debian_package_name, version)) 119 | elif constraint == '>': 120 | dependencies.add('%s (>> %s)' % (debian_package_name, version)) 121 | elif constraint in ('<=', '>='): 122 | dependencies.add('%s (%s %s)' % (debian_package_name, constraint, version)) 123 | else: 124 | msg = "Conversion specifier not supported! (%r used by Python package %s)" 125 | raise Exception(msg % (constraint, self.python_name)) 126 | else: 127 | dependencies.add(debian_package_name) 128 | dependencies = sorted(dependencies) 129 | logger.debug("Debian dependencies of %s: %r", self, dependencies) 130 | return dependencies 131 | 132 | @cached_property 133 | def debian_description(self): 134 | """ 135 | Get a minimal description for the converted Debian package. 136 | 137 | Includes the name of the Python package and the date at which the 138 | package was converted. 139 | """ 140 | text = ["Python package", self.python_name, "converted by py2deb on"] 141 | # The %e directive (not documented in the Python standard library but 142 | # definitely available on Linux which is the only platform that py2deb 143 | # targets, for obvious reasons :-) includes a leading space for single 144 | # digit day-of-month numbers. I don't like that, fixed width fields are 145 | # an artefact of 30 years ago and have no place in my software 146 | # (generally speaking :-). This explains the split/compact duo. 147 | text.extend(time.strftime('%B %e, %Y at %H:%M').split()) 148 | return ' '.join(text) 149 | 150 | @cached_property 151 | def debian_maintainer(self): 152 | """ 153 | Get the package maintainer name and e-mail address. 154 | 155 | The name and e-mail address are combined into a single string that can 156 | be embedded in a Debian package (in the format ``name ``). The 157 | metadata is retrieved as follows: 158 | 159 | 1. If the environment variable ``$DEBFULLNAME`` is defined then its 160 | value is taken to be the name of the maintainer (this logic was 161 | added in `#25`_). If ``$DEBEMAIL`` is set as well that will be 162 | incorporated into the result. 163 | 164 | 2. The Python package maintainer name and email address are looked up 165 | in the package metadata and if found these are used. 166 | 167 | 3. The Python package author name and email address are looked up in 168 | the package metadata and if found these are used. 169 | 170 | 4. Finally if all else fails the text "Unknown" is returned. 171 | 172 | .. _#25: https://github.com/paylogic/py2deb/pull/25 173 | """ 174 | if "DEBFULLNAME" in os.environ: 175 | maintainer = os.environ["DEBFULLNAME"] 176 | maintainer_email = os.environ.get("DEBEMAIL") 177 | elif self.metadata.maintainer: 178 | maintainer = self.metadata.maintainer 179 | maintainer_email = self.metadata.maintainer_email 180 | elif self.metadata.author: 181 | maintainer = self.metadata.author 182 | maintainer_email = self.metadata.author_email 183 | else: 184 | maintainer = None 185 | maintainer_email = None 186 | if maintainer and maintainer_email: 187 | return '%s <%s>' % (maintainer, maintainer_email.strip('<>')) 188 | else: 189 | return maintainer or 'Unknown' 190 | 191 | @cached_property 192 | def debian_name(self): 193 | """The name of the converted Debian package (a string).""" 194 | return self.converter.transform_name(self.python_name, *self.requirement.pip_requirement.extras) 195 | 196 | @cached_property 197 | def debian_provides(self): 198 | """ 199 | A symbolic name for the role the package provides (a string). 200 | 201 | When a Python package provides "extras" those extras are encoded into 202 | the name of the generated Debian package, to represent the additional 203 | dependencies versus the package without extras. 204 | 205 | However the package including extras definitely also satisfies a 206 | dependency on the package without extras, so a ``Provides: ...`` 207 | control field is added to the Debian package that contains the 208 | converted package name *without extras*. 209 | """ 210 | if self.requirement.pip_requirement.extras: 211 | return self.converter.transform_name(self.python_name) 212 | else: 213 | return '' 214 | 215 | @cached_property 216 | def debian_version(self): 217 | """ 218 | The version of the Debian package (a string). 219 | 220 | Reformats :attr:`python_version` using 221 | :func:`.normalize_package_version()`. 222 | """ 223 | return normalize_package_version( 224 | self.python_version, prerelease_workaround=self.converter.prerelease_workaround 225 | ) 226 | 227 | @cached_property 228 | def existing_archive(self): 229 | """ 230 | Find ``*.deb`` archive for current package name and version. 231 | 232 | :returns: 233 | 234 | The pathname of the found archive (a string) or :data:`None` if no 235 | existing archive is found. 236 | """ 237 | return self.converter.repository.get_package( 238 | self.debian_name, self.debian_version, "all" 239 | ) or self.converter.repository.get_package( 240 | self.debian_name, self.debian_version, self.converter.debian_architecture 241 | ) 242 | 243 | @cached_property 244 | def has_custom_install_prefix(self): 245 | """ 246 | Check whether package is being installed under custom installation prefix. 247 | 248 | :returns: 249 | 250 | :data:`True` if the package is being installed under a custom 251 | installation prefix, :data:`False` otherwise. 252 | 253 | A custom installation prefix is an installation prefix whose ``bin`` 254 | directory is (likely) not available on the default executable search 255 | path (the environment variable ``$PATH``). 256 | """ 257 | return self.converter.install_prefix not in KNOWN_INSTALL_PREFIXES 258 | 259 | @cached_property 260 | def metadata(self): 261 | """ 262 | Get the Python package metadata. 263 | 264 | The metadata is loaded from the ``PKG-INFO`` file generated by 265 | :pypi:`pip` when it unpacked the source distribution archive. Results 266 | in a pkginfo.UnpackedSDist_ object. 267 | 268 | .. _pkginfo.UnpackedSDist: http://pythonhosted.org/pkginfo/distributions.html 269 | """ 270 | return UnpackedSDist(self.find_egg_info_file()) 271 | 272 | @cached_property 273 | def namespace_packages(self): 274 | """ 275 | Get the Python `namespace packages`_ defined by the Python package. 276 | 277 | :returns: A list of dotted names (strings). 278 | 279 | When :attr:`setuptools_namespaces` is available that will be used, 280 | otherwise we fall back to :attr:`pkgutil_namespaces`. This order of 281 | preference may be switched in the future, but not until 282 | :attr:`pkgutil_namespaces` has seen more thorough testing: 283 | 284 | - Support for :attr:`setuptools_namespaces` was added to py2deb in 285 | release 0.22 (2015) so this is fairly mature code that has seen 286 | thousands of executions between 2015-2020. 287 | 288 | - Support for :attr:`pkgutil_namespaces` was added in August 2020 so 289 | this is new (and complicated) code that hasn't seen a lot of use yet. 290 | Out of conservativeness on my part this is nested in the 'else' 291 | branch (to reduce the scope of potential regressions). 292 | 293 | Additionally computing :attr:`setuptools_namespaces` is very cheap 294 | (all it has to do is search for and read one text file) compared 295 | to :attr:`pkgutil_namespaces` (which needs to recursively search 296 | a directory tree for ``__init__.py`` files and parse each file 297 | it finds to determine whether it's relevant). 298 | 299 | .. _namespace packages: https://packaging.python.org/guides/packaging-namespace-packages/ 300 | """ 301 | if self.setuptools_namespaces: 302 | return self.setuptools_namespaces 303 | else: 304 | return sorted(set(ns['name'] for ns in self.pkgutil_namespaces)) 305 | 306 | @cached_property 307 | def namespace_style(self): 308 | """ 309 | Get the style of Python `namespace packages`_ in use by this package. 310 | 311 | :returns: One of the strings ``pkgutil``, ``setuptools`` or ``none``. 312 | """ 313 | # We check setuptools_namespaces first because it's cheaper and the 314 | # code has been battle tested (in contrast to pkgutil_namespaces). 315 | if self.setuptools_namespaces: 316 | return "setuptools" 317 | elif self.pkgutil_namespaces: 318 | return "pkgutil" 319 | else: 320 | return "none" 321 | 322 | @cached_property 323 | def namespaces(self): 324 | """ 325 | Get the Python `namespace packages`_ defined by the Python package. 326 | 327 | :returns: A list of unique tuples of strings. The tuples are sorted by 328 | increasing length (the number of strings in each tuple) so 329 | that e.g. ``zope`` is guaranteed to sort before 330 | ``zope.app``. 331 | 332 | This property processes the result of :attr:`namespace_packages` 333 | into a more easily usable format. Here's an example of the difference 334 | between :attr:`namespace_packages` and :attr:`namespaces`: 335 | 336 | >>> from py2deb.converter import PackageConverter 337 | >>> converter = PackageConverter() 338 | >>> package = next(converter.get_source_distributions(['zope.app.cache'])) 339 | >>> package.namespace_packages 340 | ['zope', 'zope.app'] 341 | >>> package.namespaces 342 | [('zope',), ('zope', 'app')] 343 | 344 | The value of this property is used by 345 | :func:`~py2deb.hooks.initialize_namespaces()` and 346 | :func:`~py2deb.hooks.cleanup_namespaces()` during installation and 347 | removal of the generated package. 348 | """ 349 | namespaces = set() 350 | for namespace_package in self.namespace_packages: 351 | dotted_name = [] 352 | for component in namespace_package.split('.'): 353 | dotted_name.append(component) 354 | namespaces.add(tuple(dotted_name)) 355 | return sorted(namespaces, key=lambda n: len(n)) 356 | 357 | @cached_property 358 | def pkgutil_namespaces(self): 359 | """ 360 | Namespace packages declared through :mod:`pkgutil`. 361 | 362 | :returns: 363 | 364 | A list of dictionaries similar to those returned by 365 | :func:`.find_pkgutil_namespaces()`. 366 | 367 | For details about this type of namespace packages please refer to 368 | . 369 | 370 | The implementation of this property lives in a separate module (refer 371 | to :func:`.find_pkgutil_namespaces()`) in order to compartmentalize the 372 | complexity of reliably identifying namespace packages defined using 373 | :mod:`pkgutil`. 374 | """ 375 | return list(find_pkgutil_namespaces(self.requirement.source_directory)) 376 | 377 | @property 378 | def python_name(self): 379 | """The name of the Python package (a string).""" 380 | return self.requirement.name 381 | 382 | @cached_property 383 | def python_requirements(self): 384 | """ 385 | Find the installation requirements of the Python package. 386 | 387 | :returns: 388 | 389 | A list of :std:doc:`pkg_resources.Requirement ` 390 | objects. 391 | 392 | This property used to be implemented by manually parsing the 393 | ``requires.txt`` file generated by :pypi:`pip` when it unpacks 394 | a distribution archive. 395 | 396 | While this implementation was eventually enhanced to supported named 397 | extras, it never supported environment markers. 398 | 399 | Since then this property has been reimplemented to use 400 | :std:doc:`pkg_resources.Distribution.requires() ` so 401 | that environment markers are supported. 402 | 403 | If the new implementation fails the property falls back to the old 404 | implementation (as a precautionary measure to avoid unexpected side 405 | effects of the new implementation). 406 | """ 407 | try: 408 | dist = self.requirement.pip_requirement.get_dist() 409 | extras = self.requirement.pip_requirement.extras 410 | requirements = list(dist.requires(extras)) 411 | except Exception: 412 | logger.warning("Failed to determine installation requirements of %s " 413 | "using pkg-resources, falling back to old implementation.", 414 | self, exc_info=True) 415 | requirements = self.python_requirements_fallback 416 | logger.debug("Python requirements of %s: %r", self, requirements) 417 | return requirements 418 | 419 | @cached_property 420 | def python_requirements_fallback(self): 421 | """Fall-back implementation of :attr:`python_requirements`.""" 422 | requirements = [] 423 | filename = self.find_egg_info_file('requires.txt') 424 | if filename: 425 | selected_extras = set(extra.lower() for extra in self.requirement.pip_requirement.extras) 426 | current_extra = None 427 | with open(filename) as handle: 428 | for line in handle: 429 | line = line.strip() 430 | if line.startswith('['): 431 | current_extra = line.strip('[]').lower() 432 | elif line and (current_extra is None or current_extra in selected_extras): 433 | requirements.append(Requirement.parse(line)) 434 | return requirements 435 | 436 | @property 437 | def python_version(self): 438 | """The version of the Python package (a string).""" 439 | return self.requirement.version 440 | 441 | @cached_property 442 | def setuptools_namespaces(self): 443 | """ 444 | Namespace packages declared through :pypi:`setuptools`. 445 | 446 | :returns: A list of dotted names (strings). 447 | 448 | For details about this type of namespace packages please refer to 449 | . 450 | """ 451 | logger.debug("Searching for pkg_resources-style namespace packages of '%s' ..", self.python_name) 452 | dotted_names = [] 453 | namespace_packages_file = self.find_egg_info_file('namespace_packages.txt') 454 | if namespace_packages_file: 455 | with open(namespace_packages_file) as handle: 456 | for line in handle: 457 | line = line.strip() 458 | if line: 459 | dotted_names.append(line) 460 | return dotted_names 461 | 462 | @cached_property 463 | def vcs_revision(self): 464 | """ 465 | The VCS revision of the Python package. 466 | 467 | This works by parsing the ``.hg_archival.txt`` file generated by the 468 | ``hg archive`` command so for now this only supports Python source 469 | distributions exported from Mercurial repositories. 470 | """ 471 | filename = os.path.join(self.requirement.source_directory, '.hg_archival.txt') 472 | if os.path.isfile(filename): 473 | with open(filename) as handle: 474 | for line in handle: 475 | name, _, value = line.partition(':') 476 | if name.strip() == 'node': 477 | return value.strip() 478 | 479 | def convert(self): 480 | """ 481 | Convert current package from Python package to Debian package. 482 | 483 | :returns: The pathname of the generated ``*.deb`` archive. 484 | """ 485 | with TemporaryDirectory(prefix='py2deb-build-') as build_directory: 486 | 487 | # Prepare the absolute pathname of the Python interpreter on the 488 | # target system. This pathname will be embedded in the first line 489 | # of executable scripts (including the post-installation and 490 | # pre-removal scripts). 491 | python_executable = '/usr/bin/%s' % python_version() 492 | 493 | # Unpack the binary distribution archive provided by pip-accel inside our build directory. 494 | build_install_prefix = os.path.join(build_directory, self.converter.install_prefix.lstrip('/')) 495 | self.converter.pip_accel.bdists.install_binary_dist( 496 | members=self.transform_binary_dist(python_executable), 497 | prefix=build_install_prefix, 498 | python=python_executable, 499 | virtualenv_compatible=False, 500 | ) 501 | 502 | # Determine the directory (at build time) where the *.py files for 503 | # Python modules are located (the site-packages equivalent). 504 | if self.has_custom_install_prefix: 505 | build_modules_directory = os.path.join(build_install_prefix, 'lib') 506 | else: 507 | # The /py*/ pattern below is intended to match both /pythonX.Y/ and /pypyX.Y/. 508 | dist_packages_directories = glob.glob(os.path.join(build_install_prefix, 'lib/py*/dist-packages')) 509 | if len(dist_packages_directories) != 1: 510 | msg = "Expected to find a single 'dist-packages' directory inside converted package!" 511 | raise Exception(msg) 512 | build_modules_directory = dist_packages_directories[0] 513 | 514 | # Determine the directory (at installation time) where the *.py 515 | # files for Python modules are located. 516 | install_modules_directory = os.path.join('/', os.path.relpath(build_modules_directory, build_directory)) 517 | 518 | # Execute a user defined command inside the directory where the Python modules are installed. 519 | command = self.converter.scripts.get(self.python_name.lower()) 520 | if command: 521 | execute(command, directory=build_modules_directory, logger=logger) 522 | 523 | # Determine the package's dependencies, starting with the currently 524 | # running version of Python and the Python requirements converted 525 | # to Debian packages. 526 | dependencies = [python_version()] + self.debian_dependencies 527 | 528 | # Check if the converted package contains any compiled *.so files. 529 | object_files = find_object_files(build_directory) 530 | if object_files: 531 | # Strip debugging symbols from the object files. 532 | strip_object_files(object_files) 533 | # Determine system dependencies by analyzing the linkage of the 534 | # *.so file(s) found in the converted package. 535 | dependencies += find_system_dependencies(object_files) 536 | 537 | # Make up some control file fields ... :-) 538 | architecture = self.determine_package_architecture(object_files) 539 | control_fields = unparse_control_fields(dict(package=self.debian_name, 540 | version=self.debian_version, 541 | maintainer=self.debian_maintainer, 542 | description=self.debian_description, 543 | architecture=architecture, 544 | depends=dependencies, 545 | provides=self.debian_provides, 546 | priority='optional', 547 | section='python')) 548 | 549 | # Automatically add the Mercurial global revision id when available. 550 | if self.vcs_revision: 551 | control_fields['Vcs-Hg'] = self.vcs_revision 552 | 553 | # Apply user defined control field overrides from `stdeb.cfg'. 554 | control_fields = self.load_control_field_overrides(control_fields) 555 | 556 | # Create the DEBIAN directory. 557 | debian_directory = os.path.join(build_directory, 'DEBIAN') 558 | os.mkdir(debian_directory) 559 | 560 | # Generate the DEBIAN/control file. 561 | control_file = os.path.join(debian_directory, 'control') 562 | logger.debug("Saving control file fields to %s: %s", control_file, control_fields) 563 | with open(control_file, 'wb') as handle: 564 | control_fields.dump(handle) 565 | 566 | # Lintian is a useful tool to find mistakes in Debian binary 567 | # packages however Lintian checks from the perspective of a package 568 | # included in the official Debian repositories. Because py2deb 569 | # doesn't and probably never will generate such packages some 570 | # messages emitted by Lintian are useless (they merely point out 571 | # how the internals of py2deb work). Because of this we silence 572 | # `known to be irrelevant' messages from Lintian using overrides. 573 | if self.converter.lintian_ignore: 574 | overrides_directory = os.path.join( 575 | build_directory, 'usr', 'share', 'lintian', 'overrides', 576 | ) 577 | overrides_file = os.path.join(overrides_directory, self.debian_name) 578 | os.makedirs(overrides_directory) 579 | with open(overrides_file, 'w') as handle: 580 | for tag in self.converter.lintian_ignore: 581 | handle.write('%s: %s\n' % (self.debian_name, tag)) 582 | 583 | # Find the alternatives relevant to the package we're building. 584 | alternatives = set((link, path) for link, path in self.converter.alternatives 585 | if os.path.isfile(os.path.join(build_directory, path.lstrip('/')))) 586 | 587 | # Remove __init__.py files that define "pkgutil-style namespace 588 | # packages" and let the maintainer scripts generate these files 589 | # instead. If we don't do this these __init__.py files will cause 590 | # dpkg file conflicts. 591 | if self.namespace_style == 'pkgutil': 592 | for ns in self.pkgutil_namespaces: 593 | module_in_build_directory = os.path.join(build_modules_directory, ns['relpath']) 594 | logger.debug("Removing pkgutil-style namespace package file: %s", module_in_build_directory) 595 | os.remove(module_in_build_directory) 596 | 597 | # Generate post-installation and pre-removal maintainer scripts. 598 | self.generate_maintainer_script(filename=os.path.join(debian_directory, 'postinst'), 599 | python_executable=python_executable, 600 | function='post_installation_hook', 601 | package_name=self.debian_name, 602 | alternatives=alternatives, 603 | modules_directory=install_modules_directory, 604 | namespaces=self.namespaces, 605 | namespace_style=self.namespace_style) 606 | self.generate_maintainer_script(filename=os.path.join(debian_directory, 'prerm'), 607 | python_executable=python_executable, 608 | function='pre_removal_hook', 609 | package_name=self.debian_name, 610 | alternatives=alternatives, 611 | modules_directory=install_modules_directory, 612 | namespaces=self.namespaces) 613 | 614 | # Enable a user defined Python callback to manipulate the resulting 615 | # binary package before it's turned into a *.deb archive (e.g. 616 | # manipulate the contents or change the package metadata). 617 | if self.converter.python_callback: 618 | logger.debug("Invoking user defined Python callback ..") 619 | self.converter.python_callback(self.converter, self, build_directory) 620 | logger.debug("User defined Python callback finished!") 621 | 622 | return build_package(directory=build_directory, 623 | check_package=self.converter.lintian_enabled, 624 | copy_files=False) 625 | 626 | def determine_package_architecture(self, has_shared_object_files): 627 | """ 628 | Determine binary architecture that Debian package should be tagged with. 629 | 630 | :param has_shared_objects: 631 | 632 | :data:`True` if the package contains ``*.so`` files, :data:`False` 633 | otherwise. 634 | 635 | :returns: 636 | 637 | The architecture string, 'all' or one of the values of 638 | :attr:`~py2deb.converter.PackageConverter.debian_architecture`. 639 | 640 | 641 | If a package contains ``*.so`` files we're dealing with a compiled 642 | Python module. To determine the applicable architecture, we take the 643 | Debian architecture reported by 644 | :attr:`~py2deb.converter.PackageConverter.debian_architecture`. 645 | """ 646 | logger.debug("Checking package architecture ..") 647 | if has_shared_object_files: 648 | logger.debug("Package contains shared object files, tagging with %s architecture.", 649 | self.converter.debian_architecture) 650 | return self.converter.debian_architecture 651 | else: 652 | logger.debug("Package doesn't contain shared object files, dealing with a portable package.") 653 | return 'all' 654 | 655 | def find_egg_info_file(self, pattern=''): 656 | """ 657 | Find :pypi:`pip` metadata files in unpacked source distributions. 658 | 659 | :param pattern: The :mod:`glob` pattern to search for (a string). 660 | :returns: A list of matched filenames (strings). 661 | 662 | When pip unpacks a source distribution archive it creates a directory 663 | ``pip-egg-info`` which contains the package metadata in a declarative 664 | and easy to parse format. This method finds such metadata files. 665 | """ 666 | full_pattern = os.path.join(self.requirement.source_directory, 'pip-egg-info', '*.egg-info', pattern) 667 | logger.debug("Looking for %r file(s) using pattern %r ..", pattern, full_pattern) 668 | matches = glob.glob(full_pattern) 669 | if len(matches) > 1: 670 | msg = "Source distribution directory of %s (%s) contains multiple *.egg-info directories: %s" 671 | raise Exception(msg % (self.requirement.project_name, self.requirement.version, concatenate(matches))) 672 | elif matches: 673 | logger.debug("Matched %s: %s.", pluralize(len(matches), "file", "files"), concatenate(matches)) 674 | return matches[0] 675 | else: 676 | logger.debug("No matching %r files found.", pattern) 677 | 678 | def generate_maintainer_script(self, filename, python_executable, function, **arguments): 679 | """ 680 | Generate a post-installation or pre-removal maintainer script. 681 | 682 | :param filename: 683 | 684 | The pathname of the maintainer script (a string). 685 | 686 | :param python_executable: 687 | 688 | The absolute pathname of the Python interpreter on the target 689 | system (a string). 690 | 691 | :param function: 692 | 693 | The name of the function in the :mod:`py2deb.hooks` module to be 694 | called when the maintainer script is run (a string). 695 | 696 | :param arguments: 697 | 698 | Any keyword arguments to the function in the :mod:`py2deb.hooks` 699 | are serialized to text using :func:`repr()` and embedded inside the 700 | generated maintainer script. 701 | """ 702 | # Read the py2deb/hooks.py script. 703 | py2deb_directory = os.path.dirname(os.path.abspath(__file__)) 704 | hooks_script = os.path.join(py2deb_directory, 'hooks.py') 705 | with open(hooks_script) as handle: 706 | contents = handle.read() 707 | blocks = contents.split('\n\n') 708 | # Generate the shebang / hashbang line. 709 | blocks.insert(0, '#!%s' % python_executable) 710 | # Generate the call to the top level function. 711 | encoded_arguments = ', '.join('%s=%r' % (k, v) for k, v in arguments.items()) 712 | blocks.append('%s(%s)' % (function, encoded_arguments)) 713 | # Write the maintainer script. 714 | with open(filename, 'w') as handle: 715 | handle.write('\n\n'.join(blocks)) 716 | handle.write('\n') 717 | # Make sure the maintainer script is executable. 718 | os.chmod(filename, 0o755) 719 | 720 | def load_control_field_overrides(self, control_fields): 721 | """ 722 | Apply user defined control field overrides. 723 | 724 | :param control_fields: 725 | 726 | The control field defaults constructed by py2deb (a 727 | :class:`deb_pkg_tools.deb822.Deb822` object). 728 | 729 | :returns: 730 | 731 | The merged defaults and overrides (a 732 | :class:`deb_pkg_tools.deb822.Deb822` object). 733 | 734 | Looks for an ``stdeb.cfg`` file inside the Python package's source 735 | distribution and if found it merges the overrides into the control 736 | fields that will be embedded in the generated Debian binary package. 737 | 738 | This method first applies any overrides defined in the ``DEFAULT`` 739 | section and then it applies any overrides defined in the section whose 740 | normalized name (see :func:`~py2deb.utils.package_names_match()`) 741 | matches that of the Python package. 742 | """ 743 | py2deb_cfg = os.path.join(self.requirement.source_directory, 'stdeb.cfg') 744 | if not os.path.isfile(py2deb_cfg): 745 | logger.debug("Control field overrides file not found (%s).", py2deb_cfg) 746 | else: 747 | logger.debug("Loading control field overrides from %s ..", py2deb_cfg) 748 | parser = configparser.RawConfigParser() 749 | parser.read(py2deb_cfg) 750 | # Prepare to load the overrides from the DEFAULT section and 751 | # the section whose name matches that of the Python package. 752 | # DEFAULT is processed first on purpose. 753 | section_names = ['DEFAULT'] 754 | # Match the normalized package name instead of the raw package 755 | # name because `python setup.py egg_info' normalizes 756 | # underscores in package names to dashes which can bite 757 | # unsuspecting users. For what it's worth, PEP-8 discourages 758 | # underscores in package names but doesn't forbid them: 759 | # https://www.python.org/dev/peps/pep-0008/#package-and-module-names 760 | section_names.extend(section_name for section_name in parser.sections() 761 | if package_names_match(section_name, self.python_name)) 762 | for section_name in section_names: 763 | if parser.has_section(section_name): 764 | overrides = dict(parser.items(section_name)) 765 | logger.debug("Found %i control file field override(s) in section %s of %s: %r", 766 | len(overrides), section_name, py2deb_cfg, overrides) 767 | control_fields = merge_control_fields(control_fields, overrides) 768 | return control_fields 769 | 770 | def transform_binary_dist(self, interpreter): 771 | """ 772 | Build Python package and transform directory layout. 773 | 774 | :param interpreter: 775 | 776 | The absolute pathname of the Python interpreter that should be 777 | referenced by executable scripts in the binary distribution (a 778 | string). 779 | 780 | :returns: 781 | 782 | An iterable of tuples with two values each: 783 | 784 | 1. A :class:`tarfile.TarInfo` object; 785 | 2. A file-like object. 786 | 787 | Builds the Python package (using :pypi:`pip-accel`) and changes the 788 | names of the files included in the package to match the layout 789 | corresponding to the given conversion options. 790 | """ 791 | # Detect whether we're running on PyPy (it needs special handling). 792 | if platform.python_implementation() == 'PyPy': 793 | on_pypy = True 794 | normalized_pypy_path = 'lib/pypy%i.%i/site-packages/' % sys.version_info[:2] 795 | if sys.version_info[0] == 3: 796 | # The file /usr/lib/pypy3/dist-packages/README points to 797 | # /usr/lib/pypy3/lib-python/3/site.py which states that in 798 | # PyPy 3 /usr/lib/python3/dist-packages is shared between 799 | # cPython and PyPy. 800 | normalized_pypy_segment = '/python3/' 801 | else: 802 | # The file /usr/lib/pypy/dist-packages/README points to 803 | # /usr/lib/pypy/lib-python/2.7/site.py which states that in 804 | # PyPy 2 /usr/lib/pypy/dist-packages is used for 805 | # "Debian addons" however when you run the interpreter and 806 | # inspect sys.path you'll find that /usr/lib/pypy/dist-packages 807 | # is being used instead of the directory. This might 808 | # be a documentation bug? 809 | normalized_pypy_segment = '/pypy/' 810 | else: 811 | on_pypy = False 812 | for member, handle in self.converter.pip_accel.bdists.get_binary_dist(self.requirement): 813 | is_executable = member.name.startswith('bin/') 814 | # Note that at this point the installation prefix has already been 815 | # stripped from `member.name' by the get_binary_dist() method. 816 | if on_pypy: 817 | # Normalize PyPy virtual environment layout: 818 | # 819 | # 1. cPython uses /lib/pythonX.Y/(dist|site)-packages/ 820 | # 2. PyPy uses /site-packages/ (a top level directory) 821 | # 822 | # In this if branch we change 2 to look like 1 so that the 823 | # following if/else branches don't need to care about the 824 | # difference. 825 | member.name = re.sub('^(dist|site)-packages/', normalized_pypy_path, member.name) 826 | if self.has_custom_install_prefix: 827 | # Strip the complete /usr/lib/pythonX.Y/site-packages/ prefix 828 | # so we can replace it with the custom installation prefix. 829 | member.name = re.sub(r'lib/(python|pypy)\d+(\.\d+)*/(dist|site)-packages/', 'lib/', member.name) 830 | # Rewrite executable Python scripts so they know about the 831 | # custom installation prefix. 832 | if is_executable: 833 | handle = embed_install_prefix(handle, os.path.join(self.converter.install_prefix, 'lib')) 834 | else: 835 | if on_pypy: 836 | # Normalize the PyPy "versioned directory segment" (it differs 837 | # between virtual environments versus system wide installations). 838 | member.name = re.sub(r'/pypy\d(\.\d)?/', normalized_pypy_segment, member.name) 839 | # Rewrite /site-packages/ to /dist-packages/. For details see 840 | # https://wiki.debian.org/Python#Deviations_from_upstream. 841 | member.name = member.name.replace('/site-packages/', '/dist-packages/') 842 | # Update the interpreter reference in the first line of executable scripts. 843 | if is_executable: 844 | handle = self.update_shebang(handle, interpreter) 845 | yield member, handle 846 | 847 | def update_shebang(self, handle, interpreter): 848 | """ 849 | Update the shebang_ of executable scripts. 850 | 851 | :param handle: 852 | 853 | A file-like object containing an executable. 854 | 855 | :param interpreter: 856 | 857 | The absolute pathname of the Python interpreter that should be 858 | referenced by the script (a string). 859 | 860 | :returns: 861 | 862 | A file-like object. 863 | 864 | Normally :pypi:`pip-accel` is responsible for updating interpreter 865 | references in executable scripts, however there's a bug in pip-accel 866 | where it assumes that the string 'python' will appear literally in the 867 | shebang (which isn't true when running on PyPy). 868 | 869 | .. note:: Of course this bug should be fixed in pip-accel however that 870 | project is in limbo while I decide whether to reinvigorate or 871 | kill it (the second of which implies needing to make a whole 872 | lot of changes to py2deb). 873 | 874 | .. _shebang: https://en.wikipedia.org/wiki/Shebang_(Unix) 875 | """ 876 | if detect_python_script(handle): 877 | lines = handle.readlines() 878 | lines[0] = b'#!' + interpreter.encode('ascii') + b'\n' 879 | handle = BytesIO(b''.join(lines)) 880 | handle.seek(0) 881 | return handle 882 | 883 | def __str__(self): 884 | """The name, version and extras of the package encoded in a human readable string.""" 885 | version = [self.python_version] 886 | extras = self.requirement.pip_requirement.extras 887 | if extras: 888 | version.append("extras: %s" % concatenate(sorted(extras))) 889 | return "%s (%s)" % (self.python_name, ', '.join(version)) 890 | -------------------------------------------------------------------------------- /py2deb/utils.py: -------------------------------------------------------------------------------- 1 | # Utility functions for py2deb. 2 | # 3 | # Authors: 4 | # - Arjan Verwer 5 | # - Peter Odding 6 | # Last Change: August 6, 2020 7 | # URL: https://py2deb.readthedocs.io 8 | 9 | """The :mod:`py2deb.utils` module contains miscellaneous code.""" 10 | 11 | # Standard library modules. 12 | import logging 13 | import os 14 | import platform 15 | import re 16 | import shlex 17 | import shutil 18 | import sys 19 | import tempfile 20 | 21 | # External dependencies. 22 | from property_manager import PropertyManager, cached_property, required_property 23 | from deb_pkg_tools.package import find_package_archives 24 | from six import BytesIO 25 | 26 | # Initialize a logger. 27 | logger = logging.getLogger(__name__) 28 | 29 | integer_pattern = re.compile('([0-9]+)') 30 | """Compiled regular expression to match a consecutive run of digits.""" 31 | 32 | PYTHON_EXECUTABLE_PATTERN = re.compile(r'^(pypy|python)(\d(\.\d)?)?m?$') 33 | """ 34 | A compiled regular expression to match Python interpreter executable names. 35 | 36 | The following are examples of program names that match this pattern: 37 | 38 | - pypy 39 | - pypy2.7 40 | - pypy3 41 | - python 42 | - python2 43 | - python2.7 44 | - python3m 45 | """ 46 | 47 | 48 | class PackageRepository(PropertyManager): 49 | 50 | """ 51 | Very simply abstraction for a directory containing ``*.deb`` archives. 52 | 53 | Used by :class:`py2deb.converter.PackageConverter` to recognize which 54 | Python packages have previously been converted (and so can be skipped). 55 | """ 56 | 57 | def __init__(self, directory): 58 | """ 59 | Initialize a :class:`PackageRepository` object. 60 | 61 | :param directory: The pathname of a directory containing ``*.deb`` archives (a string). 62 | """ 63 | super(PackageRepository, self).__init__(directory=directory) 64 | 65 | @cached_property 66 | def archives(self): 67 | """ 68 | A sorted list of package archives in :attr:`directory`. 69 | 70 | The value of :attr:`archives` is computed using 71 | :func:`deb_pkg_tools.package.find_package_archives()`. 72 | 73 | An example: 74 | 75 | >>> from py2deb import PackageRepository 76 | >>> repo = PackageRepository('/tmp') 77 | >>> repo.archives 78 | [PackageFile(name='py2deb', version='0.1', architecture='all', 79 | filename='/tmp/py2deb_0.1_all.deb'), 80 | PackageFile(name='py2deb-cached-property', version='0.1.5', architecture='all', 81 | filename='/tmp/py2deb-cached-property_0.1.5_all.deb'), 82 | PackageFile(name='py2deb-chardet', version='2.2.1', architecture='all', 83 | filename='/tmp/py2deb-chardet_2.2.1_all.deb'), 84 | PackageFile(name='py2deb-coloredlogs', version='0.5', architecture='all', 85 | filename='/tmp/py2deb-coloredlogs_0.5_all.deb'), 86 | PackageFile(name='py2deb-deb-pkg-tools', version='1.20.4', architecture='all', 87 | filename='/tmp/py2deb-deb-pkg-tools_1.20.4_all.deb'), 88 | PackageFile(name='py2deb-docutils', version='0.11', architecture='all', 89 | filename='/tmp/py2deb-docutils_0.11_all.deb'), 90 | PackageFile(name='py2deb-executor', version='1.2', architecture='all', 91 | filename='/tmp/py2deb-executor_1.2_all.deb'), 92 | PackageFile(name='py2deb-html2text', version='2014.4.5', architecture='all', 93 | filename='/tmp/py2deb-html2text_2014.4.5_all.deb'), 94 | PackageFile(name='py2deb-humanfriendly', version='1.8.2', architecture='all', 95 | filename='/tmp/py2deb-humanfriendly_1.8.2_all.deb'), 96 | PackageFile(name='py2deb-pkginfo', version='1.1', architecture='all', 97 | filename='/tmp/py2deb-pkginfo_1.1_all.deb'), 98 | PackageFile(name='py2deb-python-debian', version='0.1.21-nmu2', architecture='all', 99 | filename='/tmp/py2deb-python-debian_0.1.21-nmu2_all.deb'), 100 | PackageFile(name='py2deb-six', version='1.6.1', architecture='all', 101 | filename='/tmp/py2deb-six_1.6.1_all.deb')] 102 | 103 | """ 104 | return find_package_archives(self.directory) 105 | 106 | @required_property 107 | def directory(self): 108 | """The pathname of a directory containing ``*.deb`` archives (a string).""" 109 | 110 | def get_package(self, package, version, architecture): 111 | """ 112 | Find a package in the repository. 113 | 114 | :param package: The name of the package (a string). 115 | :param version: The version of the package (a string). 116 | :param architecture: The architecture of the package (a string). 117 | :returns: A :class:`deb_pkg_tools.package.PackageFile` object or :data:`None`. 118 | 119 | Here's an example: 120 | 121 | >>> from py2deb import PackageRepository 122 | >>> repo = PackageRepository('/tmp') 123 | >>> repo.get_package('py2deb', '0.1', 'all') 124 | PackageFile(name='py2deb', version='0.1', architecture='all', filename='/tmp/py2deb_0.1_all.deb') 125 | """ 126 | for archive in self.archives: 127 | if archive.name == package and archive.version == version and archive.architecture == architecture: 128 | return archive 129 | 130 | 131 | class TemporaryDirectory(object): 132 | 133 | """ 134 | Easy temporary directory creation & cleanup using the :keyword:`with` statement. 135 | 136 | Here's an example of how to use this: 137 | 138 | .. code-block:: python 139 | 140 | with TemporaryDirectory() as directory: 141 | # Do something useful here. 142 | assert os.path.isdir(directory) 143 | """ 144 | 145 | def __init__(self, **options): 146 | """ 147 | Initialize context manager that manages creation & cleanup of temporary directory. 148 | 149 | :param options: Any keyword arguments are passed on to :func:`tempfile.mkdtemp()`. 150 | """ 151 | self.options = options 152 | 153 | def __enter__(self): 154 | """Create the temporary directory.""" 155 | self.temporary_directory = tempfile.mkdtemp(**self.options) 156 | logger.debug("Created temporary directory: %s", self.temporary_directory) 157 | return self.temporary_directory 158 | 159 | def __exit__(self, exc_type, exc_value, traceback): 160 | """Destroy the temporary directory.""" 161 | logger.debug("Cleaning up temporary directory: %s", self.temporary_directory) 162 | shutil.rmtree(self.temporary_directory) 163 | del self.temporary_directory 164 | 165 | 166 | def compact_repeating_words(words): 167 | """ 168 | Remove adjacent repeating words. 169 | 170 | :param words: 171 | 172 | An iterable of words (strings), assumed to 173 | already be normalized (lowercased). 174 | 175 | :returns: 176 | 177 | An iterable of words with adjacent repeating 178 | words replaced by a single word. 179 | 180 | This is used to avoid awkward word repetitions in the package name 181 | conversion algorithm. Here's an example of what I mean: 182 | 183 | >>> from py2deb import compact_repeating_words 184 | >>> name_prefix = 'python' 185 | >>> package_name = 'python-mcrypt' 186 | >>> combined_words = [name_prefix] + package_name.split('-') 187 | >>> print(list(combined_words)) 188 | ['python', 'python', 'mcrypt'] 189 | >>> compacted_words = compact_repeating_words(combined_words) 190 | >>> print(list(compacted_words)) 191 | ['python', 'mcrypt'] 192 | """ 193 | last_word = None 194 | for word in words: 195 | if word != last_word: 196 | yield word 197 | last_word = word 198 | 199 | 200 | def convert_package_name(python_package_name, name_prefix=None, extras=()): 201 | """ 202 | Convert a Python package name to a Debian package name. 203 | 204 | :param python_package_name: 205 | 206 | The name of a Python package as found on PyPI (a string). 207 | 208 | :param name_prefix: 209 | 210 | The name prefix to apply (a string or :data:`None`, in which case the 211 | result of :func:`default_name_prefix()` is used instead). 212 | 213 | :returns: 214 | 215 | A Debian package name (a string). 216 | """ 217 | # Apply the name prefix. 218 | if not name_prefix: 219 | name_prefix = default_name_prefix() 220 | debian_package_name = '%s-%s' % (name_prefix, python_package_name) 221 | # Normalize casing and special characters. 222 | debian_package_name = normalize_package_name(debian_package_name) 223 | # Compact repeating words (to avoid package names like 'python-python-debian'). 224 | debian_package_name = '-'.join(compact_repeating_words(debian_package_name.split('-'))) 225 | # If a requirement includes extras this changes the dependencies of the 226 | # package. Because Debian doesn't have this concept we encode the names of 227 | # the extras in the name of the package. 228 | if extras: 229 | words = [debian_package_name] 230 | words.extend(sorted(extra.lower() for extra in extras)) 231 | debian_package_name = '-'.join(words) 232 | return debian_package_name 233 | 234 | 235 | def default_name_prefix(): 236 | """ 237 | Get the default package name prefix for the Python version we're running. 238 | 239 | :returns: One of the strings ``python``, ``python3`` or ``pypy``. 240 | """ 241 | implementation = 'pypy' if platform.python_implementation() == 'PyPy' else 'python' 242 | if sys.version_info[0] == 3: 243 | implementation += '3' 244 | return implementation 245 | 246 | 247 | def detect_python_script(handle): 248 | """ 249 | Detect whether a file-like object contains an executable Python script. 250 | 251 | :param handle: 252 | 253 | A file-like object (assumed to contain an executable). 254 | 255 | :returns: 256 | 257 | :data:`True` if the program name in the shebang_ of the script 258 | references a known Python interpreter, :data:`False` otherwise. 259 | """ 260 | command = extract_shebang_command(handle) 261 | program = extract_shebang_program(command) 262 | return PYTHON_EXECUTABLE_PATTERN.match(program) is not None 263 | 264 | 265 | def embed_install_prefix(handle, install_prefix): 266 | """ 267 | Embed Python snippet that adds custom installation prefix to module search path. 268 | 269 | :param handle: A file-like object containing an executable Python script. 270 | :param install_prefix: The pathname of the custom installation prefix (a string). 271 | :returns: A file-like object containing the modified Python script. 272 | """ 273 | # Make sure the first line of the file contains something that looks like a 274 | # Python hashbang so we don't try to embed Python code in files like shell 275 | # scripts :-). 276 | if detect_python_script(handle): 277 | lines = handle.readlines() 278 | # We need to choose where to inject our line into the Python script. 279 | # This is trickier than it might seem at first, because of conflicting 280 | # concerns: 281 | # 282 | # 1) We want our line to be the first one to be executed so that any 283 | # later imports respect the custom installation prefix. 284 | # 285 | # 2) Our line cannot be the very first line because we would break the 286 | # hashbang of the script, without which it won't be executable. 287 | # 288 | # 3) Python has the somewhat obscure `from __future__ import ...' 289 | # statement which must precede all other statements. 290 | # 291 | # Our first step is to skip all comments, taking care of point two. 292 | insertion_point = 0 293 | while insertion_point < len(lines) and lines[insertion_point].startswith(b'#'): 294 | insertion_point += 1 295 | # The next step is to bump the insertion point if we find any `from 296 | # __future__ import ...' statements. 297 | for i, line in enumerate(lines): 298 | if re.match(b'^\\s*from\\s+__future__\\s+import\\s+', line): 299 | insertion_point = i + 1 300 | lines.insert(insertion_point, ('import sys; sys.path.insert(0, %r)\n' % install_prefix).encode('UTF-8')) 301 | # Turn the modified contents back into a file-like object. 302 | handle = BytesIO(b''.join(lines)) 303 | else: 304 | # Reset the file pointer of handle, so its contents can be read again later. 305 | handle.seek(0) 306 | return handle 307 | 308 | 309 | def extract_shebang_command(handle): 310 | """ 311 | Extract the shebang_ command line from an executable script. 312 | 313 | :param handle: A file-like object (assumed to contain an executable). 314 | :returns: The command in the shebang_ line (a string). 315 | 316 | The seek position is expected to be at the start of the file and will be 317 | reset afterwards, before this function returns. It is not an error if the 318 | executable contains binary data. 319 | 320 | .. _shebang: https://en.wikipedia.org/wiki/Shebang_(Unix) 321 | """ 322 | try: 323 | if handle.read(2) == b'#!': 324 | data = handle.readline() 325 | text = data.decode('UTF-8') 326 | return text.strip() 327 | else: 328 | return '' 329 | finally: 330 | handle.seek(0) 331 | 332 | 333 | def extract_shebang_program(command): 334 | """ 335 | Extract the program name from a shebang_ command line. 336 | 337 | :param command: The result of :func:`extract_shebang_command()`. 338 | :returns: The program name in the shebang_ command line (a string). 339 | """ 340 | tokens = shlex.split(command) 341 | if len(tokens) >= 2 and os.path.basename(tokens[0]) == 'env': 342 | tokens = tokens[1:] 343 | return os.path.basename(tokens[0]) if tokens else '' 344 | 345 | 346 | def normalize_package_name(python_package_name): 347 | """ 348 | Normalize Python package name to be used as Debian package name. 349 | 350 | :param python_package_name: 351 | 352 | The name of a Python package as found on PyPI (a string). 353 | 354 | :returns: 355 | 356 | The normalized name (a string). 357 | 358 | >>> from py2deb import normalize_package_name 359 | >>> normalize_package_name('MySQL-python') 360 | 'mysql-python' 361 | >>> normalize_package_name('simple_json') 362 | 'simple-json' 363 | """ 364 | return re.sub('[^a-z0-9]+', '-', python_package_name.lower()).strip('-') 365 | 366 | 367 | def normalize_package_version(python_package_version, prerelease_workaround=True): 368 | """ 369 | Normalize Python package version to be used as Debian package version. 370 | 371 | :param python_package_version: 372 | 373 | The version of a Python package (a string). 374 | 375 | :param prerelease_workaround: 376 | 377 | :data:`True` to enable the pre-release handling documented below, 378 | :data:`False` to restore the old behavior. 379 | 380 | Reformats Python package versions to comply with the Debian policy manual. 381 | All characters except alphanumerics, dot (``.``) and plus (``+``) are 382 | replaced with dashes (``-``). 383 | 384 | The PEP 440 pre-release identifiers 'a', 'b', 'c' and 'rc' are prefixed by 385 | a tilde (``~``) to replicate the intended ordering in Debian versions, also 386 | the identifier 'c' is translated into 'rc'. Refer to `issue #8 387 | `_ for details. 388 | """ 389 | # We need to avoid normalizing "local version labels" (naming from PEP 440) 390 | # because these may contain strings such as SCM hashes that should not be 391 | # altered, so we split the version string into the "public version 392 | # identifier" and "local version label" and only apply normalization to the 393 | # "public version identifier". 394 | public_version, delimiter, local_version = python_package_version.partition('+') 395 | # Lowercase and remove invalid characters from the "public version identifier". 396 | public_version = re.sub('[^a-z0-9.+]+', '-', public_version.lower()).strip('-') 397 | if prerelease_workaround: 398 | # Translate the PEP 440 pre-release identifier 'c' to 'rc'. 399 | public_version = re.sub(r'(\d)c(\d)', r'\1rc\2', public_version) 400 | # Replicate the intended ordering of PEP 440 pre-release versions (a, b, rc). 401 | public_version = re.sub(r'(\d)(a|b|rc)(\d)', r'\1~\2\3', public_version) 402 | # Restore the local version label (without any normalization). 403 | if local_version: 404 | public_version = public_version + '+' + local_version 405 | # Make sure the "Debian revision" contains a digit. If we don't find one we 406 | # add it ourselves, to prevent dpkg and apt from aborting (!) as soon as 407 | # they see an invalid Debian revision... 408 | if '-' in public_version: 409 | components = public_version.split('-') 410 | if len(components) > 1 and not re.search('[0-9]', components[-1]): 411 | components.append('1') 412 | public_version = '-'.join(components) 413 | return public_version 414 | 415 | 416 | def package_names_match(a, b): 417 | """ 418 | Check whether two Python package names are equal. 419 | 420 | :param a: The name of the first Python package (a string). 421 | :param b: The name of the second Python package (a string). 422 | :returns: :data:`True` if the package names match, :data:`False` if they don't. 423 | 424 | Uses :func:`normalize_package_name()` to normalize both names before 425 | comparing them for equality. This makes sure differences in case and dashes 426 | versus underscores are ignored. 427 | """ 428 | return normalize_package_name(a) == normalize_package_name(b) 429 | 430 | 431 | def python_version(): 432 | """ 433 | Find the version of Python we're running. 434 | 435 | :returns: A string like ``python2.7``, ``python3.8``, ``pypy`` or ``pypy3``. 436 | 437 | This specifically returns a name that matches both of the following: 438 | 439 | - The name of the Debian package providing the current Python version. 440 | - The name of the interpreter executable for the current Python version. 441 | """ 442 | if platform.python_implementation() == 'PyPy': 443 | python_version = 'pypy' 444 | if sys.version_info[0] == 3: 445 | python_version += '3' 446 | else: 447 | python_version = 'python%d.%d' % sys.version_info[:2] 448 | logger.debug("Detected Python version: %s", python_version) 449 | return python_version 450 | 451 | 452 | def tokenize_version(version_number): 453 | """ 454 | Tokenize a string containing a version number. 455 | 456 | :param version_number: The string to tokenize. 457 | :returns: A list of strings. 458 | """ 459 | return [t for t in integer_pattern.split(version_number) if t] 460 | -------------------------------------------------------------------------------- /requirements-checks.txt: -------------------------------------------------------------------------------- 1 | # Python packages required to run `make check'. 2 | flake8 >= 2.6.0 3 | flake8-docstrings >= 0.2.8 4 | pyflakes >= 1.2.3 5 | -------------------------------------------------------------------------------- /requirements-tests.txt: -------------------------------------------------------------------------------- 1 | # Test suite requirements. 2 | coverage >= 4.2 3 | pytest >= 3.0.4 4 | pytest-cov >= 2.4.0 5 | 6 | # The following packages are part of the setup_requires of transitive 7 | # requirements like pytest, unfortunately when they're installed automatically 8 | # (nested inside the main "pip install" run) the nested pip commands try to 9 | # install via wheels, which is broken on PyPy. By lifting these dependencies to 10 | # the top level our --no-binary=:all: choice should hopefully be respected. 11 | pytest-runner 12 | setuptools-scm 13 | -------------------------------------------------------------------------------- /requirements-travis.txt: -------------------------------------------------------------------------------- 1 | --requirement=requirements-checks.txt 2 | --requirement=requirements-tests.txt 3 | --requirement=requirements.txt 4 | coveralls 5 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # Installation requirements. 2 | 3 | coloredlogs >= 0.5 4 | deb-pkg-tools >= 5.2 5 | executor >= 21.0 6 | humanfriendly >= 8.0 7 | pip-accel >= 0.25, <= 0.43 8 | pkginfo >= 1.1 9 | property-manager >= 2.3.1 10 | six >= 1.6.1 11 | -------------------------------------------------------------------------------- /scripts/pypy.py: -------------------------------------------------------------------------------- 1 | """ 2 | Downgrade to pip < 20.2 when running on PyPy. 3 | 4 | Unfortunately pip 20.2 (the most recent release at the time of writing) breaks 5 | compatibility with PyPy, thereby causing Travis CI builds of py2deb to fail as 6 | well. For details please refer to https://github.com/pypa/pip/issues/8653. 7 | """ 8 | 9 | import pip 10 | import platform 11 | import subprocess 12 | import sys 13 | 14 | from distutils.version import LooseVersion 15 | 16 | if platform.python_implementation() == "PyPy": 17 | installed_release = LooseVersion(pip.__version__) 18 | known_bad_release = LooseVersion("20.2") 19 | if installed_release >= known_bad_release: 20 | sys.stderr.write("[scripts/pypy.py] Removing incompatible pip release ..\n") 21 | subprocess.call([sys.executable, "-m", "pip", "uninstall", "--yes", "pip"]) 22 | sys.stderr.write("[scripts/pypy.py] Installing compatible pip using setuptools ..\n") 23 | subprocess.check_call([sys.executable, "-m", "easy_install", "--always-unzip", "pip < 20.2"]) 24 | -------------------------------------------------------------------------------- /scripts/pypy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | # This shell script is run by tox from the project directory, one level up. 4 | # This explains why we have to use `scripts/' in the pathname below. 5 | 6 | # Downgrade to pip < 20.2 when running on PyPy inside tox. 7 | echo "[scripts/pypy.sh] pip before downgrade: $(pip --version)" >&2 8 | python scripts/pypy.py 9 | echo "[scripts/pypy.sh] pip after downgrade: $(pip --version)" >&2 10 | 11 | # Continue installing packages as normal. 12 | python -m pip install "$@" 13 | -------------------------------------------------------------------------------- /scripts/travis.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | # This Bash script is responsible for running the py2deb test suite on the 4 | # Travis CI hosted continuous integration service. Some notes about this 5 | # script: 6 | # 7 | # 1. Travis CI provides Python 2.6, 2.7 and 3.4 installations based on a Chef 8 | # cookbook that uses pyenv which means that the Python installations used 9 | # are custom compiled and not managed using Debian packages. This is a 10 | # problem for py2deb because it depends on the correct functioning of 11 | # dpkg-shlibdeps which expects Python to be installed using Debian 12 | # packages. 13 | # 14 | # 2. A dozen convoluted workarounds can be constructed to work around this. 15 | # I've decided to go with a fairly simple one that I know very well and 16 | # which has worked very well for the local testing that I've been doing for 17 | # months: Using the `deadsnakes PPA' to install various Python versions 18 | # using Debian packages. 19 | 20 | # The following Debian system packages are required for all builds. 21 | REQUIRED_SYSTEM_PACKAGES="dpkg-dev fakeroot lintian" 22 | 23 | main () { 24 | msg "Preparing Travis CI test environment .." 25 | case "$TOXENV" in 26 | py27) 27 | # At the time of writing Travis CI workers are running Ubuntu 12.04 which 28 | # includes Python 2.7 as the default system wide Python version so we 29 | # don't need the deadsnakes PPA. 30 | install_with_apt_get python2.7 python2.7-dev 31 | ;; 32 | py35) 33 | # We need to get Python 3.5 from the deadsnakes PPA. 34 | install_with_deadsnakes_ppa python3.5 python3.5-dev 35 | ;; 36 | py36) 37 | # We need to get Python 3.6 from the deadsnakes PPA. 38 | install_with_deadsnakes_ppa python3.6 python3.6-dev 39 | ;; 40 | py37) 41 | # We need to get Python 3.7 from the deadsnakes PPA. 42 | install_with_deadsnakes_ppa python3.7 python3.7-dev 43 | ;; 44 | pypy) 45 | # Get PyPy 2 from the official PyPy PPA. 46 | install_with_pypy_ppa pypy pypy-dev 47 | ;; 48 | pypy3) 49 | # Get PyPy 3 from the official PyPy PPA. 50 | install_with_pypy_ppa pypy3 pypy3-dev 51 | ;; 52 | *) 53 | # Make sure .travis.yml and .travis.sh don't get out of sync. 54 | die "Unsupported Python version requested! (\$TOXENV not set)" 55 | ;; 56 | esac 57 | } 58 | 59 | install_with_deadsnakes_ppa () { 60 | msg "Installing deadsnakes PPA .." 61 | sudo add-apt-repository --yes ppa:deadsnakes/ppa 62 | install_with_apt_get "$@" 63 | } 64 | 65 | install_with_pypy_ppa () { 66 | msg "Installing PyPy PPA .." 67 | sudo add-apt-repository --yes ppa:pypy/ppa 68 | install_with_apt_get "$@" 69 | } 70 | 71 | install_with_apt_get () { 72 | export DEBIAN_FRONTEND=noninteractive 73 | msg "Installing with apt-get: $REQUIRED_SYSTEM_PACKAGES $*" 74 | sudo apt-get update --quiet --quiet 75 | sudo apt-get install --yes --quiet $REQUIRED_SYSTEM_PACKAGES "$@" 76 | } 77 | 78 | die () { 79 | msg "Error: $*" 80 | exit 1 81 | } 82 | 83 | msg () { 84 | echo "[travis.sh] $*" >&2 85 | } 86 | 87 | main "$@" 88 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | # Enable building of universal wheels so we can publish wheel 2 | # distribution archives to PyPI (the Python package index) 3 | # that are compatible with Python 2 as well as Python 3. 4 | 5 | [wheel] 6 | universal=1 7 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Setup script for the `py2deb' package. 4 | 5 | # Author: Peter Odding 6 | # Last Change: August 4, 2020 7 | # URL: https://py2deb.readthedocs.io 8 | 9 | """ 10 | Setup script for the `py2deb` package. 11 | 12 | **python setup.py install** 13 | Install from the working directory into the current Python environment. 14 | 15 | **python setup.py sdist** 16 | Build a source distribution archive. 17 | 18 | **python setup.py bdist_wheel** 19 | Build a wheel distribution archive. 20 | """ 21 | 22 | # Standard library modules. 23 | import codecs 24 | import os 25 | import re 26 | import sys 27 | 28 | # De-facto standard solution for Python packaging. 29 | from setuptools import find_packages, setup 30 | 31 | 32 | def get_contents(*args): 33 | """Get the contents of a file relative to the source distribution directory.""" 34 | with codecs.open(get_absolute_path(*args), 'r', 'UTF-8') as handle: 35 | return handle.read() 36 | 37 | 38 | def get_version(*args): 39 | """Extract the version number from a Python module.""" 40 | contents = get_contents(*args) 41 | metadata = dict(re.findall('__([a-z]+)__ = [\'"]([^\'"]+)', contents)) 42 | return metadata['version'] 43 | 44 | 45 | def get_install_requires(): 46 | """Get the conditional dependencies for source distributions.""" 47 | install_requires = get_requirements('requirements.txt') 48 | if 'bdist_wheel' not in sys.argv: 49 | if sys.version_info[:2] <= (2, 6) or sys.version_info[:2] == (3, 0): 50 | install_requires.append('importlib') 51 | return sorted(install_requires) 52 | 53 | 54 | def get_extras_require(): 55 | """Get the conditional dependencies for wheel distributions.""" 56 | extras_require = {} 57 | if have_environment_marker_support(): 58 | expression = ':python_version == "2.6" or python_version == "3.0"' 59 | extras_require[expression] = ['importlib'] 60 | return extras_require 61 | 62 | 63 | def get_absolute_path(*args): 64 | """Transform relative pathnames into absolute pathnames.""" 65 | return os.path.join(os.path.dirname(os.path.abspath(__file__)), *args) 66 | 67 | 68 | def get_requirements(*args): 69 | """Get requirements from pip requirement files.""" 70 | requirements = set() 71 | with open(get_absolute_path(*args)) as handle: 72 | for line in handle: 73 | # Strip comments. 74 | line = re.sub(r'^#.*|\s#.*', '', line) 75 | # Ignore empty lines 76 | if line and not line.isspace(): 77 | requirements.add(re.sub(r'\s+', '', line)) 78 | return sorted(requirements) 79 | 80 | 81 | def have_environment_marker_support(): 82 | """ 83 | Check whether setuptools has support for PEP-426 environment markers. 84 | 85 | Based on the ``setup.py`` script of the ``pytest`` package: 86 | https://bitbucket.org/pytest-dev/pytest/src/default/setup.py 87 | """ 88 | try: 89 | from pkg_resources import parse_version 90 | from setuptools import __version__ 91 | return parse_version(__version__) >= parse_version('0.7.2') 92 | except Exception: 93 | return False 94 | 95 | 96 | setup( 97 | name='py2deb', 98 | version=get_version('py2deb', '__init__.py'), 99 | description="Python to Debian package converter", 100 | long_description=get_contents('README.rst'), 101 | url='https://py2deb.readthedocs.io', 102 | author="Peter Odding & Arjan Verwer (Paylogic International)", 103 | author_email='peter.odding@paylogic.com', 104 | license='MIT', 105 | packages=find_packages(), 106 | entry_points=dict(console_scripts=[ 107 | 'py2deb = py2deb.cli:main', 108 | ]), 109 | install_requires=get_install_requires(), 110 | extras_require=get_extras_require(), 111 | test_suite='py2deb.tests', 112 | include_package_data=True, 113 | python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*', 114 | classifiers=[ 115 | 'Development Status :: 5 - Production/Stable', 116 | 'Intended Audience :: Developers', 117 | 'Intended Audience :: Information Technology', 118 | 'Intended Audience :: System Administrators', 119 | 'License :: OSI Approved :: MIT License', 120 | 'Operating System :: POSIX :: Linux', 121 | 'Programming Language :: Python', 122 | 'Programming Language :: Python :: 2', 123 | 'Programming Language :: Python :: 2.7', 124 | 'Programming Language :: Python :: 3', 125 | 'Programming Language :: Python :: 3.5', 126 | 'Programming Language :: Python :: 3.6', 127 | 'Programming Language :: Python :: 3.7', 128 | 'Programming Language :: Python :: Implementation :: CPython', 129 | 'Programming Language :: Python :: Implementation :: PyPy', 130 | 'Topic :: Software Development :: Build Tools', 131 | 'Topic :: Software Development :: Libraries :: Python Modules', 132 | 'Topic :: System :: Archiving :: Packaging', 133 | 'Topic :: System :: Installation/Setup', 134 | 'Topic :: System :: Software Distribution', 135 | 'Topic :: System :: Systems Administration', 136 | 'Topic :: Utilities', 137 | ]) 138 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | # Tox (http://tox.testrun.org/) is a tool for running Python test suites on 2 | # multiple versions of Python with a single command. This configuration file 3 | # will run the test suite on all supported Python versions. To use it, 4 | # `pip-accel install tox' and then run `tox' from this directory. 5 | 6 | [tox] 7 | envlist = py27, py34, py35, py36, py37, pypy, pypy3 8 | 9 | [testenv] 10 | deps = 11 | --requirement=requirements-tests.txt 12 | --constraint=constraints.txt 13 | commands = py.test --cov {posargs} 14 | passenv = TRAVIS 15 | 16 | [pytest] 17 | addopts = -p no:logging --verbose 18 | python_files = py2deb/tests.py 19 | 20 | [flake8] 21 | exclude = .tox 22 | extend-ignore = D211,D401,D412 23 | max-line-length = 120 24 | 25 | # The following sections force Tox to create virtual environments based on 26 | # Python binaries that are (assumed to be) installed using Debian packages 27 | # because this is required for py2deb to function properly. This forces Tox to 28 | # sidestep the custom compiled Python binaries that are used on Travis CI by 29 | # default. See https://github.com/paylogic/py2deb/issues/3. 30 | 31 | [testenv:py27] 32 | basepython = /usr/bin/python2.7 33 | 34 | [testenv:py35] 35 | basepython = /usr/bin/python3.5 36 | 37 | [testenv:py36] 38 | basepython = /usr/bin/python3.6 39 | 40 | [testenv:py37] 41 | basepython = /usr/bin/python3.7 42 | 43 | [testenv:pypy] 44 | basepython = /usr/bin/pypy 45 | install_command = {toxinidir}/scripts/pypy.sh {opts} {packages} 46 | 47 | [testenv:pypy3] 48 | basepython = /usr/bin/pypy3 49 | install_command = {toxinidir}/scripts/pypy.sh {opts} {packages} 50 | --------------------------------------------------------------------------------