├── .github └── workflows │ └── tox.yml ├── .gitignore ├── .readthedocs.yml ├── CONTRIBUTING.md ├── LICENSE ├── MANIFEST.in ├── NOTICE ├── README.md ├── docs ├── api.rst ├── cli-reference.rst ├── conf.py ├── django.rst ├── history.rst ├── index.rst └── requirements.txt ├── logo.png ├── pyproject.toml ├── setup.cfg ├── src └── shiv │ ├── __init__.py │ ├── __main__.py │ ├── __version__.py │ ├── bootstrap │ ├── __init__.py │ ├── environment.py │ ├── filelock.py │ └── interpreter.py │ ├── builder.py │ ├── cli.py │ ├── constants.py │ ├── info.py │ └── pip.py ├── test ├── conftest.py ├── package │ ├── hello │ │ ├── __init__.py │ │ └── script.sh │ └── setup.py ├── test.zip ├── test_bootstrap.py ├── test_builder.py ├── test_cli.py └── test_pip.py └── tox.ini /.github/workflows/tox.yml: -------------------------------------------------------------------------------- 1 | name: ci 2 | 3 | on: 4 | - push 5 | - pull_request 6 | 7 | jobs: 8 | build: 9 | runs-on: ${{ matrix.platform }} 10 | strategy: 11 | max-parallel: 4 12 | matrix: 13 | platform: [ubuntu-latest, macos-latest, windows-latest] 14 | python-version: [3.8, 3.9, '3.10', '3.11'] 15 | 16 | steps: 17 | - uses: actions/checkout@v2 18 | 19 | - name: set up python ${{ matrix.python-version }} 20 | uses: actions/setup-python@v2 21 | with: 22 | python-version: ${{ matrix.python-version }} 23 | 24 | - name: install dependencies 25 | run: | 26 | python -m pip install --disable-pip-version-check --upgrade pip 27 | python -m pip install tox tox-gh-actions coverage 28 | 29 | - name: test with tox 30 | run: tox 31 | env: 32 | PLATFORM: ${{ matrix.platform }} 33 | 34 | - name: coverage 35 | run: "python -m coverage xml" 36 | 37 | - name: "upload coverage to codecov" 38 | if: ${{ github.event_name == 'push' }} 39 | uses: "codecov/codecov-action@v1" 40 | with: 41 | token: ${{ secrets.CODECOV_TOKEN }} 42 | fail_ci_if_error: true 43 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # Distribution / packaging 7 | .Python 8 | env/ 9 | build/ 10 | develop-eggs/ 11 | dist/ 12 | downloads/ 13 | eggs/ 14 | .eggs/ 15 | lib/ 16 | lib64/ 17 | parts/ 18 | sdist/ 19 | var/ 20 | *.egg-info/ 21 | .installed.cfg 22 | *.egg 23 | 24 | # PyInstaller 25 | # Usually these files are written by a python script from a template 26 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 27 | *.manifest 28 | *.spec 29 | 30 | # Installer logs 31 | pip-log.txt 32 | pip-delete-this-directory.txt 33 | 34 | # Unit test / coverage reports 35 | htmlcov/ 36 | .tox/ 37 | .coverage 38 | .coverage.* 39 | .cache 40 | nosetests.xml 41 | coverage.xml 42 | *,cover 43 | .hypothesis/ 44 | 45 | # Translations 46 | *.mo 47 | *.pot 48 | 49 | # Django stuff: 50 | *.log 51 | 52 | # Sphinx documentation 53 | docs/_build/ 54 | 55 | # PyBuilder 56 | target/ 57 | 58 | # pyenv python configuration file 59 | .python-version 60 | .pytest_cache/ 61 | .venv/ 62 | venv/ 63 | activate 64 | 65 | # mypy 66 | .mypy_cache 67 | 68 | # IDEA 69 | .idea/ 70 | *.iml 71 | *.ipr 72 | *.iws 73 | 74 | # VS Code 75 | .vscode/ 76 | 77 | # Gradle 78 | .gradle/ 79 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | build: 2 | os: ubuntu-22.04 3 | tools: 4 | python: "3.11" 5 | 6 | python: 7 | install: 8 | - method: pip 9 | path: . 10 | extra_requirements: 11 | - rtd 12 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Contribution Agreement 2 | ====================== 3 | 4 | As a contributor, you represent that the code you submit is your 5 | original work or that of your employer (in which case you represent you 6 | have the right to bind your employer). By submitting code, you (and, if 7 | applicable, your employer) are licensing the submitted code to LinkedIn 8 | and the open source community subject to the BSD 2-Clause license. 9 | 10 | Responsible Disclosure of Security Vulnerabilities 11 | ================================================== 12 | 13 | Please do not file reports on Github for security issues. 14 | Please review the guidelines on at (link to more info). 15 | Reports should be encrypted using PGP (link to PGP key) and sent to 16 | security@linkedin.com preferably with the title "Github linkedin/ - ". 17 | 18 | Tips for Getting Your Pull Request Accepted 19 | =========================================== 20 | 21 | 1. Make sure all new features are tested and the tests pass. 22 | 2. Bug fixes must include a test case demonstrating the error that it fixes. 23 | 3. Open an issue first and seek advice for your change before submitting 24 | a pull request. Large features which have never been discussed are 25 | unlikely to be accepted. **You have been warned.** 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 2-CLAUSE LICENSE 2 | 3 | Copyright 2017 LinkedIn Corporation. 4 | All Rights Reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | 1. Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | 2. Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 17 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 20 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 23 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md 2 | include LICENSE NOTICE 3 | recursive-include docs *.rst 4 | recursive-include test *.py 5 | recursive-include test *.sh 6 | recursive-include test *.zip 7 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Copyright [2017] LinkedIn Corporation 2 | All Rights Reserved. 3 | Licensed under the BSD 2-Clause License (the "License"). See License in the project root for license information. 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![pypi](https://img.shields.io/pypi/v/shiv.svg)](https://pypi.python.org/pypi/shiv) 2 | [![ci](https://github.com/linkedin/shiv/workflows/ci/badge.svg)](https://github.com/linkedin/shiv/actions?query=workflow%3Aci) 3 | [![codecov](https://codecov.io/gh/linkedin/shiv/branch/master/graph/badge.svg)](https://codecov.io/gh/linkedin/shiv) 4 | [![docs](https://readthedocs.org/projects/shiv/badge/?version=latest)](http://shiv.readthedocs.io/en/latest/?badge=latest) 5 | [![license](https://img.shields.io/badge/License-BSD%202--Clause-orange.svg)](https://opensource.org/licenses/BSD-2-Clause) 6 | [![supported](https://img.shields.io/pypi/pyversions/shiv.svg)](https://pypi.python.org/pypi/shiv) 7 | 8 | ![snake](https://github.com/linkedin/shiv/raw/main/logo.png) 9 | 10 | # shiv 11 | shiv is a command line utility for building fully self-contained Python zipapps as outlined in [PEP 441](https://www.python.org/dev/peps/pep-0441/), but with all their dependencies included! 12 | 13 | shiv's primary goal is making distributing Python applications fast & easy. 14 | 15 | 📗 Full documentation can be found [here](http://shiv.readthedocs.io/en/latest/). 16 | 17 | ### System Requirements 18 | 19 | - Python >= 3.8 20 | - linux/osx/windows 21 | 22 | ### quickstart 23 | 24 | shiv has a few command line options of its own and accepts almost all options passable to `pip install`. 25 | 26 | ##### simple cli example 27 | 28 | Creating an executable of flake8 with shiv: 29 | 30 | ```sh 31 | $ shiv -c flake8 -o ~/bin/flake8 flake8 32 | $ ~/bin/flake8 --version 33 | 3.7.8 (mccabe: 0.6.1, pycodestyle: 2.5.0, pyflakes: 2.1.1) CPython 3.7.4 on Darwin 34 | ``` 35 | 36 | `-c flake8` specifies the console script that should be invoked when the executable runs, `-o ~/bin/flake8` specifies the location of the generated executable file and `flake8` is the dependency that should be installed from PyPI. 37 | 38 | Creating an interactive executable with the boto library: 39 | 40 | ```sh 41 | $ shiv -o boto.pyz boto 42 | Collecting boto 43 | Installing collected packages: boto 44 | Successfully installed boto-2.49.0 45 | $ ./boto.pyz 46 | Python 3.7.4 (v3.7.4:e09359112e, Jul 8 2019, 14:54:52) 47 | [Clang 6.0 (clang-600.0.57)] on darwin 48 | Type "help", "copyright", "credits" or "license" for more information. 49 | (InteractiveConsole) 50 | >>> import boto 51 | >>> boto.__version__ 52 | '2.49.0' 53 | ``` 54 | 55 | ### installing 56 | 57 | You can install shiv by simply downloading a release from https://github.com/linkedin/shiv/releases or via `pip` / `pypi`: 58 | 59 | ```sh 60 | pip install shiv 61 | ``` 62 | 63 | You can even create a pyz _of_ shiv _using_ shiv! 64 | 65 | ```sh 66 | python3 -m venv . 67 | source bin/activate 68 | pip install shiv 69 | shiv -c shiv -o shiv shiv 70 | ``` 71 | 72 | ### developing 73 | 74 | We'd love contributions! Getting bootstrapped to develop is easy: 75 | 76 | ```sh 77 | git clone git@github.com:linkedin/shiv.git 78 | cd shiv 79 | python3 -m venv venv 80 | source ./venv/bin/activate 81 | python3 -m pip install --upgrade build 82 | python3 -m build 83 | python3 -m pip install -e . 84 | ``` 85 | 86 | Don't forget to run and write tests: 87 | 88 | ```sh 89 | python3 -m pip install tox 90 | tox 91 | ``` 92 | 93 | To build documentation when you changed something in `docs`: 94 | 95 | ```sh 96 | python3 -m pip install -r docs/requirements.txt 97 | sphinx-build docs build/html 98 | ``` 99 | 100 | ### gotchas 101 | 102 | Zipapps created with shiv are not guaranteed to be cross-compatible with other architectures. For example, a `pyz` 103 | file built on a Mac may only work on other Macs, likewise for RHEL, etc. This usually only applies to zipapps that have C extensions in their dependencies. If all your dependencies are pure python, then chances are the `pyz` _will_ work on other platforms. Just something to be aware of. 104 | 105 | Zipapps created with shiv *will* extract themselves into `~/.shiv`, unless overridden via 106 | `SHIV_ROOT`. If you create many utilities with shiv, you may want to occasionally clean this 107 | directory. 108 | 109 | --- 110 | 111 | ### acknowledgements 112 | 113 | Similar projects: 114 | 115 | * [PEX](https://github.com/pantsbuild/pex) 116 | * [pyzzer](https://pypi.org/project/pyzzer/#description) 117 | * [superzippy](https://github.com/brownhead/superzippy) 118 | 119 | Logo by Juliette Carvalho 120 | -------------------------------------------------------------------------------- /docs/api.rst: -------------------------------------------------------------------------------- 1 | Shiv API 2 | ======== 3 | 4 | .. automodule:: shiv 5 | :members: 6 | :show-inheritance: 7 | 8 | cli 9 | --- 10 | 11 | .. automodule:: shiv.cli 12 | :members: 13 | :show-inheritance: 14 | 15 | constants 16 | --- 17 | 18 | .. automodule:: shiv.constants 19 | :members: 20 | :show-inheritance: 21 | 22 | builder 23 | ------- 24 | 25 | .. automodule:: shiv.builder 26 | :members: 27 | :show-inheritance: 28 | 29 | pip 30 | --- 31 | 32 | .. automodule:: shiv.pip 33 | :members: 34 | :show-inheritance: 35 | 36 | bootstrap 37 | --------- 38 | 39 | .. automodule:: shiv.bootstrap 40 | :members: 41 | :show-inheritance: 42 | 43 | bootstrap.environment 44 | --------------------- 45 | 46 | .. automodule:: shiv.bootstrap.environment 47 | :members: 48 | :show-inheritance: 49 | 50 | bootstrap.interpreter 51 | --------------------- 52 | 53 | .. automodule:: shiv.bootstrap.interpreter 54 | :members: 55 | :show-inheritance: 56 | -------------------------------------------------------------------------------- /docs/cli-reference.rst: -------------------------------------------------------------------------------- 1 | ********************** 2 | Complete CLI Reference 3 | ********************** 4 | 5 | This is a full reference of the project's command line tools, 6 | with the same information as you get from using the :option:`-h` option. 7 | It is generated from source code and thus always up to date. 8 | 9 | 10 | Available Commands 11 | ================== 12 | 13 | .. contents:: 14 | :local: 15 | 16 | .. click:: shiv.cli:main 17 | :prog: shiv 18 | :show-nested: 19 | 20 | .. click:: shiv.info:main 21 | :prog: shiv-info 22 | :show-nested: 23 | 24 | 25 | Additional Hints 26 | ================ 27 | 28 | Choosing a Python Interpreter Path 29 | ---------------------------------- 30 | 31 | A good overall interpreter path as passed into :option:`--python` is ``/usr/bin/env python3``. 32 | If you want to make sure your code runs on the Python version you tested it on, 33 | include the minor version (e.g. ``… python3.8``) – use what fits your circumstances best. 34 | 35 | On Windows, the Python launcher ``py`` knows how to handle shebangs using ``env``, 36 | so it's overall the best choice if you target multiple platforms with a pure Python zipapp. 37 | 38 | Also note that you can always fix the shebang during installation of a zipapp using this: 39 | 40 | .. code-block:: shell 41 | 42 | python3 -m zipapp -p '/usr/bin/env python3.7' -o ~/bin/foo foo.pyz 43 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import sys 3 | 4 | from pathlib import Path 5 | 6 | here = Path(__file__).parent 7 | sys.path.insert(0, str(Path(here.parent, 'src').absolute())) 8 | 9 | extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode', 'sphinx_click.ext', 'sphinx.ext.intersphinx'] 10 | source_suffix = '.rst' 11 | master_doc = 'index' 12 | project = u'shiv' 13 | # noinspection PyShadowingBuiltins 14 | copyright = u"{year}, LinkedIn".format(year=datetime.datetime.now().year) 15 | pygments_style = 'sphinx' 16 | html_theme = 'default' 17 | intersphinx_mapping = {'python': ('https://docs.python.org/3', None)} 18 | -------------------------------------------------------------------------------- /docs/django.rst: -------------------------------------------------------------------------------- 1 | Deploying django apps 2 | --------------------- 3 | 4 | Because of how shiv works, you can ship entire django apps with shiv, even including the database if you want! 5 | 6 | Defining an entrypoint 7 | ====================== 8 | 9 | First, we will need an entrypoint. 10 | 11 | We'll call it ``main.py``, and store it at ``//main.py`` (alongside ``wsgi.py``) 12 | 13 | .. code-block:: python 14 | 15 | import os 16 | import sys 17 | 18 | import django 19 | 20 | # setup django 21 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", ".settings") 22 | django.setup() 23 | 24 | try: 25 | production = sys.argv[1] == "production" 26 | except IndexError: 27 | production = False 28 | 29 | if production: 30 | import gunicorn.app.wsgiapp as wsgi 31 | 32 | # This is just a simple way to supply args to gunicorn 33 | sys.argv = [".", ".wsgi", "--bind=0.0.0.0:80"] 34 | 35 | wsgi.run() 36 | else: 37 | from django.core.management import call_command 38 | 39 | call_command("runserver") 40 | 41 | *This is meant as an example. While it's fully production-ready, you might want to tweak it according to your project's needs.* 42 | 43 | 44 | Build script 45 | ============ 46 | 47 | Next, we'll create a simple bash script that will build a zipapp for us. 48 | 49 | Save it as ``build.sh`` (next to manage.py) 50 | 51 | .. code-block:: sh 52 | 53 | #!/usr/bin/env bash 54 | 55 | # clean old build 56 | rm -r dist .pyz 57 | 58 | # include the dependencies from `pip freeze` 59 | pip install -r <(pip freeze) --target dist/ 60 | 61 | # or, if you're using pipenv 62 | # pip install -r <(pipenv lock -r) --target dist/ 63 | 64 | # specify which files to be included in the build 65 | # You probably want to specify what goes here 66 | cp -r \ 67 | -t dist \ 68 | manage.py db.sqlite3 69 | 70 | # finally, build! 71 | shiv --site-packages dist --compressed -p '/usr/bin/env python3' -o .pyz -e .main 72 | 73 | And then, you can just do the following 74 | 75 | .. code-block:: sh 76 | 77 | $ ./build.sh 78 | 79 | $ ./.pyz 80 | 81 | # In production - 82 | 83 | $ ./.pyz production 84 | -------------------------------------------------------------------------------- /docs/history.rst: -------------------------------------------------------------------------------- 1 | Motivation & Comparisons 2 | ======================== 3 | 4 | Why? 5 | ---- 6 | 7 | At LinkedIn we ship hundreds of command line utilities to every machine in our data-centers and all 8 | of our employees workstations. The vast majority of these utilities are written in Python. In 9 | addition to these utilities we also have many internal libraries that are uprev'd daily. 10 | 11 | Because of differences in iteration rate and the inherent problems present when dealing with such a 12 | huge dependency graph, we need to package the executables discretely. Initially we took advantage 13 | of the great open source tool `PEX `_. PEX elegantly solved the 14 | isolated packaging requirement we had by including all of a tool's dependencies inside of a single 15 | binary file that we could then distribute! 16 | 17 | However, as our tools matured and picked up additional dependencies, we became acutely aware of the 18 | performance issues being imposed on us by ``pkg_resources``'s 19 | `Issue 510 `_. Since PEX leans heavily on 20 | ``pkg_resources`` to bootstrap its environment, we found ourselves at an impass: lose out on the 21 | ability to neatly package our tools in favor of invocation speed, or impose a few second 22 | performance penalty for the benefit of easy packaging. 23 | 24 | After spending some time investigating extricating pkg_resources from PEX, we decided to start from 25 | a clean slate and thus ``shiv`` was created. 26 | 27 | How? 28 | ---- 29 | 30 | Shiv exploits the same features of Python as PEX, packing ``__main__.py`` into a zipfile with a 31 | shebang prepended (akin to zipapps, as defined by 32 | `PEP 441 `_), extracting a dependency directory and 33 | injecting said dependencies at runtime. We have to credit the great work by @wickman, @kwlzn, 34 | @jsirois and the other PEX contributors for laying the groundwork! 35 | 36 | The primary differences between PEX and shiv are: 37 | 38 | * ``shiv`` completely avoids the use of ``pkg_resources``. If it is included by a transitive 39 | dependency, the performance implications are mitigated by limiting the length of ``sys.path``. 40 | Internally, at LinkedIn, we always include the 41 | `-s `_ and 42 | `-E `_ Python interpreter flags by 43 | specifying ``--python "/path/to/python -sE"``, which ensures a clean environment. 44 | * Instead of shipping our binary with downloaded wheels inside, we package an entire site-packages 45 | directory, as installed by ``pip``. We then bootstrap that directory post-extraction via the 46 | stdlib's ``site.addsitedir`` function. That way, everything works out of the box: namespace 47 | packages, real filesystem access, etc. 48 | 49 | Because we optimize for a shorter ``sys.path`` and don't include ``pkg_resources`` in the critical 50 | path, executables created with ``shiv`` can outperform ones created with PEX by almost 2x. In most 51 | cases the executables created with ``shiv`` are even faster than running a script from within a 52 | virtualenv! 53 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | shiv 🔪 2 | ==================== 3 | 4 | Shiv is a command line utility for building fully self-contained Python zipapps as outlined in `PEP 441 `_ 5 | but with all their dependencies included! 6 | 7 | Shiv's primary goal is making distributing Python applications fast & easy. 8 | 9 | How it works 10 | ------------ 11 | 12 | Internally, shiv includes two major components: a *builder* and a small *bootstrap* runtime. 13 | 14 | Building 15 | ^^^^^^^^ 16 | 17 | In order to build self-contained, single-artifact executables, shiv leverages ``pip`` to stage your project's dependencies 18 | and then uses the features described in `PEP 441 `_ to create a "zipapp". 19 | 20 | The primary feature of PEP 441 that ``shiv`` uses is Python's ability to implicitly execute a `__main__.py` file inside of a zip archive. 21 | Here's an example of the feature in action: 22 | 23 | .. code-block:: sh 24 | 25 | $ echo "print('hello world')" >> __main__.py 26 | $ zip archive.zip __main__.py 27 | adding: __main__.py (stored 0%) 28 | $ python3 ./archive.zip 29 | hello world 30 | 31 | ``shiv`` expands on this functionality by packing your dependencies into the same zip and adding a specialized `__main__.py` that instructs the Python interpreter to 32 | unpack those dependencies to a known location. Then, at runtime, adds those dependencies to your interpreter's search path, and that's it! 33 | 34 | .. note:: 35 | "Conventional" zipapps don't include any dependencies, which is what sets shiv apart from the stdlib zipapp module. 36 | 37 | ``shiv`` accepts only a few command line parameters of its own, `described here `_, and under the covers, **any unprocessed parameters are 38 | delegated to** ``pip install``. This allows users to fully leverage all the functionality that pip provides. 39 | 40 | For example, if you wanted to create an executable for ``flake8``, you'd specify the required 41 | dependencies (in this case, simply ``flake8``), the callable (either via ``-e`` for a setuptools-style entry 42 | point or ``-c`` for a bare console_script name), and the output file: 43 | 44 | .. code-block:: sh 45 | 46 | $ shiv -c flake8 -o ~/bin/flake8 flake8 47 | 48 | Let's break this command down, 49 | 50 | * ``shiv`` is the command itself. 51 | * ``-c flake8`` specifies the ``console_script`` for flake8 (`defined here `_) 52 | * ``-o ~/bin/flake8`` specifies the ``outfile`` 53 | * ``flake8`` is a dependency (this portion of the command is delegated to ``pip install``) 54 | 55 | This creates an executable (``~/bin/flake8``) containing all the dependencies specified (``flake8``) 56 | that, when invoked, executes the provided console_script (``flake8``)! 57 | 58 | If you were to omit the entry point/console script flag, invoking the resulting executable would drop you into an interpreter that 59 | is bootstrapped with the dependencies you've specified. This can be useful for creating a single-artifact executable 60 | Python environment: 61 | 62 | .. code-block:: sh 63 | 64 | $ shiv httpx -o httpx.pyz --quiet 65 | $ ./httpx.pyz 66 | Python 3.7.7 (default, Mar 10 2020, 16:11:21) 67 | [Clang 11.0.0 (clang-1100.0.33.12)] on darwin 68 | Type "help", "copyright", "credits" or "license" for more information. 69 | (InteractiveConsole) 70 | >>> import httpx 71 | >>> httpx.get("https://shiv.readthedocs.io") 72 | 73 | 74 | This is particularly useful for running scripts without needing to create a virtual environment or contaminate your Python 75 | environment, since the ``pyz`` files can be used as a shebang! 76 | 77 | .. code-block:: sh 78 | 79 | $ cat << EOF > tryme.py 80 | > #!/usr/bin/env httpx.pyz 81 | > 82 | > import httpx 83 | > url = "https://shiv.readthedocs.io" 84 | > response = httpx.get(url) 85 | > print(f"Got {response.status_code} from {url}!") 86 | > 87 | > EOF 88 | $ chmod +x tryme.py 89 | $ ./tryme.py 90 | Got 200 from https://shiv.readthedocs.io! 91 | 92 | Bootstrapping 93 | ^^^^^^^^^^^^^ 94 | 95 | As mentioned above, when you run an executable created with ``shiv``, a special bootstrap function is called. 96 | This function unpacks the dependencies into a uniquely named subdirectory of ``~/.shiv`` and then runs your entry point 97 | (or interactive interpreter) with those dependencies added to your interpreter's search path (``sys.path``). 98 | 99 | To improve performance, once the dependencies have been extracted to disk, any further invocations will re-use the 'cached' 100 | site-packages unless they are deleted or moved. 101 | 102 | .. note:: 103 | 104 | Dependencies are extracted (rather than loaded into memory from the zipapp itself) for two reasons. 105 | 106 | **1.) Because of limitations of third-party and binary dependencies.** 107 | 108 | Just as an example, shared objects loaded via the dlopen syscall require a regular filesystem. 109 | In addition, many libraries also expect a filesystem in order to do things like building paths via ``__file__`` (which doesn't work when a module is imported from a zip), etc. 110 | To learn more, check out this resource about the setuptools `"zip_safe" flag `_. 111 | 112 | **2.) Performance reasons** 113 | 114 | Decompressing files takes time, and if we loaded the dependencies from the zip file every time it would significantly slow down invocation speed. 115 | 116 | Preamble 117 | ^^^^^^^^ 118 | 119 | As an application packager, you may want to run some sanity checks or clean up tasks when users execute a pyz. 120 | For such a use case, ``shiv`` provides a ``--preamble`` flag. 121 | Any executable script passed to that flag will be packed into the zipapp and invoked during bootstrapping (*after* extracting dependencies but *before* invoking an entry point / console script). 122 | 123 | If the preamble file is written in Python (e.g. ends in ``.py``) then shiv will inject three variables into the runtime that may be useful to preamble authors: 124 | 125 | * ``archive``: (a string) path to the current PYZ file 126 | * ``env``: an instance of the `Environment `_ object. 127 | * ``site_packages``: a :py:class:`pathlib.Path` of the directory where the current PYZ's site_packages were extracted to during bootstrap. 128 | 129 | For an example, a preamble file that cleans up prior extracted ``~/.shiv`` directories might look like: 130 | 131 | .. code-block:: py 132 | 133 | #!/usr/bin/env python3 134 | 135 | import shutil 136 | 137 | from pathlib import Path 138 | 139 | # These variables are injected by shiv.bootstrap 140 | site_packages: Path 141 | env: "shiv.bootstrap.environment.Environment" 142 | 143 | # Get a handle of the current PYZ's site_packages directory 144 | current = site_packages.parent 145 | 146 | # The parent directory of the site_packages directory is our shiv cache 147 | cache_path = current.parent 148 | 149 | 150 | name, build_id = current.name.split('_') 151 | 152 | if __name__ == "__main__": 153 | for path in cache_path.iterdir(): 154 | if path.name.startswith(f"{name}_") and not path.name.endswith(build_id): 155 | shutil.rmtree(path) 156 | 157 | Hello World 158 | ^^^^^^^^^^^ 159 | 160 | Here's an example of how to set up a hello-world executable using ``shiv``. 161 | 162 | First, create a new project: 163 | 164 | .. code-block:: sh 165 | 166 | $ mkdir hello-world 167 | $ cd hello-world 168 | 169 | Add some code. 170 | 171 | .. code-block:: python 172 | :caption: hello.py 173 | 174 | def main(): 175 | print("Hello world") 176 | 177 | if __name__ == "__main__": 178 | main() 179 | 180 | Second, create a Python package using your preferred workflow (for this example, I'll simply create a minimal ``setup.py`` file). 181 | 182 | .. code-block:: python 183 | :caption: setup.py 184 | 185 | from setuptools import setup 186 | 187 | setup( 188 | name="hello-world", 189 | version="0.0.1", 190 | description="Greet the world.", 191 | py_modules=["hello"], 192 | entry_points={ 193 | "console_scripts": ["hello=hello:main"], 194 | }, 195 | ) 196 | 197 | That's it! We now have a proper Python package, so we can use ``shiv`` to create a single-artifact executable for it. 198 | 199 | .. code-block:: sh 200 | 201 | $ shiv -c hello -o hello . 202 | 203 | .. note:: 204 | 205 | Notice the ``.`` at the end of the ``shiv`` invocation. That is referring to the local package that we just created. 206 | You can think of it as analogous to running ``pip install .`` 207 | 208 | That's it! Our example should now execute as expected. 209 | 210 | .. code-block:: sh 211 | 212 | $ ./hello 213 | Hello world 214 | 215 | Influencing Runtime 216 | ------------------- 217 | 218 | Whenever you are creating a zipapp with ``shiv``, you can specify a few flags that influence the runtime. 219 | For example, the ``-c/--console-script`` and ``-e/--entry-point`` options already mentioned in this doc. 220 | To see the full list of command line options, see this page. 221 | 222 | In addition to options that are settable during zipapp creation, there are a number of environment variables 223 | you can specify to influence a zipapp created with shiv at run time. 224 | 225 | SHIV_ROOT 226 | ^^^^^^^^^ 227 | 228 | This should be populated with a full path, it overrides ``~/.shiv`` as the default base dir for shiv's extraction cache. 229 | 230 | This is useful if you want to collect the contents of a zipapp to inspect them, or if you want to make a quick edit to 231 | a source file, but don't want to taint the extraction cache. 232 | 233 | SHIV_INTERPRETER 234 | ^^^^^^^^^^^^^^^^ 235 | 236 | This is a boolean that bypasses and console_script or entry point baked into your pyz. Useful for 237 | dropping into an interactive session in the environment of a built cli utility. 238 | 239 | SHIV_ENTRY_POINT / SHIV_MODULE 240 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 241 | 242 | .. note:: Same functionality as ``-e/--entry-point`` at build time 243 | 244 | This should be populated with a setuptools-style callable, e.g. "module.main:main". This will 245 | execute the pyz with whatever callable entry point you supply. Useful for sharing a single pyz 246 | across many callable 'scripts'. 247 | 248 | SHIV_CONSOLE_SCRIPT 249 | ^^^^^^^^^^^^^^^^^^^ 250 | 251 | .. note:: Same functionality as ``-c/--console-script` at build time 252 | 253 | Similar to the SHIV_ENTRY_POINT and SHIV_MODULE environment variables, SHIV_CONSOLE_SCRIPT overrides any value 254 | provided at build time. 255 | 256 | SHIV_FORCE_EXTRACT 257 | ^^^^^^^^^^^^^^^^^^ 258 | 259 | This forces re-extraction of dependencies even if they've already been extracted. If you make 260 | hotfixes/modifications to the 'cached' dependencies, this will overwrite them. 261 | 262 | SHIV_EXTEND_PYTHONPATH 263 | ^^^^^^^^^^^^^^^^^^^^^^ 264 | 265 | .. note:: Same functionality as ``-E/--extend-pythonpath`` at build time. 266 | 267 | This is a boolean that adds the modules bundled into the zipapp into the ``PYTHONPATH`` environment 268 | variable. It is not needed for most applications, but if an application calls Python as a 269 | subprocess, expecting to be able to import the modules bundled in the zipapp, this will allow it 270 | to do so successfully. 271 | 272 | 273 | SHIV_PREPEND_PYTHONPATH 274 | ^^^^^^^^^^^^^^^^^^^^^^^ 275 | 276 | The value of this environment variable will be prepended to ``sys.path`` during the bootstrap process 277 | before running the application. This is useful to load code from additional locations that are outside 278 | the shiv-created file, for example for debugging purposes. This variable takes precedence over 279 | ``PYTHONPATH``. 280 | 281 | Reproducibility 282 | ^^^^^^^^^^^^^^^ 283 | 284 | ``shiv`` supports the ability to create reproducible artifacts. By using the ``--reproducible`` command line option or 285 | by setting the ``SOURCE_DATE_EPOCH`` environment variable during zipapp creation. When this option is selected, if the 286 | inputs do not change, the output should be idempotent. 287 | 288 | For more information, see https://reproducible-builds.org/. 289 | 290 | Table of Contents 291 | ================= 292 | 293 | .. toctree:: 294 | :maxdepth: 2 295 | 296 | cli-reference 297 | history 298 | api 299 | django 300 | 301 | Indices and tables 302 | ================== 303 | 304 | * :ref:`genindex` 305 | * :ref:`modindex` 306 | * :ref:`search` 307 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | # Requirements to build docs on RTD 2 | sphinx-click 3 | -------------------------------------------------------------------------------- /logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linkedin/shiv/a353d10ecd785c6bd67ccd08b642437e7204be57/logo.png -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools", "wheel"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [black] 6 | line_length = 120 7 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 120 3 | max-complexity = 15 4 | exclude = docs test/package/setup.py 5 | extend-ignore = C901 6 | 7 | [tool:pytest] 8 | addopts = --ignore build/ --ignore dist/ test/ 9 | 10 | [mypy] 11 | mypy_path = src/ 12 | strict_optional = yes 13 | disallow_untyped_defs = no 14 | 15 | [isort] 16 | line_length = 120 17 | indent=' ' 18 | multi_line_output=3 19 | lines_between_types = 1 20 | include_trailing_comma = true 21 | use_parentheses = true 22 | not_skip = __init__.py 23 | sections = FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER 24 | 25 | [metadata] 26 | name = shiv 27 | version = attr: shiv.__version__ 28 | description = A command line utility for building fully self contained Python zipapps. 29 | long_description = file: README.md 30 | long_description_content_type = text/markdown 31 | url = https://github.com/linkedin/shiv 32 | author = Loren Carvalho 33 | author_email = loren@linkedin.com 34 | license = BSD License 35 | license_file = LICENSE 36 | classifiers = 37 | License :: OSI Approved :: BSD License 38 | Programming Language :: Python :: 3 :: Only 39 | Programming Language :: Python :: 3 40 | Programming Language :: Python :: 3.8 41 | Programming Language :: Python :: 3.9 42 | Programming Language :: Python :: 3.10 43 | Programming Language :: Python :: 3.11 44 | 45 | [options] 46 | package_dir = 47 | =src 48 | packages = find: 49 | install_requires = 50 | click>=6.7,!=7.0 51 | pip>=9.0.3 52 | setuptools 53 | python_requires = >=3.8 54 | include_package_data = True 55 | 56 | [options.extras_require] 57 | rtd = 58 | sphinx-click 59 | 60 | [options.packages.find] 61 | where=src 62 | 63 | [options.entry_points] 64 | console_scripts = 65 | shiv = shiv.cli:main 66 | shiv-info = shiv.info:main 67 | 68 | [bdist_wheel] 69 | universal = True 70 | -------------------------------------------------------------------------------- /src/shiv/__init__.py: -------------------------------------------------------------------------------- 1 | from .__version__ import __version__ # noqa 2 | -------------------------------------------------------------------------------- /src/shiv/__main__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Shim for package execution (python3 -m shiv ...). 3 | """ 4 | from .cli import main 5 | 6 | if __name__ == "__main__": # pragma: no cover 7 | main() 8 | -------------------------------------------------------------------------------- /src/shiv/__version__.py: -------------------------------------------------------------------------------- 1 | __version__ = "1.1.0" 2 | -------------------------------------------------------------------------------- /src/shiv/bootstrap/__init__.py: -------------------------------------------------------------------------------- 1 | import compileall 2 | import hashlib 3 | import os 4 | import runpy 5 | import shutil 6 | import site 7 | import subprocess 8 | import sys 9 | import zipfile 10 | 11 | from contextlib import contextmanager, suppress 12 | from functools import partial 13 | from importlib import import_module 14 | from pathlib import Path 15 | 16 | from .environment import Environment 17 | from .filelock import FileLock 18 | from .interpreter import execute_interpreter 19 | 20 | 21 | def run(module): # pragma: no cover 22 | """Run a module in a scrubbed environment. 23 | 24 | If a single pyz has multiple callers, we want to remove these vars as we no longer need them 25 | and they can cause subprocesses to fail with a ModuleNotFoundError. 26 | 27 | :param Callable module: The entry point to invoke the pyz with. 28 | """ 29 | with suppress(KeyError): 30 | del os.environ[Environment.MODULE] 31 | 32 | with suppress(KeyError): 33 | del os.environ[Environment.ENTRY_POINT] 34 | 35 | with suppress(KeyError): 36 | del os.environ[Environment.CONSOLE_SCRIPT] 37 | 38 | sys.exit(module()) 39 | 40 | 41 | @contextmanager 42 | def current_zipfile(): 43 | """A function to vend the current zipfile, if any""" 44 | if zipfile.is_zipfile(sys.argv[0]): 45 | with zipfile.ZipFile(sys.argv[0]) as fd: 46 | yield fd 47 | else: 48 | yield None 49 | 50 | 51 | def import_string(import_name): 52 | """Returns a callable for a given setuptools style import string 53 | 54 | :param str import_name: A console_scripts style import string 55 | """ 56 | import_name = str(import_name).replace(":", ".") 57 | 58 | try: 59 | import_module(import_name) 60 | 61 | except ImportError: 62 | if "." not in import_name: 63 | # this is a case like "import name", where continuing to the 64 | # next style of import would not improve the situation, so 65 | # we raise here. 66 | raise 67 | 68 | else: 69 | return sys.modules[import_name] 70 | 71 | # this is a case where the previous attempt may have failed due to 72 | # not being importable. ("not a package", etc) 73 | module_name, obj_name = import_name.rsplit(".", 1) 74 | 75 | try: 76 | module = __import__(module_name, None, None, [obj_name]) 77 | 78 | except ImportError: 79 | # Recurse to support importing modules not yet set up by the parent module 80 | # (or package for that matter) 81 | module = import_string(module_name) 82 | 83 | try: 84 | return getattr(module, obj_name) 85 | 86 | except AttributeError as e: 87 | raise ImportError(e) 88 | 89 | 90 | def cache_path(archive, root_dir, build_id): 91 | """Returns a ~/.shiv cache directory for unzipping site-packages during bootstrap. 92 | 93 | :param ZipFile archive: The zipfile object we are bootstrapping from. 94 | :param str root_dir: Optional, either a path or environment variable pointing to a SHIV_ROOT. 95 | :param str build_id: The build id generated at zip creation. 96 | """ 97 | 98 | if root_dir: 99 | 100 | if root_dir.startswith("$"): 101 | root_dir = os.environ.get(root_dir[1:], root_dir[1:]) 102 | 103 | root_dir = Path(root_dir).expanduser() 104 | 105 | root = root_dir or Path("~/.shiv").expanduser() 106 | name = Path(archive.filename).resolve().name 107 | return root / f"{name}_{build_id}" 108 | 109 | 110 | def extract_site_packages(archive, target_path, compile_pyc=False, compile_workers=0, force=False): 111 | """Extract everything in site-packages to a specified path. 112 | 113 | :param ZipFile archive: The zipfile object we are bootstrapping from. 114 | :param Path target_path: The path to extract our zip to. 115 | :param bool compile_pyc: A boolean to dictate whether we pre-compile pyc. 116 | :param int compile_workers: An int representing the number of pyc compiler workers. 117 | :param bool force: A boolean to dictate whether or not we force extraction. 118 | """ 119 | parent = target_path.parent 120 | target_path_tmp = Path(parent, target_path.name + ".tmp") 121 | lock = Path(parent, f".{target_path.name}_lock") 122 | 123 | # If this is the first time that a pyz is being extracted, we'll need to create the ~/.shiv dir 124 | if not parent.exists(): 125 | parent.mkdir(parents=True, exist_ok=True) 126 | 127 | with FileLock(lock): 128 | 129 | # we acquired a lock, it's possible that prior invocation was holding the lock and has 130 | # completed bootstrapping, so let's check (again) if we need to do any work 131 | if not target_path.exists() or force: 132 | 133 | # extract our site-packages 134 | for fileinfo in archive.infolist(): 135 | 136 | if fileinfo.filename.startswith("site-packages"): 137 | extracted = archive.extract(fileinfo.filename, target_path_tmp) 138 | 139 | # restore original permissions 140 | os.chmod(extracted, fileinfo.external_attr >> 16) 141 | 142 | if compile_pyc: 143 | compileall.compile_dir(target_path_tmp, quiet=2, workers=compile_workers) 144 | 145 | # if using `force` we will need to delete our target path 146 | if target_path.exists(): 147 | shutil.rmtree(str(target_path)) 148 | 149 | # atomic move 150 | shutil.move(str(target_path_tmp), str(target_path)) 151 | 152 | 153 | def get_first_sitedir_index(): 154 | for index, part in enumerate(sys.path): 155 | if Path(part).stem in ("site-packages", "dist-packages"): 156 | return index 157 | 158 | 159 | def extend_python_path(environ, additional_paths): 160 | """Create or extend a PYTHONPATH variable with the frozen environment we are bootstrapping with.""" 161 | 162 | # we don't want to clobber any existing PYTHONPATH value, so check for it. 163 | python_path = environ["PYTHONPATH"].split(os.pathsep) if "PYTHONPATH" in environ else [] 164 | python_path.extend(additional_paths) 165 | 166 | # put it back into the environment so that PYTHONPATH contains the shiv-manipulated paths 167 | # and any pre-existing PYTHONPATH values with no duplicates. 168 | environ["PYTHONPATH"] = os.pathsep.join(sorted(set(python_path), key=python_path.index)) 169 | 170 | 171 | def ensure_no_modify(site_packages, hashes): 172 | """Compare the sha256 hash of the unpacked source files to the files when they were added to the pyz.""" 173 | 174 | for path in site_packages.rglob("**/*.py"): 175 | 176 | if hashlib.sha256(path.read_bytes()).hexdigest() != hashes.get(str(path.relative_to(site_packages))): 177 | raise RuntimeError( 178 | "A Python source file has been modified! File: {}. " 179 | "Try again with SHIV_FORCE_EXTRACT=1 to overwrite the modified source file(s).".format(str(path)) 180 | ) 181 | 182 | 183 | def prepend_pythonpath(env): 184 | """Prepend the sys.path with the value of SHIV_PREPEND_PYTHONPATH, if set.""" 185 | if env.prepend_pythonpath: 186 | sys.path.insert(0, env.prepend_pythonpath) 187 | 188 | 189 | def bootstrap(): # pragma: no cover 190 | """Actually bootstrap our shiv environment.""" 191 | 192 | # get a handle of the currently executing zip file 193 | with current_zipfile() as archive: 194 | 195 | # create an environment object (a combination of env vars and json metadata) 196 | env = Environment.from_json(archive.read("environment.json").decode()) 197 | 198 | # get a site-packages directory (from env var or via build id) 199 | site_packages = cache_path(archive, env.root, env.build_id) / "site-packages" 200 | 201 | # determine if first run or forcing extract 202 | if not site_packages.exists() or env.force_extract: 203 | extract_site_packages( 204 | archive, 205 | site_packages.parent, 206 | env.compile_pyc, 207 | env.compile_workers, 208 | env.force_extract, 209 | ) 210 | 211 | # get sys.path's length 212 | length = len(sys.path) 213 | 214 | # Find the first instance of an existing site-packages on sys.path 215 | index = get_first_sitedir_index() or length 216 | 217 | # copy sys.path to determine diff 218 | sys_path_before = sys.path.copy() 219 | 220 | # append site-packages using the stdlib blessed way of extending path 221 | # so as to handle .pth files correctly 222 | site.addsitedir(site_packages) 223 | 224 | # reorder to place our site-packages before any others found 225 | sys.path = sys.path[:index] + sys.path[length:] + sys.path[index:length] 226 | 227 | # Prepend the sys.path if environment variable is set 228 | prepend_pythonpath(env) 229 | 230 | # determine newly added paths 231 | new_paths = [p for p in sys.path if p not in sys_path_before] 232 | 233 | # check if source files have been modified, if required 234 | if env.no_modify: 235 | ensure_no_modify(site_packages, env.hashes) 236 | 237 | # add any new paths to the environment, if requested 238 | if env.extend_pythonpath: 239 | extend_python_path(os.environ, new_paths) 240 | 241 | # if a preamble script was provided, run it 242 | if env.preamble: 243 | 244 | # path to the preamble 245 | preamble_bin = site_packages / "bin" / env.preamble 246 | 247 | if preamble_bin.suffix == ".py": 248 | runpy.run_path( 249 | str(preamble_bin), 250 | init_globals={"archive": sys.argv[0], "env": env, "site_packages": site_packages}, 251 | run_name="__main__", 252 | ) 253 | 254 | else: 255 | subprocess.run([preamble_bin]) 256 | 257 | # first check if we should drop into interactive mode 258 | if not env.interpreter: 259 | 260 | # do entry point import and call 261 | if env.entry_point is not None and not env.script: 262 | run(import_string(env.entry_point)) 263 | 264 | elif env.script is not None: 265 | run(partial(runpy.run_path, str(site_packages / "bin" / env.script), run_name="__main__")) 266 | 267 | # all other options exhausted, drop into interactive mode 268 | execute_interpreter() 269 | 270 | 271 | if __name__ == "__main__": 272 | bootstrap() 273 | -------------------------------------------------------------------------------- /src/shiv/bootstrap/environment.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module contains the ``Environment`` object, which combines settings decided at build time with 3 | overrides defined at runtime (via environment variables). 4 | """ 5 | import json 6 | import os 7 | from typing import Any, Dict, Optional 8 | 9 | 10 | def str_bool(v) -> bool: 11 | if not isinstance(v, bool): 12 | return str(v).lower() in ("yes", "true", "t", "1") 13 | return v 14 | 15 | 16 | class Environment: 17 | INTERPRETER: str = "SHIV_INTERPRETER" 18 | ENTRY_POINT: str = "SHIV_ENTRY_POINT" 19 | CONSOLE_SCRIPT: str = "SHIV_CONSOLE_SCRIPT" 20 | MODULE: str = "SHIV_MODULE" 21 | ROOT: str = "SHIV_ROOT" 22 | FORCE_EXTRACT: str = "SHIV_FORCE_EXTRACT" 23 | COMPILE_PYC: str = "SHIV_COMPILE_PYC" 24 | COMPILE_WORKERS: str = "SHIV_COMPILE_WORKERS" 25 | EXTEND_PYTHONPATH: str = "SHIV_EXTEND_PYTHONPATH" 26 | PREPEND_PYTHONPATH: str = "SHIV_PREPEND_PYTHONPATH" 27 | 28 | def __init__( 29 | self, 30 | built_at: str, 31 | shiv_version: str, 32 | always_write_cache: bool = False, 33 | build_id: Optional[str] = None, 34 | compile_pyc: bool = True, 35 | entry_point: Optional[str] = None, 36 | extend_pythonpath: bool = False, 37 | prepend_pythonpath: Optional[str] = None, 38 | hashes: Optional[Dict[str, Any]] = None, 39 | no_modify: bool = False, 40 | reproducible: bool = False, 41 | script: Optional[str] = None, 42 | preamble: Optional[str] = None, 43 | root: Optional[str] = None, 44 | ) -> None: 45 | self.shiv_version: str = shiv_version 46 | self.always_write_cache: bool = always_write_cache 47 | self.build_id: Optional[str] = build_id 48 | self.built_at: str = built_at 49 | self.hashes: Optional[Dict[str, Any]] = hashes or {} 50 | self.no_modify: bool = no_modify 51 | self.reproducible: bool = reproducible 52 | self.preamble: Optional[str] = preamble 53 | 54 | # properties 55 | self._entry_point: Optional[str] = entry_point 56 | self._compile_pyc: bool = compile_pyc 57 | self._extend_pythonpath: bool = extend_pythonpath 58 | self._prepend_pythonpath: Optional[str] = prepend_pythonpath 59 | self._root: Optional[str] = root 60 | self._script: Optional[str] = script 61 | 62 | @classmethod 63 | def from_json(cls, json_data) -> "Environment": 64 | return Environment(**json.loads(json_data)) 65 | 66 | def to_json(self) -> str: 67 | return json.dumps( 68 | # we strip the leading underscores to retain properties (such as _entry_point) 69 | {key.lstrip("_"): value for key, value in self.__dict__.items()} 70 | ) 71 | 72 | @property 73 | def entry_point(self) -> Optional[str]: 74 | return os.environ.get(self.ENTRY_POINT, os.environ.get(self.MODULE, self._entry_point)) 75 | 76 | @property 77 | def script(self) -> Optional[str]: 78 | return os.environ.get(self.CONSOLE_SCRIPT, self._script) 79 | 80 | @property 81 | def interpreter(self) -> Optional[str]: 82 | return os.environ.get(self.INTERPRETER, None) 83 | 84 | @property 85 | def root(self) -> Optional[str]: 86 | root = os.environ.get(self.ROOT, self._root) 87 | return root 88 | 89 | @property 90 | def force_extract(self) -> bool: 91 | return str_bool(os.environ.get(self.FORCE_EXTRACT, self.always_write_cache)) 92 | 93 | @property 94 | def compile_pyc(self) -> bool: 95 | return str_bool(os.environ.get(self.COMPILE_PYC, self._compile_pyc)) 96 | 97 | @property 98 | def extend_pythonpath(self) -> Optional[bool]: 99 | return str_bool(os.environ.get(self.EXTEND_PYTHONPATH, self._extend_pythonpath)) 100 | 101 | @property 102 | def prepend_pythonpath(self) -> Optional[str]: 103 | """Prepend the given path to sys.path.""" 104 | return os.environ.get(self.PREPEND_PYTHONPATH, self._prepend_pythonpath) 105 | 106 | @property 107 | def compile_workers(self) -> int: 108 | try: 109 | return int(os.environ.get(self.COMPILE_WORKERS, 0)) 110 | except ValueError: 111 | return 0 112 | -------------------------------------------------------------------------------- /src/shiv/bootstrap/filelock.py: -------------------------------------------------------------------------------- 1 | """A simple low-feature cross-platform file lock implementation. 2 | 3 | Code is based on github.com/benediktschmitt/py-filelock 4 | """ 5 | 6 | import os 7 | import time 8 | 9 | try: 10 | import msvcrt # type: ignore 11 | except ImportError: 12 | msvcrt = None # type: ignore 13 | 14 | try: 15 | import fcntl # type: ignore 16 | except ImportError: 17 | fcntl = None # type: ignore 18 | 19 | OPEN_MODE = os.O_RDWR | os.O_CREAT | os.O_TRUNC 20 | 21 | 22 | def acquire_win(lock_file): # pragma: no cover 23 | """Acquire a lock file on windows.""" 24 | try: 25 | fd = os.open(lock_file, OPEN_MODE) 26 | except OSError: 27 | pass 28 | else: 29 | try: 30 | msvcrt.locking(fd, msvcrt.LK_NBLCK, 1) 31 | except (IOError, OSError): 32 | os.close(fd) 33 | else: 34 | return fd 35 | 36 | 37 | def acquire_nix(lock_file): # pragma: no cover 38 | """Acquire a lock file on linux or osx.""" 39 | fd = os.open(lock_file, OPEN_MODE) 40 | 41 | try: 42 | fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) 43 | except (IOError, OSError): 44 | os.close(fd) 45 | else: 46 | return fd 47 | 48 | 49 | class FileLock: 50 | """A rudimentary file lock class.""" 51 | 52 | def __init__(self, lock_file): 53 | # The path to the lock file. 54 | self.lock_file = lock_file 55 | 56 | # The file descriptor for the lock file 57 | self.lock_file_fd = None 58 | 59 | @property 60 | def is_locked(self): 61 | """This property signals if we are holding the lock.""" 62 | return self.lock_file_fd is not None 63 | 64 | def __enter__(self, poll_intervall=0.01): 65 | 66 | while not self.is_locked: 67 | 68 | if msvcrt: 69 | self.lock_file_fd = acquire_win(self.lock_file) 70 | elif fcntl: 71 | self.lock_file_fd = acquire_nix(self.lock_file) 72 | 73 | time.sleep(poll_intervall) 74 | 75 | return self 76 | 77 | def __exit__(self, exc_type, exc_value, traceback): 78 | 79 | if self.is_locked: 80 | 81 | fd = self.lock_file_fd 82 | self.lock_file_fd = None 83 | 84 | if msvcrt: 85 | msvcrt.locking(fd, msvcrt.LK_UNLCK, 1) 86 | elif fcntl: 87 | fcntl.flock(fd, fcntl.LOCK_UN) 88 | 89 | os.close(fd) 90 | -------------------------------------------------------------------------------- /src/shiv/bootstrap/interpreter.py: -------------------------------------------------------------------------------- 1 | """ 2 | The code in this module is adapted from https://github.com/pantsbuild/pex/blob/master/pex/pex.py 3 | 4 | It is used to enter an interactive interpreter session from an executable created with ``shiv``. 5 | """ 6 | import code 7 | import runpy 8 | import sys 9 | 10 | from pathlib import Path 11 | 12 | 13 | def _exec_function(ast, globals_map): 14 | locals_map = globals_map 15 | exec(ast, globals_map, locals_map) 16 | return locals_map 17 | 18 | 19 | def execute_content(name, content, argv0=None): 20 | argv0 = argv0 or name 21 | 22 | try: 23 | ast = compile(content, name, "exec", flags=0, dont_inherit=1) 24 | except SyntaxError: 25 | raise RuntimeError(f"Unable to parse {name}. Is it a Python script? Syntax correct?") 26 | 27 | sys.argv[0] = argv0 28 | globals_ = globals().copy() 29 | globals_["__name__"] = "__main__" 30 | globals_["__file__"] = name 31 | _exec_function(ast, globals_) 32 | 33 | 34 | def execute_module(module_name): 35 | runpy.run_module(module_name, run_name="__main__") 36 | 37 | 38 | def execute_interpreter(): 39 | args = sys.argv[1:] 40 | 41 | if args: 42 | 43 | arg = args[0] 44 | 45 | if arg == "-c": 46 | content = args[1] 47 | sys.argv = [arg, *args[2:]] 48 | execute_content("-c ", content, argv0=arg) 49 | 50 | elif arg == "-m": 51 | module = args[1] 52 | sys.argv = args[1:] 53 | execute_module(module) 54 | 55 | else: 56 | if arg == "-": 57 | content = sys.stdin.read() 58 | 59 | else: 60 | try: 61 | content = Path(arg).read_text() 62 | except (FileNotFoundError, IsADirectoryError, PermissionError) as e: 63 | raise RuntimeError(f"Could not open '{arg}' in the environment [{sys.argv[0]}]: {e}") 64 | 65 | sys.argv = args 66 | execute_content(arg, content) 67 | 68 | else: 69 | code.interact() 70 | -------------------------------------------------------------------------------- /src/shiv/builder.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module is a modified implementation of Python's "zipapp" module. 3 | 4 | We've copied a lot of zipapp's code here in order to backport support for compression. 5 | https://docs.python.org/3.7/library/zipapp.html#cmdoption-zipapp-c 6 | """ 7 | import hashlib 8 | import os 9 | import sys 10 | import time 11 | import zipapp 12 | import zipfile 13 | 14 | from datetime import datetime, timezone 15 | from itertools import chain 16 | from pathlib import Path 17 | from stat import S_IFMT, S_IMODE, S_IXGRP, S_IXOTH, S_IXUSR 18 | from types import ModuleType 19 | from typing import Any, Generator, IO, Iterator, List, Optional, Tuple, Union 20 | 21 | from . import bootstrap 22 | from .bootstrap.environment import Environment 23 | from .constants import BINPRM_ERROR, BUILD_AT_TIMESTAMP_FORMAT 24 | 25 | try: 26 | import importlib.resources as importlib_resources # type: ignore 27 | except ImportError: 28 | # noinspection PyUnresolvedReferences 29 | import importlib_resources # type: ignore 30 | 31 | # N.B.: `importlib.resources.{contents,is_resource,path}` are deprecated in 3.11 and gone in 3.13. 32 | if sys.version_info < (3, 11): 33 | def iter_package_files(package: Union[str, ModuleType]) -> Iterator[Tuple[Path, str]]: 34 | for bootstrap_file in importlib_resources.contents(bootstrap): 35 | if importlib_resources.is_resource(bootstrap, bootstrap_file): 36 | with importlib_resources.path(bootstrap, bootstrap_file) as path: 37 | yield (path, bootstrap_file) 38 | else: 39 | def iter_package_files(package: Union[str, ModuleType]) -> Iterator[Tuple[Path, str]]: 40 | for resource in importlib_resources.files(package).iterdir(): 41 | if resource.is_file(): 42 | with importlib_resources.as_file(resource) as path: 43 | yield (path, resource.name) 44 | 45 | # Typical maximum length for a shebang line 46 | BINPRM_BUF_SIZE = 128 47 | 48 | # zipapp __main__.py template 49 | MAIN_TEMPLATE = """\ 50 | # -*- coding: utf-8 -*- 51 | import {module} 52 | {module}.{fn}() 53 | """ 54 | 55 | 56 | def write_file_prefix(f: IO[Any], interpreter: str) -> None: 57 | """Write a shebang line. 58 | 59 | :param f: An open file handle. 60 | :param interpreter: A path to a python interpreter. 61 | """ 62 | # if the provided path is too long for a shebang we should error out 63 | if len(interpreter) > BINPRM_BUF_SIZE: 64 | sys.exit(BINPRM_ERROR) 65 | 66 | f.write(b"#!" + interpreter.encode(sys.getfilesystemencoding()) + b"\n") 67 | 68 | 69 | def write_to_zipapp( 70 | archive: zipfile.ZipFile, 71 | arcname: str, 72 | data: bytes, 73 | date_time: Tuple[int, int, int, int, int, int], 74 | compression: int, 75 | stat: Optional[os.stat_result] = None, 76 | ) -> None: 77 | """Write a file or a bytestring to a ZipFile as a separate entry and update contents_hash as a side effect.""" 78 | 79 | zinfo = zipfile.ZipInfo(arcname, date_time=date_time) 80 | zinfo.compress_type = compression 81 | 82 | if stat: 83 | zinfo.external_attr = (S_IMODE(stat.st_mode) | S_IFMT(stat.st_mode)) << 16 84 | 85 | archive.writestr(zinfo, data) 86 | 87 | 88 | def rglob_follow_symlinks(path: Path, glob: str) -> Generator[Path, None, None]: 89 | """Path.rglob extended to follow symlinks, while we wait for Python 3.13.""" 90 | for p in path.rglob('*'): 91 | if p.is_symlink() and p.is_dir(): 92 | yield from chain([p], rglob_follow_symlinks(p, glob)) 93 | else: 94 | yield p 95 | 96 | 97 | def create_archive( 98 | sources: List[Path], target: Path, interpreter: str, main: str, env: Environment, compressed: bool = True 99 | ) -> None: 100 | """Create an application archive from SOURCE. 101 | 102 | This function is a heavily modified version of stdlib's 103 | `zipapp.create_archive `_ 104 | 105 | """ 106 | 107 | # Check that main has the right format. 108 | mod, sep, fn = main.partition(":") 109 | mod_ok = all(part.isidentifier() for part in mod.split(".")) 110 | fn_ok = all(part.isidentifier() for part in fn.split(".")) 111 | if not (sep == ":" and mod_ok and fn_ok): 112 | raise zipapp.ZipAppError("Invalid entry point: " + main) 113 | 114 | # Collect our timestamp data 115 | main_py = MAIN_TEMPLATE.format(module=mod, fn=fn) 116 | timestamp = datetime.strptime(env.built_at, BUILD_AT_TIMESTAMP_FORMAT).replace(tzinfo=timezone.utc).timestamp() 117 | zipinfo_datetime: Tuple[int, int, int, int, int, int] = time.gmtime(int(timestamp))[0:6] 118 | 119 | with target.open(mode="wb") as fd: 120 | 121 | # Write shebang. 122 | write_file_prefix(fd, interpreter) 123 | 124 | # Determine compression. 125 | compression = zipfile.ZIP_DEFLATED if compressed else zipfile.ZIP_STORED 126 | 127 | # Pack zipapp with dependencies. 128 | with zipfile.ZipFile(fd, "w", compression=compression) as archive: 129 | 130 | site_packages = Path("site-packages") 131 | contents_hash = hashlib.sha256() 132 | 133 | for source in sources: 134 | 135 | # Glob is known to return results in non-deterministic order. 136 | # We need to sort them by in-archive paths to ensure 137 | # that archive contents are reproducible. 138 | # 139 | # NOTE: https://github.com/linkedin/shiv/issues/236 140 | # this special rglob function can be replaced with "rglob('*', follow_symlinks=True)" 141 | # when Python 3.13 becomes the lowest supported version 142 | for path in sorted(rglob_follow_symlinks(source, "*"), key=str): 143 | 144 | # Skip compiled files and directories (as they are not required to be present in the zip). 145 | if path.suffix == ".pyc" or path.is_dir(): 146 | continue 147 | 148 | data = path.read_bytes() 149 | 150 | # update the contents hash 151 | contents_hash.update(data) 152 | # take filenames into account as well - build_id should change if a file is moved or renamed 153 | contents_hash.update(str(path.relative_to(source)).encode()) 154 | 155 | arcname = str(site_packages / path.relative_to(source)) 156 | 157 | write_to_zipapp(archive, arcname, data, zipinfo_datetime, compression, stat=path.stat()) 158 | 159 | if env.build_id is None: 160 | # Now that we have a hash of all the source files, use it as our build id if the user did not 161 | # specify a custom one. 162 | env.build_id = contents_hash.hexdigest() 163 | 164 | # now let's add the shiv bootstrap code. 165 | bootstrap_target = Path("_bootstrap") 166 | 167 | for path, name in iter_package_files(bootstrap): 168 | data = path.read_bytes() 169 | 170 | write_to_zipapp( 171 | archive, 172 | str(bootstrap_target / name), 173 | data, 174 | zipinfo_datetime, 175 | compression, 176 | stat=path.stat(), 177 | ) 178 | 179 | # Write environment info in json file. 180 | # 181 | # The environment file contains build_id which is a SHA-256 checksum of all **site-packages** contents. 182 | # the bootstrap code, environment.json and __main__.py are not used to calculate the checksum, is it's 183 | # only used for local caching of site-packages and these files are always read from archive. 184 | write_to_zipapp(archive, "environment.json", env.to_json().encode("utf-8"), zipinfo_datetime, compression) 185 | 186 | # write __main__ 187 | write_to_zipapp(archive, "__main__.py", main_py.encode("utf-8"), zipinfo_datetime, compression) 188 | 189 | # Make pyz executable (on windows this is no-op). 190 | target.chmod(target.stat().st_mode | S_IXUSR | S_IXGRP | S_IXOTH) 191 | -------------------------------------------------------------------------------- /src/shiv/cli.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | import os 3 | import shutil 4 | import sys 5 | import time 6 | 7 | from configparser import ConfigParser 8 | from datetime import datetime 9 | from pathlib import Path 10 | from tempfile import TemporaryDirectory 11 | from typing import List, Optional 12 | 13 | import click 14 | 15 | from . import __version__ 16 | from . import builder, pip 17 | from .bootstrap.environment import Environment 18 | from .constants import ( 19 | BUILD_AT_TIMESTAMP_FORMAT, 20 | DEFAULT_SHEBANG, 21 | DISALLOWED_ARGS, 22 | DISALLOWED_PIP_ARGS, 23 | NO_ENTRY_POINT, 24 | NO_OUTFILE, 25 | NO_PIP_ARGS_OR_SITE_PACKAGES, 26 | SOURCE_DATE_EPOCH_DEFAULT, 27 | SOURCE_DATE_EPOCH_ENV, 28 | ) 29 | 30 | 31 | def find_entry_point(site_packages_dirs: List[Path], console_script: str) -> str: 32 | """Find a console_script in a site-packages directory. 33 | 34 | Console script metadata is stored in entry_points.txt per setuptools 35 | convention. This function searches all entry_points.txt files and 36 | returns the import string for a given console_script argument. 37 | 38 | :param site_packages_dirs: Paths to site-packages directories on disk. 39 | :param console_script: A console_script string. 40 | """ 41 | 42 | config_parser = ConfigParser() 43 | 44 | for site_packages in site_packages_dirs: 45 | # noinspection PyTypeChecker 46 | config_parser.read(site_packages.rglob("entry_points.txt")) 47 | 48 | return config_parser["console_scripts"][console_script] 49 | 50 | 51 | def console_script_exists(site_packages_dirs: List[Path], console_script: str) -> bool: 52 | """Return true if the console script with provided name exists in one of the site-packages directories. 53 | 54 | Console script is expected to be in the 'bin' directory of site packages. 55 | 56 | :param site_packages_dirs: Paths to site-packages directories on disk. 57 | :param console_script: A console script name. 58 | """ 59 | 60 | for site_packages in site_packages_dirs: 61 | 62 | if (site_packages / "bin" / console_script).exists(): 63 | return True 64 | 65 | return False 66 | 67 | 68 | def copytree(src: Path, dst: Path) -> None: 69 | """A utility function for syncing directories. 70 | 71 | This function is based on shutil.copytree. In Python versions that are 72 | older than 3.8, shutil.copytree would raise FileExistsError if the "dst" 73 | directory already existed. 74 | 75 | """ 76 | 77 | # Make our target (if it doesn't already exist). 78 | dst.mkdir(parents=True, exist_ok=True) 79 | 80 | for path in src.iterdir(): # type: Path 81 | 82 | # If we encounter a subdirectory, recurse. 83 | if path.is_dir(): 84 | copytree(path, dst / path.relative_to(src)) 85 | 86 | else: 87 | shutil.copy2(str(path), str(dst / path.relative_to(src))) 88 | 89 | 90 | @click.command(context_settings=dict(help_option_names=["-h", "--help", "--halp"], ignore_unknown_options=True)) 91 | @click.version_option(version=__version__, prog_name="shiv") 92 | @click.option( 93 | "--entry-point", "-e", default=None, help="The entry point to invoke (takes precedence over --console-script)." 94 | ) 95 | @click.option("--console-script", "-c", default=None, help="The console_script to invoke.") 96 | @click.option("--output-file", "-o", help="The path to the output file for shiv to create.") 97 | @click.option( 98 | "--python", 99 | "-p", 100 | help=( 101 | "The python interpreter to set as the shebang, a.k.a. whatever you want after '#!' " 102 | "(default is '/usr/bin/env python3')" 103 | ), 104 | ) 105 | @click.option( 106 | "--site-packages", 107 | help="The path to an existing site-packages directory to copy into the zipapp.", 108 | type=click.Path(exists=True), 109 | multiple=True, 110 | ) 111 | @click.option( 112 | "--build-id", 113 | default=None, 114 | help=( 115 | "Use a custom build id instead of the default (a SHA256 hash of the contents of the build). " 116 | "Warning: must be unique per build!" 117 | ), 118 | ) 119 | @click.option("--compressed/--uncompressed", default=True, help="Whether or not to compress your zip.") 120 | @click.option( 121 | "--compile-pyc", 122 | is_flag=True, 123 | help="Whether or not to compile pyc files during initial bootstrap.", 124 | ) 125 | @click.option( 126 | "--extend-pythonpath", 127 | "-E", 128 | is_flag=True, 129 | help="Add the contents of the zipapp to PYTHONPATH (for subprocesses).", 130 | ) 131 | @click.option( 132 | "--reproducible", 133 | is_flag=True, 134 | help=( 135 | "Generate a reproducible zipapp by overwriting all files timestamps to a default value. " 136 | "Timestamp can be overwritten by SOURCE_DATE_EPOCH env variable. " 137 | "Note: If SOURCE_DATE_EPOCH is set, this option will be implicitly set to true." 138 | ), 139 | ) 140 | @click.option( 141 | "--no-modify", 142 | is_flag=True, 143 | help=( 144 | "If specified, this modifies the runtime of the zipapp to raise " 145 | "a RuntimeException if the source files (in ~/.shiv or SHIV_ROOT) have been modified. " 146 | """It's recommended to use Python's "--check-hash-based-pycs always" option with this feature.""" 147 | ), 148 | ) 149 | @click.option( 150 | "--preamble", 151 | type=click.Path(exists=True), 152 | help=( 153 | "Provide a path to a preamble script that is invoked by shiv's runtime after bootstrapping the environment, " 154 | "but before invoking your entry point." 155 | ), 156 | ) 157 | @click.option("--root", type=click.Path(), help="Override the 'root' path (default is ~/.shiv).") 158 | @click.argument("pip_args", nargs=-1, type=click.UNPROCESSED) 159 | def main( 160 | output_file: str, 161 | entry_point: Optional[str], 162 | console_script: Optional[str], 163 | python: Optional[str], 164 | site_packages: Optional[str], 165 | build_id: Optional[str], 166 | compressed: bool, 167 | compile_pyc: bool, 168 | extend_pythonpath: bool, 169 | reproducible: bool, 170 | no_modify: bool, 171 | preamble: Optional[str], 172 | root: Optional[str], 173 | pip_args: List[str], 174 | ) -> None: 175 | """ 176 | Shiv is a command line utility for building fully self-contained Python zipapps 177 | as outlined in PEP 441, but with all their dependencies included! 178 | """ 179 | 180 | if not pip_args and not site_packages: 181 | sys.exit(NO_PIP_ARGS_OR_SITE_PACKAGES) 182 | 183 | if output_file is None: 184 | sys.exit(NO_OUTFILE) 185 | 186 | # check for disallowed pip arguments 187 | for disallowed in DISALLOWED_ARGS: 188 | for supplied_arg in pip_args: 189 | if supplied_arg in disallowed: 190 | sys.exit(DISALLOWED_PIP_ARGS.format(arg=supplied_arg, reason=DISALLOWED_ARGS[disallowed])) 191 | 192 | if build_id is not None: 193 | click.secho( 194 | "Warning! You have overridden the default build-id behavior, " 195 | "executables created by shiv must have unique build IDs or unexpected behavior could occur.", 196 | fg="yellow", 197 | ) 198 | 199 | sources: List[Path] = [] 200 | 201 | with TemporaryDirectory() as tmp_site_packages: 202 | 203 | # If both site_packages and pip_args are present, we need to copy the site_packages 204 | # dir into our staging area (tmp_site_packages) as pip may modify the contents. 205 | if site_packages: 206 | if pip_args: 207 | for sp in site_packages: 208 | copytree(Path(sp), Path(tmp_site_packages)) 209 | else: 210 | sources.extend([Path(p).expanduser() for p in site_packages]) 211 | 212 | if pip_args: 213 | # Install dependencies into staged site-packages. 214 | pip.install(["--target", tmp_site_packages] + list(pip_args)) 215 | 216 | if preamble: 217 | bin_dir = Path(tmp_site_packages, "bin") 218 | bin_dir.mkdir(exist_ok=True) 219 | shutil.copy(Path(preamble).absolute(), bin_dir / Path(preamble).name) 220 | 221 | sources.append(Path(tmp_site_packages).absolute()) 222 | 223 | if no_modify: 224 | # if no_modify is specified, we need to build a map of source files and their 225 | # sha256 hashes, to be checked at runtime: 226 | hashes = {} 227 | 228 | for source in sources: 229 | for path in source.rglob("**/*.py"): 230 | hashes[str(path.relative_to(source))] = hashlib.sha256(path.read_bytes()).hexdigest() 231 | 232 | # if entry_point is a console script, get the callable and null out the console_script variable 233 | # so that we avoid modifying sys.argv in bootstrap.py 234 | if entry_point is None and console_script is not None: 235 | try: 236 | entry_point = find_entry_point(sources, console_script) 237 | except KeyError: 238 | if not console_script_exists(sources, console_script): 239 | sys.exit(NO_ENTRY_POINT.format(entry_point=console_script)) 240 | else: 241 | console_script = None 242 | 243 | # Some projects need reproducible artifacts, so they can use SOURCE_DATE_EPOCH 244 | # environment variable to specify the timestamps in the zipapp. 245 | timestamp = int( 246 | os.environ.get(SOURCE_DATE_EPOCH_ENV, SOURCE_DATE_EPOCH_DEFAULT if reproducible else time.time()) 247 | ) 248 | 249 | # create runtime environment metadata 250 | env = Environment( 251 | built_at=datetime.utcfromtimestamp(timestamp).strftime(BUILD_AT_TIMESTAMP_FORMAT), 252 | build_id=build_id, 253 | entry_point=entry_point, 254 | script=console_script, 255 | compile_pyc=compile_pyc, 256 | extend_pythonpath=extend_pythonpath, 257 | shiv_version=__version__, 258 | no_modify=no_modify, 259 | reproducible=reproducible, 260 | preamble=Path(preamble).name if preamble else None, 261 | root=root, 262 | ) 263 | 264 | if no_modify: 265 | env.hashes = hashes 266 | 267 | # create the zip 268 | builder.create_archive( 269 | sources, 270 | target=Path(output_file).expanduser(), 271 | interpreter=python or DEFAULT_SHEBANG, 272 | main="_bootstrap:bootstrap", 273 | env=env, 274 | compressed=compressed, 275 | ) 276 | 277 | 278 | if __name__ == "__main__": # pragma: no cover 279 | main() 280 | -------------------------------------------------------------------------------- /src/shiv/constants.py: -------------------------------------------------------------------------------- 1 | """This module contains various error messages.""" 2 | from typing import Dict, Tuple 3 | 4 | # errors: 5 | DISALLOWED_PIP_ARGS = "\nYou supplied a disallowed pip argument! '{arg}'\n\n{reason}\n" 6 | NO_PIP_ARGS_OR_SITE_PACKAGES = "\nYou must supply PIP ARGS or --site-packages!\n" 7 | NO_OUTFILE = "\nYou must provide an output file option! (--output-file/-o)\n" 8 | NO_ENTRY_POINT = "\nNo entry point '{entry_point}' found in console_scripts or the bin dir!\n" 9 | BINPRM_ERROR = "\nShebang is too long, it would exceed BINPRM_BUF_SIZE! Consider /usr/bin/env\n" 10 | 11 | # pip 12 | PIP_INSTALL_ERROR = "\nPip install failed!\n" 13 | PIP_REQUIRE_VIRTUALENV = "PIP_REQUIRE_VIRTUALENV" 14 | DISALLOWED_ARGS: Dict[Tuple[str, ...], str] = { 15 | ("-t", "--target"): "Shiv already supplies a target internally, so overriding is not allowed.", 16 | ( 17 | "--editable", 18 | ): "Editable installs don't actually install via pip (they are just linked), so they are not allowed.", 19 | ("-d", "--download"): "Shiv needs to actually perform an install, not merely a download.", 20 | ("--user", "--prefix"): "Which conflicts with Shiv's internal use of '--target'.", 21 | } 22 | 23 | SOURCE_DATE_EPOCH_ENV = "SOURCE_DATE_EPOCH" 24 | # This is the timestamp for beginning of the day Jan 1 1980, which is the minimum timestamp 25 | # value you can use in zip archives 26 | SOURCE_DATE_EPOCH_DEFAULT = 315554400 27 | BUILD_AT_TIMESTAMP_FORMAT = "%Y-%m-%d %H:%M:%S" 28 | 29 | # The default shebang to use at the top of the pyz. 30 | # We use "/usr/bin/env" here because it's cross-platform compatible 31 | # https://docs.python.org/3/using/windows.html#shebang-lines 32 | DEFAULT_SHEBANG = "/usr/bin/env python3" 33 | -------------------------------------------------------------------------------- /src/shiv/info.py: -------------------------------------------------------------------------------- 1 | import json 2 | import zipfile 3 | 4 | import click 5 | 6 | 7 | @click.command(context_settings=dict(help_option_names=["-h", "--help", "--halp"])) 8 | @click.option("--json", "-j", "print_as_json", is_flag=True, help="output as plain json") 9 | @click.argument("pyz") 10 | def main(print_as_json, pyz): 11 | """A simple utility to print debugging information about PYZ files created with ``shiv``""" 12 | 13 | zip_file = zipfile.ZipFile(pyz) 14 | data = json.loads(zip_file.read("environment.json")) 15 | 16 | if print_as_json: 17 | click.echo(json.dumps(data, indent=4, sort_keys=True)) 18 | 19 | else: 20 | click.echo() 21 | click.secho("pyz file: ", fg="green", bold=True, nl=False) 22 | click.secho(pyz, fg="white") 23 | click.echo() 24 | 25 | for key, value in data.items(): 26 | click.secho(f"{key}: ", fg="blue", bold=True, nl=False) 27 | 28 | if key == "hashes": 29 | click.secho(json.dumps(value, sort_keys=True, indent=2)) 30 | else: 31 | click.secho(f"{value}", fg="white") 32 | 33 | click.echo() 34 | -------------------------------------------------------------------------------- /src/shiv/pip.py: -------------------------------------------------------------------------------- 1 | import contextlib 2 | import os 3 | import subprocess 4 | import sys 5 | 6 | from typing import Generator, List 7 | 8 | import click 9 | 10 | from .bootstrap import extend_python_path, get_first_sitedir_index 11 | from .constants import PIP_INSTALL_ERROR, PIP_REQUIRE_VIRTUALENV 12 | 13 | 14 | @contextlib.contextmanager 15 | def clean_pip_env() -> Generator[None, None, None]: 16 | """A context manager for temporarily removing 'PIP_REQUIRE_VIRTUALENV' from the environment. 17 | 18 | Since shiv installs via `--target`, we need to ignore venv requirements if they exist. 19 | 20 | """ 21 | require_venv = os.environ.pop(PIP_REQUIRE_VIRTUALENV, None) 22 | 23 | try: 24 | yield 25 | 26 | finally: 27 | if require_venv is not None: 28 | os.environ[PIP_REQUIRE_VIRTUALENV] = require_venv 29 | 30 | 31 | def install(args: List[str]) -> None: 32 | """`pip install` as a function. 33 | 34 | Accepts a list of pip arguments. 35 | 36 | .. code-block:: py 37 | 38 | >>> install(['numpy', '--target', 'site-packages']) 39 | Collecting numpy 40 | Downloading numpy-1.13.3-cp35-cp35m-manylinux1_x86_64.whl (16.9MB) 41 | 100% || 16.9MB 53kB/s 42 | Installing collected packages: numpy 43 | Successfully installed numpy-1.13.3 44 | 45 | """ 46 | 47 | with clean_pip_env(): 48 | 49 | # if being invoked as a pyz, we must ensure we have access to our own 50 | # site-packages when subprocessing since there is no guarantee that pip 51 | # will be available 52 | subprocess_env = os.environ.copy() 53 | sitedir_index = get_first_sitedir_index() 54 | extend_python_path(subprocess_env, sys.path[sitedir_index:]) 55 | 56 | process = subprocess.Popen( 57 | [sys.executable, "-m", "pip", "--disable-pip-version-check", "install", *args], 58 | stdout=subprocess.PIPE, 59 | stderr=subprocess.STDOUT, 60 | env=subprocess_env, 61 | universal_newlines=True, 62 | ) 63 | 64 | for output in process.stdout: # type: ignore 65 | if output: 66 | click.echo(output.rstrip()) 67 | 68 | if process.wait() > 0: 69 | sys.exit(PIP_INSTALL_ERROR) 70 | -------------------------------------------------------------------------------- /test/conftest.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from contextlib import contextmanager 4 | from pathlib import Path 5 | 6 | import pytest 7 | 8 | from shiv.bootstrap.environment import Environment 9 | 10 | 11 | @pytest.fixture 12 | def zip_location(): 13 | return Path(__file__).absolute().parent / "test.zip" 14 | 15 | 16 | @pytest.fixture 17 | def package_location(): 18 | return Path(__file__).absolute().parent / "package" 19 | 20 | 21 | @pytest.fixture 22 | def sp(): 23 | return [Path(__file__).absolute().parent / "sp" / "site-packages"] 24 | 25 | 26 | @pytest.fixture 27 | def env(): 28 | return Environment( 29 | built_at=str("2019-01-01 12:12:12"), 30 | build_id=str("test_id"), 31 | entry_point="test_entry_point", 32 | script="test_console_script", 33 | compile_pyc=False, 34 | extend_pythonpath=False, 35 | shiv_version="0.0.1", 36 | ) 37 | 38 | 39 | @pytest.fixture 40 | def env_var(): 41 | 42 | @contextmanager 43 | def _env_var(key, value): 44 | os.environ[key] = value 45 | yield 46 | del os.environ[key] 47 | 48 | return _env_var 49 | -------------------------------------------------------------------------------- /test/package/hello/__init__.py: -------------------------------------------------------------------------------- 1 | def main(): 2 | print("hello world") 3 | -------------------------------------------------------------------------------- /test/package/hello/script.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | exit 0 4 | -------------------------------------------------------------------------------- /test/package/setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | from setuptools import setup 4 | 5 | setup( 6 | name="hello", 7 | packages=["hello"], 8 | package_data={"": ["script.sh"]}, 9 | entry_points={"console_scripts": ["hello = hello:main"]}, 10 | ) 11 | -------------------------------------------------------------------------------- /test/test.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linkedin/shiv/a353d10ecd785c6bd67ccd08b642437e7204be57/test/test.zip -------------------------------------------------------------------------------- /test/test_bootstrap.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | from code import interact 5 | from datetime import datetime 6 | from pathlib import Path 7 | from site import addsitedir 8 | from unittest import mock 9 | from uuid import uuid4 10 | from zipfile import ZipFile 11 | 12 | import pytest 13 | 14 | from shiv.bootstrap import ( 15 | cache_path, 16 | current_zipfile, 17 | ensure_no_modify, 18 | extend_python_path, 19 | extract_site_packages, 20 | get_first_sitedir_index, 21 | import_string, 22 | prepend_pythonpath, 23 | ) 24 | from shiv.bootstrap.environment import Environment 25 | from shiv.bootstrap.filelock import FileLock 26 | from shiv.pip import install 27 | 28 | 29 | class TestBootstrap: 30 | def test_import_string(self): 31 | assert import_string("site.addsitedir") == addsitedir 32 | assert import_string("site:addsitedir") == addsitedir 33 | assert import_string("code.interact") == interact 34 | assert import_string("code:interact") == interact 35 | 36 | # test things not already imported 37 | func = import_string("os.path:join") 38 | from os.path import join 39 | 40 | assert func == join 41 | 42 | # test something already imported 43 | import shiv 44 | 45 | assert import_string("shiv") == shiv == sys.modules["shiv"] 46 | 47 | # test bogus imports raise properly 48 | with pytest.raises(ImportError): 49 | import_string("this is bogus!") 50 | 51 | def test_is_zipfile(self, zip_location): 52 | with mock.patch.object(sys, "argv", [zip_location]): 53 | with current_zipfile() as zipfile: 54 | assert isinstance(zipfile, ZipFile) 55 | 56 | # When the tests are run via tox, sys.argv[0] is the full path to 'pytest.EXE', 57 | # i.e. a native launcher created by pip to from console_scripts entry points. 58 | # These are indeed a form of zip files, thus the following assertion could fail. 59 | @pytest.mark.skipif(os.name == "nt", reason="this may give false positive on win") 60 | def test_argv0_is_not_zipfile(self): 61 | with current_zipfile() as zipfile: 62 | assert not zipfile 63 | 64 | def test_cache_path(self, env_var): 65 | mock_zip = mock.MagicMock(spec=ZipFile) 66 | mock_zip.filename = "test" 67 | uuid = str(uuid4()) 68 | 69 | assert cache_path(mock_zip, 'foo', uuid) == Path("foo", f"test_{uuid}") 70 | 71 | with env_var("FOO", "foo"): 72 | assert cache_path(mock_zip, '$FOO', uuid) == Path("foo", f"test_{uuid}") 73 | 74 | def test_first_sitedir_index(self): 75 | with mock.patch.object(sys, "path", ["site-packages", "dir", "dir", "dir"]): 76 | assert get_first_sitedir_index() == 0 77 | 78 | with mock.patch.object(sys, "path", []): 79 | assert get_first_sitedir_index() is None 80 | 81 | @pytest.mark.parametrize("nested", (False, True)) 82 | @pytest.mark.parametrize("compile_pyc", (False, True)) 83 | @pytest.mark.parametrize("force", (False, True)) 84 | def test_extract_site_packages(self, tmp_path, zip_location, nested, compile_pyc, force): 85 | 86 | zipfile = ZipFile(str(zip_location)) 87 | target = tmp_path / "test" 88 | 89 | if nested: 90 | # we want to test for not-yet-created shiv root dirs 91 | target = target / "nested" / "root" 92 | 93 | if force: 94 | # we want to make sure we overwrite if the target exists when using force 95 | target.mkdir(parents=True, exist_ok=True) 96 | 97 | # Do the extraction (of our empty zip file) 98 | extract_site_packages(zipfile, target, compile_pyc, force=force) 99 | 100 | site_packages = target / "site-packages" 101 | assert site_packages.exists() 102 | assert site_packages.is_dir() 103 | assert Path(site_packages, "test").exists() 104 | assert Path(site_packages, "test").is_file() 105 | 106 | @pytest.mark.parametrize("additional_paths", (["test"], ["test", ".pth"])) 107 | def test_extend_path(self, additional_paths): 108 | 109 | env = {} 110 | 111 | extend_python_path(env, additional_paths) 112 | assert env["PYTHONPATH"] == os.pathsep.join(additional_paths) 113 | 114 | def test_extend_path_existing_pythonpath(self): 115 | """When PYTHONPATH exists, extending it preserves the existing values.""" 116 | env = {"PYTHONPATH": "hello"} 117 | 118 | extend_python_path(env, ["test", ".pth"]) 119 | assert env["PYTHONPATH"] == os.pathsep.join(["hello", "test", ".pth"]) 120 | 121 | @pytest.mark.parametrize("extra_path", [ 122 | None, 123 | "/path/to/other_package", 124 | ]) 125 | def test_prepend_pythonpath(self, env, extra_path): 126 | # Save old path to be able to restore it after executing the test 127 | old_path = sys.path.copy() 128 | 129 | env._prepend_pythonpath = extra_path 130 | prepend_pythonpath(env) 131 | if extra_path is not None: 132 | assert len(sys.path) == len(old_path) + 1 133 | assert sys.path[0] == extra_path 134 | else: 135 | assert len(sys.path) == len(old_path) 136 | 137 | # Cleanup 138 | sys.path = old_path 139 | 140 | 141 | class TestEnvironment: 142 | def test_overrides(self, env_var): 143 | now = str(datetime.now()) 144 | version = "0.0.1" 145 | env = Environment(now, version) 146 | 147 | assert env.built_at == now 148 | assert env.shiv_version == version 149 | 150 | assert env.entry_point is None 151 | with env_var("SHIV_ENTRY_POINT", "test"): 152 | assert env.entry_point == "test" 153 | 154 | assert env.script is None 155 | with env_var("SHIV_CONSOLE_SCRIPT", "test"): 156 | assert env.script == "test" 157 | 158 | assert env.interpreter is None 159 | with env_var("SHIV_INTERPRETER", "1"): 160 | assert env.interpreter is not None 161 | 162 | assert env.root is None 163 | with env_var("SHIV_ROOT", "tmp"): 164 | assert env.root == "tmp" 165 | 166 | assert env.force_extract is False 167 | with env_var("SHIV_FORCE_EXTRACT", "1"): 168 | assert env.force_extract is True 169 | 170 | assert env.compile_pyc is True 171 | with env_var("SHIV_COMPILE_PYC", "False"): 172 | assert env.compile_pyc is False 173 | 174 | assert env.extend_pythonpath is False 175 | with env_var("SHIV_EXTEND_PYTHONPATH", "1"): 176 | assert env.compile_pyc is True 177 | 178 | assert env.compile_workers == 0 179 | with env_var("SHIV_COMPILE_WORKERS", "1"): 180 | assert env.compile_workers == 1 181 | 182 | # ensure that non-digits are ignored 183 | with env_var("SHIV_COMPILE_WORKERS", "one bazillion"): 184 | assert env.compile_workers == 0 185 | 186 | assert env.prepend_pythonpath is None 187 | with env_var(Environment.PREPEND_PYTHONPATH, "/path/to/other_package"): 188 | assert env.prepend_pythonpath == "/path/to/other_package" 189 | 190 | def test_roundtrip(self): 191 | now = str(datetime.now()) 192 | version = "0.0.1" 193 | env = Environment(now, version) 194 | env_as_json = env.to_json() 195 | env_from_json = Environment.from_json(env_as_json) 196 | assert env.__dict__ == env_from_json.__dict__ 197 | 198 | def test_lock(self, tmp_path): 199 | with FileLock(str(tmp_path / "lockfile")) as f: 200 | assert f.is_locked 201 | 202 | assert not f.is_locked 203 | 204 | @pytest.mark.skipif( 205 | os.name == "nt", reason="windows creates .exe files for entry points, which are not reproducible :(" 206 | ) 207 | def test_ensure_no_modify(self, tmp_path, package_location): 208 | 209 | # Populate a site-packages dir 210 | site_packages = tmp_path / "site-packages" 211 | install(["-t", str(site_packages), str(package_location)]) 212 | 213 | for test_hash in [{"abc": "123"}, {"hello/__init__.py": "123"}]: 214 | with pytest.raises(RuntimeError): 215 | ensure_no_modify(site_packages, test_hash) 216 | 217 | # the hash of the only source file the test package provides 218 | hashes = {"hello/__init__.py": "1e8d5b8a6839487a4211229f69b76a5f901515dcad7f111a4bdd5b30d9e96020"} 219 | 220 | ensure_no_modify(site_packages, hashes) 221 | -------------------------------------------------------------------------------- /test/test_builder.py: -------------------------------------------------------------------------------- 1 | import os 2 | import stat 3 | import sys 4 | import tempfile 5 | import zipfile 6 | 7 | from pathlib import Path 8 | from zipapp import ZipAppError 9 | 10 | import pytest 11 | 12 | from shiv.builder import create_archive, rglob_follow_symlinks, write_file_prefix 13 | 14 | UGOX = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH 15 | 16 | 17 | def tmp_write_prefix(interpreter): 18 | with tempfile.TemporaryFile() as fd: 19 | write_file_prefix(fd, interpreter) 20 | fd.seek(0) 21 | written = fd.read() 22 | 23 | return written 24 | 25 | 26 | class TestBuilder: 27 | @pytest.mark.parametrize( 28 | "interpreter,expected", 29 | [ 30 | ("/usr/bin/python", b"#!/usr/bin/python\n"), 31 | ("/usr/bin/env python", b"#!/usr/bin/env python\n"), 32 | ("/some/other/path/python -sE", b"#!/some/other/path/python -sE\n"), 33 | ], 34 | ) 35 | def test_file_prefix(self, interpreter, expected): 36 | assert tmp_write_prefix(interpreter) == expected 37 | 38 | def test_binprm_error(self): 39 | with pytest.raises(SystemExit): 40 | tmp_write_prefix(f"/{'c' * 200}/python") 41 | 42 | def test_rglob_follow_symlinks(self, tmp_path): 43 | real_dir = tmp_path / 'real_dir' 44 | real_dir.mkdir() 45 | real_file = real_dir / 'real_file' 46 | real_file.touch() 47 | sym_dir = tmp_path / 'sym_dir' 48 | sym_dir.symlink_to(real_dir) 49 | sym_file = sym_dir / real_file.name 50 | assert sorted(rglob_follow_symlinks(tmp_path, '*'), key=str) == [real_dir, real_file, sym_dir, sym_file] 51 | 52 | def test_create_archive(self, sp, env): 53 | with tempfile.TemporaryDirectory() as tmpdir: 54 | target = Path(tmpdir, "test.zip") 55 | 56 | # create an archive 57 | create_archive(sp, target, sys.executable, "code:interact", env) 58 | 59 | # create one again (to ensure we overwrite) 60 | create_archive(sp, target, sys.executable, "code:interact", env) 61 | 62 | assert zipfile.is_zipfile(str(target)) 63 | 64 | with pytest.raises(ZipAppError): 65 | create_archive(sp, target, sys.executable, "alsjdbas,,,", env) 66 | 67 | @pytest.mark.skipif(os.name == "nt", reason="windows has no concept of execute permissions") 68 | def test_archive_permissions(self, sp, env): 69 | with tempfile.TemporaryDirectory() as tmpdir: 70 | target = Path(tmpdir, "test.zip") 71 | create_archive(sp, target, sys.executable, "code:interact", env) 72 | 73 | assert target.stat().st_mode & UGOX == UGOX 74 | -------------------------------------------------------------------------------- /test/test_cli.py: -------------------------------------------------------------------------------- 1 | import contextlib 2 | import hashlib 3 | import json 4 | import os 5 | import stat 6 | import subprocess 7 | import sys 8 | 9 | from pathlib import Path 10 | 11 | import pytest 12 | 13 | from click.testing import CliRunner 14 | from shiv.cli import console_script_exists, find_entry_point, main 15 | from shiv.constants import DISALLOWED_ARGS, DISALLOWED_PIP_ARGS, NO_OUTFILE, NO_PIP_ARGS_OR_SITE_PACKAGES 16 | from shiv.info import main as info_main 17 | from shiv.pip import install 18 | 19 | UGOX = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH 20 | 21 | 22 | @contextlib.contextmanager 23 | def mocked_sys_prefix(): 24 | attribute_to_mock = "real_prefix" if hasattr(sys, "real_prefix") else "base_prefix" 25 | original = getattr(sys, attribute_to_mock) 26 | setattr(sys, attribute_to_mock, "/fake/dir") 27 | yield 28 | setattr(sys, attribute_to_mock, original) 29 | 30 | 31 | class TestCLI: 32 | @pytest.fixture 33 | def shiv_root(self, monkeypatch, tmp_path): 34 | os.environ["SHIV_ROOT"] = str(tmp_path) 35 | yield tmp_path 36 | os.environ.pop("SHIV_ROOT") 37 | 38 | @pytest.fixture 39 | def runner(self): 40 | """Returns a click test runner.""" 41 | 42 | def invoke(args, env=None): 43 | args.extend(["-p", "/usr/bin/env python3"]) 44 | return CliRunner().invoke(main, args, env=env) 45 | 46 | return invoke 47 | 48 | @pytest.fixture 49 | def info_runner(self): 50 | """Returns a click test runner (for shiv-info).""" 51 | 52 | return lambda args: CliRunner().invoke(info_main, args) 53 | 54 | def test_find_entry_point(self, tmpdir, package_location): 55 | """Test that we can find console_script metadata.""" 56 | install(["-t", str(tmpdir), str(package_location)]) 57 | assert find_entry_point([Path(tmpdir)], "hello") == "hello:main" 58 | 59 | def test_find_entry_point_two_points(self, tmpdir, package_location): 60 | """Test that we can find console_script metadata.""" 61 | install(["-t", str(tmpdir), str(package_location)]) 62 | assert find_entry_point([Path(tmpdir)], "hello") == "hello:main" 63 | 64 | def test_console_script_exists(self, tmp_path, package_location): 65 | """Test that we can check console_script presence.""" 66 | install_dir = tmp_path / "install" 67 | install(["-t", str(install_dir), str(package_location)]) 68 | empty_dir = tmp_path / "empty" 69 | empty_dir.mkdir() 70 | 71 | assert console_script_exists([empty_dir, install_dir], "hello.exe" if os.name == "nt" else "hello") 72 | 73 | def test_no_args(self, runner): 74 | """This should fail with a warning about supplying pip arguments""" 75 | 76 | result = runner([]) 77 | 78 | assert result.exit_code == 1 79 | assert NO_PIP_ARGS_OR_SITE_PACKAGES in result.output 80 | 81 | def test_no_outfile(self, runner): 82 | """This should fail with a warning about not providing an outfile""" 83 | 84 | result = runner(["-e", "test", "flask"]) 85 | 86 | assert result.exit_code == 1 87 | assert NO_OUTFILE in result.output 88 | 89 | @pytest.mark.parametrize("arg", [arg for tup in DISALLOWED_ARGS.keys() for arg in tup]) 90 | def test_disallowed_args(self, runner, arg): 91 | """This method tests that all the potential disallowed arguments match their error messages.""" 92 | 93 | # run shiv with a disallowed argument 94 | result = runner(["-o", "tmp", arg]) 95 | 96 | # get the 'reason' message: 97 | reason = next(iter([DISALLOWED_ARGS[disallowed] for disallowed in DISALLOWED_ARGS if arg in disallowed])) 98 | 99 | assert result.exit_code == 1 100 | 101 | # assert we got the correct reason 102 | assert DISALLOWED_PIP_ARGS.format(arg=arg, reason=reason) in result.output 103 | 104 | @pytest.mark.parametrize("compile_option", [["--compile-pyc"], ["--build-id", "42424242"], []]) 105 | @pytest.mark.parametrize("force", ["yes", "no"]) 106 | def test_hello_world(self, runner, info_runner, shiv_root, package_location, compile_option, force): 107 | output_file = shiv_root / "test.pyz" 108 | 109 | result = runner(["-e", "hello:main", "-o", str(output_file), str(package_location), *compile_option]) 110 | 111 | # check that the command successfully completed 112 | assert result.exit_code == 0 113 | 114 | # ensure the created file actually exists 115 | assert output_file.exists() 116 | 117 | # build env 118 | env = {**os.environ, "SHIV_FORCE_EXTRACT": force} 119 | 120 | # now run the produced zipapp 121 | proc = subprocess.run([str(output_file)], stdout=subprocess.PIPE, shell=True, env=env) 122 | 123 | assert proc.stdout.decode() == "hello world" + os.linesep 124 | 125 | # now run shiv-info on the produced zipapp 126 | result = info_runner([str(output_file)]) 127 | 128 | # check the rc and output 129 | assert result.exit_code == 0 130 | assert f"pyz file: {str(output_file)}" in result.output 131 | 132 | # ensure that executable permissions were retained (skip test on windows) 133 | if os.name != "nt": 134 | build_id = json.loads(info_runner([str(output_file), "--json"]).output)["build_id"] 135 | if "--build-id" in compile_option: 136 | assert build_id == compile_option[1] 137 | assert ( 138 | Path(shiv_root, f"{output_file.name}_{build_id}", "site-packages", "hello", "script.sh").stat().st_mode 139 | & UGOX 140 | == UGOX 141 | ) 142 | 143 | @pytest.mark.parametrize("extend_path", [["--extend-pythonpath"], ["-E"], []]) 144 | def test_extend_pythonpath(self, shiv_root, runner, extend_path): 145 | 146 | output_file = Path(shiv_root, "test_pythonpath.pyz") 147 | package_dir = Path(shiv_root, "package") 148 | main_script = Path(package_dir, "env.py") 149 | 150 | # noinspection PyPep8Naming 151 | MAIN_PROG = "\n".join(["import os", "def main():", " print(os.environ.get('PYTHONPATH', ''))"]) 152 | 153 | package_dir.mkdir() 154 | main_script.write_text(MAIN_PROG) 155 | 156 | result = runner(["-e", "env:main", "-o", str(output_file), "--site-packages", str(package_dir), *extend_path]) 157 | 158 | # check that the command successfully completed 159 | assert result.exit_code == 0 160 | 161 | # ensure the created file actually exists 162 | assert output_file.exists() 163 | 164 | # now run the produced zipapp and confirm shiv_root is in PYTHONPATH 165 | proc = subprocess.run([str(output_file)], stdout=subprocess.PIPE, shell=True, env=os.environ) 166 | 167 | pythonpath_has_root = str(shiv_root) in proc.stdout.decode() 168 | if extend_path: 169 | assert pythonpath_has_root 170 | 171 | def test_multiple_site_packages(self, shiv_root, runner): 172 | output_file = shiv_root / "test_multiple_sp.pyz" 173 | package_dir = shiv_root / "package" 174 | main_script = package_dir / "hello.py" 175 | 176 | env_code = "\n".join(["import os", "def hello():", " print('hello!')"]) 177 | 178 | package_dir.mkdir() 179 | main_script.write_text(env_code) 180 | 181 | other_package_dir = shiv_root / "dependent_package" 182 | main_script = package_dir / "hello_client.py" 183 | 184 | env_client_code = "\n".join(["import os", "from hello import hello", "def main():", " hello()"]) 185 | 186 | other_package_dir.mkdir() 187 | main_script.write_text(env_client_code) 188 | 189 | result = runner( 190 | [ 191 | "-e", 192 | "hello_client:main", 193 | "-o", 194 | str(output_file), 195 | "--site-packages", 196 | str(package_dir), 197 | "--site-packages", 198 | str(other_package_dir), 199 | ] 200 | ) 201 | 202 | # check that the command successfully completed 203 | assert result.exit_code == 0 204 | 205 | # ensure the created file actually exists 206 | assert output_file.exists() 207 | 208 | # now run the produced zipapp and confirm that output is ok 209 | proc = subprocess.run([str(output_file)], stdout=subprocess.PIPE, shell=True, env=os.environ) 210 | assert "hello!" in proc.stdout.decode() 211 | 212 | def test_no_entrypoint(self, shiv_root, runner, package_location): 213 | 214 | output_file = shiv_root / "test.pyz" 215 | 216 | result = runner(["-o", str(output_file), str(package_location)]) 217 | 218 | # check that the command successfully completed 219 | assert result.exit_code == 0 220 | 221 | # ensure the created file actually exists 222 | assert output_file.exists() 223 | 224 | # now run the produced zipapp 225 | proc = subprocess.run( 226 | [str(output_file)], 227 | input=b"import hello;print(hello)", 228 | stdout=subprocess.PIPE, 229 | stderr=subprocess.PIPE, 230 | shell=True, 231 | env=os.environ, 232 | ) 233 | 234 | assert proc.returncode == 0 235 | assert "hello" in proc.stdout.decode() 236 | 237 | @pytest.mark.skipif( 238 | os.name == "nt", reason="windows creates .exe files for entry points, which are not reproducible :(" 239 | ) 240 | def test_results_are_binary_identical_with_env_and_build_id(self, shiv_root, runner, package_location): 241 | first_output_file = shiv_root / "test_one.pyz" 242 | second_output_file = shiv_root / "test_two.pyz" 243 | 244 | result_one = runner( 245 | ["-e", "hello:main", "-o", str(first_output_file), "--reproducible", "--no-modify", str(package_location)], 246 | env={"SOURCE_DATE_EPOCH": "1234567890"}, 247 | ) # 2009-02-13 23:31:30 UTC 248 | 249 | result_two = runner( 250 | ["-e", "hello:main", "-o", str(second_output_file), "--reproducible", "--no-modify", str(package_location)], 251 | env={"SOURCE_DATE_EPOCH": "1234567890"}, 252 | ) # 2009-02-13 23:31:30 UTC 253 | 254 | # check that both commands successfully completed 255 | assert result_one.exit_code == 0 256 | assert result_two.exit_code == 0 257 | 258 | # check that both executables are binary identical 259 | assert ( 260 | hashlib.md5(first_output_file.read_bytes()).hexdigest() 261 | == hashlib.md5(second_output_file.read_bytes()).hexdigest() 262 | ) 263 | 264 | # finally, check that one of the result works 265 | proc = subprocess.run( 266 | [str(first_output_file)], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, env=os.environ, 267 | ) 268 | 269 | assert proc.returncode == 0 270 | assert "hello" in proc.stdout.decode() 271 | 272 | @pytest.mark.skipif( 273 | os.name == "nt", reason="can't run a shell script on windows" 274 | ) 275 | @pytest.mark.parametrize( 276 | "preamble, contents", 277 | [ 278 | ("preamble.py", "#!/usr/bin/env python3\nprint('hello from preamble')"), 279 | ("preamble.sh", "#!/bin/sh\necho 'hello from preamble'"), 280 | ], 281 | ) 282 | def test_preamble(self, preamble, contents, shiv_root, runner, package_location, tmp_path): 283 | """Test the --preamble argument.""" 284 | 285 | output_file = shiv_root / "test.pyz" 286 | preamble = tmp_path / preamble 287 | preamble.write_text(contents) 288 | preamble.chmod(preamble.stat().st_mode | stat.S_IEXEC) 289 | 290 | result = runner( 291 | ["-e", "hello:main", "--preamble", str(preamble), "-o", str(output_file), str(package_location)] 292 | ) 293 | 294 | # check that the command successfully completed 295 | assert result.exit_code == 0 296 | 297 | # ensure the created file actually exists 298 | assert output_file.exists() 299 | 300 | # now run the produced zipapp 301 | proc = subprocess.run( 302 | [str(output_file)], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, env=os.environ, 303 | ) 304 | 305 | assert proc.returncode == 0 306 | assert proc.stdout.decode().splitlines() == ["hello from preamble", "hello world"] 307 | 308 | def test_preamble_no_pip(self, shiv_root, runner, package_location, tmp_path): 309 | """Test that the preamble script is created even with no pip installed packages.""" 310 | 311 | output_file = shiv_root / "test.pyz" 312 | target = tmp_path / "target" 313 | preamble = tmp_path / "preamble.py" 314 | preamble.write_text("#!/usr/bin/env python3\nprint('hello from preamble')") 315 | preamble.chmod(preamble.stat().st_mode | stat.S_IEXEC) 316 | 317 | # first, by installing our test package into a target 318 | install(["-t", str(target), str(package_location)]) 319 | result = runner( 320 | ["-e", "hello:main", "--preamble", str(preamble), "-o", str(output_file), "--site-packages", target] 321 | ) 322 | 323 | # check that the command successfully completed 324 | assert result.exit_code == 0 325 | 326 | # ensure the created file actually exists 327 | assert output_file.exists() 328 | 329 | # now run the produced zipapp 330 | proc = subprocess.run( 331 | [str(output_file)], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, env=os.environ, 332 | ) 333 | 334 | assert proc.returncode == 0 335 | assert proc.stdout.decode().splitlines() == ["hello from preamble", "hello world"] 336 | 337 | def test_alternate_root(self, runner, package_location, tmp_path): 338 | """Test that the --root argument properly sets the extraction root.""" 339 | 340 | output_file = tmp_path / "test.pyz" 341 | shiv_root = tmp_path / "root" 342 | result = runner( 343 | ["-e", "hello:main", "--root", str(shiv_root), "-o", str(output_file), str(package_location)] 344 | ) 345 | 346 | # check that the command successfully completed 347 | assert result.exit_code == 0 348 | 349 | # ensure the created file actually exists 350 | assert output_file.exists() 351 | 352 | # now run the produced zipapp 353 | proc = subprocess.run( 354 | [str(output_file)], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, env=os.environ, 355 | ) 356 | 357 | assert proc.returncode == 0 358 | assert "hello" in proc.stdout.decode() 359 | assert shiv_root.exists() 360 | 361 | def test_alternate_root_environment_variable(self, runner, package_location, tmp_path, env_var): 362 | """Test that the --root argument works with environment variables.""" 363 | 364 | output_file = tmp_path / "test.pyz" 365 | shiv_root_var = "NEW_ROOT" 366 | shiv_root_path = tmp_path / 'new_root' 367 | result = runner( 368 | ["-e", "hello:main", "--root", "$" + shiv_root_var, "-o", str(output_file), str(package_location)] 369 | ) 370 | 371 | with env_var(shiv_root_var, str(shiv_root_path)): 372 | 373 | # check that the command successfully completed 374 | assert result.exit_code == 0 375 | 376 | # ensure the created file actually exists 377 | assert output_file.exists() 378 | 379 | # now run the produced zipapp 380 | proc = subprocess.run( 381 | [str(output_file)], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, env=os.environ, 382 | ) 383 | 384 | assert proc.returncode == 0 385 | assert "hello" in proc.stdout.decode() 386 | assert shiv_root_path.exists() 387 | -------------------------------------------------------------------------------- /test/test_pip.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from shiv.constants import PIP_REQUIRE_VIRTUALENV 4 | from shiv.pip import clean_pip_env 5 | 6 | 7 | def test_clean_pip_env(monkeypatch): 8 | 9 | before_env_var = "test" 10 | monkeypatch.setenv(PIP_REQUIRE_VIRTUALENV, before_env_var) 11 | 12 | with clean_pip_env(): 13 | assert PIP_REQUIRE_VIRTUALENV not in os.environ 14 | 15 | assert os.environ.get(PIP_REQUIRE_VIRTUALENV) == before_env_var 16 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | isolated_build = True 3 | envlist = py38, py39, py310, py311 4 | 5 | [gh-actions] 6 | python = 7 | 3.8: py38 8 | 3.9: py39 9 | 3.10: py310 10 | 3.11: py311 11 | 12 | [testenv] 13 | commands= 14 | coverage run -m pytest 15 | mypy src/ 16 | flake8 src/ test/ 17 | deps= 18 | coverage 19 | pytest 20 | mypy 21 | flake8 22 | --------------------------------------------------------------------------------