├── .github └── workflows │ ├── main.yml │ └── pypi.yml ├── .gitignore ├── .pre-commit-config.yaml ├── LICENSE ├── README.md ├── requirements-dev.txt ├── setup.cfg ├── setup.py ├── tests ├── __init__.py └── virtualenv_tools_test.py ├── tox.ini └── virtualenv_tools.py /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | on: 2 | pull_request: 3 | push: 4 | branches: [master, test-me-*] 5 | 6 | jobs: 7 | main: 8 | runs-on: ubuntu-latest 9 | strategy: 10 | matrix: 11 | include: 12 | - env: py38 13 | py: '3.8' 14 | - env: py39 15 | py: '3.9' 16 | - env: py310 17 | py: '3.10' 18 | - env: py311 19 | py: '3.11' 20 | - env: pypy3.9 21 | py: 'pypy-3.9' 22 | steps: 23 | - uses: actions/checkout@v3 24 | - uses: actions/setup-python@v4 25 | with: 26 | python-version: ${{ matrix.py }} 27 | - run: pip install tox 28 | - run: tox -e ${{ matrix.env }} 29 | -------------------------------------------------------------------------------- /.github/workflows/pypi.yml: -------------------------------------------------------------------------------- 1 | name: pypi 2 | 3 | on: 4 | push: 5 | tags: 6 | - v* 7 | branches: 8 | - main 9 | 10 | jobs: 11 | test: 12 | runs-on: ubuntu-latest 13 | strategy: 14 | matrix: 15 | include: 16 | - env: py38 17 | py: '3.8' 18 | - env: py39 19 | py: '3.9' 20 | - env: py310 21 | py: '3.10' 22 | - env: py311 23 | py: '3.11' 24 | - env: pypy3.9 25 | py: 'pypy-3.9' 26 | steps: 27 | - uses: actions/checkout@v3 28 | - uses: actions/setup-python@v4 29 | with: 30 | python-version: ${{ matrix.py }} 31 | - run: pip install tox 32 | - run: tox -e ${{ matrix.env }} 33 | pypi: 34 | needs: test 35 | runs-on: ubuntu-latest 36 | permissions: 37 | id-token: write 38 | steps: 39 | - uses: actions/checkout@v3 40 | - uses: actions/setup-python@v4 41 | with: 42 | python-version: '3.10' 43 | - run: python setup.py sdist 44 | - uses: pypa/gh-action-pypi-publish@v1.10.3 45 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.egg-info 2 | *.pyc 3 | *.pyo 4 | .DS_Store 5 | /.cache 6 | /.coverage 7 | /.tox 8 | /build 9 | /dist 10 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v4.4.0 4 | hooks: 5 | - id: trailing-whitespace 6 | - id: end-of-file-fixer 7 | - id: check-docstring-first 8 | - id: check-yaml 9 | - id: debug-statements 10 | - id: name-tests-test 11 | - id: requirements-txt-fixer 12 | - repo: https://github.com/pycqa/flake8 13 | rev: 6.0.0 14 | hooks: 15 | - id: flake8 16 | - repo: https://github.com/pre-commit/mirrors-autopep8 17 | rev: v2.0.2 18 | hooks: 19 | - id: autopep8 20 | - repo: https://github.com/asottile/reorder_python_imports 21 | rev: v3.9.0 22 | hooks: 23 | - id: reorder-python-imports 24 | args: [--py38-plus] 25 | - repo: https://github.com/asottile/pyupgrade 26 | rev: v3.4.0 27 | hooks: 28 | - id: pyupgrade 29 | args: [--py38-plus] 30 | - repo: https://github.com/asottile/setup-cfg-fmt 31 | rev: v2.2.0 32 | hooks: 33 | - id: setup-cfg-fmt 34 | - repo: https://github.com/pre-commit/mirrors-mypy 35 | rev: v1.3.0 36 | hooks: 37 | - id: mypy 38 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012 by Fireteam Ltd., see AUTHORS for more details. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are 5 | met: 6 | 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above 11 | copyright notice, this list of conditions and the following 12 | disclaimer in the documentation and/or other materials provided 13 | with the distribution. 14 | 15 | * The names of the contributors may not be used to endorse or 16 | promote products derived from this software without specific 17 | prior written permission. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://github.com/Yelp/virtualenv-tools/actions/workflows/main.yml/badge.svg?query=branch%3Amain)](https://github.com/Yelp/virtualenv-tools/actions/workflows/main.yml) 2 | [![Coverage Status](https://img.shields.io/coveralls/Yelp/virtualenv-tools.svg?branch=master)](https://coveralls.io/r/Yelp/virtualenv-tools) 3 | [![PyPI version](https://badge.fury.io/py/virtualenv-tools3.svg)](https://pypi.python.org/pypi/virtualenv-tools3) 4 | 5 | virtualenv-tools3 6 | -------- 7 | 8 | virtualenv-tools3 is a fork of [the original 9 | virtualenv-tools](https://github.com/fireteam/virtualenv-tools) (now 10 | unmaintained) which adds support for Python 3, among other things. Full patch 11 | details are below. 12 | 13 | ## yelp patches 14 | 15 | ### yelp4 16 | 17 | * Add python3 support 18 | * Drop python2.6 support 19 | * 100% test coverage 20 | * Removes `$VENV/local` instead of fixing up symlinks 21 | * Removed `--reinitialize`, instead run `virtualenv $VENV -p $PYTHON` 22 | * Rewrite .pth files to relative paths 23 | 24 | 25 | ### yelp3 26 | 27 | * default output much more concise, added a --verbose option 28 | * improved fault tolerance, in the case of: 29 | * corrupt pyc files 30 | * broken symlinks 31 | * unexpected directories 32 | * no-changes-needed is a success case (idempotency exits 0) 33 | 34 | 35 | ### yelp1 36 | 37 | * --update now works more generally and reliably (e.g. virtualenv --python=python2.7) 38 | 39 | 40 | ## virtualenv-tools 41 | 42 | This repository contains scripts we're using at Fireteam for our 43 | deployment of Python code. We're using them in combination with 44 | salt to build code on one server on a self contained virtualenv 45 | and then move that over to the destination servers to run. 46 | 47 | ### Why not virtualenv --relocatable? 48 | 49 | For starters: because it does not work. relocatable is very 50 | limited in what it does and it works at runtime instead of 51 | making the whole thing actually move to the new location. We 52 | ran into a ton of issues with it and it is currently in the 53 | process of being phased out. 54 | 55 | ### Why would I want to use it? 56 | 57 | The main reason you want to use this is for build caching. You 58 | have one folder where one virtualenv exists, you install the 59 | latest version of your codebase and all extensions in there, then 60 | you can make the virtualenv relocate to a target location, put it 61 | into a tarball, distribute it to all servers and done! 62 | 63 | ### Example flow: 64 | 65 | First time: create the build cache 66 | 67 | ``` 68 | $ mkdir /tmp/build-cache 69 | $ virtualenv --distribute /tmp/build-cache 70 | ``` 71 | 72 | Now every time you build: 73 | 74 | ``` 75 | $ . /tmp/build-cache/bin/activate 76 | $ pip install YourApplication 77 | ``` 78 | 79 | Build done, package up and copy to whatever location you want to have it. 80 | 81 | Once unpacked on the target server, use the virtualenv tools to 82 | update the paths and make the virtualenv magically work in the new 83 | location. For instance we deploy things to a path with the 84 | hash of the commit in: 85 | 86 | ``` 87 | $ virtualenv-tools --update-path /srv/your-application/ 88 | ``` 89 | 90 | Compile once, deploy whereever. Virtualenvs are completely self 91 | contained. In order to switch the current version all you need to 92 | do is to relink the builds. 93 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | covdefaults 2 | coverage 3 | pytest 4 | virtualenv 5 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = virtualenv_tools3 3 | version = 3.1.1 4 | description = A set of tools for virtualenv 5 | long_description = file: README.md 6 | long_description_content_type = text/markdown 7 | url = http://github.com/Yelp/virtualenv-tools 8 | author = Fireteam Ltd.; Yelp, Inc. 9 | author_email = opensource@yelp.com 10 | license_file = LICENSE 11 | classifiers = 12 | License :: OSI Approved :: BSD License 13 | Programming Language :: Python 14 | Programming Language :: Python :: 3 15 | Programming Language :: Python :: 3 :: Only 16 | Programming Language :: Python :: Implementation :: CPython 17 | Programming Language :: Python :: Implementation :: PyPy 18 | 19 | [options] 20 | py_modules = virtualenv_tools 21 | python_requires = >=3.8 22 | 23 | [options.entry_points] 24 | console_scripts = 25 | virtualenv-tools = virtualenv_tools:main 26 | 27 | [coverage:run] 28 | plugins = covdefaults 29 | 30 | [bdist_wheel] 31 | universal = True 32 | 33 | [mypy] 34 | check_untyped_defs = true 35 | disallow_any_generics = true 36 | disallow_incomplete_defs = true 37 | disallow_untyped_defs = true 38 | warn_redundant_casts = true 39 | warn_unused_ignores = true 40 | 41 | [mypy-testing.*] 42 | disallow_untyped_defs = false 43 | 44 | [mypy-tests.*] 45 | disallow_untyped_defs = false 46 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | setup() 3 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/virtualenv-tools/a57fffec596757a8592d56637c2b27819afe9f3d/tests/__init__.py -------------------------------------------------------------------------------- /tests/virtualenv_tools_test.py: -------------------------------------------------------------------------------- 1 | import collections 2 | import platform 3 | import shlex 4 | import subprocess 5 | import sys 6 | 7 | import pytest 8 | 9 | import virtualenv_tools 10 | 11 | 12 | def auto_namedtuple(**kwargs): 13 | return collections.namedtuple('ns', tuple(kwargs))(**kwargs) 14 | 15 | 16 | @pytest.fixture 17 | def venv(tmpdir): 18 | app_before = tmpdir.join('before').ensure_dir() 19 | app_before.join('mymodule.py').write( 20 | "if __name__ == '__main__':\n" 21 | " print('ohai!')\n" 22 | ) 23 | app_before.join('setup.py').write( 24 | 'from setuptools import setup\n' 25 | 'setup(name="mymodule", py_modules=["mymodule"])\n' 26 | ) 27 | venv_before = app_before.join('venv') 28 | app_after = tmpdir.join('after') 29 | venv_after = app_after.join('venv') 30 | 31 | cmd = (sys.executable, '-m', 'virtualenv', venv_before.strpath) 32 | subprocess.check_call(cmd) 33 | subprocess.check_call(( 34 | venv_before.join('bin/pip').strpath, 35 | 'install', '-e', app_before.strpath, 36 | )) 37 | yield auto_namedtuple( 38 | app_before=app_before, app_after=app_after, 39 | before=venv_before, after=venv_after, 40 | ) 41 | 42 | 43 | def run(before, after, args=()): 44 | ret = virtualenv_tools.main( 45 | (before.strpath, f'--update-path={after.strpath}') + args, 46 | ) 47 | assert ret == 0 48 | 49 | 50 | @pytest.mark.parametrize('helpargs', ((), ('--help',))) 51 | def test_help(capsys, helpargs): 52 | with pytest.raises(SystemExit): 53 | virtualenv_tools.main(helpargs) 54 | out, err = capsys.readouterr() 55 | assert 'usage: ' in out + err 56 | 57 | 58 | def test_already_up_to_date(venv, capsys): 59 | run(venv.before, venv.before) 60 | out, _ = capsys.readouterr() 61 | assert out == 'Already up-to-date: {0} ({0})\n'.format(venv.before) 62 | 63 | 64 | def test_bourne_shell_exec(venv, capsys): 65 | venv.before.join('bin').join('bourne.py').write( 66 | "#!/bin/sh\n" 67 | f"""'''exec' {venv.before.strpath}/bin/python "$0" "$@"\n""" 68 | "' '''\n") 69 | 70 | run(venv.before, venv.after) 71 | out, _ = capsys.readouterr() 72 | expected = 'Updated: {0} ({0} -> {1})\n'.format(venv.before, venv.after) 73 | assert out == expected 74 | 75 | 76 | def test_each_part_idempotent(tmpdir, venv, capsys): 77 | activate = venv.before.join('bin/activate') 78 | before_activate_contents = activate.read() 79 | run(venv.before, venv.after) 80 | capsys.readouterr() 81 | # Write the activate file to trick the logic into rerunning 82 | activate.write(before_activate_contents) 83 | run(venv.before, venv.after, args=('--verbose',)) 84 | out, _ = capsys.readouterr() 85 | # Should only update our activate file: 86 | expected = 'A {0}\nUpdated: {1} ({1} -> {2})\n'.format( 87 | activate, venv.before, venv.after, 88 | ) 89 | assert out == expected 90 | 91 | 92 | def _assert_activated_sys_executable(path): 93 | exe = subprocess.check_output(( 94 | 'bash', '-c', 95 | ". {} && python -c 'import sys; print(sys.executable)'".format( 96 | shlex.quote(path.join('bin/activate').strpath), 97 | ) 98 | )).decode().strip() 99 | assert exe == path.join('bin/python').strpath 100 | 101 | 102 | def _assert_mymodule_output(path): 103 | out = subprocess.check_output( 104 | (path.join('bin/python').strpath, '-m', 'mymodule'), 105 | # Run from '/' to ensure we're not importing from . 106 | cwd='/', 107 | ).decode() 108 | assert out == 'ohai!\n' 109 | 110 | 111 | def assert_virtualenv_state(path): 112 | _assert_activated_sys_executable(path) 113 | _assert_mymodule_output(path) 114 | 115 | 116 | def test_move(venv, capsys): 117 | assert_virtualenv_state(venv.before) 118 | run(venv.before, venv.after) 119 | out, _ = capsys.readouterr() 120 | expected = 'Updated: {0} ({0} -> {1})\n'.format(venv.before, venv.after) 121 | assert out == expected 122 | venv.app_before.move(venv.app_after) 123 | assert_virtualenv_state(venv.after) 124 | 125 | 126 | def test_move_non_ascii_script(venv, capsys): 127 | # We have a script with non-ascii bytes which we 128 | # want to install non-editable. 129 | venv.app_before.join('mymodule.py').write_binary( 130 | b"#!/usr/bin/env python\n" 131 | b'"""Copyright: \xc2\xa9 Me"""\n' 132 | b"if __name__ == '__main__':\n" 133 | b" print('ohai!')\n" 134 | ) 135 | venv.app_before.join('setup.py').write( 136 | 'from setuptools import setup\n' 137 | 'setup(' 138 | ' name="mymodule", ' 139 | ' py_modules=["mymodule"], ' 140 | ' scripts=["mymodule.py"], ' 141 | ')\n' 142 | ) 143 | subprocess.check_call(( 144 | venv.before.join('bin/pip').strpath, 145 | 'install', '--upgrade', venv.app_before.strpath, 146 | )) 147 | 148 | assert_virtualenv_state(venv.before) 149 | run(venv.before, venv.after) 150 | out, _ = capsys.readouterr() 151 | expected = 'Updated: {0} ({0} -> {1})\n'.format(venv.before, venv.after) 152 | assert out == expected 153 | venv.app_before.move(venv.app_after) 154 | assert_virtualenv_state(venv.after) 155 | 156 | 157 | def test_move_with_auto(venv, capsys): 158 | venv.app_before.move(venv.app_after) 159 | ret = virtualenv_tools.main(('--update-path=auto', venv.after.strpath)) 160 | out, _ = capsys.readouterr() 161 | expected = 'Updated: {1} ({0} -> {1})\n'.format(venv.before, venv.after) 162 | assert ret == 0 163 | assert out == expected 164 | assert_virtualenv_state(venv.after) 165 | 166 | 167 | if platform.python_implementation() == 'PyPy': # pragma: pypy cover 168 | libdir_fmt = 'lib/pypy{}.{}/' 169 | else: # pragma: pypy no cover 170 | libdir_fmt = 'lib/python{}.{}' 171 | 172 | 173 | def test_bad_pyc(venv, capsys): 174 | libdir = libdir_fmt.format(*sys.version_info[:2]) 175 | bad_pyc = venv.before.join(libdir, 'bad.pyc') 176 | bad_pyc.write_binary(b'I am a very naughty pyc\n') 177 | # Retries on failures as well 178 | for _ in range(2): 179 | with pytest.raises(ValueError): 180 | run(venv.before, venv.after) 181 | out, _ = capsys.readouterr() 182 | assert out == f'Error in {bad_pyc.strpath}\n' 183 | 184 | 185 | def test_dir_oddities(venv): 186 | bindir = venv.before.join('bin') 187 | # A directory existing in the bin dir 188 | bindir.join('im_a_directory').ensure_dir() 189 | # A broken symlink 190 | bindir.join('bad_symlink').mksymlinkto('/i/dont/exist') 191 | # A file with a shebang-looking start, but not actually 192 | bindir.join('not-an-exe').write('#!\nohai') 193 | run(venv.before, venv.after) 194 | 195 | 196 | def test_verbose(venv, capsys): 197 | run(venv.before, venv.after, args=('--verbose',)) 198 | out, _ = capsys.readouterr() 199 | # Lots of output 200 | assert len(out.splitlines()) > 10 201 | 202 | 203 | def test_non_absolute_error(capsys): 204 | ret = virtualenv_tools.main(('--update-path', 'notabs')) 205 | out, _ = capsys.readouterr() 206 | assert ret == 1 207 | assert out == '--update-path must be absolute: notabs\n' 208 | 209 | 210 | @pytest.fixture 211 | def fake_venv_quoted(tmpdir): 212 | tmpdir.join('bin').ensure_dir() 213 | tmpdir.join('lib/python2.7/site-packages').ensure_dir() 214 | tmpdir.join('bin/activate').write('VIRTUAL_ENV="/venv"\n') 215 | yield tmpdir 216 | 217 | 218 | @pytest.fixture 219 | def fake_venv(tmpdir): 220 | tmpdir.join('bin').ensure_dir() 221 | tmpdir.join('lib/python2.7/site-packages').ensure_dir() 222 | tmpdir.join('bin/activate').write('VIRTUAL_ENV=/venv\n') 223 | yield tmpdir 224 | 225 | 226 | def test_not_a_virtualenv_missing_site_packages(fake_venv, capsys): 227 | fake_venv.join('lib/python2.7/site-packages').remove() 228 | ret = virtualenv_tools.main(('--update-path=auto', fake_venv.strpath)) 229 | out, _ = capsys.readouterr() 230 | assert ret == 1 231 | expected = '{} is not a virtualenv: not a directory: {}\n'.format( 232 | fake_venv, fake_venv.join('lib/python2.7/site-packages'), 233 | ) 234 | assert out == expected 235 | 236 | 237 | def test_not_a_virtualenv_missing_bindir(fake_venv, capsys): 238 | fake_venv.join('bin').remove() 239 | ret = virtualenv_tools.main(('--update-path=auto', fake_venv.strpath)) 240 | out, _ = capsys.readouterr() 241 | assert ret == 1 242 | expected = '{} is not a virtualenv: not a directory: {}\n'.format( 243 | fake_venv, fake_venv.join('bin'), 244 | ) 245 | assert out == expected 246 | 247 | 248 | def test_not_a_virtualenv_missing_activate_file(fake_venv, capsys): 249 | fake_venv.join('bin/activate').remove() 250 | ret = virtualenv_tools.main(('--update-path=auto', fake_venv.strpath)) 251 | out, _ = capsys.readouterr() 252 | assert ret == 1 253 | expected = '{} is not a virtualenv: not a file: {}\n'.format( 254 | fake_venv, fake_venv.join('bin/activate'), 255 | ) 256 | assert out == expected 257 | 258 | 259 | def test_not_a_virtualenv_missing_versioned_lib_directory(fake_venv, capsys): 260 | fake_venv.join('lib/python2.7').remove() 261 | ret = virtualenv_tools.main(('--update-path=auto', fake_venv.strpath)) 262 | out, _ = capsys.readouterr() 263 | assert ret == 1 264 | expected = '{} is not a virtualenv: not a directory: {}\n'.format( 265 | fake_venv, fake_venv.join('lib/python#.#'), 266 | ) 267 | assert out == expected 268 | 269 | 270 | def test_virtualenv_path_works_with_quoted_path(fake_venv_quoted, capsys): 271 | ret = virtualenv_tools.main( 272 | ('--update-path=auto', fake_venv_quoted.strpath) 273 | ) 274 | out, _ = capsys.readouterr() 275 | assert ret == 0 276 | assert out.startswith('Updated: ') 277 | assert '/venv ->' in out 278 | 279 | 280 | def test_virtualenv_path_works_with_nonquoted_path(fake_venv, capsys): 281 | ret = virtualenv_tools.main(('--update-path=auto', fake_venv.strpath)) 282 | out, _ = capsys.readouterr() 283 | assert ret == 0 284 | assert out.startswith('Updated: ') 285 | assert '/venv ->' in out 286 | 287 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | project = virtualenv_tools 3 | envlist = py,pypy3 4 | 5 | [testenv] 6 | deps = -rrequirements-dev.txt 7 | commands = 8 | coverage erase 9 | coverage run -m pytest {posargs:tests} 10 | coverage report 11 | 12 | [testenv:venv] 13 | envdir = venv-{[tox]project} 14 | commands = 15 | 16 | [pep8] 17 | ignore = E265,E501,W504 18 | -------------------------------------------------------------------------------- /virtualenv_tools.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | move-virtualenv 4 | ~~~~~~~~~~~~~~~ 5 | 6 | A helper script that moves virtualenvs to a new location. 7 | 8 | It only supports POSIX based virtualenvs and at the moment. 9 | 10 | :copyright: (c) 2012 by Fireteam Ltd. 11 | :license: BSD, see LICENSE for more details. 12 | """ 13 | from __future__ import annotations 14 | 15 | import argparse 16 | import marshal 17 | import os.path 18 | import re 19 | import shlex 20 | import shutil 21 | import sys 22 | from types import CodeType 23 | from typing import NamedTuple 24 | from typing import Sequence 25 | 26 | 27 | ACTIVATION_SCRIPTS = [ 28 | 'activate', 29 | 'activate.csh', 30 | 'activate.fish', 31 | 'activate.xsh', 32 | ] 33 | _pybin_match = re.compile(r'^python\d+\.\d+$') 34 | _pypy_match = re.compile(r'^pypy\d+.\d+$') 35 | _activation_path_re = re.compile( 36 | r'^(?:set -gx |setenv |)VIRTUAL_ENV[ =][\'"]?(.*?)[\'"]?\s*$', 37 | ) 38 | VERBOSE = False 39 | # magic length 40 | # + 4 byte timestamp 41 | # + 4 byte "size" hint was added to pyc files 42 | # PEP 552 (implemented in python 3.7) extends this by another word 43 | MAGIC_LENGTH = 4 + 4 + 4 + 4 44 | 45 | 46 | def debug(msg: str) -> None: 47 | if VERBOSE: 48 | print(msg) 49 | 50 | 51 | def update_activation_script(script_filename: str, new_path: str) -> None: 52 | """Updates the paths for the activate shell scripts.""" 53 | with open(script_filename) as f: 54 | lines = list(f) 55 | 56 | def _handle_sub(match: re.Match[str]) -> str: 57 | text = match.group() 58 | start, end = match.span() 59 | g_start, g_end = match.span(1) 60 | return text[:(g_start - start)] + new_path + text[(g_end - end):] 61 | 62 | changed = False 63 | for idx, line in enumerate(lines): 64 | new_line = _activation_path_re.sub(_handle_sub, line) 65 | if line != new_line: 66 | lines[idx] = new_line 67 | changed = True 68 | 69 | if changed: 70 | debug('A %s' % script_filename) 71 | with open(script_filename, 'w') as f: 72 | f.writelines(lines) 73 | 74 | 75 | def path_is_within(path: bytes, within: bytes) -> bool: 76 | relpath = os.path.relpath(path, within) 77 | return not relpath.startswith(b'.') 78 | 79 | 80 | def update_script( 81 | script_filename: str, 82 | old_path_s: str, 83 | new_path_s: str, 84 | ) -> None: 85 | """Updates shebang lines for actual scripts.""" 86 | filesystem_encoding = sys.getfilesystemencoding() 87 | old_path = old_path_s.encode(filesystem_encoding) 88 | new_path = new_path_s.encode(filesystem_encoding) 89 | 90 | with open(script_filename, 'rb') as f: 91 | if f.read(2) != b'#!': 92 | return 93 | f.seek(0) 94 | lines = list(f) 95 | 96 | # is this a python script being run under a bourne exec call 97 | if ( 98 | len(lines) >= 2 and 99 | lines[0] == b'#!/bin/sh\n' and 100 | lines[1].startswith(b"'''exec' ") 101 | ): 102 | args = lines[1].strip().split() 103 | 104 | if path_is_within(args[1], old_path): 105 | new_bin = os.path.join( 106 | new_path, 107 | os.path.relpath(args[1], old_path) 108 | ) 109 | else: 110 | return 111 | 112 | args[1] = new_bin 113 | lines[1] = b' '.join(args) + b'\n' 114 | else: 115 | args = lines[0][2:].strip().split() 116 | 117 | if not args: 118 | return 119 | 120 | if path_is_within(args[0], old_path): 121 | new_bin = os.path.join( 122 | new_path, 123 | os.path.relpath(args[0], old_path) 124 | ) 125 | else: 126 | return 127 | 128 | args[0] = new_bin 129 | lines[0] = b'#!' + b' '.join(args) + b'\n' 130 | 131 | debug('S %s' % script_filename) 132 | with open(script_filename, 'wb') as f: 133 | f.writelines(lines) 134 | 135 | 136 | def update_scripts( 137 | bin_dir: str, 138 | orig_path: str, 139 | new_path: str, 140 | activation: bool = False, 141 | ) -> None: 142 | """Updates all scripts in the bin folder.""" 143 | for fname in os.listdir(bin_dir): 144 | path = os.path.join(bin_dir, fname) 145 | if fname in ACTIVATION_SCRIPTS and activation: 146 | update_activation_script(path, new_path) 147 | elif os.path.isfile(path): 148 | update_script(path, orig_path, new_path) 149 | 150 | 151 | def update_pyc(filename: str, new_path: str) -> None: 152 | """Updates the filenames stored in pyc files.""" 153 | with open(filename, 'rb') as rf: 154 | magic = rf.read(MAGIC_LENGTH) 155 | try: 156 | code = marshal.load(rf) 157 | except Exception: 158 | print('Error in %s' % filename) 159 | raise 160 | 161 | def _process(code: CodeType) -> CodeType: 162 | consts = [] 163 | for const in code.co_consts: 164 | if type(const) is CodeType: 165 | const = _process(const) 166 | consts.append(const) 167 | if new_path != code.co_filename or consts != list(code.co_consts): 168 | code = code.replace(co_filename=new_path, co_consts=tuple(consts)) 169 | return code 170 | 171 | new_code = _process(code) 172 | 173 | if new_code is not code: 174 | debug('B %s' % filename) 175 | with open(filename, 'wb') as wf: 176 | wf.write(magic) 177 | marshal.dump(new_code, wf) 178 | 179 | 180 | def update_pycs(lib_dir: str, new_path: str) -> None: 181 | """Walks over all pyc files and updates their paths.""" 182 | def get_new_path(filename: str) -> str: 183 | filename = os.path.normpath(filename) 184 | return os.path.join(new_path, filename[len(lib_dir) + 1:]) 185 | 186 | for dirname, dirnames, filenames in os.walk(lib_dir): 187 | for filename in filenames: 188 | if ( 189 | filename.endswith(('.pyc', '.pyo')) and 190 | # python 2, virtualenv 20.x symlinks os.pyc 191 | not os.path.islink(os.path.join(dirname, filename)) 192 | ): 193 | filename = os.path.join(dirname, filename) 194 | local_path = get_new_path(filename) 195 | update_pyc(filename, local_path) 196 | 197 | 198 | def _update_pth_file(pth_filename: str, orig_path: str) -> None: 199 | with open(pth_filename) as f: 200 | lines = f.readlines() 201 | changed = False 202 | for i, line in enumerate(lines): 203 | val = line.strip() 204 | if val.startswith('import ') or not os.path.isabs(val): 205 | continue 206 | changed = True 207 | relto_original = os.path.relpath(val, orig_path) 208 | # If we are moving a pypy venv the site-packages directory 209 | # is in a different location than if we are moving a cpython venv 210 | relto_pth = os.path.join( 211 | '../../..', # venv/lib/pythonX.X/site-packages 212 | relto_original 213 | ) 214 | lines[i] = f'{relto_pth}\n' 215 | if changed: 216 | with open(pth_filename, 'w') as f: 217 | f.write(''.join(lines)) 218 | debug(f'P {pth_filename}') 219 | 220 | 221 | def update_pth_files(site_packages: str, orig_path: str) -> None: 222 | """Converts /full/paths in pth files to relative relocatable paths.""" 223 | for filename in os.listdir(site_packages): 224 | filename = os.path.join(site_packages, filename) 225 | if filename.endswith('.pth') and os.path.isfile(filename): 226 | _update_pth_file(filename, orig_path) 227 | 228 | 229 | def remove_local(base: str) -> None: 230 | """On some systems virtualenv seems to have something like a local 231 | directory with symlinks. This directory is safe to remove in modern 232 | versions of virtualenv. Delete it. 233 | """ 234 | local_dir = os.path.join(base, 'local') 235 | if os.path.exists(local_dir): # pragma: no cover (not all systems) 236 | debug(f'D {local_dir}') 237 | shutil.rmtree(local_dir) 238 | 239 | 240 | def update_paths(venv: Virtualenv, new_path: str) -> None: 241 | """Updates all paths in a virtualenv to a new one.""" 242 | update_scripts(venv.bin_dir, venv.orig_path, new_path) 243 | for lib_dir in venv.lib_dirs: 244 | update_pycs(lib_dir, new_path) 245 | update_pth_files(venv.site_packages, venv.orig_path) 246 | remove_local(venv.path) 247 | update_scripts(venv.bin_dir, venv.orig_path, new_path, activation=True) 248 | 249 | 250 | def get_orig_path(venv_path: str) -> str: 251 | """This helps us know whether someone has tried to relocate the 252 | virtualenv 253 | """ 254 | activate_path = os.path.join(venv_path, 'bin/activate') 255 | 256 | with open(activate_path) as activate: 257 | venv_var_prefix = 'VIRTUAL_ENV=' 258 | for line in activate: 259 | # virtualenv 20 changes the position 260 | if line.startswith(venv_var_prefix): 261 | return shlex.split(line[len(venv_var_prefix):])[0] 262 | else: 263 | raise AssertionError( 264 | 'Could not find VIRTUAL_ENV= in activation script: %s' % 265 | activate_path 266 | ) 267 | 268 | 269 | class NotAVirtualenvError(ValueError): 270 | def __str__(self) -> str: 271 | return '{} is not a virtualenv: not a {}: {}'.format(*self.args) 272 | 273 | 274 | class Virtualenv(NamedTuple): 275 | path: str 276 | bin_dir: str 277 | lib_dirs: list[str] 278 | site_packages: str 279 | orig_path: str 280 | 281 | 282 | def _get_original_state(path: str) -> Virtualenv: 283 | is_pypy = os.path.isfile(os.path.join(path, 'bin', 'pypy')) 284 | bin_dir = os.path.join(path, 'bin') 285 | base_lib_dir = os.path.join(path, 'lib') 286 | activate_file = os.path.join(bin_dir, 'activate') 287 | 288 | for dir_path in (bin_dir, base_lib_dir): 289 | if not os.path.isdir(dir_path): 290 | raise NotAVirtualenvError(path, 'directory', dir_path) 291 | if not os.path.isfile(activate_file): 292 | raise NotAVirtualenvError(path, 'file', activate_file) 293 | 294 | matcher = _pypy_match if is_pypy else _pybin_match 295 | lib_dirs = [ 296 | os.path.join(base_lib_dir, potential_lib_dir) 297 | for potential_lib_dir in os.listdir(base_lib_dir) 298 | if matcher.match(potential_lib_dir) 299 | ] 300 | if len(lib_dirs) != 1: 301 | raise NotAVirtualenvError( 302 | path, 303 | 'directory', 304 | os.path.join(base_lib_dir, 'pypy#.#)' if is_pypy else 'python#.#'), 305 | ) 306 | lib_dir, = lib_dirs 307 | 308 | site_packages = os.path.join(lib_dir, 'site-packages') 309 | if not os.path.isdir(site_packages): 310 | raise NotAVirtualenvError(path, 'directory', site_packages) 311 | 312 | return Virtualenv( 313 | path=path, 314 | bin_dir=bin_dir, 315 | lib_dirs=[lib_dir], 316 | site_packages=site_packages, 317 | orig_path=get_orig_path(path), 318 | ) 319 | 320 | 321 | def main(argv: Sequence[str] | None = None) -> int: 322 | parser = argparse.ArgumentParser() 323 | parser.add_argument( 324 | '--update-path', 325 | required=True, 326 | help=( 327 | 'Update the path for all required executables and helper files ' 328 | 'that are supported to the new python prefix. You can also set ' 329 | 'this to "auto" for autodetection.' 330 | ), 331 | ) 332 | parser.add_argument( 333 | '--verbose', action='store_true', help='show a listing of changes', 334 | ) 335 | parser.add_argument('path', default='.', nargs='?') 336 | args = parser.parse_args(argv) 337 | 338 | global VERBOSE 339 | VERBOSE = args.verbose 340 | 341 | if args.update_path == 'auto': 342 | update_path = os.path.abspath(args.path) 343 | else: 344 | update_path = args.update_path 345 | 346 | if not os.path.isabs(update_path): 347 | print(f'--update-path must be absolute: {update_path}') 348 | return 1 349 | 350 | try: 351 | venv = _get_original_state(path=args.path) 352 | except NotAVirtualenvError as e: 353 | print(e) 354 | return 1 355 | 356 | if venv.orig_path == update_path: 357 | print(f'Already up-to-date: {venv.path} ({update_path})') 358 | return 0 359 | 360 | update_paths(venv, update_path) 361 | print(f'Updated: {venv.path} ({venv.orig_path} -> {update_path})') 362 | return 0 363 | 364 | 365 | if __name__ == '__main__': 366 | raise SystemExit(main()) 367 | --------------------------------------------------------------------------------