├── .activate.sh ├── .coveragerc ├── .deactivate.sh ├── .github └── workflows │ └── ci.yml ├── .gitignore ├── .pre-commit-config.yaml ├── BUGS.md ├── CI ├── coverage └── requirements.txt ├── LICENSE ├── Makefile ├── README.md ├── RELEASE.md ├── TODO.md ├── benchmark ├── .gitignore ├── cold ├── env ├── faster.log ├── main ├── noop ├── results ├── setup ├── vanilla.log └── warm ├── docs ├── .gitignore ├── Makefile └── source │ ├── _static │ ├── custom.css │ └── venv-update.svg │ ├── _templates │ ├── about.html │ └── layout.html │ ├── benchmarks.rst │ ├── conf.py │ ├── favicon.ico │ ├── index.rst │ ├── internal-pypi.rst │ ├── pip-faster.rst │ └── venv-update.rst ├── pip_faster.py ├── pytest.ini ├── requirements.d ├── CI.txt ├── _lint.txt ├── coverage.txt ├── dev.txt ├── docs.txt ├── import_tests.txt ├── lint.txt └── test.txt ├── requirements.txt ├── setup.py ├── test ├── tests ├── conftest.py ├── functional │ ├── __init__.py │ ├── args.py │ ├── conflict_test.py │ ├── faster.py │ ├── get_installed_test.py │ ├── pip_faster.py │ ├── relocation_test.py │ ├── simple_test.py │ └── validation.py ├── regression │ ├── __init__.py │ └── pip_faster.py ├── testing │ ├── __init__.py │ ├── capture_subprocess.py │ ├── capture_subprocess_test.py │ ├── fix_coverage.py │ ├── make_sdists.py │ ├── packages │ │ ├── Weird-casing_pacKAGE │ │ │ ├── README │ │ │ ├── setup.py │ │ │ └── weird_casing_package.py │ │ ├── cant_wheel_package │ │ │ ├── README │ │ │ └── setup.py │ │ ├── circular-dep-a │ │ │ ├── README │ │ │ └── setup.py │ │ ├── circular-dep-b │ │ │ ├── README │ │ │ └── setup.py │ │ ├── conflicting_package │ │ │ ├── README │ │ │ └── setup.py │ │ ├── dependant_package │ │ │ ├── README │ │ │ └── setup.py │ │ ├── dotted_package_name │ │ │ ├── README │ │ │ ├── dotted_package_name.py │ │ │ ├── setup.py │ │ │ └── wheelme │ │ ├── implicit_dependency │ │ │ ├── README │ │ │ └── setup.py │ │ ├── many_versions_package_1 │ │ │ ├── README │ │ │ └── setup.py │ │ ├── many_versions_package_2.1 │ │ │ ├── README │ │ │ └── setup.py │ │ ├── many_versions_package_2 │ │ │ ├── README │ │ │ └── setup.py │ │ ├── many_versions_package_3 │ │ │ ├── README │ │ │ └── setup.py │ │ ├── many_versions_package_4 │ │ │ ├── README │ │ │ └── setup.py │ │ ├── project_with_c │ │ │ ├── README │ │ │ ├── project_with_c.c │ │ │ └── setup.py │ │ ├── pure_python_package │ │ │ ├── README │ │ │ ├── pure_python_package.py │ │ │ └── setup.py │ │ ├── pure_python_package_2 │ │ │ ├── README │ │ │ ├── pure_python_package.py │ │ │ └── setup.py │ │ ├── slow_python_package │ │ │ ├── README │ │ │ ├── setup.py │ │ │ └── slow_python_package.py │ │ └── wheeled_package │ │ │ ├── README │ │ │ ├── setup.py │ │ │ └── wheelme │ ├── python_lib.py │ └── python_lib_test.py └── unit │ ├── __init__.py │ ├── fix_coverage_test.py │ ├── patch.py │ └── simple_test.py ├── tox.ini ├── venv_update.py └── wheelme /.activate.sh: -------------------------------------------------------------------------------- 1 | venv/bin/activate -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | parallel = True 3 | branch = True 4 | source = 5 | $TOP/tests 6 | venv_update 7 | pip_faster 8 | data_file = $TOP/.coverage 9 | 10 | [report] 11 | exclude_lines = 12 | # Have to re-enable the standard pragma 13 | \#.*:pragma:nocover: 14 | \#.*never returns 15 | \#.*doesn't return 16 | 17 | # Don't complain if tests don't hit defensive assertion code: 18 | ^\s*raise AssertionError\b 19 | ^\s*raise NotImplementedError\b 20 | ^\s*return NotImplemented\b 21 | 22 | # Don't complain if tests don't hit re-raise of unexpected errors: 23 | ^\s*raise$ 24 | 25 | # if main is covered, we're good: 26 | ^\s*exit\(main\(\)\)$ 27 | 28 | partial_branches = 29 | \#.*:pragma:nobranch: 30 | show_missing = True 31 | 32 | [html] 33 | directory = $TOP/coverage-html 34 | 35 | # vim:ft=dosini 36 | -------------------------------------------------------------------------------- /.deactivate.sh: -------------------------------------------------------------------------------- 1 | deactivate 2 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: venv-update-ci 3 | on: push 4 | jobs: 5 | lint: 6 | runs-on: ubuntu-18.04 7 | steps: 8 | - uses: actions/checkout@v2 9 | - uses: actions/setup-python@v2 10 | with: 11 | python-version: 3.6 12 | - run: pip install tox==3.21.2 13 | - run: pip install -r CI/requirements.txt 14 | - run: tox -e lint 15 | py36: 16 | runs-on: ubuntu-18.04 17 | steps: 18 | - uses: actions/checkout@v2 19 | - uses: actions/setup-python@v2 20 | with: 21 | python-version: 3.6 22 | - run: pip install tox==3.21.2 23 | - run: pip install -r CI/requirements.txt 24 | - run: tox -e py36 25 | py27: 26 | runs-on: ubuntu-18.04 27 | steps: 28 | - uses: actions/checkout@v2 29 | - uses: actions/setup-python@v2 30 | with: 31 | python-version: 3.6 32 | - run: pip install tox==3.21.2 33 | - run: pip install -r CI/requirements.txt 34 | - run: tox -e py27 35 | pypy3: 36 | runs-on: ubuntu-18.04 37 | steps: 38 | - uses: actions/checkout@v2 39 | - uses: actions/setup-python@v2 40 | with: 41 | python-version: pypy-3.6 42 | - run: pip install tox==3.21.2 43 | - run: pip install -r CI/requirements.txt 44 | - run: tox -e pypy3 45 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # only ignored in this directory, specifically 2 | /build/ 3 | /.coverage* 4 | /.idea 5 | /.project 6 | /.pydevproject 7 | /.tox 8 | /coverage-html 9 | /dist 10 | /tmp*/ 11 | /venv/ 12 | /.pytest_cache 13 | 14 | # ignored anywhere 15 | *.egg-info/ 16 | *.iml 17 | *.py[co] 18 | .*.sw[a-z] 19 | *.tmp 20 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v2.1.0 4 | hooks: 5 | - id: check-added-large-files 6 | - id: check-case-conflict 7 | - id: check-docstring-first 8 | - id: check-yaml 9 | - id: debug-statements 10 | - id: detect-private-key 11 | - id: double-quote-string-fixer 12 | - id: end-of-file-fixer 13 | - repo: https://gitlab.com/pycqa/flake8 14 | rev: 3.7.4 15 | hooks: 16 | - id: flake8 17 | exclude: ^docs/ 18 | - repo: https://github.com/pre-commit/mirrors-autopep8 19 | rev: v1.4.3 20 | hooks: 21 | - id: autopep8 22 | - repo: https://github.com/asottile/reorder_python_imports 23 | rev: v1.3.5 24 | hooks: 25 | - id: reorder-python-imports 26 | args: [ 27 | --add-import, 'from __future__ import absolute_import', 28 | --add-import, 'from __future__ import print_function', 29 | --add-import, 'from __future__ import unicode_literals', 30 | --application-directories, '.:tests', 31 | ] 32 | - repo: https://github.com/asottile/pyupgrade 33 | rev: v1.11.1 34 | hooks: 35 | - id: pyupgrade 36 | -------------------------------------------------------------------------------- /BUGS.md: -------------------------------------------------------------------------------- 1 | This is a simple listing of bugs previously encountered: 2 | 3 | Known Bugs 4 | ============ 5 | This is just a place to brain-dump bugs I've found so I don't go insane trying to remember them. 6 | It's much lighter weight than filing tickets, and I like that it's version controlled. 7 | 8 | (none, at the moment) 9 | 10 | 11 | Annoyances 12 | ========== 13 | 14 | * `capture_subprocess` doesn't properly proxy tty input/output. 15 | This means I can't simply insert a `import pudb; pudb.set_trace` to debug tests. 16 | see: https://github.com/bukzor/ptyproxy 17 | 18 | * CI sometimes craps out with "pypi server never became ready!" 19 | I've never had this happen locally. 20 | 21 | * CI sometimes fails with missing coverage. 22 | Looking closely, some of the test workers entirely failed to report in. 23 | I've never had this happen locally, although I can sometimes 24 | reproduce it in a io-constrained docker container. 25 | 26 | 27 | Fixed, Not Tested 28 | ================= 29 | 30 | * venv-update can `rm -rf .`, if '.' is its first argument. 31 | Fix: check for $DEST/bin/python before removal 32 | 33 | * venv-update shows the same `> virtualenv venv` line twice in a row 34 | 35 | * the first venv-update fails with "filename too long" in a download cache file, 36 | but subsequent run succeeds 37 | TESTCASE: add a super-extra-really-obscenely-long-named-package 38 | FIX: don't set download-cache within pip-faster. i dont think it was speeding anything up. 39 | the wheels are what matter. 40 | 41 | * if the "outer" pip is >6, installing pip1.5 shows "a valid SSLContext is not available" and 42 | "a newer pip is available" we can suppress these with PIP_NO_PIP_VERSION_CHECK and 43 | python -W 'ignore' 44 | 45 | 46 | Magically Fixed 47 | =============== 48 | 49 | * `print 1; print 2` is coming from somewhere during py.test -s 50 | 51 | * Explosion when argparse is not installed: 52 | 53 | $ pip-faster install argparse 54 | Traceback (most recent call last): 55 | File "/nail/home/buck/trees/yelp/pip-faster/venv/bin/pip-faster", line 5, in 56 | from pkg_resources import load_entry_point 57 | File "/nail/home/buck/trees/yelp/pip-faster/venv/lib/python2.6/site-packages/pkg_resources.py", line 2749, in 58 | working_set = WorkingSet._build_master() 59 | File "/nail/home/buck/trees/yelp/pip-faster/venv/lib/python2.6/site-packages/pkg_resources.py", line 444, in _build_master 60 | ws.require(__requires__) 61 | File "/nail/home/buck/trees/yelp/pip-faster/venv/lib/python2.6/site-packages/pkg_resources.py", line 725, in require 62 | needed = self.resolve(parse_requirements(requirements)) 63 | File "/nail/home/buck/trees/yelp/pip-faster/venv/lib/python2.6/site-packages/pkg_resources.py", line 628, in resolve 64 | raise DistributionNotFound(req) 65 | pkg_resources.DistributionNotFound: argparse 66 | 67 | Fixed and Tested 68 | ================ 69 | 70 | pip-faster install: 71 | 72 | * Explosion on "Requirement already satisfied" -- `'InstallRequirement' object has no attribute 'best_installed'` 73 | Fix: factor out the best_installed attribute entirely -- yay 74 | 75 | * install incurs build time twice 76 | Fix: nasty hack to remove non-wheel source and replace with unzipped wheel 77 | 78 | * wheel-install can install a prior version 79 | Fix: only do the wheel search with pinned requirements 80 | 81 | * Cause: a prior prune uninstalled argparse, but pip-faster depends on it, transitively, via wheel 82 | Planned fix: for the purposes of pruning, pip-faster should be added to the list of requirements 83 | Stopgap fix: whitelist argparse along with pip-faster, pip, setuptools, and wheel to never be pruned 84 | 85 | test: 86 | * during make-sdists, the setup.py for pip-faster went missing, once 87 | Cause: parallel test fixtures were stomping on each others' egg-info 88 | Fix: set a --egg-dir for egg-info 89 | -------------------------------------------------------------------------------- /CI/coverage: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -eux 3 | export TOP=$PWD 4 | 5 | coverage xml 6 | 7 | pip-faster install codecov 8 | codecov -e TOXENV --file coverage.xml 9 | -------------------------------------------------------------------------------- /CI/requirements.txt: -------------------------------------------------------------------------------- 1 | # minimal set of packages for github actions to run our test suites 2 | codecov 3 | tox==3.21.2 4 | virtualenv>=20.0.8 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Yelp 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | export PYTHON?=python3.6 2 | export REQUIREMENTS?=requirements.txt 3 | 4 | .PHONY: all 5 | all: lint test 6 | 7 | .PHONY: lint 8 | lint: venv 9 | venv/bin/pre-commit install -f --install-hooks 10 | venv/bin/pre-commit run --all-files 11 | 12 | .PHONY: test tests 13 | test tests: venv 14 | . venv/bin/activate && ./test $(ARGS) 15 | 16 | venv: setup.py requirements.txt requirements.d/* Makefile 17 | ./venv_update.py venv= --python=$(PYTHON) venv install= -r $(REQUIREMENTS) bootstrap-deps= -e . 18 | 19 | .PHONY: docs 20 | docs: venv 21 | make -C docs serve 22 | 23 | .PHONY: clean 24 | clean: 25 | rm -rf .tox 26 | find -name '*.pyc' -print0 | xargs -0 -r -P4 rm 27 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | venv-update 2 | =========== 3 | Quickly and exactly synchronize a large python project's virtualenv with its 4 | [requirements](https://pip.pypa.io/en/stable/user_guide/#requirements-files). 5 | 6 | [![PyPI version](https://badge.fury.io/py/venv-update.svg)](https://pypi.python.org/pypi/venv-update) 7 | [![Documentation](https://readthedocs.org/projects/venv-update/badge/?version=master)](http://venv-update.readthedocs.org/en/master/) 8 | 9 | 10 | Please see http://venv-update.readthedocs.org/en/master/ for the complete documentation. 11 | 12 | 13 | How to Contribute 14 | ----------------- 15 | 16 | 1. Fork this repository on github: https://help.github.com/articles/fork-a-repo/ 17 | 2. Clone it: https://help.github.com/articles/cloning-a-repository/ 18 | 3. Make a feature branch for your changes: 19 | 20 | git remote add upstream https://github.com/Yelp/venv-update.git 21 | git fetch upstream 22 | git checkout upstream/master -b my-feature-branch 23 | 24 | 4. Make sure the test suite works before you start: 25 | 26 | source .activate.sh 27 | make test 28 | 29 | 5. Commit patches: http://gitref.org/basic/ 30 | 6. Push to github: `git pull && git push origin` 31 | 7. Send a pull request: https://help.github.com/articles/creating-a-pull-request/ 32 | 33 | 34 | ### Running tests: ### 35 | 36 | Run a particular test: 37 | 38 | py.test tests/functional/simple_test.py::test_downgrade 39 | 40 | 41 | See all output from a test: 42 | 43 | py.test -s -k downgrade 44 | 45 | 46 | Check coverage of a single test: 47 | 48 | ./test tests/functional/simple_test.py::test_downgrade 49 | 50 | 51 | Yelpers 52 | ======= 53 | To develop and run tests suites on a devbox, make sure to: 54 | 55 | 1. Python 3.6.0 on a xenial devbox breaks coverage. Use a bionic devbox instead. 56 | 57 | 2. Override pip.conf to use public pypi. Don't forget to delete it after you're done! 58 | ``` 59 | $ cat ~/.pip/pip.conf 60 | [global] 61 | index-url = https://pypi.org/simple/ 62 | ``` 63 | 64 | 3. `sudo apt-get install pypy-dev` so TOXENV=pypy doesn't fail spectacularly 65 | -------------------------------------------------------------------------------- /RELEASE.md: -------------------------------------------------------------------------------- 1 | **NOTE:** This guide is only useful for the owners of the venv-update project. 2 | 3 | 1. start on the latest master 4 | 1. bump `venv_update.py` 5 | 1. bump the URL in the `docs/source/index.rst` curl command with the new 6 | version tag 7 | 1. `git commit -m "this is {{version}}"` 8 | 1. `git tag v{{version}}` 9 | 1. `git push origin master --tags` 10 | 1. upload to pypi 11 | 1. if you need to set up pypi auth, `python setup.py register` and follow the prompts 12 | 1. `python setup.py sdist bdist_wheel` 13 | 1. `twine upload --skip-existing dist/*` 14 | 1. `fetch-python-package venv-update` -- upload to pypi.yelpcorp 15 | -------------------------------------------------------------------------------- /TODO.md: -------------------------------------------------------------------------------- 1 | NEXT 2 | ==== 3 | We plan to work on these in our next version. 4 | 5 | * support for `pip>6` 6 | 7 | 8 | 9 | BACKLOG 10 | ======= 11 | 12 | (none, at the moment) 13 | 14 | 15 | ARCHIVE 16 | ======= 17 | 18 | v1.0: tox support 19 | ----------------- 20 | 21 | 22 | * venv-update reports its *own* version 23 | 24 | - [x] testing 25 | - [x] PIP\_FIND\_LINKS=https://... 26 | - [x] pip-faster install nonsense 27 | 28 | 29 | - [x] dogfood venv-update during travis, tox 30 | - [x] recommended tox config: `install_command=pip-faster --prune {opts} {packages}` 31 | 32 | * change host-python requirement to simply virtualenv, any version 33 | 34 | - [x] missing tests 35 | - [x] non-default requirements file(s) 36 | - [x] run from virtualenv which doesn't have virtualenv installed 37 | - [x] update an active virtualenv which wasn't created by venv-update 38 | 39 | * fixed upgrade-install of unpinned requirements is ~50% slower than vanilla pip. 40 | compare: 41 | 42 | - time pip install --find-links file://$HOME/.pip/wheelhouse --upgrade -r 43 | requirements.d/dev.txt -vvvv > fast 44 | 45 | real 0m3.107s 46 | user 0m2.424s 47 | sys 0m0.427s 48 | 49 | - time make venv > slow 50 | 51 | real 0m4.870s 52 | user 0m4.021s 53 | sys 0m0.631s 54 | 55 | * change host-python requirement to simply virtualenv, any version 56 | 57 | * STRETCH GOAL: pip6 support 58 | 59 | 60 | v2.0: stuff we really want 61 | -------------------------- 62 | 63 | * pip6 support 64 | 65 | * test against select older virtualenv(pip) versions 66 | 67 | * speed up the (git|hg)+ case. `pip install` is currently much faster. 68 | 69 | * new requirement: move the venv, still be able to us `./bin/thatscript` and `source activate && thatscript` 70 | 71 | 72 | 73 | LATER: Things that I want to do, but would put me past my deadline: 74 | ------------------------------------------------------------ 75 | 76 | * coverage: 105, 124, 135, 184, 242, 281, 285-292, 297-298, 326-328, 461, 465 77 | 78 | * populate wheels into the cache *during* build (rather than separate step, beforehand. 79 | This would shave 5s off all scenarios. 80 | see: https://github.com/pypa/pip/issues/2140 81 | see also: https://github.com/pypa/pip/pull/1572 82 | 83 | * On ubuntu stock python2.7 I have to rm -rf $VIRTUAL_ENV/local 84 | to avoid the AssertionError('unexpected third case') 85 | https://bitbucket.org/ned/coveragepy/issue/340/keyerror-subpy 86 | 87 | * I could remove the --cov-config with a small patch to pytest-cov 88 | use os.path.abspath(options.cov_config) in postprocess(options) 89 | 90 | * coverage.py adds some helpful warnings to stderr, with no way to quiet them. 91 | there's already an issue (#2 i think?), just needs a patch 92 | 93 | * pytest-timeout won't actually time-out with floating-point, zero, or negative input 94 | 95 | * Make doubly sure these test scenarios are covered: 96 | * each of these should behave similarly whether caused by the user 97 | (mucking between venv-updates) or the requirements file: 98 | * upgrade 99 | * downgrade 100 | * add 101 | * delete 102 | 103 | * Go through all my forks/branches and see how close i can get back to master 104 | some of this stuff has been merged 105 | wait till March 2014 106 | 107 | * pip micro-bug: pip wheel unzips found wheels even with --no-deps 108 | 109 | * make a fixture containing the necessary wheels for the enable_coverage function 110 | 111 | * pytest-timeout is causing flakiness: 112 | https://bitbucket.org/flub/pytest-timeout/issue/8/internalerror-valueerror-signal-only-works 113 | 114 | * get README.md working as the long_description on pypi 115 | 116 | * try out pbr (?) 117 | http://docs.openstack.org/developer/pbr/#usage 118 | -------------------------------------------------------------------------------- /benchmark/.gitignore: -------------------------------------------------------------------------------- 1 | requirements.txt 2 | cache/ 3 | venv/ 4 | -------------------------------------------------------------------------------- /benchmark/cold: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eux 3 | 4 | rm -rf venv cache 5 | virtualenv venv 6 | source ./setup 7 | 8 | # `--pre` see: https://github.com/pypa/pip/issues/3511 9 | time "$PIP" install --pre plone 10 | "$PIP" freeze | sort > requirements.txt 11 | cat requirements.txt 12 | -------------------------------------------------------------------------------- /benchmark/env: -------------------------------------------------------------------------------- 1 | #!/not/executable/bash 2 | export HOMEBREW=$HOME/prefices/brew 3 | export LIBRARY_PATH=$HOMEBREW/lib:$HOMEBREW/lib64 4 | export CPATH=$HOMEBREW/include 5 | export XDG_CACHE_HOME=$PWD/cache 6 | -------------------------------------------------------------------------------- /benchmark/main: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eu 3 | 4 | now() { 5 | date +"%F %T.%N" 6 | } 7 | 8 | 9 | onetest() { 10 | TEST="$1" 11 | echo 12 | echo START -- "$TEST" 13 | ./"$TEST" 14 | echo END -- "$TEST" 15 | } 16 | 17 | alltests() { 18 | now 19 | time onetest cold 20 | now 21 | time onetest noop 22 | now 23 | time onetest warm 24 | now 25 | } 26 | 27 | export PIP="pip" 28 | export INSTALL="pip" 29 | (time alltests) 2>&1 | tee vanilla.log 30 | 31 | export PIP="pip-faster" 32 | export INSTALL="-e .." 33 | (time alltests) 2>&1 | tee faster.log 34 | -------------------------------------------------------------------------------- /benchmark/noop: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eux 3 | 4 | source ./setup 5 | 6 | time "$PIP" install --upgrade -r requirements.txt 7 | 8 | # assert that we made a noop 9 | diff -us requirements.txt <("$PIP" freeze | sort) 10 | -------------------------------------------------------------------------------- /benchmark/results: -------------------------------------------------------------------------------- 1 | benchmark: installing plone and its dependencies (260 packages) 2 | last run: 2016-02-24 3 | 4 | pip 8.0.2: 5 | cold: 6 | 4m37.612s 7 | 4m39.762s 8 | 4m39.717s 9 | noop: 10 | 0m6.890s 11 | 0m7.112s 12 | 0m7.436s 13 | warm: 14 | 0m44.684s 15 | 0m44.614s 16 | 0m43.272s 17 | 18 | 19 | pip-faster: 20 | cold: 21 | 4m16.163s 22 | 4m21.282s 23 | 4m14.038s 24 | noop: 25 | 0m2.399s 26 | 0m2.389s 27 | 0m2.335s 28 | warm: 29 | 0m30.410s 30 | 0m21.303s 31 | 0m21.323s 32 | -------------------------------------------------------------------------------- /benchmark/setup: -------------------------------------------------------------------------------- 1 | #!/not/executable/bash 2 | source ./env 3 | set +ux 4 | source ./venv/bin/activate 5 | set -ux 6 | 7 | ./venv/bin/pip install --upgrade $INSTALL wheel setuptools 8 | venv-update --version || true 9 | pip --version 10 | python -c 'import wheel; print wheel.__version__' 11 | python -c 'import setuptools; print setuptools.__version__' 12 | -------------------------------------------------------------------------------- /benchmark/warm: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eux 3 | 4 | rm -rf venv 5 | virtualenv venv 6 | ./noop 7 | -------------------------------------------------------------------------------- /docs/.gitignore: -------------------------------------------------------------------------------- 1 | build/ 2 | port 3 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 21 | 22 | .PHONY: help 23 | help: 24 | @echo "Please use \`make ' where is one of" 25 | @echo " html to make standalone HTML files" 26 | @echo " dirhtml to make HTML files named index.html in directories" 27 | @echo " singlehtml to make a single large HTML file" 28 | @echo " pickle to make pickle files" 29 | @echo " json to make JSON files" 30 | @echo " htmlhelp to make HTML files and a HTML help project" 31 | @echo " qthelp to make HTML files and a qthelp project" 32 | @echo " applehelp to make an Apple Help Book" 33 | @echo " devhelp to make HTML files and a Devhelp project" 34 | @echo " epub to make an epub" 35 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 36 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 37 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 38 | @echo " text to make text files" 39 | @echo " man to make manual pages" 40 | @echo " texinfo to make Texinfo files" 41 | @echo " info to make Texinfo files and run them through makeinfo" 42 | @echo " gettext to make PO message catalogs" 43 | @echo " changes to make an overview of all changed/added/deprecated items" 44 | @echo " xml to make Docutils-native XML files" 45 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 46 | @echo " linkcheck to check all external links for integrity" 47 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 48 | @echo " coverage to run coverage check of the documentation (if enabled)" 49 | 50 | .PHONY: clean 51 | clean: 52 | rm -rf $(BUILDDIR)/* 53 | 54 | .PHONY: html 55 | html: 56 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 57 | @echo 58 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 59 | 60 | .PHONY: dirhtml 61 | dirhtml: 62 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 63 | @echo 64 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 65 | 66 | .PHONY: singlehtml 67 | singlehtml: 68 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 69 | @echo 70 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 71 | 72 | .PHONY: pickle 73 | pickle: 74 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 75 | @echo 76 | @echo "Build finished; now you can process the pickle files." 77 | 78 | .PHONY: json 79 | json: 80 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 81 | @echo 82 | @echo "Build finished; now you can process the JSON files." 83 | 84 | .PHONY: htmlhelp 85 | htmlhelp: 86 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 87 | @echo 88 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 89 | ".hhp project file in $(BUILDDIR)/htmlhelp." 90 | 91 | .PHONY: qthelp 92 | qthelp: 93 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 94 | @echo 95 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 96 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 97 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/pip-faster.qhcp" 98 | @echo "To view the help file:" 99 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/pip-faster.qhc" 100 | 101 | .PHONY: applehelp 102 | applehelp: 103 | $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp 104 | @echo 105 | @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." 106 | @echo "N.B. You won't be able to view it unless you put it in" \ 107 | "~/Library/Documentation/Help or install it in your application" \ 108 | "bundle." 109 | 110 | .PHONY: devhelp 111 | devhelp: 112 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 113 | @echo 114 | @echo "Build finished." 115 | @echo "To view the help file:" 116 | @echo "# mkdir -p $$HOME/.local/share/devhelp/pip-faster" 117 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/pip-faster" 118 | @echo "# devhelp" 119 | 120 | .PHONY: epub 121 | epub: 122 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 123 | @echo 124 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 125 | 126 | .PHONY: latex 127 | latex: 128 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 129 | @echo 130 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 131 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 132 | "(use \`make latexpdf' here to do that automatically)." 133 | 134 | .PHONY: latexpdf 135 | latexpdf: 136 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 137 | @echo "Running LaTeX files through pdflatex..." 138 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 139 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 140 | 141 | .PHONY: latexpdfja 142 | latexpdfja: 143 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 144 | @echo "Running LaTeX files through platex and dvipdfmx..." 145 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 146 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 147 | 148 | .PHONY: text 149 | text: 150 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 151 | @echo 152 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 153 | 154 | .PHONY: man 155 | man: 156 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 157 | @echo 158 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 159 | 160 | .PHONY: texinfo 161 | texinfo: 162 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 163 | @echo 164 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 165 | @echo "Run \`make' in that directory to run these through makeinfo" \ 166 | "(use \`make info' here to do that automatically)." 167 | 168 | .PHONY: info 169 | info: 170 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 171 | @echo "Running Texinfo files through makeinfo..." 172 | make -C $(BUILDDIR)/texinfo info 173 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 174 | 175 | .PHONY: gettext 176 | gettext: 177 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 178 | @echo 179 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 180 | 181 | .PHONY: changes 182 | changes: 183 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 184 | @echo 185 | @echo "The overview file is in $(BUILDDIR)/changes." 186 | 187 | .PHONY: linkcheck 188 | linkcheck: 189 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 190 | @echo 191 | @echo "Link check complete; look for any errors in the above output " \ 192 | "or in $(BUILDDIR)/linkcheck/output.txt." 193 | 194 | .PHONY: doctest 195 | doctest: 196 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 197 | @echo "Testing of doctests in the sources finished, look at the " \ 198 | "results in $(BUILDDIR)/doctest/output.txt." 199 | 200 | .PHONY: coverage 201 | coverage: 202 | $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage 203 | @echo "Testing of coverage in the sources finished, look at the " \ 204 | "results in $(BUILDDIR)/coverage/python.txt." 205 | 206 | .PHONY: xml 207 | xml: 208 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 209 | @echo 210 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 211 | 212 | .PHONY: pseudoxml 213 | pseudoxml: 214 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 215 | @echo 216 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 217 | 218 | port: 219 | ephemeral-port-reserve > port 220 | 221 | .PHONY: serve 222 | serve: HOST := $(shell hostname -f) 223 | serve: port 224 | sphinx-autobuild -a -s 1 -r ".*~" -r '\..*\.sw[a-z]' source/ build/html/ -H $(HOST) -p $(shell cat port) 225 | -------------------------------------------------------------------------------- /docs/source/_static/custom.css: -------------------------------------------------------------------------------- 1 | pre { 2 | padding: 7px 7px; 3 | font-size: 13px; 4 | } 5 | 6 | img.logo { 7 | width: 100%; 8 | } 9 | -------------------------------------------------------------------------------- /docs/source/_static/venv-update.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /docs/source/_templates/about.html: -------------------------------------------------------------------------------- 1 | {% include "!about.html"%} 2 | 3 |

4 | 5 | https://codecov.io/github/Yelp/venv-update/coverage.svg?branch=master 9 | 10 |

11 | 12 | 13 |

14 | 15 | https://img.shields.io/travis/Yelp/venv-update/master.svg?label=travis-ci 19 | 20 |

21 | -------------------------------------------------------------------------------- /docs/source/_templates/layout.html: -------------------------------------------------------------------------------- 1 | {%- extends "!layout.html" %} 2 | {%- block extrahead %} 3 | {{ super() }} 4 | {%- if favicon %} 5 | 6 | {%- endif %} 7 | {% endblock %} 8 | -------------------------------------------------------------------------------- /docs/source/benchmarks.rst: -------------------------------------------------------------------------------- 1 | .. _benchmarks: 2 | 3 | Benchmarks 4 | ========== 5 | 6 | You can find the set of scripts used to derive these numbers at: 7 | https://github.com/Yelp/venv-update/tree/master/benchmark 8 | 9 | .. literalinclude:: ../../benchmark/results 10 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # venv-update documentation build configuration file, created by 4 | # sphinx-quickstart on Mon Feb 22 11:13:18 2016. 5 | # 6 | # This file is execfile()d with the current directory set to its 7 | # containing dir. 8 | # 9 | # Note that not all possible configuration values are present in this 10 | # autogenerated file. 11 | # 12 | # All configuration values have a default; values that are commented out 13 | # serve to show the default. 14 | from __future__ import absolute_import 15 | from __future__ import print_function 16 | from __future__ import unicode_literals 17 | 18 | from pkg_resources import parse_version 19 | 20 | import venv_update 21 | 22 | # -- General configuration ------------------------------------------------ 23 | 24 | # If your documentation needs a minimal Sphinx version, state it here. 25 | #needs_sphinx = '1.0' 26 | 27 | # Add any Sphinx extension module names here, as strings. They can be 28 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 29 | # ones. 30 | extensions = [ 31 | 'sphinx.ext.autodoc', 32 | 'sphinx.ext.doctest', 33 | 'sphinx.ext.todo', 34 | 'sphinx.ext.coverage', 35 | 'sphinx.ext.viewcode', 36 | ] 37 | 38 | # Add any paths that contain templates here, relative to this directory. 39 | templates_path = ['_templates'] 40 | 41 | # The suffix(es) of source filenames. 42 | # You can specify multiple suffix as a list of string: 43 | # source_suffix = ['.rst', '.md'] 44 | source_suffix = '.rst' 45 | 46 | # The encoding of source files. 47 | #source_encoding = 'utf-8-sig' 48 | 49 | # The master toctree document. 50 | master_doc = 'index' 51 | 52 | # General information about the project. 53 | project = 'venv-update' 54 | #copyright = u'2016, Buck Evan' 55 | author = 'Buck Evan' 56 | 57 | # The version info for the project you're documenting, acts as replacement for 58 | # |version| and |release|, also used in various other places throughout the 59 | # built documents. 60 | # 61 | # The full version, including alpha/beta/rc tags. 62 | release = venv_update.__version__ 63 | # The short X.Y version. 64 | version = parse_version(release).base_version 65 | 66 | 67 | # The language for content autogenerated by Sphinx. Refer to documentation 68 | # for a list of supported languages. 69 | # 70 | # This is also used if you do content translation via gettext catalogs. 71 | # Usually you set "language" from the command line for these cases. 72 | language = None 73 | 74 | # There are two options for replacing |today|: either, you set today to some 75 | # non-false value, then it is used: 76 | #today = '' 77 | # Else, today_fmt is used as the format for a strftime call. 78 | #today_fmt = '%B %d, %Y' 79 | 80 | # List of patterns, relative to source directory, that match files and 81 | # directories to ignore when looking for source files. 82 | exclude_patterns = [] 83 | 84 | # The reST default role (used for this markup: `text`) to use for all 85 | # documents. 86 | #default_role = None 87 | 88 | # If true, '()' will be appended to :func: etc. cross-reference text. 89 | #add_function_parentheses = True 90 | 91 | # If true, the current module name will be prepended to all description 92 | # unit titles (such as .. function::). 93 | #add_module_names = True 94 | 95 | # If true, sectionauthor and moduleauthor directives will be shown in the 96 | # output. They are ignored by default. 97 | #show_authors = False 98 | 99 | # The name of the Pygments (syntax highlighting) style to use. 100 | pygments_style = 'sphinx' 101 | 102 | # A list of ignored prefixes for module index sorting. 103 | #modindex_common_prefix = [] 104 | 105 | # If true, keep warnings as "system message" paragraphs in the built documents. 106 | #keep_warnings = False 107 | 108 | # If true, `todo` and `todoList` produce output, else they produce nothing. 109 | todo_include_todos = False 110 | 111 | 112 | # -- Options for HTML output ---------------------------------------------- 113 | html_context = { 114 | 'css_files': ['_static/custom.css'], 115 | } 116 | 117 | # The theme to use for HTML and HTML Help pages. See the documentation for 118 | # a list of builtin themes. 119 | html_theme = 'alabaster' 120 | 121 | # Theme options are theme-specific and customize the look and feel of a theme 122 | # further. For a list of options available for each theme, see the 123 | # documentation. 124 | html_theme_options = { 125 | 'github_user': 'Yelp', 126 | 'github_repo': 'venv-update', 127 | 'github_banner': True, 128 | 'github_type': 'star', 129 | 'show_related': True, 130 | 'sidebar_includehidden': True, 131 | } 132 | 133 | # Add any paths that contain custom themes here, relative to this directory. 134 | #html_theme_path = [] 135 | 136 | # The name for this set of Sphinx documents. If None, it defaults to 137 | # " v documentation". 138 | #html_title = None 139 | 140 | # A shorter title for the navigation bar. Default is the same as html_title. 141 | #html_short_title = None 142 | 143 | # The name of an image file (relative to this directory) to place at the top 144 | # of the sidebar. 145 | html_logo = '_static/venv-update.svg' 146 | 147 | # The name of an image file (within the static path) to use as favicon of the 148 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 149 | # pixels large. 150 | html_favicon = 'favicon.ico' 151 | 152 | # Add any paths that contain custom static files (such as style sheets) here, 153 | # relative to this directory. They are copied after the builtin static files, 154 | # so a file named "default.css" will overwrite the builtin "default.css". 155 | html_static_path = ['_static'] 156 | 157 | # Add any extra paths that contain custom files (such as robots.txt or 158 | # .htaccess) here, relative to this directory. These files are copied 159 | # directly to the root of the documentation. 160 | html_extra_path = ['favicon.ico'] 161 | 162 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 163 | # using the given strftime format. 164 | #html_last_updated_fmt = '%b %d, %Y' 165 | 166 | # If true, SmartyPants will be used to convert quotes and dashes to 167 | # typographically correct entities. 168 | #html_use_smartypants = True 169 | 170 | # Custom sidebar templates, maps document names to template names. 171 | html_sidebars = { 172 | '**': [ 173 | 'about.html', 174 | 'localtoc.html', 175 | 'navigation.html', 176 | 'relations.html', 177 | 'searchbox.html', 178 | ] 179 | } 180 | 181 | # Additional templates that should be rendered to pages, maps page names to 182 | # template names. 183 | #html_additional_pages = {} 184 | 185 | # If false, no module index is generated. 186 | #html_domain_indices = True 187 | 188 | # If false, no index is generated. 189 | #html_use_index = True 190 | 191 | # If true, the index is split into individual pages for each letter. 192 | #html_split_index = False 193 | 194 | # If true, links to the reST sources are added to the pages. 195 | #html_show_sourcelink = True 196 | 197 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 198 | #html_show_sphinx = True 199 | 200 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 201 | #html_show_copyright = True 202 | 203 | # If true, an OpenSearch description file will be output, and all pages will 204 | # contain a tag referring to it. The value of this option must be the 205 | # base URL from which the finished HTML is served. 206 | #html_use_opensearch = '' 207 | 208 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 209 | #html_file_suffix = None 210 | 211 | # Language to be used for generating the HTML full-text search index. 212 | # Sphinx supports the following languages: 213 | # 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' 214 | # 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' 215 | #html_search_language = 'en' 216 | 217 | # A dictionary with options for the search language support, empty by default. 218 | # Now only 'ja' uses this config value 219 | #html_search_options = {'type': 'default'} 220 | 221 | # The name of a javascript file (relative to the configuration directory) that 222 | # implements a search results scorer. If empty, the default will be used. 223 | #html_search_scorer = 'scorer.js' 224 | 225 | # Output file base name for HTML help builder. 226 | htmlhelp_basename = 'venv-updatedoc' 227 | 228 | # -- Options for LaTeX output --------------------------------------------- 229 | 230 | latex_elements = { 231 | # The paper size ('letterpaper' or 'a4paper'). 232 | #'papersize': 'letterpaper', 233 | 234 | # The font size ('10pt', '11pt' or '12pt'). 235 | #'pointsize': '10pt', 236 | 237 | # Additional stuff for the LaTeX preamble. 238 | #'preamble': '', 239 | 240 | # Latex figure (float) alignment 241 | #'figure_align': 'htbp', 242 | } 243 | 244 | # Grouping the document tree into LaTeX files. List of tuples 245 | # (source start file, target name, title, 246 | # author, documentclass [howto, manual, or own class]). 247 | latex_documents = [ 248 | (master_doc, 'venv-update.tex', 'venv-update Documentation', 249 | 'Buck Evan', 'manual'), 250 | ] 251 | 252 | # The name of an image file (relative to this directory) to place at the top of 253 | # the title page. 254 | #latex_logo = None 255 | 256 | # For "manual" documents, if this is true, then toplevel headings are parts, 257 | # not chapters. 258 | #latex_use_parts = False 259 | 260 | # If true, show page references after internal links. 261 | #latex_show_pagerefs = False 262 | 263 | # If true, show URL addresses after external links. 264 | #latex_show_urls = False 265 | 266 | # Documents to append as an appendix to all manuals. 267 | #latex_appendices = [] 268 | 269 | # If false, no module index is generated. 270 | #latex_domain_indices = True 271 | 272 | 273 | # -- Options for manual page output --------------------------------------- 274 | 275 | # One entry per manual page. List of tuples 276 | # (source start file, name, description, authors, manual section). 277 | man_pages = [ 278 | (master_doc, 'venv-update', 'venv-update Documentation', 279 | [author], 1) 280 | ] 281 | 282 | # If true, show URL addresses after external links. 283 | #man_show_urls = False 284 | 285 | 286 | # -- Options for Texinfo output ------------------------------------------- 287 | 288 | # Grouping the document tree into Texinfo files. List of tuples 289 | # (source start file, target name, title, author, 290 | # dir menu entry, description, category) 291 | texinfo_documents = [ 292 | (master_doc, 'venv-update', 'venv-update Documentation', 293 | author, 'venv-update', 'One line description of project.', 294 | 'Miscellaneous'), 295 | ] 296 | 297 | # Documents to append as an appendix to all manuals. 298 | #texinfo_appendices = [] 299 | 300 | # If false, no module index is generated. 301 | #texinfo_domain_indices = True 302 | 303 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 304 | #texinfo_show_urls = 'footnote' 305 | 306 | # If true, do not generate a @detailmenu in the "Top" node's menu. 307 | #texinfo_no_detailmenu = False 308 | 309 | 310 | def process_docstring(unused_app, what, name, unused_obj, unused_options, lines): 311 | if (what, name) == ('module', 'venv_update'): 312 | for i, line in enumerate(lines): 313 | lines[i] = ' ' + line 314 | lines[:0] = ('.. sourcecode:: bash', '', ' $ venv-update --help') 315 | else: 316 | print('MISS:', (what, name)) 317 | 318 | 319 | def setup(app): 320 | app.connect(str('autodoc-process-docstring'), process_docstring) 321 | -------------------------------------------------------------------------------- /docs/source/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/venv-update/5fb5491bd421fdd8ef3cff3faa5d4846b5985ec8/docs/source/favicon.ico -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | venv-update: quick, exact 2 | ========================= 3 | `Issues `_ | 4 | `Github `_ | 5 | `CI `_ | 6 | `PyPI `_ 7 | 8 | Release v\ |release| (:ref:`Installation`) 9 | 10 | .. toctree:: 11 | :hidden: 12 | 13 | Documentation overview 14 | venv-update 15 | pip-faster 16 | 17 | Introduction 18 | ------------ 19 | 20 | venv-update is an `MIT-Licensed`_ tool to quickly and exactly synchronize 21 | a large python project's virtualenv with its `requirements`_. 22 | 23 | This project ships as two separable components: ``pip-faster`` and 24 | ``venv-update``. 25 | 26 | Both are designed for use on large projects with hundreds of requirements and 27 | are used daily by Yelp_ engineers. 28 | 29 | 30 | Why? 31 | ---- 32 | 33 | Generating a repeatable build of a virtualenv has many edge cases. If 34 | a requirement is removed, it should be uninstalled when the virtualenv is 35 | updated. If the version of python has changed, the only reliable solution is to 36 | re-build the virtualenv from scratch. Initially, this was exactly how we 37 | implemented updates of our virtualenv, but it slowed things down terribly. 38 | ``venv-update`` handles all of these edge cases and more, without completely 39 | starting from scratch (in the usual case). 40 | 41 | In a large application, best practice is to "pin" versions, with requirements 42 | like ``package-x==1.2.3`` in order to ensure that dev, staging, test, and 43 | production will all use the same code. Currently ``pip`` will always reach out 44 | to PyPI to list the versions of ``package-x`` regardless of whether the package 45 | is already installed, or whether its `wheel`_ can be found in the local cache. 46 | ``pip-faster`` adds these optimizations and others. 47 | 48 | 49 | .. _venv-update: 50 | 51 | ``venv-update`` 52 | --------------- 53 | 54 | A small script designed to keep a virtualenv in sync with a changing list of 55 | requirements. The contract of ``venv-update`` is this: 56 | 57 | The virtualenv state will be exactly the same as if you deleted and rebuilt 58 | it from scratch, but will get there in *much* less time. 59 | 60 | The needs of venv-update are what drove the development of pip-faster. 61 | For more, see :ref:`venv-update-details`. 62 | 63 | 64 | ``pip-faster`` 65 | -------------- 66 | 67 | pip-faster is a drop-in replacement for pip. ``pip-faster``'s contract is: 68 | 69 | Take the same arguments and give the same results as ``pip``, just more quickly. 70 | 71 | This is *especially* true in the case of pinned requirements (e.g. ``package-x==1.2.3``). 72 | If you're also using venv-update (which we heartily recommend!), you can view 73 | pip-faster as an implementation detail. For more, see :ref:`pip-faster-details`. 74 | 75 | 76 | `How much` faster? 77 | ~~~~~~~~~~~~~~~~~~ 78 | 79 | 80 | If we install `plone`_ (a large python application with more than 250 81 | dependencies) we get these numbers: 82 | 83 | +---------+--------------+--------------+---------------+ 84 | | testcase| pip v8.0.2 | pip-faster | improvement | 85 | +=========+==============+==============+===============+ 86 | | cold | 4m 39s | 4m 16s | 8% | 87 | +---------+--------------+--------------+---------------+ 88 | | noop | 7.11s | 2.40s | **196%** | 89 | +---------+--------------+--------------+---------------+ 90 | | warm | 44.6s | 21.3s | **109%** | 91 | +---------+--------------+--------------+---------------+ 92 | 93 | In the "cold" case, all caches are completely empty. 94 | In the "noop" case nothing needs to be done in order to update the 95 | virtualenv. 96 | In the "warm" case caches are fully populated, but the virtualenv has been 97 | completely deleted. 98 | 99 | The :ref:`benchmarks` page has more detail. 100 | 101 | 102 | .. _installation: 103 | 104 | Installation 105 | ------------ 106 | 107 | Because ``venv-update`` is meant to be the entry-point for creating your 108 | virtualenv_ directory and installing your packages, it's not meant to be 109 | installed via pip; that would require a virtualenv to already exist! 110 | 111 | Instead, the script is designed to be `vendored` (directly checked in) to your 112 | project. It has no dependencies other than `virtualenv`_ and the standard 113 | Python library. 114 | 115 | 116 | .. parsed-literal:: 117 | 118 | curl -o venv-update ``_ 119 | chmod +x venv-update 120 | 121 | 122 | Usage 123 | ------------ 124 | 125 | By default, running ``venv-update`` will create a virtualenv named ``venv`` in the 126 | current directory, using ``requirements.txt`` in the current directory. This 127 | should be the desired default for most projects. 128 | 129 | If you need more control, you can pass additional options to both 130 | ``virtualenv`` and ``pip``. The command-line help gives more detail: 131 | 132 | .. automodule:: venv_update 133 | 134 | ... in your ``Makefile`` 135 | ~~~~~~~~~~~~~~~~~~~~~~~~ 136 | 137 | venv-update is a good fit for use with make because it is idempotent and should 138 | never fail, under normal circumstances. Here's an example Makefile: 139 | 140 | .. sourcecode:: make 141 | 142 | venv: requirements.txt 143 | ./bin/venv-update 144 | 145 | .PHONY: run-some-script 146 | run-some-script: venv 147 | ./venv/bin/some-script 148 | 149 | 150 | ... with tox 151 | ~~~~~~~~~~~~ 152 | 153 | tox_ is a useful tool for testing libraries against multiple versions of 154 | the Python interpreter. You can speed it up by telling it to use venv-update 155 | for dependency installation; not only will it avoid network access and prefer 156 | wheels, but it's also better at syncing a virtualenv (whereas tox will often 157 | throw out an entire virtualenv and start over). 158 | 159 | To start using venv-update inside tox, copy the venv-update script into 160 | your project (for example, at ``bin/venv-update``). 161 | 162 | Then, apply a change like this to your ``tox.ini`` file: 163 | 164 | .. sourcecode:: diff 165 | 166 | 167 | [tox] 168 | envlist = py27,py34 169 | + skipsdist = true 170 | 171 | [testenv] 172 | + venv_update = 173 | + {toxinidir}/bin/venv-update \ 174 | + venv= {envdir} \ 175 | + install= -r {toxinidir}/requirements.txt {toxinidir} 176 | - deps = -rrequirements.txt 177 | commands = 178 | + {[testenv]venv_update} 179 | py.test tests/ 180 | pre-commit run --all-files 181 | 182 | The exact changes will of course vary, but above is a general template. The 183 | two changes are: running venv-update as the first test command, and 184 | removing the list of ``deps`` (so that tox will never invalidate your 185 | virtualenv itself; we want to let venv-update manage that instead). 186 | The ``skipsdist`` avoids installing your package twice. In tox<2, it also 187 | prevents all of your packages dependencies from being installed by pip-slower. 188 | 189 | 190 | .. _MIT-Licensed: https://github.com/Yelp/pip-faster/blob/latest/COPYING 191 | .. _requirements: https://pip.pypa.io/en/stable/user_guide/#requirements-files 192 | .. _plone: https://en.wikipedia.org/wiki/Plone_(software) 193 | .. _pip: https://pip.pypa.io/en/stable/ 194 | .. _virtualenv: https://virtualenv.readthedocs.org/en/latest/ 195 | .. _wheel: https://wheel.readthedocs.org/en/latest/ 196 | .. _tox: https://tox.readthedocs.org/en/latest/ 197 | .. _yelp: https://www.yelp.com/ 198 | 199 | .. vim:textwidth=79:shiftwidth=3:noshiftround: 200 | -------------------------------------------------------------------------------- /docs/source/internal-pypi.rst: -------------------------------------------------------------------------------- 1 | Internal PyPI Servers 2 | --------------------- 3 | 4 | Under linux, performance will be much better if you use an internal PyPI server 5 | instead of the `public PyPI`_. 6 | 7 | Besides the potentially lesser latency, an internal PyPI server allows for 8 | uploading binary wheels compiled for Linux. Unlike OS X or Windows, installing 9 | projects like lxml on Linux is normally extremely slow since they will need to 10 | be compiled during every installation. 11 | 12 | pip-faster improves this by only compiling on the first installation for each 13 | user (this is also the default behavior for pip >= 7), but this doesn't help 14 | for the first run. 15 | 16 | Using an internal PyPI server which allows uploading of Linux wheels can 17 | improve speed greatly. Unfortunately, these wheels are guaranteed compatible 18 | only with the same Linux distribution they were compiled on, so this only works 19 | if your developers work in very homogeneous environments. 20 | 21 | For both venv-update and pip-faster, you can specify an index server by setting 22 | the ``$PIP_INDEX_URL`` environment variable (or ``$PIP_EXTRA_INDEX_URL`` if you 23 | want to supplement but not replace the default PyPI). For pip-faster you can 24 | also use ``-i`` or ``-e``, just like in regular pip. 25 | 26 | 27 | .. _public PyPI: https://pypi.python.org/pypi 28 | -------------------------------------------------------------------------------- /docs/source/pip-faster.rst: -------------------------------------------------------------------------------- 1 | .. _pip-faster-details: 2 | 3 | ``pip-faster`` in detail 4 | ======================== 5 | 6 | 7 | By design ``pip-faster`` maintains the interface of pip, and should only have a 8 | few desirable behavior differences, listed below. 9 | 10 | #. ``pip-faster`` adds optional "prune" capability to the ``pip install`` 11 | subcommand. ``pip-faster install --prune`` will *uninstall* any installed 12 | packages that are not required (as arguments to the same install command). 13 | This is used by default in :ref:`venv-update` to implement reproducible builds. 14 | 15 | #. We've taken great care to reduce the number of round-trips to PyPI, which 16 | makes up the majority of time spent on what should be a no-op update. For 17 | example, if you're installing a specific version of a package which we 18 | already have cached, there's no need to talk to PyPI, but vanilla pip will. 19 | 20 | #. Packages are downloaded and `wheeled`_ before installation (if they 21 | aren't available from PyPI as wheels). If the virtualenv needs to be rebuilt, 22 | or you use the same requirement in another project, the wheel can be reused. 23 | This greatly speeds up installation of projects like lxml or numpy which have 24 | a slow-to-compile binary component. 25 | 26 | Mainline pip recently added this feature (in pip 7.0, 2015-05-21). We plan 27 | to merge, but this isn't currently an urgent work item; all of our use cases 28 | are satisfied. However, patches `are` welcome. 29 | 30 | #. ``pip-faster`` will refuse to install package versions which conflict (we 31 | generally consider this a feature); stock pip, on the other hand, will 32 | happily install conflicting packages. Similarly, pip-faster detects circular 33 | dependencies and unsatisfied dependencies and throws an error where stock 34 | pip would not. 35 | 36 | 37 | Installation 38 | ------------ 39 | 40 | You can ``pip install venv-update`` to get ``pip-faster``, the same way you 41 | would any other Python tool, but if you're using :ref:`venv-update` it's not 42 | necessary to install pip-faster; the venv-update script will install the 43 | correct version inside your virtualenv for you. 44 | 45 | 46 | .. _wheeled: https://wheel.readthedocs.org/en/latest/ 47 | 48 | .. toctree:: 49 | :hidden: 50 | 51 | internal-pypi 52 | benchmarks 53 | 54 | .. vim:textwidth=79:sts=3:shiftwidth=3:noshiftround: 55 | -------------------------------------------------------------------------------- /docs/source/venv-update.rst: -------------------------------------------------------------------------------- 1 | .. _venv-update-details: 2 | 3 | ``venv-update`` in detail 4 | ========================= 5 | 6 | venv-update is a small script whose job is to idempotently ensure the existence 7 | and correctness of a project's virtualenv, and synchronize it with its 8 | requirements. 9 | 10 | We like to call venv-update from our Makefiles to create and maintain a 11 | virtualenv. It does the following: 12 | 13 | * Ensures a virtualenv exists at the specified location with the specified 14 | Python version, and that it is valid. It will create or recreate a virtualenv 15 | as necessary to ensure that one venv-update invocation is all that's needed. 16 | 17 | * Calculates the difference in packages derived from the ``requirements.txt`` 18 | files and the installed packages. Packages will be uninstalled, upgraded, or 19 | installed as necessary. 20 | 21 | The goal is that venv-update will put you in the same state as if you wipe 22 | away your virtualenv and rebuild it with ``pip install``, but much more 23 | quickly. 24 | 25 | * Takes advantage of ``pip-faster`` for package installation (see below) to 26 | avoid network access and rebuilding packages as much as possible. 27 | 28 | For reference, a project with 250 dependencies which are all pinned can run a 29 | no-op venv-update in ~2 seconds with no network access. The running time when 30 | changes are needed is dominated by the time it takes to download and install 31 | packages, but is generally quite fast (on the order of ~10 seconds). 32 | 33 | 34 | Customizing the install command 35 | ------------------------------- 36 | 37 | If you don't like pip-faster, for whatever reason, ``venv-update`` provides 38 | sufficient control to use "plain-old" pip, or any other command for that 39 | matter. 40 | 41 | To tell venv-update to install and run pip rather than pip-faster:: 42 | 43 | venv-update install-command= pip install --upgrade bootstrap-deps= 'pip>8' 44 | 45 | .. vim:textwidth=79:shiftwidth=3:noshiftround: 46 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | addopts = --durations=0 -vv -rfE --doctest-modules 3 | norecursedirs = 4 | venv 5 | .tox 6 | tmp* 7 | *tmp 8 | testing/packages 9 | build 10 | python_files = 11 | *.py 12 | python_classes = 13 | Test 14 | Describe 15 | python_functions = 16 | test_ 17 | it_ 18 | -------------------------------------------------------------------------------- /requirements.d/CI.txt: -------------------------------------------------------------------------------- 1 | ../CI/requirements.txt -------------------------------------------------------------------------------- /requirements.d/_lint.txt: -------------------------------------------------------------------------------- 1 | # packages needed for linting, but not for prod 2 | flake8 3 | # Workaround for: 4 | # Error: version conflict: virtualenv 16.7.10 (venv/lib/python3.6/site-packages) <-> virtualenv>=20.0.8 (from pre-commit->-r requirements.d/_lint.txt (line 3)) 5 | pre-commit<=1.21.0 6 | -------------------------------------------------------------------------------- /requirements.d/coverage.txt: -------------------------------------------------------------------------------- 1 | # Minimal set of packages to get coverage working 2 | # NOTE: versions must be specified in post-order or else we get the wrong versions. 3 | 4 | coverage 5 | coverage-enable-subprocess 6 | -------------------------------------------------------------------------------- /requirements.d/dev.txt: -------------------------------------------------------------------------------- 1 | # minimal set of packages for committers 2 | -e . 3 | -r test.txt 4 | -r _lint.txt 5 | -r CI.txt 6 | -r docs.txt 7 | 8 | pudb 9 | -------------------------------------------------------------------------------- /requirements.d/docs.txt: -------------------------------------------------------------------------------- 1 | # packages necessary to build, view and/or upload our documentation 2 | sphinx 3 | sphinx-autobuild 4 | -------------------------------------------------------------------------------- /requirements.d/import_tests.txt: -------------------------------------------------------------------------------- 1 | # minimal set of requirements to import (but not run) test code 2 | # this is used by the lint environment. 3 | -r coverage.txt 4 | 5 | pytest>=3.6.0 6 | ephemeral-port-reserve 7 | pip>=10.0.0,<=18.1 8 | wheel 9 | six 10 | -------------------------------------------------------------------------------- /requirements.d/lint.txt: -------------------------------------------------------------------------------- 1 | # packages necessary to `make lint` 2 | -r import_tests.txt 3 | # mixin pattern, only to avoid the diamond-include issue 4 | -r _lint.txt 5 | -------------------------------------------------------------------------------- /requirements.d/test.txt: -------------------------------------------------------------------------------- 1 | # minimal set of packages to run tests (under tox, for example) 2 | -r import_tests.txt 3 | 4 | virtualenv>=20.0.8 5 | pytest-xdist 6 | 7 | pypiserver 8 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | -r requirements.d/dev.txt 2 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from __future__ import absolute_import 3 | from __future__ import print_function 4 | from __future__ import unicode_literals 5 | 6 | from setuptools import find_packages 7 | from setuptools import setup 8 | 9 | 10 | # https://github.com/pypa/python-packaging-user-guide/blob/master/source/single_source_version.rst 11 | def read(*names, **kwargs): 12 | import io 13 | import os 14 | with io.open( 15 | os.path.join(os.path.dirname(__file__), *names), 16 | encoding=kwargs.get('encoding', 'utf8') 17 | ) as fp: 18 | return fp.read() 19 | 20 | 21 | def find_version(*file_paths): 22 | import re 23 | version_file = read(*file_paths) 24 | version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", 25 | version_file, re.M) 26 | if version_match: 27 | return version_match.group(1) 28 | raise RuntimeError('Unable to find version string.') 29 | 30 | 31 | def main(): 32 | setup( 33 | name='venv-update', 34 | version=find_version('venv_update.py'), 35 | description="quickly and exactly synchronize a large project's virtualenv with its requirements", 36 | url='https://github.com/Yelp/venv-update', 37 | author='Buck Evan', 38 | author_email='buck@yelp.com', 39 | platforms='all', 40 | license='MIT', 41 | classifiers=[ 42 | 'License :: OSI Approved :: MIT License', 43 | 'Programming Language :: Python :: 2', 44 | 'Programming Language :: Python :: 2.7', 45 | 'Programming Language :: Python :: 3', 46 | 'Programming Language :: Python :: 3.5', 47 | 'Programming Language :: Python :: 3.6', 48 | 'Programming Language :: Python :: Implementation :: PyPy', 49 | 'Topic :: System :: Archiving :: Packaging', 50 | 'Operating System :: Unix', 51 | 'Intended Audience :: Developers', 52 | 'Development Status :: 4 - Beta', 53 | 'Environment :: Console', 54 | ], 55 | py_modules=['venv_update', 'pip_faster'], 56 | packages=find_packages(exclude=('tests*',)), 57 | install_requires=[ 58 | 'pip>=10.0.0,<=18.1', 59 | 'wheel>0.25.0', # 0.25.0 causes get_tag AssertionError in python3 60 | 'setuptools>=0.8.0', # 0.7 causes "'sys_platform' not defined" when installing wheel >0.25 61 | ], 62 | entry_points={ 63 | 'console_scripts': [ 64 | 'venv-update = venv_update:main', 65 | 'pip-faster = pip_faster:main', 66 | ], 67 | }, 68 | keywords=['pip', 'virtualenv'], 69 | options={ 70 | 'bdist_wheel': { 71 | 'universal': 1, 72 | } 73 | }, 74 | ) # :pragma:nocover: covered by tox 75 | 76 | 77 | if __name__ == '__main__': 78 | exit(main()) 79 | -------------------------------------------------------------------------------- /test: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eu 3 | export TOP=$(readlink -f ${TOP:-.}) 4 | export SITEPACKAGES=${SITEPACKAGES:-.} 5 | NCPU=$(getconf _NPROCESSORS_CONF) 6 | 7 | if python -c 'import platform; exit(not(platform.python_implementation() == "PyPy"))'; then 8 | PYPY=true 9 | else 10 | PYPY=false 11 | fi 12 | 13 | coverage-report() { 14 | # reporting: 15 | cd $TOP 16 | coverage combine 17 | python ./tests/testing/fix_coverage.py '/site-packages/' $TOP 18 | unset COVERAGE_PROCESS_START 19 | coverage combine 20 | 21 | # for unknown reasons, I have to do the fixup again after stopping profiling to avoid stray test results 22 | python ./tests/testing/fix_coverage.py '/site-packages/' $TOP 23 | 24 | if ${CI:-false}; then 25 | ./CI/coverage 26 | fi 27 | coverage report --fail-under 90 28 | coverage report --fail-under 100 venv_update.py pip_faster.py 29 | } 30 | fail() { 31 | if ! $PYPY; then coverage-report; fi 32 | echo 'FAIL' 33 | } 34 | trap fail ERR 35 | 36 | 37 | if [ "$*" ]; then 38 | set -- -n0 "$@" 39 | else 40 | # default arguments 41 | set -- $TOP/tests $SITEPACKAGES/{venv_update,pip_faster}.py 42 | fi 43 | 44 | if "${CI:-false}"; then 45 | # Under CI, we don't get to use all the CPU. 46 | NCPU=$((NCPU > 5? NCPU/5 : 1)) 47 | fi 48 | 49 | set -x 50 | # unquoted is intentional. we want to expand words. 51 | set -- ${PYTEST_OPTIONS:-} "$@" 52 | if $PYPY; then 53 | # coverage under pypy takes too dang long: 54 | # https://bitbucket.org/pypy/pypy/issue/1871/10x-slower-than-cpython-under-coverage#comment-14404182 55 | # pypy can oom on travis; let's use less workers 56 | py.test -n $NCPU "$@" 57 | else 58 | # clean up anything left behind from before: 59 | rm -f $TOP/.coverage $TOP/.coverage.* 60 | 61 | # See: http://nedbatchelder.com/code/coverage/subprocess.html 62 | export COVERAGE_PROCESS_START=$TOP/.coveragerc 63 | 64 | # run the tests! 65 | py.test -n $NCPU "$@" 66 | coverage-report 67 | fi 68 | 69 | echo 'SUCCESS' 70 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | from __future__ import unicode_literals 4 | 5 | import os 6 | import socket 7 | import subprocess 8 | import sys 9 | import time 10 | from contextlib import contextmanager 11 | from errno import ECONNREFUSED 12 | 13 | import pytest 14 | import six 15 | from ephemeral_port_reserve import reserve 16 | 17 | from testing import run 18 | from testing import TOP 19 | from venv_update import colorize 20 | 21 | 22 | ENV_WHITELIST = ( 23 | # allows coverage of subprocesses 24 | 'COVERAGE_PROCESS_START', 25 | # used in the configuration of coverage 26 | 'TOP', 27 | # let's not fill up the root partition, please 28 | 'TMPDIR', 29 | # these help my debugger not freak out 30 | 'HOME', 31 | 'TERM', 32 | ) 33 | 34 | 35 | @pytest.fixture(autouse=True) 36 | def fixed_environment_variables(): 37 | orig_environ = os.environ.copy() 38 | 39 | for var in dict(os.environ): 40 | if var not in ENV_WHITELIST: 41 | del os.environ[var] 42 | 43 | # disable casual interaction with python.org 44 | os.environ['PIP_INDEX_URL'] = '(total garbage)' 45 | os.environ['PYTHONWARNINGS'] = 'ignore:Support for Python 3.0-3.2 has been dropped.:UserWarning' 46 | 47 | # Fixes argparse help tests. 48 | os.environ['COLUMNS'] = '1000' 49 | 50 | # normalize $PATH 51 | from sys import executable 52 | from os import defpath 53 | from os.path import dirname 54 | assert defpath.startswith(':') 55 | os.environ['PATH'] = dirname(executable) + defpath 56 | yield 57 | os.environ.clear() 58 | os.environ.update(orig_environ) 59 | 60 | 61 | @pytest.fixture 62 | def tmpdir(tmpdir): 63 | """override tmpdir to provide a $HOME and $TMPDIR""" 64 | home = tmpdir.ensure('home', dir=True) 65 | tmp = tmpdir.ensure('tmp', dir=True) 66 | 67 | orig_environ = os.environ.copy() 68 | os.environ['HOME'] = str(home) 69 | os.environ['TMPDIR'] = str(tmp) 70 | 71 | with tmpdir.as_cwd(): # TODO: remove all the tmpdir.chdir() 72 | yield tmpdir 73 | 74 | os.environ.clear() 75 | os.environ.update(orig_environ) 76 | 77 | 78 | @pytest.fixture(scope='session') 79 | def pypi_packages(): 80 | package_temp = TOP.join('build/test-packages') 81 | with TOP.as_cwd(): 82 | run( 83 | sys.executable, 84 | 'tests/testing/make_sdists.py', 85 | 'tests/testing/packages', 86 | '.', # we need venv-update to be installable too 87 | str(package_temp), 88 | ) 89 | 90 | yield package_temp 91 | 92 | 93 | @pytest.fixture(scope='session') 94 | def pypi_port(): 95 | yield reserve() 96 | 97 | 98 | @contextmanager 99 | def start_pypi_server(packages, port, pypi_fallback): 100 | port = str(port) 101 | cmd = ( 102 | 'pypi-server', '-v', 103 | '-i', '127.0.0.1', 104 | '-p', port, 105 | # Default fallback is HTTP, which is no longer supported. 106 | # TODO: revert after a new pypiserver is released with this patch: 107 | # https://github.com/pypiserver/pypiserver/pull/182 108 | '--fallback-url', 'https://pypi.python.org/simple', 109 | ) 110 | if not pypi_fallback: 111 | cmd += ('--disable-fallback',) 112 | cmd += (str(packages),) 113 | print(colorize(cmd)) 114 | server = subprocess.Popen(cmd, cwd=str(TOP), stderr=1) 115 | 116 | limit = 10 117 | poll = .1 118 | pypi_url = 'http://localhost:' + str(port) 119 | while True: 120 | if server.poll() is not None: 121 | raise AssertionError('pypi ended! (code %i)' % server.returncode) 122 | elif service_up(pypi_url): 123 | break 124 | elif limit > 0: 125 | time.sleep(poll) 126 | limit -= poll 127 | else: 128 | raise AssertionError('pypi server never became ready!') 129 | 130 | os.environ['PIP_INDEX_URL'] = pypi_url + '/simple' 131 | try: 132 | yield pypi_url 133 | finally: 134 | server.terminate() 135 | server.wait() 136 | 137 | 138 | @pytest.fixture 139 | def pypi_server(pypi_packages, pypi_port): 140 | with start_pypi_server(pypi_packages, pypi_port, False) as pypi_url: 141 | yield pypi_url 142 | 143 | 144 | @pytest.fixture 145 | def pypi_server_with_fallback(pypi_packages, pypi_port): 146 | with start_pypi_server(pypi_packages, pypi_port, True) as pypi_url: 147 | yield pypi_url 148 | 149 | 150 | def ioerror_to_errno(error): # :pragma:nocover: all of these cases are exceptional and quite rare 151 | if isinstance(error.errno, int): 152 | return error.errno 153 | # urllib throws an IOError with a string in the errno slot -.- 154 | elif len(error.args) > 1 and isinstance(error.args[1], socket.error): 155 | return error.args[1].errno 156 | elif len(error.args) == 1 and isinstance(error.args[0], socket.error): 157 | return error.args[0].errno 158 | elif hasattr(error, 'code') and isinstance(error.code, int): 159 | return error.code 160 | else: 161 | raise ValueError('Could not find error number: %r' % error) 162 | 163 | 164 | def service_up(url): 165 | try: 166 | response = six.moves.urllib.request.urlopen(url) 167 | except IOError as error: 168 | if ioerror_to_errno(error) in (ECONNREFUSED, 404): 169 | print('pypi not up:', error) 170 | return False 171 | else: 172 | raise 173 | 174 | print('pypi response:', response) 175 | return response.getcode() == 200 176 | 177 | 178 | def pytest_assertrepr_compare(config, op, left, right): # TODO: unit-test :pragma:nocover: 179 | if op == 'in' and '\n' in left: 180 | # Convert 'in' comparisons to '==' comparisons, for more usable error messaging. 181 | # Truncate the right-hand-side such that it has the longest common prefix with the LHS, 182 | # and the longest common suffix as well. 183 | # Given the diff of the two, this should pinpoint the difference. 184 | beginning = end = None 185 | for i in range(len(left)): 186 | if beginning and end: 187 | break 188 | 189 | if beginning is None and left[:i + 1] not in right: 190 | beginning = left[:i] 191 | 192 | if end is None and left[-i - 1:] not in right: 193 | end = left[-i:] 194 | 195 | right = right.split(beginning, 1)[-1].rsplit(end, 1)[0] 196 | right = ''.join((beginning, right, end)) 197 | 198 | from _pytest.assertion.util import assertrepr_compare 199 | return assertrepr_compare(config, '==', left, right) 200 | -------------------------------------------------------------------------------- /tests/functional/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/venv-update/5fb5491bd421fdd8ef3cff3faa5d4846b5985ec8/tests/functional/__init__.py -------------------------------------------------------------------------------- /tests/functional/args.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | from __future__ import unicode_literals 4 | 5 | from testing import strip_coverage_warnings 6 | from testing import venv_update 7 | from venv_update import __doc__ as HELP_OUTPUT 8 | from venv_update import __version__ as VERSION 9 | 10 | 11 | def test_help(): 12 | assert HELP_OUTPUT 13 | assert HELP_OUTPUT.startswith('usage:') 14 | last_line = HELP_OUTPUT.rsplit('\n', 2)[-2].strip() 15 | assert last_line.startswith('Please send issues to: https://') 16 | 17 | out, err = venv_update('--help') 18 | assert strip_coverage_warnings(err) == '' 19 | assert out == HELP_OUTPUT 20 | 21 | out, err = venv_update('-h') 22 | assert strip_coverage_warnings(err) == '' 23 | assert out == HELP_OUTPUT 24 | 25 | 26 | def test_version(): 27 | assert VERSION 28 | 29 | out, err = venv_update('--version') 30 | assert strip_coverage_warnings(err) == '' 31 | assert out == VERSION + '\n' 32 | 33 | out, err = venv_update('-V') 34 | assert strip_coverage_warnings(err) == '' 35 | assert out == VERSION + '\n' 36 | 37 | 38 | def test_bad_option(): 39 | import pytest 40 | from subprocess import CalledProcessError 41 | 42 | with pytest.raises(CalledProcessError) as excinfo: 43 | venv_update('venv') 44 | out, err = excinfo.value.result 45 | assert strip_coverage_warnings(err) == '''\ 46 | invalid option: venv 47 | Try --help for more information. 48 | ''' 49 | assert out == '' 50 | -------------------------------------------------------------------------------- /tests/functional/conflict_test.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | from __future__ import unicode_literals 4 | 5 | import os 6 | import time 7 | from subprocess import CalledProcessError 8 | from sysconfig import get_python_version 9 | 10 | import pytest 11 | 12 | import testing as T 13 | from testing.python_lib import PYTHON_LIB 14 | 15 | 16 | def assert_venv_marked_invalid(venv): 17 | """we mark a virtualenv as invalid by bumping its timestamp back by a day""" 18 | venv_age = time.time() - os.path.getmtime(venv.strpath) 19 | assert venv_age / 60 / 60 / 24 > 1 20 | 21 | 22 | def assert_something_went_wrong(out): 23 | assert out.endswith( 24 | 'Something went wrong! ' 25 | "Sending 'venv' back in time, so make knows it's invalid.\n" 26 | ) 27 | 28 | 29 | @pytest.mark.usefixtures('pypi_server') 30 | def test_conflicting_reqs(tmpdir): 31 | tmpdir.chdir() 32 | T.requirements(''' 33 | dependant_package 34 | conflicting_package 35 | ''') 36 | 37 | with pytest.raises(CalledProcessError) as excinfo: 38 | T.venv_update() 39 | assert excinfo.value.returncode == 1 40 | out, err = excinfo.value.result 41 | 42 | err = T.strip_coverage_warnings(err) 43 | err = T.strip_pip_warnings(err) 44 | assert err.strip() == ( 45 | "conflicting-package 1 has requirement many-versions-package<2, but you'll " 46 | 'have many-versions-package 3 which is incompatible.\n' 47 | # TODO: do we still need to append our own error? 48 | 'Error: version conflict: many-versions-package 3 (venv/{}) ' 49 | '<-> many-versions-package<2 ' 50 | '(from conflicting_package->-r requirements.txt (line 3))'.format( 51 | PYTHON_LIB, 52 | ) 53 | ) 54 | 55 | out = T.uncolor(out) 56 | assert_something_went_wrong(out) 57 | 58 | assert_venv_marked_invalid(tmpdir.join('venv')) 59 | 60 | 61 | @pytest.mark.usefixtures('pypi_server') 62 | def test_multiple_issues(tmpdir): 63 | # Make it a bit worse. The output should show all three issues. 64 | tmpdir.chdir() 65 | T.enable_coverage() 66 | 67 | T.requirements('dependant_package\n-r %s/requirements.d/coverage.txt' % T.TOP) 68 | T.venv_update() 69 | 70 | T.run('./venv/bin/pip', 'uninstall', '--yes', 'implicit_dependency') 71 | T.requirements(''' 72 | dependant_package 73 | conflicting_package 74 | pure_python_package==0.1.0 75 | ''') 76 | 77 | with pytest.raises(CalledProcessError) as excinfo: 78 | T.venv_update() 79 | assert excinfo.value.returncode == 1 80 | out, err = excinfo.value.result 81 | 82 | err = T.strip_coverage_warnings(err) 83 | err = T.strip_pip_warnings(err) 84 | 85 | err = err.splitlines() 86 | # pip outputs conflict lines in a non-consistent order 87 | assert set(err[:3]) == { 88 | "conflicting-package 1 has requirement many-versions-package<2, but you'll have many-versions-package 3 which is incompatible.", # noqa 89 | "dependant-package 1 has requirement pure-python-package>=0.2.1, but you'll have pure-python-package 0.1.0 which is incompatible.", # noqa 90 | 'Error: version conflict: pure-python-package 0.1.0 (venv/{lib}) <-> pure-python-package>=0.2.1 (from dependant_package->-r requirements.txt (line 2))'.format( # noqa 91 | lib=PYTHON_LIB, 92 | ), 93 | } 94 | # TODO: do we still need to append our own error? 95 | assert '\n'.join(err[3:]) == ( 96 | 'Error: version conflict: many-versions-package 3 ' 97 | '(venv/{lib}) <-> many-versions-package<2 ' 98 | '(from conflicting_package->-r requirements.txt (line 3))'.format( 99 | lib=PYTHON_LIB, 100 | ) 101 | ) 102 | 103 | out = T.uncolor(out) 104 | assert_something_went_wrong(out) 105 | 106 | assert_venv_marked_invalid(tmpdir.join('venv')) 107 | 108 | 109 | @pytest.mark.usefixtures('pypi_server') 110 | def test_editable_egg_conflict(tmpdir): 111 | conflicting_package = tmpdir / 'tmp/conflicting_package' 112 | many_versions_package_2 = tmpdir / 'tmp/many_versions_package_2' 113 | 114 | from shutil import copytree 115 | copytree( 116 | str(T.TOP / 'tests/testing/packages/conflicting_package'), 117 | str(conflicting_package), 118 | ) 119 | 120 | copytree( 121 | str(T.TOP / 'tests/testing/packages/many_versions_package_2'), 122 | str(many_versions_package_2), 123 | ) 124 | 125 | with many_versions_package_2.as_cwd(): 126 | from sys import executable as python 127 | T.run(python, 'setup.py', 'bdist_egg', '--dist-dir', str(conflicting_package)) 128 | 129 | with tmpdir.as_cwd(): 130 | T.enable_coverage() 131 | T.requirements('-e %s' % conflicting_package) 132 | with pytest.raises(CalledProcessError) as excinfo: 133 | T.venv_update() 134 | assert excinfo.value.returncode == 1 135 | out, err = excinfo.value.result 136 | 137 | err = T.strip_coverage_warnings(err) 138 | err = T.strip_pip_warnings(err) 139 | assert err.strip() == ( 140 | 'Error: version conflict: many-versions-package 2 ' 141 | '(tmp/conflicting_package/many_versions_package-2-py{}.egg) ' 142 | '<-> many_versions_package<2 ' 143 | '(from conflicting-package==1->-r requirements.txt (line 1))'.format( 144 | get_python_version(), 145 | ) 146 | ) 147 | 148 | out = T.uncolor(out) 149 | assert_something_went_wrong(out) 150 | 151 | assert_venv_marked_invalid(tmpdir.join('venv')) 152 | -------------------------------------------------------------------------------- /tests/functional/faster.py: -------------------------------------------------------------------------------- 1 | """attempt to show that pip-faster is... faster""" 2 | from __future__ import absolute_import 3 | from __future__ import print_function 4 | from __future__ import unicode_literals 5 | 6 | import contextlib 7 | import sys 8 | 9 | import pytest 10 | from py._path.local import LocalPath as Path 11 | 12 | from testing import enable_coverage 13 | from testing import install_coverage 14 | from testing import pip_freeze 15 | from testing import requirements 16 | from testing import venv_update 17 | from venv_update import __version__ 18 | 19 | 20 | def time_savings(tmpdir, between): 21 | """install twice, and the second one should be faster, due to whl caching""" 22 | with tmpdir.as_cwd(): 23 | enable_coverage() 24 | 25 | @contextlib.contextmanager 26 | def venv_setup(): 27 | # First just set up a blank virtualenv, this'll bypass the 28 | # bootstrap when we're actually testing for speed 29 | if not Path('venv').exists(): 30 | requirements('') 31 | venv_update() 32 | install_coverage() 33 | 34 | # Now the actual requirements we'll install 35 | requirements('\n'.join(( 36 | 'project_with_c', 37 | 'pure_python_package==0.2.1', 38 | 'slow_python_package==0.1.0', 39 | 'dependant_package', 40 | 'many_versions_package>=2,<3', 41 | '' 42 | ))) 43 | 44 | yield 45 | 46 | expected = '\n'.join(( 47 | 'dependant-package==1', 48 | 'implicit-dependency==1', 49 | 'many-versions-package==2.1', 50 | 'project-with-c==0.1.0', 51 | 'pure-python-package==0.2.1', 52 | 'slow-python-package==0.1.0', 53 | 'venv-update==%s' % __version__, 54 | '' 55 | )) 56 | assert pip_freeze() == expected 57 | 58 | from time import time 59 | 60 | with venv_setup(): 61 | start = time() 62 | venv_update( 63 | PIP_VERBOSE='1', 64 | PIP_RETRIES='0', 65 | PIP_TIMEOUT='0', 66 | ) 67 | time1 = time() - start 68 | 69 | between() 70 | 71 | with venv_setup(): 72 | start = time() 73 | # second install should also need no network access 74 | # these are localhost addresses with arbitrary invalid ports 75 | venv_update( 76 | PIP_VERBOSE='1', 77 | PIP_RETRIES='0', 78 | PIP_TIMEOUT='0', 79 | http_proxy='http://127.0.0.1:111111', 80 | https_proxy='https://127.0.0.1:222222', 81 | ftp_proxy='ftp://127.0.0.1:333333', 82 | ) 83 | time2 = time() - start 84 | 85 | print() 86 | print('%.3fs originally' % time1) 87 | print('%.3fs subsequently' % time2) 88 | 89 | difference = time1 - time2 90 | print('%.2fs speedup' % difference) 91 | 92 | ratio = time1 / time2 93 | percent = (ratio - 1) * 100 94 | print('%.2f%% speedup' % percent) 95 | return difference 96 | 97 | 98 | @pytest.mark.usefixtures('pypi_server') 99 | def test_noop_install_faster(tmpdir): 100 | def do_nothing(): 101 | pass 102 | 103 | # the slow-python-package takes five seconds to compile 104 | assert time_savings(tmpdir, between=do_nothing) > 6 105 | 106 | 107 | @pytest.mark.skipif('__pypy__' in sys.builtin_module_names, reason='slower under pypy for some reason') 108 | @pytest.mark.usefixtures('pypi_server_with_fallback', 'pypi_packages') 109 | def test_cached_clean_install_faster(tmpdir): 110 | def clean(): 111 | venv = tmpdir.join('venv') 112 | assert venv.isdir() 113 | venv.remove() 114 | assert not venv.exists() 115 | 116 | # the slow-python-package takes five seconds to compile 117 | assert time_savings(tmpdir, between=clean) > 5 118 | -------------------------------------------------------------------------------- /tests/functional/get_installed_test.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | from __future__ import unicode_literals 4 | 5 | import pytest 6 | 7 | from pip_faster import PY2 8 | from testing import run 9 | from venv_update import __version__ 10 | 11 | ALWAYS = {'pip', 'setuptools', 'venv-update', 'wheel'} 12 | 13 | 14 | def get_installed(): 15 | out, err = run('myvenv/bin/python', '-c', '''\ 16 | import pip_faster as p 17 | for p in sorted(p.reqnames(p.pip_get_installed())): 18 | print(p)''') 19 | 20 | assert err == '' 21 | out = set(out.split()) 22 | 23 | # Most python distributions which have argparse in the stdlib fail to 24 | # expose it to setuptools as an installed package (it seems all but ubuntu 25 | # do this). This results in argparse sometimes being installed locally, 26 | # sometimes not, even for a specific version of python. 27 | # We normalize by never looking at argparse =/ 28 | out -= {'argparse'} 29 | 30 | # these will always be present 31 | assert ALWAYS.issubset(out) 32 | return sorted(out - ALWAYS) 33 | 34 | 35 | @pytest.mark.usefixtures('pypi_server_with_fallback') 36 | def test_pip_get_installed(tmpdir): 37 | tmpdir.chdir() 38 | 39 | run('virtualenv', 'myvenv') 40 | run('rm', '-rf', 'myvenv/local') 41 | run('myvenv/bin/pip', 'install', 'venv-update==' + __version__) 42 | 43 | assert get_installed() == [] 44 | 45 | run( 46 | 'myvenv/bin/pip', 'install', 47 | 'pytest', 48 | 'git+git://github.com/bukzor/cov-core.git@master#egg=cov-core', 49 | '-e', 'git+git://github.com/bukzor/pytest-cov.git@master#egg=pytest-cov', 50 | ) 51 | 52 | expected = [ 53 | 'attrs', 54 | 'cov-core', 55 | 'coverage', 56 | 'importlib-metadata', 57 | 'packaging', 58 | 'pluggy', 59 | 'py', 60 | 'pyparsing', 61 | 'pytest', 62 | 'pytest-cov', 63 | 'zipp', 64 | ] 65 | 66 | if PY2: # :pragma:nocover: 67 | expected.extend([ 68 | 'atomicwrites', 69 | 'backports.functools-lru-cache', 70 | 'configparser', 71 | 'contextlib2', 72 | 'funcsigs', 73 | 'more-itertools', 74 | 'pathlib2', 75 | 'scandir', 76 | 'six', 77 | 'wcwidth', 78 | ]) 79 | else: 80 | # PY3 81 | expected.extend([ 82 | 'iniconfig', 83 | 'toml', 84 | 'typing-extensions', 85 | ]) 86 | 87 | # This seems prone to failure when deps can/will change. 88 | assert get_installed() == sorted(expected) 89 | 90 | run('myvenv/bin/pip', 'uninstall', '--yes', *expected) 91 | # Python2.7 workaround for: Can't uninstall 'pytest-cov'. No files were found to uninstall. 92 | assert get_installed() == [] if not PY2 else ['pytest-cov'] 93 | 94 | run('myvenv/bin/pip', 'install', 'flake8==2.5.0') 95 | assert {'flake8', 'mccabe', 'pep8', 'pyflakes'}.issubset(set(get_installed())) 96 | 97 | run('myvenv/bin/pip', 'uninstall', '--yes', 'flake8') 98 | assert {'mccabe', 'pep8', 'pyflakes'}.issubset(set(get_installed())) 99 | -------------------------------------------------------------------------------- /tests/functional/pip_faster.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | from __future__ import unicode_literals 4 | 5 | import os 6 | from subprocess import CalledProcessError 7 | 8 | import pytest 9 | 10 | from testing import cached_wheels 11 | from testing import install_coverage 12 | from testing import pip_freeze 13 | from testing import requirements 14 | from testing import run 15 | from testing import strip_pip_warnings 16 | from testing import TOP 17 | from testing import uncolor 18 | from venv_update import __version__ 19 | 20 | 21 | def it_shows_help_for_prune(): 22 | out, err = run('pip-faster', 'install', '--help') 23 | assert ''' 24 | --prune Uninstall any non-required packages. 25 | --no-prune Do not uninstall any non-required packages. 26 | 27 | Package Index Options''' in out 28 | assert err == '' 29 | 30 | 31 | @pytest.mark.usefixtures('pypi_server') 32 | def it_installs_stuff(tmpdir): 33 | venv = tmpdir.join('venv') 34 | install_coverage(venv) 35 | 36 | assert pip_freeze(str(venv)) == '''\ 37 | coverage==ANY 38 | coverage-enable-subprocess==1.0 39 | ''' 40 | 41 | pip = venv.join('bin/pip').strpath 42 | run(pip, 'install', 'venv-update==' + __version__) 43 | 44 | assert [ 45 | req.split('==')[0] 46 | for req in pip_freeze(str(venv)).split() 47 | ] == [ 48 | 'coverage', 49 | 'coverage-enable-subprocess', 50 | 'venv-update', 51 | ] 52 | 53 | run(str(venv.join('bin/pip-faster')), 'install', 'pure_python_package') 54 | 55 | assert 'pure-python-package==0.2.1' in pip_freeze(str(venv)).split('\n') 56 | 57 | 58 | @pytest.mark.usefixtures('pypi_server') 59 | def it_installs_stuff_from_requirements_file(tmpdir): 60 | venv = tmpdir.join('venv') 61 | install_coverage(venv) 62 | 63 | pip = venv.join('bin/pip').strpath 64 | run(pip, 'install', 'venv-update==' + __version__) 65 | 66 | # An arbitrary small package: pure_python_package 67 | requirements('pure_python_package\nproject_with_c') 68 | 69 | run(str(venv.join('bin/pip-faster')), 'install', '-r', 'requirements.txt') 70 | 71 | frozen_requirements = pip_freeze(str(venv)).split('\n') 72 | 73 | assert 'pure-python-package==0.2.1' in frozen_requirements 74 | assert 'project-with-c==0.1.0' in frozen_requirements 75 | 76 | 77 | @pytest.mark.usefixtures('pypi_server') 78 | def it_installs_stuff_with_dash_e_without_wheeling(tmpdir): 79 | venv = tmpdir.join('venv') 80 | install_coverage(venv) 81 | 82 | pip = venv.join('bin/pip').strpath 83 | run(pip, 'install', 'venv-update==' + __version__) 84 | 85 | # Install a package from git with no extra dependencies in editable mode. 86 | # 87 | # We need to install a package from VCS instead of the filesystem because 88 | # otherwise we aren't testing that editable requirements aren't wheeled 89 | # (and instead might just be testing that local paths aren't wheeled). 90 | requirements('-e git+git://github.com/Yelp/dumb-init.git@87545be699a13d0fd31f67199b7782ebd446437e#egg=dumb-init') # noqa 91 | 92 | run(str(venv.join('bin/pip-faster')), 'install', '-r', 'requirements.txt') 93 | 94 | frozen_requirements = pip_freeze(str(venv)).split('\n') 95 | assert set(frozen_requirements) == { 96 | '-e git://github.com/Yelp/dumb-init.git@87545be699a13d0fd31f67199b7782ebd446437e#egg=dumb_init', # noqa 97 | 'coverage-enable-subprocess==1.0', 98 | 'coverage==ANY', 99 | 'venv-update==' + __version__, 100 | '', 101 | } 102 | 103 | # we shouldn't wheel things installed editable 104 | assert not tuple(cached_wheels(tmpdir)) 105 | 106 | 107 | @pytest.mark.usefixtures('pypi_server') 108 | def it_caches_downloaded_wheels_from_pypi(tmpdir): 109 | venv = tmpdir.join('venv') 110 | install_coverage() 111 | 112 | pip = venv.join('bin/pip').strpath 113 | run(pip, 'install', 'venv-update==' + __version__) 114 | 115 | run( 116 | venv.join('bin/pip-faster').strpath, 'install', 117 | # One of the few wheeled things on our pypi 118 | 'wheeled-package', 119 | ) 120 | 121 | expected = {'wheeled-package'} 122 | assert {wheel.name for wheel in cached_wheels(tmpdir)} == expected 123 | 124 | 125 | @pytest.mark.usefixtures('pypi_server') 126 | def it_caches_downloaded_wheels_extra_index_url(tmpdir): 127 | venv = tmpdir.join('venv') 128 | install_coverage() 129 | 130 | pip = venv.join('bin/pip').strpath 131 | run(pip, 'install', 'venv-update==' + __version__) 132 | 133 | index = os.environ.pop('PIP_INDEX_URL') 134 | run( 135 | venv.join('bin/pip-faster').strpath, 'install', 136 | # bogus index url just to test `--extra-index-url` 137 | '--index-url', 'file://{}'.format(tmpdir), 138 | '--extra-index-url', index, 139 | 'wheeled-package', 140 | ) 141 | 142 | expected = {'wheeled-package'} 143 | assert {wheel.name for wheel in cached_wheels(tmpdir)} == expected 144 | 145 | 146 | @pytest.mark.usefixtures('pypi_server') 147 | def it_doesnt_wheel_local_dirs(tmpdir): 148 | venv = tmpdir.join('venv') 149 | install_coverage(venv) 150 | 151 | pip = venv.join('bin/pip').strpath 152 | run(pip, 'install', 'venv-update==' + __version__) 153 | 154 | run( 155 | venv.join('bin/pip-faster').strpath, 156 | 'install', 157 | TOP.join('tests/testing/packages/dependant_package').strpath, 158 | ) 159 | 160 | frozen_requirements = pip_freeze(str(venv)).split('\n') 161 | assert set(frozen_requirements) == { 162 | 'coverage==ANY', 163 | 'coverage-enable-subprocess==1.0', 164 | 'dependant-package==1', 165 | 'implicit-dependency==1', 166 | 'many-versions-package==3', 167 | 'pure-python-package==0.2.1', 168 | 'venv-update==' + __version__, 169 | '', 170 | } 171 | 172 | assert {wheel.name for wheel in cached_wheels(tmpdir)} == { 173 | 'implicit-dependency', 174 | 'many-versions-package', 175 | 'pure-python-package', 176 | } 177 | 178 | 179 | @pytest.mark.usefixtures('pypi_server') 180 | def it_doesnt_wheel_git_repos(tmpdir): 181 | venv = tmpdir.join('venv') 182 | install_coverage(venv) 183 | 184 | pip = venv.join('bin/pip').strpath 185 | run(pip, 'install', 'venv-update==' + __version__) 186 | 187 | run( 188 | venv.join('bin/pip-faster').strpath, 189 | 'install', 190 | 'git+git://github.com/Yelp/dumb-init.git@87545be699a13d0fd31f67199b7782ebd446437e#egg=dumb-init', # noqa 191 | ) 192 | 193 | frozen_requirements = pip_freeze(str(venv)).split('\n') 194 | assert set(frozen_requirements) == { 195 | 'coverage-enable-subprocess==1.0', 196 | 'coverage==ANY', 197 | 'dumb-init==0.5.0', 198 | 'venv-update==' + __version__, 199 | '', 200 | } 201 | 202 | assert not tuple(cached_wheels(tmpdir)) 203 | 204 | 205 | @pytest.mark.usefixtures('pypi_server') 206 | def it_can_handle_requirements_already_met(tmpdir): 207 | venv = tmpdir.join('venv') 208 | install_coverage(venv) 209 | 210 | pip = venv.join('bin/pip').strpath 211 | run(pip, 'install', 'venv-update==' + __version__) 212 | 213 | requirements('many-versions-package==1') 214 | 215 | run(str(venv.join('bin/pip-faster')), 'install', '-r', 'requirements.txt') 216 | assert 'many-versions-package==1\n' in pip_freeze(str(venv)) 217 | 218 | run(str(venv.join('bin/pip-faster')), 'install', '-r', 'requirements.txt') 219 | assert 'many-versions-package==1\n' in pip_freeze(str(venv)) 220 | 221 | 222 | @pytest.mark.usefixtures('pypi_server') 223 | def it_gives_proper_error_without_requirements(tmpdir): 224 | venv = tmpdir.join('venv') 225 | install_coverage(venv) 226 | 227 | pip = venv.join('bin/pip').strpath 228 | run(pip, 'install', 'venv-update==' + __version__) 229 | 230 | with pytest.raises(CalledProcessError) as exc_info: 231 | run(str(venv.join('bin/pip-faster')), 'install') 232 | _, err = exc_info.value.result 233 | assert err.startswith('ERROR: You must give at least one requirement to install') 234 | 235 | 236 | @pytest.mark.usefixtures('pypi_server') 237 | def it_can_handle_a_bad_findlink(tmpdir): 238 | venv = tmpdir.join('venv') 239 | install_coverage(venv) 240 | 241 | pip = venv.join('bin/pip').strpath 242 | run(pip, 'install', 'venv-update==' + __version__) 243 | 244 | out, err = run( 245 | str(venv.join('bin/pip-faster')), 246 | 'install', '-vvv', 247 | '--find-links', 'git+wat://not/a/thing', 248 | 'pure-python-package', 249 | ) 250 | out = uncolor(out) 251 | err = strip_pip_warnings(err) 252 | 253 | expected = '''\ 254 | Successfully built pure-python-package 255 | Installing collected packages: pure-python-package 256 | ''' 257 | assert expected in out 258 | # Between this there's: 259 | # 'changing mode of .../venv/bin/pure-python-script to 775' 260 | # but that depends on umask 261 | _, rest = out.split(expected) 262 | expected2 = '''\ 263 | Successfully installed pure-python-package-0.2.1 264 | Cleaning up... 265 | ''' 266 | assert expected2 in rest 267 | assert "Url 'git+wat://not/a/thing' is ignored." in err 268 | assert 'pure-python-package==0.2.1' in pip_freeze(str(venv)).split('\n') 269 | 270 | 271 | @pytest.mark.usefixtures('pypi_server') 272 | def it_considers_equals_star_not_pinned(tmpdir): 273 | venv = tmpdir.join('venv') 274 | install_coverage(venv) 275 | 276 | pip = venv.join('bin/pip').strpath 277 | run(pip, 'install', 'venv-update==' + __version__) 278 | 279 | run( 280 | str(venv.join('bin/pip-faster')), 281 | 'install', 'many-versions-package==2', 282 | ) 283 | run( 284 | str(venv.join('bin/pip-faster')), 285 | 'install', '--upgrade', 'many-versions-package==2.*', 286 | ) 287 | assert 'many-versions-package==2.1' in pip_freeze(str(venv)).split('\n') 288 | 289 | 290 | @pytest.mark.usefixtures('pypi_server') 291 | def test_no_conflicts_when_no_deps_specified(tmpdir): 292 | venv = tmpdir.join('venv') 293 | install_coverage(venv) 294 | 295 | pip = venv.join('bin/pip').strpath 296 | run(pip, 'install', 'venv-update==' + __version__) 297 | 298 | pkgdir = tmpdir.join('pkgdir').ensure_dir() 299 | setup_py = pkgdir.join('setup.py') 300 | 301 | def _setup_py(many_versions_package_version): 302 | setup_py.write( 303 | 'from setuptools import setup\n' 304 | 'setup(\n' 305 | ' name="pkg",\n' 306 | ' install_requires=["many-versions-package=={}"],\n' 307 | ')\n'.format(many_versions_package_version) 308 | ) 309 | 310 | cmd = ( 311 | venv.join('bin/pip-faster').strpath, 'install', '--upgrade', 312 | pkgdir.strpath, 313 | ) 314 | 315 | _setup_py('1') 316 | run(*cmd) 317 | 318 | _setup_py('2') 319 | # Should not complain about conflicts since we specified `--no-deps` 320 | run(*cmd + ('--no-deps',)) 321 | -------------------------------------------------------------------------------- /tests/functional/relocation_test.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | from __future__ import unicode_literals 4 | 5 | import pytest 6 | 7 | from testing import Path 8 | from testing import requirements 9 | from testing import run 10 | from testing import venv_update 11 | 12 | 13 | @pytest.mark.usefixtures('pypi_server') 14 | def test_relocatable(tmpdir): 15 | tmpdir.chdir() 16 | requirements('') 17 | venv_update() 18 | 19 | Path('venv').rename('relocated') 20 | 21 | python = 'relocated/bin/python' 22 | assert Path(python).exists() 23 | run(python, '-m', 'pip.__main__', '--version') 24 | -------------------------------------------------------------------------------- /tests/functional/simple_test.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | from __future__ import unicode_literals 4 | 5 | import re 6 | import sys 7 | from subprocess import CalledProcessError 8 | 9 | import pytest 10 | from py._path.local import LocalPath as Path 11 | 12 | from testing import cached_wheels 13 | from testing import enable_coverage 14 | from testing import install_coverage 15 | from testing import OtherPython 16 | from testing import pip_freeze 17 | from testing import requirements 18 | from testing import run 19 | from testing import strip_coverage_warnings 20 | from testing import strip_pip_warnings 21 | from testing import TOP 22 | from testing import uncolor 23 | from testing import venv_update 24 | from venv_update import __version__ 25 | 26 | 27 | @pytest.mark.usefixtures('pypi_server') 28 | def test_trivial(tmpdir): 29 | tmpdir.chdir() 30 | requirements('') 31 | enable_coverage() 32 | venv_update() 33 | # Originally suggested by none other than @bukzor in: 34 | # https://github.com/pypa/virtualenv/issues/118 35 | # This directory now just causes problems (especially with relocating) 36 | # since the debian issue has been fixed. 37 | assert not tmpdir.join('venv', 'local').exists() 38 | 39 | 40 | @pytest.mark.usefixtures('pypi_server_with_fallback') 41 | def test_install_custom_path_and_requirements(tmpdir): 42 | """Show that we can install to a custom directory with a custom 43 | requirements file.""" 44 | tmpdir.chdir() 45 | requirements( 46 | 'mccabe==0.6.0\n', 47 | path='requirements2.txt', 48 | ) 49 | enable_coverage() 50 | venv_update('venv=', 'venv2', 'install=', '-r', 'requirements2.txt') 51 | assert pip_freeze('venv2') == '\n'.join(( 52 | 'mccabe==0.6.0', 53 | 'venv-update==' + __version__, 54 | '' 55 | )) 56 | 57 | 58 | @pytest.mark.usefixtures('pypi_server') 59 | def test_arguments_version(tmpdir): 60 | """Show that we can pass arguments through to virtualenv""" 61 | tmpdir.chdir() 62 | enable_coverage() 63 | 64 | # should show virtualenv version, successfully 65 | out, err = venv_update('venv=', '--version') 66 | err = strip_pip_warnings(err) 67 | if sys.version_info < (3, 0): 68 | import virtualenv 69 | assert 'virtualenv {}'.format(virtualenv.__version__) in err 70 | else: 71 | assert err == '' 72 | 73 | out = uncolor(out) 74 | lines = out.splitlines() 75 | if sys.version_info < (3, 0): 76 | assert lines[-1] == '> virtualenv --version', repr(lines) 77 | else: 78 | assert lines[-2] == '> virtualenv --version', repr(lines) 79 | 80 | 81 | @pytest.mark.skipif('__pypy__' in sys.builtin_module_names, reason="site-packages doesn't show up under pypy for some reason") 82 | @pytest.mark.usefixtures('pypi_server') 83 | def test_arguments_system_packages(tmpdir): 84 | """Show that we can pass arguments through to virtualenv""" 85 | tmpdir.chdir() 86 | requirements('') 87 | 88 | venv_update('venv=', '--system-site-packages', 'venv') 89 | # virtualenv>20 doesn't set sys.real_prefix anymore. The accepted method 90 | # for checking if we are in a virtual environment is to check for base_prefix 91 | # See: https://github.com/pypa/virtualenv/issues/1622 92 | out, err = run('venv/bin/python', '-c', '''\ 93 | import sys 94 | non_venv_prefix = sys.real_prefix if hasattr(sys, "real_prefix") else sys.base_prefix 95 | for p in sys.path: 96 | if p.startswith(non_venv_prefix) and p.endswith("-packages"): 97 | print(p) 98 | break 99 | ''') 100 | assert err == '' 101 | out = out.rstrip('\n') 102 | assert out and Path(out).isdir() 103 | 104 | 105 | @pytest.mark.skipif(sys.version_info < (3, 0), reason='fails on Python2 + not worth figuring out why') 106 | @pytest.mark.usefixtures('pypi_server') 107 | def test_eggless_url(tmpdir): 108 | tmpdir.chdir() 109 | 110 | enable_coverage() 111 | 112 | # An arbitrary url requirement. 113 | requirements('-e file://' + str(TOP / 'tests/testing/packages/pure_python_package')) 114 | 115 | venv_update() 116 | assert '#egg=pure_python_package' in pip_freeze() 117 | 118 | 119 | @pytest.mark.usefixtures('pypi_server') 120 | def test_not_installable_thing(tmpdir): 121 | tmpdir.chdir() 122 | enable_coverage() 123 | 124 | install_coverage() 125 | 126 | requirements('not-a-real-package-plz') 127 | with pytest.raises(CalledProcessError): 128 | venv_update() 129 | 130 | 131 | @pytest.mark.usefixtures('pypi_server', 'pypi_packages') 132 | def test_doesnt_use_cache_without_index_server(tmpdir): 133 | tmpdir.chdir() 134 | enable_coverage() 135 | 136 | requirements('pure-python-package==0.2.1') 137 | venv_update() 138 | 139 | tmpdir.join('venv').remove() 140 | install_coverage() 141 | 142 | cmd = ('pip-command=', 'pip-faster', 'install') 143 | with pytest.raises(CalledProcessError): 144 | venv_update(*(cmd + ('--no-index',))) 145 | # But it would succeed if we gave it an index 146 | venv_update(*cmd) 147 | 148 | 149 | @pytest.mark.usefixtures('pypi_server', 'pypi_packages') 150 | def test_extra_index_url_doesnt_cache(tmpdir): 151 | tmpdir.chdir() 152 | enable_coverage() 153 | install_coverage() 154 | 155 | requirements('pure-python-package==0.2.1') 156 | venv_update( 157 | 'pip-command=', 'pip-faster', 'install', 158 | '--extra-index-url=https://pypi.python.org/simple', 159 | ) 160 | 161 | assert not tuple(cached_wheels(tmpdir)) 162 | 163 | 164 | @pytest.mark.usefixtures('pypi_server_with_fallback') 165 | def test_scripts_left_behind(tmpdir): 166 | tmpdir.chdir() 167 | requirements('') 168 | 169 | venv_update() 170 | 171 | # an arbitrary small package with a script: pep8 172 | script_path = Path('venv/bin/pep8') 173 | assert not script_path.exists() 174 | 175 | run('venv/bin/pip', 'install', 'pep8') 176 | assert script_path.exists() 177 | 178 | venv_update() 179 | assert not script_path.exists() 180 | 181 | 182 | def assert_timestamps(*reqs): 183 | firstreq = Path(reqs[0]) 184 | lastreq = Path(reqs[-1]) 185 | args = ['install='] + sum([['-r', req] for req in reqs], []) 186 | 187 | venv_update(*args) 188 | 189 | assert firstreq.mtime() < Path('venv').mtime() 190 | 191 | # garbage, to cause a failure 192 | lastreq.write('-w wat') 193 | 194 | with pytest.raises(CalledProcessError) as excinfo: 195 | venv_update(*args) 196 | 197 | assert excinfo.value.returncode == 1 198 | assert firstreq.mtime() > Path('venv').mtime() 199 | 200 | # blank requirements should succeed 201 | lastreq.write('') 202 | 203 | venv_update(*args) 204 | assert firstreq.mtime() < Path('venv').mtime() 205 | 206 | 207 | @pytest.mark.usefixtures('pypi_server') 208 | def test_timestamps_single(tmpdir): 209 | tmpdir.chdir() 210 | requirements('') 211 | assert_timestamps('requirements.txt') 212 | 213 | 214 | @pytest.mark.usefixtures('pypi_server') 215 | def test_timestamps_multiple(tmpdir): 216 | tmpdir.chdir() 217 | requirements('') 218 | Path('requirements2.txt').write('') 219 | assert_timestamps('requirements.txt', 'requirements2.txt') 220 | 221 | 222 | def pipe_output(read, write): 223 | from os import environ 224 | environ = environ.copy() 225 | 226 | from subprocess import Popen 227 | vupdate = Popen( 228 | ('venv-update', 'venv=', '--version'), 229 | env=environ, 230 | stderr=write, 231 | stdout=write, 232 | close_fds=True, 233 | ) 234 | 235 | from os import close 236 | from testing.capture_subprocess import read_all 237 | close(write) 238 | result = read_all(read) 239 | vupdate.wait() 240 | 241 | result = result.decode('US-ASCII') 242 | print(result) 243 | uncolored = uncolor(result) 244 | assert uncolored.startswith('> ') 245 | # FIXME: Sometimes this is 'python -m', sometimes 'python2.7 -m'. Weird. 246 | import virtualenv 247 | split_uncolored = uncolored.strip().split('\n') 248 | version_cmd_index = split_uncolored.index('> virtualenv --version') 249 | assert 'virtualenv {}'.format(virtualenv.__version__) in split_uncolored[version_cmd_index + 1] 250 | 251 | return result, uncolored 252 | 253 | 254 | @pytest.mark.usefixtures('pypi_server') 255 | def test_colored_tty(tmpdir): 256 | tmpdir.chdir() 257 | 258 | from os import openpty 259 | read, write = openpty() 260 | 261 | from testing.capture_subprocess import pty_normalize_newlines 262 | pty_normalize_newlines(read) 263 | 264 | out, uncolored = pipe_output(read, write) 265 | 266 | assert out != uncolored 267 | 268 | 269 | @pytest.mark.usefixtures('pypi_server') 270 | def test_uncolored_pipe(tmpdir): 271 | tmpdir.chdir() 272 | 273 | from os import pipe 274 | read, write = pipe() 275 | 276 | out, uncolored = pipe_output(read, write) 277 | 278 | assert out == uncolored 279 | 280 | 281 | @pytest.mark.usefixtures('pypi_server') 282 | def test_args_backward(tmpdir): 283 | tmpdir.chdir() 284 | enable_coverage() 285 | requirements('') 286 | 287 | with pytest.raises(CalledProcessError) as excinfo: 288 | venv_update('venv=', 'requirements.txt') 289 | assert excinfo.value.returncode == 2 290 | out, err = excinfo.value.result 291 | err = strip_coverage_warnings(err) 292 | err = strip_pip_warnings(err) 293 | out = uncolor(out) 294 | assert '> virtualenv requirements.txt' in out 295 | assert 'virtualenv: error: argument dest: the destination requirements.txt already exists and is a file' in err 296 | 297 | assert Path('requirements.txt').isfile() 298 | assert Path('requirements.txt').read() == '' 299 | assert not Path('myvenv').exists() 300 | 301 | 302 | @pytest.mark.usefixtures('pypi_server') 303 | def test_wrong_wheel(tmpdir): 304 | tmpdir.chdir() 305 | 306 | requirements('pure_python_package==0.1.0') 307 | venv_update('venv=', 'venv1') 308 | # A different python 309 | # Before fixing, this would install argparse using the `py2-none-any` 310 | # wheel, even on py3 311 | other_python = OtherPython() 312 | ret2out, _ = venv_update('venv=', 'venv2', '-p' + other_python.interpreter, 'install=', '-vv', '-r', 'requirements.txt') 313 | 314 | assert ''' 315 | No wheel found locally for pinned requirement pure_python_package==0.1.0 (from -r requirements.txt (line 1)) 316 | ''' in uncolor(ret2out) 317 | 318 | 319 | def flake8_older(): 320 | requirements('''\ 321 | flake8==2.0 322 | # last pyflakes release before 0.8 was 0.7.3 323 | pyflakes<0.8 324 | 325 | # simply to prevent these from drifting: 326 | mccabe<=0.3 327 | pep8<=1.5.7 328 | 329 | -r %s/requirements.d/coverage.txt 330 | ''' % TOP) 331 | venv_update() 332 | assert pip_freeze() == '\n'.join(( 333 | 'coverage==ANY', 334 | 'coverage-enable-subprocess==1.0', 335 | 'flake8==2.0', 336 | 'mccabe==0.3', 337 | 'pep8==1.5.7', 338 | 'pyflakes==0.7.3', 339 | 'venv-update==' + __version__, 340 | '' 341 | )) 342 | 343 | 344 | def flake8_newer(): 345 | requirements('''\ 346 | flake8==2.2.5 347 | # we expect 0.8.1 348 | pyflakes<=0.8.1 349 | 350 | # simply to prevent these from drifting: 351 | mccabe<=0.3 352 | pep8<=1.5.7 353 | 354 | -r %s/requirements.d/coverage.txt 355 | ''' % TOP) 356 | venv_update() 357 | assert pip_freeze() == '\n'.join(( 358 | 'coverage==ANY', 359 | 'coverage-enable-subprocess==1.0', 360 | 'flake8==2.2.5', 361 | 'mccabe==0.3', 362 | 'pep8==1.5.7', 363 | 'pyflakes==0.8.1', 364 | 'venv-update==' + __version__, 365 | '' 366 | )) 367 | 368 | 369 | @pytest.mark.usefixtures('pypi_server_with_fallback') 370 | def test_upgrade(tmpdir): 371 | tmpdir.chdir() 372 | flake8_older() 373 | flake8_newer() 374 | 375 | 376 | @pytest.mark.usefixtures('pypi_server_with_fallback') 377 | def test_downgrade(tmpdir): 378 | tmpdir.chdir() 379 | flake8_newer() 380 | flake8_older() 381 | 382 | 383 | @pytest.mark.usefixtures('pypi_server') 384 | def test_package_name_normalization(tmpdir): 385 | with tmpdir.as_cwd(): 386 | enable_coverage() 387 | requirements('WEIRD_cAsing-packAge') 388 | 389 | venv_update() 390 | assert '\nweird-CASING-pACKage==' in pip_freeze() 391 | 392 | 393 | @pytest.mark.usefixtures('pypi_server') 394 | @pytest.mark.parametrize('install_req', ('dotted.package-name', 'dotted-package-name')) 395 | def test_package_name_normalization_with_dots(tmpdir, install_req): 396 | """Packages with dots should be installable with either dots or dashes.""" 397 | with tmpdir.as_cwd(): 398 | enable_coverage() 399 | requirements(install_req) 400 | 401 | venv_update() 402 | assert pip_freeze().startswith('dotted.package-name==') 403 | 404 | 405 | @pytest.mark.usefixtures('pypi_server') 406 | def test_override_requirements_file(tmpdir): 407 | tmpdir.chdir() 408 | enable_coverage() 409 | requirements('') 410 | Path('.').join('requirements-bootstrap.txt').write('''\ 411 | venv-update==%s 412 | pure_python_package 413 | ''' % __version__) 414 | out, err = venv_update( 415 | 'bootstrap-deps=', '-r', 'requirements-bootstrap.txt', 416 | ) 417 | err = strip_pip_warnings(err) 418 | # pip>=10 doesn't complain about installing an empty requirements file. 419 | assert err == '' 420 | 421 | out = uncolor(out) 422 | # installing venv-update may downgrade / upgrade pip 423 | out = re.sub(' pip-[0-9.]+ ', ' ', out) 424 | assert '\n> pip install -r requirements-bootstrap.txt\n' in out 425 | assert ( 426 | '\nSuccessfully installed pure-python-package-0.2.1 venv-update-%s' % __version__ 427 | ) in out 428 | assert '\n Successfully uninstalled pure-python-package-0.2.1\n' in out 429 | 430 | expected = '\n'.join(( 431 | 'venv-update==' + __version__, 432 | '' 433 | )) 434 | assert pip_freeze() == expected 435 | 436 | 437 | @pytest.mark.usefixtures('pypi_server') 438 | def test_cant_wheel_package(tmpdir): 439 | with tmpdir.as_cwd(): 440 | enable_coverage() 441 | install_coverage() 442 | requirements('cant-wheel-package\npure-python-package') 443 | 444 | out, err = venv_update() 445 | err = strip_pip_warnings(err) 446 | assert err.strip() == 'Failed building wheel for cant-wheel-package' 447 | 448 | out = uncolor(out) 449 | 450 | assert '''\ 451 | Installing collected packages: cant-wheel-package, pure-python-package 452 | Running setup.py install for cant-wheel-package ... done 453 | Successfully installed cant-wheel-package-0.1.0 pure-python-package-0.2.1 454 | ''' in out # noqa 455 | assert pip_freeze().startswith( 456 | 'cant-wheel-package==0.1.0\n' 457 | ) 458 | 459 | 460 | @pytest.mark.usefixtures('pypi_server') 461 | def test_has_extras(tmpdir): 462 | with tmpdir.as_cwd(): 463 | enable_coverage() 464 | install_coverage() 465 | requirements('pure-python-package[my-extra]') 466 | 467 | for _ in range(2): 468 | venv_update() 469 | 470 | expected = '\n'.join(( 471 | 'implicit-dependency==1', 472 | 'pure-python-package==0.2.1', 473 | 'venv-update==' + __version__, 474 | '' 475 | )) 476 | assert pip_freeze() == expected 477 | -------------------------------------------------------------------------------- /tests/functional/validation.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | from __future__ import unicode_literals 4 | 5 | import fileinput 6 | 7 | import pytest 8 | 9 | from testing import enable_coverage 10 | from testing import OtherPython 11 | from testing import Path 12 | from testing import pip_freeze 13 | from testing import requirements 14 | from testing import run 15 | from testing import strip_pip_warnings 16 | from testing import TOP 17 | from testing import uncolor 18 | from testing import venv_update 19 | from testing import venv_update_symlink_pwd 20 | from venv_update import __version__ 21 | 22 | 23 | def assert_c_extension_runs(): 24 | out, err = run('venv/bin/c-extension-script') 25 | assert err == '' 26 | assert out == 'hello world\n' 27 | 28 | out, err = run('sh', '-c', '. venv/bin/activate && c-extension-script') 29 | assert err == '' 30 | assert out == 'hello world\n' 31 | 32 | 33 | def assert_python_version(version): 34 | outputs = run('sh', '-c', '. venv/bin/activate && python -c "import sys; print(sys.version)"') 35 | 36 | # older versions of python output on stderr, newer on stdout, but we dont care too much which 37 | assert '' in outputs 38 | actual_version = ''.join(outputs) 39 | assert actual_version.startswith(version) 40 | return actual_version 41 | 42 | 43 | @pytest.mark.usefixtures('pypi_server') 44 | def test_python_versions(tmpdir): 45 | tmpdir.chdir() 46 | enable_coverage() 47 | requirements('project-with-c') 48 | 49 | other_python = OtherPython() 50 | venv_update('venv=', '--python=' + other_python.interpreter, 'venv') 51 | assert_c_extension_runs() 52 | assert_python_version(other_python.version_prefix) 53 | 54 | from sys import executable as python 55 | venv_update('venv=', '--python=' + python, 'venv') 56 | assert_c_extension_runs() 57 | from sys import version 58 | assert_python_version(version) 59 | 60 | venv_update('venv=', '--python=' + other_python.interpreter, 'venv') 61 | assert_c_extension_runs() 62 | assert_python_version(other_python.version_prefix) 63 | 64 | 65 | @pytest.mark.usefixtures('pypi_server') 66 | def test_virtualenv_moved(tmpdir): 67 | """if you move the virtualenv and venv-update again, the old will be blown away, and things will work""" 68 | original_path = 'original' 69 | new_path = 'new_dir' 70 | 71 | with tmpdir.mkdir(original_path).as_cwd(): 72 | enable_coverage() 73 | requirements('project_with_c') 74 | venv_update() 75 | assert_c_extension_runs() 76 | 77 | with tmpdir.as_cwd(): 78 | Path(original_path).rename(new_path) 79 | 80 | with tmpdir.join(new_path).as_cwd(): 81 | with pytest.raises(OSError) as excinfo: 82 | assert_c_extension_runs() 83 | # python >= 3.3 raises FileNotFoundError 84 | assert excinfo.type.__name__ in ('OSError', 'FileNotFoundError') 85 | assert excinfo.value.args[0] == 2 # no such file 86 | 87 | venv_update() 88 | assert_c_extension_runs() 89 | 90 | 91 | @pytest.mark.usefixtures('pypi_server') 92 | def test_recreate_active_virtualenv(tmpdir): 93 | with tmpdir.as_cwd(): 94 | enable_coverage() 95 | 96 | run('virtualenv', 'venv') 97 | run('venv/bin/pip', 'install', '-r', str(TOP / 'requirements.d/coverage.txt')) 98 | 99 | requirements('project_with_c') 100 | venv_update_symlink_pwd() 101 | run('venv/bin/python', 'venv_update.py') 102 | 103 | assert_c_extension_runs() 104 | 105 | 106 | @pytest.mark.usefixtures('pypi_server') 107 | def test_update_while_active(tmpdir): 108 | tmpdir.chdir() 109 | enable_coverage() 110 | requirements('') 111 | 112 | venv_update() 113 | assert 'project-with-c' not in pip_freeze() 114 | 115 | # An arbitrary small package: project_with_c 116 | requirements('project_with_c') 117 | 118 | venv_update_symlink_pwd() 119 | out, err = run('sh', '-c', '. venv/bin/activate && python venv_update.py venv= venv --python=venv/bin/python') 120 | out = uncolor(out) 121 | err = strip_pip_warnings(err) 122 | 123 | assert err == '' 124 | assert out.startswith('''\ 125 | > virtualenv venv --python=venv/bin/python 126 | Keeping valid virtualenv from previous run. 127 | ''') 128 | assert 'project-with-c' in pip_freeze() 129 | 130 | 131 | @pytest.mark.usefixtures('pypi_server') 132 | def test_update_invalidated_while_active(tmpdir): 133 | tmpdir.chdir() 134 | enable_coverage() 135 | requirements('') 136 | 137 | venv_update() 138 | assert 'project-with-c' not in pip_freeze() 139 | 140 | # An arbitrary small package: project_with_c 141 | requirements('project-with-c') 142 | 143 | venv_update_symlink_pwd() 144 | out, err = run('sh', '-c', '. venv/bin/activate && python venv_update.py venv= --system-site-packages venv') 145 | 146 | err = strip_pip_warnings(err) 147 | assert err == '' 148 | out = uncolor(out) 149 | assert out.startswith('''\ 150 | > virtualenv --system-site-packages venv 151 | Removing invalidated virtualenv. (system-site-packages changed, to True) 152 | ''') 153 | assert 'project-with-c' in pip_freeze() 154 | 155 | 156 | @pytest.mark.usefixtures('pypi_server') 157 | def test_update_invalidated_missing_activate(tmpdir): 158 | with tmpdir.as_cwd(): 159 | enable_coverage() 160 | requirements('') 161 | 162 | venv_update() 163 | tmpdir.join('venv/bin/activate').remove() 164 | 165 | out, err = venv_update() 166 | err = strip_pip_warnings(err) 167 | assert err.strip() == "sh: 1: .: Can't open venv/bin/activate" 168 | out = uncolor(out) 169 | assert out.startswith('''\ 170 | > virtualenv venv 171 | Removing invalidated virtualenv. (could not inspect metadata) 172 | ''') 173 | 174 | 175 | @pytest.mark.usefixtures('pypi_server') 176 | def test_update_invalidated_missing_pyvenv_cfg(tmpdir): 177 | with tmpdir.as_cwd(): 178 | enable_coverage() 179 | requirements('') 180 | 181 | venv_update() 182 | tmpdir.join('venv/pyvenv.cfg').remove() 183 | 184 | out, _ = venv_update() 185 | out = uncolor(out) 186 | assert out.startswith('''\ 187 | > virtualenv venv 188 | Removing invalidated virtualenv. (virtualenv created with virtualenv<20) 189 | ''') 190 | 191 | 192 | @pytest.mark.usefixtures('pypi_server') 193 | def test_update_invalidated_changed_base_executable(tmpdir): 194 | with tmpdir.as_cwd(): 195 | enable_coverage() 196 | requirements('') 197 | 198 | from sys import executable as python 199 | venv_update('venv=', '--python=' + python, 'venv') 200 | 201 | for line in fileinput.input(str(tmpdir.join('venv/pyvenv.cfg')), inplace=True): 202 | if line.startswith('base-executable = '): 203 | print('base-executable = /some/other/python') 204 | else: 205 | print(line, end='') 206 | out, _ = venv_update('venv=', '--python=' + python, 'venv') 207 | out = uncolor(out) 208 | assert out.startswith('''\ 209 | > virtualenv --python={} venv 210 | Removing invalidated virtualenv. (base executable python version changed'''.format(python)) 211 | 212 | 213 | @pytest.mark.usefixtures('pypi_server') 214 | def it_gives_the_same_python_version_as_we_started_with(tmpdir): 215 | other_python = OtherPython() 216 | with tmpdir.as_cwd(): 217 | requirements('') 218 | 219 | # first simulate some unrelated use of venv-update 220 | # this guards against statefulness in the venv-update scratch dir 221 | venv_update('venv=', 'unrelated_venv', 'pip-command=', 'true') 222 | 223 | run('virtualenv', '--python', other_python.interpreter, 'venv') 224 | initial_version = assert_python_version(other_python.version_prefix) 225 | 226 | venv_update_symlink_pwd() 227 | out, err = run('./venv/bin/python', 'venv_update.py') 228 | 229 | err = strip_pip_warnings(err) 230 | assert err == '' 231 | out = uncolor(out) 232 | assert out.startswith('''\ 233 | > virtualenv venv 234 | Keeping valid virtualenv from previous run. 235 | > rm -rf venv/local 236 | > pip install venv-update=={} 237 | '''.format(__version__)) 238 | 239 | final_version = assert_python_version(other_python.version_prefix) 240 | assert final_version == initial_version 241 | -------------------------------------------------------------------------------- /tests/regression/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/venv-update/5fb5491bd421fdd8ef3cff3faa5d4846b5985ec8/tests/regression/__init__.py -------------------------------------------------------------------------------- /tests/regression/pip_faster.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | from __future__ import unicode_literals 4 | 5 | import sys 6 | 7 | import pytest 8 | 9 | from testing import cached_wheels 10 | from testing import enable_coverage 11 | from testing import install_coverage 12 | from testing import Path 13 | from testing import pip_freeze 14 | from testing import run 15 | from testing import strip_pip_warnings 16 | from testing import uncolor 17 | from venv_update import __version__ 18 | 19 | 20 | def make_venv(): 21 | enable_coverage() 22 | venv = Path('venv') 23 | run('virtualenv', venv.strpath) 24 | install_coverage(venv.strpath) 25 | 26 | pip = venv.join('bin/pip').strpath 27 | run(pip, 'install', 'venv-update==' + __version__) 28 | return venv 29 | 30 | 31 | @pytest.mark.usefixtures('pypi_server', 'tmpdir') 32 | def test_circular_dependencies(): 33 | """pip-faster should be able to install packages with circular 34 | dependencies.""" 35 | venv = make_venv() 36 | 37 | out, err = run( 38 | venv.join('bin/pip-faster').strpath, 39 | 'install', 40 | '-vv', # show debug logging 41 | 'circular-dep-a', 42 | ) 43 | err = strip_pip_warnings(err) 44 | assert err.strip() == ( 45 | 'Circular dependency! circular-dep-a==1.0 ' 46 | '(from circular-dep-b==1.0->circular-dep-a)' 47 | ) 48 | out = uncolor(out) 49 | assert ''' 50 | tracing: circular-dep-a 51 | already queued: circular-dep-b==1.0 (from circular-dep-a) 52 | tracing: circular-dep-b==1.0 (from circular-dep-a) 53 | ''' in out 54 | 55 | frozen_requirements = pip_freeze(str(venv)).split('\n') 56 | assert 'circular-dep-a==1.0' in frozen_requirements 57 | assert 'circular-dep-b==1.0' in frozen_requirements 58 | 59 | 60 | @pytest.mark.usefixtures('pypi_server') 61 | @pytest.mark.skipif( 62 | sys.version_info > (3, 0), 63 | reason='ancient versions are not py3 compatible, even for install', 64 | ) 65 | @pytest.mark.parametrize('reqs', [ 66 | # new setuptools and old pip 67 | [ 68 | 'setuptools==18.2', 69 | # Non-SNI compatible clients (i.e. pip<2.7.9) cannot access public pypi anymore. 70 | # pip==6.0 is the earliest supported version. See https://github.com/pypa/pypi-support/issues/978 71 | 'pip==6.0', 72 | ], 73 | ]) 74 | def test_old_pip_and_setuptools(tmpdir, reqs): 75 | """We should be able to use pip-faster's wheel building even if we have 76 | ancient pip and setuptools. 77 | 78 | https://github.com/Yelp/venv-update/issues/33 79 | """ 80 | tmpdir.chdir() 81 | 82 | # 1. Create an empty virtualenv. 83 | # 2. Install old pip/setuptools that don't support wheel building. 84 | # 3. Install pip-faster. 85 | # 4. Install pure-python-package and assert it was wheeled during install. 86 | tmpdir.join('venv') 87 | venv = Path('venv') 88 | run('virtualenv', venv.strpath) 89 | 90 | # We need to add public PyPI as an extra URL since we're installing 91 | # packages (setuptools and pip) which aren't available from our PyPI fixture. 92 | from os import environ 93 | environ['PIP_EXTRA_INDEX_URL'] = 'https://pypi.org/simple/' 94 | try: 95 | pip = venv.join('bin/pip').strpath 96 | for req in reqs: 97 | run(pip, 'install', '--', req) 98 | run(pip, 'install', 'venv-update==' + __version__) 99 | finally: 100 | del environ['PIP_EXTRA_INDEX_URL'] 101 | 102 | run(str(venv.join('bin/pip-faster')), 'install', 'pure_python_package') 103 | 104 | # it was installed 105 | assert 'pure-python-package==0.2.1' in pip_freeze(str(venv)).split('\n') 106 | 107 | # it was wheeled 108 | wheel_names = [wheel.name for wheel in cached_wheels(tmpdir)] 109 | assert 'pure-python-package' in wheel_names 110 | 111 | 112 | @pytest.mark.usefixtures('tmpdir') 113 | def test_install_whl_over_http(pypi_server): 114 | whl_url = pypi_server + '/packages/wheeled_package-0.2.0-py2.py3-none-any.whl' 115 | venv = make_venv() 116 | 117 | out, err = run(str(venv.join('bin/pip-faster')), 'install', whl_url) 118 | err = strip_pip_warnings(err) 119 | assert err == '' 120 | out = uncolor(out) 121 | assert out == '''\ 122 | Looking in indexes: {server}/simple 123 | Collecting wheeled-package==0.2.0 from {server}/packages/wheeled_package-0.2.0-py2.py3-none-any.whl 124 | Downloading {server}/packages/wheeled_package-0.2.0-py2.py3-none-any.whl 125 | Installing collected packages: wheeled-package 126 | Successfully installed wheeled-package-0.2.0 127 | '''.format(server=pypi_server) 128 | -------------------------------------------------------------------------------- /tests/testing/__init__.py: -------------------------------------------------------------------------------- 1 | # NOTE WELL: No side-effects are allowed in __init__ files. This means you! 2 | from __future__ import absolute_import 3 | from __future__ import print_function 4 | from __future__ import unicode_literals 5 | 6 | import os 7 | from re import compile as Regex 8 | from re import DOTALL 9 | from re import MULTILINE 10 | 11 | from pip._internal.wheel import Wheel 12 | from py._path.local import LocalPath as Path 13 | 14 | TOP = Path(__file__) / '../../..' 15 | COVERAGE_REQS = TOP.join('requirements.d/coverage.txt') 16 | 17 | 18 | def requirements(reqs, path='requirements.txt'): 19 | """Write a requirements.txt file to the current working directory.""" 20 | Path(path).write(reqs) 21 | 22 | 23 | def run(*cmd, **env): 24 | if env: 25 | from os import environ 26 | tmp = env 27 | env = environ.copy() 28 | env.update(tmp) 29 | else: 30 | env = None 31 | 32 | from .capture_subprocess import capture_subprocess 33 | from venv_update import info, colorize 34 | info('\nTEST> ' + colorize(cmd)) 35 | out, err = capture_subprocess(cmd, env=env) 36 | err = strip_coverage_warnings(err) 37 | return out, err 38 | 39 | 40 | def venv_update(*args, **env): 41 | # we get coverage for free via the (patched) pytest-cov plugin 42 | return run('venv-update', *args, **env) 43 | 44 | 45 | def venv_update_symlink_pwd(): 46 | # I wish I didn't need this =/ 47 | # surely there's a better way -.- 48 | # NOTE: `pip install TOP` causes an infinite copyfiles loop, under tox >.< 49 | from venv_update import __file__ as venv_update_path, dotpy 50 | 51 | # symlink so that we get coverage, where possible 52 | venv_update_path = Path(dotpy(venv_update_path)) 53 | local_vu = Path(venv_update_path.basename) 54 | local_vu.mksymlinkto(venv_update_path) 55 | 56 | 57 | # coverage.py adds some helpful warnings to stderr, with no way to quiet them. 58 | coverage_warnings_regex = Regex( 59 | r'^Coverage.py warning: (%s)\n' % '|'.join(( 60 | r'Module .* was never imported\.', 61 | r'No data was collected\.', 62 | r'Module venv_update was previously imported, but not measured\.', 63 | )), 64 | flags=MULTILINE, 65 | ) 66 | 67 | 68 | def strip_coverage_warnings(stderr): 69 | return coverage_warnings_regex.sub('', stderr) 70 | 71 | 72 | # pip adds some helpful warnings to stderr, with no way to quiet them. 73 | pip_warnings_regex = Regex( 74 | '|'.join(( 75 | r"^ Url '[^']*/\.cache/pip-faster/wheelhouse' is ignored: it is neither a file nor a directory\.\n", 76 | r'^You are using pip version [0-9.]+, however version [0-9.]+ is available\.\n', 77 | r"^You should consider upgrading via the 'pip install --upgrade pip' command\.\n", 78 | r"^DEPRECATION: Python 2\.7 will reach the end of its life on January 1st, 2020\. Please upgrade your Python as Python 2\.7 won't be maintained after that date\. A future version of pip will drop support for Python 2\.7\.\n", # noqa: E501 79 | r"^DEPRECATION: Python 2\.7 will reach the end of its life on January 1st, 2020\. Please upgrade your Python as Python 2\.7 won't be maintained after that date\. A future version of pip will drop support for Python 2\.7\. More details about Python 2 support in pip, can be found at https://pip\.pypa\.io/en/latest/development/release-process/#python-2-support\n", # noqa: E501 80 | r'DEPRECATION: Python 2\.7 reached the end of its life on January 1st, 2020\. Please upgrade your Python as Python 2\.7 is no longer maintained\. pip 21\.0 will drop support for Python 2\.7 in January 2021\. More details about Python 2 support in pip can be found at https://pip\.pypa\.io/en/latest/development/release-process/#python-2-support pip 21\.0 will remove support for this functionality\.', # noqa: E501 81 | r"^DEPRECATION: Python 3\.4 support has been deprecated. pip 19\.1 will be the last one supporting it\. Please upgrade your Python as Python 3\.4 won't be maintained after March 2019 \(cf PEP 429\)\.\n", # noqa: E501 82 | r'^DEPRECATION: A future version of pip will drop support for Python 2\.7\.\n', # noqa: E501 83 | r'^DEPRECATION: A future version of pip will drop support for Python 2\.7\. More details about Python 2 support in pip, can be found at https://pip\.pypa\.io/en/latest/development/release-process/#python-2-support\n', # noqa: E501 84 | r'\S+ UserWarning: Setuptools will stop working on Python 2.*warnings.warn.*$', # noqa: E501 85 | )), 86 | flags=MULTILINE | DOTALL, 87 | ) 88 | 89 | 90 | def strip_pip_warnings(stderr): 91 | stderr = pip_warnings_regex.sub('', stderr) 92 | return stderr.strip() 93 | 94 | 95 | def uncolor(text): 96 | # the colored_tty, uncolored_pipe tests cover this pretty well. 97 | from re import sub 98 | text = sub('\033\\[[^A-z]*[A-z]', '', text) 99 | text = sub('.\b', '', text) 100 | return sub('[^\n\r]*\r', '', text) 101 | 102 | 103 | def pip_freeze(venv='venv'): 104 | from os.path import join 105 | out, err = run(join(venv, 'bin', 'pip'), 'freeze', '--local', '--all') 106 | 107 | # Most python distributions which have argparse in the stdlib fail to 108 | # expose it to setuptools as an installed package (it seems all but ubuntu 109 | # do this). This results in argparse sometimes being installed locally, 110 | # sometimes not, even for a specific version of python. 111 | # We normalize by never looking at argparse =/ 112 | import re 113 | out = re.sub(r'argparse==[\d.]+\n', '', out, count=1) 114 | 115 | # We'll always have `pip`, `setuptools`, and `wheel` -- filter those out 116 | for pkg in ('pip', 'setuptools', 'wheel'): 117 | reg = re.compile(r'((?<=\n)|^){}==.*(\n|$)'.format(pkg)) 118 | assert reg.search(out) 119 | out = reg.sub('', out) 120 | # The `coverage` version is floating 121 | out = re.sub('^coverage==.*', 'coverage==ANY', out, count=1) 122 | 123 | err = strip_pip_warnings(err) 124 | assert err == '' 125 | return out 126 | 127 | 128 | def install_coverage(venv='venv'): 129 | venv = Path(venv) 130 | if not venv.exists(): 131 | run('virtualenv', str(venv)) 132 | run(str(venv.join('bin/python')), '-m', 'pip.__main__', 'install', '-r', str(COVERAGE_REQS)) 133 | 134 | 135 | def enable_coverage(): 136 | from venv_update import Scratch 137 | install_coverage(Scratch().venv) 138 | 139 | 140 | class OtherPython(object): 141 | """represents a python interpreter that doesn't match the "current" interpreter's version""" 142 | 143 | def __init__(self): 144 | import sys 145 | if sys.version_info[0] <= 2: 146 | self.interpreter = 'python3' 147 | self.version_prefix = '3.' 148 | else: 149 | self.interpreter = 'python2.7' 150 | self.version_prefix = '2.7.' 151 | 152 | 153 | def cached_wheels(tmpdir): 154 | for _, _, filenames in os.walk( 155 | tmpdir.join('home', '.cache', 'pip-faster', 'wheelhouse').strpath, 156 | ): 157 | for filename in filenames: 158 | assert filename.endswith('.whl'), filename 159 | yield Wheel(filename) 160 | -------------------------------------------------------------------------------- /tests/testing/capture_subprocess.py: -------------------------------------------------------------------------------- 1 | """ 2 | Show a command's output in realtime and capture its outputs as strings, 3 | without deadlocking or temporary files. 4 | 5 | This should maybe be a package in its own right, someday. 6 | """ 7 | from __future__ import absolute_import 8 | from __future__ import print_function 9 | from __future__ import unicode_literals 10 | 11 | import os 12 | from subprocess import CalledProcessError 13 | from subprocess import Popen 14 | 15 | 16 | # posix standard file descriptors 17 | STDIN, STDOUT, STDERR = range(3) 18 | PY3 = (str is not bytes) 19 | 20 | if hasattr(os, 'set_inheritable'): 21 | # os.set_inheritable only exists in py3 pylint:disable=no-member 22 | set_inheritable = os.set_inheritable 23 | else: 24 | def set_inheritable(*_): 25 | pass 26 | 27 | 28 | class Pipe(object): 29 | """a convenience object, wrapping os.pipe()""" 30 | 31 | def __init__(self): 32 | self.read, self.write = os.pipe() 33 | # emulate old, inheritable os.pipe in py34 34 | set_inheritable(self.read, True) 35 | set_inheritable(self.write, True) 36 | 37 | def readonly(self): 38 | """close the write end of the pipe. idempotent.""" 39 | os.close(self.write) 40 | 41 | 42 | def pty_normalize_newlines(fd): 43 | r""" 44 | Twiddle the tty flags such that \n won't get munged to \r\n. 45 | Details: 46 | https://docs.python.org/2/library/termios.html 47 | http://ftp.gnu.org/old-gnu/Manuals/glibc-2.2.3/html_chapter/libc_17.html#SEC362 48 | """ 49 | import termios as T 50 | attrs = T.tcgetattr(fd) 51 | attrs[1] &= ~(T.ONLCR | T.OPOST) 52 | T.tcsetattr(fd, T.TCSANOW, attrs) 53 | 54 | 55 | class Pty(Pipe): 56 | """Represent a pty as a pipe""" 57 | 58 | def __init__(self): 59 | self.read, self.write = os.openpty() 60 | pty_normalize_newlines(self.read) 61 | 62 | 63 | def read_block(fd, block=4 * 1024): 64 | """Read up to 4k bytes from fd. 65 | Returns empty-string upon end of file. 66 | """ 67 | from os import read 68 | try: 69 | return read(fd, block) 70 | except OSError as error: 71 | if error.errno == 5: 72 | # pty end-of-file, sometimes: 73 | # http://bugs.python.org/issue21090#msg231093 74 | return b'' 75 | else: 76 | raise 77 | 78 | 79 | def read_all(fd): 80 | """My own read loop, bc the one in python3.4 is derpy atm: 81 | http://bugs.python.org/issue21090#msg231093 82 | """ 83 | from os import close 84 | 85 | result = [] 86 | lastread = None 87 | while lastread != b'': 88 | lastread = read_block(fd) 89 | result.append(lastread) 90 | close(fd) 91 | return b''.join(result) 92 | 93 | 94 | class Tee(object): 95 | """send output from read_fd to each of write_fds 96 | call .join() to get a complete copy of output 97 | """ 98 | 99 | def __init__(self, read_fd, *write_fds): 100 | self.read = read_fd 101 | self.write = write_fds 102 | self._result = [] 103 | 104 | from threading import Thread 105 | self.thread = Thread(target=self.tee) 106 | self.thread.start() 107 | 108 | def tee(self): 109 | line = read_block(self.read) 110 | while line != b'': 111 | self._result.append(line) 112 | for w in self.write: 113 | os.write(w, line) 114 | line = read_block(self.read) 115 | os.close(self.read) 116 | 117 | def join(self): 118 | self.thread.join() 119 | return b''.join(self._result) 120 | 121 | 122 | def capture_subprocess(cmd, encoding='UTF-8', **popen_kwargs): 123 | """Run a command, showing its usual outputs in real time, 124 | and return its stdout, stderr output as strings. 125 | 126 | No temporary files are used. 127 | """ 128 | stdout = Pty() # libc uses full buffering for stdout if it doesn't see a tty 129 | stderr = Pipe() 130 | 131 | # deadlocks occur if we have any write-end of a pipe open more than once 132 | # best practice: close any used write pipes just after spawn 133 | outputter = Popen( 134 | cmd, 135 | stdout=stdout.write, 136 | stderr=stderr.write, 137 | **popen_kwargs 138 | ) 139 | stdout.readonly() # deadlock otherwise 140 | stderr.readonly() # deadlock otherwise 141 | 142 | # start one tee each on the original stdout and stderr 143 | # writing each to three places: 144 | # 1. the original destination 145 | # 2. a pipe just for that one stream 146 | stdout_tee = Tee(stdout.read, STDOUT) 147 | stderr_tee = Tee(stderr.read, STDERR) 148 | 149 | # clean up left-over processes and pipes: 150 | exit_code = outputter.wait() 151 | result = (stdout_tee.join(), stderr_tee.join()) 152 | 153 | result = tuple( 154 | bytestring.decode(encoding) 155 | for bytestring in result 156 | ) 157 | 158 | if exit_code == 0: 159 | return result 160 | else: 161 | error = CalledProcessError(exit_code, cmd) 162 | error.result = result 163 | raise error 164 | 165 | 166 | def main(): 167 | import sys 168 | capture_subprocess(sys.argv[1:]) 169 | 170 | 171 | if __name__ == '__main__': 172 | exit(main()) 173 | -------------------------------------------------------------------------------- /tests/testing/capture_subprocess_test.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | from __future__ import unicode_literals 4 | 5 | from .capture_subprocess import capture_subprocess 6 | from testing import coverage_warnings_regex 7 | 8 | 9 | def make_outputter(): 10 | """Create the the outputter.py script, for the demonstration.""" 11 | with open('outputter.py', 'w') as outputter: 12 | outputter.write('''\ 13 | from __future__ import print_function 14 | from __future__ import division 15 | from sys import stdout, stderr 16 | from time import sleep 17 | from random import random, seed 18 | seed(0) # unpredictable, but repeatable 19 | 20 | # system should not deadlock for any given value of these parameters. 21 | LINES = 4000 22 | TIME = 0 23 | WIDTH = 179 24 | ERROR_RATIO = .20 25 | 26 | for i in range(LINES): 27 | if random() > ERROR_RATIO: 28 | char = '.' 29 | file = stdout 30 | else: 31 | char = '%' 32 | file = stderr 33 | 34 | for j in range(WIDTH): 35 | print(char, file=file, end='') 36 | file.flush() 37 | sleep(TIME / LINES / WIDTH) 38 | print(file=file) 39 | file.flush() 40 | ''') 41 | 42 | 43 | def test_capture_subprocess(tmpdir): 44 | tmpdir.chdir() 45 | make_outputter() 46 | 47 | cmd = ('python', 'outputter.py') 48 | stdout, stderr = capture_subprocess(cmd) 49 | 50 | stderr = coverage_warnings_regex.sub('', stderr) 51 | 52 | assert stdout.count('\n') == 3207 53 | assert stderr.count('\n') == 793 54 | 55 | assert stdout.strip('.\n') == '' 56 | assert stderr.strip('%\n') == '' 57 | 58 | 59 | def test_cli(): 60 | from venv_update import check_output 61 | from sys import executable 62 | from testing.capture_subprocess import __file__ as script 63 | output = check_output((executable, script, 'echo', 'ok')) 64 | assert output == 'ok\n' 65 | -------------------------------------------------------------------------------- /tests/testing/fix_coverage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from __future__ import absolute_import 3 | from __future__ import print_function 4 | from __future__ import unicode_literals 5 | 6 | import os.path 7 | 8 | import coverage.env 9 | from coverage.data import CoverageData 10 | coverage.env.TESTING = 'true' 11 | 12 | 13 | def merge_coverage(coverage_data, from_path, to_path): 14 | new_coverage_data = CoverageData() 15 | assert coverage_data._filename != new_coverage_data._filename 16 | 17 | for filename in coverage_data.measured_files(): 18 | result_filename = filename.split(from_path)[-1] 19 | if filename != result_filename: 20 | result_filename = result_filename.lstrip('/') 21 | result_filename = os.path.join(to_path, result_filename) 22 | result_filename = os.path.abspath(result_filename) 23 | assert os.path.exists(result_filename), result_filename 24 | 25 | new_coverage_data.add_arcs( 26 | {result_filename: coverage_data.arcs(filename)} 27 | ) 28 | 29 | return new_coverage_data 30 | 31 | 32 | def fix_coverage(from_path, to_path): 33 | os.rename('.coverage', '.coverage.orig') 34 | coverage_data = CoverageData(basename='.coverage.orig') 35 | coverage_data.read() 36 | new_coverage_data = merge_coverage(coverage_data, from_path, to_path) 37 | new_coverage_data.write() 38 | 39 | 40 | def main(): 41 | from sys import argv 42 | from_path, to_path = argv[1:] 43 | fix_coverage(from_path, to_path) 44 | 45 | 46 | if __name__ == '__main__': 47 | exit(main()) 48 | -------------------------------------------------------------------------------- /tests/testing/make_sdists.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Build a collection of packages, to be used as a pytest fixture. 4 | 5 | This script is reentrant IFF the destinations are not shared. 6 | """ 7 | from __future__ import absolute_import 8 | from __future__ import print_function 9 | from __future__ import unicode_literals 10 | 11 | import os 12 | from contextlib import contextmanager 13 | from sys import executable as python 14 | 15 | 16 | @contextmanager 17 | def chdir(path): 18 | if os.getcwd() == str(path): 19 | yield 20 | return 21 | 22 | from sys import stdout 23 | stdout.write('cd %s\n' % path) 24 | with path.as_cwd(): 25 | yield 26 | 27 | 28 | def run(cmd): 29 | from sys import stdout 30 | from subprocess import check_call 31 | from pipes import quote 32 | cmd_string = ' '.join(quote(arg) for arg in cmd) 33 | stdout.write('%s\n' % (cmd_string)) 34 | check_call(cmd) 35 | 36 | 37 | def make_copy(setuppy, dst): 38 | pkg = setuppy.dirpath().basename 39 | copy = dst.join('src', pkg).ensure(dir=True) 40 | 41 | # egg-info is also not reentrant-safe: it briefly blanks SOURCES.txt 42 | with chdir(setuppy.dirpath()): 43 | run((python, 'setup.py', '--quiet', 'egg_info', '--egg-base', str(copy))) 44 | 45 | from glob import glob 46 | sources = copy.join('*/SOURCES.txt') 47 | sources, = glob(str(sources)) 48 | sources = open(sources).read().splitlines() 49 | 50 | for source in sources: 51 | source = setuppy.dirpath().join(source, abs=1) 52 | dest = copy.join(source.relto(setuppy)) 53 | dest.dirpath().ensure(dir=True) 54 | source.copy(dest) 55 | return copy 56 | 57 | 58 | def sdist(setuppy, dst): 59 | copy = make_copy(setuppy, dst) 60 | with chdir(copy): 61 | run( 62 | (python, 'setup.py', '--quiet', 'sdist', '--dist-dir', str(dst)), 63 | ) 64 | 65 | 66 | def build_one(src, dst): 67 | setuppy = src.join('setup.py') 68 | if setuppy.exists(): 69 | sdist(setuppy, dst) 70 | 71 | if src.join('wheelme').exists(): 72 | copy = make_copy(setuppy, dst) 73 | wheel(copy, dst) 74 | 75 | return True 76 | 77 | 78 | def build_all(sources, dst): 79 | for source in sources: 80 | if build_one(source, dst): 81 | continue 82 | for source in sorted(source.listdir()): 83 | build_one(source, dst) 84 | 85 | 86 | def wheel(src, dst): 87 | build = dst.join('build') 88 | build.ensure_dir() 89 | run(( 90 | python, '-m', 'pip', 91 | 'wheel', 92 | '--quiet', 93 | '--build-dir', str(build), 94 | '--wheel-dir', str(dst), 95 | str(src) 96 | )) 97 | build.remove() # pip1.5 wheel doesn't clean up its build =/ 98 | 99 | 100 | def download_sdist(source, destination): 101 | run(( 102 | python, '-m', 'pip', 103 | 'download', 104 | '--quiet', 105 | '--no-deps', 106 | '--no-binary', ':all:', 107 | '--build-dir', str(destination.join('build')), 108 | '--dest', str(destination), 109 | str(source), 110 | )) 111 | 112 | 113 | def do_build(sources, destination): 114 | build_all(sources, destination) 115 | wheel('virtualenv>=20.0.8', destination) 116 | wheel('coverage-enable-subprocess', destination) 117 | download_sdist('coverage', destination) 118 | download_sdist('coverage-enable-subprocess', destination) 119 | 120 | 121 | def random_string(): 122 | """return a short suffix that shouldn't collide with any subsequent calls""" 123 | import os 124 | import base64 125 | 126 | return '.'.join(( 127 | str(os.getpid()), 128 | base64.urlsafe_b64encode(os.urandom(3)).decode('US-ASCII'), 129 | )) 130 | 131 | 132 | def flock(path, blocking=True): 133 | import os 134 | fd = os.open(path, os.O_CREAT) 135 | 136 | import fcntl 137 | flags = fcntl.LOCK_EX # exclusive 138 | if not blocking: # :pragma:nobranch: 139 | flags |= fcntl.LOCK_NB # non-blocking 140 | 141 | try: 142 | fcntl.flock(fd, flags) 143 | except IOError as error: # :pragma:nocover: not always hit 144 | if error.errno == 11: # EAGAIN: lock held 145 | return None 146 | else: 147 | raise 148 | else: 149 | return fd 150 | 151 | 152 | def make_sdists(sources, destination): 153 | destination.dirpath().ensure(dir=True) 154 | 155 | lock = destination.new(ext='lock') 156 | if flock(lock.strpath, blocking=False) is None: # :pragma:nocover: not always hit 157 | print('lock held; waiting for other thread...') 158 | flock(lock.strpath, blocking=True) 159 | return 160 | 161 | staging = destination.new(ext=random_string()) 162 | staging.ensure(dir=True) 163 | 164 | do_build(sources, staging) 165 | if destination.islink(): # :pragma:nocover: 166 | old = destination.readlink() 167 | else: 168 | old = None 169 | 170 | link = staging.new(ext='ln') 171 | link.mksymlinkto(staging, absolute=False) 172 | link.rename(destination) 173 | 174 | if old is not None: # :pragma:nocover: 175 | destination.dirpath(old).remove() 176 | 177 | 178 | def main(): 179 | assert 'PIP_INDEX_URL' not in os.environ, os.environ['PIP_INDEX_URL'] 180 | from sys import argv 181 | argv = argv[1:] 182 | sources, destination = argv[:-1], argv[-1] 183 | 184 | from py._path.local import LocalPath 185 | sources = tuple([LocalPath(src) for src in sources]) 186 | destination = LocalPath(destination) 187 | 188 | return make_sdists(sources, destination) 189 | 190 | 191 | if __name__ == '__main__': 192 | exit(main()) 193 | -------------------------------------------------------------------------------- /tests/testing/packages/Weird-casing_pacKAGE/README: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/venv-update/5fb5491bd421fdd8ef3cff3faa5d4846b5985ec8/tests/testing/packages/Weird-casing_pacKAGE/README -------------------------------------------------------------------------------- /tests/testing/packages/Weird-casing_pacKAGE/setup.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | from __future__ import unicode_literals 4 | 5 | from setuptools import setup 6 | 7 | 8 | setup( 9 | name=str('weird_CASING-pACKage'), 10 | version='0.1.0', 11 | url='example.com', 12 | author='nobody', 13 | author_email='nobody@example.com', 14 | py_modules=[str('weird_casing_package')], 15 | options={ 16 | 'bdist_wheel': { 17 | 'universal': 1, 18 | } 19 | }, 20 | ) 21 | -------------------------------------------------------------------------------- /tests/testing/packages/Weird-casing_pacKAGE/weird_casing_package.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | from __future__ import unicode_literals 4 | 5 | 6 | def main(): 7 | pass 8 | -------------------------------------------------------------------------------- /tests/testing/packages/cant_wheel_package/README: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/venv-update/5fb5491bd421fdd8ef3cff3faa5d4846b5985ec8/tests/testing/packages/cant_wheel_package/README -------------------------------------------------------------------------------- /tests/testing/packages/cant_wheel_package/setup.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | from __future__ import unicode_literals 4 | 5 | from setuptools import setup 6 | 7 | 8 | class broken_bdist_wheel(object): 9 | """This isn't even a valid command class.""" 10 | 11 | 12 | setup( 13 | name=str('cant_wheel_package'), 14 | version='0.1.0', 15 | url='example.com', 16 | author='nobody', 17 | author_email='nobody@example.com', 18 | cmdclass={'bdist_wheel': broken_bdist_wheel}, 19 | ) 20 | -------------------------------------------------------------------------------- /tests/testing/packages/circular-dep-a/README: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/venv-update/5fb5491bd421fdd8ef3cff3faa5d4846b5985ec8/tests/testing/packages/circular-dep-a/README -------------------------------------------------------------------------------- /tests/testing/packages/circular-dep-a/setup.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | from __future__ import unicode_literals 4 | 5 | from setuptools import setup 6 | 7 | 8 | setup( 9 | name=str('circular-dep-a'), 10 | version='1.0', 11 | url='example.com', 12 | author='nobody', 13 | author_email='nobody@example.com', 14 | install_requires=[ 15 | 'circular-dep-b==1.0', 16 | ], 17 | options={ 18 | 'bdist_wheel': { 19 | 'universal': 1, 20 | } 21 | }, 22 | ) 23 | -------------------------------------------------------------------------------- /tests/testing/packages/circular-dep-b/README: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/venv-update/5fb5491bd421fdd8ef3cff3faa5d4846b5985ec8/tests/testing/packages/circular-dep-b/README -------------------------------------------------------------------------------- /tests/testing/packages/circular-dep-b/setup.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | from __future__ import unicode_literals 4 | 5 | from setuptools import setup 6 | 7 | 8 | setup( 9 | name=str('circular-dep-b'), 10 | version='1.0', 11 | url='example.com', 12 | author='nobody', 13 | author_email='nobody@example.com', 14 | install_requires=[ 15 | 'circular-dep-a==1.0', 16 | ], 17 | options={ 18 | 'bdist_wheel': { 19 | 'universal': 1, 20 | } 21 | }, 22 | ) 23 | -------------------------------------------------------------------------------- /tests/testing/packages/conflicting_package/README: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/venv-update/5fb5491bd421fdd8ef3cff3faa5d4846b5985ec8/tests/testing/packages/conflicting_package/README -------------------------------------------------------------------------------- /tests/testing/packages/conflicting_package/setup.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | from __future__ import unicode_literals 4 | 5 | from setuptools import setup 6 | 7 | 8 | setup( 9 | name=str('conflicting_package'), 10 | version='1', 11 | url='example.com', 12 | author='nobody', 13 | author_email='nobody@example.com', 14 | install_requires=[ 15 | 'many_versions_package<2', 16 | ], 17 | options={ 18 | 'bdist_wheel': { 19 | 'universal': 1, 20 | } 21 | }, 22 | ) 23 | -------------------------------------------------------------------------------- /tests/testing/packages/dependant_package/README: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/venv-update/5fb5491bd421fdd8ef3cff3faa5d4846b5985ec8/tests/testing/packages/dependant_package/README -------------------------------------------------------------------------------- /tests/testing/packages/dependant_package/setup.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | from __future__ import unicode_literals 4 | 5 | from setuptools import setup 6 | 7 | 8 | setup( 9 | name=str('dependant_package'), 10 | version='1', 11 | url='example.com', 12 | author='nobody', 13 | author_email='nobody@example.com', 14 | install_requires=[ 15 | 'many_versions_package>=2,<4', 16 | 'implicit_dependency', 17 | 'pure_python_package>=0.2.1', 18 | ], 19 | options={ 20 | 'bdist_wheel': { 21 | 'universal': 1, 22 | } 23 | }, 24 | ) 25 | -------------------------------------------------------------------------------- /tests/testing/packages/dotted_package_name/README: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/venv-update/5fb5491bd421fdd8ef3cff3faa5d4846b5985ec8/tests/testing/packages/dotted_package_name/README -------------------------------------------------------------------------------- /tests/testing/packages/dotted_package_name/dotted_package_name.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | from __future__ import unicode_literals 4 | 5 | 6 | def main(): 7 | pass 8 | -------------------------------------------------------------------------------- /tests/testing/packages/dotted_package_name/setup.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | from __future__ import unicode_literals 4 | 5 | from setuptools import setup 6 | 7 | 8 | setup( 9 | name=str('dotted.package-name'), 10 | version='0.1.0', 11 | url='example.com', 12 | author='nobody', 13 | author_email='nobody@example.com', 14 | py_modules=[str('dotted_package_name')], 15 | options={ 16 | 'bdist_wheel': { 17 | 'universal': 1, 18 | } 19 | }, 20 | ) 21 | -------------------------------------------------------------------------------- /tests/testing/packages/dotted_package_name/wheelme: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/venv-update/5fb5491bd421fdd8ef3cff3faa5d4846b5985ec8/tests/testing/packages/dotted_package_name/wheelme -------------------------------------------------------------------------------- /tests/testing/packages/implicit_dependency/README: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/venv-update/5fb5491bd421fdd8ef3cff3faa5d4846b5985ec8/tests/testing/packages/implicit_dependency/README -------------------------------------------------------------------------------- /tests/testing/packages/implicit_dependency/setup.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | from __future__ import unicode_literals 4 | 5 | from setuptools import setup 6 | 7 | 8 | setup( 9 | name=str('implicit_dependency'), 10 | version='1', 11 | url='example.com', 12 | author='nobody', 13 | author_email='nobody@example.com', 14 | options={ 15 | 'bdist_wheel': { 16 | 'universal': 1, 17 | } 18 | }, 19 | ) 20 | -------------------------------------------------------------------------------- /tests/testing/packages/many_versions_package_1/README: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/venv-update/5fb5491bd421fdd8ef3cff3faa5d4846b5985ec8/tests/testing/packages/many_versions_package_1/README -------------------------------------------------------------------------------- /tests/testing/packages/many_versions_package_1/setup.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | from __future__ import unicode_literals 4 | 5 | from setuptools import setup 6 | 7 | 8 | setup( 9 | name=str('many_versions_package'), 10 | version='1', 11 | url='example.com', 12 | author='nobody', 13 | author_email='nobody@example.com', 14 | options={ 15 | 'bdist_wheel': { 16 | 'universal': 1, 17 | } 18 | }, 19 | ) 20 | -------------------------------------------------------------------------------- /tests/testing/packages/many_versions_package_2.1/README: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/venv-update/5fb5491bd421fdd8ef3cff3faa5d4846b5985ec8/tests/testing/packages/many_versions_package_2.1/README -------------------------------------------------------------------------------- /tests/testing/packages/many_versions_package_2.1/setup.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | from __future__ import unicode_literals 4 | 5 | from setuptools import setup 6 | 7 | 8 | setup( 9 | name=str('many_versions_package'), 10 | version='2.1', 11 | url='example.com', 12 | author='nobody', 13 | author_email='nobody@example.com', 14 | options={ 15 | 'bdist_wheel': { 16 | 'universal': 1, 17 | } 18 | }, 19 | ) 20 | -------------------------------------------------------------------------------- /tests/testing/packages/many_versions_package_2/README: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/venv-update/5fb5491bd421fdd8ef3cff3faa5d4846b5985ec8/tests/testing/packages/many_versions_package_2/README -------------------------------------------------------------------------------- /tests/testing/packages/many_versions_package_2/setup.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | from __future__ import unicode_literals 4 | 5 | from setuptools import setup 6 | 7 | 8 | setup( 9 | name=str('many_versions_package'), 10 | version='2', 11 | url='example.com', 12 | author='nobody', 13 | author_email='nobody@example.com', 14 | options={ 15 | 'bdist_wheel': { 16 | 'universal': 1, 17 | } 18 | }, 19 | ) 20 | -------------------------------------------------------------------------------- /tests/testing/packages/many_versions_package_3/README: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/venv-update/5fb5491bd421fdd8ef3cff3faa5d4846b5985ec8/tests/testing/packages/many_versions_package_3/README -------------------------------------------------------------------------------- /tests/testing/packages/many_versions_package_3/setup.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | from __future__ import unicode_literals 4 | 5 | from setuptools import setup 6 | 7 | 8 | setup( 9 | name=str('many_versions_package'), 10 | version='3', 11 | url='example.com', 12 | author='nobody', 13 | author_email='nobody@example.com', 14 | options={ 15 | 'bdist_wheel': { 16 | 'universal': 1, 17 | } 18 | }, 19 | ) 20 | -------------------------------------------------------------------------------- /tests/testing/packages/many_versions_package_4/README: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/venv-update/5fb5491bd421fdd8ef3cff3faa5d4846b5985ec8/tests/testing/packages/many_versions_package_4/README -------------------------------------------------------------------------------- /tests/testing/packages/many_versions_package_4/setup.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | from __future__ import unicode_literals 4 | 5 | from setuptools import setup 6 | 7 | 8 | setup( 9 | name=str('many_versions_package'), 10 | version='4', 11 | url='example.com', 12 | author='nobody', 13 | author_email='nobody@example.com', 14 | options={ 15 | 'bdist_wheel': { 16 | 'universal': 1, 17 | } 18 | }, 19 | ) 20 | -------------------------------------------------------------------------------- /tests/testing/packages/project_with_c/README: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/venv-update/5fb5491bd421fdd8ef3cff3faa5d4846b5985ec8/tests/testing/packages/project_with_c/README -------------------------------------------------------------------------------- /tests/testing/packages/project_with_c/project_with_c.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | static PyObject* _hello_world(PyObject* self) { 4 | PyObject_Print(PyUnicode_FromString("hello world\n"), stdout, Py_PRINT_RAW); 5 | Py_RETURN_NONE; 6 | } 7 | 8 | static struct PyMethodDef methods[] = { 9 | {"hello_world", (PyCFunction)_hello_world, METH_NOARGS}, 10 | {NULL, NULL} 11 | }; 12 | 13 | #if PY_MAJOR_VERSION >= 3 14 | static struct PyModuleDef module = { 15 | PyModuleDef_HEAD_INIT, 16 | "project_with_c", 17 | NULL, 18 | -1, 19 | methods 20 | }; 21 | 22 | PyMODINIT_FUNC PyInit_project_with_c(void) { 23 | return PyModule_Create(&module); 24 | } 25 | #else 26 | PyMODINIT_FUNC initproject_with_c(void) { 27 | Py_InitModule3("project_with_c", methods, NULL); 28 | } 29 | #endif 30 | -------------------------------------------------------------------------------- /tests/testing/packages/project_with_c/setup.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | from __future__ import unicode_literals 4 | 5 | from setuptools import Extension 6 | from setuptools import setup 7 | 8 | 9 | setup( 10 | name=str('project_with_c'), 11 | version='0.1.0', 12 | url='example.com', 13 | author='nobody', 14 | author_email='nobody@example.com', 15 | ext_modules=[Extension(str('project_with_c'), [str('project_with_c.c')])], 16 | entry_points={ 17 | 'console_scripts': [ 18 | 'c-extension-script = project_with_c:hello_world', 19 | ], 20 | }, 21 | ) 22 | -------------------------------------------------------------------------------- /tests/testing/packages/pure_python_package/README: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/venv-update/5fb5491bd421fdd8ef3cff3faa5d4846b5985ec8/tests/testing/packages/pure_python_package/README -------------------------------------------------------------------------------- /tests/testing/packages/pure_python_package/pure_python_package.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | from __future__ import unicode_literals 4 | 5 | 6 | def main(): 7 | pass 8 | -------------------------------------------------------------------------------- /tests/testing/packages/pure_python_package/setup.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | from __future__ import unicode_literals 4 | 5 | from setuptools import setup 6 | 7 | 8 | setup( 9 | name=str('pure_python_package'), 10 | version='0.1.0', 11 | url='example.com', 12 | author='nobody', 13 | author_email='nobody@example.com', 14 | py_modules=[str('pure_python_package')], 15 | entry_points={ 16 | 'console_scripts': [ 17 | 'pure-python-script = pure_python_package:main', 18 | ], 19 | }, 20 | # NOT a universal wheel 21 | ) 22 | -------------------------------------------------------------------------------- /tests/testing/packages/pure_python_package_2/README: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/venv-update/5fb5491bd421fdd8ef3cff3faa5d4846b5985ec8/tests/testing/packages/pure_python_package_2/README -------------------------------------------------------------------------------- /tests/testing/packages/pure_python_package_2/pure_python_package.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | from __future__ import unicode_literals 4 | 5 | 6 | def main(): 7 | pass 8 | -------------------------------------------------------------------------------- /tests/testing/packages/pure_python_package_2/setup.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | from __future__ import unicode_literals 4 | 5 | from setuptools import setup 6 | 7 | 8 | setup( 9 | name=str('pure_python_package'), 10 | version='0.2.1', 11 | url='example.com', 12 | author='nobody', 13 | author_email='nobody@example.com', 14 | py_modules=[str('pure_python_package')], 15 | extras_require={ 16 | 'my-extra': ['implicit_dependency'], 17 | }, 18 | entry_points={ 19 | 'console_scripts': [ 20 | 'pure-python-script = pure_python_package:main', 21 | ], 22 | }, 23 | options={ 24 | 'bdist_wheel': { 25 | 'universal': 1, 26 | } 27 | }, 28 | ) 29 | -------------------------------------------------------------------------------- /tests/testing/packages/slow_python_package/README: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/venv-update/5fb5491bd421fdd8ef3cff3faa5d4846b5985ec8/tests/testing/packages/slow_python_package/README -------------------------------------------------------------------------------- /tests/testing/packages/slow_python_package/setup.py: -------------------------------------------------------------------------------- 1 | # pylint:disable=import-error,invalid-name,no-init 2 | from __future__ import absolute_import 3 | from __future__ import print_function 4 | from __future__ import unicode_literals 5 | 6 | from distutils.command.build import build as _build 7 | 8 | from setuptools import setup 9 | 10 | 11 | class build(_build): 12 | 13 | def run(self): # I actually don't know why coverage doesn't see this :pragma:nocover: 14 | # Simulate a slow package 15 | import time 16 | time.sleep(5) 17 | # old style class 18 | _build.run(self) 19 | 20 | 21 | setup( 22 | name=str('slow_python_package'), 23 | version='0.1.0', 24 | url='example.com', 25 | author='nobody', 26 | author_email='nobody@example.com', 27 | py_modules=[str('slow_python_package')], 28 | cmdclass={'build': build}, 29 | options={ 30 | 'bdist_wheel': { 31 | 'universal': 1, 32 | } 33 | }, 34 | ) 35 | -------------------------------------------------------------------------------- /tests/testing/packages/slow_python_package/slow_python_package.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/venv-update/5fb5491bd421fdd8ef3cff3faa5d4846b5985ec8/tests/testing/packages/slow_python_package/slow_python_package.py -------------------------------------------------------------------------------- /tests/testing/packages/wheeled_package/README: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/venv-update/5fb5491bd421fdd8ef3cff3faa5d4846b5985ec8/tests/testing/packages/wheeled_package/README -------------------------------------------------------------------------------- /tests/testing/packages/wheeled_package/setup.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | from __future__ import unicode_literals 4 | 5 | from setuptools import setup 6 | 7 | 8 | setup( 9 | name=str('wheeled_package'), 10 | version='0.2.0', 11 | url='example.com', 12 | author='nobody', 13 | author_email='nobody@example.com', 14 | options={ 15 | 'bdist_wheel': { 16 | 'universal': 1, 17 | } 18 | }, 19 | ) 20 | -------------------------------------------------------------------------------- /tests/testing/packages/wheeled_package/wheelme: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/venv-update/5fb5491bd421fdd8ef3cff3faa5d4846b5985ec8/tests/testing/packages/wheeled_package/wheelme -------------------------------------------------------------------------------- /tests/testing/python_lib.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from __future__ import absolute_import 3 | from __future__ import print_function 4 | from __future__ import unicode_literals 5 | 6 | import os 7 | import sys 8 | from distutils.sysconfig import get_python_lib 9 | 10 | PYTHON_LIB = os.path.relpath(get_python_lib(), sys.prefix) 11 | 12 | if __name__ == '__main__': 13 | print(PYTHON_LIB) 14 | -------------------------------------------------------------------------------- /tests/testing/python_lib_test.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | from __future__ import unicode_literals 4 | 5 | 6 | def test_cli(): 7 | from venv_update import check_output 8 | from sys import executable 9 | from testing.python_lib import __file__ as script 10 | output = check_output((executable, script)) 11 | output = output.splitlines() 12 | assert len(output) == 1 13 | output = output[0] 14 | assert output.endswith('site-packages') 15 | 16 | from testing import Path 17 | import sys 18 | path = Path(sys.prefix).join(output) 19 | assert path.basename == 'site-packages' 20 | assert path.check(dir=True) 21 | -------------------------------------------------------------------------------- /tests/unit/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/venv-update/5fb5491bd421fdd8ef3cff3faa5d4846b5985ec8/tests/unit/__init__.py -------------------------------------------------------------------------------- /tests/unit/fix_coverage_test.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | from __future__ import unicode_literals 4 | 5 | from coverage.data import CoverageData 6 | 7 | from testing.fix_coverage import merge_coverage 8 | 9 | 10 | def test_fix_coverage(tmpdir): 11 | base_file = tmpdir.join('foo.py') 12 | base_file.ensure() 13 | sub_file = tmpdir.join('site-packages/foo.py') 14 | sub_file.ensure() 15 | unrelated_file = tmpdir.join('bar.py') 16 | unrelated_file.ensure() 17 | 18 | coverage_data = CoverageData(basename='.coverage.orig') 19 | coverage_data.add_arcs({ 20 | str(base_file): {(1, 2): None}, 21 | str(sub_file): {(3, 4): None}, 22 | str(unrelated_file): {(5, 6): None}, 23 | }) 24 | 25 | assert coverage_data.lines(base_file) == [1, 2] 26 | assert coverage_data.lines(sub_file) == [3, 4] 27 | assert coverage_data.lines(unrelated_file) == [5, 6] 28 | 29 | new_coverage_data = merge_coverage(coverage_data, '/site-packages/', str(tmpdir)) 30 | 31 | # The new file should contain all the lines and arcs 32 | assert new_coverage_data.lines(base_file) == [1, 2, 3, 4] 33 | assert new_coverage_data.arcs(base_file) == [(1, 2), (3, 4)] 34 | assert new_coverage_data.lines(unrelated_file) == [5, 6] 35 | assert new_coverage_data.arcs(unrelated_file) == [(5, 6)] 36 | 37 | # And it should not contain the original, un-merged names. 38 | assert sub_file not in new_coverage_data.measured_files() 39 | -------------------------------------------------------------------------------- /tests/unit/patch.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | from __future__ import unicode_literals 4 | 5 | import pip_faster as P 6 | 7 | 8 | def test_patch(): 9 | d = {1: 1, 2: 2} 10 | patches = {2: 3}.items() 11 | orig = P.patch(d, patches) 12 | 13 | assert d == {1: 1, 2: 3} 14 | assert orig == {2: 2} 15 | 16 | try: 17 | P.patch(d, {3: 3}.items()) 18 | raise AssertionError('expected KeyError') 19 | except KeyError: 20 | pass 21 | 22 | 23 | def test_patched(): 24 | d = {1: 1, 2: 2} 25 | patches = {2: 3} 26 | 27 | before = d.copy() 28 | with P.patched(d, patches) as orig: 29 | assert d == {1: 1, 2: 3} 30 | assert orig == {2: 2} 31 | 32 | assert d == before 33 | 34 | try: 35 | with P.patched(d, {3: 3}): 36 | raise AssertionError('expected KeyError') 37 | except KeyError: 38 | pass 39 | -------------------------------------------------------------------------------- /tests/unit/simple_test.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | from __future__ import unicode_literals 4 | 5 | import pytest 6 | 7 | import pip_faster 8 | import venv_update 9 | from testing import Path 10 | 11 | 12 | def test_importable(): 13 | assert venv_update 14 | 15 | 16 | def test_pip_get_installed(): 17 | installed = pip_faster.pip_get_installed() 18 | installed = pip_faster.reqnames(installed) 19 | installed = sorted(installed) 20 | print(installed) 21 | assert 'pip' in installed 22 | 23 | 24 | @pytest.mark.parametrize('filename,expected', [ 25 | ('foo.py', 'foo.py'), 26 | ('foo.pyc', 'foo.py'), 27 | ('foo.pye', 'foo.pye'), 28 | ('../foo.pyc', '../foo.py'), 29 | ('/a/b/c/foo.pyc', '/a/b/c/foo.py'), 30 | ('bar.pyd', 'bar.py'), 31 | ('baz.pyo', 'baz.py'), 32 | ]) 33 | def test_dotpy(filename, expected): 34 | assert venv_update.dotpy(filename) == expected 35 | 36 | 37 | @pytest.mark.parametrize('args', [ 38 | ('-h',), 39 | ('a', '-h',), 40 | ('-h', 'b'), 41 | ('--help',), 42 | ('a', '--help',), 43 | ('--help', 'b'), 44 | ]) 45 | def test_parseargs_help(args, capsys): 46 | from venv_update import __doc__ as HELP_OUTPUT 47 | with pytest.raises(SystemExit) as excinfo: 48 | assert venv_update.parseargs(args) 49 | 50 | out, err = capsys.readouterr() 51 | assert err == '' 52 | assert out == HELP_OUTPUT 53 | assert excinfo.value.code == 0 54 | 55 | 56 | @pytest.mark.parametrize('args,expected', [ 57 | ( 58 | ('1', 'foo'), 59 | '1 foo', 60 | ), ( 61 | ('1 foo',), 62 | "'1 foo'", 63 | ), ( 64 | (r'''she said "hi", she said 'bye' ''',), 65 | r"""'she said "hi", she said '"'"'bye'"'"' '""", 66 | ), 67 | ]) 68 | def test_shellescape(args, expected): 69 | assert venv_update.shellescape(args) == expected 70 | 71 | 72 | @pytest.mark.parametrize('path,expected', [ 73 | ( 74 | '1', 75 | '1', 76 | ), ( 77 | '2 foo', 78 | "'2 foo'", 79 | ), ( 80 | '../foo', 81 | '../foo', 82 | ), 83 | ]) 84 | def test_shellescape_relpath(path, expected, tmpdir): 85 | tmpdir.chdir() 86 | tmpfile = tmpdir.join(path) 87 | args = (tmpfile.strpath,) 88 | assert venv_update.shellescape(args) == expected 89 | assert expected != tmpfile.strpath 90 | 91 | 92 | def test_shellescape_relpath_longer(tmpdir): 93 | tmpdir.chdir() 94 | path = Path('/a/b') 95 | args = (path.strpath,) 96 | assert venv_update.shellescape(args) == path.strpath 97 | 98 | 99 | @pytest.mark.parametrize('req,expected', [ 100 | ('foo', False), 101 | ('foo==1', True), 102 | ('bar<3,==2,>1', True), 103 | ('quux<3,!=2,>1', False), 104 | ('wat==2,!=2', True), 105 | ('wat-more==2,==3', True), 106 | ]) 107 | def test_is_req_pinned(req, expected): 108 | from pkg_resources import Requirement 109 | req = Requirement.parse(req) 110 | assert pip_faster.is_req_pinned(req) is expected 111 | 112 | 113 | def test_is_req_pinned_null(): 114 | assert pip_faster.is_req_pinned(None) is False 115 | 116 | 117 | def test_wait_for_all_subprocesses(monkeypatch): 118 | class _nonlocal(object): 119 | wait = 10 120 | thrown = False 121 | 122 | def fakewait(): 123 | if _nonlocal.wait <= 0: 124 | _nonlocal.thrown = True 125 | raise OSError(10, 'No child process') 126 | else: 127 | _nonlocal.wait -= 1 128 | 129 | import os 130 | monkeypatch.setattr(os, 'wait', fakewait) 131 | venv_update.wait_for_all_subprocesses() 132 | 133 | assert _nonlocal.wait == 0 134 | assert _nonlocal.thrown is True 135 | 136 | 137 | def test_samefile(tmpdir): 138 | with tmpdir.as_cwd(): 139 | a = tmpdir.ensure('a') 140 | b = tmpdir.ensure('b') 141 | tmpdir.join('c').mksymlinkto(a, absolute=True) 142 | tmpdir.join('d').mksymlinkto(b, absolute=False) 143 | 144 | assert venv_update.samefile('a', 'b') is False 145 | assert venv_update.samefile('a', 'x') is False 146 | assert venv_update.samefile('x', 'a') is False 147 | 148 | assert venv_update.samefile('a', 'a') is True 149 | assert venv_update.samefile('a', 'c') is True 150 | assert venv_update.samefile('d', 'b') is True 151 | 152 | 153 | def passwd(): 154 | import os 155 | import pwd 156 | return pwd.getpwuid(os.getuid()) 157 | 158 | 159 | def test_user_cache_dir(): 160 | assert venv_update.user_cache_dir() == passwd().pw_dir + '/.cache' 161 | 162 | from os import environ 163 | environ['HOME'] = '/foo/bar' 164 | assert venv_update.user_cache_dir() == '/foo/bar/.cache' 165 | 166 | environ['XDG_CACHE_HOME'] = '/quux/bar' 167 | assert venv_update.user_cache_dir() == '/quux/bar' 168 | 169 | 170 | def test_get_python_version(): 171 | import sys 172 | 173 | expected = '.'.join(str(part) for part in sys.version_info) 174 | actual = venv_update.get_python_version(sys.executable) 175 | assert expected == actual 176 | 177 | assert venv_update.get_python_version('total garbage') is None 178 | 179 | from subprocess import CalledProcessError 180 | with pytest.raises(CalledProcessError) as excinfo: 181 | venv_update.get_python_version('/bin/false') 182 | assert excinfo.value.returncode == 1 183 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | # These should match the travis env list 3 | envlist = py27,py36,pypy3,lint 4 | skipsdist=True 5 | 6 | [testenv] 7 | # `changedir` ensures we run against the installed, rather than the working directory: 8 | passenv = 9 | # For codecov 10 | CI TOXENV 11 | PYTEST_OPTIONS 12 | changedir = 13 | {envtmpdir} 14 | setenv = 15 | TOP={toxinidir} 16 | SITEPACKAGES={envsitepackagesdir} 17 | venv_update = 18 | pip install {toxinidir} # this is only because we're using an un-released version of venv-update 19 | {toxinidir}/venv_update.py \ 20 | venv= {envdir} \ 21 | install= 22 | commands = 23 | {[testenv]venv_update} -r {toxinidir}/requirements.d/test.txt {toxinidir} 24 | pip-faster freeze --all 25 | {toxinidir}/test {posargs} 26 | 27 | [testenv:latest-pip] 28 | commands = 29 | sed -i 's/,<=18.1//g' {toxinidir}/setup.py {toxinidir}/requirements.d/import_tests.txt 30 | pip install git+git://github.com/pypa/pip 31 | {[testenv]commands} 32 | 33 | [testenv:lint] 34 | commands = 35 | {[testenv]venv_update} -r {toxinidir}/requirements.d/lint.txt 36 | pre-commit run --all-files 37 | 38 | [testenv:docs] 39 | deps = -rrequirements.d/docs.txt 40 | changedir = docs 41 | commands = sphinx-build -b html -d build/doctrees source build/html 42 | 43 | [pep8] 44 | ignore=E265,E266,W504 45 | 46 | [flake8] 47 | max-line-length=131 48 | max-complexity=12 49 | -------------------------------------------------------------------------------- /venv_update.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | '''\ 4 | usage: venv-update [-hV] [options] 5 | 6 | Update a (possibly non-existent) virtualenv directory using a pip requirements 7 | file. When this script completes, the virtualenv directory should contain the 8 | same packages as if it were deleted then rebuilt. 9 | 10 | venv-update uses "trailing equal" options (e.g. venv=) to delimit groups of 11 | (conventional, dashed) options to pass to wrapped commands (virtualenv and pip). 12 | 13 | Options: 14 | venv= parameters are passed to virtualenv 15 | default: {venv=} 16 | install= options to pip-command 17 | default: {install=} 18 | pip-command= is run after the virtualenv directory is bootstrapped 19 | default: {pip-command=} 20 | bootstrap-deps= dependencies to install before pip-command= is run 21 | default: {bootstrap-deps=} 22 | 23 | Examples: 24 | # install requirements.txt to "venv" 25 | venv-update 26 | 27 | # install requirements.txt to "myenv" 28 | venv-update venv= myenv 29 | 30 | # install requirements.txt to "myenv" using Python 3.4 31 | venv-update venv= -ppython3.4 myenv 32 | 33 | # install myreqs.txt to "venv" 34 | venv-update install= -r myreqs.txt 35 | 36 | # install requirements.txt to "venv", verbosely 37 | venv-update venv= venv -vvv install= -r requirements.txt -vvv 38 | 39 | # install requirements.txt to "venv", without pip-faster --update --prune 40 | venv-update pip-command= pip install 41 | 42 | We strongly recommend that you keep the default value of pip-command= in order 43 | to quickly and reproducibly install your requirements. You can override the 44 | packages installed during bootstrapping, prior to pip-command=, by setting 45 | bootstrap-deps= 46 | 47 | Pip options are also controllable via environment variables. 48 | See https://pip.readthedocs.org/en/stable/user_guide/#environment-variables 49 | For example: 50 | PIP_INDEX_URL=https://pypi.example.com/simple venv-update 51 | 52 | Please send issues to: https://github.com/yelp/venv-update 53 | ''' 54 | from __future__ import absolute_import 55 | from __future__ import print_function 56 | from __future__ import unicode_literals 57 | 58 | import os 59 | from os.path import exists 60 | from os.path import join 61 | from subprocess import CalledProcessError 62 | 63 | # https://github.com/Yelp/venv-update/issues/227 64 | # https://stackoverflow.com/a/53193892 65 | # On OS X, Python "framework" builds set a `__PYVENV_LAUNCHER__` environment 66 | # variable when executed, which gets inherited by child processes and cause 67 | # certain Python builds to put incorrect packages onto their path. This causes 68 | # weird bugs with venv-update like import errors calling pip and infinite 69 | # exec() loops trying to activate a virtualenv. 70 | # 71 | # To fix this we just delete the environment variable. 72 | os.environ.pop('__PYVENV_LAUNCHER__', None) 73 | 74 | __version__ = '4.0.0' 75 | DEFAULT_VIRTUALENV_PATH = 'venv' 76 | DEFAULT_OPTION_VALUES = { 77 | 'venv=': (DEFAULT_VIRTUALENV_PATH,), 78 | 'install=': ('-r', 'requirements.txt',), 79 | 'pip-command=': ('pip-faster', 'install', '--upgrade', '--prune'), 80 | 'bootstrap-deps=': ('venv-update==' + __version__,), 81 | } 82 | __doc__ = __doc__.format( 83 | **{key: ' '.join(val) for key, val in DEFAULT_OPTION_VALUES.items()} 84 | ) 85 | 86 | # This script must not rely on anything other than 87 | # stdlib>=2.6 and virtualenv>1.11 88 | 89 | 90 | def parseargs(argv): 91 | '''handle --help, --version and our double-equal ==options''' 92 | args = [] 93 | options = {} 94 | key = None 95 | for arg in argv: 96 | if arg in DEFAULT_OPTION_VALUES: 97 | key = arg.strip('=').replace('-', '_') 98 | options[key] = () 99 | elif key is None: 100 | args.append(arg) 101 | else: 102 | options[key] += (arg,) 103 | 104 | if set(args) & {'-h', '--help'}: 105 | print(__doc__, end='') 106 | exit(0) 107 | elif set(args) & {'-V', '--version'}: 108 | print(__version__) 109 | exit(0) 110 | elif args: 111 | exit('invalid option: %s\nTry --help for more information.' % args[0]) 112 | 113 | return options 114 | 115 | 116 | def timid_relpath(arg): 117 | """convert an argument to a relative path, carefully""" 118 | # TODO-TEST: unit tests 119 | from os.path import isabs, relpath, sep 120 | if isabs(arg): 121 | result = relpath(arg) 122 | if result.count(sep) + 1 < arg.count(sep): 123 | return result 124 | 125 | return arg 126 | 127 | 128 | def shellescape(args): 129 | from pipes import quote 130 | return ' '.join(quote(timid_relpath(arg)) for arg in args) 131 | 132 | 133 | def colorize(cmd): 134 | from os import isatty 135 | 136 | if isatty(1): 137 | template = '\033[36m>\033[m \033[32m{0}\033[m' 138 | else: 139 | template = '> {0}' 140 | 141 | return template.format(shellescape(cmd)) 142 | 143 | 144 | def run(cmd): 145 | from subprocess import check_call 146 | check_call(('echo', colorize(cmd))) 147 | check_call(cmd) 148 | 149 | 150 | def info(msg): 151 | # use a subprocess to ensure correct output interleaving. 152 | from subprocess import check_call 153 | check_call(('echo', msg)) 154 | 155 | 156 | def check_output(cmd): 157 | from subprocess import Popen, PIPE 158 | process = Popen(cmd, stdout=PIPE) 159 | output, _ = process.communicate() 160 | if process.returncode: 161 | raise CalledProcessError(process.returncode, cmd) 162 | else: 163 | assert process.returncode == 0 164 | return output.decode('UTF-8') 165 | 166 | 167 | def samefile(file1, file2): 168 | if not exists(file1) or not exists(file2): 169 | return False 170 | else: 171 | from os.path import samefile 172 | return samefile(file1, file2) 173 | 174 | 175 | def exec_(argv): # never returns 176 | """Wrapper to os.execv which shows the command and runs any atexit handlers (for coverage's sake). 177 | Like os.execv, this function never returns. 178 | """ 179 | # info('EXEC' + colorize(argv)) # TODO: debug logging by environment variable 180 | 181 | # in python3, sys.exitfunc has gone away, and atexit._run_exitfuncs seems to be the only pubic-ish interface 182 | # https://hg.python.org/cpython/file/3.4/Modules/atexitmodule.c#l289 183 | import atexit 184 | atexit._run_exitfuncs() 185 | 186 | from os import execv 187 | execv(argv[0], argv) 188 | 189 | 190 | class Scratch(object): 191 | 192 | def __init__(self): 193 | self.dir = join(user_cache_dir(), 'venv-update', __version__) 194 | self.venv = join(self.dir, 'venv') 195 | self.python = venv_python(self.venv) 196 | self.src = join(self.dir, 'src') 197 | 198 | 199 | def exec_scratch_virtualenv(args): 200 | """ 201 | goals: 202 | - get any random site-packages off of the pythonpath 203 | - ensure we can import virtualenv 204 | - ensure that we're not using the interpreter that we may need to delete 205 | - idempotency: do nothing if the above goals are already met 206 | """ 207 | scratch = Scratch() 208 | if not exists(scratch.python): 209 | run(('virtualenv', scratch.venv)) 210 | 211 | if not exists(join(scratch.src, 'virtualenv')): 212 | scratch_python = venv_python(scratch.venv) 213 | # TODO: do we allow user-defined override of which version of virtualenv to install? 214 | tmp = scratch.src + '.tmp' 215 | run((scratch_python, '-m', 'pip.__main__', 'install', 'virtualenv>=20.0.8', '--target', tmp)) 216 | from os import rename 217 | rename(tmp, scratch.src) 218 | 219 | import sys 220 | from os.path import realpath 221 | # We want to compare the paths themselves as sometimes sys.path is the same 222 | # as scratch.venv, but with a suffix of bin/.. 223 | if realpath(sys.prefix) != realpath(scratch.venv): 224 | # TODO-TEST: sometimes we would get a stale version of venv-update 225 | exec_((scratch.python, dotpy(__file__)) + args) # never returns 226 | 227 | # TODO-TEST: the original venv-update's directory was on sys.path (when using symlinking) 228 | sys.path[0] = scratch.src 229 | 230 | 231 | def get_original_path(venv_path): # TODO-TEST: a unit test 232 | """This helps us know whether someone has tried to relocate the virtualenv""" 233 | return check_output(('sh', '-c', '. %s; printf "$VIRTUAL_ENV"' % venv_executable(venv_path, 'activate'))) 234 | 235 | 236 | def get_python_version(interpreter): 237 | if not exists(interpreter): 238 | return None 239 | 240 | cmd = (interpreter, '-c', 'import sys; print(".".join(str(p) for p in sys.version_info))') 241 | return check_output(cmd).strip() 242 | 243 | 244 | def invalid_virtualenv_reason(venv_path, source_python, destination_python, virtualenv_system_site_packages): 245 | try: 246 | orig_path = get_original_path(venv_path) 247 | except CalledProcessError: 248 | return 'could not inspect metadata' 249 | if not samefile(orig_path, venv_path): 250 | return 'virtualenv moved {} -> {}'.format(timid_relpath(orig_path), timid_relpath(venv_path)) 251 | 252 | pyvenv_cfg_path = join(venv_path, 'pyvenv.cfg') 253 | 254 | if not exists(pyvenv_cfg_path): 255 | return 'virtualenv created with virtualenv<20' 256 | 257 | # Avoid using pathlib.Path which doesn't exist in python2 and 258 | # hack around configparser's inability to handle sectionless config 259 | # files: https://bugs.python.org/issue22253 260 | from configparser import ConfigParser 261 | pyvenv_cfg = ConfigParser() 262 | with open(pyvenv_cfg_path, 'r') as f: 263 | pyvenv_cfg.read_string('[root]\n' + f.read()) 264 | if pyvenv_cfg.getboolean('root', 'include-system-site-packages', fallback=False) != virtualenv_system_site_packages: 265 | return 'system-site-packages changed, to %s' % virtualenv_system_site_packages 266 | if source_python is None: 267 | return 268 | 269 | destination_version = pyvenv_cfg.get('root', 'version_info', fallback=None) 270 | source_version = get_python_version(source_python) 271 | if source_version != destination_version: 272 | return 'python version changed {} -> {}'.format(destination_version, source_version) 273 | 274 | base_executable = pyvenv_cfg.get('root', 'base-executable', fallback=None) 275 | base_executable_version = get_python_version(base_executable) 276 | if base_executable_version != destination_version: 277 | return 'base executable python version changed {} -> {}'.format(destination_version, base_executable_version) 278 | 279 | 280 | def ensure_virtualenv(args, return_values): 281 | """Ensure we have a valid virtualenv.""" 282 | 283 | from sys import argv 284 | argv[:] = ('virtualenv',) + args 285 | info(colorize(argv)) 286 | 287 | import virtualenv 288 | 289 | run_virtualenv = True 290 | filtered_args = [a for a in args if not a.startswith('-')] 291 | if filtered_args: 292 | venv_path = return_values.venv_path = filtered_args[0] if filtered_args else None 293 | if venv_path == DEFAULT_VIRTUALENV_PATH: 294 | from os.path import abspath, basename, dirname 295 | args = ('--prompt=({})'.format(basename(dirname(abspath(venv_path)))),) + args 296 | 297 | # Validate existing virtualenv if there is one 298 | # there are two python interpreters involved here: 299 | # 1) the interpreter we're instructing virtualenv to copy 300 | python_options_arg = [a for a in args if a.startswith('-p') or a.startswith('--python')] 301 | if not python_options_arg: 302 | source_python = None 303 | else: 304 | virtualenv_session = virtualenv.session_via_cli(args) 305 | source_python = virtualenv_session._interpreter.executable 306 | # 2) the interpreter virtualenv will create 307 | destination_python = venv_python(venv_path) 308 | 309 | if exists(destination_python): 310 | # Check if --system-site-packages is a passed-in option 311 | dummy_session = virtualenv.session_via_cli(args) 312 | virtualenv_system_site_packages = dummy_session.creator.enable_system_site_package 313 | 314 | reason = invalid_virtualenv_reason(venv_path, source_python, destination_python, virtualenv_system_site_packages) 315 | if reason: 316 | info('Removing invalidated virtualenv. (%s)' % reason) 317 | run(('rm', '-rf', venv_path)) 318 | else: 319 | info('Keeping valid virtualenv from previous run.') 320 | run_virtualenv = False # looks good! we're done here. 321 | 322 | if run_virtualenv: 323 | raise_on_failure(lambda: virtualenv.cli_run(args), ignore_return=True) 324 | 325 | # There might not be a venv_path if doing something like "venv= --version" 326 | # and not actually asking virtualenv to make a venv. 327 | if return_values.venv_path is not None: 328 | run(('rm', '-rf', join(return_values.venv_path, 'local'))) 329 | 330 | 331 | def wait_for_all_subprocesses(): 332 | from os import wait 333 | try: 334 | while True: 335 | wait() 336 | except OSError as error: 337 | if error.errno == 10: # no child processes 338 | return 339 | else: 340 | raise 341 | 342 | 343 | def touch(filename, timestamp): 344 | """set the mtime of a file""" 345 | if timestamp is not None: 346 | timestamp = (timestamp, timestamp) # atime, mtime 347 | 348 | from os import utime 349 | utime(filename, timestamp) 350 | 351 | 352 | def mark_venv_valid(venv_path): 353 | wait_for_all_subprocesses() 354 | touch(venv_path, None) 355 | 356 | 357 | def mark_venv_invalid(venv_path): 358 | # LBYL, to attempt to avoid any exception during exception handling 359 | from os.path import isdir 360 | if venv_path and isdir(venv_path): 361 | info('') 362 | info("Something went wrong! Sending '%s' back in time, so make knows it's invalid." % timid_relpath(venv_path)) 363 | wait_for_all_subprocesses() 364 | touch(venv_path, 0) 365 | 366 | 367 | def dotpy(filename): 368 | if filename.endswith(('.pyc', '.pyo', '.pyd')): 369 | return filename[:-1] 370 | else: 371 | return filename 372 | 373 | 374 | def venv_executable(venv_path, executable): 375 | return join(venv_path, 'bin', executable) 376 | 377 | 378 | def venv_python(venv_path): 379 | return venv_executable(venv_path, 'python') 380 | 381 | 382 | def user_cache_dir(): 383 | # stolen from pip.utils.appdirs.user_cache_dir 384 | from os import getenv 385 | from os.path import expanduser 386 | return getenv('XDG_CACHE_HOME', expanduser('~/.cache')) 387 | 388 | 389 | def venv_update( 390 | venv=DEFAULT_OPTION_VALUES['venv='], 391 | install=DEFAULT_OPTION_VALUES['install='], 392 | pip_command=DEFAULT_OPTION_VALUES['pip-command='], 393 | bootstrap_deps=DEFAULT_OPTION_VALUES['bootstrap-deps='], 394 | ): 395 | """we have an arbitrary python interpreter active, (possibly) outside the virtualenv we want. 396 | 397 | make a fresh venv at the right spot, make sure it has pip-faster, and use it 398 | """ 399 | 400 | # SMELL: mutable argument as return value 401 | class return_values(object): 402 | venv_path = None 403 | 404 | try: 405 | ensure_virtualenv(venv, return_values) 406 | if return_values.venv_path is None: 407 | return 408 | # invariant: the final virtualenv exists, with the right python version 409 | raise_on_failure(lambda: pip_faster(return_values.venv_path, pip_command, install, bootstrap_deps)) 410 | except BaseException: 411 | mark_venv_invalid(return_values.venv_path) 412 | raise 413 | else: 414 | mark_venv_valid(return_values.venv_path) 415 | 416 | 417 | def execfile_(filename): 418 | with open(filename) as code: 419 | code = compile(code.read(), filename, 'exec') 420 | exec(code, {'__file__': filename}) 421 | 422 | 423 | def pip_faster(venv_path, pip_command, install, bootstrap_deps): 424 | """install and run pip-faster""" 425 | # activate the virtualenv 426 | execfile_(venv_executable(venv_path, 'activate_this.py')) 427 | 428 | # disable a useless warning 429 | # FIXME: ensure a "true SSLContext" is available 430 | from os import environ 431 | environ['PIP_DISABLE_PIP_VERSION_CHECK'] = '1' 432 | 433 | # we always have to run the bootstrap, because the presense of an 434 | # executable doesn't imply the right version. pip is able to validate the 435 | # version in the fastpath case quickly anyway. 436 | run(('pip', 'install') + bootstrap_deps) 437 | 438 | run(pip_command + install) 439 | 440 | 441 | def raise_on_failure(mainfunc, ignore_return=False): 442 | """raise if and only if mainfunc fails""" 443 | try: 444 | errors = mainfunc() 445 | if not ignore_return and errors: 446 | exit(errors) 447 | except CalledProcessError as error: 448 | exit(error.returncode) 449 | except SystemExit as error: 450 | if error.code: 451 | raise 452 | except KeyboardInterrupt: # I don't plan to test-cover this. :pragma:nocover: 453 | exit(1) 454 | 455 | 456 | def main(): 457 | from sys import argv 458 | args = tuple(argv[1:]) 459 | 460 | # process --help before we create any side-effects. 461 | options = parseargs(args) 462 | exec_scratch_virtualenv(args) 463 | return venv_update(**options) 464 | 465 | 466 | if __name__ == '__main__': 467 | exit(main()) 468 | -------------------------------------------------------------------------------- /wheelme: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/venv-update/5fb5491bd421fdd8ef3cff3faa5d4846b5985ec8/wheelme --------------------------------------------------------------------------------