├── .coveragerc ├── .gitignore ├── .travis.yml ├── LICENSE.txt ├── MANIFEST.in ├── Makefile ├── README.rst ├── appveyor.yml ├── docs ├── conf.py ├── developers.rst ├── index.rst └── users.rst ├── pip_accel ├── __init__.py ├── __main__.py ├── bdist.py ├── caches │ ├── __init__.py │ ├── local.py │ └── s3.py ├── cli.py ├── compat.py ├── config.py ├── deps │ ├── __init__.py │ └── debian.ini ├── exceptions.py ├── req.py ├── tests.py └── utils.py ├── requirements-flake8.txt ├── requirements-rtd.txt ├── requirements-testing.txt ├── requirements.txt ├── scripts ├── appveyor.py ├── collect-test-coverage.sh ├── prepare-test-environment.cmd ├── prepare-test-environment.sh └── retry-command ├── setup.py └── tox.ini /.coveragerc: -------------------------------------------------------------------------------- 1 | # .coveragerc: Configuration file for coverage.py. 2 | # http://nedbatchelder.com/code/coverage/ 3 | 4 | [run] 5 | source = pip_accel 6 | omit = pip_accel/tests.py 7 | 8 | [report] 9 | exclude_lines = raise NotImplementedError() 10 | 11 | # vim: ft=dosini 12 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.egg 2 | *.pyc 3 | .tox/ 4 | build/ 5 | dist/ 6 | pip_accel.egg-info/ 7 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # .travis.yml: Configuration for continuous integration (automated tests) 2 | # hosted on Travis CI, see https://travis-ci.org/paylogic/pip-accel. 3 | 4 | sudo: required 5 | language: python 6 | env: BOTO_CONFIG=/tmp/nowhere 7 | python: 8 | - "2.6" 9 | - "2.7" 10 | - "3.4" 11 | - "3.5" 12 | - "pypy" 13 | before_install: 14 | - scripts/retry-command sudo apt-get update 15 | install: 16 | - scripts/retry-command gem install fakes3 17 | - scripts/retry-command pip install coveralls --requirement=requirements-flake8.txt 18 | - scripts/prepare-test-environment.sh 19 | script: 20 | - make check 21 | - scripts/collect-test-coverage.sh 22 | after_success: 23 | - scripts/retry-command coveralls 24 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016 Peter Odding and Paylogic International 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.rst 2 | include *.txt 3 | include pip_accel/deps/*.ini 4 | include tox.ini 5 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for the pip accelerator. 2 | # 3 | # Author: Peter Odding 4 | # Last Change: January 17, 2016 5 | # URL: https://github.com/paylogic/pip-accel 6 | 7 | WORKON_HOME ?= $(HOME)/.virtualenvs 8 | VIRTUAL_ENV ?= $(WORKON_HOME)/pip-accel 9 | PATH := $(VIRTUAL_ENV)/bin:$(PATH) 10 | SHELL = /bin/bash 11 | 12 | default: 13 | @echo 'Makefile for the pip accelerator' 14 | @echo 15 | @echo 'Usage:' 16 | @echo 17 | @echo ' make install install the package in a virtual environment' 18 | @echo ' make reset recreate the virtual environment' 19 | @echo ' make test run the tests and collect coverage' 20 | @echo ' make check check the coding style' 21 | @echo ' make docs update documentation using Sphinx' 22 | @echo ' make publish publish changes to GitHub/PyPI' 23 | @echo ' make clean cleanup all temporary files' 24 | @echo 25 | 26 | install: 27 | test -d "$(VIRTUAL_ENV)" || mkdir -p "$(VIRTUAL_ENV)" 28 | test -x "$(VIRTUAL_ENV)/bin/python" || virtualenv "$(VIRTUAL_ENV)" 29 | test -x "$(VIRTUAL_ENV)/bin/pip" || easy_install pip 30 | pip uninstall --yes --quiet pip-accel &>/dev/null || true 31 | pip install --quiet --editable . 32 | pip-accel install --quiet --requirement=requirements-testing.txt 33 | 34 | reset: 35 | rm -Rf "$(VIRTUAL_ENV)" 36 | make --no-print-directory clean install 37 | 38 | test: install 39 | scripts/prepare-test-environment.sh 40 | scripts/collect-test-coverage.sh 41 | coverage html 42 | 43 | tox: install 44 | (test -x "$(VIRTUAL_ENV)/bin/tox" \ 45 | || pip-accel install --quiet tox) \ 46 | && tox 47 | 48 | detox: install 49 | (test -x "$(VIRTUAL_ENV)/bin/detox" \ 50 | || pip-accel install --quiet detox) \ 51 | && COVERAGE=no detox 52 | 53 | check: install 54 | (test -x "$(VIRTUAL_ENV)/bin/flake8" \ 55 | || pip-accel install --quiet --requirement requirements-flake8.txt) \ 56 | && flake8 57 | 58 | docs: install 59 | test -x "$(VIRTUAL_ENV)/bin/sphinx-build" || pip-accel install --quiet sphinx 60 | cd docs && sphinx-build -b html -d build/doctrees . build/html 61 | 62 | publish: install 63 | git push origin && git push --tags origin 64 | test -x "$(VIRTUAL_ENV)/bin/twine" || pip-accel install --quiet twine 65 | make clean && python setup.py sdist && twine upload dist/* 66 | 67 | clean: 68 | rm -Rf \ 69 | *.egg \ 70 | .cache/ \ 71 | .coverage \ 72 | .coverage.* \ 73 | .tox/ \ 74 | build/ \ 75 | dist/ \ 76 | docs/build/ \ 77 | htmlcov/ 78 | find -name __pycache__ -exec rm -Rf {} \; &>/dev/null || true 79 | find -type f -name '*.py[co]' -delete 80 | 81 | .PHONY: default install reset test tox detox check docs publish clean 82 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | pip-accel: Accelerator for pip, the Python package manager 2 | ========================================================== 3 | 4 | .. image:: https://travis-ci.org/paylogic/pip-accel.svg?branch=master 5 | :target: https://travis-ci.org/paylogic/pip-accel 6 | 7 | .. image:: https://coveralls.io/repos/paylogic/pip-accel/badge.svg?branch=master 8 | :target: https://coveralls.io/r/paylogic/pip-accel?branch=master 9 | 10 | The pip-accel program is a wrapper for pip_, the Python package manager. It 11 | accelerates the usage of pip to initialize `Python virtual environments`_ given 12 | one or more `requirements files`_. It does so by combining the following two 13 | approaches: 14 | 15 | 1. Source distribution downloads are cached and used to generate a `local index 16 | of source distribution archives`_. If all your dependencies are pinned to 17 | absolute versions whose source distribution downloads were previously 18 | cached, pip-accel won't need a network connection at all! This is one of the 19 | reasons why pip can be so slow: given absolute pinned dependencies available 20 | in the download cache it will still scan PyPI_ and distribution websites. 21 | 22 | 2. `Binary distributions`_ are used to speed up the process of installing 23 | dependencies with binary components (like M2Crypto_ and LXML_). Instead of 24 | recompiling these dependencies again for every virtual environment we 25 | compile them once and cache the result as a binary ``*.tar.gz`` 26 | distribution. 27 | 28 | In addition, since version 0.9 pip-accel contains a simple mechanism that 29 | detects missing system packages when a build fails and prompts the user whether 30 | to install the missing dependencies and retry the build. 31 | 32 | The pip-accel program is currently tested on cPython 2.6, 2.7, 3.4 and 3.5 and 33 | PyPy (2.7). The automated test suite regularly runs on Ubuntu Linux (`Travis 34 | CI`_) as well as Microsoft Windows (AppVeyor_). In addition to these platforms 35 | pip-accel should work fine on most UNIX systems (e.g. Mac OS X). 36 | 37 | .. contents:: 38 | 39 | Status 40 | ------ 41 | 42 | Paylogic_ uses pip-accel to quickly and reliably initialize virtual 43 | environments on its farm of continuous integration slaves which are constantly 44 | running unit tests (this was one of the original use cases for which pip-accel 45 | was developed). We also use it on our build servers. 46 | 47 | When pip-accel was originally developed PyPI_ was sometimes very unreliable 48 | (PyPI wasn't `behind a CDN`_ back then). Because of the CDN, PyPI is much more 49 | reliable nowadays however pip-accel still has its place: 50 | 51 | - The CDN doesn't help for distribution sites, which are as unreliably as they 52 | have always been. 53 | 54 | - By using pip-accel you can make Python deployments completely independent 55 | from internet connectivity. 56 | 57 | - Because pip-accel caches compiled binary packages it can still provide a nice 58 | speed boost over using plain pip. 59 | 60 | Usage 61 | ----- 62 | 63 | The pip-accel command supports all subcommands and options supported by pip, 64 | however it is of course only useful for the ``pip install`` subcommand. So for 65 | example: 66 | 67 | .. code-block:: bash 68 | 69 | $ pip-accel install -r requirements.txt 70 | 71 | Alternatively you can also run pip-accel as follows, but note that this 72 | requires Python 2.7 or higher (it specifically doesn't work on Python 2.6): 73 | 74 | .. code-block:: bash 75 | 76 | $ python -m pip_accel install -r requirements.txt 77 | 78 | If you pass a ``-v`` or ``--verbose`` option then pip and pip-accel will both 79 | use verbose output. The ``-q`` or ``--quiet`` option is also supported. 80 | 81 | Based on the user running pip-accel the following file locations are used by 82 | default: 83 | 84 | ============================= ========================= ======================================= 85 | Root user All other users Purpose 86 | ============================= ========================= ======================================= 87 | ``/var/cache/pip-accel`` ``~/.pip-accel`` Used to store the source/binary indexes 88 | ============================= ========================= ======================================= 89 | 90 | This default can be overridden by defining the environment variable 91 | ``PIP_ACCEL_CACHE``. 92 | 93 | Configuration 94 | ~~~~~~~~~~~~~ 95 | 96 | For most users the default configuration of pip-accel should be fine. If you do 97 | want to change pip-accel's defaults you do so by setting environment variables 98 | and/or adding configuration options to a configuration file. This is because 99 | pip-accel shares its command line interface with pip and adding support for 100 | command line options specific to pip-accel is non trivial and may end up 101 | causing more confusion than it's worth :-). For an overview of the available 102 | configuration options and corresponding environment variables please refer to 103 | the `documentation of the pip_accel.config module`_. 104 | 105 | How fast is it? 106 | --------------- 107 | 108 | To give you an idea of how effective pip-accel is, below are the results of a 109 | test to build a virtual environment for one of the internal code bases of 110 | Paylogic_. This code base requires more than 40 dependencies including several 111 | packages that need compilation with SWIG and a C compiler: 112 | 113 | ========= ================================ =========== =============== 114 | Program Description Duration Percentage 115 | ========= ================================ =========== =============== 116 | pip Default configuration 444 seconds 100% (baseline) 117 | pip With download cache (first run) 416 seconds 94% 118 | pip With download cache (second run) 318 seconds 72% 119 | pip-accel First run 397 seconds 89% 120 | pip-accel Second run 30 seconds 7% 121 | ========= ================================ =========== =============== 122 | 123 | Alternative cache backends 124 | -------------------------- 125 | 126 | Bundled with pip-accel are a local cache backend (which stores distribution 127 | archives on the local file system) and an `Amazon S3`_ backend (see below). 128 | 129 | Both of these cache backends are registered with pip-accel using a generic 130 | pluggable cache backend registration mechanism. This mechanism makes it 131 | possible to register additional cache backends without modifying pip-accel. If 132 | you are interested in the details please refer to pip-accel's ``setup.py`` 133 | script and the two simple Python modules that define the bundled backends. 134 | 135 | If you've written a cache backend that you think may be valuable to others, 136 | please feel free to open an issue or pull request on GitHub in order to get 137 | your backend bundled with pip-accel. 138 | 139 | Storing the binary cache on Amazon S3 140 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 141 | 142 | You can configure pip-accel to store its binary cache files in an `Amazon S3`_ 143 | bucket. In this case Amazon S3 is treated as a second level cache, only used if 144 | the local file system cache can't satisfy a dependency. If the dependency is 145 | not found in the Amazon S3 bucket, the package is built and cached locally (as 146 | usual) but then also saved to the Amazon S3 bucket. This functionality can be 147 | useful for continuous integration build worker boxes that are ephemeral and 148 | don't have persistent local storage to store the pip-accel binary cache. 149 | 150 | To get started you need to install pip-accel as follows: 151 | 152 | .. code-block:: bash 153 | 154 | $ pip install 'pip-accel[s3]' 155 | 156 | The ``[s3]`` part enables the Amazon S3 cache backend by installing the Boto_ 157 | package. Once installed you can use the following environment variables to 158 | configure the Amazon S3 cache backend: 159 | 160 | ``$PIP_ACCEL_S3_BUCKET`` 161 | The name of the Amazon S3 bucket in which binary distribution archives should 162 | be cached. This environment variable is required to enable the Amazon S3 cache 163 | backend. 164 | 165 | ``$PIP_ACCEL_S3_PREFIX`` 166 | The optional prefix to apply to all Amazon S3 keys. This enables name spacing 167 | based on the environment in which pip-accel is running (to isolate the binary 168 | caches of ABI incompatible systems). *The user is currently responsible for 169 | choosing a suitable prefix.* 170 | 171 | ``$PIP_ACCEL_S3_READONLY`` 172 | If this option is set pip-accel will skip uploading to the Amazon S3 bucket. 173 | This means pip-accel will use the configured Amazon S3 bucket to "warm up" 174 | your local cache but it will never write to the bucket, so you can use read 175 | only credentials. Of course you will need to run at least one instance of 176 | pip-accel that does have write permissions, so this setup is best suited to 177 | teams working around e.g. a continuous integration (CI) server, where the CI 178 | server primes the cache and developers use the cache in read only mode. 179 | 180 | You can also set these options from a configuration file, please refer to the 181 | `documentation of the pip_accel.config module`_. You will also need to set AWS 182 | credentials, either in a `.boto file`_ or in the ``$AWS_ACCESS_KEY_ID`` and 183 | ``$AWS_SECRET_ACCESS_KEY`` environment variables (refer to the Boto 184 | documentation for details). 185 | 186 | Using S3 compatible storage services 187 | ```````````````````````````````````` 188 | 189 | If you want to point pip-accel at an `S3 compatible storage service`_ that is 190 | *not* Amazon S3 you can `override the S3 API URL`_ using a configuration option 191 | or environment variable. For example the pip-accel test suite first installs 192 | and starts FakeS3_ and then sets ``PIP_ACCEL_S3_URL=http://localhost:12345`` to 193 | point pip-accel at the FakeS3 server (in order to test the Amazon S3 cache 194 | backend without actually having to pay for an Amazon S3 bucket :-). For more 195 | details please refer to the documentation of the `Amazon S3 cache backend`_. 196 | 197 | Caching of setup requirements 198 | ----------------------------- 199 | 200 | Since version 0.38 pip-accel instructs setuptools to cache setup requirements 201 | in a subdirectory of pip-accel's data directory (see the eggs_cache_ option) to 202 | avoid recompilation of setup requirements. This works by injecting a symbolic 203 | link called ``.eggs`` into unpacked source distribution directories before pip 204 | or pip-accel runs the setup script. 205 | 206 | The use of the ``.eggs`` directory was added in setuptools version 7.0 which is 207 | why pip-accel now requires setuptools 7.0 or higher to be installed. This 208 | dependency was added because the whole point of pip-accel is to work well out 209 | of the box, shielding the user from surprising behavior like setup requirements 210 | slowing things down and breaking offline installation. 211 | 212 | Dependencies on system packages 213 | ------------------------------- 214 | 215 | Since version 0.9 pip-accel contains a simple mechanism that detects missing 216 | system packages when a build fails and prompts the user whether to install the 217 | missing dependencies and retry the build. Currently only Debian Linux and 218 | derivative Linux distributions are supported, although support for other 219 | platforms should be easy to add. This functionality currently works based on 220 | configuration files that define dependencies of Python packages on system 221 | packages. This means the results should be fairly reliable, but every single 222 | dependency needs to be manually defined... 223 | 224 | Here's what it looks like in practice:: 225 | 226 | 2013-06-16 01:01:53 wheezy-vm INFO Building binary distribution of python-mcrypt (1.1) .. 227 | 2013-06-16 01:01:53 wheezy-vm ERROR Failed to build binary distribution of python-mcrypt! (version: 1.1) 228 | 2013-06-16 01:01:53 wheezy-vm INFO Build output (will probably provide a hint as to what went wrong): 229 | 230 | gcc -pthread -fno-strict-aliasing -DNDEBUG -g -fwrapv -O2 -Wall -Wstrict-prototypes -fPIC -DVERSION="1.1" -I/usr/include/python2.7 -c mcrypt.c -o build/temp.linux-i686-2.7/mcrypt.o 231 | mcrypt.c:23:20: fatal error: mcrypt.h: No such file or directory 232 | error: command 'gcc' failed with exit status 1 233 | 234 | 2013-06-16 01:01:53 wheezy-vm INFO python-mcrypt: Checking for missing dependencies .. 235 | 2013-06-16 01:01:53 wheezy-vm INFO You seem to be missing 1 dependency: libmcrypt-dev 236 | 2013-06-16 01:01:53 wheezy-vm INFO I can install it for you with this command: sudo apt-get install --yes libmcrypt-dev 237 | Do you want me to install this dependency? [y/N] y 238 | 2013-06-16 01:02:05 wheezy-vm INFO Got permission to install missing dependency. 239 | 240 | The following extra packages will be installed: 241 | libmcrypt4 242 | Suggested packages: 243 | mcrypt 244 | The following NEW packages will be installed: 245 | libmcrypt-dev libmcrypt4 246 | 0 upgraded, 2 newly installed, 0 to remove and 68 not upgraded. 247 | Unpacking libmcrypt4 (from .../libmcrypt4_2.5.8-3.1_i386.deb) ... 248 | Unpacking libmcrypt-dev (from .../libmcrypt-dev_2.5.8-3.1_i386.deb) ... 249 | Setting up libmcrypt4 (2.5.8-3.1) ... 250 | Setting up libmcrypt-dev (2.5.8-3.1) ... 251 | 252 | 2013-06-16 01:02:13 wheezy-vm INFO Successfully installed 1 missing dependency. 253 | 2013-06-16 01:02:13 wheezy-vm INFO Building binary distribution of python-mcrypt (1.1) .. 254 | 2013-06-16 01:02:14 wheezy-vm INFO Copying binary distribution python-mcrypt-1.1.linux-i686.tar.gz to cache as python-mcrypt:1.1:py2.7.tar.gz. 255 | 256 | Integrating with tox 257 | -------------------- 258 | 259 | You can tell Tox_ to use pip-accel using a small shell script that first uses 260 | pip to install pip-accel, then uses pip-accel to bootstrap the virtual 261 | environment. You can find details about this in `issue #30 on GitHub`_. 262 | 263 | Control flow of pip-accel 264 | ------------------------- 265 | 266 | The way pip-accel works is not very intuitive but it is very effective. Below 267 | is an overview of the control flow. Once you take a look at the code you'll 268 | notice that the steps below are all embedded in a loop that retries several 269 | times. This is mostly because of step 2 (downloading the source 270 | distributions). 271 | 272 | 1. Run ``pip install --download=... --no-index -r requirements.txt`` to unpack 273 | source distributions available in the local source index. This is the first 274 | step because pip-accel should accept `requirements.txt` files as input but 275 | it will manually install dependencies from cached binary distributions 276 | (without using pip or easy_install): 277 | 278 | - If the command succeeds it means all dependencies are already available as 279 | downloaded source distributions. We'll parse the verbose pip output of step 280 | 1 to find the direct and transitive dependencies (names and versions) 281 | defined in `requirements.txt` and use them as input for step 3. 282 | Go to step 3. 283 | 284 | - If the command fails it probably means not all dependencies are available 285 | as local source distributions yet so we should download them. Go to step 2. 286 | 287 | 2. Run ``pip install --download=... -r requirements.txt`` to download missing 288 | source distributions to the download cache: 289 | 290 | - If the command fails it means that pip encountered errors while scanning 291 | PyPI_, scanning a distribution website, downloading a source distribution 292 | or unpacking a source distribution. Usually these kinds of errors are 293 | intermittent so retrying a few times is worth a shot. Go to step 2. 294 | 295 | - If the command succeeds it means all dependencies are now available as 296 | local source distributions; we don't need the network anymore! Go to step 1. 297 | 298 | 3. Run ``python setup.py bdist_dumb --format=gztar`` for each dependency that 299 | doesn't have a cached binary distribution yet (taking version numbers into 300 | account). Go to step 4. 301 | 302 | 4. Install all dependencies from binary distributions based on the list of 303 | direct and transitive dependencies obtained in step 1. We have to do these 304 | installations manually because easy_install nor pip support binary 305 | ``*.tar.gz`` distributions. 306 | 307 | Contact 308 | ------- 309 | 310 | If you have questions, bug reports, suggestions, etc. please create an issue on 311 | the `GitHub project page`_. The latest version of pip-accel will always be 312 | available on GitHub. The internal API documentation is `hosted on Read The 313 | Docs`_. 314 | 315 | License 316 | ------- 317 | 318 | This software is licensed under the `MIT license`_ just like pip_ (on which 319 | pip-accel is based). 320 | 321 | © 2016 Peter Odding and Paylogic_ International. 322 | 323 | 324 | .. External references: 325 | .. _.boto file: http://boto.readthedocs.org/en/latest/boto_config_tut.html 326 | .. _Amazon S3 cache backend: http://pip-accel.readthedocs.org/en/latest/developers.html#module-pip_accel.caches.s3 327 | .. _Amazon S3: http://aws.amazon.com/s3/ 328 | .. _AppVeyor: https://ci.appveyor.com/project/xolox/pip-accel 329 | .. _behind a CDN: http://mail.python.org/pipermail/distutils-sig/2013-May/020848.html 330 | .. _Binary distributions: http://docs.python.org/2/distutils/builtdist.html 331 | .. _Boto: https://github.com/boto/boto 332 | .. _documentation of the pip_accel.config module: http://pip-accel.readthedocs.org/en/latest/developers.html#module-pip_accel.config 333 | .. _eggs_cache: http://pip-accel.readthedocs.org/en/latest/developers.html#pip_accel.config.Config.binary_cache 334 | .. _FakeS3: https://github.com/jubos/fake-s3 335 | .. _GitHub project page: https://github.com/paylogic/pip-accel 336 | .. _hosted on Read The Docs: https://pip-accel.readthedocs.org/ 337 | .. _issue #30 on GitHub: https://github.com/paylogic/pip-accel/issues/30 338 | .. _local index of source distribution archives: http://www.pip-installer.org/en/latest/cookbook.html#fast-local-installs 339 | .. _LXML: https://pypi.python.org/pypi/lxml 340 | .. _M2Crypto: https://pypi.python.org/pypi/M2Crypto 341 | .. _MIT license: http://en.wikipedia.org/wiki/MIT_License 342 | .. _override the S3 API URL: http://pip-accel.readthedocs.org/en/latest/developers.html#pip_accel.config.Config.s3_cache_url 343 | .. _Paylogic: http://www.paylogic.com/ 344 | .. _pip: http://www.pip-installer.org/ 345 | .. _PyPI: http://pypi.python.org/ 346 | .. _Python virtual environments: http://www.virtualenv.org/ 347 | .. _requirements files: http://www.pip-installer.org/en/latest/cookbook.html#requirements-files 348 | .. _S3 compatible storage service: http://en.wikipedia.org/wiki/Amazon_S3#S3_API_and_competing_services 349 | .. _Tox: https://tox.readthedocs.org/ 350 | .. _Travis CI: https://travis-ci.org/paylogic/pip-accel 351 | -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | # appveyor.yml: Configuration for continuous integration (automated tests) 2 | # hosted on AppVeyor, see https://ci.appveyor.com/project/xolox/pip-accel. 3 | # 4 | # This uses a forked coveralls-python until my pull request is merged: 5 | # https://github.com/coagulant/coveralls-python/pull/97 6 | 7 | version: 1.0.{build} 8 | clone_depth: 1 9 | environment: 10 | PYTHON: C:\Python27 11 | COVERALLS_REPO_TOKEN: 12 | secure: DCxZQaYFWVR0zWqjTPXhhlRlLdmKNMS2qDUwIR8jRar13clunOqJIaXn+vKInS7g 13 | PYWIN32_URL: "https://downloads.sourceforge.net/project/pywin32/pywin32/Build%20219/pywin32-219.win32-py2.7.exe" 14 | install: 15 | - cmd: 'gem install fakes3' 16 | - ps: (new-object net.webclient).DownloadFile($env:PYWIN32_URL, 'c:\\pywin32.exe') 17 | - cmd: '"%PYTHON%\Scripts\easy_install.exe" c:\\pywin32.exe' 18 | - cmd: '"%PYTHON%\Scripts\pip.exe" install --quiet https://github.com/coagulant/coveralls-python/archive/master.zip' 19 | - cmd: 'scripts\prepare-test-environment.cmd' 20 | build: off 21 | test_script: 22 | - cmd: '"%PYTHON%\Scripts\py.test.exe" --cov' 23 | - cmd: 'echo py.test finished' 24 | on_success: 25 | - cmd: 'echo sending coverage' 26 | - cmd: '"%PYTHON%\Scripts\coveralls.exe"' 27 | - cmd: 'echo finished sending coverage' 28 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Accelerator for pip, the Python package manager. 2 | # 3 | # Author: Peter Odding 4 | # Last Change: May 18, 2016 5 | # URL: https://github.com/paylogic/pip-accel 6 | 7 | """Sphinx documentation configuration for the `pip-accel` project.""" 8 | 9 | import os 10 | import sys 11 | 12 | # Add the pip_accel source distribution's root directory to the module path. 13 | sys.path.insert(0, os.path.abspath('..')) 14 | 15 | # -- General configuration ----------------------------------------------------- 16 | 17 | # Sphinx extension module names. 18 | extensions = [ 19 | 'sphinx.ext.autodoc', 20 | 'sphinx.ext.graphviz', 21 | 'sphinx.ext.inheritance_diagram', 22 | 'sphinx.ext.intersphinx', 23 | 'humanfriendly.sphinx', 24 | ] 25 | 26 | # Paths that contain templates, relative to this directory. 27 | templates_path = ['templates'] 28 | 29 | # The suffix of source filenames. 30 | source_suffix = '.rst' 31 | 32 | # The master toctree document. 33 | master_doc = 'index' 34 | 35 | # General information about the project. 36 | project = u'pip-accel' 37 | copyright = u'2016, Peter Odding and Paylogic International' 38 | 39 | # The version info for the project you're documenting, acts as replacement for 40 | # |version| and |release|, also used in various other places throughout the 41 | # built documents. 42 | 43 | # Find the package version and make it the release. 44 | from pip_accel import __version__ as pip_accel_version # NOQA 45 | 46 | # The short X.Y version. 47 | version = '.'.join(pip_accel_version.split('.')[:2]) 48 | 49 | # The full version, including alpha/beta/rc tags. 50 | release = pip_accel_version 51 | 52 | # The language for content autogenerated by Sphinx. Refer to documentation 53 | # for a list of supported languages. 54 | language = 'en' 55 | 56 | # List of patterns, relative to source directory, that match files and 57 | # directories to ignore when looking for source files. 58 | exclude_patterns = ['build'] 59 | 60 | # If true, '()' will be appended to :func: etc. cross-reference text. 61 | add_function_parentheses = True 62 | 63 | # http://sphinx-doc.org/ext/autodoc.html#confval-autodoc_member_order 64 | autodoc_member_order = 'bysource' 65 | 66 | # The name of the Pygments (syntax highlighting) style to use. 67 | pygments_style = 'sphinx' 68 | 69 | # Refer to the Python standard library. 70 | # From: http://twistedmatrix.com/trac/ticket/4582. 71 | intersphinx_mapping = dict( 72 | boto=('http://boto.readthedocs.org/en/latest/', None), 73 | coloredlogs=('http://coloredlogs.readthedocs.org/en/latest/', None), 74 | humanfriendly=('http://humanfriendly.readthedocs.org/en/latest/', None), 75 | python=('http://docs.python.org', None), 76 | ) 77 | 78 | # -- Options for HTML output --------------------------------------------------- 79 | 80 | # The theme to use for HTML and HTML Help pages. See the documentation for 81 | # a list of builtin themes. 82 | html_theme = 'default' 83 | 84 | # Output file base name for HTML help builder. 85 | htmlhelp_basename = 'pip-acceldoc' 86 | -------------------------------------------------------------------------------- /docs/developers.rst: -------------------------------------------------------------------------------- 1 | Documentation for the pip accelerator API 2 | ========================================= 3 | 4 | On this page you can find the complete API documentation of pip-accel 5 | |release|. 6 | 7 | A note about backwards compatibility 8 | ------------------------------------ 9 | 10 | Please note that pip-accel has not yet reached a 1.0 version and until that 11 | time arbitrary changes to the API can be made. To clarify that statement: 12 | 13 | - On the one hand I value API stability and I've built a dozen tools on top of 14 | pip-accel myself so I don't think too lightly about breaking backwards 15 | compatibility :-) 16 | 17 | - On the other hand if I see opportunities to simplify the code base or make 18 | things more robust I will go ahead and do it. Furthermore the implementation 19 | of pip-accel is dictated (to a certain extent) by pip and this certainly 20 | influences the API. For example API changes may be necessary to facilitate 21 | the upgrade to pip 1.5.x (the current version of pip-accel is based on pip 22 | 1.4.x). 23 | 24 | In pip-accel 0.16 a completely new API was introduced and support for the old 25 | "API" was dropped. The goal of the new API is to last for quite a while but of 26 | course only time will tell if that plan is going to work out :-) 27 | 28 | The Python API of pip-accel 29 | --------------------------- 30 | 31 | Here are the relevant Python modules that make up pip-accel: 32 | 33 | .. contents:: 34 | :local: 35 | 36 | :mod:`pip_accel` 37 | ~~~~~~~~~~~~~~~~ 38 | 39 | .. automodule:: pip_accel 40 | :members: 41 | 42 | :mod:`pip_accel.config` 43 | ~~~~~~~~~~~~~~~~~~~~~~~ 44 | 45 | .. automodule:: pip_accel.config 46 | :members: 47 | 48 | :mod:`pip_accel.req` 49 | ~~~~~~~~~~~~~~~~~~~~~~~ 50 | 51 | .. automodule:: pip_accel.req 52 | :members: 53 | 54 | :mod:`pip_accel.bdist` 55 | ~~~~~~~~~~~~~~~~~~~~~~ 56 | 57 | .. automodule:: pip_accel.bdist 58 | :members: 59 | 60 | :mod:`pip_accel.caches` 61 | ~~~~~~~~~~~~~~~~~~~~~~~ 62 | 63 | .. automodule:: pip_accel.caches 64 | :members: 65 | 66 | :mod:`pip_accel.caches.local` 67 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 68 | 69 | .. automodule:: pip_accel.caches.local 70 | :members: 71 | 72 | :mod:`pip_accel.caches.s3` 73 | ~~~~~~~~~~~~~~~~~~~~~~~~~~ 74 | 75 | .. automodule:: pip_accel.caches.s3 76 | :members: 77 | 78 | :mod:`pip_accel.deps` 79 | ~~~~~~~~~~~~~~~~~~~~~ 80 | 81 | .. automodule:: pip_accel.deps 82 | :members: 83 | 84 | :mod:`pip_accel.utils` 85 | ~~~~~~~~~~~~~~~~~~~~~~ 86 | 87 | .. automodule:: pip_accel.utils 88 | :members: 89 | 90 | :mod:`pip_accel.exceptions` 91 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~ 92 | 93 | .. automodule:: pip_accel.exceptions 94 | :members: 95 | 96 | :mod:`pip_accel.tests` 97 | ~~~~~~~~~~~~~~~~~~~~~~ 98 | 99 | .. automodule:: pip_accel.tests 100 | :members: 101 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | Documentation for the pip accelerator 2 | ===================================== 3 | 4 | The pip accelerator makes `pip `_ (the Python 5 | package manager) faster by keeping pip off the internet when possible and by 6 | caching compiled binary distributions. It can bring a 10 minute run of ``pip`` 7 | down to less than a minute. You can find the pip accelerator in the following 8 | places: 9 | 10 | - The source code lives on `GitHub `_ 11 | - Downloads are available in the `Python Package Index `_ 12 | - Online documentation is hosted by `Read The Docs `_ 13 | 14 | This is the documentation for version |release| of the pip accelerator. The 15 | documentation consists of two parts: 16 | 17 | - The documentation for users of the ``pip-accel`` command 18 | - The documentation for developers who wish to extend and/or embed the 19 | functionality of ``pip-accel`` 20 | 21 | Introduction & usage 22 | -------------------- 23 | 24 | The first part of the documentation is the readme which is targeted at users of 25 | the ``pip-accel`` command. Here are the topics discussed in the readme: 26 | 27 | .. toctree:: 28 | :maxdepth: 2 29 | 30 | users.rst 31 | 32 | Internal API documentation 33 | -------------------------- 34 | 35 | The second part of the documentation is targeted at developers who wish to 36 | extend and/or embed the functionality of ``pip-accel``. Here are the contents 37 | of the API documentation: 38 | 39 | .. toctree:: 40 | :maxdepth: 3 41 | 42 | developers.rst 43 | -------------------------------------------------------------------------------- /docs/users.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../README.rst 2 | -------------------------------------------------------------------------------- /pip_accel/__init__.py: -------------------------------------------------------------------------------- 1 | # Accelerator for pip, the Python package manager. 2 | # 3 | # Author: Peter Odding 4 | # Last Change: March 14, 2016 5 | # URL: https://github.com/paylogic/pip-accel 6 | # 7 | # TODO Permanently store logs in the pip-accel directory (think about log rotation). 8 | # TODO Maybe we should save the output of `python setup.py bdist_dumb` somewhere as well? 9 | 10 | """ 11 | Top level functionality of `pip-accel`. 12 | 13 | The Python module :mod:`pip_accel` defines the classes that implement the 14 | top level functionality of the pip accelerator. Instead of using the 15 | ``pip-accel`` command you can also use the pip accelerator as a Python module, 16 | in this case you'll probably want to start by taking a look at 17 | the :class:`PipAccelerator` class. 18 | 19 | Wheel support 20 | ------------- 21 | 22 | During the upgrade to pip 6 support for installation of wheels_ was added to 23 | pip-accel. The ``pip-accel`` command line program now downloads and installs 24 | wheels when available for a given requirement, but part of pip-accel's Python 25 | API defaults to the more conservative choice of allowing callers to opt-in to 26 | wheel support. 27 | 28 | This is because previous versions of pip-accel would only download source 29 | distributions and pip-accel provides the functionality to convert those source 30 | distributions to "dumb binary distributions". This functionality is exposed to 31 | callers who may depend on this mode of operation. So for now users of the 32 | Python API get to decide whether they're interested in wheels or not. 33 | 34 | Setuptools upgrade 35 | ~~~~~~~~~~~~~~~~~~ 36 | 37 | If the requirement set includes wheels and ``setuptools >= 0.8`` is not yet 38 | installed, it will be added to the requirement set and installed together with 39 | the other requirement(s) in order to enable the usage of distributions 40 | installed from wheels (their metadata is different). 41 | 42 | .. _wheels: https://pypi.python.org/pypi/wheel 43 | """ 44 | 45 | # Standard library modules. 46 | import logging 47 | import os 48 | import os.path 49 | import shutil 50 | import sys 51 | import tempfile 52 | 53 | # Modules included in our package. 54 | from pip_accel.bdist import BinaryDistributionManager 55 | from pip_accel.compat import basestring 56 | from pip_accel.exceptions import EnvironmentMismatchError, NothingToDoError 57 | from pip_accel.req import Requirement, TransactionalUpdate 58 | from pip_accel.utils import ( 59 | create_file_url, 60 | hash_files, 61 | is_installed, 62 | makedirs, 63 | match_option, 64 | match_option_with_value, 65 | requirement_is_installed, 66 | same_directories, 67 | uninstall, 68 | ) 69 | 70 | # External dependencies. 71 | from humanfriendly import concatenate, Timer, pluralize 72 | from pip import index as pip_index_module 73 | from pip import wheel as pip_wheel_module 74 | from pip.commands import install as pip_install_module 75 | from pip.commands.install import InstallCommand 76 | from pip.exceptions import DistributionNotFound 77 | from pip.req import InstallRequirement 78 | 79 | # Semi-standard module versioning. 80 | __version__ = '0.43' 81 | 82 | # Initialize a logger for this module. 83 | logger = logging.getLogger(__name__) 84 | 85 | 86 | class PipAccelerator(object): 87 | 88 | """ 89 | Accelerator for pip, the Python package manager. 90 | 91 | The :class:`PipAccelerator` class brings together the top level logic of 92 | pip-accel. This top level logic was previously just a collection of 93 | functions but that became more unwieldy as the amount of internal state 94 | increased. The :class:`PipAccelerator` class is intended to make it 95 | (relatively) easy to build something on top of pip and pip-accel. 96 | """ 97 | 98 | def __init__(self, config, validate=True): 99 | """ 100 | Initialize the pip accelerator. 101 | 102 | :param config: The pip-accel configuration (a :class:`.Config` 103 | object). 104 | :param validate: :data:`True` to run :func:`validate_environment()`, 105 | :data:`False` otherwise. 106 | """ 107 | self.config = config 108 | self.bdists = BinaryDistributionManager(self.config) 109 | if validate: 110 | self.validate_environment() 111 | self.initialize_directories() 112 | self.clean_source_index() 113 | # Keep a list of build directories created by pip-accel. 114 | self.build_directories = [] 115 | # We hold on to returned Requirement objects so we can remove their 116 | # temporary sources after pip-accel has finished. 117 | self.reported_requirements = [] 118 | # Keep a list of `.eggs' symbolic links created by pip-accel. 119 | self.eggs_links = [] 120 | 121 | def validate_environment(self): 122 | """ 123 | Make sure :data:`sys.prefix` matches ``$VIRTUAL_ENV`` (if defined). 124 | 125 | This may seem like a strange requirement to dictate but it avoids hairy 126 | issues like `documented here `_. 127 | 128 | The most sneaky thing is that ``pip`` doesn't have this problem 129 | (de-facto) because ``virtualenv`` copies ``pip`` wherever it goes... 130 | (``pip-accel`` on the other hand has to be installed by the user). 131 | """ 132 | environment = os.environ.get('VIRTUAL_ENV') 133 | if environment: 134 | if not same_directories(sys.prefix, environment): 135 | raise EnvironmentMismatchError(""" 136 | You are trying to install packages in environment #1 which 137 | is different from environment #2 where pip-accel is 138 | installed! Please install pip-accel under environment #1 to 139 | install packages there. 140 | 141 | Environment #1: {environment} (defined by $VIRTUAL_ENV) 142 | 143 | Environment #2: {prefix} (Python's installation prefix) 144 | """, environment=environment, prefix=sys.prefix) 145 | 146 | def initialize_directories(self): 147 | """Automatically create local directories required by pip-accel.""" 148 | makedirs(self.config.source_index) 149 | makedirs(self.config.eggs_cache) 150 | 151 | def clean_source_index(self): 152 | """ 153 | Cleanup broken symbolic links in the local source distribution index. 154 | 155 | The purpose of this method requires some context to understand. Let me 156 | preface this by stating that I realize I'm probably overcomplicating 157 | things, but I like to preserve forward / backward compatibility when 158 | possible and I don't feel like dropping everyone's locally cached 159 | source distribution archives without a good reason to do so. With that 160 | out of the way: 161 | 162 | - Versions of pip-accel based on pip 1.4.x maintained a local source 163 | distribution index based on a directory containing symbolic links 164 | pointing directly into pip's download cache. When files were removed 165 | from pip's download cache, broken symbolic links remained in 166 | pip-accel's local source distribution index directory. This resulted 167 | in very confusing error messages. To avoid this 168 | :func:`clean_source_index()` cleaned up broken symbolic links 169 | whenever pip-accel was about to invoke pip. 170 | 171 | - More recent versions of pip (6.x) no longer support the same style of 172 | download cache that contains source distribution archives that can be 173 | re-used directly by pip-accel. To cope with the changes in pip 6.x 174 | new versions of pip-accel tell pip to download source distribution 175 | archives directly into the local source distribution index directory 176 | maintained by pip-accel. 177 | 178 | - It is very reasonable for users of pip-accel to have multiple 179 | versions of pip-accel installed on their system (imagine a dozen 180 | Python virtual environments that won't all be updated at the same 181 | time; this is the situation I always find myself in :-). These 182 | versions of pip-accel will be sharing the same local source 183 | distribution index directory. 184 | 185 | - All of this leads up to the local source distribution index directory 186 | containing a mixture of symbolic links and regular files with no 187 | obvious way to atomically and gracefully upgrade the local source 188 | distribution index directory while avoiding fights between old and 189 | new versions of pip-accel :-). 190 | 191 | - I could of course switch to storing the new local source distribution 192 | index in a differently named directory (avoiding potential conflicts 193 | between multiple versions of pip-accel) but then I would have to 194 | introduce a new configuration option, otherwise everyone who has 195 | configured pip-accel to store its source index in a non-default 196 | location could still be bitten by compatibility issues. 197 | 198 | For now I've decided to keep using the same directory for the local 199 | source distribution index and to keep cleaning up broken symbolic 200 | links. This enables cooperating between old and new versions of 201 | pip-accel and avoids trashing user's local source distribution indexes. 202 | The main disadvantage is that pip-accel is still required to clean up 203 | broken symbolic links... 204 | """ 205 | cleanup_timer = Timer() 206 | cleanup_counter = 0 207 | for entry in os.listdir(self.config.source_index): 208 | pathname = os.path.join(self.config.source_index, entry) 209 | if os.path.islink(pathname) and not os.path.exists(pathname): 210 | logger.warn("Cleaning up broken symbolic link: %s", pathname) 211 | os.unlink(pathname) 212 | cleanup_counter += 1 213 | logger.debug("Cleaned up %i broken symbolic links from source index in %s.", cleanup_counter, cleanup_timer) 214 | 215 | def install_from_arguments(self, arguments, **kw): 216 | """ 217 | Download, unpack, build and install the specified requirements. 218 | 219 | This function is a simple wrapper for :func:`get_requirements()`, 220 | :func:`install_requirements()` and :func:`cleanup_temporary_directories()` 221 | that implements the default behavior of the pip accelerator. If you're 222 | extending or embedding pip-accel you may want to call the underlying 223 | methods instead. 224 | 225 | If the requirement set includes wheels and ``setuptools >= 0.8`` is not 226 | yet installed, it will be added to the requirement set and installed 227 | together with the other requirement(s) in order to enable the usage of 228 | distributions installed from wheels (their metadata is different). 229 | 230 | :param arguments: The command line arguments to ``pip install ..`` (a 231 | list of strings). 232 | :param kw: Any keyword arguments are passed on to 233 | :func:`install_requirements()`. 234 | :returns: The result of :func:`install_requirements()`. 235 | """ 236 | try: 237 | requirements = self.get_requirements(arguments, use_wheels=self.arguments_allow_wheels(arguments)) 238 | have_wheels = any(req.is_wheel for req in requirements) 239 | if have_wheels and not self.setuptools_supports_wheels(): 240 | logger.info("Preparing to upgrade to setuptools >= 0.8 to enable wheel support ..") 241 | requirements.extend(self.get_requirements(['setuptools >= 0.8'])) 242 | if requirements: 243 | if '--user' in arguments: 244 | from site import USER_BASE 245 | kw.setdefault('prefix', USER_BASE) 246 | return self.install_requirements(requirements, **kw) 247 | else: 248 | logger.info("Nothing to do! (requirements already installed)") 249 | return 0 250 | finally: 251 | self.cleanup_temporary_directories() 252 | 253 | def setuptools_supports_wheels(self): 254 | """ 255 | Check whether setuptools should be upgraded to ``>= 0.8`` for wheel support. 256 | 257 | :returns: :data:`True` when setuptools 0.8 or higher is already 258 | installed, :data:`False` otherwise (it needs to be upgraded). 259 | """ 260 | return requirement_is_installed('setuptools >= 0.8') 261 | 262 | def get_requirements(self, arguments, max_retries=None, use_wheels=False): 263 | """ 264 | Use pip to download and unpack the requested source distribution archives. 265 | 266 | :param arguments: The command line arguments to ``pip install ...`` (a 267 | list of strings). 268 | :param max_retries: The maximum number of times that pip will be asked 269 | to download distribution archives (this helps to 270 | deal with intermittent failures). If this is 271 | :data:`None` then :attr:`~.Config.max_retries` is 272 | used. 273 | :param use_wheels: Whether pip and pip-accel are allowed to use wheels_ 274 | (:data:`False` by default for backwards compatibility 275 | with callers that use pip-accel as a Python API). 276 | 277 | .. warning:: Requirements which are already installed are not included 278 | in the result. If this breaks your use case consider using 279 | pip's ``--ignore-installed`` option. 280 | """ 281 | arguments = self.decorate_arguments(arguments) 282 | # Demote hash sum mismatch log messages from CRITICAL to DEBUG (hiding 283 | # implementation details from users unless they want to see them). 284 | with DownloadLogFilter(): 285 | with SetupRequiresPatch(self.config, self.eggs_links): 286 | # Use a new build directory for each run of get_requirements(). 287 | self.create_build_directory() 288 | # Check whether -U or --upgrade was given. 289 | if any(match_option(a, '-U', '--upgrade') for a in arguments): 290 | logger.info("Checking index(es) for new version (-U or --upgrade was given) ..") 291 | else: 292 | # If -U or --upgrade wasn't given and all requirements can be 293 | # satisfied using the archives in pip-accel's local source 294 | # index we don't need pip to connect to PyPI looking for new 295 | # versions (that will just slow us down). 296 | try: 297 | return self.unpack_source_dists(arguments, use_wheels=use_wheels) 298 | except DistributionNotFound: 299 | logger.info("We don't have all distribution archives yet!") 300 | # Get the maximum number of retries from the configuration if the 301 | # caller didn't specify a preference. 302 | if max_retries is None: 303 | max_retries = self.config.max_retries 304 | # If not all requirements are available locally we use pip to 305 | # download the missing source distribution archives from PyPI (we 306 | # retry a couple of times in case pip reports recoverable 307 | # errors). 308 | for i in range(max_retries): 309 | try: 310 | return self.download_source_dists(arguments, use_wheels=use_wheels) 311 | except Exception as e: 312 | if i + 1 < max_retries: 313 | # On all but the last iteration we swallow exceptions 314 | # during downloading. 315 | logger.warning("pip raised exception while downloading distributions: %s", e) 316 | else: 317 | # On the last iteration we don't swallow exceptions 318 | # during downloading because the error reported by pip 319 | # is the most sensible error for us to report. 320 | raise 321 | logger.info("Retrying after pip failed (%i/%i) ..", i + 1, max_retries) 322 | 323 | def decorate_arguments(self, arguments): 324 | """ 325 | Change pathnames of local files into ``file://`` URLs with ``#md5=...`` fragments. 326 | 327 | :param arguments: The command line arguments to ``pip install ...`` (a 328 | list of strings). 329 | :returns: A copy of the command line arguments with pathnames of local 330 | files rewritten to ``file://`` URLs. 331 | 332 | When pip-accel calls pip to download missing distribution archives and 333 | the user specified the pathname of a local distribution archive on the 334 | command line, pip will (by default) *not* copy the archive into the 335 | download directory if an archive for the same package name and 336 | version is already present. 337 | 338 | This can lead to the confusing situation where the user specifies a 339 | local distribution archive to install, a different (older) archive for 340 | the same package and version is present in the download directory and 341 | `pip-accel` installs the older archive instead of the newer archive. 342 | 343 | To avoid this confusing behavior, the :func:`decorate_arguments()` 344 | method rewrites the command line arguments given to ``pip install`` so 345 | that pathnames of local archives are changed into ``file://`` URLs that 346 | include a fragment with the hash of the file's contents. Here's an 347 | example: 348 | 349 | - Local pathname: ``/tmp/pep8-1.6.3a0.tar.gz`` 350 | - File URL: ``file:///tmp/pep8-1.6.3a0.tar.gz#md5=19cbf0b633498ead63fb3c66e5f1caf6`` 351 | 352 | When pip fills the download directory and encounters a previously 353 | cached distribution archive it will check the hash, realize the 354 | contents have changed and replace the archive in the download 355 | directory. 356 | """ 357 | arguments = list(arguments) 358 | for i, value in enumerate(arguments): 359 | is_constraint_file = (i >= 1 and match_option(arguments[i - 1], '-c', '--constraint')) 360 | is_requirement_file = (i >= 1 and match_option(arguments[i - 1], '-r', '--requirement')) 361 | if not is_constraint_file and not is_requirement_file and os.path.isfile(value): 362 | arguments[i] = '%s#md5=%s' % (create_file_url(value), hash_files('md5', value)) 363 | return arguments 364 | 365 | def unpack_source_dists(self, arguments, use_wheels=False): 366 | """ 367 | Find and unpack local source distributions and discover their metadata. 368 | 369 | :param arguments: The command line arguments to ``pip install ...`` (a 370 | list of strings). 371 | :param use_wheels: Whether pip and pip-accel are allowed to use wheels_ 372 | (:data:`False` by default for backwards compatibility 373 | with callers that use pip-accel as a Python API). 374 | :returns: A list of :class:`pip_accel.req.Requirement` objects. 375 | :raises: Any exceptions raised by pip, for example 376 | :exc:`pip.exceptions.DistributionNotFound` when not all 377 | requirements can be satisfied. 378 | 379 | This function checks whether there are local source distributions 380 | available for all requirements, unpacks the source distribution 381 | archives and finds the names and versions of the requirements. By using 382 | the ``pip install --download`` command we avoid reimplementing the 383 | following pip features: 384 | 385 | - Parsing of ``requirements.txt`` (including recursive parsing). 386 | - Resolution of possibly conflicting pinned requirements. 387 | - Unpacking source distributions in multiple formats. 388 | - Finding the name & version of a given source distribution. 389 | """ 390 | unpack_timer = Timer() 391 | logger.info("Unpacking distribution(s) ..") 392 | with PatchedAttribute(pip_install_module, 'PackageFinder', CustomPackageFinder): 393 | requirements = self.get_pip_requirement_set(arguments, use_remote_index=False, use_wheels=use_wheels) 394 | logger.info("Finished unpacking %s in %s.", pluralize(len(requirements), "distribution"), unpack_timer) 395 | return requirements 396 | 397 | def download_source_dists(self, arguments, use_wheels=False): 398 | """ 399 | Download missing source distributions. 400 | 401 | :param arguments: The command line arguments to ``pip install ...`` (a 402 | list of strings). 403 | :param use_wheels: Whether pip and pip-accel are allowed to use wheels_ 404 | (:data:`False` by default for backwards compatibility 405 | with callers that use pip-accel as a Python API). 406 | :raises: Any exceptions raised by pip. 407 | """ 408 | download_timer = Timer() 409 | logger.info("Downloading missing distribution(s) ..") 410 | requirements = self.get_pip_requirement_set(arguments, use_remote_index=True, use_wheels=use_wheels) 411 | logger.info("Finished downloading distribution(s) in %s.", download_timer) 412 | return requirements 413 | 414 | def get_pip_requirement_set(self, arguments, use_remote_index, use_wheels=False): 415 | """ 416 | Get the unpacked requirement(s) specified by the caller by running pip. 417 | 418 | :param arguments: The command line arguments to ``pip install ...`` (a 419 | list of strings). 420 | :param use_remote_index: A boolean indicating whether pip is allowed to 421 | connect to the main package index 422 | (http://pypi.python.org by default). 423 | :param use_wheels: Whether pip and pip-accel are allowed to use wheels_ 424 | (:data:`False` by default for backwards compatibility 425 | with callers that use pip-accel as a Python API). 426 | :returns: A :class:`pip.req.RequirementSet` object created by pip. 427 | :raises: Any exceptions raised by pip. 428 | """ 429 | # Compose the pip command line arguments. This is where a lot of the 430 | # core logic of pip-accel is hidden and it uses some esoteric features 431 | # of pip so this method is heavily commented. 432 | command_line = [] 433 | # Use `--download' to instruct pip to download requirement(s) into 434 | # pip-accel's local source distribution index directory. This has the 435 | # following documented side effects (see `pip install --help'): 436 | # 1. It disables the installation of requirements (without using the 437 | # `--no-install' option which is deprecated and slated for removal 438 | # in pip 7.x). 439 | # 2. It ignores requirements that are already installed (because 440 | # pip-accel doesn't actually need to re-install requirements that 441 | # are already installed we will have work around this later, but 442 | # that seems fairly simple to do). 443 | command_line.append('--download=%s' % self.config.source_index) 444 | # Use `--find-links' to point pip at pip-accel's local source 445 | # distribution index directory. This ensures that source distribution 446 | # archives are never downloaded more than once (regardless of the HTTP 447 | # cache that was introduced in pip 6.x). 448 | command_line.append('--find-links=%s' % create_file_url(self.config.source_index)) 449 | # Use `--no-binary=:all:' to ignore wheel distributions by default in 450 | # order to preserve backwards compatibility with callers that expect a 451 | # requirement set consisting only of source distributions that can be 452 | # converted to `dumb binary distributions'. 453 | if not use_wheels and self.arguments_allow_wheels(arguments): 454 | command_line.append('--no-binary=:all:') 455 | # Use `--no-index' to force pip to only consider source distribution 456 | # archives contained in pip-accel's local source distribution index 457 | # directory. This enables pip-accel to ask pip "Can the local source 458 | # distribution index satisfy all requirements in the given requirement 459 | # set?" which enables pip-accel to keep pip off the internet unless 460 | # absolutely necessary :-). 461 | if not use_remote_index: 462 | command_line.append('--no-index') 463 | # Use `--no-clean' to instruct pip to unpack the source distribution 464 | # archives and *not* clean up the unpacked source distributions 465 | # afterwards. This enables pip-accel to replace pip's installation 466 | # logic with cached binary distribution archives. 467 | command_line.append('--no-clean') 468 | # Use `--build-directory' to instruct pip to unpack the source 469 | # distribution archives to a temporary directory managed by pip-accel. 470 | # We will clean up the build directory when we're done using the 471 | # unpacked source distributions. 472 | command_line.append('--build-directory=%s' % self.build_directory) 473 | # Append the user's `pip install ...' arguments to the command line 474 | # that we just assembled. 475 | command_line.extend(arguments) 476 | logger.info("Executing command: pip install %s", ' '.join(command_line)) 477 | # Clear the build directory to prevent PreviousBuildDirError exceptions. 478 | self.clear_build_directory() 479 | # During the pip 6.x upgrade pip-accel switched to using `pip install 480 | # --download' which can produce an interactive prompt as described in 481 | # issue 51 [1]. The documented way [2] to get rid of this interactive 482 | # prompt is pip's --exists-action option, but due to what is most 483 | # likely a bug in pip this doesn't actually work. The environment 484 | # variable $PIP_EXISTS_ACTION does work however, so if the user didn't 485 | # set it we will set a reasonable default for them. 486 | # [1] https://github.com/paylogic/pip-accel/issues/51 487 | # [2] https://pip.pypa.io/en/latest/reference/pip.html#exists-action-option 488 | os.environ.setdefault('PIP_EXISTS_ACTION', 'w') 489 | # Initialize and run the `pip install' command. 490 | command = InstallCommand() 491 | opts, args = command.parse_args(command_line) 492 | if not opts.ignore_installed: 493 | # If the user didn't supply the -I, --ignore-installed option we 494 | # will forcefully disable the option. Refer to the documentation of 495 | # the AttributeOverrides class for further details. 496 | opts = AttributeOverrides(opts, ignore_installed=False) 497 | requirement_set = command.run(opts, args) 498 | # Make sure the output of pip and pip-accel are not intermingled. 499 | sys.stdout.flush() 500 | if requirement_set is None: 501 | raise NothingToDoError(""" 502 | pip didn't generate a requirement set, most likely you 503 | specified an empty requirements file? 504 | """) 505 | else: 506 | return self.transform_pip_requirement_set(requirement_set) 507 | 508 | def transform_pip_requirement_set(self, requirement_set): 509 | """ 510 | Transform pip's requirement set into one that `pip-accel` can work with. 511 | 512 | :param requirement_set: The :class:`pip.req.RequirementSet` object 513 | reported by pip. 514 | :returns: A list of :class:`pip_accel.req.Requirement` objects. 515 | 516 | This function converts the :class:`pip.req.RequirementSet` object 517 | reported by pip into a list of :class:`pip_accel.req.Requirement` 518 | objects. 519 | """ 520 | filtered_requirements = [] 521 | for requirement in requirement_set.requirements.values(): 522 | # The `satisfied_by' property is set by pip when a requirement is 523 | # already satisfied (i.e. a version of the package that satisfies 524 | # the requirement is already installed) and -I, --ignore-installed 525 | # is not used. We filter out these requirements because pip never 526 | # unpacks distributions for these requirements, so pip-accel can't 527 | # do anything useful with such requirements. 528 | if requirement.satisfied_by: 529 | continue 530 | # The `constraint' property marks requirement objects that 531 | # constrain the acceptable version(s) of another requirement but 532 | # don't define a requirement themselves, so we filter them out. 533 | if requirement.constraint: 534 | continue 535 | # All other requirements are reported to callers. 536 | filtered_requirements.append(requirement) 537 | self.reported_requirements.append(requirement) 538 | return sorted([Requirement(self.config, r) for r in filtered_requirements], 539 | key=lambda r: r.name.lower()) 540 | 541 | def install_requirements(self, requirements, **kw): 542 | """ 543 | Manually install a requirement set from binary and/or wheel distributions. 544 | 545 | :param requirements: A list of :class:`pip_accel.req.Requirement` objects. 546 | :param kw: Any keyword arguments are passed on to 547 | :func:`~pip_accel.bdist.BinaryDistributionManager.install_binary_dist()`. 548 | :returns: The number of packages that were just installed (an integer). 549 | """ 550 | install_timer = Timer() 551 | install_types = [] 552 | if any(not req.is_wheel for req in requirements): 553 | install_types.append('binary') 554 | if any(req.is_wheel for req in requirements): 555 | install_types.append('wheel') 556 | logger.info("Installing from %s distributions ..", concatenate(install_types)) 557 | # Track installed files by default (unless the caller specifically opted out). 558 | kw.setdefault('track_installed_files', True) 559 | num_installed = 0 560 | for requirement in requirements: 561 | # When installing setuptools we need to uninstall distribute, 562 | # otherwise distribute will shadow setuptools and all sorts of 563 | # strange issues can occur (e.g. upgrading to the latest 564 | # setuptools to gain wheel support and then having everything 565 | # blow up because distribute doesn't know about wheels). 566 | if requirement.name == 'setuptools' and is_installed('distribute'): 567 | uninstall('distribute') 568 | if requirement.is_editable: 569 | logger.debug("Installing %s in editable form using pip.", requirement) 570 | with TransactionalUpdate(requirement): 571 | command = InstallCommand() 572 | opts, args = command.parse_args(['--no-deps', '--editable', requirement.source_directory]) 573 | command.run(opts, args) 574 | elif requirement.is_wheel: 575 | logger.info("Installing %s wheel distribution using pip ..", requirement) 576 | with TransactionalUpdate(requirement): 577 | wheel_version = pip_wheel_module.wheel_version(requirement.source_directory) 578 | pip_wheel_module.check_compatibility(wheel_version, requirement.name) 579 | requirement.pip_requirement.move_wheel_files(requirement.source_directory) 580 | else: 581 | logger.info("Installing %s binary distribution using pip-accel ..", requirement) 582 | with TransactionalUpdate(requirement): 583 | binary_distribution = self.bdists.get_binary_dist(requirement) 584 | self.bdists.install_binary_dist(binary_distribution, **kw) 585 | num_installed += 1 586 | logger.info("Finished installing %s in %s.", 587 | pluralize(num_installed, "requirement"), 588 | install_timer) 589 | return num_installed 590 | 591 | def arguments_allow_wheels(self, arguments): 592 | """ 593 | Check whether the given command line arguments allow the use of wheels. 594 | 595 | :param arguments: A list of strings with command line arguments. 596 | :returns: :data:`True` if the arguments allow wheels, :data:`False` if 597 | they disallow wheels. 598 | 599 | Contrary to what the name of this method implies its implementation 600 | actually checks if the user hasn't *disallowed* the use of wheels using 601 | the ``--no-use-wheel`` option (deprecated in pip 7.x) or the 602 | ``--no-binary=:all:`` option (introduced in pip 7.x). This is because 603 | wheels are "opt out" in recent versions of pip. I just didn't like the 604 | method name ``arguments_dont_disallow_wheels`` ;-). 605 | """ 606 | return not ('--no-use-wheel' in arguments or match_option_with_value(arguments, '--no-binary', ':all:')) 607 | 608 | def create_build_directory(self): 609 | """Create a new build directory for pip to unpack its archives.""" 610 | self.build_directories.append(tempfile.mkdtemp(prefix='pip-accel-build-dir-')) 611 | 612 | def clear_build_directory(self): 613 | """Clear the build directory where pip unpacks the source distribution archives.""" 614 | stat = os.stat(self.build_directory) 615 | shutil.rmtree(self.build_directory) 616 | os.makedirs(self.build_directory, stat.st_mode) 617 | 618 | def cleanup_temporary_directories(self): 619 | """Delete the build directories and any temporary directories created by pip.""" 620 | while self.build_directories: 621 | shutil.rmtree(self.build_directories.pop()) 622 | for requirement in self.reported_requirements: 623 | requirement.remove_temporary_source() 624 | while self.eggs_links: 625 | symbolic_link = self.eggs_links.pop() 626 | if os.path.islink(symbolic_link): 627 | os.unlink(symbolic_link) 628 | 629 | @property 630 | def build_directory(self): 631 | """Get the pathname of the current build directory (a string).""" 632 | if not self.build_directories: 633 | self.create_build_directory() 634 | return self.build_directories[-1] 635 | 636 | 637 | class DownloadLogFilter(logging.Filter): 638 | 639 | """ 640 | Rewrite log messages emitted by pip's ``pip.download`` module. 641 | 642 | When pip encounters hash mismatches it logs a message with the severity 643 | :data:`~logging.CRITICAL`, however because of the interaction between 644 | pip-accel and pip hash mismatches are to be expected and handled gracefully 645 | (refer to :func:`~PipAccelerator.decorate_arguments()` for details). The 646 | :class:`DownloadLogFilter` context manager changes the severity of these 647 | log messages to :data:`~logging.DEBUG` in order to avoid confusing users of 648 | pip-accel. 649 | """ 650 | 651 | KEYWORDS = ("doesn't", "match", "expected", "hash") 652 | 653 | def __enter__(self): 654 | """Enable the download log filter.""" 655 | self.logger = logging.getLogger('pip.download') 656 | self.logger.addFilter(self) 657 | 658 | def __exit__(self, exc_type=None, exc_value=None, traceback=None): 659 | """Disable the download log filter.""" 660 | self.logger.removeFilter(self) 661 | 662 | def filter(self, record): 663 | """Change the severity of selected log records.""" 664 | if isinstance(record.msg, basestring): 665 | message = record.msg.lower() 666 | if all(kw in message for kw in self.KEYWORDS): 667 | record.levelname = 'DEBUG' 668 | record.levelno = logging.DEBUG 669 | return 1 670 | 671 | 672 | class SetupRequiresPatch(object): 673 | 674 | """ 675 | Monkey patch to enable caching of setup requirements. 676 | 677 | This context manager monkey patches ``InstallRequirement.run_egg_info()`` 678 | to enable caching of setup requirements. It works by creating a symbolic 679 | link called ``.eggs`` in the source directory of unpacked Python source 680 | distributions which points to a shared directory inside the pip-accel 681 | data directory. This can only work on platforms that support 682 | :func:`os.symlink()`` but should fail gracefully elsewhere. 683 | 684 | The :class:`SetupRequiresPatch` context manager doesn't clean up the 685 | symbolic links because doing so would remove the link when it is still 686 | being used. Instead the context manager builds up a list of created links 687 | so that pip-accel can clean these up when it is known that the symbolic 688 | links are no longer needed. 689 | 690 | For more information about this hack please refer to `issue 49 691 | `_. 692 | """ 693 | 694 | def __init__(self, config, created_links=None): 695 | """ 696 | Initialize a :class:`SetupRequiresPatch` object. 697 | 698 | :param config: A :class:`~pip_accel.config.Config` object. 699 | :param created_links: A list where newly created symbolic links are 700 | added to (so they can be cleaned up later). 701 | """ 702 | self.config = config 703 | self.patch = None 704 | self.created_links = created_links 705 | 706 | def __enter__(self): 707 | """Enable caching of setup requirements (by patching the ``run_egg_info()`` method).""" 708 | if self.patch is None: 709 | created_links = self.created_links 710 | original_method = InstallRequirement.run_egg_info 711 | shared_directory = self.config.eggs_cache 712 | 713 | def run_egg_info_wrapper(self, *args, **kw): 714 | # Heads up: self is an `InstallRequirement' object here! 715 | link_name = os.path.join(self.source_dir, '.eggs') 716 | try: 717 | logger.debug("Creating symbolic link: %s -> %s", link_name, shared_directory) 718 | os.symlink(shared_directory, link_name) 719 | if created_links is not None: 720 | created_links.append(link_name) 721 | except Exception as e: 722 | # Always log the failure, but only include a traceback if 723 | # it looks like symbolic links should be supported on the 724 | # current platform (os.symlink() is available). 725 | logger.debug("Failed to create symbolic link! (continuing without)", 726 | exc_info=not isinstance(e, AttributeError)) 727 | # Execute the real run_egg_info() method. 728 | return original_method(self, *args, **kw) 729 | 730 | # Install the wrapper method for the duration of the context manager. 731 | self.patch = PatchedAttribute(InstallRequirement, 'run_egg_info', run_egg_info_wrapper) 732 | self.patch.__enter__() 733 | 734 | def __exit__(self, exc_type=None, exc_value=None, traceback=None): 735 | """Undo the changes that enable caching of setup requirements.""" 736 | if self.patch is not None: 737 | self.patch.__exit__(exc_type, exc_value, traceback) 738 | self.patch = None 739 | 740 | 741 | class CustomPackageFinder(pip_index_module.PackageFinder): 742 | 743 | """ 744 | Custom :class:`pip.index.PackageFinder` to keep pip off the internet. 745 | 746 | This class customizes :class:`pip.index.PackageFinder` to enforce what 747 | the ``--no-index`` option does for the default package index but doesn't do 748 | for package indexes registered with the ``--index=`` option in requirements 749 | files. Judging by pip's documentation the fact that this has to be monkey 750 | patched seems like a bug / oversight in pip (IMHO). 751 | """ 752 | 753 | @property 754 | def index_urls(self): 755 | """Dummy list of index URLs that is always empty.""" 756 | return [] 757 | 758 | @index_urls.setter 759 | def index_urls(self, value): 760 | """Dummy setter for index URLs that ignores the value set.""" 761 | pass 762 | 763 | @property 764 | def dependency_links(self): 765 | """Dummy list of dependency links that is always empty.""" 766 | return [] 767 | 768 | @dependency_links.setter 769 | def dependency_links(self, value): 770 | """Dummy setter for dependency links that ignores the value set.""" 771 | pass 772 | 773 | 774 | class PatchedAttribute(object): 775 | 776 | """ 777 | Context manager to temporarily patch an object attribute. 778 | 779 | This context manager changes the value of an object attribute when the 780 | context is entered and restores the original value when the context is 781 | exited. 782 | """ 783 | 784 | def __init__(self, object, attribute, value, enabled=True): 785 | """ 786 | Initialize a :class:`PatchedAttribute` object. 787 | 788 | :param object: The object whose attribute should be patched. 789 | :param attribute: The name of the attribute to be patched (a string). 790 | :param value: The temporary value for the attribute. 791 | :param enabled: :data:`True` to patch the attribute, :data:`False` to 792 | do nothing instead. This enables conditional attribute 793 | patching while unconditionally using the 794 | :keyword:`with` statement. 795 | """ 796 | self.object = object 797 | self.attribute = attribute 798 | self.patched_value = value 799 | self.original_value = None 800 | self.enabled = enabled 801 | 802 | def __enter__(self): 803 | """Change the object attribute when entering the context.""" 804 | if self.enabled: 805 | self.original_value = getattr(self.object, self.attribute) 806 | setattr(self.object, self.attribute, self.patched_value) 807 | 808 | def __exit__(self, exc_type=None, exc_value=None, traceback=None): 809 | """Restore the object attribute when leaving the context.""" 810 | if self.enabled: 811 | setattr(self.object, self.attribute, self.original_value) 812 | 813 | 814 | class AttributeOverrides(object): 815 | 816 | """ 817 | :class:`AttributeOverrides` enables overriding of object attributes. 818 | 819 | During the pip 6.x upgrade pip-accel switched to using ``pip install 820 | --download`` which unintentionally broke backwards compatibility with 821 | previous versions of pip-accel as documented in `issue 52`_. 822 | 823 | The reason for this is that when pip is given the ``--download`` option it 824 | internally enables ``--ignore-installed`` (which can be problematic for 825 | certain use cases as described in `issue 52`_). There is no documented way 826 | to avoid this behavior, so instead pip-accel resorts to monkey patching to 827 | restore backwards compatibility. 828 | 829 | :class:`AttributeOverrides` is used to replace pip's parsed command line 830 | options object with an object that defers all attribute access (gets and 831 | sets) to the original options object but always reports 832 | ``ignore_installed`` as :data:`False`, even after it was set to :data:`True` by pip 833 | (as described above). 834 | 835 | .. _issue 52: https://github.com/paylogic/pip-accel/issues/52 836 | """ 837 | 838 | def __init__(self, opts, **overrides): 839 | """ 840 | Construct an :class:`AttributeOverrides` instance. 841 | 842 | :param opts: The object to which attribute access is deferred. 843 | :param overrides: The attributes whose value should be overridden. 844 | """ 845 | object.__setattr__(self, 'opts', opts) 846 | object.__setattr__(self, 'overrides', overrides) 847 | 848 | def __getattr__(self, name): 849 | """ 850 | Get an attribute's value from overrides or by deferring attribute access. 851 | 852 | :param name: The name of the attribute (a string). 853 | :returns: The attribute's value. 854 | """ 855 | if name in self.overrides: 856 | logger.debug("AttributeOverrides() getting %s from overrides ..", name) 857 | return self.overrides[name] 858 | else: 859 | logger.debug("AttributeOverrides() getting %s by deferring attribute access ..", name) 860 | return getattr(self.opts, name) 861 | 862 | def __setattr__(self, name, value): 863 | """ 864 | Set an attribute's value (unless it has an override). 865 | 866 | :param name: The name of the attribute (a string). 867 | :param value: The new value for the attribute. 868 | """ 869 | if name in self.overrides: 870 | logger.debug("AttributeOverrides() refusing to set %s=%r (attribute has override) ..", name, value) 871 | else: 872 | logger.debug("AttributeOverrides() setting %s=%r by deferring attribute access ..", name, value) 873 | setattr(self.opts, name, value) 874 | -------------------------------------------------------------------------------- /pip_accel/__main__.py: -------------------------------------------------------------------------------- 1 | # Accelerator for pip, the Python package manager. 2 | # 3 | # Author: Peter Odding 4 | # Last Change: March 14, 2016 5 | # URL: https://github.com/paylogic/pip-accel 6 | 7 | """ 8 | Enable running `pip-accel` as ``python -m pip_accel ...``. 9 | 10 | This module provides a uniform (platform independent) syntax for invoking 11 | `pip-accel`, that is to say the command line ``python -m pip_accel ...`` works 12 | the same on Windows, Linux and Mac OS X. 13 | 14 | This requires Python 2.7 or higher (it specifically doesn't work on Python 15 | 2.6). The way ``__main__`` modules work is documented under the documentation 16 | of the `python -m`_ construct. 17 | 18 | .. _python -m: https://docs.python.org/2/using/cmdline.html#cmdoption-m 19 | """ 20 | 21 | from pip_accel.cli import main 22 | 23 | if __name__ == '__main__': 24 | main() 25 | -------------------------------------------------------------------------------- /pip_accel/bdist.py: -------------------------------------------------------------------------------- 1 | # Accelerator for pip, the Python package manager. 2 | # 3 | # Author: Peter Odding 4 | # Last Change: November 7, 2015 5 | # URL: https://github.com/paylogic/pip-accel 6 | 7 | """ 8 | Functions to manipulate Python binary distribution archives. 9 | 10 | The functions in this module are used to create, transform and install from 11 | binary distribution archives (which are not supported by tools like 12 | easy_install and pip). 13 | """ 14 | 15 | # Standard library modules. 16 | import errno 17 | import fnmatch 18 | import logging 19 | import os 20 | import os.path 21 | import pipes 22 | import re 23 | import shutil 24 | import stat 25 | import subprocess 26 | import sys 27 | import tarfile 28 | import tempfile 29 | import time 30 | 31 | # External dependencies. 32 | from humanfriendly import Spinner, Timer, concatenate 33 | 34 | # Modules included in our package. 35 | from pip_accel.caches import CacheManager 36 | from pip_accel.deps import SystemPackageManager 37 | from pip_accel.exceptions import BuildFailed, InvalidSourceDistribution, NoBuildOutput 38 | from pip_accel.utils import AtomicReplace, compact, makedirs 39 | 40 | # Initialize a logger for this module. 41 | logger = logging.getLogger(__name__) 42 | 43 | 44 | class BinaryDistributionManager(object): 45 | 46 | """Generates and transforms Python binary distributions.""" 47 | 48 | def __init__(self, config): 49 | """ 50 | Initialize the binary distribution manager. 51 | 52 | :param config: The pip-accel configuration (a :class:`.Config` 53 | object). 54 | """ 55 | self.config = config 56 | self.cache = CacheManager(config) 57 | self.system_package_manager = SystemPackageManager(config) 58 | 59 | def get_binary_dist(self, requirement): 60 | """ 61 | Get or create a cached binary distribution archive. 62 | 63 | :param requirement: A :class:`.Requirement` object. 64 | :returns: An iterable of tuples with two values each: A 65 | :class:`tarfile.TarInfo` object and a file-like object. 66 | 67 | Gets the cached binary distribution that was previously built for the 68 | given requirement. If no binary distribution has been cached yet, a new 69 | binary distribution is built and added to the cache. 70 | 71 | Uses :func:`build_binary_dist()` to build binary distribution 72 | archives. If this fails with a build error :func:`get_binary_dist()` 73 | will use :class:`.SystemPackageManager` to check for and install 74 | missing system packages and retry the build when missing system 75 | packages were installed. 76 | """ 77 | cache_file = self.cache.get(requirement) 78 | if cache_file: 79 | if self.needs_invalidation(requirement, cache_file): 80 | logger.info("Invalidating old %s binary (source has changed) ..", requirement) 81 | cache_file = None 82 | else: 83 | logger.debug("%s hasn't been cached yet, doing so now.", requirement) 84 | if not cache_file: 85 | # Build the binary distribution. 86 | try: 87 | raw_file = self.build_binary_dist(requirement) 88 | except BuildFailed: 89 | logger.warning("Build of %s failed, checking for missing dependencies ..", requirement) 90 | if self.system_package_manager.install_dependencies(requirement): 91 | raw_file = self.build_binary_dist(requirement) 92 | else: 93 | raise 94 | # Transform the binary distribution archive into a form that we can re-use. 95 | fd, transformed_file = tempfile.mkstemp(prefix='pip-accel-bdist-', suffix='.tar.gz') 96 | try: 97 | archive = tarfile.open(transformed_file, 'w:gz') 98 | try: 99 | for member, from_handle in self.transform_binary_dist(raw_file): 100 | archive.addfile(member, from_handle) 101 | finally: 102 | archive.close() 103 | # Push the binary distribution archive to all available backends. 104 | with open(transformed_file, 'rb') as handle: 105 | self.cache.put(requirement, handle) 106 | finally: 107 | # Close file descriptor before removing the temporary file. 108 | # Without closing Windows is complaining that the file cannot 109 | # be removed because it is used by another process. 110 | os.close(fd) 111 | # Cleanup the temporary file. 112 | os.remove(transformed_file) 113 | # Get the absolute pathname of the file in the local cache. 114 | cache_file = self.cache.get(requirement) 115 | # Enable checksum based cache invalidation. 116 | self.persist_checksum(requirement, cache_file) 117 | archive = tarfile.open(cache_file, 'r:gz') 118 | try: 119 | for member in archive.getmembers(): 120 | yield member, archive.extractfile(member.name) 121 | finally: 122 | archive.close() 123 | 124 | def needs_invalidation(self, requirement, cache_file): 125 | """ 126 | Check whether a cached binary distribution needs to be invalidated. 127 | 128 | :param requirement: A :class:`.Requirement` object. 129 | :param cache_file: The pathname of a cached binary distribution (a string). 130 | :returns: :data:`True` if the cached binary distribution needs to be 131 | invalidated, :data:`False` otherwise. 132 | """ 133 | if self.config.trust_mod_times: 134 | return requirement.last_modified > os.path.getmtime(cache_file) 135 | else: 136 | checksum = self.recall_checksum(cache_file) 137 | return checksum and checksum != requirement.checksum 138 | 139 | def recall_checksum(self, cache_file): 140 | """ 141 | Get the checksum of the input used to generate a binary distribution archive. 142 | 143 | :param cache_file: The pathname of the binary distribution archive (a string). 144 | :returns: The checksum (a string) or :data:`None` (when no checksum is available). 145 | """ 146 | # EAFP instead of LBYL because of concurrency between pip-accel 147 | # processes (https://docs.python.org/2/glossary.html#term-lbyl). 148 | checksum_file = '%s.txt' % cache_file 149 | try: 150 | with open(checksum_file) as handle: 151 | contents = handle.read() 152 | return contents.strip() 153 | except IOError as e: 154 | if e.errno == errno.ENOENT: 155 | # Gracefully handle missing checksum files. 156 | return None 157 | else: 158 | # Don't swallow exceptions we don't expect! 159 | raise 160 | 161 | def persist_checksum(self, requirement, cache_file): 162 | """ 163 | Persist the checksum of the input used to generate a binary distribution. 164 | 165 | :param requirement: A :class:`.Requirement` object. 166 | :param cache_file: The pathname of a cached binary distribution (a string). 167 | 168 | .. note:: The checksum is only calculated and persisted when 169 | :attr:`~.Config.trust_mod_times` is :data:`False`. 170 | """ 171 | if not self.config.trust_mod_times: 172 | checksum_file = '%s.txt' % cache_file 173 | with AtomicReplace(checksum_file) as temporary_file: 174 | with open(temporary_file, 'w') as handle: 175 | handle.write('%s\n' % requirement.checksum) 176 | 177 | def build_binary_dist(self, requirement): 178 | """ 179 | Build a binary distribution archive from an unpacked source distribution. 180 | 181 | :param requirement: A :class:`.Requirement` object. 182 | :returns: The pathname of a binary distribution archive (a string). 183 | :raises: :exc:`.BinaryDistributionError` when the original command 184 | and the fall back both fail to produce a binary distribution 185 | archive. 186 | 187 | This method uses the following command to build binary distributions: 188 | 189 | .. code-block:: sh 190 | 191 | $ python setup.py bdist_dumb --format=tar 192 | 193 | This command can fail for two main reasons: 194 | 195 | 1. The package is missing binary dependencies. 196 | 2. The ``setup.py`` script doesn't (properly) implement ``bdist_dumb`` 197 | binary distribution format support. 198 | 199 | The first case is dealt with in :func:`get_binary_dist()`. To deal 200 | with the second case this method falls back to the following command: 201 | 202 | .. code-block:: sh 203 | 204 | $ python setup.py bdist 205 | 206 | This fall back is almost never needed, but there are Python packages 207 | out there which require this fall back (this method was added because 208 | the installation of ``Paver==1.2.3`` failed, see `issue 37`_ for 209 | details about that). 210 | 211 | .. _issue 37: https://github.com/paylogic/pip-accel/issues/37 212 | """ 213 | try: 214 | return self.build_binary_dist_helper(requirement, ['bdist_dumb', '--format=tar']) 215 | except (BuildFailed, NoBuildOutput): 216 | logger.warning("Build of %s failed, falling back to alternative method ..", requirement) 217 | return self.build_binary_dist_helper(requirement, ['bdist', '--formats=gztar']) 218 | 219 | def build_binary_dist_helper(self, requirement, setup_command): 220 | """ 221 | Convert an unpacked source distribution to a binary distribution. 222 | 223 | :param requirement: A :class:`.Requirement` object. 224 | :param setup_command: A list of strings with the arguments to 225 | ``setup.py``. 226 | :returns: The pathname of the resulting binary distribution (a string). 227 | :raises: :exc:`.BuildFailed` when the build reports an error (e.g. 228 | because of missing binary dependencies like system 229 | libraries). 230 | :raises: :exc:`.NoBuildOutput` when the build does not produce the 231 | expected binary distribution archive. 232 | """ 233 | build_timer = Timer() 234 | # Make sure the source distribution contains a setup script. 235 | setup_script = os.path.join(requirement.source_directory, 'setup.py') 236 | if not os.path.isfile(setup_script): 237 | msg = "Directory %s (%s %s) doesn't contain a source distribution!" 238 | raise InvalidSourceDistribution(msg % (requirement.source_directory, requirement.name, requirement.version)) 239 | # Let the user know what's going on. 240 | build_text = "Building %s binary distribution" % requirement 241 | logger.info("%s ..", build_text) 242 | # Cleanup previously generated distributions. 243 | dist_directory = os.path.join(requirement.source_directory, 'dist') 244 | if os.path.isdir(dist_directory): 245 | logger.debug("Cleaning up previously generated distributions in %s ..", dist_directory) 246 | shutil.rmtree(dist_directory) 247 | # Let the user know (approximately) which command is being executed 248 | # (I don't think it's necessary to show them the nasty details :-). 249 | logger.debug("Executing external command: %s", 250 | ' '.join(map(pipes.quote, [self.config.python_executable, 'setup.py'] + setup_command))) 251 | # Compose the command line needed to build the binary distribution. 252 | # This nasty command line forces the use of setuptools (instead of 253 | # distutils) just like pip does. This will cause the `*.egg-info' 254 | # metadata to be written to a directory instead of a file, which 255 | # (amongst other things) enables tracking of installed files. 256 | command_line = [ 257 | self.config.python_executable, '-c', 258 | ';'.join([ 259 | 'import setuptools', 260 | '__file__=%r' % setup_script, 261 | r"exec(compile(open(__file__).read().replace('\r\n', '\n'), __file__, 'exec'))", 262 | ]) 263 | ] + setup_command 264 | # Redirect all output of the build to a temporary file. 265 | fd, temporary_file = tempfile.mkstemp() 266 | try: 267 | # Start the build. 268 | build = subprocess.Popen(command_line, cwd=requirement.source_directory, stdout=fd, stderr=fd) 269 | # Wait for the build to finish and provide feedback to the user in the mean time. 270 | spinner = Spinner(label=build_text, timer=build_timer) 271 | while build.poll() is None: 272 | spinner.step() 273 | # Don't tax the CPU too much. 274 | time.sleep(0.2) 275 | spinner.clear() 276 | # Make sure the build succeeded and produced a binary distribution archive. 277 | try: 278 | # If the build reported an error we'll try to provide the user with 279 | # some hints about what went wrong. 280 | if build.returncode != 0: 281 | raise BuildFailed("Failed to build {name} ({version}) binary distribution!", 282 | name=requirement.name, version=requirement.version) 283 | # Check if the build created the `dist' directory (the os.listdir() 284 | # call below will raise an exception if we don't check for this). 285 | if not os.path.isdir(dist_directory): 286 | raise NoBuildOutput("Build of {name} ({version}) did not produce a binary distribution archive!", 287 | name=requirement.name, version=requirement.version) 288 | # Check if we can find the binary distribution archive. 289 | filenames = os.listdir(dist_directory) 290 | if len(filenames) != 1: 291 | variables = dict(name=requirement.name, 292 | version=requirement.version, 293 | filenames=concatenate(sorted(filenames))) 294 | raise NoBuildOutput(""" 295 | Build of {name} ({version}) produced more than one 296 | distribution archive! (matches: {filenames}) 297 | """, **variables) 298 | except Exception as e: 299 | # Decorate the exception with the output of the failed build. 300 | with open(temporary_file) as handle: 301 | build_output = handle.read() 302 | enhanced_message = compact(""" 303 | {message} 304 | 305 | Please check the build output because it will probably 306 | provide a hint about what went wrong. 307 | 308 | Build output: 309 | 310 | {output} 311 | """, message=e.args[0], output=build_output.strip()) 312 | e.args = (enhanced_message,) 313 | raise 314 | logger.info("Finished building %s in %s.", requirement.name, build_timer) 315 | return os.path.join(dist_directory, filenames[0]) 316 | finally: 317 | # Close file descriptor before removing the temporary file. 318 | # Without closing Windows is complaining that the file cannot 319 | # be removed because it is used by another process. 320 | os.close(fd) 321 | os.unlink(temporary_file) 322 | 323 | def transform_binary_dist(self, archive_path): 324 | """ 325 | Transform binary distributions into a form that can be cached for future use. 326 | 327 | :param archive_path: The pathname of the original binary distribution archive. 328 | :returns: An iterable of tuples with two values each: 329 | 330 | 1. A :class:`tarfile.TarInfo` object. 331 | 2. A file-like object. 332 | 333 | This method transforms a binary distribution archive created by 334 | :func:`build_binary_dist()` into a form that can be cached for future 335 | use. This comes down to making the pathnames inside the archive 336 | relative to the `prefix` that the binary distribution was built for. 337 | """ 338 | # Copy the tar archive file by file so we can rewrite the pathnames. 339 | logger.debug("Transforming binary distribution: %s.", archive_path) 340 | archive = tarfile.open(archive_path, 'r') 341 | for member in archive.getmembers(): 342 | # Some source distribution archives on PyPI that are distributed as ZIP 343 | # archives contain really weird permissions: the world readable bit is 344 | # missing. I've encountered this with the httplib2 (0.9) and 345 | # google-api-python-client (1.2) packages. I assume this is a bug of 346 | # some kind in the packaging process on "their" side. 347 | if member.mode & stat.S_IXUSR: 348 | # If the owner has execute permissions we'll give everyone read and 349 | # execute permissions (only the owner gets write permissions). 350 | member.mode = 0o755 351 | else: 352 | # If the owner doesn't have execute permissions we'll give everyone 353 | # read permissions (only the owner gets write permissions). 354 | member.mode = 0o644 355 | # In my testing the `dumb' tar files created with the `python 356 | # setup.py bdist' and `python setup.py bdist_dumb' commands contain 357 | # pathnames that are relative to `/' in one way or another: 358 | # 359 | # - In almost all cases the pathnames look like this: 360 | # 361 | # ./home/peter/.virtualenvs/pip-accel/lib/python2.7/site-packages/pip_accel/__init__.py 362 | # 363 | # - After working on pip-accel for several years I encountered 364 | # a pathname like this (Python 2.6 on Mac OS X 10.10.5): 365 | # 366 | # Users/peter/.virtualenvs/pip-accel/lib/python2.6/site-packages/pip_accel/__init__.py 367 | # 368 | # Both of the above pathnames are relative to `/' but in different 369 | # ways :-). The following normpath(join('/', ...))) pathname 370 | # manipulation logic is intended to handle both cases. 371 | original_pathname = member.name 372 | absolute_pathname = os.path.normpath(os.path.join('/', original_pathname)) 373 | if member.isdev(): 374 | logger.warn("Ignoring device file: %s.", absolute_pathname) 375 | elif not member.isdir(): 376 | modified_pathname = os.path.relpath(absolute_pathname, self.config.install_prefix) 377 | if os.path.isabs(modified_pathname): 378 | logger.warn("Failed to transform pathname in binary distribution" 379 | " to relative path! (original: %r, modified: %r)", 380 | original_pathname, modified_pathname) 381 | else: 382 | # Rewrite /usr/local to /usr (same goes for all prefixes of course). 383 | modified_pathname = re.sub('^local/', '', modified_pathname) 384 | # Rewrite /dist-packages/ to /site-packages/. For details see 385 | # https://wiki.debian.org/Python#Deviations_from_upstream. 386 | if self.config.on_debian: 387 | modified_pathname = modified_pathname.replace('/dist-packages/', '/site-packages/') 388 | # Enable operators to debug the transformation process. 389 | logger.debug("Transformed %r -> %r.", original_pathname, modified_pathname) 390 | # Get the file data from the input archive. 391 | handle = archive.extractfile(original_pathname) 392 | # Yield the modified metadata and a handle to the data. 393 | member.name = modified_pathname 394 | yield member, handle 395 | archive.close() 396 | 397 | def install_binary_dist(self, members, virtualenv_compatible=True, prefix=None, 398 | python=None, track_installed_files=False): 399 | """ 400 | Install a binary distribution into the given prefix. 401 | 402 | :param members: An iterable of tuples with two values each: 403 | 404 | 1. A :class:`tarfile.TarInfo` object. 405 | 2. A file-like object. 406 | :param prefix: The "prefix" under which the requirements should be 407 | installed. This will be a pathname like ``/usr``, 408 | ``/usr/local`` or the pathname of a virtual environment. 409 | Defaults to :attr:`.Config.install_prefix`. 410 | :param python: The pathname of the Python executable to use in the shebang 411 | line of all executable Python scripts inside the binary 412 | distribution. Defaults to :attr:`.Config.python_executable`. 413 | :param virtualenv_compatible: Whether to enable workarounds to make the 414 | resulting filenames compatible with 415 | virtual environments (defaults to 416 | :data:`True`). 417 | :param track_installed_files: If this is :data:`True` (not the default for 418 | this method because of backwards 419 | compatibility) pip-accel will create 420 | ``installed-files.txt`` as required by 421 | pip to properly uninstall packages. 422 | 423 | This method installs a binary distribution created by 424 | :class:`build_binary_dist()` into the given prefix (a directory like 425 | ``/usr``, ``/usr/local`` or a virtual environment). 426 | """ 427 | # TODO This is quite slow for modules like Django. Speed it up! Two choices: 428 | # 1. Run the external tar program to unpack the archive. This will 429 | # slightly complicate the fixing up of hashbangs. 430 | # 2. Using links? The plan: We can maintain a "seed" environment under 431 | # $PIP_ACCEL_CACHE and use symbolic and/or hard links to populate other 432 | # places based on the "seed" environment. 433 | module_search_path = set(map(os.path.normpath, sys.path)) 434 | prefix = os.path.normpath(prefix or self.config.install_prefix) 435 | python = os.path.normpath(python or self.config.python_executable) 436 | installed_files = [] 437 | for member, from_handle in members: 438 | pathname = member.name 439 | if virtualenv_compatible: 440 | # Some binary distributions include C header files (see for example 441 | # the greenlet package) however the subdirectory of include/ in a 442 | # virtual environment is a symbolic link to a subdirectory of 443 | # /usr/include/ so we should never try to install C header files 444 | # inside the directory pointed to by the symbolic link. Instead we 445 | # implement the same workaround that pip uses to avoid this 446 | # problem. 447 | pathname = re.sub('^include/', 'include/site/', pathname) 448 | if self.config.on_debian and '/site-packages/' in pathname: 449 | # On Debian based system wide Python installs the /site-packages/ 450 | # directory is not in Python's module search path while 451 | # /dist-packages/ is. We try to be compatible with this. 452 | match = re.match('^(.+?)/site-packages', pathname) 453 | if match: 454 | site_packages = os.path.normpath(os.path.join(prefix, match.group(0))) 455 | dist_packages = os.path.normpath(os.path.join(prefix, match.group(1), 'dist-packages')) 456 | if dist_packages in module_search_path and site_packages not in module_search_path: 457 | pathname = pathname.replace('/site-packages/', '/dist-packages/') 458 | pathname = os.path.join(prefix, pathname) 459 | if track_installed_files: 460 | # Track the installed file's absolute pathname. 461 | installed_files.append(pathname) 462 | directory = os.path.dirname(pathname) 463 | if not os.path.isdir(directory): 464 | logger.debug("Creating directory: %s ..", directory) 465 | makedirs(directory) 466 | logger.debug("Creating file: %s ..", pathname) 467 | with open(pathname, 'wb') as to_handle: 468 | contents = from_handle.read() 469 | if contents.startswith(b'#!/'): 470 | contents = self.fix_hashbang(contents, python) 471 | to_handle.write(contents) 472 | os.chmod(pathname, member.mode) 473 | if track_installed_files: 474 | self.update_installed_files(installed_files) 475 | 476 | def fix_hashbang(self, contents, python): 477 | """ 478 | Rewrite hashbangs_ to use the correct Python executable. 479 | 480 | :param contents: The contents of the script whose hashbang should be 481 | fixed (a string). 482 | :param python: The absolute pathname of the Python executable (a 483 | string). 484 | :returns: The modified contents of the script (a string). 485 | 486 | .. _hashbangs: http://en.wikipedia.org/wiki/Shebang_(Unix) 487 | """ 488 | lines = contents.splitlines() 489 | if lines: 490 | hashbang = lines[0] 491 | # Get the base name of the command in the hashbang. 492 | executable = os.path.basename(hashbang) 493 | # Deal with hashbangs like `#!/usr/bin/env python'. 494 | executable = re.sub(b'^env ', b'', executable) 495 | # Only rewrite hashbangs that actually involve Python. 496 | if re.match(b'^python(\\d+(\\.\\d+)*)?$', executable): 497 | lines[0] = b'#!' + python.encode('ascii') 498 | logger.debug("Rewriting hashbang %r to %r!", hashbang, lines[0]) 499 | contents = b'\n'.join(lines) 500 | return contents 501 | 502 | def update_installed_files(self, installed_files): 503 | """ 504 | Track the files installed by a package so pip knows how to remove the package. 505 | 506 | This method is used by :func:`install_binary_dist()` (which collects 507 | the list of installed files for :func:`update_installed_files()`). 508 | 509 | :param installed_files: A list of absolute pathnames (strings) with the 510 | files that were just installed. 511 | """ 512 | # Find the *.egg-info directory where installed-files.txt should be created. 513 | pkg_info_files = [fn for fn in installed_files if fnmatch.fnmatch(fn, '*.egg-info/PKG-INFO')] 514 | # I'm not (yet) sure how reliable the above logic is, so for now 515 | # I'll err on the side of caution and only act when the results 516 | # seem to be reliable. 517 | if len(pkg_info_files) != 1: 518 | logger.warning("Not tracking installed files (couldn't reliably determine *.egg-info directory)") 519 | else: 520 | egg_info_directory = os.path.dirname(pkg_info_files[0]) 521 | installed_files_path = os.path.join(egg_info_directory, 'installed-files.txt') 522 | logger.debug("Tracking installed files in %s ..", installed_files_path) 523 | with open(installed_files_path, 'w') as handle: 524 | for pathname in installed_files: 525 | handle.write('%s\n' % os.path.relpath(pathname, egg_info_directory)) 526 | -------------------------------------------------------------------------------- /pip_accel/caches/__init__.py: -------------------------------------------------------------------------------- 1 | # Accelerator for pip, the Python package manager. 2 | # 3 | # Author: Peter Odding 4 | # Last Change: October 31, 2015 5 | # URL: https://github.com/paylogic/pip-accel 6 | 7 | """ 8 | Support for multiple cache backends. 9 | 10 | This module defines an abstract base class (:class:`AbstractCacheBackend`) 11 | to be inherited by custom cache backends in order to easily integrate them in 12 | pip-accel. The cache backends included in pip-accel are built on top of the 13 | same mechanism. 14 | 15 | Additionally this module defines :class:`CacheManager` which makes it 16 | possible to merge the available cache backends into a single logical cache 17 | which automatically disables backends that report errors. 18 | """ 19 | 20 | # Standard library modules. 21 | import logging 22 | 23 | # Modules included in our package. 24 | from pip_accel.compat import WINDOWS 25 | from pip_accel.exceptions import CacheBackendDisabledError 26 | from pip_accel.utils import get_python_version 27 | 28 | # External dependencies. 29 | from humanfriendly import concatenate, pluralize 30 | from pkg_resources import iter_entry_points 31 | 32 | # Initialize a logger for this module. 33 | logger = logging.getLogger(__name__) 34 | 35 | # Initialize the registry of cache backends. 36 | registered_backends = set() 37 | 38 | # On Windows it is not allowed to have colons in filenames so we use a dollar sign instead. 39 | FILENAME_PATTERN = 'v%i\\%s$%s$%s.tar.gz' if WINDOWS else 'v%i/%s:%s:%s.tar.gz' 40 | 41 | 42 | class CacheBackendMeta(type): 43 | 44 | """Metaclass to intercept cache backend definitions.""" 45 | 46 | def __init__(cls, name, bases, dict): 47 | """Intercept cache backend definitions.""" 48 | type.__init__(cls, name, bases, dict) 49 | registered_backends.add(cls) 50 | 51 | 52 | class AbstractCacheBackend(object): 53 | 54 | """ 55 | Abstract base class for implementations of pip-accel cache backends. 56 | 57 | Subclasses of this class are used by pip-accel to store Python distribution 58 | archives in order to accelerate performance and gain independence of 59 | external systems like PyPI and distribution sites. 60 | 61 | .. note:: This base class automatically registers subclasses at definition 62 | time, providing a simple and elegant registration mechanism for 63 | custom backends. This technique uses metaclasses and was 64 | originally based on the article `Using Metaclasses to Create 65 | Self-Registering Plugins 66 | `_. 67 | 68 | I've since had to introduce some additional magic to make this 69 | mechanism compatible with both Python 2.x and Python 3.x because 70 | the syntax for metaclasses is very much incompatible and I refuse 71 | to write separate implementations for both :-). 72 | """ 73 | 74 | PRIORITY = 0 75 | 76 | def __init__(self, config): 77 | """ 78 | Initialize a cache backend. 79 | 80 | :param config: The pip-accel configuration (a :class:`.Config` 81 | object). 82 | """ 83 | self.config = config 84 | 85 | def get(self, filename): 86 | """ 87 | Get a previously cached distribution archive from the cache. 88 | 89 | :param filename: The expected filename of the distribution archive (a 90 | string). 91 | :returns: The absolute pathname of a local file or :data:`None` when the 92 | distribution archive hasn't been cached. 93 | 94 | This method is called by `pip-accel` before fetching or building a 95 | distribution archive, in order to check whether a previously cached 96 | distribution archive is available for re-use. 97 | """ 98 | raise NotImplementedError() 99 | 100 | def put(self, filename, handle): 101 | """ 102 | Store a newly built distribution archive in the cache. 103 | 104 | :param filename: The filename of the distribution archive (a string). 105 | :param handle: A file-like object that provides access to the 106 | distribution archive. 107 | 108 | This method is called by `pip-accel` after fetching or building a 109 | distribution archive, in order to cache the distribution archive. 110 | """ 111 | raise NotImplementedError() 112 | 113 | def __repr__(self): 114 | """Generate a textual representation of the cache backend.""" 115 | return self.__class__.__name__ 116 | 117 | 118 | # Obscure syntax gymnastics to define a class with a metaclass whose 119 | # definition is compatible with Python 2.x as well as Python 3.x. 120 | # See also: https://wiki.python.org/moin/PortingToPy3k/BilingualQuickRef#metaclasses 121 | AbstractCacheBackend = CacheBackendMeta('AbstractCacheBackend', 122 | AbstractCacheBackend.__bases__, 123 | dict(AbstractCacheBackend.__dict__)) 124 | 125 | 126 | class CacheManager(object): 127 | 128 | """ 129 | Interface to treat multiple cache backends as a single one. 130 | 131 | The cache manager automatically disables cache backends that raise 132 | exceptions on ``get()`` and ``put()`` operations. 133 | """ 134 | 135 | def __init__(self, config): 136 | """ 137 | Initialize a cache manager. 138 | 139 | Automatically initializes instances of all registered cache backends 140 | based on setuptools' support for entry points which makes it possible 141 | for external Python packages to register additional cache backends 142 | without any modifications to pip-accel. 143 | 144 | :param config: The pip-accel configuration (a :class:`.Config` 145 | object). 146 | """ 147 | self.config = config 148 | for entry_point in iter_entry_points('pip_accel.cache_backends'): 149 | logger.debug("Importing cache backend: %s", entry_point.module_name) 150 | __import__(entry_point.module_name) 151 | # Initialize instances of all registered cache backends (sorted by 152 | # priority so that e.g. the local file system is checked before S3). 153 | self.backends = sorted((b(self.config) for b in registered_backends if b != AbstractCacheBackend), 154 | key=lambda b: b.PRIORITY) 155 | logger.debug("Initialized %s: %s", 156 | pluralize(len(self.backends), "cache backend"), 157 | concatenate(map(repr, self.backends))) 158 | 159 | def get(self, requirement): 160 | """ 161 | Get a distribution archive from any of the available caches. 162 | 163 | :param requirement: A :class:`.Requirement` object. 164 | :returns: The absolute pathname of a local file or :data:`None` when the 165 | distribution archive is missing from all available caches. 166 | """ 167 | filename = self.generate_filename(requirement) 168 | for backend in list(self.backends): 169 | try: 170 | pathname = backend.get(filename) 171 | if pathname is not None: 172 | return pathname 173 | except CacheBackendDisabledError as e: 174 | logger.debug("Disabling %s because it requires configuration: %s", backend, e) 175 | self.backends.remove(backend) 176 | except Exception as e: 177 | logger.exception("Disabling %s because it failed: %s", backend, e) 178 | self.backends.remove(backend) 179 | 180 | def put(self, requirement, handle): 181 | """ 182 | Store a distribution archive in all of the available caches. 183 | 184 | :param requirement: A :class:`.Requirement` object. 185 | :param handle: A file-like object that provides access to the 186 | distribution archive. 187 | """ 188 | filename = self.generate_filename(requirement) 189 | for backend in list(self.backends): 190 | handle.seek(0) 191 | try: 192 | backend.put(filename, handle) 193 | except CacheBackendDisabledError as e: 194 | logger.debug("Disabling %s because it requires configuration: %s", backend, e) 195 | self.backends.remove(backend) 196 | except Exception as e: 197 | logger.exception("Disabling %s because it failed: %s", backend, e) 198 | self.backends.remove(backend) 199 | 200 | def generate_filename(self, requirement): 201 | """ 202 | Generate a distribution archive filename for a package. 203 | 204 | :param requirement: A :class:`.Requirement` object. 205 | :returns: The filename of the distribution archive (a string) 206 | including a single leading directory component to indicate 207 | the cache format revision. 208 | """ 209 | return FILENAME_PATTERN % (self.config.cache_format_revision, 210 | requirement.name, requirement.version, 211 | get_python_version()) 212 | -------------------------------------------------------------------------------- /pip_accel/caches/local.py: -------------------------------------------------------------------------------- 1 | # Accelerator for pip, the Python package manager. 2 | # 3 | # Author: Peter Odding 4 | # Last Change: October 31, 2015 5 | # URL: https://github.com/paylogic/pip-accel 6 | 7 | """ 8 | Local file system cache backend. 9 | 10 | This module implements the local cache backend which stores distribution 11 | archives on the local file system. This is a very simple cache backend, all it 12 | does is create directories and write local files. The only trick here is that 13 | new binary distribution archives are written to temporary files which are 14 | then moved into place atomically using :func:`os.rename()` to avoid partial 15 | reads caused by running multiple invocations of pip-accel at the same time 16 | (which happened in `issue 25`_). 17 | 18 | .. _issue 25: https://github.com/paylogic/pip-accel/issues/25 19 | """ 20 | 21 | # Standard library modules. 22 | import logging 23 | import os 24 | import shutil 25 | 26 | # Modules included in our package. 27 | from pip_accel.caches import AbstractCacheBackend 28 | from pip_accel.utils import AtomicReplace, makedirs 29 | 30 | # Initialize a logger for this module. 31 | logger = logging.getLogger(__name__) 32 | 33 | 34 | class LocalCacheBackend(AbstractCacheBackend): 35 | 36 | """The local cache backend stores Python distribution archives on the local file system.""" 37 | 38 | PRIORITY = 10 39 | 40 | def get(self, filename): 41 | """ 42 | Check if a distribution archive exists in the local cache. 43 | 44 | :param filename: The filename of the distribution archive (a string). 45 | :returns: The pathname of a distribution archive on the local file 46 | system or :data:`None`. 47 | """ 48 | pathname = os.path.join(self.config.binary_cache, filename) 49 | if os.path.isfile(pathname): 50 | logger.debug("Distribution archive exists in local cache (%s).", pathname) 51 | return pathname 52 | else: 53 | logger.debug("Distribution archive doesn't exist in local cache (%s).", pathname) 54 | return None 55 | 56 | def put(self, filename, handle): 57 | """ 58 | Store a distribution archive in the local cache. 59 | 60 | :param filename: The filename of the distribution archive (a string). 61 | :param handle: A file-like object that provides access to the 62 | distribution archive. 63 | """ 64 | file_in_cache = os.path.join(self.config.binary_cache, filename) 65 | logger.debug("Storing distribution archive in local cache: %s", file_in_cache) 66 | makedirs(os.path.dirname(file_in_cache)) 67 | # Stream the contents of the distribution archive to a temporary file 68 | # to avoid race conditions (e.g. partial reads) between multiple 69 | # processes that are using the local cache at the same time. 70 | with AtomicReplace(file_in_cache) as temporary_file: 71 | with open(temporary_file, 'wb') as temporary_file_handle: 72 | shutil.copyfileobj(handle, temporary_file_handle) 73 | logger.debug("Finished caching distribution archive in local cache.") 74 | -------------------------------------------------------------------------------- /pip_accel/caches/s3.py: -------------------------------------------------------------------------------- 1 | # Accelerator for pip, the Python package manager. 2 | # 3 | # Authors: 4 | # - Adam Feuer 5 | # - Peter Odding 6 | # Last Change: March 4, 2016 7 | # URL: https://github.com/paylogic/pip-accel 8 | # 9 | # A word of warning: Do *not* use the cached_property decorator here, because 10 | # it interacts badly with the metaclass magic performed by the base class. I 11 | # wasted about an hour trying to get it to work but it became more and more 12 | # apparent that it was never going to work the way I wanted it to :-) 13 | 14 | """ 15 | Amazon S3 cache backend. 16 | 17 | This module implements a cache backend that stores distribution archives in a 18 | user defined `Amazon S3 `_ bucket. To enable this 19 | backend you need to define the configuration option 20 | :attr:`~.Config.s3_cache_bucket` and configure your Amazon S3 API 21 | credentials (see the readme for details). 22 | 23 | Using S3 compatible storage services 24 | ------------------------------------ 25 | 26 | The Amazon S3 API has been implemented in several open source projects and 27 | dozens of online services. To use pip-accel with an S3 compatible storage 28 | service you can override the :attr:`~.Config.s3_cache_url` option. The 29 | pip-accel test suite actually uses this option to test the S3 cache backend by 30 | running FakeS3_ in the background and pointing pip-accel at the FakeS3 server. 31 | Below are some usage notes that may be relevant for people evaluating this 32 | option. 33 | 34 | **Secure connections** 35 | Boto_ has to be told whether to make a "secure" connection to the S3 API and 36 | pip-accel assumes the ``https://`` URL scheme implies a secure connection 37 | while the ``http://`` URL scheme implies a non-secure connection. 38 | 39 | **Calling formats** 40 | Boto_ has the concept of "calling formats" for the S3 API and to connect to 41 | the official Amazon S3 API pip-accel needs to specify the "sub-domain calling 42 | format" or the API calls will fail. When you specify a nonstandard S3 API URL 43 | pip-accel tells Boto to use the "ordinary calling format" instead. This 44 | differentiation will undoubtedly not be correct in all cases. If this is 45 | bothering you then feel free to open an issue on GitHub to make pip-accel more 46 | flexible in this regard. 47 | 48 | **Credentials** 49 | If you don't specify S3 API credentials and the connection attempt to S3 fails 50 | with "NoAuthHandlerFound: No handler was ready to authenticate" pip-accel will 51 | fall back to an anonymous connection attempt. If that fails as well the S3 52 | cache backend is disabled. It may be useful to note here that the pip-accel 53 | test suite uses FakeS3_ and the anonymous connection fall back works fine. 54 | 55 | A note about robustness 56 | ----------------------- 57 | 58 | The Amazon S3 cache backend implemented in :mod:`pip_accel.caches.s3` is 59 | specifically written to gracefully disable itself when it encounters known 60 | errors such as: 61 | 62 | - The configuration option :attr:`~.Config.s3_cache_bucket` is not set (i.e. 63 | the user hasn't configured the backend yet). 64 | 65 | - The :mod:`boto` package is not installed (i.e. the user ran ``pip install 66 | pip-accel`` instead of ``pip install 'pip-accel[s3]'``). 67 | 68 | - The connection to the S3 API can't be established (e.g. because API 69 | credentials haven't been correctly configured). 70 | 71 | - The connection to the configured S3 bucket can't be established (e.g. because 72 | the bucket doesn't exist or the configured credentials don't provide access to 73 | the bucket). 74 | 75 | Additionally :class:`~pip_accel.caches.CacheManager` automatically disables 76 | cache backends that raise exceptions on 77 | :class:`~pip_accel.caches.AbstractCacheBackend.get()` and 78 | :class:`~pip_accel.caches.AbstractCacheBackend.put()` operations. The end 79 | result is that when the S3 backend fails you will just revert to using the 80 | cache on the local file system. 81 | 82 | Optionally if you are using read only credentials you can disable 83 | :class:`~S3CacheBackend.put()` operations by setting the configuration 84 | option :attr:`~.Config.s3_cache_readonly`. 85 | 86 | ---- 87 | 88 | .. _FakeS3: https://github.com/jubos/fake-s3 89 | .. _Boto: https://github.com/boto/boto 90 | """ 91 | 92 | # Standard library modules. 93 | import logging 94 | import os 95 | 96 | # External dependencies. 97 | from humanfriendly import coerce_boolean, Timer 98 | 99 | # Modules included in our package. 100 | from pip_accel import PatchedAttribute 101 | from pip_accel.caches import AbstractCacheBackend 102 | from pip_accel.compat import PY3, urlparse 103 | from pip_accel.exceptions import CacheBackendDisabledError, CacheBackendError 104 | from pip_accel.utils import AtomicReplace, makedirs 105 | 106 | # Initialize a logger for this module. 107 | logger = logging.getLogger(__name__) 108 | 109 | # The name of the boto.config configuration section that controls general 110 | # settings like the number of retries and the HTTP socket timeout. 111 | BOTO_CONFIG_SECTION = 'Boto' 112 | 113 | # The name of the boto.config option that controls the number of retries. 114 | BOTO_CONFIG_NUM_RETRIES_OPTION = 'num_retries' 115 | 116 | # The name of the boto.config option that controls the HTTP socket timeout. 117 | BOTO_CONFIG_SOCKET_TIMEOUT_OPTION = 'http_socket_timeout' 118 | 119 | # The `coloredlogs' package installs a logging handler on the root logger which 120 | # means all loggers automatically write their log messages to the standard 121 | # error stream. In the case of Boto this is a bit confusing because Boto logs 122 | # messages with the ERROR severity even when nothing is wrong, because it 123 | # tries to connect to the Amazon EC2 metadata service which is (obviously) not 124 | # available outside of Amazon EC2: 125 | # 126 | # boto[6851] DEBUG Retrieving credentials from metadata server. 127 | # boto[6851] ERROR Caught exception reading instance data 128 | # 129 | # To avoid confusing users of pip-accel (i.e. this is not an error because it's 130 | # properly handled) we silence the Boto logger. To avoid annoying people who 131 | # actually want to debug Boto we'll also provide an escape hatch in the form of 132 | # an environment variable. 133 | if coerce_boolean(os.environ.get('PIP_ACCEL_SILENCE_BOTO', 'true')): 134 | logging.getLogger('boto').setLevel(logging.FATAL) 135 | 136 | 137 | class S3CacheBackend(AbstractCacheBackend): 138 | 139 | """The S3 cache backend stores distribution archives in a user defined Amazon S3 bucket.""" 140 | 141 | PRIORITY = 20 142 | 143 | def get(self, filename): 144 | """ 145 | Download a distribution archive from the configured Amazon S3 bucket. 146 | 147 | :param filename: The filename of the distribution archive (a string). 148 | :returns: The pathname of a distribution archive on the local file 149 | system or :data:`None`. 150 | :raises: :exc:`.CacheBackendError` when any underlying method fails. 151 | """ 152 | timer = Timer() 153 | self.check_prerequisites() 154 | with PatchedBotoConfig(): 155 | # Check if the distribution archive is available. 156 | raw_key = self.get_cache_key(filename) 157 | logger.info("Checking if distribution archive is available in S3 bucket: %s", raw_key) 158 | key = self.s3_bucket.get_key(raw_key) 159 | if key is None: 160 | logger.debug("Distribution archive is not available in S3 bucket.") 161 | else: 162 | # Download the distribution archive to the local binary index. 163 | # TODO Shouldn't this use LocalCacheBackend.put() instead of 164 | # implementing the same steps manually?! 165 | logger.info("Downloading distribution archive from S3 bucket ..") 166 | file_in_cache = os.path.join(self.config.binary_cache, filename) 167 | makedirs(os.path.dirname(file_in_cache)) 168 | with AtomicReplace(file_in_cache) as temporary_file: 169 | key.get_contents_to_filename(temporary_file) 170 | logger.debug("Finished downloading distribution archive from S3 bucket in %s.", timer) 171 | return file_in_cache 172 | 173 | def put(self, filename, handle): 174 | """ 175 | Upload a distribution archive to the configured Amazon S3 bucket. 176 | 177 | If the :attr:`~.Config.s3_cache_readonly` configuration option is 178 | enabled this method does nothing. 179 | 180 | :param filename: The filename of the distribution archive (a string). 181 | :param handle: A file-like object that provides access to the 182 | distribution archive. 183 | :raises: :exc:`.CacheBackendError` when any underlying method fails. 184 | """ 185 | if self.config.s3_cache_readonly: 186 | logger.info('Skipping upload to S3 bucket (using S3 in read only mode).') 187 | else: 188 | timer = Timer() 189 | self.check_prerequisites() 190 | with PatchedBotoConfig(): 191 | from boto.s3.key import Key 192 | raw_key = self.get_cache_key(filename) 193 | logger.info("Uploading distribution archive to S3 bucket: %s", raw_key) 194 | key = Key(self.s3_bucket) 195 | key.key = raw_key 196 | try: 197 | key.set_contents_from_file(handle) 198 | except Exception as e: 199 | logger.info("Encountered error writing to S3 bucket, " 200 | "falling back to read only mode (exception: %s)", e) 201 | self.config.s3_cache_readonly = True 202 | else: 203 | logger.info("Finished uploading distribution archive to S3 bucket in %s.", timer) 204 | 205 | @property 206 | def s3_bucket(self): 207 | """ 208 | Connect to the user defined Amazon S3 bucket. 209 | 210 | Called on demand by :func:`get()` and :func:`put()`. Caches its 211 | return value so that only a single connection is created. 212 | 213 | :returns: A :class:`boto.s3.bucket.Bucket` object. 214 | :raises: :exc:`.CacheBackendDisabledError` when the user hasn't 215 | defined :attr:`.Config.s3_cache_bucket`. 216 | :raises: :exc:`.CacheBackendError` when the connection to the Amazon 217 | S3 bucket fails. 218 | """ 219 | if not hasattr(self, 'cached_bucket'): 220 | self.check_prerequisites() 221 | with PatchedBotoConfig(): 222 | from boto.exception import BotoClientError, BotoServerError, S3ResponseError 223 | # The following try/except block translates unexpected exceptions 224 | # raised by Boto into a CacheBackendError exception. 225 | try: 226 | # The following try/except block handles the expected exception 227 | # raised by Boto when an Amazon S3 bucket does not exist. 228 | try: 229 | logger.debug("Connecting to Amazon S3 bucket: %s", self.config.s3_cache_bucket) 230 | self.cached_bucket = self.s3_connection.get_bucket(self.config.s3_cache_bucket) 231 | except S3ResponseError as e: 232 | if e.status == 404 and self.config.s3_cache_create_bucket: 233 | logger.info("Amazon S3 bucket doesn't exist yet, creating it now: %s", 234 | self.config.s3_cache_bucket) 235 | self.s3_connection.create_bucket(self.config.s3_cache_bucket) 236 | self.cached_bucket = self.s3_connection.get_bucket(self.config.s3_cache_bucket) 237 | else: 238 | # Don't swallow exceptions we can't handle. 239 | raise 240 | except (BotoClientError, BotoServerError): 241 | raise CacheBackendError(""" 242 | Failed to connect to the configured Amazon S3 bucket 243 | {bucket}! Are you sure the bucket exists and is accessible 244 | using the provided credentials? The Amazon S3 cache backend 245 | will be disabled for now. 246 | """, bucket=repr(self.config.s3_cache_bucket)) 247 | return self.cached_bucket 248 | 249 | @property 250 | def s3_connection(self): 251 | """ 252 | Connect to the Amazon S3 API. 253 | 254 | If the connection attempt fails because Boto can't find credentials the 255 | attempt is retried once with an anonymous connection. 256 | 257 | Called on demand by :attr:`s3_bucket`. 258 | 259 | :returns: A :class:`boto.s3.connection.S3Connection` object. 260 | :raises: :exc:`.CacheBackendError` when the connection to the Amazon 261 | S3 API fails. 262 | """ 263 | if not hasattr(self, 'cached_connection'): 264 | self.check_prerequisites() 265 | with PatchedBotoConfig(): 266 | import boto 267 | from boto.exception import BotoClientError, BotoServerError, NoAuthHandlerFound 268 | from boto.s3.connection import S3Connection, SubdomainCallingFormat, OrdinaryCallingFormat 269 | try: 270 | # Configure the number of retries and the socket timeout used 271 | # by Boto. Based on the snippet given in the following email: 272 | # https://groups.google.com/d/msg/boto-users/0osmP0cUl5Y/X4NdlMGWKiEJ 273 | if not boto.config.has_section(BOTO_CONFIG_SECTION): 274 | boto.config.add_section(BOTO_CONFIG_SECTION) 275 | boto.config.set(BOTO_CONFIG_SECTION, 276 | BOTO_CONFIG_NUM_RETRIES_OPTION, 277 | str(self.config.s3_cache_retries)) 278 | boto.config.set(BOTO_CONFIG_SECTION, 279 | BOTO_CONFIG_SOCKET_TIMEOUT_OPTION, 280 | str(self.config.s3_cache_timeout)) 281 | logger.debug("Connecting to Amazon S3 API ..") 282 | endpoint = urlparse(self.config.s3_cache_url) 283 | host, _, port = endpoint.netloc.partition(':') 284 | kw = dict( 285 | host=host, 286 | port=int(port) if port else None, 287 | is_secure=(endpoint.scheme == 'https'), 288 | calling_format=(SubdomainCallingFormat() if host == S3Connection.DefaultHost 289 | else OrdinaryCallingFormat()), 290 | ) 291 | try: 292 | self.cached_connection = S3Connection(**kw) 293 | except NoAuthHandlerFound: 294 | logger.debug("Amazon S3 API credentials missing, retrying with anonymous connection ..") 295 | self.cached_connection = S3Connection(anon=True, **kw) 296 | except (BotoClientError, BotoServerError): 297 | raise CacheBackendError(""" 298 | Failed to connect to the Amazon S3 API! Most likely your 299 | credentials are not correctly configured. The Amazon S3 300 | cache backend will be disabled for now. 301 | """) 302 | return self.cached_connection 303 | 304 | def get_cache_key(self, filename): 305 | """ 306 | Compose an S3 cache key based on :attr:`.Config.s3_cache_prefix` and the given filename. 307 | 308 | :param filename: The filename of the distribution archive (a string). 309 | :returns: The cache key for the given filename (a string). 310 | """ 311 | return '/'.join(filter(None, [self.config.s3_cache_prefix, filename])) 312 | 313 | def check_prerequisites(self): 314 | """ 315 | Validate the prerequisites required to use the Amazon S3 cache backend. 316 | 317 | Makes sure the Amazon S3 cache backend is configured 318 | (:attr:`.Config.s3_cache_bucket` is defined by the user) and 319 | :mod:`boto` is available for use. 320 | 321 | :raises: :exc:`.CacheBackendDisabledError` when a prerequisite fails. 322 | """ 323 | if not self.config.s3_cache_bucket: 324 | raise CacheBackendDisabledError(""" 325 | To use Amazon S3 as a cache you have to set the environment 326 | variable $PIP_ACCEL_S3_BUCKET and configure your Amazon S3 API 327 | credentials (see the documentation for details). 328 | """) 329 | try: 330 | __import__('boto') 331 | except ImportError: 332 | raise CacheBackendDisabledError(""" 333 | Boto is required to use Amazon S3 as a cache but it looks like 334 | Boto is not installed! You can resolve this issue by installing 335 | pip-accel using the command `pip install pip-accel[s3]'. The 336 | Amazon S3 cache backend will be disabled for now. 337 | """) 338 | 339 | 340 | class PatchedBotoConfig(PatchedAttribute): 341 | 342 | """ 343 | Monkey patch for Boto's configuration handling. 344 | 345 | Boto's configuration handling is kind of broken on Python 3 `as documented 346 | here `_. The :class:`PatchedBotoConfig` 347 | class implements a context manager that temporarily patches Boto to work 348 | around the bug. 349 | 350 | Without this monkey patch it is impossible to configure the number of 351 | retries on Python 3 which makes the pip-accel test suite horribly slow. 352 | """ 353 | 354 | def __init__(self): 355 | """Initialize a :class:`PatchedBotoConfig` object.""" 356 | from boto import config 357 | from boto.pyami.config import Config, ConfigParser 358 | self.instance = config 359 | self.unbound_method = ConfigParser.get 360 | super(PatchedBotoConfig, self).__init__( 361 | object=Config, 362 | attribute='get', 363 | value=self.get, 364 | enabled=PY3, 365 | ) 366 | 367 | def get(self, section, name, default=None, **kw): 368 | """Replacement for :func:`boto.pyami.config.Config.get()`.""" 369 | try: 370 | return self.unbound_method(self.instance, section, name, **kw) 371 | except Exception: 372 | return default 373 | -------------------------------------------------------------------------------- /pip_accel/cli.py: -------------------------------------------------------------------------------- 1 | # Accelerator for pip, the Python package manager. 2 | # 3 | # Author: Peter Odding 4 | # Last Change: November 14, 2015 5 | # URL: https://github.com/paylogic/pip-accel 6 | 7 | """Command line interface for the ``pip-accel`` program.""" 8 | 9 | # Standard library modules. 10 | import logging 11 | import os 12 | import sys 13 | import textwrap 14 | 15 | # Modules included in our package. 16 | from pip_accel import PipAccelerator 17 | from pip_accel.config import Config 18 | from pip_accel.exceptions import NothingToDoError 19 | from pip_accel.utils import match_option 20 | 21 | # External dependencies. 22 | import coloredlogs 23 | 24 | # Initialize a logger for this module. 25 | logger = logging.getLogger(__name__) 26 | 27 | 28 | def main(): 29 | """The command line interface for the ``pip-accel`` program.""" 30 | arguments = sys.argv[1:] 31 | # If no arguments are given, the help text of pip-accel is printed. 32 | if not arguments: 33 | usage() 34 | sys.exit(0) 35 | # If no install subcommand is given we pass the command line straight 36 | # to pip without any changes and exit immediately afterwards. 37 | if 'install' not in arguments: 38 | # This will not return. 39 | os.execvp('pip', ['pip'] + arguments) 40 | else: 41 | arguments = [arg for arg in arguments if arg != 'install'] 42 | config = Config() 43 | # Initialize logging output. 44 | coloredlogs.install( 45 | fmt=config.log_format, 46 | level=config.log_verbosity, 47 | ) 48 | # Adjust verbosity based on -v, -q, --verbose, --quiet options. 49 | for argument in list(arguments): 50 | if match_option(argument, '-v', '--verbose'): 51 | coloredlogs.increase_verbosity() 52 | elif match_option(argument, '-q', '--quiet'): 53 | coloredlogs.decrease_verbosity() 54 | # Perform the requested action(s). 55 | try: 56 | accelerator = PipAccelerator(config) 57 | accelerator.install_from_arguments(arguments) 58 | except NothingToDoError as e: 59 | # Don't print a traceback for this (it's not very user friendly) and 60 | # exit with status zero to stay compatible with pip. For more details 61 | # please refer to https://github.com/paylogic/pip-accel/issues/47. 62 | logger.warning("%s", e) 63 | sys.exit(0) 64 | except Exception: 65 | logger.exception("Caught unhandled exception!") 66 | sys.exit(1) 67 | 68 | 69 | def usage(): 70 | """Print a usage message to the terminal.""" 71 | print(textwrap.dedent(""" 72 | Usage: pip-accel [PIP_ARGS] 73 | 74 | The pip-accel program is a wrapper for pip, the Python package manager. It 75 | accelerates the usage of pip to initialize Python virtual environments given 76 | one or more requirements files. The pip-accel command supports all subcommands 77 | and options supported by pip, however the only added value is in the "pip 78 | install" subcommand. 79 | 80 | For more information please refer to the GitHub project page 81 | at https://github.com/paylogic/pip-accel 82 | """).strip()) 83 | -------------------------------------------------------------------------------- /pip_accel/compat.py: -------------------------------------------------------------------------------- 1 | # Accelerator for pip, the Python package manager. 2 | # 3 | # Author: Peter Odding 4 | # Last Change: March 4, 2016 5 | # URL: https://github.com/paylogic/pip-accel 6 | 7 | """Operating system detection and Python version compatibility.""" 8 | 9 | # Standard library modules. 10 | import sys 11 | 12 | # Inform static code analysis tools about our intention to expose the 13 | # following variables. This avoids 'imported but unused' warnings. 14 | __all__ = ( 15 | 'WINDOWS', 16 | 'StringIO', 17 | 'configparser', 18 | 'pathname2url', 19 | 'urljoin', 20 | 'urlparse', 21 | ) 22 | 23 | WINDOWS = sys.platform.startswith('win') 24 | """:data:`True` if running on Windows, :data:`False` otherwise.""" 25 | 26 | # Compatibility between Python 2 and 3. 27 | try: 28 | # Python 2. 29 | basestring = basestring 30 | import ConfigParser as configparser 31 | from StringIO import StringIO 32 | from urllib import pathname2url 33 | from urlparse import urljoin, urlparse 34 | PY3 = False 35 | except (ImportError, NameError): 36 | # Python 3. 37 | basestring = str 38 | import configparser 39 | from io import StringIO 40 | from urllib.parse import urljoin, urlparse 41 | from urllib.request import pathname2url 42 | PY3 = True 43 | -------------------------------------------------------------------------------- /pip_accel/config.py: -------------------------------------------------------------------------------- 1 | # Accelerator for pip, the Python package manager. 2 | # 3 | # Author: Peter Odding 4 | # Last Change: January 10, 2016 5 | # URL: https://github.com/paylogic/pip-accel 6 | 7 | """ 8 | Configuration handling for `pip-accel`. 9 | 10 | This module defines the :class:`Config` class which is used throughout the 11 | pip accelerator. At runtime an instance of :class:`Config` is created and 12 | passed down like this: 13 | 14 | .. digraph:: config_dependency_injection 15 | 16 | node [fontsize=10, shape=rect] 17 | 18 | PipAccelerator -> BinaryDistributionManager 19 | BinaryDistributionManager -> CacheManager 20 | CacheManager -> LocalCacheBackend 21 | CacheManager -> S3CacheBackend 22 | BinaryDistributionManager -> SystemPackageManager 23 | 24 | The :class:`.PipAccelerator` class receives its configuration object from 25 | its caller. Usually this will be :func:`.main()` but when pip-accel is used 26 | as a Python API the person embedding or extending pip-accel is responsible for 27 | providing the configuration object. This is intended as a form of `dependency 28 | injection`_ that enables non-default configurations to be injected into 29 | pip-accel. 30 | 31 | Support for runtime configuration 32 | --------------------------------- 33 | 34 | The properties of the :class:`Config` class can be set at runtime using 35 | regular attribute assignment syntax. This overrides the default values of the 36 | properties (whether based on environment variables, configuration files or hard 37 | coded defaults). 38 | 39 | Support for configuration files 40 | ------------------------------- 41 | 42 | You can use a configuration file to permanently configure certain options of 43 | pip-accel. If ``/etc/pip-accel.conf`` and/or ``~/.pip-accel/pip-accel.conf`` 44 | exist they are automatically loaded. You can also set the environment variable 45 | ``$PIP_ACCEL_CONFIG`` to load a configuration file in a non-default location. 46 | If all three files exist the system wide file is loaded first, then the user 47 | specific file is loaded and then the file set by the environment variable is 48 | loaded (duplicate settings are overridden by the configuration file that's 49 | loaded last). 50 | 51 | Here is an example of the available options: 52 | 53 | .. code-block:: ini 54 | 55 | [pip-accel] 56 | auto-install = yes 57 | max-retries = 3 58 | data-directory = ~/.pip-accel 59 | s3-bucket = my-shared-pip-accel-binary-cache 60 | s3-prefix = ubuntu-trusty-amd64 61 | s3-readonly = yes 62 | 63 | Note that the configuration options shown above are just examples, they are not 64 | meant to represent the configuration defaults. 65 | 66 | ---- 67 | 68 | .. _dependency injection: http://en.wikipedia.org/wiki/Dependency_injection 69 | """ 70 | 71 | # Standard library modules. 72 | import logging 73 | import os 74 | import os.path 75 | import sys 76 | 77 | # Modules included in our package. 78 | from pip_accel.compat import configparser 79 | from pip_accel.utils import is_root, expand_path 80 | 81 | # External dependencies. 82 | from coloredlogs import DEFAULT_LOG_FORMAT 83 | from cached_property import cached_property 84 | from humanfriendly import coerce_boolean, parse_path 85 | 86 | # Initialize a logger for this module. 87 | logger = logging.getLogger(__name__) 88 | 89 | # The locations of the user specific and system wide configuration files. 90 | LOCAL_CONFIG = '~/.pip-accel/pip-accel.conf' 91 | GLOBAL_CONFIG = '/etc/pip-accel.conf' 92 | 93 | 94 | class Config(object): 95 | 96 | """Configuration of the pip accelerator.""" 97 | 98 | def __init__(self, load_configuration_files=True, load_environment_variables=True): 99 | """ 100 | Initialize the configuration of the pip accelerator. 101 | 102 | :param load_configuration_files: If this is :data:`True` (the default) then 103 | configuration files in known locations 104 | are automatically loaded. 105 | :param load_environment_variables: If this is :data:`True` (the default) then 106 | environment variables are used to 107 | initialize the configuration. 108 | """ 109 | self.overrides = {} 110 | self.configuration = {} 111 | self.environment = os.environ if load_environment_variables else {} 112 | if load_configuration_files: 113 | for filename in self.available_configuration_files: 114 | self.load_configuration_file(filename) 115 | 116 | @cached_property 117 | def available_configuration_files(self): 118 | """A list of strings with the absolute pathnames of the available configuration files.""" 119 | known_files = [GLOBAL_CONFIG, LOCAL_CONFIG, self.environment.get('PIP_ACCEL_CONFIG')] 120 | absolute_paths = [parse_path(pathname) for pathname in known_files if pathname] 121 | return [pathname for pathname in absolute_paths if os.path.isfile(pathname)] 122 | 123 | def load_configuration_file(self, configuration_file): 124 | """ 125 | Load configuration defaults from a configuration file. 126 | 127 | :param configuration_file: The pathname of a configuration file (a 128 | string). 129 | :raises: :exc:`Exception` when the configuration file cannot be 130 | loaded. 131 | """ 132 | configuration_file = parse_path(configuration_file) 133 | logger.debug("Loading configuration file: %s", configuration_file) 134 | parser = configparser.RawConfigParser() 135 | files_loaded = parser.read(configuration_file) 136 | if len(files_loaded) != 1: 137 | msg = "Failed to load configuration file! (%s)" 138 | raise Exception(msg % configuration_file) 139 | elif not parser.has_section('pip-accel'): 140 | msg = "Missing 'pip-accel' section in configuration file! (%s)" 141 | raise Exception(msg % configuration_file) 142 | else: 143 | self.configuration.update(parser.items('pip-accel')) 144 | 145 | def __setattr__(self, name, value): 146 | """ 147 | Override the value of a property at runtime. 148 | 149 | :param name: The name of the property to override (a string). 150 | :param value: The overridden value of the property. 151 | """ 152 | attribute = getattr(self, name, None) 153 | if isinstance(attribute, (property, cached_property)): 154 | self.overrides[name] = value 155 | else: 156 | self.__dict__[name] = value 157 | 158 | def get(self, property_name=None, environment_variable=None, configuration_option=None, default=None): 159 | """ 160 | Internal shortcut to get a configuration option's value. 161 | 162 | :param property_name: The name of the property that users can set on 163 | the :class:`Config` class (a string). 164 | :param environment_variable: The name of the environment variable (a 165 | string). 166 | :param configuration_option: The name of the option in the 167 | configuration file (a string). 168 | :param default: The default value. 169 | :returns: The value of the environment variable or configuration file 170 | option or the default value. 171 | """ 172 | if self.overrides.get(property_name) is not None: 173 | return self.overrides[property_name] 174 | elif environment_variable and self.environment.get(environment_variable): 175 | return self.environment[environment_variable] 176 | elif self.configuration.get(configuration_option) is not None: 177 | return self.configuration[configuration_option] 178 | else: 179 | return default 180 | 181 | @cached_property 182 | def cache_format_revision(self): 183 | """ 184 | The revision of the binary distribution cache format in use (an integer). 185 | 186 | This number is encoded in the directory name of the binary cache so 187 | that multiple revisions can peacefully coexist. When pip-accel breaks 188 | backwards compatibility this number is bumped so that pip-accel starts 189 | using a new directory. 190 | """ 191 | return 7 192 | 193 | @cached_property 194 | def source_index(self): 195 | """ 196 | The absolute pathname of pip-accel's source index directory (a string). 197 | 198 | This is the ``sources`` subdirectory of :data:`data_directory`. 199 | """ 200 | return self.get(property_name='source_index', 201 | default=os.path.join(self.data_directory, 'sources')) 202 | 203 | @cached_property 204 | def binary_cache(self): 205 | """ 206 | The absolute pathname of pip-accel's binary cache directory (a string). 207 | 208 | This is the ``binaries`` subdirectory of :data:`data_directory`. 209 | """ 210 | return self.get(property_name='binary_cache', 211 | default=os.path.join(self.data_directory, 'binaries')) 212 | 213 | @cached_property 214 | def eggs_cache(self): 215 | """ 216 | The absolute pathname of pip-accel's eggs cache directory (a string). 217 | 218 | This is the ``eggs`` subdirectory of :data:`data_directory`. It is used 219 | to cache setup requirements which avoids continuous rebuilding of setup 220 | requirements. 221 | """ 222 | return self.get(property_name='eggs_cache', 223 | default=os.path.join(self.data_directory, 'eggs')) 224 | 225 | @cached_property 226 | def data_directory(self): 227 | """ 228 | The absolute pathname of the directory where pip-accel's data files are stored (a string). 229 | 230 | - Environment variable: ``$PIP_ACCEL_CACHE`` 231 | - Configuration option: ``data-directory`` 232 | - Default: ``/var/cache/pip-accel`` if running as ``root``, ``~/.pip-accel`` otherwise 233 | """ 234 | return expand_path(self.get(property_name='data_directory', 235 | environment_variable='PIP_ACCEL_CACHE', 236 | configuration_option='data-directory', 237 | default='/var/cache/pip-accel' if is_root() else '~/.pip-accel')) 238 | 239 | @cached_property 240 | def on_debian(self): 241 | """:data:`True` if running on a Debian derived system, :data:`False` otherwise.""" 242 | return self.get(property_name='on_debian', 243 | default=os.path.exists('/etc/debian_version')) 244 | 245 | @cached_property 246 | def install_prefix(self): 247 | """ 248 | The absolute pathname of the installation prefix to use (a string). 249 | 250 | This property is based on :data:`sys.prefix` except that when 251 | :data:`sys.prefix` is ``/usr`` and we're running on a Debian derived 252 | system ``/usr/local`` is used instead. 253 | 254 | The reason for this is that on Debian derived systems only apt (dpkg) 255 | should be allowed to touch files in ``/usr/lib/pythonX.Y/dist-packages`` 256 | and ``python setup.py install`` knows this (see the ``posix_local`` 257 | installation scheme in ``/usr/lib/pythonX.Y/sysconfig.py`` on Debian 258 | derived systems). Because pip-accel replaces ``python setup.py 259 | install`` it has to replicate this logic. Inferring all of this from 260 | the :mod:`sysconfig` module would be nice but that module wasn't 261 | available in Python 2.6. 262 | """ 263 | return self.get(property_name='install_prefix', 264 | default='/usr/local' if sys.prefix == '/usr' and self.on_debian else sys.prefix) 265 | 266 | @cached_property 267 | def python_executable(self): 268 | """The absolute pathname of the Python executable (a string).""" 269 | return self.get(property_name='python_executable', 270 | default=sys.executable or os.path.join(self.install_prefix, 'bin', 'python')) 271 | 272 | @cached_property 273 | def auto_install(self): 274 | """ 275 | Whether automatic installation of missing system packages is enabled. 276 | 277 | :data:`True` if automatic installation of missing system packages is 278 | enabled, :data:`False` if it is disabled, :data:`None` otherwise (in this case 279 | the user will be prompted at the appropriate time). 280 | 281 | - Environment variable: ``$PIP_ACCEL_AUTO_INSTALL`` (refer to 282 | :func:`~humanfriendly.coerce_boolean()` for details on how the 283 | value of the environment variable is interpreted) 284 | - Configuration option: ``auto-install`` (also parsed using 285 | :func:`~humanfriendly.coerce_boolean()`) 286 | - Default: :data:`None` 287 | """ 288 | value = self.get(property_name='auto_install', 289 | environment_variable='PIP_ACCEL_AUTO_INSTALL', 290 | configuration_option='auto-install') 291 | if value is not None: 292 | return coerce_boolean(value) 293 | 294 | @cached_property 295 | def log_format(self): 296 | """ 297 | The format of log messages written to the terminal. 298 | 299 | - Environment variable: ``$PIP_ACCEL_LOG_FORMAT`` 300 | - Configuration option: ``log-format`` 301 | - Default: :data:`~coloredlogs.DEFAULT_LOG_FORMAT` 302 | """ 303 | return self.get(property_name='log_format', 304 | environment_variable='PIP_ACCEL_LOG_FORMAT', 305 | configuration_option='log-format', 306 | default=DEFAULT_LOG_FORMAT) 307 | 308 | @cached_property 309 | def log_verbosity(self): 310 | """ 311 | The verbosity of log messages written to the terminal. 312 | 313 | - Environment variable: ``$PIP_ACCEL_LOG_VERBOSITY`` 314 | - Configuration option: ``log-verbosity`` 315 | - Default: 'INFO' (a string). 316 | """ 317 | return self.get(property_name='log_verbosity', 318 | environment_variable='PIP_ACCEL_LOG_VERBOSITY', 319 | configuration_option='log-verbosity', 320 | default='INFO') 321 | 322 | @cached_property 323 | def max_retries(self): 324 | """ 325 | The number of times to retry ``pip install --download`` if it fails. 326 | 327 | - Environment variable: ``$PIP_ACCEL_MAX_RETRIES`` 328 | - Configuration option: ``max-retries`` 329 | - Default: ``3`` 330 | """ 331 | value = self.get(property_name='max_retries', 332 | environment_variable='PIP_ACCEL_MAX_RETRIES', 333 | configuration_option='max-retries') 334 | try: 335 | n = int(value) 336 | if n >= 0: 337 | return n 338 | except: 339 | return 3 340 | 341 | @cached_property 342 | def trust_mod_times(self): 343 | """ 344 | Whether to trust file modification times for cache invalidation. 345 | 346 | - Environment variable: ``$PIP_ACCEL_TRUST_MOD_TIMES`` 347 | - Configuration option: ``trust-mod-times`` 348 | - Default: :data:`True` unless the AppVeyor_ continuous integration 349 | environment is detected (see `issue 62`_). 350 | 351 | .. _AppVeyor: http://www.appveyor.com 352 | .. _issue 62: https://github.com/paylogic/pip-accel/issues/62 353 | """ 354 | on_appveyor = coerce_boolean(os.environ.get('APPVEYOR', 'False')) 355 | return coerce_boolean(self.get(property_name='trust_mod_times', 356 | environment_variable='PIP_ACCEL_TRUST_MOD_TIMES', 357 | configuration_option='trust-mod-times', 358 | default=(not on_appveyor))) 359 | 360 | @cached_property 361 | def s3_cache_url(self): 362 | """ 363 | The URL of the Amazon S3 API endpoint to use. 364 | 365 | By default this points to the official Amazon S3 API endpoint. You can 366 | change this option if you're running a local Amazon S3 compatible 367 | storage service that you want pip-accel to use. 368 | 369 | - Environment variable: ``$PIP_ACCEL_S3_URL`` 370 | - Configuration option: ``s3-url`` 371 | - Default: ``https://s3.amazonaws.com`` 372 | 373 | For details please refer to the :mod:`pip_accel.caches.s3` module. 374 | """ 375 | return self.get(property_name='s3_cache_url', 376 | environment_variable='PIP_ACCEL_S3_URL', 377 | configuration_option='s3-url', 378 | default='https://s3.amazonaws.com') 379 | 380 | @cached_property 381 | def s3_cache_bucket(self): 382 | """ 383 | Name of Amazon S3 bucket where binary distributions are cached (a string or :data:`None`). 384 | 385 | - Environment variable: ``$PIP_ACCEL_S3_BUCKET`` 386 | - Configuration option: ``s3-bucket`` 387 | - Default: :data:`None` 388 | 389 | For details please refer to the :mod:`pip_accel.caches.s3` module. 390 | """ 391 | return self.get(property_name='s3_cache_bucket', 392 | environment_variable='PIP_ACCEL_S3_BUCKET', 393 | configuration_option='s3-bucket') 394 | 395 | @cached_property 396 | def s3_cache_create_bucket(self): 397 | """ 398 | Whether to automatically create the Amazon S3 bucket when it's missing. 399 | 400 | - Environment variable: ``$PIP_ACCEL_S3_CREATE_BUCKET`` 401 | - Configuration option: ``s3-create-bucket`` 402 | - Default: :data:`False` 403 | 404 | For details please refer to the :mod:`pip_accel.caches.s3` module. 405 | """ 406 | return coerce_boolean(self.get(property_name='s3_cache_create_bucket', 407 | environment_variable='PIP_ACCEL_S3_CREATE_BUCKET', 408 | configuration_option='s3-create-bucket', 409 | default=False)) 410 | 411 | @cached_property 412 | def s3_cache_prefix(self): 413 | """ 414 | Cache prefix for binary distribution archives in Amazon S3 bucket (a string or :data:`None`). 415 | 416 | - Environment variable: ``$PIP_ACCEL_S3_PREFIX`` 417 | - Configuration option: ``s3-prefix`` 418 | - Default: :data:`None` 419 | 420 | For details please refer to the :mod:`pip_accel.caches.s3` module. 421 | """ 422 | return self.get(property_name='s3_cache_prefix', 423 | environment_variable='PIP_ACCEL_S3_PREFIX', 424 | configuration_option='s3-prefix') 425 | 426 | @cached_property 427 | def s3_cache_readonly(self): 428 | """ 429 | Whether the Amazon S3 bucket is considered read only. 430 | 431 | If this is :data:`True` then the Amazon S3 bucket will only be used for 432 | :class:`~pip_accel.caches.s3.S3CacheBackend.get()` operations (all 433 | :class:`~pip_accel.caches.s3.S3CacheBackend.put()` operations will 434 | be disabled). 435 | 436 | - Environment variable: ``$PIP_ACCEL_S3_READONLY`` (refer to 437 | :func:`~humanfriendly.coerce_boolean()` for details on how the 438 | value of the environment variable is interpreted) 439 | - Configuration option: ``s3-readonly`` (also parsed using 440 | :func:`~humanfriendly.coerce_boolean()`) 441 | - Default: :data:`False` 442 | 443 | For details please refer to the :mod:`pip_accel.caches.s3` module. 444 | """ 445 | return coerce_boolean(self.get(property_name='s3_cache_readonly', 446 | environment_variable='PIP_ACCEL_S3_READONLY', 447 | configuration_option='s3-readonly', 448 | default=False)) 449 | 450 | @cached_property 451 | def s3_cache_timeout(self): 452 | """ 453 | The socket timeout in seconds for connections to Amazon S3 (an integer). 454 | 455 | This value is injected into Boto's configuration to override the 456 | default socket timeout used for connections to Amazon S3. 457 | 458 | - Environment variable: ``$PIP_ACCEL_S3_TIMEOUT`` 459 | - Configuration option: ``s3-timeout`` 460 | - Default: ``60`` (`Boto's default`_) 461 | 462 | .. _Boto's default: http://boto.readthedocs.org/en/latest/boto_config_tut.html 463 | """ 464 | value = self.get(property_name='s3_cache_timeout', 465 | environment_variable='PIP_ACCEL_S3_TIMEOUT', 466 | configuration_option='s3-timeout') 467 | try: 468 | n = int(value) 469 | if n >= 0: 470 | return n 471 | except: 472 | return 60 473 | 474 | @cached_property 475 | def s3_cache_retries(self): 476 | """ 477 | The number of times to retry failed requests to Amazon S3 (an integer). 478 | 479 | This value is injected into Boto's configuration to override the 480 | default number of times to retry failed requests to Amazon S3. 481 | 482 | - Environment variable: ``$PIP_ACCEL_S3_RETRIES`` 483 | - Configuration option: ``s3-retries`` 484 | - Default: ``5`` (`Boto's default`_) 485 | """ 486 | value = self.get(property_name='s3_cache_retries', 487 | environment_variable='PIP_ACCEL_S3_RETRIES', 488 | configuration_option='s3-retries') 489 | try: 490 | n = int(value) 491 | if n >= 0: 492 | return n 493 | except: 494 | return 5 495 | -------------------------------------------------------------------------------- /pip_accel/deps/__init__.py: -------------------------------------------------------------------------------- 1 | # Extension of pip-accel that deals with dependencies on system packages. 2 | # 3 | # Author: Peter Odding 4 | # Last Change: October 31, 2015 5 | # URL: https://github.com/paylogic/pip-accel 6 | 7 | """ 8 | System package dependency handling. 9 | 10 | The :mod:`pip_accel.deps` module is an extension of pip-accel that deals 11 | with dependencies on system packages. Currently only Debian Linux and 12 | derivative Linux distributions are supported by this extension but it should be 13 | fairly easy to add support for other platforms. 14 | 15 | The interface between pip-accel and :class:`SystemPackageManager` focuses on 16 | :func:`~SystemPackageManager.install_dependencies()` (the other methods are 17 | used internally). 18 | """ 19 | 20 | # Standard library modules. 21 | import logging 22 | import os 23 | import shlex 24 | import subprocess 25 | 26 | # Modules included in our package. 27 | from pip_accel.compat import WINDOWS, configparser 28 | from pip_accel.exceptions import DependencyInstallationFailed, DependencyInstallationRefused, SystemDependencyError 29 | from pip_accel.utils import is_root 30 | 31 | # External dependencies. 32 | from humanfriendly import Timer, concatenate, format, pluralize 33 | from humanfriendly.prompts import prompt_for_confirmation 34 | 35 | # Initialize a logger for this module. 36 | logger = logging.getLogger(__name__) 37 | 38 | 39 | class SystemPackageManager(object): 40 | 41 | """Interface to the system's package manager.""" 42 | 43 | def __init__(self, config): 44 | """ 45 | Initialize the system package dependency manager. 46 | 47 | :param config: The pip-accel configuration (a :class:`.Config` 48 | object). 49 | """ 50 | # Defaults for unsupported systems. 51 | self.list_command = 'true' 52 | self.install_command = 'true' 53 | self.dependencies = {} 54 | # Keep a reference to the pip-accel configuration. 55 | self.config = config 56 | # Initialize the platform specific package manager interface. 57 | directory = os.path.dirname(os.path.abspath(__file__)) 58 | for filename in sorted(os.listdir(directory)): 59 | pathname = os.path.join(directory, filename) 60 | if filename.endswith('.ini') and os.path.isfile(pathname): 61 | logger.debug("Loading configuration from %s ..", pathname) 62 | parser = configparser.RawConfigParser() 63 | parser.read(pathname) 64 | # Check if the package manager is supported. 65 | supported_command = parser.get('commands', 'supported') 66 | logger.debug("Checking if configuration is supported: %s", supported_command) 67 | with open(os.devnull, 'wb') as null_device: 68 | if subprocess.call(supported_command, shell=True, 69 | stdout=null_device, 70 | stderr=subprocess.STDOUT) == 0: 71 | logger.debug("System package manager configuration is supported!") 72 | # Get the commands to list and install system packages. 73 | self.list_command = parser.get('commands', 'list') 74 | self.install_command = parser.get('commands', 'install') 75 | # Get the known dependencies. 76 | self.dependencies = dict((n.lower(), v.split()) for n, v 77 | in parser.items('dependencies')) 78 | logger.debug("Loaded dependencies of %s: %s", 79 | pluralize(len(self.dependencies), "Python package"), 80 | concatenate(sorted(self.dependencies))) 81 | else: 82 | logger.debug("Command failed, assuming configuration doesn't apply ..") 83 | 84 | def install_dependencies(self, requirement): 85 | """ 86 | Install missing dependencies for the given requirement. 87 | 88 | :param requirement: A :class:`.Requirement` object. 89 | :returns: :data:`True` when missing system packages were installed, 90 | :data:`False` otherwise. 91 | :raises: :exc:`.DependencyInstallationRefused` when automatic 92 | installation is disabled or refused by the operator. 93 | :raises: :exc:`.DependencyInstallationFailed` when the installation 94 | of missing system packages fails. 95 | 96 | If `pip-accel` fails to build a binary distribution, it will call this 97 | method as a last chance to install missing dependencies. If this 98 | function does not raise an exception, `pip-accel` will retry the build 99 | once. 100 | """ 101 | install_timer = Timer() 102 | missing_dependencies = self.find_missing_dependencies(requirement) 103 | if missing_dependencies: 104 | # Compose the command line for the install command. 105 | install_command = shlex.split(self.install_command) + missing_dependencies 106 | # Prepend `sudo' to the command line? 107 | if not WINDOWS and not is_root(): 108 | # FIXME Ideally this should properly detect the presence of `sudo'. 109 | # Or maybe this should just be embedded in the *.ini files? 110 | install_command.insert(0, 'sudo') 111 | # Always suggest the installation command to the operator. 112 | logger.info("You seem to be missing %s: %s", 113 | pluralize(len(missing_dependencies), "dependency", "dependencies"), 114 | concatenate(missing_dependencies)) 115 | logger.info("You can install %s with this command: %s", 116 | "it" if len(missing_dependencies) == 1 else "them", " ".join(install_command)) 117 | if self.config.auto_install is False: 118 | # Refuse automatic installation and don't prompt the operator when the configuration says no. 119 | self.installation_refused(requirement, missing_dependencies, "automatic installation is disabled") 120 | # Get the operator's permission to install the missing package(s). 121 | if self.config.auto_install: 122 | logger.info("Got permission to install %s (via auto_install option).", 123 | pluralize(len(missing_dependencies), "dependency", "dependencies")) 124 | elif self.confirm_installation(requirement, missing_dependencies, install_command): 125 | logger.info("Got permission to install %s (via interactive prompt).", 126 | pluralize(len(missing_dependencies), "dependency", "dependencies")) 127 | else: 128 | logger.error("Refused installation of missing %s!", 129 | "dependency" if len(missing_dependencies) == 1 else "dependencies") 130 | self.installation_refused(requirement, missing_dependencies, "manual installation was refused") 131 | if subprocess.call(install_command) == 0: 132 | logger.info("Successfully installed %s in %s.", 133 | pluralize(len(missing_dependencies), "dependency", "dependencies"), 134 | install_timer) 135 | return True 136 | else: 137 | logger.error("Failed to install %s.", 138 | pluralize(len(missing_dependencies), "dependency", "dependencies")) 139 | msg = "Failed to install %s required by Python package %s! (%s)" 140 | raise DependencyInstallationFailed(msg % (pluralize(len(missing_dependencies), 141 | "system package", "system packages"), 142 | requirement.name, 143 | concatenate(missing_dependencies))) 144 | return False 145 | 146 | def find_missing_dependencies(self, requirement): 147 | """ 148 | Find missing dependencies of a Python package. 149 | 150 | :param requirement: A :class:`.Requirement` object. 151 | :returns: A list of strings with system package names. 152 | """ 153 | known_dependencies = self.find_known_dependencies(requirement) 154 | if known_dependencies: 155 | installed_packages = self.find_installed_packages() 156 | logger.debug("Checking for missing dependencies of %s ..", requirement.name) 157 | missing_dependencies = sorted(set(known_dependencies).difference(installed_packages)) 158 | if missing_dependencies: 159 | logger.debug("Found %s: %s", 160 | pluralize(len(missing_dependencies), "missing dependency", "missing dependencies"), 161 | concatenate(missing_dependencies)) 162 | else: 163 | logger.info("All known dependencies are already installed.") 164 | return missing_dependencies 165 | 166 | def find_known_dependencies(self, requirement): 167 | """ 168 | Find the known dependencies of a Python package. 169 | 170 | :param requirement: A :class:`.Requirement` object. 171 | :returns: A list of strings with system package names. 172 | """ 173 | logger.info("Checking for known dependencies of %s ..", requirement.name) 174 | known_dependencies = sorted(self.dependencies.get(requirement.name.lower(), [])) 175 | if known_dependencies: 176 | logger.info("Found %s: %s", pluralize(len(known_dependencies), "known dependency", "known dependencies"), 177 | concatenate(known_dependencies)) 178 | else: 179 | logger.info("No known dependencies... Maybe you have a suggestion?") 180 | return known_dependencies 181 | 182 | def find_installed_packages(self): 183 | """ 184 | Find the installed system packages. 185 | 186 | :returns: A list of strings with system package names. 187 | :raises: :exc:`.SystemDependencyError` when the command to list the 188 | installed system packages fails. 189 | """ 190 | list_command = subprocess.Popen(self.list_command, shell=True, stdout=subprocess.PIPE) 191 | stdout, stderr = list_command.communicate() 192 | if list_command.returncode != 0: 193 | raise SystemDependencyError("The command to list the installed system packages failed! ({command})", 194 | command=self.list_command) 195 | installed_packages = sorted(stdout.decode().split()) 196 | logger.debug("Found %i installed system package(s): %s", len(installed_packages), installed_packages) 197 | return installed_packages 198 | 199 | def installation_refused(self, requirement, missing_dependencies, reason): 200 | """ 201 | Raise :exc:`.DependencyInstallationRefused` with a user friendly message. 202 | 203 | :param requirement: A :class:`.Requirement` object. 204 | :param missing_dependencies: A list of strings with missing dependencies. 205 | :param reason: The reason why installation was refused (a string). 206 | """ 207 | msg = "Missing %s (%s) required by Python package %s (%s) but %s!" 208 | raise DependencyInstallationRefused( 209 | msg % (pluralize(len(missing_dependencies), "system package", "system packages"), 210 | concatenate(missing_dependencies), 211 | requirement.name, 212 | requirement.version, 213 | reason) 214 | ) 215 | 216 | def confirm_installation(self, requirement, missing_dependencies, install_command): 217 | """ 218 | Ask the operator's permission to install missing system packages. 219 | 220 | :param requirement: A :class:`.Requirement` object. 221 | :param missing_dependencies: A list of strings with missing dependencies. 222 | :param install_command: A list of strings with the command line needed 223 | to install the missing dependencies. 224 | :raises: :exc:`.DependencyInstallationRefused` when the operator refuses. 225 | """ 226 | try: 227 | return prompt_for_confirmation(format( 228 | "Do you want me to install %s %s?", 229 | "this" if len(missing_dependencies) == 1 else "these", 230 | "dependency" if len(missing_dependencies) == 1 else "dependencies", 231 | ), default=True) 232 | except KeyboardInterrupt: 233 | # Control-C is a negative response but doesn't 234 | # otherwise interrupt the program flow. 235 | return False 236 | -------------------------------------------------------------------------------- /pip_accel/deps/debian.ini: -------------------------------------------------------------------------------- 1 | ; Dependencies of known Python packages on Debian system packages. 2 | ; 3 | ; Author: Peter Odding 4 | ; Last Change: September 22, 2015 5 | ; URL: https://github.com/paylogic/pip-accel 6 | ; 7 | ; This configuration file defines dependencies of Python packages on Debian 8 | ; system packages. The left side of each line is the name of the package as 9 | ; used on the Python Package Index (these names are case insensitive just like 10 | ; PyPI and pip). The right side is a space separated list of Debian system 11 | ; package names. 12 | 13 | [commands] 14 | supported = test -e /etc/debian_version 15 | list = dpkg -l | awk '/^ii/ {print $2}' 16 | install = apt-get install --yes 17 | 18 | [dependencies] 19 | cffi = libffi-dev python-dev 20 | cryptography = libssl-dev python-dev 21 | lxml = libxml2-dev libxslt1-dev python-dev 22 | M2Crypto = libssl-dev python-dev swig 23 | Mercurial = python-dev 24 | MySQL-python = libmysqlclient-dev python-dev 25 | Pillow = libexif-dev libfreetype6-dev libjpeg-dev liblcms1-dev libtiff4-dev zlib1g-dev python-dev 26 | python-mcrypt = libmcrypt-dev python-dev 27 | PyXMLSec = libxmlsec1-dev 28 | PyYAML = libyaml-dev python-dev 29 | scipy = gfortran libatlas-base-dev 30 | -------------------------------------------------------------------------------- /pip_accel/exceptions.py: -------------------------------------------------------------------------------- 1 | # Accelerator for pip, the Python package manager. 2 | # 3 | # Author: Peter Odding 4 | # Last Change: October 31, 2015 5 | # URL: https://github.com/paylogic/pip-accel 6 | 7 | """ 8 | Exceptions for structured error handling. 9 | 10 | This module defines named exceptions raised by pip-accel when it encounters 11 | error conditions that: 12 | 13 | 1. Already require structured handling inside pip-accel 14 | 2. May require structured handling by callers of pip-accel 15 | 16 | Yes, I know, I just made your lovely and elegant Python look a whole lot like 17 | Java! I guess the message to take away here is that (in my opinion) structured 18 | error handling helps to build robust software that acknowledges failures exist 19 | and tries to deal with them (even if only by clearly recognizing a problem and 20 | giving up when there's nothing useful to do!). 21 | 22 | Hierarchy of exceptions 23 | ----------------------- 24 | 25 | If you're interested in implementing structured handling of exceptions reported 26 | by pip-accel the following diagram may help by visualizing the hierarchy: 27 | 28 | .. inheritance-diagram:: EnvironmentMismatchError UnknownDistributionFormat InvalidSourceDistribution \ 29 | BuildFailed NoBuildOutput CacheBackendError CacheBackendDisabledError \ 30 | DependencyInstallationRefused DependencyInstallationFailed 31 | :parts: 1 32 | 33 | ---- 34 | """ 35 | 36 | from pip_accel.utils import compact 37 | 38 | 39 | class PipAcceleratorError(Exception): 40 | 41 | """Base exception for all exception types explicitly raised by :mod:`pip_accel`.""" 42 | 43 | def __init__(self, text, **kw): 44 | """ 45 | Initialize a :class:`PipAcceleratorError` object. 46 | 47 | Accepts the same arguments as :func:`.compact()`. 48 | """ 49 | super(PipAcceleratorError, self).__init__(compact(text, **kw)) 50 | 51 | 52 | class NothingToDoError(PipAcceleratorError): 53 | 54 | """ 55 | Custom exception raised on empty requirement sets. 56 | 57 | Raised by :func:`~pip_accel.PipAccelerator.get_pip_requirement_set()` 58 | when pip doesn't report an error but also doesn't generate a requirement 59 | set (this happens when the user specifies an empty requirements file). 60 | """ 61 | 62 | 63 | class EnvironmentMismatchError(PipAcceleratorError): 64 | 65 | """ 66 | Custom exception raised when a cross-environment action is attempted. 67 | 68 | Raised by :func:`~pip_accel.PipAccelerator.validate_environment()` when 69 | it detects a mismatch between :data:`sys.prefix` and ``$VIRTUAL_ENV``. 70 | """ 71 | 72 | 73 | class UnknownDistributionFormat(PipAcceleratorError): 74 | 75 | """ 76 | Custom exception raised on unrecognized distribution archives. 77 | 78 | Raised by :attr:`~pip_accel.req.Requirement.is_wheel` when it cannot 79 | discern whether a given unpacked distribution is a source distribution or a 80 | wheel distribution. 81 | """ 82 | 83 | 84 | class BinaryDistributionError(PipAcceleratorError): 85 | 86 | """Base class for exceptions related to the generation of binary distributions.""" 87 | 88 | 89 | class InvalidSourceDistribution(BinaryDistributionError): 90 | 91 | """ 92 | Custom exception raised when a source distribution's setup script is missing. 93 | 94 | Raised by :func:`~pip_accel.bdist.BinaryDistributionManager.build_binary_dist()` 95 | when the given directory doesn't contain a Python source distribution. 96 | """ 97 | 98 | 99 | class BuildFailed(BinaryDistributionError): 100 | 101 | """ 102 | Custom exception raised when a binary distribution build fails. 103 | 104 | Raised by :func:`~pip_accel.bdist.BinaryDistributionManager.build_binary_dist()` 105 | when a binary distribution build fails. 106 | """ 107 | 108 | 109 | class NoBuildOutput(BinaryDistributionError): 110 | 111 | """ 112 | Custom exception raised when binary distribution builds don't generate an archive. 113 | 114 | Raised by :func:`~pip_accel.bdist.BinaryDistributionManager.build_binary_dist()` 115 | when a binary distribution build fails to produce the expected binary 116 | distribution archive. 117 | """ 118 | 119 | 120 | class CacheBackendError(PipAcceleratorError): 121 | 122 | """Custom exception raised by cache backends when they fail in a controlled manner.""" 123 | 124 | 125 | class CacheBackendDisabledError(CacheBackendError): 126 | 127 | """Custom exception raised by cache backends when they require configuration.""" 128 | 129 | 130 | class SystemDependencyError(PipAcceleratorError): 131 | 132 | """Base class for exceptions related to missing system packages.""" 133 | 134 | 135 | class DependencyInstallationRefused(SystemDependencyError): 136 | 137 | """ 138 | Custom exception raised when installation of dependencies is refused. 139 | 140 | Raised by :class:`.SystemPackageManager` when one or more known to be 141 | required system packages are missing and automatic installation of missing 142 | dependencies is disabled by the operator. 143 | """ 144 | 145 | 146 | class DependencyInstallationFailed(SystemDependencyError): 147 | 148 | """ 149 | Custom exception raised when installation of dependencies fails. 150 | 151 | Raised by :class:`.SystemPackageManager` when the installation of 152 | missing system packages fails. 153 | """ 154 | -------------------------------------------------------------------------------- /pip_accel/req.py: -------------------------------------------------------------------------------- 1 | # Accelerator for pip, the Python package manager. 2 | # 3 | # Author: Peter Odding 4 | # Last Change: February 2, 2016 5 | # URL: https://github.com/paylogic/pip-accel 6 | 7 | """ 8 | Simple wrapper for pip and pkg_resources `Requirement` objects. 9 | 10 | After downloading the specified requirement(s) pip reports a "requirement set" 11 | to pip-accel. In the past pip-accel would summarize this requirement set into a 12 | list of tuples, where each tuple would contain a requirement's project name, 13 | version and source directory (basically only the information required by 14 | pip-accel remained). 15 | 16 | Recently I've started using pip-accel as a library in another project I'm 17 | working on (not yet public) and in that project I am very interested in whether 18 | a given requirement is a direct or transitive requirement. Unfortunately 19 | pip-accel did not preserve this information. 20 | 21 | That's when I decided that next to pip's :class:`pip.req.InstallRequirement` 22 | and setuptools' :class:`pkg_resources.Requirement` I would introduce yet 23 | another type of requirement object... It's basically just a summary of the 24 | other two types of requirement objects and it also provides access to the 25 | original requirement objects (for those who are interested; the interfaces are 26 | basically undocumented AFAIK). 27 | """ 28 | 29 | # Standard library modules. 30 | import glob 31 | import logging 32 | import os 33 | import re 34 | import time 35 | 36 | # Modules included in our package. 37 | from pip_accel.exceptions import UnknownDistributionFormat 38 | from pip_accel.utils import hash_files 39 | 40 | # External dependencies. 41 | from cached_property import cached_property 42 | from pip.req import InstallRequirement 43 | 44 | # The following package(s) are usually bundled with pip but may be unbundled 45 | # by redistributors and pip-accel should handle this gracefully. 46 | try: 47 | from pip._vendor.distlib.util import ARCHIVE_EXTENSIONS 48 | from pip._vendor.pkg_resources import find_distributions 49 | except ImportError: 50 | from distlib.util import ARCHIVE_EXTENSIONS 51 | from pkg_resources import find_distributions 52 | 53 | # Initialize a logger for this module. 54 | logger = logging.getLogger(__name__) 55 | 56 | 57 | class Requirement(object): 58 | 59 | """Simple wrapper for the requirement objects defined by pip and setuptools.""" 60 | 61 | def __init__(self, config, requirement): 62 | """ 63 | Initialize a requirement object. 64 | 65 | :param config: A :class:`~pip_accel.config.Config` object. 66 | :param requirement: A :class:`pip.req.InstallRequirement` object. 67 | """ 68 | self.config = config 69 | self.pip_requirement = requirement 70 | self.setuptools_requirement = requirement.req 71 | 72 | def __repr__(self): 73 | """Generate a human friendly representation of a requirement object.""" 74 | return "Requirement(name=%r, version=%r)" % (self.name, self.version) 75 | 76 | @cached_property 77 | def name(self): 78 | """ 79 | The name of the Python package (a string). 80 | 81 | This is the name used to register a package on PyPI and the name 82 | reported by commands like ``pip freeze``. Based on 83 | :attr:`pkg_resources.Requirement.project_name`. 84 | """ 85 | return self.setuptools_requirement.project_name 86 | 87 | @cached_property 88 | def version(self): 89 | """The version of the package that ``pip`` wants to install (a string).""" 90 | if self.is_wheel: 91 | return self.wheel_metadata.version 92 | else: 93 | return self.sdist_metadata['Version'] 94 | 95 | @cached_property 96 | def related_archives(self): 97 | """ 98 | The pathnames of the source distribution(s) for this requirement (a list of strings). 99 | 100 | .. note:: This property is very new in pip-accel and its logic may need 101 | some time to mature. For now any misbehavior by this property 102 | shouldn't be too much of a problem because the pathnames 103 | reported by this property are only used for cache 104 | invalidation (see the :attr:`last_modified` and 105 | :attr:`checksum` properties). 106 | """ 107 | # Escape the requirement's name for use in a regular expression. 108 | name_pattern = escape_name(self.name) 109 | # Escape the requirement's version for in a regular expression. 110 | version_pattern = re.escape(self.version) 111 | # Create a regular expression that matches any of the known source 112 | # distribution archive extensions. 113 | extension_pattern = '|'.join(re.escape(ext) for ext in ARCHIVE_EXTENSIONS if ext != '.whl') 114 | # Compose the regular expression pattern to match filenames of source 115 | # distribution archives in the local source index directory. 116 | pattern = '^%s-%s(%s)$' % (name_pattern, version_pattern, extension_pattern) 117 | # Compile the regular expression for case insensitive matching. 118 | compiled_pattern = re.compile(pattern, re.IGNORECASE) 119 | # Find the matching source distribution archives. 120 | return [os.path.join(self.config.source_index, fn) 121 | for fn in os.listdir(self.config.source_index) 122 | if compiled_pattern.match(fn)] 123 | 124 | @cached_property 125 | def last_modified(self): 126 | """ 127 | The last modified time of the requirement's source distribution archive(s) (a number). 128 | 129 | The value of this property is based on the :attr:`related_archives` 130 | property. If no related archives are found the current time is 131 | reported. In the balance between not invalidating cached binary 132 | distributions enough and invalidating them too frequently, this 133 | property causes the latter to happen. 134 | """ 135 | mtimes = list(map(os.path.getmtime, self.related_archives)) 136 | return max(mtimes) if mtimes else time.time() 137 | 138 | @cached_property 139 | def checksum(self): 140 | """ 141 | The SHA1 checksum of the requirement's source distribution archive(s) (a string). 142 | 143 | The value of this property is based on the :attr:`related_archives` 144 | property. If no related archives are found the SHA1 digest of the empty 145 | string is reported. 146 | """ 147 | return hash_files('sha1', *sorted(self.related_archives)) 148 | 149 | @cached_property 150 | def source_directory(self): 151 | """ 152 | The pathname of the directory containing the unpacked source distribution (a string). 153 | 154 | This is the directory that contains a ``setup.py`` script. Based on 155 | :attr:`pip.req.InstallRequirement.source_dir`. 156 | """ 157 | return self.pip_requirement.source_dir 158 | 159 | @cached_property 160 | def is_wheel(self): 161 | """ 162 | :data:`True` when the requirement is a wheel, :data:`False` otherwise. 163 | 164 | .. note:: To my surprise it seems to be non-trivial to determine 165 | whether a given :class:`pip.req.InstallRequirement` object 166 | produced by pip's internal Python API concerns a source 167 | distribution or a wheel distribution. 168 | 169 | There's a :class:`pip.req.InstallRequirement.is_wheel` 170 | property but I'm currently looking at a wheel distribution 171 | whose ``is_wheel`` property returns :data:`None`, apparently 172 | because the requirement's ``url`` property is also :data:`None`. 173 | 174 | Whether this is an obscure implementation detail of pip or 175 | caused by the way pip-accel invokes pip, I really can't tell 176 | (yet). 177 | """ 178 | probably_sdist = os.path.isfile(os.path.join(self.source_directory, 'setup.py')) 179 | probably_wheel = len(glob.glob(os.path.join(self.source_directory, '*.dist-info', 'WHEEL'))) > 0 180 | if probably_wheel and not probably_sdist: 181 | return True 182 | elif probably_sdist and not probably_wheel: 183 | return False 184 | elif probably_sdist and probably_wheel: 185 | variables = dict(requirement=self.setuptools_requirement, 186 | directory=self.source_directory) 187 | raise UnknownDistributionFormat(""" 188 | The unpacked distribution of {requirement} in {directory} looks 189 | like a source distribution and a wheel distribution, I'm 190 | confused! 191 | """, **variables) 192 | else: 193 | variables = dict(requirement=self.setuptools_requirement, 194 | directory=self.source_directory) 195 | raise UnknownDistributionFormat(""" 196 | The unpacked distribution of {requirement} in {directory} 197 | doesn't look like a source distribution and also doesn't look 198 | like a wheel distribution, I'm confused! 199 | """, **variables) 200 | 201 | @cached_property 202 | def is_transitive(self): 203 | """ 204 | Whether the dependency is transitive (indirect). 205 | 206 | :data:`True` when the requirement is a transitive dependency (a 207 | dependency of a dependency) or :data:`False` when the requirement is a 208 | direct dependency (specified on pip's command line or in a 209 | ``requirements.txt`` file). Based on 210 | :attr:`pip.req.InstallRequirement.comes_from`. 211 | """ 212 | return isinstance(self.pip_requirement.comes_from, InstallRequirement) 213 | 214 | @cached_property 215 | def is_direct(self): 216 | """The opposite of :attr:`Requirement.is_transitive`.""" 217 | return not self.is_transitive 218 | 219 | @cached_property 220 | def is_editable(self): 221 | """ 222 | Whether the requirement should be installed in editable mode. 223 | 224 | :data:`True` when the requirement is to be installed in editable mode 225 | (i.e. setuptools "develop mode"). Based on 226 | :attr:`pip.req.InstallRequirement.editable`. 227 | """ 228 | return self.pip_requirement.editable 229 | 230 | @cached_property 231 | def sdist_metadata(self): 232 | """Get the distribution metadata of an unpacked source distribution.""" 233 | if self.is_wheel: 234 | raise TypeError("Requirement is not a source distribution!") 235 | return self.pip_requirement.pkg_info() 236 | 237 | @cached_property 238 | def wheel_metadata(self): 239 | """Get the distribution metadata of an unpacked wheel distribution.""" 240 | if not self.is_wheel: 241 | raise TypeError("Requirement is not a wheel distribution!") 242 | for distribution in find_distributions(self.source_directory): 243 | return distribution 244 | msg = "pkg_resources didn't find a wheel distribution in %s!" 245 | raise Exception(msg % self.source_directory) 246 | 247 | def __str__(self): 248 | """Render a human friendly string describing the requirement.""" 249 | return "%s (%s)" % (self.name, self.version) 250 | 251 | 252 | class TransactionalUpdate(object): 253 | 254 | """Context manager that enables transactional package upgrades.""" 255 | 256 | def __init__(self, requirement): 257 | """ 258 | Initialize a :class:`TransactionalUpdate` object. 259 | 260 | :param requirement: A :class:`Requirement` object. 261 | """ 262 | self.requirement = requirement 263 | self.pip_requirement = requirement.pip_requirement 264 | self.in_transaction = False 265 | 266 | def __enter__(self): 267 | """Prepare package upgrades by removing conflicting installations.""" 268 | if self.pip_requirement.conflicts_with: 269 | # Let __exit__() know that it should commit or rollback. 270 | self.in_transaction = True 271 | # Remove the conflicting installation (and let the user know). 272 | logger.info("Found existing installation: %s", self.pip_requirement.conflicts_with) 273 | self.pip_requirement.uninstall(auto_confirm=True) 274 | # The uninstall() method has the unfortunate side effect of setting 275 | # `satisfied_by' (as a side effect of calling check_if_exists()) 276 | # which breaks the behavior we expect from pkg_info(). We clear the 277 | # `satisfied_by' property to avoid this strange interaction. 278 | self.pip_requirement.satisfied_by = None 279 | 280 | def __exit__(self, exc_type=None, exc_value=None, traceback=None): 281 | """Finalize or rollback a package upgrade.""" 282 | if self.in_transaction: 283 | if exc_type is None: 284 | self.pip_requirement.commit_uninstall() 285 | else: 286 | self.pip_requirement.rollback_uninstall() 287 | self.in_transaction = False 288 | 289 | 290 | def escape_name(requirement_name): 291 | """ 292 | Escape a requirement's name for use in a regular expression. 293 | 294 | This backslash-escapes all non-alphanumeric characters and replaces dashes 295 | and underscores with a character class that matches a dash or underscore 296 | (effectively treating dashes and underscores equivalently). 297 | 298 | :param requirement_name: The name of the requirement (a string). 299 | :returns: The requirement's name as a regular expression (a string). 300 | """ 301 | return re.sub('[^A-Za-z0-9]', escape_name_callback, requirement_name) 302 | 303 | 304 | def escape_name_callback(match): 305 | """ 306 | Used by :func:`escape_name()` to treat dashes and underscores as equivalent. 307 | 308 | :param match: A regular expression match object that captured a single character. 309 | :returns: A regular expression string that matches the captured character. 310 | """ 311 | character = match.group(0) 312 | return '[-_]' if character in ('-', '_') else r'\%s' % character 313 | -------------------------------------------------------------------------------- /pip_accel/utils.py: -------------------------------------------------------------------------------- 1 | # Utility functions for the pip accelerator. 2 | # 3 | # Author: Peter Odding 4 | # Last Change: January 17, 2016 5 | # URL: https://github.com/paylogic/pip-accel 6 | 7 | """ 8 | Utility functions for the pip accelerator. 9 | 10 | The :mod:`pip_accel.utils` module defines several miscellaneous/utility 11 | functions that are used throughout :mod:`pip_accel` but don't really belong 12 | with any single module. 13 | """ 14 | 15 | # Standard library modules. 16 | import errno 17 | import hashlib 18 | import logging 19 | import os 20 | import platform 21 | import sys 22 | 23 | # Modules included in our package. 24 | from pip_accel.compat import pathname2url, urljoin, WINDOWS 25 | 26 | # External dependencies. 27 | from humanfriendly import parse_path 28 | from pip.commands.uninstall import UninstallCommand 29 | 30 | # The following package(s) are usually bundled with pip but may be unbundled 31 | # by redistributors and pip-accel should handle this gracefully. 32 | try: 33 | from pip._vendor.pkg_resources import DistributionNotFound, WorkingSet, get_distribution, parse_requirements 34 | except ImportError: 35 | from pkg_resources import DistributionNotFound, WorkingSet, get_distribution, parse_requirements 36 | 37 | # Initialize a logger for this module. 38 | logger = logging.getLogger(__name__) 39 | 40 | 41 | def compact(text, **kw): 42 | """ 43 | Compact whitespace in a string and format any keyword arguments into the string. 44 | 45 | :param text: The text to compact (a string). 46 | :param kw: Any keyword arguments to apply using :func:`str.format()`. 47 | :returns: The compacted, formatted string. 48 | 49 | The whitespace compaction preserves paragraphs. 50 | """ 51 | return '\n\n'.join(' '.join(p.split()) for p in text.split('\n\n')).format(**kw) 52 | 53 | 54 | def expand_path(pathname): 55 | """ 56 | Expand the home directory in a pathname based on the effective user id. 57 | 58 | :param pathname: A pathname that may start with ``~/``, indicating the path 59 | should be interpreted as being relative to the home 60 | directory of the current (effective) user. 61 | :returns: The (modified) pathname. 62 | 63 | This function is a variant of :func:`os.path.expanduser()` that doesn't use 64 | ``$HOME`` but instead uses the home directory of the effective user id. 65 | This is basically a workaround for ``sudo -s`` not resetting ``$HOME``. 66 | """ 67 | # The following logic previously used regular expressions but that approach 68 | # turned out to be very error prone, hence the current contraption based on 69 | # direct string manipulation :-). 70 | home_directory = find_home_directory() 71 | separators = set([os.sep]) 72 | if os.altsep is not None: 73 | separators.add(os.altsep) 74 | if len(pathname) >= 2 and pathname[0] == '~' and pathname[1] in separators: 75 | pathname = os.path.join(home_directory, pathname[2:]) 76 | # Also expand environment variables. 77 | return parse_path(pathname) 78 | 79 | 80 | def create_file_url(pathname): 81 | """ 82 | Create a ``file:...`` URL from a local pathname. 83 | 84 | :param pathname: The pathname of a local file or directory (a string). 85 | :returns: A URL that refers to the local file or directory (a string). 86 | """ 87 | return urljoin('file:', pathname2url(os.path.abspath(pathname))) 88 | 89 | 90 | def find_home_directory(): 91 | """ 92 | Look up the home directory of the effective user id. 93 | 94 | :returns: The pathname of the home directory (a string). 95 | 96 | .. note:: On Windows this uses the ``%APPDATA%`` environment variable (if 97 | available) and otherwise falls back to ``~/Application Data``. 98 | """ 99 | if WINDOWS: 100 | directory = os.environ.get('APPDATA') 101 | if not directory: 102 | directory = os.path.expanduser(r'~\Application Data') 103 | else: 104 | # This module isn't available on Windows so we have to import it here. 105 | import pwd 106 | # Look up the home directory of the effective user id so we can 107 | # generate pathnames relative to the home directory. 108 | entry = pwd.getpwuid(os.getuid()) 109 | directory = entry.pw_dir 110 | return directory 111 | 112 | 113 | def is_root(): 114 | """Detect whether we're running with super user privileges.""" 115 | return False if WINDOWS else os.getuid() == 0 116 | 117 | 118 | def get_python_version(): 119 | """ 120 | Get a string identifying the currently running Python version. 121 | 122 | This function generates a string that uniquely identifies the currently 123 | running Python implementation and version. The Python implementation is 124 | discovered using :func:`platform.python_implementation()` and the major 125 | and minor version numbers are extracted from :data:`sys.version_info`. 126 | 127 | :returns: A string containing the name of the Python implementation 128 | and the major and minor version numbers. 129 | 130 | Example: 131 | 132 | >>> from pip_accel.utils import get_python_version 133 | >>> get_python_version() 134 | 'CPython-2.7' 135 | """ 136 | return '%s-%i.%i' % (platform.python_implementation(), 137 | sys.version_info[0], 138 | sys.version_info[1]) 139 | 140 | 141 | def makedirs(path, mode=0o777): 142 | """ 143 | Create a directory if it doesn't already exist (keeping concurrency in mind). 144 | 145 | :param path: The pathname of the directory to create (a string). 146 | :param mode: The mode to apply to newly created directories (an integer, 147 | defaults to the octal number ``0777``). 148 | :returns: :data:`True` when the directory was created, :data:`False` if it already 149 | existed. 150 | :raises: Any exceptions raised by :func:`os.makedirs()` except for 151 | :data:`errno.EEXIST` (this error is swallowed and :data:`False` is 152 | returned instead). 153 | """ 154 | try: 155 | os.makedirs(path, mode) 156 | return True 157 | except OSError as e: 158 | if e.errno != errno.EEXIST: 159 | # We don't want to swallow errors other than EEXIST, 160 | # because we could be obscuring a real problem. 161 | raise 162 | return False 163 | 164 | 165 | def same_directories(path1, path2): 166 | """ 167 | Check if two pathnames refer to the same directory. 168 | 169 | :param path1: The first pathname (a string). 170 | :param path2: The second pathname (a string). 171 | :returns: :data:`True` if both pathnames refer to the same directory, 172 | :data:`False` otherwise. 173 | """ 174 | if all(os.path.isdir(p) for p in (path1, path2)): 175 | try: 176 | return os.path.samefile(path1, path2) 177 | except AttributeError: 178 | # On Windows and Python 2 os.path.samefile() is unavailable. 179 | return os.path.realpath(path1) == os.path.realpath(path2) 180 | else: 181 | return False 182 | 183 | 184 | def hash_files(method, *files): 185 | """ 186 | Calculate the hexadecimal digest of one or more local files. 187 | 188 | :param method: The hash method (a string, given to :func:`hashlib.new()`). 189 | :param files: The pathname(s) of file(s) to hash (zero or more strings). 190 | :returns: The calculated hex digest (a string). 191 | """ 192 | context = hashlib.new(method) 193 | for filename in files: 194 | with open(filename, 'rb') as handle: 195 | while True: 196 | chunk = handle.read(4096) 197 | if not chunk: 198 | break 199 | context.update(chunk) 200 | return context.hexdigest() 201 | 202 | 203 | def replace_file(src, dst): 204 | """ 205 | Overwrite a file (in an atomic fashion when possible). 206 | 207 | :param src: The pathname of the source file (a string). 208 | :param dst: The pathname of the destination file (a string). 209 | """ 210 | # Try os.replace() which was introduced in Python 3.3 211 | # (this should work on POSIX as well as Windows systems). 212 | try: 213 | os.replace(src, dst) 214 | return 215 | except AttributeError: 216 | pass 217 | # Try os.rename() which is atomic on UNIX but refuses to overwrite existing 218 | # files on Windows. 219 | try: 220 | os.rename(src, dst) 221 | return 222 | except OSError as e: 223 | if e.errno != errno.EEXIST: 224 | raise 225 | # Finally we fall back to the dumb approach required only on Windows. 226 | # See https://bugs.python.org/issue8828 for a long winded discussion. 227 | os.remove(dst) 228 | os.rename(src, dst) 229 | 230 | 231 | class AtomicReplace(object): 232 | 233 | """Context manager to atomically replace a file's contents.""" 234 | 235 | def __init__(self, filename): 236 | """ 237 | Initialize a :class:`AtomicReplace` object. 238 | 239 | :param filename: The pathname of the file to replace (a string). 240 | """ 241 | self.filename = filename 242 | self.temporary_file = '%s.tmp-%i' % (filename, os.getpid()) 243 | 244 | def __enter__(self): 245 | """ 246 | Prepare to replace the file's contents. 247 | 248 | :returns: The pathname of a temporary file in the same directory as the 249 | file to replace (a string). Using this temporary file ensures 250 | that :func:`replace_file()` doesn't fail due to a 251 | cross-device rename operation. 252 | """ 253 | logger.debug("Using temporary file to avoid partial reads: %s", self.temporary_file) 254 | return self.temporary_file 255 | 256 | def __exit__(self, exc_type=None, exc_value=None, traceback=None): 257 | """Replace the file's contents (if no exception occurred) using :func:`replace_file()`.""" 258 | if exc_type is None: 259 | logger.debug("Moving temporary file into place: %s", self.filename) 260 | replace_file(self.temporary_file, self.filename) 261 | 262 | 263 | def requirement_is_installed(expr): 264 | """ 265 | Check whether a requirement is installed. 266 | 267 | :param expr: A requirement specification similar to those used in pip 268 | requirement files (a string). 269 | :returns: :data:`True` if the requirement is available (installed), 270 | :data:`False` otherwise. 271 | """ 272 | required_dist = next(parse_requirements(expr)) 273 | try: 274 | installed_dist = get_distribution(required_dist.key) 275 | return installed_dist in required_dist 276 | except DistributionNotFound: 277 | return False 278 | 279 | 280 | def is_installed(package_name): 281 | """ 282 | Check whether a package is installed in the current environment. 283 | 284 | :param package_name: The name of the package (a string). 285 | :returns: :data:`True` if the package is installed, :data:`False` otherwise. 286 | """ 287 | return package_name.lower() in (d.key.lower() for d in WorkingSet()) 288 | 289 | 290 | def uninstall(*package_names): 291 | """ 292 | Uninstall one or more packages using the Python equivalent of ``pip uninstall --yes``. 293 | 294 | The package(s) to uninstall must be installed, otherwise pip will raise an 295 | ``UninstallationError``. You can check for installed packages using 296 | :func:`is_installed()`. 297 | 298 | :param package_names: The names of one or more Python packages (strings). 299 | """ 300 | command = UninstallCommand() 301 | opts, args = command.parse_args(['--yes'] + list(package_names)) 302 | command.run(opts, args) 303 | 304 | 305 | def match_option(argument, short_option, long_option): 306 | """ 307 | Match a command line argument against a short and long option. 308 | 309 | :param argument: The command line argument (a string). 310 | :param short_option: The short option (a string). 311 | :param long_option: The long option (a string). 312 | :returns: :data:`True` if the argument matches, :data:`False` otherwise. 313 | """ 314 | return short_option[1] in argument[1:] if is_short_option(argument) else argument == long_option 315 | 316 | 317 | def is_short_option(argument): 318 | """ 319 | Check if a command line argument is a short option. 320 | 321 | :param argument: The command line argument (a string). 322 | :returns: :data:`True` if the argument is a short option, :data:`False` otherwise. 323 | """ 324 | return len(argument) >= 2 and argument[0] == '-' and argument[1] != '-' 325 | 326 | 327 | def match_option_with_value(arguments, option, value): 328 | """ 329 | Check if a list of command line options contains an option with a value. 330 | 331 | :param arguments: The command line arguments (a list of strings). 332 | :param option: The long option (a string). 333 | :param value: The expected value (a string). 334 | :returns: :data:`True` if the command line contains the option/value pair, 335 | :data:`False` otherwise. 336 | """ 337 | return ('%s=%s' % (option, value) in arguments or 338 | contains_sublist(arguments, [option, value])) 339 | 340 | 341 | def contains_sublist(lst, sublst): 342 | """ 343 | Check if one list contains the items from another list (in the same order). 344 | 345 | :param lst: The main list. 346 | :param sublist: The sublist to check for. 347 | :returns: :data:`True` if the main list contains the items from the 348 | sublist in the same order, :data:`False` otherwise. 349 | 350 | Based on `this StackOverflow answer `_. 351 | """ 352 | n = len(sublst) 353 | return any((sublst == lst[i:i + n]) for i in range(len(lst) - n + 1)) 354 | -------------------------------------------------------------------------------- /requirements-flake8.txt: -------------------------------------------------------------------------------- 1 | # Travis CI builds run flake8 with the pep257 plug-in to validate the coding 2 | # style and break the build on (unexpected) violations. However recently I 3 | # encountered a failure [1] that seems to be an incompatibility between 4 | # flake8-pep257==1.0.3 and pep257==0.7.0. This incompatibility has already 5 | # been reported [2]. 6 | # 7 | # [1] https://travis-ci.org/xolox/python-coloredlogs/builds/85401773 8 | # [2] https://github.com/Robpol86/flake8-pep257/issues/4 9 | 10 | flake8==2.4.1 11 | flake8-pep257==1.0.3 12 | pep257==0.6.0 13 | pep8==1.5.7 14 | pyflakes==0.8.1 15 | -------------------------------------------------------------------------------- /requirements-rtd.txt: -------------------------------------------------------------------------------- 1 | # This pip requirements file is used by Read the Docs to build pip-accel's 2 | # online documentation available at http://pip-accel.readthedocs.org/. 3 | # 4 | # On April 5, 2015 I released pip-accel 0.23. The Read the Docs build for 5 | # release 0.23 went fine [1] but the build of 0.24 [2] later that same day 6 | # failed. The curious thing is that 0.23 introduced large changes while 0.24 7 | # contains only minimal changes that don't look like they could have caused 8 | # the issue at hand! The actual issue in the failing build seems to be an 9 | # obscure version conflict: 10 | # 11 | # 1. pip-accel >= 0.23 uses pip >= 0.6.8. 12 | # 2. pip 6.x bundles `retrying and `six' in pip._vendor modules. 13 | # 3. retrying.retry() expects six.wraps() to exist, but it doesn't. 14 | # 15 | # Because pip bundles both `retrying' and `six' and has modified `retrying' to 16 | # import from the bundled `six' this should never be a problem, but somehow it 17 | # is, specifically and only on Read the Docs... This may have something to do 18 | # with the way Read the Docs manages their Python build environments?! 19 | # 20 | # [1] https://readthedocs.org/builds/pip-accel/2532860/ 21 | # [2] https://readthedocs.org/builds/pip-accel/2532985/ 22 | 23 | six >= 1.7.0 24 | 25 | # Make sure test requirements are also installed; the documentation includes 26 | # the test suite and so Sphinx needs to be able to import the test suite, which 27 | # means the test dependencies must be installed! 28 | 29 | --requirement=requirements-testing.txt 30 | -------------------------------------------------------------------------------- /requirements-testing.txt: -------------------------------------------------------------------------------- 1 | # We need boto to test the S3 cache backend. 2 | boto >= 2.32 3 | 4 | # We use executor to control FakeS3. 5 | executor >= 7.7 6 | 7 | # We use portalocker to avoid concurrent dpkg and apt-get processes. 8 | portalocker >= 0.5.4 9 | 10 | # We use py.test for improved error reporting. 11 | pytest >= 2.6.4 12 | 13 | # We use a pytest plug-in to collect coverage. 14 | pytest-cov >= 2.2.0 15 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | cached-property >= 0.1.5 2 | coloredlogs >= 3.0 3 | humanfriendly >= 1.44.7 4 | pip >= 7.0, < 7.2 5 | setuptools >= 7.0 6 | -------------------------------------------------------------------------------- /scripts/appveyor.py: -------------------------------------------------------------------------------- 1 | # Simple Python script that helps to understand the AppVeyor environment. 2 | # 3 | # Author: Peter Odding 4 | # Last Change: November 11, 2015 5 | # URL: https://github.com/paylogic/pip-accel 6 | 7 | """Introspection of the AppVeyor CI environment ...""" 8 | 9 | # Standard library modules. 10 | import os 11 | 12 | # External dependencies. 13 | from humanfriendly import concatenate 14 | 15 | # Test dependencies. 16 | from executor import ExternalCommandFailed, execute, get_search_path, which 17 | 18 | print("FakeS3 executables:\n%r" % which('fakes3')) 19 | 20 | for program in which('fakes3'): 21 | with open(program) as handle: 22 | contents = handle.read() 23 | delimiter = "-" * 40 24 | vertical_whitespace = "\n\n" 25 | padding = vertical_whitespace + delimiter + vertical_whitespace 26 | print(padding + ("%s:" % program) + padding + contents + padding) 27 | 28 | for program in ['fakes3', 'fakes3.bat'] + which('fakes3'): 29 | try: 30 | execute(program, '--help') 31 | except ExternalCommandFailed: 32 | print("%s doesn't work?!" % program) 33 | 34 | print("Executable search path:\n\n%s" % "\n\n".join( 35 | "%s:\n%s" % (d, concatenate(sorted(os.listdir(d)))) 36 | for d in get_search_path() if os.path.isdir(d) 37 | )) 38 | -------------------------------------------------------------------------------- /scripts/collect-test-coverage.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | # Shell script to run the pip-accel test suite. 4 | # 5 | # Author: Peter Odding 6 | # Last Change: November 11, 2015 7 | # URL: https://github.com/paylogic/pip-accel 8 | # 9 | # This shell script is used in tox.ini and .travis.yml to run 10 | # the pip-accel test suite with coverage collection enabled. 11 | 12 | main () { 13 | 14 | # The following environment variable is needed to collect coverage about 15 | # automatic installation of dependencies on system packages. Please review 16 | # the notes in the test suite (pip_accel/tests.py) if you're not sure whether 17 | # you want to run this on your system :-). 18 | if [ -n "$CI" ] || (hostname | grep -q peter); then 19 | export PIP_ACCEL_TEST_AUTO_INSTALL=true 20 | fi 21 | 22 | # Don't silence the Boto logger because it can be interesting to see how Boto 23 | # deals with FakeS3 dropping out in the middle of the test suite. 24 | export PIP_ACCEL_SILENCE_BOTO=false 25 | 26 | # Run the test suite under py.test with coverage collection enabled? 27 | if [ "$COVERAGE" != no ]; then 28 | py.test --cov "$@" 29 | else 30 | py.test "$@" 31 | fi 32 | 33 | } 34 | 35 | main "$@" 36 | -------------------------------------------------------------------------------- /scripts/prepare-test-environment.cmd: -------------------------------------------------------------------------------- 1 | :: Windows batch script to prepare for the pip-accel test suite. 2 | :: 3 | :: Author: Peter Odding 4 | :: Last Change: January 16, 2016 5 | :: URL: https://github.com/paylogic/pip-accel 6 | :: 7 | :: This Windows batch script is used to run the pip-accel test suite on 8 | :: AppVeyor CI with increased coverage collection (which requires some 9 | :: preparations). It installs/upgrades/removes several Python packages whose 10 | :: installation, upgrade and/or removal is tested in the test suite to make 11 | :: sure that the test suite starts from a known state. 12 | 13 | :: Install pip-accel in editable mode. 14 | "%PYTHON%\Scripts\pip.exe" install --quiet --editable . 15 | 16 | :: Install the test suite's dependencies. We ignore py.test wheels because of 17 | :: an obscure issue that took me hours to debug and I really don't want to get 18 | :: into it here :-(. 19 | "%PYTHON%\Scripts\pip.exe" install --no-binary=pytest --quiet --requirement=requirements-testing.txt 20 | 21 | :: Install requests==2.6.0 so the test suite can downgrade to requests==2.2.1 22 | :: (to verify that downgrading of packages works). Ideally the test suite 23 | :: should just be able to install requests==2.6.0 and then downgrade to 24 | :: requests==2.2.1 but unfortunately this doesn't work reliably in the same 25 | :: Python process due to (what looks like) caching in the pkg_resources module 26 | :: bundled with pip (which in turn causes a variety of confusing internal 27 | :: errors in pip and pip-accel). 28 | "%PYTHON%\Scripts\pip.exe" install --quiet requests==2.6.0 29 | 30 | :: Remove iPython so the test suite can install iPython in a clean environment, 31 | :: allowing the test suite to compare the files installed and removed by pip 32 | :: and pip-accel. 33 | "%PYTHON%\Scripts\pip.exe" uninstall --quiet --yes ipython 34 | 35 | :: If iPython wasn't installed to begin with, the previous command will have 36 | :: returned with a nonzero exit code. We don't want this to terminate the 37 | :: AppVeyor CI build. 38 | exit /b 0 39 | -------------------------------------------------------------------------------- /scripts/prepare-test-environment.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | # Shell script to initialize a pip-accel test environment. 4 | # 5 | # Author: Peter Odding 6 | # Last Change: March 14, 2016 7 | # URL: https://github.com/paylogic/pip-accel 8 | # 9 | # This shell script is used in tox.ini and .travis.yml to prepare 10 | # virtual environments for running the pip-accel test suite. 11 | 12 | main () { 13 | 14 | # Install the dependencies of pip-accel. 15 | msg "Installing dependencies .." 16 | pip install --quiet --requirement=requirements.txt 17 | 18 | # Install pip-accel in editable mode. The LC_ALL=C trick makes sure that the 19 | # setup.py script works regardless of the user's locale (this is all about 20 | # that little copyright symbol at the bottom of README.rst :-). 21 | msg "Installing pip-accel (in editable mode) .." 22 | LC_ALL=C pip install --quiet --editable . 23 | 24 | # Install the test suite's dependencies. We ignore py.test wheels because of 25 | # an obscure issue that took me hours to debug and I really don't want to get 26 | # into it here :-(. 27 | msg "Installing test dependencies .." 28 | pip install --no-binary=pytest --quiet --requirement=$PWD/requirements-testing.txt 29 | 30 | # Make it possible to install local working copies of selected dependencies. 31 | install_working_copies 32 | 33 | # Install requests==2.6.0 so the test suite can downgrade to requests==2.2.1 34 | # (to verify that downgrading of packages works). 35 | install_requests 36 | 37 | # Remove iPython so the test suite can install iPython in a clean environment. 38 | remove_ipython 39 | 40 | } 41 | 42 | install_working_copies () { 43 | # Once in a while I find myself working on multiple Python projects at the same 44 | # time, context switching between the projects and running each project's tests 45 | # in turn. Out of the box `tox' will not be able to locate and install my 46 | # unreleased working copies, but using this ugly hack it works fine :-). 47 | if hostname | grep -q peter; then 48 | for PROJECT in coloredlogs executor humanfriendly; do 49 | DIRECTORY="$HOME/projects/python/$PROJECT" 50 | if [ -e "$DIRECTORY" ]; then 51 | if flock -x "$DIRECTORY"; then 52 | msg "Installing working copy of $PROJECT .." 53 | # The following code to install a Python package from a git checkout is 54 | # a bit convoluted because I want to bypass pip's frustrating "let's 55 | # copy the whole project tree including a 100 MB `.tox' directory 56 | # before installing 10 KB of source code" approach. The use of a 57 | # source distribution works fine :-). 58 | cd "$DIRECTORY" 59 | rm -Rf dist 60 | python setup.py sdist &>/dev/null 61 | # Side step caching of binary wheels because we'll be building a new 62 | # one on each run anyway. 63 | pip install --no-binary=:all: --quiet dist/* 64 | fi 65 | fi 66 | done 67 | fi 68 | } 69 | 70 | install_requests () { 71 | # FWIW: Ideally the test suite should just be able to install requests==2.6.0 72 | # and then downgrade to requests==2.2.1 but unfortunately this doesn't work 73 | # reliably in the same Python process due to (what looks like) caching in the 74 | # pkg_resources module bundled with pip (which in turn causes a variety of 75 | # confusing internal errors in pip and pip-accel). 76 | msg "Installing requests (so the test suite can downgrade it) .." 77 | pip install --quiet requests==2.6.0 78 | } 79 | 80 | remove_ipython () { 81 | # Remove iPython so the test suite can install iPython in a clean 82 | # environment, allowing the test suite to compare the files installed and 83 | # removed by pip and pip-accel. 84 | msg "Removing iPython (so the test suite can install and remove it) .." 85 | pip uninstall --yes ipython &>/dev/null || true 86 | } 87 | 88 | msg () { 89 | echo "[prepare-test-environment]" "$@" >&2 90 | } 91 | 92 | main "$@" 93 | -------------------------------------------------------------------------------- /scripts/retry-command: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Shell script to retry failing commands on Travis CI. 4 | # 5 | # Author: Peter Odding 6 | # Last Change: May 3, 2015 7 | # URL: https://github.com/paylogic/pip-accel 8 | # 9 | # Commands like `apt-get update' fail quite frequently on Travis CI and 10 | # retrying builds manually gets tiresome quickly. This shell script retries the 11 | # command given in the command line arguments up to ten times before giving up 12 | # and propagating the return code. 13 | 14 | main () { 15 | local limit=10 16 | for ((i=1; i<=$limit; i+=1)); do 17 | msg "Running command ($i/$limit): $*" 18 | "$@" 19 | local status=$? 20 | if [ $status -eq 0 ]; then 21 | msg "Command succeeded, done!" 22 | return 0 23 | elif [ $i -lt $limit ]; then 24 | msg "Command failed with return code $status, retrying .." 25 | else 26 | msg "Command failed with return code $status, giving up! :-(" 27 | return 1 28 | fi 29 | done 30 | } 31 | 32 | msg () { 33 | echo "[retry-command] $*" >&2 34 | } 35 | 36 | main "$@" 37 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Accelerator for pip, the Python package manager. 4 | # 5 | # Author: Peter Odding 6 | # Last Change: May 17, 2016 7 | # URL: https://github.com/paylogic/pip-accel 8 | 9 | """Setup script for the `pip-accel` package.""" 10 | 11 | # Standard library modules. 12 | import codecs 13 | import os 14 | import re 15 | 16 | # De-facto standard solution for Python packaging. 17 | from setuptools import setup, find_packages 18 | 19 | 20 | def get_readme(): 21 | """Get the contents of the ``README.rst`` file as a Unicode string.""" 22 | with codecs.open(get_absolute_path('README.rst'), 'r', 'utf-8') as handle: 23 | return handle.read() 24 | 25 | 26 | def get_version(*args): 27 | """Get the package's version (by extracting it from the source code).""" 28 | module_path = get_absolute_path(*args) 29 | with open(module_path) as handle: 30 | for line in handle: 31 | match = re.match(r'^__version__\s*=\s*["\']([^"\']+)["\']$', line) 32 | if match: 33 | return match.group(1) 34 | raise Exception("Failed to extract version from %s!" % module_path) 35 | 36 | 37 | def get_requirements(*args): 38 | """Get requirements from pip requirement files.""" 39 | requirements = set() 40 | with open(get_absolute_path(*args)) as handle: 41 | for line in handle: 42 | # Strip comments. 43 | line = re.sub(r'^#.*|\s#.*', '', line) 44 | # Ignore empty lines 45 | if line and not line.isspace(): 46 | requirements.add(re.sub(r'\s+', '', line)) 47 | return sorted(requirements) 48 | 49 | 50 | def get_absolute_path(*args): 51 | """Transform relative pathnames into absolute pathnames.""" 52 | directory = os.path.dirname(os.path.abspath(__file__)) 53 | return os.path.join(directory, *args) 54 | 55 | 56 | setup(name='pip-accel', 57 | version=get_version('pip_accel', '__init__.py'), 58 | description='Accelerator for pip, the Python package manager', 59 | long_description=get_readme(), 60 | author='Peter Odding', 61 | author_email='peter.odding@paylogic.com', 62 | url='https://github.com/paylogic/pip-accel', 63 | packages=find_packages(), 64 | entry_points={ 65 | 'console_scripts': ['pip-accel = pip_accel.cli:main'], 66 | 'pip_accel.cache_backends': [ 67 | # The default cache backend (uses the local file system). 68 | 'local = pip_accel.caches.local', 69 | # An optional cache backend that uses Amazon S3. 70 | 's3 = pip_accel.caches.s3 [s3]', 71 | ], 72 | }, 73 | extras_require={'s3': 'boto >= 2.32'}, 74 | package_data={'pip_accel.deps': ['*.ini']}, 75 | install_requires=get_requirements('requirements.txt'), 76 | test_suite='pip_accel.tests', 77 | tests_require=get_requirements('requirements-testing.txt'), 78 | classifiers=[ 79 | 'Development Status :: 5 - Production/Stable', 80 | 'Environment :: Console', 81 | 'Intended Audience :: Developers', 82 | 'Intended Audience :: Information Technology', 83 | 'Intended Audience :: System Administrators', 84 | 'License :: OSI Approved :: MIT License', 85 | 'Operating System :: MacOS :: MacOS X', 86 | 'Operating System :: Microsoft :: Windows', 87 | 'Operating System :: POSIX :: Linux', 88 | 'Operating System :: Unix', 89 | 'Programming Language :: Python :: 2.6', 90 | 'Programming Language :: Python :: 2.7', 91 | 'Programming Language :: Python :: 3.4', 92 | 'Programming Language :: Python :: 3.5', 93 | 'Topic :: Software Development :: Build Tools', 94 | 'Topic :: Software Development :: Libraries :: Python Modules', 95 | 'Topic :: System :: Archiving :: Packaging', 96 | 'Topic :: System :: Installation/Setup', 97 | 'Topic :: System :: Software Distribution', 98 | ]) 99 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | # Tox (http://tox.testrun.org/) is a tool for running Python test suites on 2 | # multiple versions of Python with a single command. This configuration file 3 | # will run the test suite on all supported Python versions. To use it, 4 | # `pip-accel install tox' and then run `tox' from this directory. 5 | 6 | [tox] 7 | envlist = py26, py27, py34, py35, pypy 8 | 9 | [testenv] 10 | passenv = 11 | COVERAGE 12 | commands = 13 | {toxinidir}/scripts/collect-test-coverage.sh {posargs} 14 | install_command = 15 | {toxinidir}/scripts/prepare-test-environment.sh {opts} {packages} 16 | whitelist_externals = * 17 | 18 | [pytest] 19 | addopts = --verbose 20 | norecursedirs = .tox 21 | python_files = pip_accel/tests.py 22 | 23 | [flake8] 24 | exclude = .tox 25 | max-line-length = 120 26 | 27 | # D301 is ignored because of the inheritance diagram included in the 28 | # pip_accel.exceptions module. 29 | 30 | [pep257] 31 | ignore = D211,D301,D402 32 | --------------------------------------------------------------------------------