├── .gitignore ├── .pre-commit-config.yaml ├── .travis.yml ├── CHANGELOG.rst ├── CONTRIBUTING.rst ├── LICENSE ├── MANIFEST.in ├── README.rst ├── TESTING.rst ├── mkwheelhouse.py ├── pylintrc ├── setup.cfg ├── setup.py └── test ├── fixtures.py ├── requirements ├── default.txt └── exclusions.txt └── test.bats /.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | dist 3 | *.egg-info 4 | __pycache__ 5 | *.pyc 6 | .env 7 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | - repo: git://github.com/pre-commit/pre-commit-hooks 2 | sha: v0.6.1 3 | hooks: 4 | - id: check-case-conflict 5 | - id: check-merge-conflict 6 | - id: check-yaml 7 | - id: debug-statements 8 | - id: double-quote-string-fixer 9 | - id: end-of-file-fixer 10 | - id: flake8 11 | - id: trailing-whitespace 12 | 13 | - repo: git://github.com/pre-commit/mirrors-pylint 14 | sha: v1.6.4 15 | hooks: 16 | - id: pylint 17 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: python 3 | python: 4 | - 2.7 5 | - 3.3 6 | - 3.4 7 | - 3.5 8 | - 3.6 9 | before_install: 10 | - npm install bats 11 | install: 12 | - pip install -e . 13 | - pip install "file://`pwd`#egg=mkwheelhouse[tests]" 14 | before_script: 15 | - test/fixtures.py create 16 | script: 17 | - bats test/test.bats 18 | after_script: 19 | - test/fixtures.py delete 20 | deploy: 21 | provider: pypi 22 | distributions: sdist bdist_wheel 23 | user: benesch 24 | on: 25 | tags: true 26 | password: 27 | secure: NIy//LpwQrQ6tGA/+jZCjgewqj4/TwMkjM8IDP7aKKQgDImqGFhzA0otdd5vitJFui2yDetrVubukCVpoTsGZBf0vi+Oy1O77jhJV9noWQKgTjWQiDhJSdPsbbGBfiXBzLW/9ZiTTHKbQiIy6VlPjqSSYEqsj4hk0zSACojwJWM= 28 | env: 29 | global: 30 | - MKWHEELHOUSE_BUCKET_STANDARD="travis-$TRAVIS_JOB_NUMBER.mkwheelhouse.com" 31 | - MKWHEELHOUSE_BUCKET_NONSTANDARD="travis-$TRAVIS_JOB_NUMBER.eu.mkwheelhouse.com" 32 | - secure: IVLpsOkFZer8m5oO3vZDhyg/lChlO/ZdZbw2LsQkPpxjpx8nzkksJzS33N+Vyk7RD/7RkRNMag2qRAIbauqV2xyMu+mCSHp/MznBnpGBc5DO62Cc4AZrPpzTL5JCBjzxpombaoTfhLRr15wQ7jMAMBBpxsxfNWlKPO9i1icRL5g= 33 | - secure: hoS+2oDLCByaeQQ21fM3z+h8Y0Da0tx2+FloEr483Xdv01N8ChzEoRWlvl7e9WF/bI7zMWPRmrpVOKxV095Y0+wlzDXRlGOA+2qr2F5KsiYvH/2yNtO3pFMs4FjKYg0dFXZpvXNhDPion98ish6zFsSAdy2h/HnV8b2hr/8D0Tk= 34 | -------------------------------------------------------------------------------- /CHANGELOG.rst: -------------------------------------------------------------------------------- 1 | Changelog 2 | ========= 3 | 4 | All notable changes to this project will be documented in this file. 5 | This project adheres to `Semantic Versioning `_. 6 | 7 | `2.0.0`_ - 2016 December 30 8 | ----------------------------- 9 | 10 | Added 11 | ~~~~~ 12 | 13 | - Unrecognized options will be passed through to ``pip wheel``. This 14 | means all requested pip options, like ``--no-binary`` and 15 | ``--cache-dir``, are exposed through mkwheelhouse. 16 | 17 | - Canned ACL policies can be specified on the command line, avoiding the 18 | need for a bucket policy [`#12`_\ ]. Thanks, `@thenoid`_! 19 | 20 | Changed 21 | ~~~~~~~ 22 | 23 | - Invoking mkwheelhouse with no requirements (e.g., ``mkwheelhouse 24 | s3://BUCKET``) generates only a warning instead of an error. This is 25 | because a bare ``pip wheel`` is not an error, and arguments are now 26 | passed directly to ``pip wheel``. This behavior may be changed 27 | upstream; follow `pypa/pip#2720`_ for details. 28 | 29 | Fixed 30 | ~~~~~ 31 | 32 | - Bucket URLs generated when IAM roles are in use are now valid [`#11`_\ ]. 33 | Thanks, `@joshma`_! 34 | 35 | 36 | `1.1.1`_ - 2016 January 23 37 | ----------------------------- 38 | 39 | Fixed 40 | ~~~~~ 41 | 42 | - mkwheelhouse can actually build wheelhouses in S3 subdirectories 43 | [`#10`_\ ]. Thanks, `@rajiteh`_! 44 | 45 | 46 | `1.1.0`_ - 2015 April 24 47 | ----------------------------- 48 | 49 | Added 50 | ~~~~~ 51 | 52 | - mkwheelhouse learned to build wheelhouses in S3 subdirectories (key 53 | prefixes) [`#2`_\ ]. Thanks, `@j-martin`_! 54 | 55 | 56 | `1.0.0`_ - 2015 April 23 57 | ------------------------ 58 | 59 | Changed 60 | ~~~~~~~ 61 | 62 | - Boto replaced Botocore. Boto is a higher-level AWS API that provides 63 | better error messages. mkwheelhouse's documented functionality should 64 | remain unchanged, but if you were relying on error messages, you may 65 | be impacted. **[BREAKING]** 66 | - Documentation converted to reStructuredText. 67 | 68 | 69 | `0.3.1`_ - 2015 April 23 70 | ------------------------ 71 | 72 | Added 73 | ~~~~~ 74 | 75 | - This CHANGELOG. 76 | - Unofficial Python 2.6 support [`#6`_\ ]. Thanks, `@prologic`_! 77 | 78 | 79 | .. _2.0.0: https://github.com/WhoopInc/mkwheelhouse/compare/1.1.1...2.0.0 80 | .. _1.1.1: https://github.com/WhoopInc/mkwheelhouse/compare/1.1.0...1.1.1 81 | .. _1.1.0: https://github.com/WhoopInc/mkwheelhouse/compare/1.0.0...1.1.0 82 | .. _1.0.0: https://github.com/WhoopInc/mkwheelhouse/compare/0.3.1...1.0.0 83 | .. _0.3.1: https://github.com/WhoopInc/mkwheelhouse/compare/0.3.0...0.3.1 84 | 85 | .. _#2: https://github.com/WhoopInc/mkwheelhouse/pull/2 86 | .. _#6: https://github.com/WhoopInc/mkwheelhouse/pull/6 87 | .. _#8: https://github.com/WhoopInc/mkwheelhouse/pull/8 88 | .. _#10: https://github.com/WhoopInc/mkwheelhouse/issues/10 89 | .. _#11: https://github.com/WhoopInc/mkwheelhouse/issues/11 90 | .. _#12: https://github.com/WhoopInc/mkwheelhouse/pull/12 91 | 92 | .. _@j-martin: https://github.com/j-martin 93 | .. _@joshma: https://github.com/joshma 94 | .. _@prologic: https://github.com/prologic 95 | .. _@rajiteh: https://github.com/rajiteh 96 | .. _@thenoid: https://github.com/thenoid 97 | 98 | .. _pypa/pip#2720: https://github.com/pypa/pip/issues/2720 99 | -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | Contributing 2 | ============ 3 | 4 | We love contributions! Pull request away. 5 | 6 | Hacking 7 | ------- 8 | 9 | You'll need Python, of course. Then, check out the code and install the 10 | production and test dependencies: 11 | 12 | .. code:: bash 13 | 14 | $ git clone git@github.com:WhoopInc/mkwheelhouse.git 15 | $ cd mkwheelhouse 16 | $ pip install -e . 17 | $ pip install "file://`pwd`#egg=mkwheelhouse[tests]" 18 | 19 | Hack away! When you're ready to test, either `run the test 20 | suite `_ or run ``mkwheelhouse`` manually: 21 | 22 | .. code:: bash 23 | 24 | $ mkwheelhouse BUCKET PACKAGE 25 | 26 | As long as you install mkwheelhouse with ``pip install -e`` or ``setup.py 27 | develop``, the ``mkwheelhouse`` binary will point to your up-to-date 28 | copy in the Git repository. 29 | 30 | Guidelines 31 | ---------- 32 | 33 | We do ask that all contributions pass the linter and test suite. Travis 34 | will automatically run these against your contribution once you submit 35 | the pull request, but you can also run them locally as you go! 36 | 37 | Linting 38 | ~~~~~~~ 39 | 40 | .. code:: bash 41 | 42 | $ pre-commit run --all-files 43 | 44 | You can also install a pre-commit hook to lint all changed files before 45 | every commit: 46 | 47 | .. code:: bash 48 | 49 | $ pre-commit install 50 | 51 | Testing 52 | ~~~~~~~ 53 | 54 | See `TESTING `_. 55 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014 WHOOP, Inc. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.rst 2 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ## Notice: This repository is no longer maintained as of 3/26/2024 2 | 3 | mkwheelhouse 4 | ============ 5 | 6 | .. image:: https://travis-ci.org/WhoopInc/mkwheelhouse.svg?branch=master 7 | :target: https://travis-ci.org/WhoopInc/mkwheelhouse 8 | :align: right 9 | 10 | Amazon S3 wheelhouse generator. 11 | 12 | Wheels are the latest standard in distributing binary for Python. Wheels 13 | cut down scipy's installation time from 15 minutes to 15 seconds. `Learn more 14 | about wheels. `_ 15 | 16 | Usage 17 | ----- 18 | 19 | Generate wheels for all listed ``PACKAGE``\ s and their dependencies, 20 | then upload them to Amazon S3 ``BUCKET``: 21 | 22 | .. code:: bash 23 | 24 | $ mkwheelhouse [options] BUCKET [PACKAGE...] [pip-options] 25 | 26 | Then install with pip like usual, but preferring generated wheels: 27 | 28 | .. code:: bash 29 | 30 | $ pip install --find-links BUCKET/index.html PACKAGE 31 | 32 | You can also build a wheelhouse in an `S3 subdirectory`_ by specifying 33 | the full S3 path: 34 | 35 | .. code:: bash 36 | 37 | $ mkwheelhouse s3://BUCKET/SUB/DIRECTORY PACKAGE 38 | 39 | .. _S3 subdirectory: http://docs.aws.amazon.com/AmazonS3/latest/UG/FolderOperations.html 40 | 41 | Additional options 42 | ~~~~~~~~~~~~~~~~~~ 43 | 44 | - ``-h``, ``--help`` 45 | 46 | Print usage information and exit. 47 | 48 | - ``-e``, ``--exclude WHEEL_FILENAME`` 49 | 50 | Don't upload built wheel with filename WHEEL\_FILENAME. Note this is the 51 | final wheel filename, like ``argparse-1.3.0-py2.py3-none-any.whl``, 52 | *not* the bare package name. 53 | 54 | Specifying an exclusion will not remove pre-existing built wheels from 55 | S3; you'll have to remove those wheels from the bucket manually. 56 | 57 | - ``-a``, ``--acl POLICY`` 58 | 59 | Apply canned ACL policy POLICY to the uploaded wheels and index. 60 | Specifying ``public-read``, for example, will make the uploaded wheels 61 | and index publicly accessible, avoiding the need for for a bucket 62 | policy to do the same. Valid values for POLICY can be found in the 63 | `AWS documentation`_. 64 | 65 | .. _AWS documentation: http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl 66 | 67 | Any unrecognized options are passed directly to ``pip wheel``. That 68 | means mkwheelhouse supports requirements files, for example: 69 | 70 | .. code:: bash 71 | 72 | $ mkwheelhouse s3://BUCKET -r requirements.txt 73 | 74 | Need an install log? Need to compile from source? You got it. 75 | 76 | .. code:: bash 77 | 78 | $ mkwheelhouse s3://BUCKET --log log.txt --no-binary :all: numpy 79 | 80 | Consult ``pip wheel`` for a complete list of pip-options. 81 | 82 | Notes 83 | ----- 84 | 85 | - Python 2 and 3 86 | 87 | - Set a `bucket policy to make all objects publicly accessible 88 | `_ 89 | or Pip won't be able to download wheels from your wheelhouse. 90 | -------------------------------------------------------------------------------- /TESTING.rst: -------------------------------------------------------------------------------- 1 | Testing 2 | ======= 3 | 4 | No unit testing, since the project is so small. But a full suite of 5 | acceptance tests that run using `Bats: Bash Automated Testing 6 | System `_! 7 | 8 | See `the .travis.yml CI configuration <.travis.yml>`_ for a working 9 | example. 10 | 11 | Environment variables 12 | --------------------- 13 | 14 | You'll need to export the below. Recommended values included when not 15 | sensitive. 16 | 17 | .. code:: bash 18 | 19 | # AWS credentials with permissions to create S3 buckets 20 | export AWS_ACCESS_KEY_ID= 21 | export AWS_SECRET_ACCESS_KEY= 22 | 23 | # Bucket names, one for the US Standard region and one for 24 | # a non-US Standard region to test endpoint logic. 25 | export MKWHEELHOUSE_BUCKET_STANDARD="$USER.mkwheelhouse.com" 26 | export MKWHEELHOUSE_BUCKET_NONSTANDARD="$USER.eu.mkwheelhouse.com" 27 | 28 | Running tests 29 | ------------- 30 | 31 | You'll need `Bats `_ installed! 32 | Then: 33 | 34 | .. code:: bash 35 | 36 | # export env vars as described 37 | $ python setup.py develop 38 | $ test/fixtures.py create 39 | $ bats test/test.bats 40 | # hack hack hack 41 | $ test/fixtures.py delete 42 | -------------------------------------------------------------------------------- /mkwheelhouse.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from __future__ import print_function 4 | from __future__ import unicode_literals 5 | 6 | import argparse 7 | import glob 8 | import json 9 | import mimetypes 10 | import os 11 | import re 12 | import shutil 13 | import subprocess 14 | import sys 15 | import tempfile 16 | from six.moves.urllib.parse import urlparse 17 | 18 | import boto 19 | import boto.s3.connection 20 | import yattag 21 | 22 | 23 | def spawn(args, capture_output=False): 24 | print('=>', ' '.join(args)) 25 | if capture_output: 26 | return subprocess.check_output(args) 27 | return subprocess.check_call(args) 28 | 29 | 30 | class UrlSafeS3Connection(object): 31 | # Hack to work around Boto bug that generates invalid URLs when 32 | # query_auth=False and an IAM role is in use. 33 | # See: https://github.com/boto/boto/issues/2043 34 | # See: https://github.com/WhoopInc/mkwheelhouse/issues/11 35 | def __init__(self, s3_connection): 36 | self.connection = s3_connection 37 | self.security_token = s3_connection.provider.security_token 38 | 39 | def __enter__(self): 40 | self.connection.provider.security_token = '' 41 | 42 | def __exit__(self, type, value, traceback): 43 | self.connection.provider.security_token = self.security_token 44 | 45 | 46 | class Bucket(object): 47 | def __init__(self, url): 48 | if not re.match(r'^(s3:)?//', url): 49 | url = '//' + url 50 | url = urlparse(url) 51 | self.name = url.netloc 52 | self.prefix = url.path.lstrip('/') 53 | # Boto currently can't handle names with dots unless the region 54 | # is specified explicitly. 55 | # See: https://github.com/boto/boto/issues/2836 56 | self.region = self._get_region() 57 | self.s3 = boto.s3.connect_to_region( 58 | region_name=self.region, 59 | calling_format=boto.s3.connection.OrdinaryCallingFormat()) 60 | self.bucket = self.s3.get_bucket(self.name) 61 | 62 | def _get_region(self): 63 | # S3, for what appears to be backwards-compatibility 64 | # reasons, maintains a distinction between location 65 | # constraints and region endpoints. Newer regions have 66 | # equivalent regions and location constraints, so we 67 | # hardcode the non-equivalent regions here with hopefully no 68 | # automatic support future S3 regions. 69 | # 70 | # Note also that someday, Boto should handle this for us 71 | # instead of the AWS command line tools. 72 | stdout = spawn([ 73 | 'aws', 's3api', 'get-bucket-location', 74 | '--bucket', self.name], capture_output=True) 75 | location = json.loads(stdout.decode())['LocationConstraint'] 76 | if not location: 77 | return 'us-east-1' 78 | elif location == 'EU': 79 | return 'eu-west-1' 80 | else: 81 | return location 82 | 83 | def get_key(self, key): 84 | if not isinstance(key, boto.s3.key.Key): 85 | return boto.s3.key.Key(bucket=self.bucket, 86 | name=os.path.join(self.prefix, key)) 87 | return key 88 | 89 | def has_key(self, key): 90 | return self.get_key(key).exists() 91 | 92 | def generate_url(self, key): 93 | key = self.get_key(key) 94 | with UrlSafeS3Connection(self.s3): 95 | return key.generate_url(expires_in=0, query_auth=False) 96 | 97 | def list(self): 98 | return self.bucket.list(prefix=self.prefix) 99 | 100 | def sync(self, local_dir, acl): 101 | return spawn([ 102 | 'aws', 's3', 'sync', 103 | local_dir, 's3://{0}/{1}'.format(self.name, self.prefix), 104 | '--region', self.region, '--acl', acl]) 105 | 106 | def put(self, body, key, acl): 107 | key = self.get_key(key) 108 | 109 | content_type = mimetypes.guess_type(key.name)[0] 110 | if content_type: 111 | key.content_type = content_type 112 | 113 | key.set_contents_from_string(body, replace=True, policy=acl) 114 | 115 | def list_wheels(self): 116 | return [key for key in self.list() if key.name.endswith('.whl')] 117 | 118 | def make_index(self): 119 | doc, tag, text = yattag.Doc().tagtext() 120 | with tag('html'): 121 | for key in self.list_wheels(): 122 | with tag('a', href=self.generate_url(key)): 123 | text(key.name) 124 | doc.stag('br') 125 | 126 | return doc.getvalue() 127 | 128 | 129 | def build_wheels(index_url, pip_wheel_args, exclusions): 130 | build_dir = tempfile.mkdtemp(prefix='mkwheelhouse-') 131 | args = [ 132 | 'pip', 'wheel', 133 | '--wheel-dir', build_dir, 134 | '--find-links', index_url, 135 | # pip < 7 doesn't invalidate HTTP cache based on last-modified 136 | # header, so disable it. 137 | '--no-cache-dir' 138 | ] + pip_wheel_args 139 | spawn(args) 140 | for exclusion in exclusions: 141 | matches = glob.glob(os.path.join(build_dir, exclusion)) 142 | for match in matches: 143 | os.remove(match) 144 | return build_dir 145 | 146 | 147 | def parse_args(): 148 | parser = argparse.ArgumentParser( 149 | description='Generate and upload wheels to an Amazon S3 wheelhouse', 150 | usage='mkwheelhouse [options] bucket [PACKAGE...] [pip-options]', 151 | epilog='Consult `pip wheel` for valid pip-options.') 152 | parser.add_argument('-e', '--exclude', action='append', default=[], 153 | metavar='WHEEL_FILENAME', 154 | help='wheels to exclude from upload') 155 | parser.add_argument('-a', '--acl', metavar='POLICY', default='private', 156 | help='canned ACL policy to apply to uploaded objects') 157 | parser.add_argument('bucket', 158 | help='the Amazon S3 bucket to upload wheels to') 159 | args, pip_wheel_args = parser.parse_known_args() 160 | try: 161 | run(args, pip_wheel_args) 162 | except subprocess.CalledProcessError: 163 | print('mkwheelhouse: detected error in subprocess, aborting!', 164 | file=sys.stderr) 165 | 166 | 167 | def run(args, pip_wheel_args): 168 | bucket = Bucket(args.bucket) 169 | if not bucket.has_key('index.html'): 170 | bucket.put('', 'index.html', acl=args.acl) 171 | index_url = bucket.generate_url('index.html') 172 | build_dir = build_wheels(index_url, pip_wheel_args, args.exclude) 173 | bucket.sync(build_dir, acl=args.acl) 174 | bucket.put(bucket.make_index(), key='index.html', acl=args.acl) 175 | shutil.rmtree(build_dir) 176 | print('mkwheelhouse: index written to', index_url) 177 | 178 | 179 | if __name__ == '__main__': 180 | parse_args() 181 | -------------------------------------------------------------------------------- /pylintrc: -------------------------------------------------------------------------------- 1 | [MESSAGES CONTROL] 2 | disable = C0103, C0111, F0401 3 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | universal = 1 3 | 4 | [flake8] 5 | ignore = W601 6 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | install_requires = [ 4 | 'awscli >= 1.3.6', 5 | 'boto >= 2.38.0', 6 | 'yattag >= 0.9.2', 7 | 'wheel >= 0.23.0', 8 | 'pip >= 1.5.4', 9 | 'six >= 1.9.0', 10 | ] 11 | 12 | setup( 13 | name='mkwheelhouse', 14 | version='2.0.0', 15 | author='Nikhil Benesch', 16 | author_email='benesch@whoop.com', 17 | py_modules=['mkwheelhouse'], 18 | url='https://github.com/WhoopInc/mkwheelhouse', 19 | description='Amazon S3 wheelhouse generator', 20 | classifiers=[ 21 | 'License :: OSI Approved :: MIT License', 22 | 'Development Status :: 4 - Beta', 23 | 'Environment :: Console', 24 | 'Intended Audience :: Developers', 25 | 'Topic :: Software Development :: Build Tools' 26 | ], 27 | install_requires=install_requires, 28 | extras_require={ 29 | 'tests': [ 30 | 'pre-commit' 31 | ] 32 | }, 33 | entry_points={ 34 | 'console_scripts': [ 35 | 'mkwheelhouse=mkwheelhouse:parse_args' 36 | ], 37 | } 38 | ) 39 | -------------------------------------------------------------------------------- /test/fixtures.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import json 4 | import os 5 | import sys 6 | import traceback 7 | 8 | import boto 9 | from boto.s3.connection import Location, OrdinaryCallingFormat 10 | 11 | BUCKET_POLICY_PUBLIC = json.dumps({ 12 | 'Version': '2008-10-17', 13 | 'Statement': [{ 14 | 'Sid': 'AllowPublicRead', 15 | 'Effect': 'Allow', 16 | 'Principal': {'AWS': '*'}, 17 | 'Action': ['s3:GetObject'], 18 | 'Resource': ['arn:aws:s3:::%s/*'] 19 | }] 20 | }) 21 | 22 | if len(sys.argv) != 2 or sys.argv[1] not in ['create', 'delete']: 23 | sys.exit('usage: {0} [create|delete]'.format(sys.argv[0])) 24 | 25 | failed = False 26 | 27 | # We have to specify both the desired region and location or API 28 | # requests will fail in subtle ways and undocumented ways: SSL errors, 29 | # missing location constraints, etc. 30 | connections = [ 31 | (os.environ['MKWHEELHOUSE_BUCKET_NONSTANDARD'], 'eu-west-1', Location.EU), 32 | (os.environ['MKWHEELHOUSE_BUCKET_STANDARD'], 'us-east-1', Location.DEFAULT) 33 | ] 34 | 35 | 36 | def purge_bucket(name): 37 | bucket = s3.get_bucket(name) 38 | bucket.delete_keys([key.name for key in bucket.list()]) 39 | bucket.delete() 40 | 41 | 42 | for bucket_name, region, location in connections: 43 | bucket_name_private = bucket_name + '-private' 44 | s3 = boto.s3.connect_to_region(region, 45 | calling_format=OrdinaryCallingFormat()) 46 | if sys.argv[1] == 'create': 47 | b = s3.create_bucket(bucket_name, location=location) 48 | b.set_policy(BUCKET_POLICY_PUBLIC % bucket_name) 49 | s3.create_bucket(bucket_name_private, location=location) 50 | else: 51 | try: 52 | purge_bucket(bucket_name) 53 | purge_bucket(bucket_name_private) 54 | except boto.exception.S3ResponseError: 55 | traceback.print_exc() 56 | # Don't exit just yet. We may still be able to clean up the 57 | # other buckets. 58 | failed = True 59 | continue 60 | 61 | sys.exit(failed) 62 | -------------------------------------------------------------------------------- /test/requirements/default.txt: -------------------------------------------------------------------------------- 1 | pytsdb==0.2.1 2 | -------------------------------------------------------------------------------- /test/requirements/exclusions.txt: -------------------------------------------------------------------------------- 1 | unittest2==1.0.1 2 | -------------------------------------------------------------------------------- /test/test.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | missing_vars=() 4 | 5 | require_var() { 6 | [[ "${!1}" ]] || missing_vars+=("$1") 7 | } 8 | 9 | require_var MKWHEELHOUSE_BUCKET_STANDARD 10 | require_var MKWHEELHOUSE_BUCKET_NONSTANDARD 11 | 12 | if [[ ${#missing_vars[*]} -gt 0 ]]; then 13 | echo "Missing required environment variables:" 14 | printf ' %s\n' "${missing_vars[@]}" 15 | exit 1 16 | fi 17 | 18 | setup() { 19 | pip uninstall --yes jinja2 pytsdb zini &> /dev/null || true 20 | } 21 | 22 | assert_not_found() { 23 | [[ "$output" == *"No distributions at all found for $1"* || "$output" == *"No matching distribution found for $1"* ]] 24 | } 25 | 26 | @test "standard: packages only" { 27 | mkwheelhouse "$MKWHEELHOUSE_BUCKET_STANDARD" jinja2 28 | } 29 | 30 | @test "standard: -r only" { 31 | mkwheelhouse "$MKWHEELHOUSE_BUCKET_STANDARD" \ 32 | -r "$BATS_TEST_DIRNAME/requirements/default.txt" 33 | } 34 | 35 | @test "standard: --requirement only" { 36 | mkwheelhouse "$MKWHEELHOUSE_BUCKET_STANDARD" \ 37 | --requirement "$BATS_TEST_DIRNAME/requirements/default.txt" 38 | } 39 | 40 | @test "standard: --requirement and package" { 41 | mkwheelhouse "$MKWHEELHOUSE_BUCKET_STANDARD" \ 42 | --requirement "$BATS_TEST_DIRNAME/requirements/default.txt" 43 | } 44 | 45 | @test "standard: -e" { 46 | mkwheelhouse "$MKWHEELHOUSE_BUCKET_STANDARD" \ 47 | --requirement "$BATS_TEST_DIRNAME/requirements/exclusions.txt" \ 48 | -e unittest2-1.0.1-py2.py3-none-any.whl 49 | } 50 | 51 | @test "standard: --exclude" { 52 | mkwheelhouse "$MKWHEELHOUSE_BUCKET_STANDARD" \ 53 | --requirement "$BATS_TEST_DIRNAME/requirements/exclusions.txt" \ 54 | --exclude unittest2-1.0.1-py2.py3-none-any.whl 55 | } 56 | 57 | @test "standard: install with acl private (default)" { 58 | mkwheelhouse "$MKWHEELHOUSE_BUCKET_STANDARD-private" jinja2 59 | } 60 | 61 | @test "standard: no bucket specified" { 62 | run mkwheelhouse 63 | [[ "$status" -eq 2 ]] 64 | [[ "$output" == *"too few arguments"* || $output == *"the following arguments are required: bucket"* ]] 65 | } 66 | 67 | @test "standard: no package or requirement specified" { 68 | run mkwheelhouse "$MKWHEELHOUSE_BUCKET_STANDARD" 69 | [[ "$output" == *"You must give at least one requirement to wheel"* ]] 70 | } 71 | 72 | @test "standard: pip can't install excluded packages" { 73 | run pip install \ 74 | --no-index \ 75 | --find-links "https://s3.amazonaws.com/$MKWHEELHOUSE_BUCKET_STANDARD/index.html" \ 76 | unittest2 77 | [[ "$status" -eq 1 ]] 78 | assert_not_found unittest2 79 | } 80 | 81 | @test "standard: pip can install built packages" { 82 | pip install \ 83 | --no-index \ 84 | --find-links "https://s3.amazonaws.com/$MKWHEELHOUSE_BUCKET_STANDARD/index.html" \ 85 | jinja2 pytsdb 86 | } 87 | 88 | @test "standard: mkwheelhouse builds into prefix" { 89 | mkwheelhouse "$MKWHEELHOUSE_BUCKET_STANDARD/deep/rooted/fear" zini 90 | } 91 | 92 | @test "standard: pip can install built packages in prefix" { 93 | pip install \ 94 | --no-index \ 95 | --find-links "https://s3.amazonaws.com/$MKWHEELHOUSE_BUCKET_STANDARD/deep/rooted/fear/index.html" \ 96 | zini 97 | } 98 | 99 | @test "standard: pip can't install non-prefixed packages from prefix" { 100 | run pip install \ 101 | --no-index \ 102 | --find-links "https://s3.amazonaws.com/$MKWHEELHOUSE_BUCKET_STANDARD/deep/rooted/fear/index.html" \ 103 | jinja2 104 | [[ "$status" -eq 1 ]] 105 | assert_not_found jinja2 106 | } 107 | 108 | @test "standard: pip can't install packages from acl private" { 109 | run pip install \ 110 | --no-index \ 111 | --find-links "https://s3.amazonaws.com/$MKWHEELHOUSE_BUCKET_STANDARD-private/index.html" \ 112 | jinja2 113 | [[ "$status" -eq 1 ]] 114 | assert_not_found jinja2 115 | } 116 | 117 | @test "standard: install with acl public-read" { 118 | mkwheelhouse --acl public-read "$MKWHEELHOUSE_BUCKET_STANDARD-private" jinja2 119 | } 120 | 121 | @test "standard: pip can install packages from acl public-read" { 122 | pip install \ 123 | --no-index \ 124 | --find-links "https://s3.amazonaws.com/$MKWHEELHOUSE_BUCKET_STANDARD-private/index.html" \ 125 | jinja2 126 | } 127 | 128 | @test "unrecognized options are passed through to pip" { 129 | logfile="$(mktemp -u -t XXXXXmkwheelhouse)" 130 | [[ ! -f "$logfile" ]] 131 | mkwheelhouse "$MKWHEELHOUSE_BUCKET_STANDARD" --log "$logfile" six 132 | [[ -f "$logfile" ]] 133 | } 134 | 135 | @test "nonstandard: packages only" { 136 | mkwheelhouse "$MKWHEELHOUSE_BUCKET_NONSTANDARD" jinja2 137 | } 138 | 139 | @test "nonstandard: -r only" { 140 | mkwheelhouse "$MKWHEELHOUSE_BUCKET_NONSTANDARD" \ 141 | -r "$BATS_TEST_DIRNAME/requirements/default.txt" 142 | } 143 | 144 | @test "nonstandard: --requirement only" { 145 | mkwheelhouse "$MKWHEELHOUSE_BUCKET_NONSTANDARD" \ 146 | --requirement "$BATS_TEST_DIRNAME/requirements/default.txt" 147 | } 148 | 149 | @test "nonstandard: --requirement and package" { 150 | mkwheelhouse "$MKWHEELHOUSE_BUCKET_NONSTANDARD" \ 151 | --requirement "$BATS_TEST_DIRNAME/requirements/default.txt" 152 | } 153 | 154 | @test "nonstandard: -e" { 155 | mkwheelhouse "$MKWHEELHOUSE_BUCKET_NONSTANDARD" \ 156 | --requirement "$BATS_TEST_DIRNAME/requirements/exclusions.txt" \ 157 | -e unittest2-1.0.1-py2.py3-none-any.whl 158 | } 159 | 160 | @test "nonstandard: --exclude" { 161 | mkwheelhouse "$MKWHEELHOUSE_BUCKET_NONSTANDARD" \ 162 | --requirement "$BATS_TEST_DIRNAME/requirements/exclusions.txt" \ 163 | --exclude unittest2-1.0.1-py2.py3-none-any.whl 164 | } 165 | 166 | @test "nonstandard: install with acl private (default)" { 167 | mkwheelhouse "$MKWHEELHOUSE_BUCKET_NONSTANDARD-private" jinja2 168 | } 169 | 170 | @test "nonstandard: no bucket specified" { 171 | run mkwheelhouse 172 | [[ "$status" -eq 2 ]] 173 | [[ "$output" == *"too few arguments"* || $output == *"the following arguments are required: bucket"* ]] 174 | } 175 | 176 | @test "nonstandard: no package or requirement specified" { 177 | run mkwheelhouse "$MKWHEELHOUSE_BUCKET_NONSTANDARD" 178 | [[ "$output" == *"You must give at least one requirement to wheel"* ]] 179 | } 180 | 181 | @test "nonstandard: pip can't install excluded packages" { 182 | run pip install \ 183 | --no-index \ 184 | --find-links "https://s3-eu-west-1.amazonaws.com/$MKWHEELHOUSE_BUCKET_NONSTANDARD/index.html" \ 185 | unittest2 186 | [[ "$status" -eq 1 ]] 187 | assert_not_found unittest2 188 | } 189 | 190 | @test "nonstandard: pip can install built packages" { 191 | pip install \ 192 | --no-index \ 193 | --find-links "https://s3-eu-west-1.amazonaws.com/$MKWHEELHOUSE_BUCKET_NONSTANDARD/index.html" \ 194 | jinja2 pytsdb 195 | } 196 | 197 | @test "nonstandard: mkwheelhouse builds into prefix" { 198 | mkwheelhouse "$MKWHEELHOUSE_BUCKET_NONSTANDARD/deep/rooted/fear" zini 199 | } 200 | 201 | @test "nonstandard: pip can install built packages in prefix" { 202 | pip install \ 203 | --no-index \ 204 | --find-links "https://s3-eu-west-1.amazonaws.com/$MKWHEELHOUSE_BUCKET_NONSTANDARD/deep/rooted/fear/index.html" \ 205 | zini 206 | } 207 | 208 | @test "nonstandard: pip can't install non-prefixed packages from prefix" { 209 | run pip install \ 210 | --no-index \ 211 | --find-links "https://s3-eu-west-1.amazonaws.com/$MKWHEELHOUSE_BUCKET_NONSTANDARD/deep/rooted/fear/index.html" \ 212 | jinja2 213 | [[ "$status" -eq 1 ]] 214 | assert_not_found jinja2 215 | } 216 | 217 | @test "nonstandard: pip can't install packages from acl private" { 218 | run pip install \ 219 | --no-index \ 220 | --find-links "https://s3-eu-west-1.amazonaws.com/$MKWHEELHOUSE_BUCKET_NONSTANDARD-private/index.html" \ 221 | jinja2 222 | [[ "$status" -eq 1 ]] 223 | assert_not_found jinja2 224 | } 225 | 226 | @test "nonstandard: install with acl public-read" { 227 | mkwheelhouse --acl public-read "$MKWHEELHOUSE_BUCKET_NONSTANDARD-private" jinja2 228 | } 229 | 230 | @test "nonstandard: pip can install packages from acl public-read" { 231 | pip install \ 232 | --no-index \ 233 | --find-links "https://s3-eu-west-1.amazonaws.com/$MKWHEELHOUSE_BUCKET_NONSTANDARD-private/index.html" \ 234 | jinja2 235 | } 236 | --------------------------------------------------------------------------------