├── AUTHORS.txt ├── setup.py ├── tests.sh ├── .travis.yml ├── README.rst └── dirtbike └── __init__.py /AUTHORS.txt: -------------------------------------------------------------------------------- 1 | Asheesh Laroia 2 | Your name here, if you like! Help always welcome. 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from __future__ import absolute_import, division, print_function, unicode_literals 3 | 4 | from distutils.core import setup 5 | 6 | setup(name='dirtbike', 7 | version='0.1', 8 | description=( 9 | 'Convert already-installed Python modules ("distribution") to wheel' 10 | ), 11 | author='Asheesh Laroia', 12 | author_email='asheesh@asheesh.org', 13 | url='https://github.com/paulproteus/dirtbike', 14 | packages=['dirtbike'], 15 | install_requires=[ 16 | 'wheel', 17 | ], 18 | entry_points={ 19 | 'console_scripts': [ 20 | 'dirtbike = dirtbike:main', 21 | ], 22 | }, 23 | ) 24 | -------------------------------------------------------------------------------- /tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | # This is the shell script test suite for making sure we 5 | # can install, and then re-pack, a wheel. 6 | # 7 | # For this test, we use the livereload wheel, for the fun 8 | # of it. 9 | echo "Test 1. Get a wheel from PyPI, install it, remove wheel file" 10 | echo " then verify we can re-create the wheel file." 11 | echo "" 12 | 13 | wget https://pypi.python.org/packages/2.7/l/livereload/livereload-2.4.0-py2.py3-none-any.whl 14 | pip install --user livereload-2.4.0-py2.py3-none-any.whl 15 | 16 | # Now remove the upstream-provided wheel. 17 | rm livereload-2.4.0-py2.py3-none-any.whl 18 | 19 | # Now run our thing. 20 | python -c 'import dirtbike; dirtbike.make_wheel_file("livereload")' 21 | unzip -qq dist/livereload*whl 22 | if [ -f livereload/__init__.py ] ; then 23 | echo 'yay, we got livereload' 24 | rm -rf livereload 25 | else 26 | echo 'aw we did not get livereload' 27 | exit 1 28 | fi 29 | 30 | echo "Test 2. apt-get install something, then generate a wheel file" 31 | echo " using dirtbike." 32 | echo "" 33 | 34 | # Now test against a system-installed package. We install "six" in the 35 | # .travis.yml so let's extract from that. 36 | 37 | # Assert python-six is actually installed. 38 | THING=six 39 | dpkg -l python-$THING 40 | dirtbike $THING 41 | ls dist 42 | 43 | # Now extract it! 44 | unzip -qq dist/$THING*whl 45 | if [ -f $THING/__init__.py ] || [ -f "${THING}.py" ] ; then 46 | echo "yay, we got $THING" 47 | rm -rf ./"$THING" 48 | else 49 | echo "aw we did not get $THING" 50 | exit 1 51 | fi 52 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: true 2 | # Use C as a dummy language to avoid Travis-CI setting up a virtualenv 3 | # for us, etc. 4 | language: c 5 | before_install: 6 | ### Now do the fun part: trans-grade to Debian. 7 | # Remove some packages that we get from Debian anyway. 8 | - sudo apt-get --quiet=2 remove --purge -y mongodb-10gen mysql-client-core-5.5 man-db >/dev/null 9 | # Remove some packages that actively depend on locales working properly. 10 | - sudo apt-get --quiet=2 remove postgresql-9.1 postgresql-contrib-9.1 postgresql-9.2 postgresql-contrib-9.2 postgresql-9.3 postgresql-contrib-9.3 postgresql-9.4 postgresql-contrib-9.4 >/dev/null 11 | # Tell dpkg it is OK to choose the new config files, without prompting. 12 | - echo 'Dpkg::Options { "--force-confdef"; "--force-confnew"; }' | sudo dd of=/etc/apt/apt.conf.d/99conf 13 | # Trans-grade to Debian. 14 | - sudo rm /etc/apt/sources.list 15 | - echo 'deb http://mirror.mit.edu/debian/ jessie main' | sudo dd of=/etc/apt/sources.list 16 | - sudo apt-get --quiet=2 update 17 | - sudo apt-get --quiet=2 --force-yes -y install debian-archive-keyring >/dev/null 18 | - sudo apt-get --quiet=2 update 19 | - sudo rm -f /etc/dpkg/dpkg.cfg.d/multiarch # Remove historic multi-arch config. 20 | - echo -n | sudo dd of=/var/lib/dpkg/info/libc-bin.list # Pretend nothing conflicts with this package's files. 21 | - sudo dpkg -r --force-depends --force-remove-essential locales libc6-dev 22 | - sudo apt-get --quiet=2 --fix-broken install >/dev/null 23 | # Make sure locales are in a healthy shape. 24 | - echo 'en_US.UTF-8 UTF-8' | sudo dd of=/etc/locale.gen 25 | - sudo /usr/sbin/locale-gen 26 | - sudo apt-get install python-six/jessie 27 | - curl https://bootstrap.pypa.io/get-pip.py | sudo python 28 | install: 29 | - pip install --user --editable . 30 | script: /bin/bash tests.sh 31 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | dirtbike 2 | ======== 3 | 4 | .. image:: https://img.shields.io/travis/paulproteus/dirtbike.svg 5 | :target: https://travis-ci.org/paulproteus/dirtbike 6 | 7 | 8 | Purpose of the software 9 | ----------------------- 10 | 11 | This is a Python program that takes packages that are already 12 | installed on a system and converts them to wheels. 13 | 14 | That is an admittedly strange goal. The deeper purpose is to help 15 | packages like `pip` vendor their dependencies in a way compatible with 16 | the packaging policy for Debian, and hopefully other GNU/Linux 17 | distributions. 18 | 19 | Therefore, I am eager to see this tool discussed and/or adopted by 20 | Fedora, Debian, Ubuntu, and any other software distributions that 21 | distribute pip as well as other Python packages. 22 | 23 | 24 | License 25 | ------- 26 | 27 | This software is available under the terms of the same license as 28 | `pya/pip`:: 29 | 30 | Copyright (c) 2008-2014 The developers (see AUTHORS.txt file) 31 | 32 | Permission is hereby granted, free of charge, to any person 33 | obtaining a copy of this software and associated documentation files 34 | (the "Software"), to deal in the Software without restriction, 35 | including without limitation the rights to use, copy, modify, merge, 36 | publish, distribute, sublicense, and/or sell copies of the Software, 37 | and to permit persons to whom the Software is furnished to do so, 38 | subject to the following conditions: 39 | 40 | The above copyright notice and this permission notice shall be 41 | included in all copies or substantial portions of the Software. 42 | 43 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 44 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 45 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 46 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 47 | BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 48 | ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 49 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 50 | SOFTWARE. 51 | 52 | 53 | Origin of the name 54 | ------------------ 55 | 56 | The name comes from a They Might Be Giants song:: 57 | 58 | Here comes the dirt bike, 59 | Beware of the dirt bike. 60 | [...] 61 | You see I never thought I'd understand. 62 | Till that bike took me by the hand, 63 | Now I ride. 64 | 65 | The real motivation behind the name was to find a word or two that 66 | alludes to the idea of "wheel." 67 | -------------------------------------------------------------------------------- /dirtbike/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import ( 2 | absolute_import, division, print_function, unicode_literals, 3 | ) 4 | 5 | import distutils.dist 6 | import os 7 | import pkg_resources 8 | import shutil 9 | import subprocess 10 | import wheel.bdist_wheel 11 | 12 | 13 | def _mkdir_ok_exists(dirname): 14 | try: 15 | os.mkdir(dirname) 16 | except OSError as e: 17 | if e.errno == 17: # File exists 18 | pass 19 | else: 20 | raise 21 | 22 | 23 | def _mkdir_p(dirname): 24 | if not dirname: 25 | raise ValueError("I refuse to operate on false-y values.") 26 | 27 | path_components = dirname.split('/') 28 | for i in range(len(path_components)): 29 | leading_path_component = '/'.join(path_components[:i+1]) 30 | if leading_path_component: 31 | _mkdir_ok_exists(leading_path_component) 32 | 33 | 34 | def _copy_file_making_dirs_as_needed(src, dst): 35 | _mkdir_p(os.path.dirname(dst)) 36 | shutil.copy(src, dst) 37 | 38 | 39 | # NOTE: For now, this module has some Debian-specific hacks 40 | # (invocations to dpkg -L, etc.) to make it useful immediately. 41 | # 42 | # I consider that a bug, not a feature. I'd like to remove the 43 | # Debian-specific code so that this can rely entirely in the pip 44 | # pseudo-standard of "installed-files.txt" etc. However, the 45 | # python-requests package in Debian testing (at the time of writing) 46 | # does not distribute an installed-files.txt, so probably it is useful 47 | # to have the Debian-specific code in this version. 48 | # 49 | # To be semver-esque, a version with the Debian-specific code 50 | # removed would presumably have a bumped "major" version number. 51 | def _get_files_list_via_debian(distribution_obj): 52 | # Find the .egg-info directory, and then search the dpkg database 53 | # for which package provides it. 54 | path_to_egg_info = distribution_obj._provider.egg_info 55 | as_bytes = subprocess.check_output([ 56 | '/usr/bin/dpkg', '-S', path_to_egg_info]) 57 | as_text = as_bytes.decode('utf-8') 58 | pkg_name = as_text.split(':')[0] 59 | files_list_as_bytes = subprocess.check_output([ 60 | '/usr/bin/dpkg', '-L', pkg_name]) 61 | files_list_as_text = files_list_as_bytes.decode('utf-8') 62 | # Now we have all the files from the Debian package. However, 63 | # RECORD-style files lists are all relative to the site-packages 64 | # directory in which the package was installed. 65 | ret = [] 66 | base_location = distribution_obj.location 67 | for filename in files_list_as_text.split('\n'): 68 | if filename.startswith(distribution_obj.location): 69 | shortened_filename = filename[len(base_location):] 70 | if shortened_filename.startswith('/'): 71 | shortened_filename = shortened_filename[1:] 72 | ret.append(shortened_filename) 73 | return ret 74 | 75 | 76 | def _get_files_list_via_wheel_metadata(distribution_obj): 77 | '''Return either the bytes of the RECORD file, or something 78 | equivalent. I hear installed-files.txt is a thing, but I haven't 79 | found evidence for that yet.''' 80 | try: 81 | # If we're lucky, the information for what files are installed on the 82 | # system are available in RECORD, aka wheel metadata. 83 | return distribution_obj.get_metadata('RECORD').split('\n') 84 | except IOError as e: 85 | # Let's find the path to an egg-info file and ask dpkg for the 86 | # file metadata. 87 | if e.errno == 2: 88 | return None 89 | raise 90 | 91 | 92 | def make_wheel_file(distribution_name): 93 | # Grab the metadata for the installed version of this 94 | # distribution. 95 | installed_package_metadata = pkg_resources.get_distribution( 96 | distribution_name) 97 | 98 | # Create Distribution object so that the wheel.bdist_wheel 99 | # machinery can operate happily. We copy any metadata we need out 100 | # of the installed package metadata into this thing. 101 | dummy_dist_distribution_obj = distutils.dist.Distribution(attrs={ 102 | 'name': installed_package_metadata.project_name, 103 | 'version': installed_package_metadata.version}) 104 | 105 | wheel_generator = wheel.bdist_wheel.bdist_wheel( 106 | dummy_dist_distribution_obj) 107 | 108 | # Copy files from the system into a place where wheel_generator 109 | # will look. 110 | # 111 | # FIXME: Pull this out of wheel_generator. 112 | BUILD_PREFIX = 'build/bdist.linux-x86_64/wheel/' 113 | 114 | if os.path.exists(BUILD_PREFIX): 115 | raise ValueError( 116 | "Yikes, I am afraid of inconsistent state and will bail out.") 117 | 118 | files_list = _get_files_list_via_wheel_metadata(installed_package_metadata) 119 | if not files_list: 120 | # Let's try the Debian-specific hack, just in case. 121 | files_list = _get_files_list_via_debian(installed_package_metadata) 122 | if not files_list: 123 | # Well, I don't know what to do. 124 | raise RuntimeError("Cannot find files for this package. Bailing.") 125 | 126 | for line in files_list: 127 | # NOTE: We currently ignore hashes and any other metadata. 128 | filename = line.split(',')[0] 129 | 130 | # The list of files sometimes contains the empty 131 | # string. That's not much of a file, so we don't bother adding 132 | # it to the archive. 133 | if not filename: 134 | continue 135 | 136 | # NOTE: If a file is not in the "location" that the installed 137 | # package supposedly lives in, we skip it. This means we are 138 | # likely to skip console scripts. 139 | if filename.startswith('/'): 140 | abspath = os.path.abspath(filename) 141 | else: 142 | abspath = os.path.abspath(installed_package_metadata.location + 143 | '/' + filename) 144 | 145 | found = False 146 | isfile = False 147 | if (abspath.startswith(installed_package_metadata.location) and 148 | os.path.exists(abspath)): 149 | found = True 150 | isfile = os.path.isfile(abspath) 151 | 152 | # Print a warning in the case that some file is missing, then skip it. 153 | if not found: 154 | print('Skipping', abspath, 155 | 'because we could not find it in the metadata location.') 156 | continue 157 | 158 | # Skip directories. 159 | if found and not isfile: 160 | continue 161 | 162 | # Skip the dist-info directory, since bdist_wheel will 163 | # recreate it for us. 164 | if '.dist-info' in abspath: 165 | continue 166 | 167 | # Skip any *.pyc files or files that are in a __pycache__ directory. 168 | if (('__pycache__' in abspath) or 169 | abspath.endswith('.pyc')): 170 | continue 171 | 172 | # Since we actually do seem to want this file, let us copy it 173 | # into the BUILD_PREFIX. 174 | _copy_file_making_dirs_as_needed( 175 | abspath, 176 | os.path.abspath(BUILD_PREFIX + '/' + filename)) 177 | 178 | # Call finalize_options() to tell bdist_wheel we are done playing 179 | # with metadata. 180 | wheel_generator.finalize_options() 181 | 182 | wheel_generator.run() # OMG Rofl? 183 | return wheel_generator 184 | 185 | 186 | def main(): 187 | import sys 188 | make_wheel_file(sys.argv[1]) 189 | --------------------------------------------------------------------------------